diff options
| author | Ruslan Kovalev <ruslan.a.kovalev@gmail.com> | 2022-02-10 16:46:45 +0300 |
|---|---|---|
| committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:46:45 +0300 |
| commit | 9123176b341b6f2658cff5132482b8237c1416c8 (patch) | |
| tree | 49e222ea1c5804306084bb3ae065bb702625360f /library | |
| parent | 59e19371de37995fcb36beb16cd6ec030af960bc (diff) | |
| download | ydb-9123176b341b6f2658cff5132482b8237c1416c8.tar.gz | |
Restoring authorship annotation for Ruslan Kovalev <ruslan.a.kovalev@gmail.com>. Commit 2 of 2.
Diffstat (limited to 'library')
298 files changed, 11838 insertions, 11838 deletions
diff --git a/library/README.md b/library/README.md index 778ccbfef9..fc418bef7b 100644 --- a/library/README.md +++ b/library/README.md @@ -30,7 +30,7 @@ library * spam filter rules; * coefficients for ML; * etc. - + 11. All OSS (OpenSource Software) ready code **MUST** be accompanied by macro [LICENCE](https://docs.yandex-team.ru/ya-make/manual/common/macros#licence(license...)) in `ya.make`. 12. All language specific aspects are defined by `<language>` committee: see `library/<language>/README.md`. diff --git a/library/cpp/archive/yarchive.cpp b/library/cpp/archive/yarchive.cpp index 07e1a0b2a9..1becc3e5da 100644 --- a/library/cpp/archive/yarchive.cpp +++ b/library/cpp/archive/yarchive.cpp @@ -327,15 +327,15 @@ public: inline TBlob ObjectBlobByKey(const TStringBuf key) const { TBlob subBlob = BlobByKey(key); - + if (UseDecompression) { TArchiveInputStream st(subBlob); return TBlob::FromStream(st); } else { return subBlob; - } + } } - + inline TBlob BlobByKey(const TStringBuf key) const { const auto it = Dict_.find(key); @@ -386,9 +386,9 @@ TAutoPtr<IInputStream> TArchiveReader::ObjectByKey(const TStringBuf key) const { } TBlob TArchiveReader::ObjectBlobByKey(const TStringBuf key) const { - return Impl_->ObjectBlobByKey(key); -} - + return Impl_->ObjectBlobByKey(key); +} + TBlob TArchiveReader::BlobByKey(const TStringBuf key) const { return Impl_->BlobByKey(key); } diff --git a/library/cpp/archive/yarchive.h b/library/cpp/archive/yarchive.h index 516698e71d..8120bcb940 100644 --- a/library/cpp/archive/yarchive.h +++ b/library/cpp/archive/yarchive.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include "models_archive_reader.h" diff --git a/library/cpp/bit_io/bitinout_ut.cpp b/library/cpp/bit_io/bitinout_ut.cpp index 375e89abde..23a1ddf344 100644 --- a/library/cpp/bit_io/bitinout_ut.cpp +++ b/library/cpp/bit_io/bitinout_ut.cpp @@ -1,19 +1,19 @@ -#include "bitinput.h" -#include "bitoutput.h" - +#include "bitinput.h" +#include "bitoutput.h" + #include <library/cpp/testing/unittest/registar.h> - + #include <util/stream/buffer.h> -#include <util/generic/buffer.h> - -namespace NBitIO { - static const char BITS_REF[] = +#include <util/generic/buffer.h> + +namespace NBitIO { + static const char BITS_REF[] = "00100010 01000000 00000000 00100111 11011111 01100111 11010101 00010100 " "00100010 01100011 11100011 00110000 11011011 11011111 01001100 00110101 " "10011110 01011111 01010000 00000110 00011011 00100110 00010100 01110011 " "00001010 10101010 10101010 10101010 10101010 10101010 10101010 10101010 " "10110101 01010101 01010101 01010101 01010101 01010101 01010101 01010101 " - "01000000"; + "01000000"; inline ui64 Bits(ui64 bytes) { return bytes << 3ULL; @@ -43,237 +43,237 @@ namespace NBitIO { inline TString PrintBits(T t, ui32 bits = Bits(sizeof(T))) { return PrintBits((char*)&t, ((char*)&t) + BytesUp(bits)); } -} - -class TBitIOTest: public TTestBase { - UNIT_TEST_SUITE(TBitIOTest); +} + +class TBitIOTest: public TTestBase { + UNIT_TEST_SUITE(TBitIOTest); UNIT_TEST(TestBitIO) - UNIT_TEST_SUITE_END(); - -private: + UNIT_TEST_SUITE_END(); + +private: using TBi = NBitIO::TBitInput; using TVec = TVector<char>; - + void static CheckBits(const TVec& v, const TString& ref, const TString& rem) { - UNIT_ASSERT_VALUES_EQUAL_C(NBitIO::PrintBits(v.begin(), v.end()), ref, rem); - } - - void DoRead(TBi& b, ui32& t) { + UNIT_ASSERT_VALUES_EQUAL_C(NBitIO::PrintBits(v.begin(), v.end()), ref, rem); + } + + void DoRead(TBi& b, ui32& t) { b.Read(t, 1, 0); // 1 - b.ReadK<3>(t, 1); // 4 + b.ReadK<3>(t, 1); // 4 b.Read(t, 5, 4); // 9 - b.ReadK<14>(t, 9); // 23 + b.ReadK<14>(t, 9); // 23 b.Read(t, 1, 23); // 24 - b.ReadK<5>(t, 24); // 29 + b.ReadK<5>(t, 24); // 29 b.Read(t, 3, 29); // 32 - } - - template <typename TBo> - void DoWrite(TBo& b, ui32 t) { - b.Write(t, 1, 0); //1 - b.Write(t, 3, 1); //4 - b.Write(t, 5, 4); //9 - b.Write(t, 14, 9); //23 - b.Write(t, 1, 23); //24 - b.Write(t, 5, 24); //29 + } + + template <typename TBo> + void DoWrite(TBo& b, ui32 t) { + b.Write(t, 1, 0); //1 + b.Write(t, 3, 1); //4 + b.Write(t, 5, 4); //9 + b.Write(t, 14, 9); //23 + b.Write(t, 1, 23); //24 + b.Write(t, 5, 24); //29 b.Write(t, 3, 29); //32 - } - - template <typename TBo> + } + + template <typename TBo> void DoWrite1(TBo& out, const TString& rem) { - out.Write(0x0C, 3); + out.Write(0x0C, 3); UNIT_ASSERT_VALUES_EQUAL_C(out.GetOffset(), 1u, (rem + ", " + ToString(__LINE__))); - out.Write(0x18, 4); + out.Write(0x18, 4); UNIT_ASSERT_VALUES_EQUAL_C(out.GetOffset(), 1u, (rem + ", " + ToString(__LINE__))); - out.Write(0x0C, 3); + out.Write(0x0C, 3); UNIT_ASSERT_VALUES_EQUAL_C(out.GetOffset(), 2u, (rem + ", " + ToString(__LINE__))); - out.Write(0x30000, 17); + out.Write(0x30000, 17); UNIT_ASSERT_VALUES_EQUAL_C(out.GetOffset(), 4u, (rem + ", " + ToString(__LINE__))); - out.Write(0x0C, 3); + out.Write(0x0C, 3); UNIT_ASSERT_VALUES_EQUAL_C(out.GetOffset(), 4u, (rem + ", " + ToString(__LINE__))); - } - - template <typename TBo> + } + + template <typename TBo> void DoWrite2(TBo& out, const TString& rem) { - out.Write(0x0C, 3); + out.Write(0x0C, 3); UNIT_ASSERT_VALUES_EQUAL_C(out.GetOffset(), 8u, (rem + ", " + ToString(__LINE__))); - - out.Write(0x42, 7); + + out.Write(0x42, 7); UNIT_ASSERT_VALUES_EQUAL_C(out.GetOffset(), 9u, (rem + ", " + ToString(__LINE__))); - - DoWrite(out, 1637415112); + + DoWrite(out, 1637415112); UNIT_ASSERT_VALUES_EQUAL_C(out.GetOffset(), 13u, (rem + ", " + ToString(__LINE__))); - - DoWrite(out, 897998715); + + DoWrite(out, 897998715); UNIT_ASSERT_VALUES_EQUAL_C(out.GetOffset(), 17u, (rem + ", " + ToString(__LINE__))); - - DoWrite(out, 201416527); + + DoWrite(out, 201416527); UNIT_ASSERT_VALUES_EQUAL_C(out.GetOffset(), 21u, (rem + ", " + ToString(__LINE__))); - - DoWrite(out, 432344219); + + DoWrite(out, 432344219); UNIT_ASSERT_VALUES_EQUAL_C(out.GetOffset(), 25u, (rem + ", " + ToString(__LINE__))); - - out.Write(0xAAAAAAAAAAAAAAAAULL, 64); + + out.Write(0xAAAAAAAAAAAAAAAAULL, 64); UNIT_ASSERT_VALUES_EQUAL_C(out.GetOffset(), 33u, (rem + ", " + ToString(__LINE__))); - - out.Write(0x5555555555555555ULL, 64); + + out.Write(0x5555555555555555ULL, 64); UNIT_ASSERT_VALUES_EQUAL_C(out.GetOffset(), 41u, (rem + ", " + ToString(__LINE__))); - } - + } + void DoBitOutput(NBitIO::TBitOutputYVector& out, const TString& rem) { - DoWrite1(out, rem); - - out.WriteWords<8>(0xabcdef); - UNIT_ASSERT_VALUES_EQUAL_C(out.GetOffset(), 8u, (rem + ", " + ToString(__LINE__))); - - DoWrite2(out, rem); - } - + DoWrite1(out, rem); + + out.WriteWords<8>(0xabcdef); + UNIT_ASSERT_VALUES_EQUAL_C(out.GetOffset(), 8u, (rem + ", " + ToString(__LINE__))); + + DoWrite2(out, rem); + } + void DoBitOutput(NBitIO::TBitOutputArray& out, const TString& rem) { - DoWrite1(out, rem); - - out.WriteWords<8>(0xabcdef); - UNIT_ASSERT_VALUES_EQUAL_C(out.GetOffset(), 8u, (rem + ", " + ToString(__LINE__))); - - DoWrite2(out, rem); - } - + DoWrite1(out, rem); + + out.WriteWords<8>(0xabcdef); + UNIT_ASSERT_VALUES_EQUAL_C(out.GetOffset(), 8u, (rem + ", " + ToString(__LINE__))); + + DoWrite2(out, rem); + } + void DoBitInput(TBi& in, const TString& rem) { - UNIT_ASSERT(!in.Eof()); - - { - ui64 val; - - val = 0; + UNIT_ASSERT(!in.Eof()); + + { + ui64 val; + + val = 0; UNIT_ASSERT_VALUES_EQUAL_C(in.GetOffset(), 0u, (rem + ": " + NBitIO::PrintBits(val))); - + UNIT_ASSERT_C(in.Read(val, 3), (rem + ": " + NBitIO::PrintBits(val)).data()); - + UNIT_ASSERT_VALUES_EQUAL_C(val, 0x4u, (rem + ": " + NBitIO::PrintBits(val))); UNIT_ASSERT_VALUES_EQUAL_C(in.GetOffset(), 1u, (rem + ": " + NBitIO::PrintBits(val))); UNIT_ASSERT_C(!in.Eof(), (rem + ", " + ToString(__LINE__)).data()); - - val = 0; + + val = 0; UNIT_ASSERT_C(in.Read(val, 4), (rem + ": " + NBitIO::PrintBits(val)).data()); - + UNIT_ASSERT_VALUES_EQUAL_C(val, 0x8u, (rem + ": " + NBitIO::PrintBits(val))); UNIT_ASSERT_VALUES_EQUAL_C(in.GetOffset(), 1u, (rem + ": " + NBitIO::PrintBits(val))); UNIT_ASSERT_C(!in.Eof(), (rem + ", " + ToString(__LINE__)).data()); - - val = 0; + + val = 0; UNIT_ASSERT_C(in.Read(val, 3), (rem + ": " + NBitIO::PrintBits(val)).data()); - + UNIT_ASSERT_VALUES_EQUAL_C(val, 0x4u, (rem + ": " + NBitIO::PrintBits(val))); UNIT_ASSERT_VALUES_EQUAL_C(in.GetOffset(), 2u, (rem + ": " + NBitIO::PrintBits(val))); UNIT_ASSERT_C(!in.Eof(), (rem + ", " + ToString(__LINE__)).data()); - - val = 0; + + val = 0; UNIT_ASSERT_C(in.Read(val, 17), (rem + ": " + NBitIO::PrintBits(val)).data()); - + UNIT_ASSERT_VALUES_EQUAL_C(val, 0x10000u, (rem + ": " + NBitIO::PrintBits(val))); UNIT_ASSERT_VALUES_EQUAL_C(in.GetOffset(), 4u, (rem + ": " + NBitIO::PrintBits(val))); UNIT_ASSERT_C(!in.Eof(), (rem + ", " + ToString(__LINE__)).data()); - + UNIT_ASSERT_VALUES_EQUAL_C(in.GetOffset(), 4u, (rem + ": " + NBitIO::PrintBits(val))); - - { - ui32 rt = 0; - in.ReadRandom(30, rt, 10, 20); + + { + ui32 rt = 0; + in.ReadRandom(30, rt, 10, 20); UNIT_ASSERT_STRINGS_EQUAL(NBitIO::PrintBits(rt).data(), "00000000 00000000 00001111 01111100"); - } - val = 0; + } + val = 0; UNIT_ASSERT_C(in.Read(val, 3), (rem + ": " + NBitIO::PrintBits(val)).data()); - + UNIT_ASSERT_VALUES_EQUAL_C(val, 0x4u, (rem + ": " + NBitIO::PrintBits(val))); UNIT_ASSERT_VALUES_EQUAL_C(in.GetOffset(), 4u, (rem + ": " + NBitIO::PrintBits(val))); UNIT_ASSERT_C(!in.Eof(), (rem + ", " + ToString(__LINE__)).data()); - - val = 0; + + val = 0; UNIT_ASSERT_C(in.ReadWords<8>(val), (rem + ": " + NBitIO::PrintBits(val)).data()); - + UNIT_ASSERT_VALUES_EQUAL_C(val, 0xabcdefU, (rem + ": " + NBitIO::PrintBits(val))); UNIT_ASSERT_VALUES_EQUAL_C(in.GetOffset(), 8u, (rem + ": " + NBitIO::PrintBits(val))); UNIT_ASSERT_C(!in.Eof(), (rem + ", " + ToString(__LINE__)).data()); - - val = 0; + + val = 0; UNIT_ASSERT_C(in.Read(val, 3), (rem + ", " + ToString(__LINE__)).data()); - + UNIT_ASSERT_VALUES_EQUAL_C(val, 0x4u, (rem + ": " + NBitIO::PrintBits(val))); UNIT_ASSERT_VALUES_EQUAL_C(in.GetOffset(), 8u, (rem + ": " + NBitIO::PrintBits(val))); UNIT_ASSERT_C(!in.Eof(), (rem + ", " + ToString(__LINE__)).data()); - - val = 0; - in.Read(val, 7); + + val = 0; + in.Read(val, 7); UNIT_ASSERT_VALUES_EQUAL_C(val, 0x42u, (rem + ": " + NBitIO::PrintBits(val))); - } - - { - ui32 v = 0; - - DoRead(in, v); + } + + { + ui32 v = 0; + + DoRead(in, v); UNIT_ASSERT_VALUES_EQUAL_C(v, 1637415112ul, (rem + ": " + NBitIO::PrintBits(v))); - DoRead(in, v); + DoRead(in, v); UNIT_ASSERT_VALUES_EQUAL_C(v, 897998715u, (rem + ": " + NBitIO::PrintBits(v))); - DoRead(in, v); + DoRead(in, v); UNIT_ASSERT_VALUES_EQUAL_C(v, 201416527u, (rem + ": " + NBitIO::PrintBits(v))); - DoRead(in, v); + DoRead(in, v); UNIT_ASSERT_VALUES_EQUAL_C(v, 432344219u, (rem + ": " + NBitIO::PrintBits(v))); - + UNIT_ASSERT_VALUES_EQUAL_C(in.GetOffset(), 25u, (rem + ": " + NBitIO::PrintBits(v))); - } - - { - ui64 v8 = 0; - in.ReadSafe(v8, 64); - + } + + { + ui64 v8 = 0; + in.ReadSafe(v8, 64); + UNIT_ASSERT_VALUES_EQUAL_C(v8, 0xAAAAAAAAAAAAAAAAULL, (rem + ": " + NBitIO::PrintBits(v8))); UNIT_ASSERT_VALUES_EQUAL_C(in.GetOffset(), 33u, (rem + ": " + NBitIO::PrintBits(v8))); - - v8 = 0; - in.ReadK<64>(v8); - + + v8 = 0; + in.ReadK<64>(v8); + UNIT_ASSERT_VALUES_EQUAL_C(v8, 0x5555555555555555ULL, (rem + ": " + NBitIO::PrintBits(v8))); UNIT_ASSERT_VALUES_EQUAL_C(in.GetOffset(), 41u, (rem + ": " + NBitIO::PrintBits(v8))); - } - - ui32 v = 0; + } + + ui32 v = 0; UNIT_ASSERT_C(!in.Eof(), (rem + ", " + ToString(__LINE__)).data()); UNIT_ASSERT_C(in.Read(v, 5), (rem + ", " + ToString(__LINE__)).data()); UNIT_ASSERT_C(in.Eof(), (rem + ", " + ToString(__LINE__)).data()); - } - - void TestBitIO() { - { - TVec vec; - - { - NBitIO::TBitOutputYVector out(&vec); - DoBitOutput(out, ToString(__LINE__)); - } - + } + + void TestBitIO() { + { + TVec vec; + + { + NBitIO::TBitOutputYVector out(&vec); + DoBitOutput(out, ToString(__LINE__)); + } + CheckBits(vec, NBitIO::BITS_REF, ToString(__LINE__).data()); - - { - TBi in(vec); - DoBitInput(in, ToString(__LINE__)); - } - } - { - TVec vec; - vec.resize(41, 0); - { - NBitIO::TBitOutputArray out(vec.begin(), vec.size()); - DoBitOutput(out, ToString(__LINE__)); - } - + + { + TBi in(vec); + DoBitInput(in, ToString(__LINE__)); + } + } + { + TVec vec; + vec.resize(41, 0); + { + NBitIO::TBitOutputArray out(vec.begin(), vec.size()); + DoBitOutput(out, ToString(__LINE__)); + } + CheckBits(vec, NBitIO::BITS_REF, ToString(__LINE__).data()); - - { - TBi in(vec); - DoBitInput(in, ToString(__LINE__)); - } - } - } -}; - -UNIT_TEST_SUITE_REGISTRATION(TBitIOTest); + + { + TBi in(vec); + DoBitInput(in, ToString(__LINE__)); + } + } + } +}; + +UNIT_TEST_SUITE_REGISTRATION(TBitIOTest); diff --git a/library/cpp/bit_io/bitinput.h b/library/cpp/bit_io/bitinput.h index 397b311783..85711eb7f9 100644 --- a/library/cpp/bit_io/bitinput.h +++ b/library/cpp/bit_io/bitinput.h @@ -1,16 +1,16 @@ #pragma once -#include "bitinput_impl.h" - +#include "bitinput_impl.h" + #include <util/system/yassert.h> -#include <util/generic/vector.h> -#include <util/generic/yexception.h> +#include <util/generic/vector.h> +#include <util/generic/yexception.h> #include <iterator> -namespace NBitIO { +namespace NBitIO { // Based on junk/solar/codecs/bitstream.h - + class TBitInput: protected TBitInputImpl { public: template <typename TVec> @@ -18,27 +18,27 @@ namespace NBitIO { : TBitInputImpl(std::begin(vec), std::end(vec)) { } - + TBitInput(const char* start, const char* end) : TBitInputImpl(start, end) { } - + bool Eof() const { return EofImpl(); } - + ui64 GetOffset() const { ui64 bo = BitOffset(); return bo / 8 + !!(bo % 8); } - - using TBitInputImpl::GetBitLength; - + + using TBitInputImpl::GetBitLength; + ui64 GetBitOffset() const { return BitOffset() % 8; } - + public: // Read with static number of bits. // Preserves what's in result. @@ -49,7 +49,7 @@ namespace NBitIO { CopyToResultK<bits>(result, r64, skipbits); return ret; } - + // Read with static number of bits. // Zeroes other bits in result. template <ui64 bits, typename T> @@ -59,42 +59,42 @@ namespace NBitIO { result = r; return res; } - + // Shortcut to impl. template <ui64 bits> Y_FORCE_INLINE bool ReadK(ui64& result) { if (bits <= 56) return ReadKImpl<bits>(result); - + ui64 r1 = 0ULL; ui64 r2 = 0ULL; - + bool ret1 = ReadKImpl<56ULL>(r1); bool ret2 = ReadKImpl<(bits > 56ULL ? bits - 56ULL : 0) /*or else we get negative param in template*/>(r2); - + result = (r2 << 56ULL) | r1; - + return ret1 & ret2; } - + // It's safe to read up to 64 bits. // Zeroes other bits in result. template <typename T> Y_FORCE_INLINE bool ReadSafe(T& result, ui64 bits) { if (bits <= 56ULL) return Read(result, bits); - + ui64 r1 = 0ULL; ui64 r2 = 0ULL; - + bool ret1 = ReadKImpl<56ULL>(r1); bool ret2 = ReadImpl(r2, bits - 56ULL); - + result = (r2 << 56ULL) | r1; - + return ret1 & ret2; } - + // It's safe to read up to 64 bits. // Preserves what's in result. template <typename T> @@ -104,7 +104,7 @@ namespace NBitIO { CopyToResult(result, r64, bits, skipbits); return ret; } - + // Do not try to read more than 56 bits at once. Split in two reads or use ReadSafe. // Zeroes other bits in result. template <typename T> @@ -114,7 +114,7 @@ namespace NBitIO { result = r64; return ret; } - + // Shortcut to impl. Y_FORCE_INLINE bool Read(ui64& result, ui64 bits) { return ReadImpl(result, bits); @@ -129,7 +129,7 @@ namespace NBitIO { CopyToResult(result, r64, bits, skipbits); return ret; } - + // Unsigned wordwise read. Underlying data is splitted in "words" of "bits(data) + 1(flag)" bits. // Like this: (unsigned char)0x2E<3> (0010 1110) <=> 1110 0101 // fddd fddd @@ -142,13 +142,13 @@ namespace NBitIO { return retCode; } - + // Shortcut to impl. template <ui64 bits> Y_FORCE_INLINE bool ReadWords(ui64& result) { return ReadWordsImpl<bits>(result); } - + Y_FORCE_INLINE bool Back(int bits) { return Seek(BitOffset() - bits); } @@ -168,4 +168,4 @@ namespace NBitIO { return ret; } }; -} +} diff --git a/library/cpp/bit_io/bitinput_impl.h b/library/cpp/bit_io/bitinput_impl.h index d1c05f9be2..b13fbef101 100644 --- a/library/cpp/bit_io/bitinput_impl.h +++ b/library/cpp/bit_io/bitinput_impl.h @@ -1,9 +1,9 @@ -#pragma once - +#pragma once + #include <util/generic/bitops.h> #include <util/system/unaligned_mem.h> -namespace NBitIO { +namespace NBitIO { class TBitInputImpl { i64 RealStart; i64 Start; @@ -12,7 +12,7 @@ namespace NBitIO { const ui32 FakeStart; char Fake[16]; const i64 FStart; - + public: TBitInputImpl(const char* start, const char* end) : RealStart((i64)start) @@ -25,11 +25,11 @@ namespace NBitIO { memcpy(Fake, (const char*)(RealStart + (FakeStart >> 3)), (Length - FakeStart) >> 3); Start = FakeStart ? RealStart : FStart; } - - ui64 GetBitLength() const { - return Length; - } - + + ui64 GetBitLength() const { + return Length; + } + protected: template <ui32 bits> Y_FORCE_INLINE bool ReadKImpl(ui64& result) { @@ -45,7 +45,7 @@ namespace NBitIO { Start = FStart; return true; } - + Y_FORCE_INLINE bool ReadImpl(ui64& result, ui32 bits) { result = (ReadUnaligned<ui64>((const void*)(Start + (BOffset >> 3))) >> (BOffset & 7)) & MaskLowerBits(bits); BOffset += bits; @@ -57,13 +57,13 @@ namespace NBitIO { return false; } Start = FStart; - return true; - } - + return true; + } + Y_FORCE_INLINE bool EofImpl() const { return BOffset >= Length; } - + Y_FORCE_INLINE ui64 BitOffset() const { return BOffset; } @@ -81,30 +81,30 @@ namespace NBitIO { Y_FORCE_INLINE static void CopyToResultK(T& result, ui64 r64, ui64 skipbits) { result = (result & ~(Mask64(bits) << skipbits)) | (r64 << skipbits); } - + template <typename T> Y_FORCE_INLINE static void CopyToResult(T& result, ui64 r64, ui64 bits, ui64 skipbits) { result = (result & InverseMaskLowerBits(bits, skipbits)) | (r64 << skipbits); } - + public: template <ui64 bits> Y_FORCE_INLINE bool ReadWordsImpl(ui64& data) { data = 0; - + const ui64 haveMore = NthBit64(bits); const ui64 mask = Mask64(bits); ui64 current = 0; ui64 byteNo = 0; - + do { if (!ReadKImpl<bits + 1>(current)) return false; - + data |= (current & mask) << (byteNo++ * bits); } while (current & haveMore); - + return true; } }; -} +} diff --git a/library/cpp/bit_io/bitoutput.h b/library/cpp/bit_io/bitoutput.h index c54a9a7835..2b886c1f02 100644 --- a/library/cpp/bit_io/bitoutput.h +++ b/library/cpp/bit_io/bitoutput.h @@ -1,19 +1,19 @@ #pragma once #include <library/cpp/deprecated/accessors/accessors.h> - + #include <util/stream/output.h> #include <util/system/yassert.h> #include <util/generic/bitops.h> -#include <util/generic/vector.h> -#include <util/generic/yexception.h> +#include <util/generic/vector.h> +#include <util/generic/yexception.h> -namespace NBitIO { +namespace NBitIO { // Based on junk/solar/codecs/bitstream.h - + // Almost all code is hard tuned for sequential write performance. // Use tools/bursttrie/benchmarks/bitstreams_benchmark to check your changes. - + inline constexpr ui64 BytesUp(ui64 bits) { return (bits + 7ULL) >> 3ULL; } @@ -25,7 +25,7 @@ namespace NBitIO { ui64 FreeBits; ui64 Active; ui64 Offset; - + public: TBitOutputBase(TStorage* storage) : Storage(storage) @@ -38,15 +38,15 @@ namespace NBitIO { ui64 GetOffset() const { return Offset + BytesUp(64ULL - FreeBits); } - + ui64 GetBitOffset() const { return (64ULL - FreeBits) & 7ULL; } - + ui64 GetByteReminder() const { return FreeBits & 7ULL; } - + public: // interface @@ -58,14 +58,14 @@ namespace NBitIO { Active |= (data & MaskLowerBits(FreeBits)) << (64ULL - FreeBits); data >>= FreeBits; - + FreeBits = 0ULL; } Flush(); } - Active |= bits ? ((data & MaskLowerBits(bits)) << (64ULL - FreeBits)) : 0; + Active |= bits ? ((data & MaskLowerBits(bits)) << (64ULL - FreeBits)) : 0; FreeBits -= bits; } @@ -73,7 +73,7 @@ namespace NBitIO { Y_FORCE_INLINE void Write(ui64 data, ui64 bits, ui64 skipbits) { Write(data >> skipbits, bits); } - + // Unsigned wordwise write. Underlying data is splitted in "words" of "bits(data) + 1(flag)" bits. // Like this: (unsigned char)0x2E<3> (0000 0010 1110) <=> 1110 0101 // fddd fddd @@ -81,32 +81,32 @@ namespace NBitIO { Y_FORCE_INLINE void WriteWords(ui64 data) { do { ui64 part = data; - + data >>= bits; part |= FastZeroIfFalse(data, NthBit64(bits)); Write(part, bits + 1ULL); } while (data); } - + Y_FORCE_INLINE ui64 /* padded bits */ Flush() { const ui64 ubytes = 8ULL - (FreeBits >> 3ULL); - + if (ubytes) { Active <<= FreeBits; Active >>= FreeBits; - + Storage->WriteData((const char*)&Active, (const char*)&Active + ubytes); Offset += ubytes; } - + const ui64 padded = FreeBits & 7; FreeBits = 64ULL; Active = 0ULL; - + return padded; } - + virtual ~TBitOutputBase() { Flush(); } @@ -116,22 +116,22 @@ namespace NBitIO { return -i64(cond) & iftrue; } }; - + template <typename TVec> class TBitOutputVectorImpl { TVec* Data; - + public: void WriteData(const char* begin, const char* end) { NAccessors::Append(*Data, begin, end); } - + TBitOutputVectorImpl(TVec* data) : Data(data) { } }; - + template <typename TVec> struct TBitOutputVector: public TBitOutputVectorImpl<TVec>, public TBitOutputBase<TBitOutputVectorImpl<TVec>> { inline TBitOutputVector(TVec* data) @@ -144,7 +144,7 @@ namespace NBitIO { class TBitOutputArrayImpl { char* Data; size_t Left; - + public: void WriteData(const char* begin, const char* end) { size_t sz = end - begin; @@ -153,14 +153,14 @@ namespace NBitIO { Data += sz; Left -= sz; } - + TBitOutputArrayImpl(char* begin, size_t len) : Data(begin) , Left(len) { } }; - + struct TBitOutputArray: public TBitOutputArrayImpl, public TBitOutputBase<TBitOutputArrayImpl> { inline TBitOutputArray(char* begin, size_t len) : TBitOutputArrayImpl(begin, len) @@ -170,15 +170,15 @@ namespace NBitIO { }; using TBitOutputYVector = TBitOutputVector<TVector<char>>; - + class TBitOutputStreamImpl { IOutputStream* Out; - + public: void WriteData(const char* begin, const char* end) { Out->Write(begin, end - begin); } - + TBitOutputStreamImpl(IOutputStream* out) : Out(out) { @@ -192,4 +192,4 @@ namespace NBitIO { { } }; -} +} diff --git a/library/cpp/bit_io/ut/ya.make b/library/cpp/bit_io/ut/ya.make index 71d3da93c3..07ee5b4997 100644 --- a/library/cpp/bit_io/ut/ya.make +++ b/library/cpp/bit_io/ut/ya.make @@ -1,7 +1,7 @@ UNITTEST_FOR(library/cpp/bit_io) OWNER( - velavokr + velavokr g:util ) diff --git a/library/cpp/blockcodecs/codecs/legacy_zstd06/legacy_zstd06.cpp b/library/cpp/blockcodecs/codecs/legacy_zstd06/legacy_zstd06.cpp index a5c63b9be4..042f031679 100644 --- a/library/cpp/blockcodecs/codecs/legacy_zstd06/legacy_zstd06.cpp +++ b/library/cpp/blockcodecs/codecs/legacy_zstd06/legacy_zstd06.cpp @@ -2,14 +2,14 @@ #include <library/cpp/blockcodecs/core/common.h> #include <library/cpp/blockcodecs/core/register.h> -#include <contrib/libs/zstd06/common/zstd.h> -#include <contrib/libs/zstd06/common/zstd_static.h> +#include <contrib/libs/zstd06/common/zstd.h> +#include <contrib/libs/zstd06/common/zstd_static.h> using namespace NBlockCodecs; namespace { - struct TZStd06Codec: public TAddLengthCodec<TZStd06Codec> { - inline TZStd06Codec(unsigned level) + struct TZStd06Codec: public TAddLengthCodec<TZStd06Codec> { + inline TZStd06Codec(unsigned level) : Level(level) , MyName(TStringBuf("zstd06_") + ToString(Level)) { diff --git a/library/cpp/blockcodecs/codecs_ut.cpp b/library/cpp/blockcodecs/codecs_ut.cpp index 54c1ce376f..bfe5a23690 100644 --- a/library/cpp/blockcodecs/codecs_ut.cpp +++ b/library/cpp/blockcodecs/codecs_ut.cpp @@ -10,22 +10,22 @@ Y_UNIT_TEST_SUITE(TBlockCodecsTest) { using namespace NBlockCodecs; - TBuffer Buffer(TStringBuf b) { - TBuffer bb; + TBuffer Buffer(TStringBuf b) { + TBuffer bb; bb.Assign(b.data(), b.size()); - return bb; - } - + return bb; + } + void TestAllAtOnce(size_t n, size_t m) { TVector<TBuffer> datas; datas.emplace_back(); - datas.push_back(Buffer("na gorshke sidel korol")); - datas.push_back(Buffer(TStringBuf("", 1))); - datas.push_back(Buffer(" ")); - datas.push_back(Buffer(" ")); - datas.push_back(Buffer(" ")); - datas.push_back(Buffer(" ")); + datas.push_back(Buffer("na gorshke sidel korol")); + datas.push_back(Buffer(TStringBuf("", 1))); + datas.push_back(Buffer(" ")); + datas.push_back(Buffer(" ")); + datas.push_back(Buffer(" ")); + datas.push_back(Buffer(" ")); { TStringStream data; @@ -49,15 +49,15 @@ Y_UNIT_TEST_SUITE(TBlockCodecsTest) { } for (size_t j = 0; j < datas.size(); ++j) { - const TBuffer& data = datas[j]; + const TBuffer& data = datas[j]; TString res; - + try { - TBuffer e, d; - c->Encode(data, e); - c->Decode(e, d); - d.AsString(res); - UNIT_ASSERT_EQUAL(NBlockCodecs::TData(res), NBlockCodecs::TData(data)); + TBuffer e, d; + c->Encode(data, e); + c->Decode(e, d); + d.AsString(res); + UNIT_ASSERT_EQUAL(NBlockCodecs::TData(res), NBlockCodecs::TData(data)); } catch (...) { Cerr << c->Name() << "(" << res.Quote() << ")(" << TString{NBlockCodecs::TData(data)}.Quote() << ")" << Endl; diff --git a/library/cpp/blockcodecs/core/codecs.cpp b/library/cpp/blockcodecs/core/codecs.cpp index 3df2ca698e..21506e812b 100644 --- a/library/cpp/blockcodecs/core/codecs.cpp +++ b/library/cpp/blockcodecs/core/codecs.cpp @@ -1,5 +1,5 @@ #include "codecs.h" -#include "common.h" +#include "common.h" #include "register.h" #include <util/ysaveload.h> diff --git a/library/cpp/blockcodecs/core/common.h b/library/cpp/blockcodecs/core/common.h index cd8e5899dd..f05df4d334 100644 --- a/library/cpp/blockcodecs/core/common.h +++ b/library/cpp/blockcodecs/core/common.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include "codecs.h" @@ -17,7 +17,7 @@ #include <util/generic/algorithm.h> #include <util/generic/mem_copy.h> -namespace NBlockCodecs { +namespace NBlockCodecs { struct TDecompressError: public TDataError { TDecompressError(int code) { *this << "cannot decompress (errcode " << code << ")"; @@ -26,14 +26,14 @@ namespace NBlockCodecs { TDecompressError(size_t exp, size_t real) { *this << "broken input (expected len: " << exp << ", got: " << real << ")"; } - }; - + }; + struct TCompressError: public TDataError { TCompressError(int code) { *this << "cannot compress (errcode " << code << ")"; } - }; - + }; + struct TNullCodec: public ICodec { size_t DecompressedLength(const TData& in) const override { return in.size(); @@ -92,8 +92,8 @@ namespace NBlockCodecs { const auto len = ReadUnaligned<ui64>(in.data()); if (!len) - return 0; - + return 0; + Base()->DoDecompress(TData(in).Skip(sizeof(len)), out, len); return len; } diff --git a/library/cpp/build_info/build_info.h b/library/cpp/build_info/build_info.h index f8f825c12d..a494870ba3 100644 --- a/library/cpp/build_info/build_info.h +++ b/library/cpp/build_info/build_info.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include "sandbox.h" #include "build_info_static.h" diff --git a/library/cpp/cache/cache.h b/library/cpp/cache/cache.h index 632da7e93c..6dc997076d 100644 --- a/library/cpp/cache/cache.h +++ b/library/cpp/cache/cache.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <util/generic/algorithm.h> #include <util/generic/ptr.h> diff --git a/library/cpp/cgiparam/cgiparam.h b/library/cpp/cgiparam/cgiparam.h index 128388b6e0..87d1ab0ad4 100644 --- a/library/cpp/cgiparam/cgiparam.h +++ b/library/cpp/cgiparam/cgiparam.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <library/cpp/iterator/iterate_values.h> diff --git a/library/cpp/charset/codepage.h b/library/cpp/charset/codepage.h index c8b44a2f29..30a02a4610 100644 --- a/library/cpp/charset/codepage.h +++ b/library/cpp/charset/codepage.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include "doccodes.h" diff --git a/library/cpp/charset/doccodes.h b/library/cpp/charset/doccodes.h index e53026411c..75c87adf9e 100644 --- a/library/cpp/charset/doccodes.h +++ b/library/cpp/charset/doccodes.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once enum ECharset { CODES_UNSUPPORTED = -2, // valid but unsupported encoding diff --git a/library/cpp/charset/recyr.hh b/library/cpp/charset/recyr.hh index 233cd9bcc0..5ec8734bcf 100644 --- a/library/cpp/charset/recyr.hh +++ b/library/cpp/charset/recyr.hh @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <cstdlib> diff --git a/library/cpp/charset/recyr_int.hh b/library/cpp/charset/recyr_int.hh index ac4fc9eed9..353af53305 100644 --- a/library/cpp/charset/recyr_int.hh +++ b/library/cpp/charset/recyr_int.hh @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <util/charset/recode_result.h> #include <util/charset/utf8.h> diff --git a/library/cpp/charset/wide.h b/library/cpp/charset/wide.h index 7e8b979e80..32d30e849e 100644 --- a/library/cpp/charset/wide.h +++ b/library/cpp/charset/wide.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include "codepage.h" #include "iconv.h" @@ -182,21 +182,21 @@ namespace NDetail { return TBasicStringBuf<TCharTo>(dstbuf, dstFinalSize); } - // special implementation for robust utf8 functions - template <typename TResult> + // special implementation for robust utf8 functions + template <typename TResult> TWtringBuf RecodeUTF8Robust(const TStringBuf src, TResult& dst) { - // make enough room for re-coded string + // make enough room for re-coded string wchar16* dstbuf = TRecodeResultOps<TResult>::Reserve(dst, src.size() * TRecodeTraits<wchar16>::ReserveSize); - - // do re-coding - size_t written = 0; + + // do re-coding + size_t written = 0; UTF8ToWide<true>(src.data(), src.size(), dstbuf, written); - - // truncate result back to proper size - TRecodeResultOps<TResult>::Truncate(dst, written); - return TWtringBuf(dstbuf, written); - } - + + // truncate result back to proper size + TRecodeResultOps<TResult>::Truncate(dst, written); + return TWtringBuf(dstbuf, written); + } + template <typename TCharFrom> inline typename TRecodeTraits<TCharFrom>::TStringTo Recode(const TBasicStringBuf<TCharFrom> src, ECharset encoding) { typename TRecodeTraits<TCharFrom>::TStringTo res; @@ -207,17 +207,17 @@ namespace NDetail { // Write result into @dst. Return string-buffer pointing to re-coded content of @dst. -template <bool robust> +template <bool robust> inline TWtringBuf CharToWide(const TStringBuf src, TUtf16String& dst, ECharset encoding) { - if (robust && CODES_UTF8 == encoding) + if (robust && CODES_UTF8 == encoding) return ::NDetail::RecodeUTF8Robust(src, dst); return ::NDetail::Recode<char>(src, dst, encoding); } inline TWtringBuf CharToWide(const TStringBuf src, TUtf16String& dst, ECharset encoding) { return ::NDetail::Recode<char>(src, dst, encoding); -} - +} + inline TStringBuf WideToChar(const TWtringBuf src, TString& dst, ECharset encoding) { return ::NDetail::Recode<wchar16>(src, dst, encoding); } @@ -251,11 +251,11 @@ inline TUtf16String CharToWide(const char* text, size_t len, const CodePage& cp) } //! calls either to @c UTF8ToWide or @c CharToWide depending on the encoding type -template <bool robust> +template <bool robust> inline TUtf16String CharToWide(const char* text, size_t len, ECharset enc) { if (NCodepagePrivate::NativeCodepage(enc)) { if (enc == CODES_UTF8) - return UTF8ToWide<robust>(text, len); + return UTF8ToWide<robust>(text, len); return CharToWide(text, len, *CodePageByCharset(enc)); } diff --git a/library/cpp/codecs/README.md b/library/cpp/codecs/README.md index 26fa96da59..42646ccd97 100644 --- a/library/cpp/codecs/README.md +++ b/library/cpp/codecs/README.md @@ -1,8 +1,8 @@ This is a library of compression algorithms with a unified interface and serialization. See also library/cpp/codecs/static, where a support for statically compiled dictionaries is implemented. - + All algorithms have a common `ICodec` interface (described in codecs.h). - + The `ICodec` interface has the following methods:\ `virtual ui8 ICodec::Encode (TMemoryRegion, TBuffer&) const;`\ - Input - memory region. Output - filled buffer and the rest of the last byte, if it was not filled to the end.\ @@ -27,9 +27,9 @@ The `ICodec` interface has the following methods:\ For example, it allows you to save information about which combination of codecs was in use (see below).\ `virtual void Learn(ISequenceReader*);`\ - The interface for teaching codecs that use information about the distribution of data. - + In addition, the library has a number of utilities that allow a more flexible use of it. - + In the `ICodec` class the following methods are available:\ `static TCodecPtr GetInstance(const TString& name);`\ - Creation of a codec instance by a symbolic name\ diff --git a/library/cpp/codecs/codecs.cpp b/library/cpp/codecs/codecs.cpp index d2265dd9f9..b17a3156d2 100644 --- a/library/cpp/codecs/codecs.cpp +++ b/library/cpp/codecs/codecs.cpp @@ -1,69 +1,69 @@ -#include "codecs.h" -#include "tls_cache.h" - -#include <util/stream/mem.h> - -namespace NCodecs { +#include "codecs.h" +#include "tls_cache.h" + +#include <util/stream/mem.h> + +namespace NCodecs { void ICodec::Store(IOutputStream* out, TCodecPtr p) { if (!p.Get()) { ::Save(out, (ui16)0); return; } - + Y_ENSURE_EX(p->AlreadyTrained(), TCodecException() << "untrained codec " << p->GetName()); const TString& n = p->GetName(); Y_VERIFY(n.size() <= Max<ui16>()); ::Save(out, (ui16)n.size()); out->Write(n.data(), n.size()); p->Save(out); - } - + } + TCodecPtr ICodec::Restore(IInputStream* in) { ui16 l = 0; ::Load(in, l); - + if (!l) { return nullptr; } - + TString n; n.resize(l); - + Y_ENSURE_EX(in->Load(n.begin(), l) == l, TCodecException()); - + TCodecPtr p = ICodec::GetInstance(n); p->Load(in); p->Trained = true; return p; } - + TCodecPtr ICodec::RestoreFromString(TStringBuf s) { TMemoryInput minp{s.data(), s.size()}; return Restore(&minp); } - + TString ICodec::GetNameSafe(TCodecPtr p) { return !p ? TString("none") : p->GetName(); } - + ui8 TPipelineCodec::Encode(TStringBuf in, TBuffer& out) const { size_t res = Traits().ApproximateSizeOnEncode(in.size()); out.Reserve(res); out.Clear(); - + if (Pipeline.empty()) { out.Append(in.data(), in.size()); return 0; } else if (Pipeline.size() == 1) { return Pipeline.front()->Encode(in, out); } - + ui8 freelastbits = 0; - + auto buffer = TBufferTlsCache::TlsInstance().Item(); TBuffer& tmp = buffer.Get(); tmp.Reserve(res); - + for (auto it = Pipeline.begin(); it != Pipeline.end(); ++it) { if (it != Pipeline.begin()) { tmp.Clear(); @@ -72,15 +72,15 @@ namespace NCodecs { } freelastbits = (*it)->Encode(in, out); } - + return freelastbits; - } - + } + void TPipelineCodec::Decode(TStringBuf in, TBuffer& out) const { size_t res = Traits().ApproximateSizeOnDecode(in.size()); out.Reserve(res); out.Clear(); - + if (Pipeline.empty()) { out.Append(in.data(), in.size()); return; @@ -88,12 +88,12 @@ namespace NCodecs { Pipeline.front()->Decode(in, out); return; } - + auto buffer = TBufferTlsCache::TlsInstance().Item(); - + TBuffer& tmp = buffer.Get(); tmp.Reserve(res); - + for (TPipeline::const_reverse_iterator it = Pipeline.rbegin(); it != Pipeline.rend(); ++it) { if (it != Pipeline.rbegin()) { tmp.Clear(); @@ -101,40 +101,40 @@ namespace NCodecs { in = TStringBuf{tmp.data(), tmp.size()}; } (*it)->Decode(in, out); - } - } - + } + } + void TPipelineCodec::Save(IOutputStream* out) const { for (const auto& it : Pipeline) it->Save(out); - } - + } + void TPipelineCodec::Load(IInputStream* in) { for (const auto& it : Pipeline) { it->Load(in); it->SetTrained(true); } - } - + } + void TPipelineCodec::SetTrained(bool t) { for (const auto& it : Pipeline) { it->SetTrained(t); } - } - + } + TPipelineCodec& TPipelineCodec::AddCodec(TCodecPtr codec) { if (!codec) return *this; - + TCodecTraits tr = codec->Traits(); - + if (!MyName) { MyTraits.AssumesStructuredInput = tr.AssumesStructuredInput; MyTraits.SizeOfInputElement = tr.SizeOfInputElement; } else { MyName.append(':'); } - + MyName.append(codec->GetName()); MyTraits.PreservesPrefixGrouping &= tr.PreservesPrefixGrouping; MyTraits.PaddingBit = tr.PaddingBit; @@ -144,27 +144,27 @@ namespace NCodecs { MyTraits.SizeOnEncodeMultiplier *= tr.SizeOnEncodeMultiplier; MyTraits.SizeOnDecodeMultiplier *= tr.SizeOnDecodeMultiplier; MyTraits.RecommendedSampleSize = Max(MyTraits.RecommendedSampleSize, tr.RecommendedSampleSize); - + Pipeline.push_back(codec); return *this; - } - + } + void TPipelineCodec::DoLearnX(ISequenceReader& in, double sampleSizeMult) { if (!Traits().NeedsTraining) { return; } - + if (Pipeline.size() == 1) { Pipeline.back()->Learn(in); return; } - + TVector<TBuffer> trainingInput; - + TStringBuf r; while (in.NextRegion(r)) { trainingInput.emplace_back(r.data(), r.size()); - } + } TBuffer buff; for (const auto& it : Pipeline) { @@ -176,8 +176,8 @@ namespace NCodecs { buff.Swap(bit); } } - } - + } + bool TPipelineCodec::AlreadyTrained() const { for (const auto& it : Pipeline) { if (!it->AlreadyTrained()) @@ -185,6 +185,6 @@ namespace NCodecs { } return true; - } - -} + } + +} diff --git a/library/cpp/codecs/codecs.h b/library/cpp/codecs/codecs.h index aa7c24b4c6..cc5e72b285 100644 --- a/library/cpp/codecs/codecs.h +++ b/library/cpp/codecs/codecs.h @@ -1,63 +1,63 @@ -#pragma once - -#include "sample.h" - -#include <util/generic/bt_exception.h> -#include <util/generic/hash.h> -#include <util/generic/ptr.h> -#include <util/generic/singleton.h> - -#include <util/stream/input.h> -#include <util/stream/output.h> - +#pragma once + +#include "sample.h" + +#include <util/generic/bt_exception.h> +#include <util/generic/hash.h> +#include <util/generic/ptr.h> +#include <util/generic/singleton.h> + +#include <util/stream/input.h> +#include <util/stream/output.h> + #include <util/string/cast.h> -#include <util/string/vector.h> -#include <util/system/tls.h> -#include <util/ysaveload.h> - -namespace NCodecs { +#include <util/string/vector.h> +#include <util/system/tls.h> +#include <util/ysaveload.h> + +namespace NCodecs { class TCodecException: public TWithBackTrace<yexception> {}; - + class ICodec; - + using TCodecPtr = TIntrusivePtr<ICodec>; using TCodecConstPtr = TIntrusiveConstPtr<ICodec>; - + struct TCodecTraits { ui32 RecommendedSampleSize = 0; ui16 SizeOfInputElement = 1; ui8 SizeOnEncodeMultiplier = 1; ui8 SizeOnEncodeAddition = 0; ui8 SizeOnDecodeMultiplier = 1; - + bool NeedsTraining = false; bool PreservesPrefixGrouping = false; bool Irreversible = false; bool PaddingBit = 0; bool AssumesStructuredInput = false; - + size_t ApproximateSizeOnEncode(size_t sz) const { return sz * SizeOnEncodeMultiplier + SizeOnEncodeAddition; } - + size_t ApproximateSizeOnDecode(size_t sz) const { return sz * SizeOnDecodeMultiplier; } }; - + class ICodec: public TAtomicRefCount<ICodec> { protected: bool Trained = false; TCodecTraits MyTraits; - + public: TCodecTraits Traits() const { return MyTraits; } - + // the name of the codec (or its variant) to be used in the codec registry virtual TString GetName() const = 0; - + virtual ui8 /*free bits in last byte*/ Encode(TStringBuf, TBuffer&) const = 0; virtual ui8 Encode(const TBuffer& input, TBuffer& output) const { return Encode(TStringBuf(input.Data(), input.Data() + input.Size()), output); @@ -66,16 +66,16 @@ namespace NCodecs { virtual void Decode(const TBuffer& input, TBuffer& output) const { Decode(TStringBuf(input.Data(), input.Data() + input.Size()), output); } - + virtual ~ICodec() = default; - + virtual bool AlreadyTrained() const { return !Traits().NeedsTraining || Trained; } virtual void SetTrained(bool t) { Trained = t; } - + bool TryToLearn(ISequenceReader& r) { Trained = DoTryToLearn(r); return Trained; @@ -84,32 +84,32 @@ namespace NCodecs { void Learn(ISequenceReader& r) { LearnX(r, 1); } - + template <class TIter> void Learn(TIter beg, TIter end) { Learn(beg, end, IterToStringBuf<TIter>); } - + template <class TIter, class TGetter> void Learn(TIter beg, TIter end, TGetter getter) { auto sample = GetSample(beg, end, Traits().RecommendedSampleSize, getter); TSimpleSequenceReader<TBuffer> reader{sample}; Learn(reader); } - + static TCodecPtr GetInstance(TStringBuf name); - + static TVector<TString> GetCodecsList(); - + static TString GetNameSafe(TCodecPtr p); - + static void Store(IOutputStream* out, TCodecPtr p); static TCodecPtr Restore(IInputStream* in); static TCodecPtr RestoreFromString(TStringBuf); - + protected: virtual void DoLearn(ISequenceReader&) = 0; - + virtual bool DoTryToLearn(ISequenceReader& r) { DoLearn(r); return true; @@ -119,20 +119,20 @@ namespace NCodecs { virtual void DoLearnX(ISequenceReader& r, double /*sampleSizeMultiplier*/) { DoLearn(r); } - + virtual void Save(IOutputStream*) const { } virtual void Load(IInputStream*) { } friend class TPipelineCodec; - + public: // so the pipeline codec will know to adjust the sample for the subcodecs void LearnX(ISequenceReader& r, double sampleSizeMult) { DoLearnX(r, sampleSizeMult); Trained = true; } - + template <class TIter> void LearnX(TIter beg, TIter end, double sampleSizeMult) { auto sample = GetSample(beg, end, Traits().RecommendedSampleSize * sampleSizeMult); @@ -140,54 +140,54 @@ namespace NCodecs { LearnX(reader, sampleSizeMult); } }; - + class TBasicTrivialCodec: public ICodec { public: ui8 Encode(TStringBuf in, TBuffer& out) const override { out.Assign(in.data(), in.size()); return 0; } - + void Decode(TStringBuf in, TBuffer& out) const override { Encode(in, out); } - + protected: void DoLearn(ISequenceReader&) override { } }; - + class TTrivialCodec: public TBasicTrivialCodec { public: TTrivialCodec() { MyTraits.PreservesPrefixGrouping = true; } - + static TStringBuf MyName() { return "trivial"; } - + TString GetName() const override { return ToString(MyName()); } }; - + class TTrivialTrainableCodec: public TBasicTrivialCodec { public: TTrivialTrainableCodec() { MyTraits.PreservesPrefixGrouping = true; MyTraits.NeedsTraining = true; } - + static TStringBuf MyName() { return "trivial-trainable"; } - + TString GetName() const override { return ToString(MyName()); } }; - + class TNullCodec: public ICodec { public: TNullCodec() { @@ -195,31 +195,31 @@ namespace NCodecs { MyTraits.SizeOnDecodeMultiplier = 0; MyTraits.SizeOnEncodeMultiplier = 0; } - + TString GetName() const override { return "null"; } - + ui8 Encode(TStringBuf, TBuffer& out) const override { out.Clear(); return 0; } - + void Decode(TStringBuf, TBuffer& out) const override { out.Clear(); } - + protected: void DoLearn(ISequenceReader&) override { } }; - + class TPipelineCodec: public ICodec { typedef TVector<TCodecPtr> TPipeline; - + TPipeline Pipeline; TString MyName; - + public: explicit TPipelineCodec(TCodecPtr c0 = nullptr, TCodecPtr c1 = nullptr, TCodecPtr c2 = nullptr, TCodecPtr c3 = nullptr) { MyTraits.PreservesPrefixGrouping = true; @@ -228,32 +228,32 @@ namespace NCodecs { AddCodec(c2); AddCodec(c3); } - + TString GetName() const override { return MyName; } - + ui8 Encode(TStringBuf in, TBuffer& out) const override; void Decode(TStringBuf in, TBuffer& out) const override; - + public: /* - * Add codecs in the following order: - * uncompressed -> codec0 | codec1 | ... | codecN -> compressed - */ + * Add codecs in the following order: + * uncompressed -> codec0 | codec1 | ... | codecN -> compressed + */ TPipelineCodec& AddCodec(TCodecPtr codec); - + bool AlreadyTrained() const override; void SetTrained(bool t) override; - + protected: void DoLearn(ISequenceReader& in) override { DoLearnX(in, 1); } - + void DoLearnX(ISequenceReader& in, double sampleSizeMult) override; void Save(IOutputStream* out) const override; void Load(IInputStream* in) override; }; - -} + +} diff --git a/library/cpp/codecs/codecs_registry.cpp b/library/cpp/codecs/codecs_registry.cpp index 7ccfd07a8a..17d07062ab 100644 --- a/library/cpp/codecs/codecs_registry.cpp +++ b/library/cpp/codecs/codecs_registry.cpp @@ -1,104 +1,104 @@ -#include "codecs_registry.h" -#include "delta_codec.h" -#include "huffman_codec.h" -#include "pfor_codec.h" -#include "solar_codec.h" -#include "comptable_codec.h" -#include "zstd_dict_codec.h" - +#include "codecs_registry.h" +#include "delta_codec.h" +#include "huffman_codec.h" +#include "pfor_codec.h" +#include "solar_codec.h" +#include "comptable_codec.h" +#include "zstd_dict_codec.h" + #include <library/cpp/blockcodecs/codecs.h> - -#include <util/string/builder.h> + +#include <util/string/builder.h> #include <util/string/cast.h> - -namespace NCodecs { - TCodecPtr ICodec::GetInstance(TStringBuf name) { + +namespace NCodecs { + TCodecPtr ICodec::GetInstance(TStringBuf name) { return Singleton<NPrivate::TCodecRegistry>()->GetCodec(name); - } - + } + TVector<TString> ICodec::GetCodecsList() { return Singleton<NPrivate::TCodecRegistry>()->GetCodecsList(); - } - - namespace NPrivate { - void TCodecRegistry::RegisterFactory(TFactoryPtr fac) { + } + + namespace NPrivate { + void TCodecRegistry::RegisterFactory(TFactoryPtr fac) { TVector<TString> names = fac->ListNames(); - for (const auto& name : names) { + for (const auto& name : names) { Y_VERIFY(!Registry.contains(name), "already has %s", name.data()); - Registry[name] = fac; - } + Registry[name] = fac; + } } - TCodecPtr TCodecRegistry::GetCodec(TStringBuf name) const { - using namespace NPrivate; - - if (!name || "none" == name) { - return nullptr; - } - - if (TStringBuf::npos == name.find(':')) { + TCodecPtr TCodecRegistry::GetCodec(TStringBuf name) const { + using namespace NPrivate; + + if (!name || "none" == name) { + return nullptr; + } + + if (TStringBuf::npos == name.find(':')) { Y_ENSURE_EX(Registry.contains(name), TNoCodecException(name)); - return Registry.find(name)->second->MakeCodec(name); - } else { - TPipelineCodec* pipe = new TPipelineCodec; - + return Registry.find(name)->second->MakeCodec(name); + } else { + TPipelineCodec* pipe = new TPipelineCodec; + do { - TStringBuf v = name.NextTok(':'); - pipe->AddCodec(GetCodec(v)); - } while (name); - - return pipe; - } - } - + TStringBuf v = name.NextTok(':'); + pipe->AddCodec(GetCodec(v)); + } while (name); + + return pipe; + } + } + TVector<TString> TCodecRegistry::GetCodecsList() const { - using namespace NPrivate; + using namespace NPrivate; TVector<TString> vs; - vs.push_back("none"); - - for (const auto& it : Registry) { - vs.push_back(it.first); - } - - Sort(vs.begin(), vs.end()); - return vs; - } - + vs.push_back("none"); + + for (const auto& it : Registry) { + vs.push_back(it.first); + } + + Sort(vs.begin(), vs.end()); + return vs; + } + struct TSolarCodecFactory : ICodecFactory { - TCodecPtr MakeCodec(TStringBuf name) const override { - if (TSolarCodec::MyNameShortInt() == name) { - return new TSolarCodecShortInt(); - } - if (TSolarCodec::MyName() == name) { - return new TSolarCodec(); - } + TCodecPtr MakeCodec(TStringBuf name) const override { + if (TSolarCodec::MyNameShortInt() == name) { + return new TSolarCodecShortInt(); + } + if (TSolarCodec::MyName() == name) { + return new TSolarCodec(); + } if (name.EndsWith(TStringBuf("-a"))) { return MakeCodecImpl<TAdaptiveSolarCodec>(name, name.SubStr(TSolarCodec::MyName().size()).Chop(2)); - } else { + } else { return MakeCodecImpl<TSolarCodec>(name, name.SubStr(TSolarCodec::MyName().size())); - } - } - + } + } + template <class TCodecCls> - TCodecPtr MakeCodecImpl(const TStringBuf& name, const TStringBuf& type) const { + TCodecPtr MakeCodecImpl(const TStringBuf& name, const TStringBuf& type) const { if (TStringBuf("-8k") == type) { - return new TCodecCls(1 << 13); - } + return new TCodecCls(1 << 13); + } if (TStringBuf("-16k") == type) { - return new TCodecCls(1 << 14); - } + return new TCodecCls(1 << 14); + } if (TStringBuf("-32k") == type) { - return new TCodecCls(1 << 15); - } + return new TCodecCls(1 << 15); + } if (TStringBuf("-64k") == type) { - return new TCodecCls(1 << 16); - } + return new TCodecCls(1 << 16); + } if (TStringBuf("-256k") == type) { - return new TCodecCls(1 << 18); - } - ythrow TNoCodecException(name); - } - + return new TCodecCls(1 << 18); + } + ythrow TNoCodecException(name); + } + TVector<TString> ListNames() const override { TVector<TString> vs; vs.push_back(ToString(TSolarCodec::MyName())); @@ -113,114 +113,114 @@ namespace NCodecs { vs.push_back(ToString(TSolarCodec::MyName64kAdapt())); vs.push_back(ToString(TSolarCodec::MyName256kAdapt())); vs.push_back(ToString(TSolarCodec::MyNameShortInt())); - return vs; - } - }; - + return vs; + } + }; + struct TZStdDictCodecFactory : ICodecFactory { - TCodecPtr MakeCodec(TStringBuf name) const override { - return new TZStdDictCodec(TZStdDictCodec::ParseCompressionName(name)); - } - + TCodecPtr MakeCodec(TStringBuf name) const override { + return new TZStdDictCodec(TZStdDictCodec::ParseCompressionName(name)); + } + TVector<TString> ListNames() const override { - return TZStdDictCodec::ListCompressionNames(); - } - }; - + return TZStdDictCodec::ListCompressionNames(); + } + }; + struct TCompTableCodecFactory : ICodecFactory { - TCodecPtr MakeCodec(TStringBuf name) const override { - if (TCompTableCodec::MyNameHQ() == name) { - return new TCompTableCodec(TCompTableCodec::Q_HIGH); - } else if (TCompTableCodec::MyNameLQ() == name) { - return new TCompTableCodec(TCompTableCodec::Q_LOW); - } else { - Y_ENSURE_EX(false, TNoCodecException(name)); - return nullptr; - } - } - + TCodecPtr MakeCodec(TStringBuf name) const override { + if (TCompTableCodec::MyNameHQ() == name) { + return new TCompTableCodec(TCompTableCodec::Q_HIGH); + } else if (TCompTableCodec::MyNameLQ() == name) { + return new TCompTableCodec(TCompTableCodec::Q_LOW); + } else { + Y_ENSURE_EX(false, TNoCodecException(name)); + return nullptr; + } + } + TVector<TString> ListNames() const override { TVector<TString> vs; vs.push_back(ToString(TCompTableCodec::MyNameHQ())); vs.push_back(ToString(TCompTableCodec::MyNameLQ())); - return vs; - } - }; - + return vs; + } + }; + struct TBlockCodec : ICodec { - const NBlockCodecs::ICodec* Codec; - - TBlockCodec(TStringBuf name) + const NBlockCodecs::ICodec* Codec; + + TBlockCodec(TStringBuf name) : Codec(NBlockCodecs::Codec(name)) - { - } - + { + } + TString GetName() const override { return ToString(Codec->Name()); - } - - ui8 Encode(TStringBuf r, TBuffer& b) const override { - Codec->Encode(r, b); - return 0; - } - - void Decode(TStringBuf r, TBuffer& b) const override { - // TODO: throws exception that is not TCodecException - Codec->Decode(r, b); - } - - protected: - void DoLearn(ISequenceReader&) override { - } - }; - + } + + ui8 Encode(TStringBuf r, TBuffer& b) const override { + Codec->Encode(r, b); + return 0; + } + + void Decode(TStringBuf r, TBuffer& b) const override { + // TODO: throws exception that is not TCodecException + Codec->Decode(r, b); + } + + protected: + void DoLearn(ISequenceReader&) override { + } + }; + struct TBlockCodecsFactory : ICodecFactory { using TRegistry = THashMap<TString, TCodecPtr>; - TRegistry Registry; - + TRegistry Registry; + TBlockCodecsFactory() { for (TStringBuf codec : NBlockCodecs::ListAllCodecs()) { Register(codec); } - } - - void Register(TStringBuf name) { - TCodecPtr p = Registry[name] = new TBlockCodec(name); - Registry[p->GetName()] = p; - } - - TCodecPtr MakeCodec(TStringBuf name) const override { + } + + void Register(TStringBuf name) { + TCodecPtr p = Registry[name] = new TBlockCodec(name); + Registry[p->GetName()] = p; + } + + TCodecPtr MakeCodec(TStringBuf name) const override { if (!Registry.contains(name)) { - ythrow TNoCodecException(name); - } - return Registry.find(name)->second; - } - + ythrow TNoCodecException(name); + } + return Registry.find(name)->second; + } + TVector<TString> ListNames() const override { TVector<TString> res; - for (const auto& it : Registry) { - res.push_back(it.first); - } - return res; - } - }; - + for (const auto& it : Registry) { + res.push_back(it.first); + } + return res; + } + }; + TCodecRegistry::TCodecRegistry() { - RegisterFactory(new TInstanceFactory<TTrivialCodec>); - RegisterFactory(new TInstanceFactory<TTrivialTrainableCodec>); - RegisterFactory(new TInstanceFactory<THuffmanCodec>); + RegisterFactory(new TInstanceFactory<TTrivialCodec>); + RegisterFactory(new TInstanceFactory<TTrivialTrainableCodec>); + RegisterFactory(new TInstanceFactory<THuffmanCodec>); RegisterFactory(new TInstanceFactory<TPForCodec<ui64, true>>); RegisterFactory(new TInstanceFactory<TPForCodec<ui32, true>>); - RegisterFactory(new TSolarCodecFactory); - RegisterFactory(new TZStdDictCodecFactory); - RegisterFactory(new TCompTableCodecFactory); - RegisterFactory(new TBlockCodecsFactory); - } - - } - - void RegisterCodecFactory(TCodecFactoryPtr fact) { - Singleton<NPrivate::TCodecRegistry>()->RegisterFactory(fact); - } - -} + RegisterFactory(new TSolarCodecFactory); + RegisterFactory(new TZStdDictCodecFactory); + RegisterFactory(new TCompTableCodecFactory); + RegisterFactory(new TBlockCodecsFactory); + } + + } + + void RegisterCodecFactory(TCodecFactoryPtr fact) { + Singleton<NPrivate::TCodecRegistry>()->RegisterFactory(fact); + } + +} diff --git a/library/cpp/codecs/codecs_registry.h b/library/cpp/codecs/codecs_registry.h index 31170afd62..53710310d5 100644 --- a/library/cpp/codecs/codecs_registry.h +++ b/library/cpp/codecs/codecs_registry.h @@ -1,60 +1,60 @@ -#pragma once - -#include "codecs.h" +#pragma once + +#include "codecs.h" #include <util/string/cast.h> - -namespace NCodecs { + +namespace NCodecs { struct TNoCodecException : TCodecException { TNoCodecException(TStringBuf name) { - (*this) << "unknown codec: " << name; - } - }; - + (*this) << "unknown codec: " << name; + } + }; + struct ICodecFactory : TAtomicRefCount<ICodecFactory> { - virtual ~ICodecFactory() = default; - virtual TCodecPtr MakeCodec(TStringBuf name) const = 0; + virtual ~ICodecFactory() = default; + virtual TCodecPtr MakeCodec(TStringBuf name) const = 0; virtual TVector<TString> ListNames() const = 0; - }; - - typedef TIntrusivePtr<ICodecFactory> TCodecFactoryPtr; - - namespace NPrivate { + }; + + typedef TIntrusivePtr<ICodecFactory> TCodecFactoryPtr; + + namespace NPrivate { template <typename TCodec> struct TInstanceFactory : ICodecFactory { - TCodecPtr MakeCodec(TStringBuf) const override { - return new TCodec; - } - + TCodecPtr MakeCodec(TStringBuf) const override { + return new TCodec; + } + TVector<TString> ListNames() const override { TVector<TString> vs; vs.push_back(ToString(TCodec::MyName())); - return vs; - } - }; - - class TCodecRegistry { + return vs; + } + }; + + class TCodecRegistry { using TRegistry = THashMap<TString, TIntrusivePtr<ICodecFactory>>; - TRegistry Registry; - - public: - using TFactoryPtr = TIntrusivePtr<ICodecFactory>; - - TCodecRegistry(); - - void RegisterFactory(TFactoryPtr fac); - - TCodecPtr GetCodec(TStringBuf name) const; - + TRegistry Registry; + + public: + using TFactoryPtr = TIntrusivePtr<ICodecFactory>; + + TCodecRegistry(); + + void RegisterFactory(TFactoryPtr fac); + + TCodecPtr GetCodec(TStringBuf name) const; + TVector<TString> GetCodecsList() const; - }; - - } - - void RegisterCodecFactory(TCodecFactoryPtr fact); - + }; + + } + + void RegisterCodecFactory(TCodecFactoryPtr fact); + template <typename TCodec> - void RegisterCodec() { - RegisterCodecFactory(new NPrivate::TInstanceFactory<TCodec>()); - } - -} + void RegisterCodec() { + RegisterCodecFactory(new NPrivate::TInstanceFactory<TCodec>()); + } + +} diff --git a/library/cpp/codecs/comptable_codec.cpp b/library/cpp/codecs/comptable_codec.cpp index 1eca4354c6..476b8ada80 100644 --- a/library/cpp/codecs/comptable_codec.cpp +++ b/library/cpp/codecs/comptable_codec.cpp @@ -1,108 +1,108 @@ -#include "comptable_codec.h" - +#include "comptable_codec.h" + #include <library/cpp/comptable/comptable.h> #include <util/string/cast.h> - -namespace NCodecs { + +namespace NCodecs { class TCompTableCodec::TImpl: public TAtomicRefCount<TImpl> { - public: - TImpl(EQuality q) - : Quality(q) + public: + TImpl(EQuality q) + : Quality(q) { } - - void Init() { - Compressor.Reset(new NCompTable::TChunkCompressor{(bool)Quality, Table}); - Decompressor.Reset(new NCompTable::TChunkDecompressor{(bool)Quality, Table}); - } - - ui8 Encode(TStringBuf in, TBuffer& out) const { - out.Clear(); - if (!in) { - return 0; - } - + + void Init() { + Compressor.Reset(new NCompTable::TChunkCompressor{(bool)Quality, Table}); + Decompressor.Reset(new NCompTable::TChunkDecompressor{(bool)Quality, Table}); + } + + ui8 Encode(TStringBuf in, TBuffer& out) const { + out.Clear(); + if (!in) { + return 0; + } + TVector<char> result; - Compressor->Compress(in, &result); - out.Assign(&result[0], result.size()); - return 0; - } - - void Decode(TStringBuf in, TBuffer& out) const { - out.Clear(); - if (!in) { - return; - } - + Compressor->Compress(in, &result); + out.Assign(&result[0], result.size()); + return 0; + } + + void Decode(TStringBuf in, TBuffer& out) const { + out.Clear(); + if (!in) { + return; + } + TVector<char> result; - Decompressor->Decompress(in, &result); - out.Assign(&result[0], result.size()); - } - - void DoLearn(ISequenceReader& in) { - NCompTable::TDataSampler sampler; - TStringBuf region; - while (in.NextRegion(region)) { - if (!region) { - continue; - } - - sampler.AddStat(region); - } - - sampler.BuildTable(Table); - Init(); - } - + Decompressor->Decompress(in, &result); + out.Assign(&result[0], result.size()); + } + + void DoLearn(ISequenceReader& in) { + NCompTable::TDataSampler sampler; + TStringBuf region; + while (in.NextRegion(region)) { + if (!region) { + continue; + } + + sampler.AddStat(region); + } + + sampler.BuildTable(Table); + Init(); + } + void Save(IOutputStream* out) const { - ::Save(out, Table); - } - + ::Save(out, Table); + } + void Load(IInputStream* in) { - ::Load(in, Table); - Init(); - } - - NCompTable::TCompressorTable Table; - THolder<NCompTable::TChunkCompressor> Compressor; - THolder<NCompTable::TChunkDecompressor> Decompressor; - const EQuality Quality; - static const ui32 SampleSize = Max(NCompTable::TDataSampler::Size * 4, (1 << 22) * 5); - }; - - TCompTableCodec::TCompTableCodec(EQuality q) - : Impl(new TImpl{q}) - { - MyTraits.NeedsTraining = true; - MyTraits.SizeOnEncodeMultiplier = 2; - MyTraits.SizeOnDecodeMultiplier = 10; - MyTraits.RecommendedSampleSize = TImpl::SampleSize; - } - - TCompTableCodec::~TCompTableCodec() = default; - + ::Load(in, Table); + Init(); + } + + NCompTable::TCompressorTable Table; + THolder<NCompTable::TChunkCompressor> Compressor; + THolder<NCompTable::TChunkDecompressor> Decompressor; + const EQuality Quality; + static const ui32 SampleSize = Max(NCompTable::TDataSampler::Size * 4, (1 << 22) * 5); + }; + + TCompTableCodec::TCompTableCodec(EQuality q) + : Impl(new TImpl{q}) + { + MyTraits.NeedsTraining = true; + MyTraits.SizeOnEncodeMultiplier = 2; + MyTraits.SizeOnDecodeMultiplier = 10; + MyTraits.RecommendedSampleSize = TImpl::SampleSize; + } + + TCompTableCodec::~TCompTableCodec() = default; + TString TCompTableCodec::GetName() const { return ToString(Impl->Quality ? MyNameHQ() : MyNameLQ()); - } - - ui8 TCompTableCodec::Encode(TStringBuf in, TBuffer& out) const { - return Impl->Encode(in, out); - } - - void TCompTableCodec::Decode(TStringBuf in, TBuffer& out) const { - Impl->Decode(in, out); - } - - void TCompTableCodec::DoLearn(ISequenceReader& in) { - Impl->DoLearn(in); - } - + } + + ui8 TCompTableCodec::Encode(TStringBuf in, TBuffer& out) const { + return Impl->Encode(in, out); + } + + void TCompTableCodec::Decode(TStringBuf in, TBuffer& out) const { + Impl->Decode(in, out); + } + + void TCompTableCodec::DoLearn(ISequenceReader& in) { + Impl->DoLearn(in); + } + void TCompTableCodec::Save(IOutputStream* out) const { - Impl->Save(out); - } - + Impl->Save(out); + } + void TCompTableCodec::Load(IInputStream* in) { - Impl->Load(in); - } - -} + Impl->Load(in); + } + +} diff --git a/library/cpp/codecs/comptable_codec.h b/library/cpp/codecs/comptable_codec.h index 1a10c8241e..7ba4f4c543 100644 --- a/library/cpp/codecs/comptable_codec.h +++ b/library/cpp/codecs/comptable_codec.h @@ -1,40 +1,40 @@ -#pragma once - -#include "codecs.h" - -#include <util/generic/ptr.h> - -namespace NCodecs { +#pragma once + +#include "codecs.h" + +#include <util/generic/ptr.h> + +namespace NCodecs { class TCompTableCodec: public ICodec { class TImpl; TIntrusivePtr<TImpl> Impl; - + public: enum EQuality { Q_LOW = 0, Q_HIGH = 1 }; - + explicit TCompTableCodec(EQuality q = Q_HIGH); ~TCompTableCodec() override; - + static TStringBuf MyNameHQ() { return "comptable-hq"; } static TStringBuf MyNameLQ() { return "comptable-lq"; } - + TString GetName() const override; - + ui8 Encode(TStringBuf in, TBuffer& out) const override; - + void Decode(TStringBuf in, TBuffer& out) const override; - + protected: void DoLearn(ISequenceReader& in) override; void Save(IOutputStream* out) const override; void Load(IInputStream* in) override; }; - -} + +} diff --git a/library/cpp/codecs/delta_codec.cpp b/library/cpp/codecs/delta_codec.cpp index 28d6b6e3bb..61606d6f6f 100644 --- a/library/cpp/codecs/delta_codec.cpp +++ b/library/cpp/codecs/delta_codec.cpp @@ -1,6 +1,6 @@ -#include "delta_codec.h" - -namespace NCodecs { +#include "delta_codec.h" + +namespace NCodecs { template <> TStringBuf TDeltaCodec<ui64, true>::MyName() { return "delta64-unsigned"; @@ -17,5 +17,5 @@ namespace NCodecs { TStringBuf TDeltaCodec<ui32, false>::MyName() { return "delta32-signed"; } - -} + +} diff --git a/library/cpp/codecs/delta_codec.h b/library/cpp/codecs/delta_codec.h index 7398b3ae80..21325825e6 100644 --- a/library/cpp/codecs/delta_codec.h +++ b/library/cpp/codecs/delta_codec.h @@ -1,102 +1,102 @@ -#pragma once - -#include "codecs.h" - +#pragma once + +#include "codecs.h" + #include <util/generic/array_ref.h> -#include <util/generic/typetraits.h> +#include <util/generic/typetraits.h> #include <util/generic/bitops.h> #include <util/string/cast.h> - -namespace NCodecs { + +namespace NCodecs { template <typename T = ui64, bool UnsignedDelta = true> class TDeltaCodec: public ICodec { static_assert(std::is_integral<T>::value, "expect std::is_integral<T>::value"); - + public: using TUnsigned = std::make_unsigned_t<T>; using TSigned = std::make_signed_t<T>; using TDelta = std::conditional_t<UnsignedDelta, TUnsigned, TSigned>; - + private: const TDelta MinDelta{Min<TDelta>()}; const TDelta MaxDelta{Max<TDelta>() - 1}; const TDelta InvalidDelta{MaxDelta + 1}; - + Y_FORCE_INLINE static TDelta AddSafe(TUnsigned a, TUnsigned b) { return a + b; } - + Y_FORCE_INLINE static TDelta SubSafe(TUnsigned a, TUnsigned b) { return a - b; } - + public: struct TDecoder { const TDelta InvalidDelta{Max<TDelta>()}; - + T Last = 0; T Result = 0; - + bool First = true; bool Invalid = false; - + Y_FORCE_INLINE bool Decode(TDelta t) { if (Y_UNLIKELY(First)) { First = false; Result = Last = t; return true; } - + if (Y_UNLIKELY(Invalid)) { Invalid = false; Last = 0; Result = t; return true; } - + Result = (Last += t); Invalid = t == InvalidDelta; return !Invalid; - } + } }; - + public: static TStringBuf MyName(); - + TDeltaCodec() { MyTraits.SizeOfInputElement = sizeof(T); MyTraits.AssumesStructuredInput = true; - } - + } + TString GetName() const override { return ToString(MyName()); } - + template <class TItem> static void AppendTo(TBuffer& b, TItem t) { b.Append((char*)&t, sizeof(t)); } - + ui8 Encode(TStringBuf s, TBuffer& b) const override { b.Clear(); if (s.empty()) { return 0; } - + b.Reserve(s.size()); TArrayRef<const T> tin{(const T*)s.data(), s.size() / sizeof(T)}; - + const T* it = tin.begin(); TDelta last = *(it++); AppendTo(b, last); - + TDelta maxt = SubSafe(MaxDelta, last); TDelta mint = AddSafe(MinDelta, last); - + for (; it != tin.end(); ++it) { TDelta t = *it; - + if (Y_LIKELY((t >= mint) & (t <= maxt))) { AppendTo(b, t - last); last = t; @@ -111,33 +111,33 @@ namespace NCodecs { mint = MinDelta; } } - + return 0; } - + void Decode(TStringBuf s, TBuffer& b) const override { b.Clear(); if (s.empty()) { return; - } - + } + b.Reserve(s.size()); TArrayRef<const T> tin{(const T*)s.data(), s.size() / sizeof(T)}; - + TDecoder dec; - + for (const T* it = tin.begin(); it != tin.end(); ++it) { T tmp; memcpy(&tmp, it, sizeof(tmp)); if (dec.Decode(tmp)) { AppendTo(b, dec.Result); } - } - } - + } + } + protected: void DoLearn(ISequenceReader&) override { } }; - -} + +} diff --git a/library/cpp/codecs/float_huffman.h b/library/cpp/codecs/float_huffman.h index f03fc240ce..786a8eae1d 100644 --- a/library/cpp/codecs/float_huffman.h +++ b/library/cpp/codecs/float_huffman.h @@ -5,7 +5,7 @@ #include <util/generic/strbuf.h> #include <array> - + namespace NCodecs::NFloatHuff { TString Encode(TArrayRef<const float> factors); diff --git a/library/cpp/codecs/greedy_dict/gd_builder.cpp b/library/cpp/codecs/greedy_dict/gd_builder.cpp index 2fb46029bf..561bfbca01 100644 --- a/library/cpp/codecs/greedy_dict/gd_builder.cpp +++ b/library/cpp/codecs/greedy_dict/gd_builder.cpp @@ -1,85 +1,85 @@ -#include "gd_builder.h" - +#include "gd_builder.h" + #include <library/cpp/string_utils/relaxed_escaper/relaxed_escaper.h> -#include <util/generic/algorithm.h> - -#include <util/random/shuffle.h> +#include <util/generic/algorithm.h> + +#include <util/random/shuffle.h> #include <util/stream/output.h> -#include <util/string/printf.h> -#include <util/system/rusage.h> - -namespace NGreedyDict { +#include <util/string/printf.h> +#include <util/system/rusage.h> + +namespace NGreedyDict { void TDictBuilder::RebuildCounts(ui32 maxcand, bool final) { if (!Current) { Current = MakeHolder<TEntrySet>(); Current->InitWithAlpha(); } - + TEntrySet& set = *Current; - + for (auto& it : set) it.Count = 0; - + CompoundCounts = nullptr; CompoundCountsPool.Clear(); - + if (!final) { CompoundCounts = MakeHolder<TCompoundCounts>(&CompoundCountsPool); CompoundCounts->reserve(maxcand); } - + Shuffle(Input.begin(), Input.end(), Rng); - + for (auto str : Input) { if (!final && CompoundCounts->size() > maxcand) break; - + i32 prev = -1; - + while (!!str) { TEntry* e = set.FindPrefix(str); ui32 num = e->Number; - + e->Count += 1; if (!final && prev >= 0) { (*CompoundCounts)[Compose(prev, num)] += 1; } - + prev = num; ++set.TotalCount; - } + } } - + Current->SetModelP(); - } - + } + ui32 TDictBuilder::BuildNextGeneration(ui32 maxent) { TAutoPtr<TEntrySet> newset = new TEntrySet; newset->InitWithAlpha(); maxent -= newset->size(); - + ui32 additions = 0; ui32 deletions = 0; - + { const TEntrySet& set = *Current; - + Candidates.clear(); const ui32 total = set.TotalCount; const float minpval = Settings.MinPValue; const EEntryStatTest test = Settings.StatTest; const EEntryScore score = Settings.Score; const ui32 mincnt = Settings.MinAbsCount; - + for (const auto& it : set) { const TEntry& e = it; float modelp = e.ModelP; ui32 cnt = e.Count; - + if (e.HasPrefix() && e.Count > mincnt && StatTest(test, modelp, cnt, total) > minpval) Candidates.push_back(TCandidate(-Score(score, e.Len(), modelp, cnt, total), it.Number)); } - + if (!!CompoundCounts) { for (TCompoundCounts::const_iterator it = CompoundCounts->begin(); it != CompoundCounts->end(); ++it) { const TEntry& prev = set.Get(Prev(it->first)); @@ -89,13 +89,13 @@ namespace NGreedyDict { if (cnt > mincnt && StatTest(test, modelp, cnt, total) > minpval) Candidates.push_back(TCandidate(-Score(score, prev.Len() + next.Len(), modelp, cnt, total), it->first)); } - } - + } + Sort(Candidates.begin(), Candidates.end()); - + if (Candidates.size() > maxent) Candidates.resize(maxent); - + for (const auto& candidate : Candidates) { if (IsCompound(candidate.second)) { additions++; @@ -103,40 +103,40 @@ namespace NGreedyDict { } else { newset->Add(set.Get(candidate.second).Str); } - } + } deletions = set.size() - (newset->size() - additions); - } - + } + Current = newset; Current->BuildHierarchy(); return deletions + additions; - } - + } + ui32 TDictBuilder::Build(ui32 maxentries, ui32 maxiters, ui32 mindiff) { size_t totalsz = 0; for (auto it : Input) totalsz += it.size(); - + while (maxiters) { maxiters--; - + RebuildCounts(maxentries * Settings.GrowLimit, false); - + if (Settings.Verbose) { TString mess = Sprintf("iter:%" PRIu32 " sz:%" PRIu32 " pend:%" PRIu32, maxiters, (ui32)Current->size(), (ui32)CompoundCounts->size()); Clog << Sprintf("%-110s RSS=%" PRIu32 "M", mess.data(), (ui32)(TRusage::Get().MaxRss >> 20)) << Endl; } - + ui32 diff = BuildNextGeneration(maxentries); if (Current->size() == maxentries && diff < mindiff) break; - } - + } + RebuildCounts(0, true); Current->SetScores(Settings.Score); return maxiters; - } - -} + } + +} diff --git a/library/cpp/codecs/greedy_dict/gd_builder.h b/library/cpp/codecs/greedy_dict/gd_builder.h index 7f3cea88cb..b8e9a5e37b 100644 --- a/library/cpp/codecs/greedy_dict/gd_builder.h +++ b/library/cpp/codecs/greedy_dict/gd_builder.h @@ -1,94 +1,94 @@ -#pragma once - -#include "gd_entry.h" - -#include <util/generic/hash.h> -#include <util/random/fast.h> - -namespace NGreedyDict { +#pragma once + +#include "gd_entry.h" + +#include <util/generic/hash.h> +#include <util/random/fast.h> + +namespace NGreedyDict { struct TBuildSettings { EEntryStatTest StatTest = EST_SIMPLE_NORM; EEntryScore Score = ES_LEN_SIMPLE; - + float MinPValue = 0.75; ui32 MinAbsCount = 10; ui32 GrowLimit = 10; // times of maxentries bool Verbose = false; }; - + class TDictBuilder { using TCompoundCounts = THashMap<ui64, ui32, THash<ui64>, TEqualTo<ui64>, TPoolAllocator>; using TCandidate = std::pair<float, ui64>; using TCandidates = TVector<TCandidate>; - + private: TFastRng64 Rng{0x1a5d0ac170565c1c, 0x0be7bc27, 0x6235f6f57820aa0d, 0xafdc7fb}; TStringBufs Input; - + THolder<TEntrySet> Current; - + TMemoryPool CompoundCountsPool; THolder<TCompoundCounts> CompoundCounts; - + TCandidates Candidates; - + TBuildSettings Settings; - + public: TDictBuilder(const TBuildSettings& s = TBuildSettings()) : CompoundCountsPool(8112, TMemoryPool::TLinearGrow::Instance()) , Settings(s) { } - + void SetInput(const TStringBufs& in) { Input = in; } - + const TBuildSettings& GetSettings() const { return Settings; } - + TBuildSettings& GetSettings() { return Settings; } - + void SetSettings(const TBuildSettings& s) { Settings = s; } - + TEntrySet& EntrySet() { return *Current; } - + const TEntrySet& EntrySet() const { return *Current; } - + THolder<TEntrySet> ReleaseEntrySet() { return std::move(Current); } - + ui32 /*iters*/ Build(ui32 maxentries, ui32 maxiters = 16, ui32 mindiff = 10); - + public: void RebuildCounts(ui32 maxcand, bool final); ui32 /*diff size*/ BuildNextGeneration(ui32 maxent); - + static bool IsCompound(ui64 ent) { return ent & 0xFFFFFFFF00000000ULL; } - + static ui32 Next(ui64 ent) { return ent; } static ui32 Prev(ui64 ent) { return (ent >> 32) - 1; } - + static ui64 Compose(ui32 prev, ui32 next) { return ((prev + 1ULL) << 32) | next; } }; - -} + +} diff --git a/library/cpp/codecs/greedy_dict/gd_entry.cpp b/library/cpp/codecs/greedy_dict/gd_entry.cpp index f23a754976..2c315c7f7c 100644 --- a/library/cpp/codecs/greedy_dict/gd_entry.cpp +++ b/library/cpp/codecs/greedy_dict/gd_entry.cpp @@ -1,98 +1,98 @@ -#include "gd_entry.h" -#include "gd_stats.h" - -#include <util/generic/algorithm.h> -#include <util/generic/singleton.h> - -namespace NGreedyDict { +#include "gd_entry.h" +#include "gd_stats.h" + +#include <util/generic/algorithm.h> +#include <util/generic/singleton.h> + +namespace NGreedyDict { class TAlphas { char Memory[512]; - + public: TStringBufs Alphas; - + TAlphas() { for (ui32 i = 0; i < 256; ++i) { Memory[2 * i] = (char)i; Memory[2 * i + 1] = 0; - + Alphas.push_back(TStringBuf(&Memory[2 * i], 1)); } } }; - + void TEntrySet::InitWithAlpha() { Pool.ClearKeepFirstChunk(); const TStringBufs& a = Singleton<TAlphas>()->Alphas; for (auto it : a) { Add(it); - } + } BuildHierarchy(); - } - + } + void TEntrySet::BuildHierarchy() { Sort(begin(), end(), TEntry::StrLess); - + TCompactTrieBuilder<char, ui32, TAsIsPacker<ui32>> builder(CTBF_PREFIX_GROUPED); - + for (iterator it = begin(); it != end(); ++it) { it->Number = (it - begin()); TStringBuf suff = it->Str; size_t len = 0; ui32 val = 0; - + if (builder.FindLongestPrefix(suff.data(), suff.size(), &len, &val) && len) { it->NearestPrefix = val; } - + builder.Add(suff.data(), suff.size(), it->Number); - } - + } + TBufferOutput bout; builder.Save(bout); Trie.Init(TBlob::FromBuffer(bout.Buffer())); - } - + } + TEntry* TEntrySet::FindPrefix(TStringBuf& str) { size_t len = 0; ui32 off = 0; - + if (!Trie.FindLongestPrefix(str, &len, &off)) { return nullptr; } - + str.Skip(len); return &Get(off); - } - + } + void TEntrySet::SetModelP() { for (iterator it = begin(); it != end(); ++it) { TEntry& e = *it; - + if (!e.HasPrefix()) { e.ModelP = 0; continue; } - + TStringBuf suff = e.Str; const TEntry& p = Get(e.NearestPrefix); suff.Skip(p.Len()); - + float modelp = float(p.Count + e.Count) / TotalCount; - + while (!!suff) { TEntry* pp = FindPrefix(suff); modelp *= float(pp->Count + e.Count) / TotalCount; } - + e.ModelP = modelp; - } - } - + } + } + void TEntrySet::SetScores(EEntryScore s) { for (auto& it : *this) { it.Score = Score(s, it.Len(), it.ModelP, it.Count, TotalCount); } - } - -} + } + +} diff --git a/library/cpp/codecs/greedy_dict/gd_entry.h b/library/cpp/codecs/greedy_dict/gd_entry.h index 0362fd9f99..18b5be0e15 100644 --- a/library/cpp/codecs/greedy_dict/gd_entry.h +++ b/library/cpp/codecs/greedy_dict/gd_entry.h @@ -1,42 +1,42 @@ -#pragma once - -#include "gd_stats.h" - +#pragma once + +#include "gd_stats.h" + #include <library/cpp/containers/comptrie/comptrie.h> - -#include <util/generic/ptr.h> -#include <util/generic/strbuf.h> -#include <util/generic/vector.h> - -#include <util/memory/pool.h> - -namespace NGreedyDict { + +#include <util/generic/ptr.h> +#include <util/generic/strbuf.h> +#include <util/generic/vector.h> + +#include <util/memory/pool.h> + +namespace NGreedyDict { using TStringBufs = TVector<TStringBuf>; - + struct TEntry { static const i32 NoPrefix = -1; - + TStringBuf Str; - + i32 NearestPrefix = NoPrefix; ui32 Count = 0; ui32 Number = 0; float ModelP = 0; float Score = 0; - + TEntry(TStringBuf b = TStringBuf(), ui32 cnt = 0) : Str(b) , Count(cnt) { } - + bool HasPrefix() const { return NearestPrefix != NoPrefix; } ui32 Len() const { return Str.size(); } - + static bool StrLess(const TEntry& a, const TEntry& b) { return a.Str < b.Str; } @@ -47,20 +47,20 @@ namespace NGreedyDict { return a.Score > b.Score; } }; - + class TEntrySet: public TVector<TEntry>, TNonCopyable { TMemoryPool Pool{8112}; TCompactTrie<char, ui32, TAsIsPacker<ui32>> Trie; - + public: ui32 TotalCount = 0; - + void InitWithAlpha(); - + void Add(TStringBuf a) { push_back(TStringBuf(Pool.Append(a.data(), a.size()), a.size())); } - + void Add(TStringBuf a, TStringBuf b) { size_t sz = a.size() + b.size(); char* p = (char*)Pool.Allocate(sz); @@ -68,36 +68,36 @@ namespace NGreedyDict { memcpy(p + a.size(), b.data(), b.size()); push_back(TStringBuf(p, sz)); } - + TEntry& Get(ui32 idx) { return (*this)[idx]; } - + const TEntry& Get(ui32 idx) const { return (*this)[idx]; } - + void BuildHierarchy(); - + // longest prefix TEntry* FindPrefix(TStringBuf& str); - + const TEntry* FindPrefix(TStringBuf& str) const { return ((TEntrySet*)this)->FindPrefix(str); } - + const TEntry* FirstPrefix(const TEntry& e, TStringBuf& suff) { if (!e.HasPrefix()) return nullptr; - + const TEntry& p = Get(e.NearestPrefix); suff = e.Str; suff.Skip(p.Str.size()); return &p; } - + void SetModelP(); void SetScores(EEntryScore); }; - -} + +} diff --git a/library/cpp/codecs/greedy_dict/gd_stats.h b/library/cpp/codecs/greedy_dict/gd_stats.h index 3c209fc67d..b63c4c38d2 100644 --- a/library/cpp/codecs/greedy_dict/gd_stats.h +++ b/library/cpp/codecs/greedy_dict/gd_stats.h @@ -1,10 +1,10 @@ -#pragma once - +#pragma once + #include <util/generic/ymath.h> -#include <util/generic/algorithm.h> -#include <util/generic/yexception.h> - -namespace NGreedyDict { +#include <util/generic/algorithm.h> +#include <util/generic/yexception.h> + +namespace NGreedyDict { enum EEntryScore { ES_COUNT, ES_LEN_COUNT, @@ -12,33 +12,33 @@ namespace NGreedyDict { ES_LEN_SIMPLE, ES_SOLAR }; - + enum EEntryStatTest { EST_NONE = 0, EST_SIMPLE_NORM = 2 }; - + inline float ModelP(ui32 countA, ui32 countB, ui32 total) { return float(countA) * countB / total / total; } - + // P (ab | dependent) inline float SimpleTest(float modelp, ui32 countAB, ui32 total) { float realp = float(countAB) / total; return modelp >= realp ? 0 : (realp - modelp); } - + inline float SolarTest(float modelp, ui32 countAB, ui32 total) { float realp = float(countAB) / total; return modelp >= realp ? 0 : (modelp + realp * (log(realp / modelp) - 1)); } - + // P (ab | dependent) / P (ab) inline float SimpleTestNorm(float modelp, ui32 countAB, ui32 total) { float realp = float(countAB) / total; return modelp >= realp ? 0 : (realp - modelp) / realp; } - + inline float StatTest(EEntryStatTest test, float modelp, ui32 countAB, ui32 total) { if (!total) { return 0; @@ -50,9 +50,9 @@ namespace NGreedyDict { return SimpleTestNorm(modelp, countAB, total); } Y_FAIL("no way!"); - return 0; - } - + return 0; + } + inline float Score(EEntryScore score, ui32 len, float modelp, ui32 count, ui32 total) { if (!total) { return 0; @@ -73,7 +73,7 @@ namespace NGreedyDict { return SolarTest(modelp, count, total); } Y_FAIL("no way!"); - return 0; - } - -} + return 0; + } + +} diff --git a/library/cpp/codecs/greedy_dict/ut/greedy_dict_ut.cpp b/library/cpp/codecs/greedy_dict/ut/greedy_dict_ut.cpp index 60ab9f7c30..679089a11b 100644 --- a/library/cpp/codecs/greedy_dict/ut/greedy_dict_ut.cpp +++ b/library/cpp/codecs/greedy_dict/ut/greedy_dict_ut.cpp @@ -1,282 +1,282 @@ -#include "gd_builder.h" - +#include "gd_builder.h" + #include <library/cpp/testing/unittest/registar.h> #include <library/cpp/string_utils/relaxed_escaper/relaxed_escaper.h> -#include <util/string/printf.h> +#include <util/string/printf.h> #include <util/generic/ymath.h> - -class TGreedyDictTest: public TTestBase { + +class TGreedyDictTest: public TTestBase { UNIT_TEST_SUITE(TGreedyDictTest); - UNIT_TEST(TestEntrySet) - UNIT_TEST(TestBuilder0) - UNIT_TEST(TestBuilder) + UNIT_TEST(TestEntrySet) + UNIT_TEST(TestBuilder0) + UNIT_TEST(TestBuilder) UNIT_TEST_SUITE_END(); - - void TestEntrySet() { - using namespace NGreedyDict; - - { - TEntrySet d; - - d.InitWithAlpha(); - - for (TEntrySet::const_iterator it = d.begin(); it != d.end(); ++it) { - UNIT_ASSERT_C(!it->HasPrefix(), Sprintf("%u -> %u", it->Number, it->NearestPrefix)); - UNIT_ASSERT_VALUES_EQUAL(it->Number, (ui32)(it - d.begin())); - } - - UNIT_ASSERT_VALUES_EQUAL(d.size(), 256u); - TStringBuf s = "aaabbb"; - UNIT_ASSERT_VALUES_EQUAL(d.FindPrefix(s)->Str, "a"); - UNIT_ASSERT_VALUES_EQUAL(s, "aabbb"); - UNIT_ASSERT_VALUES_EQUAL(d.FindPrefix(s)->Str, "a"); - UNIT_ASSERT_VALUES_EQUAL(s, "abbb"); - UNIT_ASSERT_VALUES_EQUAL(d.FindPrefix(s)->Str, "a"); - UNIT_ASSERT_VALUES_EQUAL(s, "bbb"); - UNIT_ASSERT_VALUES_EQUAL(d.FindPrefix(s)->Str, "b"); - UNIT_ASSERT_VALUES_EQUAL(s, "bb"); - UNIT_ASSERT_VALUES_EQUAL(d.FindPrefix(s)->Str, "b"); - UNIT_ASSERT_VALUES_EQUAL(s, "b"); - UNIT_ASSERT_VALUES_EQUAL(d.FindPrefix(s)->Str, "b"); - UNIT_ASSERT_VALUES_EQUAL(s, ""); - s = TStringBuf("", 1); - UNIT_ASSERT_VALUES_EQUAL(d.FindPrefix(s)->Str, TStringBuf("", 1)); - UNIT_ASSERT_VALUES_EQUAL(s, ""); - s = "\xFF"; - UNIT_ASSERT_VALUES_EQUAL(d.FindPrefix(s)->Str, "\xFF"); - UNIT_ASSERT_VALUES_EQUAL(s, ""); - } - { - TEntrySet d; - d.Add("a"); - d.Add("b"); - d.Add("b", "a"); - d.BuildHierarchy(); - - UNIT_ASSERT_VALUES_EQUAL(d.size(), 3u); - - TStringBuf s = "bab"; - UNIT_ASSERT_VALUES_EQUAL(d.FindPrefix(s)->Str, "ba"); - UNIT_ASSERT_VALUES_EQUAL(s, "b"); - } - { - TEntrySet d; - - d.Add("a"); - d.Add("aa"); - d.Add("aaa"); - d.Add("aab"); - d.Add("b"); - d.Add("ba"); - - d.BuildHierarchy(); - - UNIT_ASSERT_VALUES_EQUAL(d.size(), 6u); - { - TStringBuf s = "aaaaa"; - const TEntry* e = d.FindPrefix(s); - UNIT_ASSERT_VALUES_EQUAL(e->Str, "aaa"); - UNIT_ASSERT_VALUES_EQUAL(e->Number, 2u); - UNIT_ASSERT_VALUES_EQUAL(e->NearestPrefix, 1); - UNIT_ASSERT_VALUES_EQUAL(s, "aa"); - } - - { - TStringBuf s = "a"; - const TEntry* e = d.FindPrefix(s); - UNIT_ASSERT_VALUES_EQUAL(e->Str, "a"); - UNIT_ASSERT_VALUES_EQUAL(e->Number, 0u); - UNIT_ASSERT_VALUES_EQUAL(e->NearestPrefix, -1); - UNIT_ASSERT_VALUES_EQUAL(s, ""); - } - - { - TStringBuf s = "bab"; - const TEntry* e = d.FindPrefix(s); - UNIT_ASSERT_VALUES_EQUAL(e->Str, "ba"); - UNIT_ASSERT_VALUES_EQUAL(e->Number, 5u); - UNIT_ASSERT_VALUES_EQUAL(e->NearestPrefix, 4); - UNIT_ASSERT_VALUES_EQUAL(s, "b"); - } - - { - TStringBuf s = "bba"; - const TEntry* e = d.FindPrefix(s); - UNIT_ASSERT_VALUES_EQUAL(e->Str, "b"); - UNIT_ASSERT_VALUES_EQUAL(e->Number, 4u); - UNIT_ASSERT_VALUES_EQUAL(e->NearestPrefix, -1); - UNIT_ASSERT_VALUES_EQUAL(s, "ba"); - } - } - } - - void TestBuilder0() { - using namespace NGreedyDict; - ui32 a = 1, b = 11; - ui64 ab = TDictBuilder::Compose(a, b); - UNIT_ASSERT(TDictBuilder::IsCompound(ab)); - UNIT_ASSERT_VALUES_EQUAL(TDictBuilder::Prev(ab), a); - UNIT_ASSERT_VALUES_EQUAL(TDictBuilder::Next(ab), b); - } - - void FillData(NGreedyDict::TStringBufs& data) { + + void TestEntrySet() { + using namespace NGreedyDict; + + { + TEntrySet d; + + d.InitWithAlpha(); + + for (TEntrySet::const_iterator it = d.begin(); it != d.end(); ++it) { + UNIT_ASSERT_C(!it->HasPrefix(), Sprintf("%u -> %u", it->Number, it->NearestPrefix)); + UNIT_ASSERT_VALUES_EQUAL(it->Number, (ui32)(it - d.begin())); + } + + UNIT_ASSERT_VALUES_EQUAL(d.size(), 256u); + TStringBuf s = "aaabbb"; + UNIT_ASSERT_VALUES_EQUAL(d.FindPrefix(s)->Str, "a"); + UNIT_ASSERT_VALUES_EQUAL(s, "aabbb"); + UNIT_ASSERT_VALUES_EQUAL(d.FindPrefix(s)->Str, "a"); + UNIT_ASSERT_VALUES_EQUAL(s, "abbb"); + UNIT_ASSERT_VALUES_EQUAL(d.FindPrefix(s)->Str, "a"); + UNIT_ASSERT_VALUES_EQUAL(s, "bbb"); + UNIT_ASSERT_VALUES_EQUAL(d.FindPrefix(s)->Str, "b"); + UNIT_ASSERT_VALUES_EQUAL(s, "bb"); + UNIT_ASSERT_VALUES_EQUAL(d.FindPrefix(s)->Str, "b"); + UNIT_ASSERT_VALUES_EQUAL(s, "b"); + UNIT_ASSERT_VALUES_EQUAL(d.FindPrefix(s)->Str, "b"); + UNIT_ASSERT_VALUES_EQUAL(s, ""); + s = TStringBuf("", 1); + UNIT_ASSERT_VALUES_EQUAL(d.FindPrefix(s)->Str, TStringBuf("", 1)); + UNIT_ASSERT_VALUES_EQUAL(s, ""); + s = "\xFF"; + UNIT_ASSERT_VALUES_EQUAL(d.FindPrefix(s)->Str, "\xFF"); + UNIT_ASSERT_VALUES_EQUAL(s, ""); + } + { + TEntrySet d; + d.Add("a"); + d.Add("b"); + d.Add("b", "a"); + d.BuildHierarchy(); + + UNIT_ASSERT_VALUES_EQUAL(d.size(), 3u); + + TStringBuf s = "bab"; + UNIT_ASSERT_VALUES_EQUAL(d.FindPrefix(s)->Str, "ba"); + UNIT_ASSERT_VALUES_EQUAL(s, "b"); + } + { + TEntrySet d; + + d.Add("a"); + d.Add("aa"); + d.Add("aaa"); + d.Add("aab"); + d.Add("b"); + d.Add("ba"); + + d.BuildHierarchy(); + + UNIT_ASSERT_VALUES_EQUAL(d.size(), 6u); + { + TStringBuf s = "aaaaa"; + const TEntry* e = d.FindPrefix(s); + UNIT_ASSERT_VALUES_EQUAL(e->Str, "aaa"); + UNIT_ASSERT_VALUES_EQUAL(e->Number, 2u); + UNIT_ASSERT_VALUES_EQUAL(e->NearestPrefix, 1); + UNIT_ASSERT_VALUES_EQUAL(s, "aa"); + } + + { + TStringBuf s = "a"; + const TEntry* e = d.FindPrefix(s); + UNIT_ASSERT_VALUES_EQUAL(e->Str, "a"); + UNIT_ASSERT_VALUES_EQUAL(e->Number, 0u); + UNIT_ASSERT_VALUES_EQUAL(e->NearestPrefix, -1); + UNIT_ASSERT_VALUES_EQUAL(s, ""); + } + + { + TStringBuf s = "bab"; + const TEntry* e = d.FindPrefix(s); + UNIT_ASSERT_VALUES_EQUAL(e->Str, "ba"); + UNIT_ASSERT_VALUES_EQUAL(e->Number, 5u); + UNIT_ASSERT_VALUES_EQUAL(e->NearestPrefix, 4); + UNIT_ASSERT_VALUES_EQUAL(s, "b"); + } + + { + TStringBuf s = "bba"; + const TEntry* e = d.FindPrefix(s); + UNIT_ASSERT_VALUES_EQUAL(e->Str, "b"); + UNIT_ASSERT_VALUES_EQUAL(e->Number, 4u); + UNIT_ASSERT_VALUES_EQUAL(e->NearestPrefix, -1); + UNIT_ASSERT_VALUES_EQUAL(s, "ba"); + } + } + } + + void TestBuilder0() { + using namespace NGreedyDict; + ui32 a = 1, b = 11; + ui64 ab = TDictBuilder::Compose(a, b); + UNIT_ASSERT(TDictBuilder::IsCompound(ab)); + UNIT_ASSERT_VALUES_EQUAL(TDictBuilder::Prev(ab), a); + UNIT_ASSERT_VALUES_EQUAL(TDictBuilder::Next(ab), b); + } + + void FillData(NGreedyDict::TStringBufs& data) { static const char* urls[] = {"http://53.ru/car/motors/foreign/opel/tigra/", "http://abakan.24au.ru/tender/85904/", "http://anm15.gulaig.com/", "http://avto-parts.com/mercedes-benz/mercedes-benz-w220-1998-2005/category-442/category-443/", "http://ballooncousin.co.uk/", "http://benzol.ru/equipment/?id=1211&parent=514", "http://blazingseorank.com/blazing-seo-rank-free-website-analysis-to-increase-rank-and-traffic-450.html", "http://blogblaugrana.contadorwebmasters.com/", "http://bristolhash.org.uk/bh3cntct.php", "http://broker.borovichi.ru/category/item/3/1/0/8/28/257", "http://canoncompactcamerax.blogspot.com/", "http://classifieds.smashits.com/p,107881,email-to-friend.htm", "http://conferences.ksde.org/Portals/132/FallAssessment/SAVETHEDAY-FA09.pdf", "http://eway.vn/raovat/325-dien-tu-gia-dung/337-dieu-hoa/98041-b1-sua-may-lanh-quan-binh-tan-sua-may-lanh-quan-binh-chanh-hh-979676119-toan-quoc.html", "http://gallery.e2bn.org/asset73204_8-.html", "http://goplay.nsw.gov.au/activities-for-kids/by/historic-houses-trust/?startdate=2012-07-10", "http://grichards19067.multiply.com/", "http://hotkovo.egent.ru/user/89262269084/", "http://howimetyourself.com/?redirect_to=http://gomiso.com/m/suits/seasons/2/episodes/2", "http://islamqa.com/hi/ref/9014/DEAD%20PEOPLE%20GOOD%20DEEDS", "http://lapras.rutube.ru/", "http://nceluiko.ya.ru/", "http://nyanyanyanyaa.beon.ru/", "http://ozbo.com/Leaf-River-DV-7SS-7-0-MP-Game-Camera-K1-32541.html", "http://sbantom.ru/catalog/chasy/632753.html", "http://shopingoff.com/index.php?option=com_virtuemart&Itemid=65&category_id=&page=shop.browse&manufacturer_id=122&limit=32&limitstart=96", "http://shopingoff.com/katalog-odezhdy/manufacturer/62-christian-audigier.html?limit=32&start=448", "https://webwinkel.ah.nl/process?fh_location=//ecommerce/nl_NL/categories%3C%7Becommerce_shoc1%7D/it_show_product_code_1384%3E%7B10%3B20%7D/pr_startdate%3C20120519/pr_enddate%3E20120519/pr_ltc_allowed%3E%7Bbowi%7D/categories%3C%7Becommerce_shoc1_1al%7D/categories%3C%7Becommerce_shoc1_1al_1ahal%7D&&action=albert_noscript.modules.build", "http://top100.rambler.ru/navi/?theme=208/210/371&rgn=17", "http://volgogradskaya-oblast.extra-m.ru/classifieds/rabota/vakansii/banki-investicii/901467/", "http://wikien4.appspot.com/wiki/Warburg_hypothesis", "http://wola_baranowska.kamerzysta24.com.pl/", "http://www.10dot0dot0dot1.com/", "http://www.anima-redux.ru/index.php?key=gifts+teenage+girls", "http://www.aquaticabyseaworld.com/Calendar.aspx/CP/CP/CP/sp-us/CP/CP/ParkMap/Tickets/Weather.aspx", "http://www.autousa.com/360spin/2012_cadillac_ctssportwagon_3.6awdpremiumcollection.htm", "http://www.booking.com/city/gb/paignton-aireborough.html?inac=0&lang=pl", "http://www.booking.com/city/it/vodo-cadore.en.html", "http://www.booking.com/district/us/new-york/rockefeller-center.html&lang=no", "http://www.booking.com/hotel/bg/crown-fort-club.lv.html", "http://www.booking.com/hotel/ca/gouverneur-rimouski.ar.html", "http://www.booking.com/hotel/ch/l-auberge-du-chalet-a-gobet.fi.html", "http://www.booking.com/hotel/de/mark-garni.ru.html?aid=337384;label=yandex-hotel-mark-garni-68157-%7Bparam1%7D", "http://www.booking.com/hotel/de/mercure-goldschmieding-castrop-rauxel.ro.html", "http://www.booking.com/hotel/de/zollenspieker-fahrhaus.fr.html", "http://www.booking.com/hotel/es/jardin-metropolitano.ca.html", "http://www.booking.com/hotel/fr/clim.fr.html", "http://www.booking.com/hotel/fr/radisson-sas-toulouse-airport.et.html", "http://www.booking.com/hotel/gb/stgileshotel.ro.html?srfid=68c7fe42a03653a8796c84435c5299e4X16?tab=4", "http://www.booking.com/hotel/gr/rodos-park-suites.ru.html", "http://www.booking.com/hotel/id/le-grande-suites-bali.ru.html", "http://www.booking.com/hotel/it/mozart.it.html?aid=321655", "http://www.booking.com/hotel/ni/bahia-del-sol-villas.ru.html?dcid=1;dva=0", "http://www.booking.com/hotel/nl/cpschiphol.ro.html.ro.html?tab=4", "http://www.booking.com/hotel/th/laem-din.en-gb.html", "http://www.booking.com/hotel/th/tinidee-ranong.en.html", "http://www.booking.com/hotel/us/best-western-plus-merrimack-valley.hu.html", "http://www.booking.com/hotel/vn/tan-hai-long.km.html", "http://www.booking.com/landmark/au/royal-brisbane-women-s-hospital.vi.html", "http://www.booking.com/landmark/hk/nam-cheong-station.html&lang=id", "http://www.booking.com/landmark/it/spanish-steps.ca.html", "http://www.booking.com/landmark/sg/asian-civilisations-museum.html&lang=fi", "http://www.booking.com/place/fi-1376029.pt.html", "http://www.booking.com/place/tn257337.pl.html", "http://www.booking.com/region/ca/niagarafalls.ar.html&selected_currency=PLN", "http://www.booking.com/region/mx/queretaro.pt-pt.html&selected_currency=AUD", "http://www.booking.com/searchresults.en.html?city=20063074", "http://www.booking.com/searchresults.et.html?checkin=;checkout=;city=-394632", "http://www.booking.com/searchresults.lv.html?region=3936", "http://www.cevredanismanlari.com/index.php/component/k2/index.php/mevzuat/genel-yazlar/item/dosyalar/index.php?option=com_k2&view=item&id=16:iso-14001-%C3%A7evre-y%C3%B6netim-sistemi&Itemid=132&limitstart=107120", "http://www.dh-wholesaler.com/MENS-POLO-RACING-TEE-RL-p-417.html", "http://www.employabilityonline.net/", "http://www.esso.inc.ru/board/tools.php?event=profile&pname=Invinerrq", "http://www.filesurgery.ru/searchfw/kids_clothes-3.html", "http://www.furnitureandcarpetsource.com/Item.aspx?ItemID=-2107311899&ItemNum=53-T3048", "http://www.gets.cn/product/Gold-Sand-Lampwork-Glass-Beads--Flat-round--28x28x13mm_p260717.html", "http://www.gets.cn/wholesale-Sterling-Silver-Pendant-Findings-3577_S--L-Star-P-1.html?view=1&by=1", "http://www.homeandgardenadvice.com/diy/Mortgages_Loans_and_Financing/9221.html", "http://www.hongkongairport.com/eng/index.html/passenger/passenger/transport/to-from-airport/business/about-the-airport/transport/shopping/entertainment/t2/passenger/interactive-map.html", "http://www.hongkongairport.com/eng/index.html/shopping/insideshopping/all/passenger/transfer-transit/all/airline-information/shopping/entertainment/t2/business/about-the-airport/welcome.html", "http://www.hongkongairport.com/eng/index.html/transport/business/about-the-airport/transport/business/airport-authority/passenger/shopping/dining/all/dining.html", "http://www.idedge.com/index.cfm/fuseaction/category.display/category_id/298/index.cfm", "http://www.istanbulburda.com/aramalar.php", "http://www.jewelryinthenet.com/ads/AdDetail.aspx?AdID=1-0311002490689&stid=22-0111001020877", "http://www.johnnydepp.ru/forum/index.php?showtopic=1629&mode=linearplus&view=findpost&p=186977", "http://www.johnnydepp.ru/forum/index.php?showtopic=476&st=60&p=87379&", "http://www.joseleano.com/joomla/index.php/audio", "http://www.kaplicarehberi.com/tag/sakar-ilicali-kaplicalari/feed", "http://www.khaber.com.tr/arama.html?key=%C3%A7avdar", "http://www.kiz-oyunlari1.com/1783/4437/4363/1056/4170/Bump-Copter2-.html", "http://www.kiz-oyunlari1.com/3752/2612/4175/1166/3649/1047/Angelina-Oyunu.html", "http://www.kiz-oyunlari1.com/4266/3630/3665/3286/4121/301/3274/Sinir-Sinekler-.html", "http://www.kuldiga.lv/index.php?f=8&cat=371", "http://www.kuldiga.lv/index.php/img/index.php?l=lv&art_id=1836&show_c=&cat=85", "http://www.patronessa.ru/remontiruemsya/kuzovnie30raboti.html", "http://www.rapdict.org/Nu_Money?title=Talk:Nu_Money&action=edit", "http://www.serafin-phu.tabor24.com/?page=8", "http://www.shoes-store.org/brand1/Kids/Minnetonka.html", "http://www.shoes-store.org/shoes-store.xml", "http://www.way2allah.com/khotab-download-34695.htm"}; - data.clear(); + data.clear(); data.insert(data.begin(), urls, urls + Y_ARRAY_SIZE(urls)); - } - + } + typedef THashMap<TStringBuf, NGreedyDict::TEntry> TDict; - - TAutoPtr<NGreedyDict::TEntrySet> DoTestBuilder(const NGreedyDict::TBuildSettings& s, + + TAutoPtr<NGreedyDict::TEntrySet> DoTestBuilder(const NGreedyDict::TBuildSettings& s, TDict& res) { - using namespace NGreedyDict; - - TStringBufs data; - FillData(data); - - TDictBuilder b(s); - b.SetInput(data); - b.Build(256 + 128); - - TEntrySet& set = b.EntrySet(); - + using namespace NGreedyDict; + + TStringBufs data; + FillData(data); + + TDictBuilder b(s); + b.SetInput(data); + b.Build(256 + 128); + + TEntrySet& set = b.EntrySet(); + for (const auto& it : set) { if (it.Score) { res[it.Str] = it; - } - } - - return b.ReleaseEntrySet(); - } - - void DoAssertEntry(TStringBuf entry, ui32 number, i32 parent, float score, const TDict& dict) { - TDict::const_iterator it = dict.find(entry); - UNIT_ASSERT_C(it != dict.end(), entry); - UNIT_ASSERT_VALUES_EQUAL_C(it->second.Number, number, entry); - UNIT_ASSERT_VALUES_EQUAL_C(it->second.NearestPrefix, parent, entry); - UNIT_ASSERT_VALUES_EQUAL_C(round(it->second.Score * 10000), round(score * 10000), entry); - } - - void TestBuilder() { - TAutoPtr<NGreedyDict::TEntrySet> set; + } + } + + return b.ReleaseEntrySet(); + } + + void DoAssertEntry(TStringBuf entry, ui32 number, i32 parent, float score, const TDict& dict) { + TDict::const_iterator it = dict.find(entry); + UNIT_ASSERT_C(it != dict.end(), entry); + UNIT_ASSERT_VALUES_EQUAL_C(it->second.Number, number, entry); + UNIT_ASSERT_VALUES_EQUAL_C(it->second.NearestPrefix, parent, entry); + UNIT_ASSERT_VALUES_EQUAL_C(round(it->second.Score * 10000), round(score * 10000), entry); + } + + void TestBuilder() { + TAutoPtr<NGreedyDict::TEntrySet> set; THashMap<TStringBuf, NGreedyDict::TEntry> res; - NGreedyDict::TBuildSettings s; - set = DoTestBuilder(s, res); - - UNIT_ASSERT_VALUES_EQUAL(set->size(), 295u); - UNIT_ASSERT_VALUES_EQUAL(res.size(), 110u); - - DoAssertEntry("%", 37, -1, 0.00375193, res); - DoAssertEntry("%7", 38, 37, 0.00513299, res); - DoAssertEntry("&", 39, -1, 0.00794527, res); - DoAssertEntry("+", 44, -1, 0.000441404, res); - DoAssertEntry(",", 45, -1, 0.000441404, res); - DoAssertEntry("-", 46, -1, 0.0417126, res); - DoAssertEntry(".", 47, -1, 0.0196425, res); - DoAssertEntry(".com/", 48, 47, 0.0374482, res); - DoAssertEntry(".html", 49, 47, 0.0496577, res); - DoAssertEntry(".html?", 50, 49, 0.0153908, res); - DoAssertEntry(".php", 51, 47, 0.0123585, res); - DoAssertEntry(".ru/", 52, 47, 0.0150027, res); - DoAssertEntry("/", 53, -1, 0.0452439, res); - DoAssertEntry("/index", 54, 53, 0.0158905, res); - DoAssertEntry("0", 55, -1, 0.00816597, res); - DoAssertEntry("1", 56, -1, 0.0167733, res); - DoAssertEntry("10", 57, 56, 0.00530474, res); - DoAssertEntry("2", 58, -1, 0.0101523, res); - DoAssertEntry("20", 59, 58, 0.00674234, res); - DoAssertEntry("3", 60, -1, 0.01258, res); - DoAssertEntry("32", 61, 60, 0.00490697, res); - DoAssertEntry("4", 62, -1, 0.00993158, res); - DoAssertEntry("5", 63, -1, 0.00617965, res); - DoAssertEntry("6", 64, -1, 0.00971088, res); - DoAssertEntry("7", 65, -1, 0.0101523, res); - DoAssertEntry("8", 66, -1, 0.00728316, res); - DoAssertEntry("9", 67, -1, 0.00728316, res); - DoAssertEntry(":", 68, -1, 0.000662106, res); - DoAssertEntry(";", 69, -1, 0.000882807, res); - DoAssertEntry("=", 71, -1, 0.01258, res); - DoAssertEntry("?", 73, -1, 0.00397263, res); - DoAssertEntry("A", 75, -1, 0.00264842, res); - DoAssertEntry("B", 76, -1, 0.00220702, res); - DoAssertEntry("C", 77, -1, 0.00353123, res); - DoAssertEntry("D", 78, -1, 0.00375193, res); - DoAssertEntry("E", 79, -1, 0.00286912, res); - DoAssertEntry("F", 80, -1, 0.00110351, res); - DoAssertEntry("G", 81, -1, 0.00110351, res); - DoAssertEntry("H", 82, -1, 0.000220702, res); - DoAssertEntry("I", 83, -1, 0.00198632, res); - DoAssertEntry("K", 85, -1, 0.000441404, res); - DoAssertEntry("L", 86, -1, 0.00198632, res); - DoAssertEntry("M", 87, -1, 0.00154491, res); - DoAssertEntry("N", 88, -1, 0.00154491, res); - DoAssertEntry("O", 89, -1, 0.00132421, res); - DoAssertEntry("P", 90, -1, 0.00308983, res); - DoAssertEntry("R", 92, -1, 0.000662106, res); - DoAssertEntry("S", 93, -1, 0.00264842, res); - DoAssertEntry("T", 94, -1, 0.00110351, res); - DoAssertEntry("U", 95, -1, 0.000220702, res); - DoAssertEntry("V", 96, -1, 0.000441404, res); - DoAssertEntry("W", 97, -1, 0.000441404, res); - DoAssertEntry("X", 98, -1, 0.000220702, res); - DoAssertEntry("Y", 99, -1, 0.000220702, res); - DoAssertEntry("_", 105, -1, 0.00904877, res); - DoAssertEntry("a", 107, -1, 0.0505407, res); - DoAssertEntry("an", 108, 107, 0.018273, res); - DoAssertEntry("ar", 109, 107, 0.0169385, res); - DoAssertEntry("b", 110, -1, 0.0156698, res); - DoAssertEntry("c", 111, -1, 0.018539, res); - DoAssertEntry("cat", 112, 111, 0.00846732, res); - DoAssertEntry("ch", 113, 111, 0.00644872, res); - DoAssertEntry("com", 114, 111, 0.00724235, res); - DoAssertEntry("ct", 115, 111, 0.00605729, res); - DoAssertEntry("d", 116, -1, 0.020746, res); - DoAssertEntry("di", 117, 116, 0.00730659, res); - DoAssertEntry("e", 118, -1, 0.0624586, res); - DoAssertEntry("en", 119, 118, 0.0108999, res); - DoAssertEntry("ent", 120, 119, 0.00616002, res); - DoAssertEntry("f", 121, -1, 0.00860737, res); - DoAssertEntry("fi", 122, 121, 0.00423196, res); - DoAssertEntry("g", 123, -1, 0.0180975, res); - DoAssertEntry("go", 124, 123, 0.00601862, res); - DoAssertEntry("h", 125, -1, 0.010373, res); - DoAssertEntry("ho", 126, 125, 0.00570298, res); - DoAssertEntry("http://", 127, 125, 0.0494372, res); - DoAssertEntry("http://www.", 128, 127, 0.0849702, res); - DoAssertEntry("http://www.booking.com/", 129, 128, 0.071066, res); - DoAssertEntry("http://www.booking.com/hotel/", 130, 129, 0.121607, res); - DoAssertEntry("i", 131, -1, 0.0258221, res); - DoAssertEntry("id=", 132, 131, 0.00725369, res); - DoAssertEntry("im", 133, 131, 0.00373318, res); - DoAssertEntry("in", 134, 131, 0.013625, res); - DoAssertEntry("ing", 135, 134, 0.00795491, res); - DoAssertEntry("ion", 136, 131, 0.00796149, res); - DoAssertEntry("it", 137, 131, 0.00953416, res); - DoAssertEntry("j", 138, -1, 0.00132421, res); - DoAssertEntry("k", 139, -1, 0.0134628, res); - DoAssertEntry("l", 140, -1, 0.0381814, res); - DoAssertEntry("m", 141, -1, 0.0174354, res); - DoAssertEntry("mer", 142, 141, 0.00711846, res); - DoAssertEntry("n", 143, -1, 0.0132421, res); - DoAssertEntry("o", 144, -1, 0.0302362, res); - DoAssertEntry("on", 145, 144, 0.00802271, res); - DoAssertEntry("ou", 146, 144, 0.00414545, res); - DoAssertEntry("p", 147, -1, 0.0225116, res); - DoAssertEntry("port", 148, 147, 0.0123532, res); - DoAssertEntry("q", 149, -1, 0.00176561, res); - DoAssertEntry("r", 150, -1, 0.0401677, res); - DoAssertEntry("ran", 151, 150, 0.00686918, res); - DoAssertEntry("s", 152, -1, 0.0487751, res); - DoAssertEntry("sho", 153, 152, 0.0113876, res); - DoAssertEntry("t", 154, -1, 0.0379607, res); - DoAssertEntry("u", 155, -1, 0.0211874, res); - DoAssertEntry("v", 156, -1, 0.00595895, res); - DoAssertEntry("vi", 157, 156, 0.00480673, res); - DoAssertEntry("w", 158, -1, 0.00816597, res); - DoAssertEntry("x", 159, -1, 0.00375193, res); - DoAssertEntry("y", 160, -1, 0.0130214, res); - DoAssertEntry("z", 161, -1, 0.00353123, res); - } -}; - -UNIT_TEST_SUITE_REGISTRATION(TGreedyDictTest); + NGreedyDict::TBuildSettings s; + set = DoTestBuilder(s, res); + + UNIT_ASSERT_VALUES_EQUAL(set->size(), 295u); + UNIT_ASSERT_VALUES_EQUAL(res.size(), 110u); + + DoAssertEntry("%", 37, -1, 0.00375193, res); + DoAssertEntry("%7", 38, 37, 0.00513299, res); + DoAssertEntry("&", 39, -1, 0.00794527, res); + DoAssertEntry("+", 44, -1, 0.000441404, res); + DoAssertEntry(",", 45, -1, 0.000441404, res); + DoAssertEntry("-", 46, -1, 0.0417126, res); + DoAssertEntry(".", 47, -1, 0.0196425, res); + DoAssertEntry(".com/", 48, 47, 0.0374482, res); + DoAssertEntry(".html", 49, 47, 0.0496577, res); + DoAssertEntry(".html?", 50, 49, 0.0153908, res); + DoAssertEntry(".php", 51, 47, 0.0123585, res); + DoAssertEntry(".ru/", 52, 47, 0.0150027, res); + DoAssertEntry("/", 53, -1, 0.0452439, res); + DoAssertEntry("/index", 54, 53, 0.0158905, res); + DoAssertEntry("0", 55, -1, 0.00816597, res); + DoAssertEntry("1", 56, -1, 0.0167733, res); + DoAssertEntry("10", 57, 56, 0.00530474, res); + DoAssertEntry("2", 58, -1, 0.0101523, res); + DoAssertEntry("20", 59, 58, 0.00674234, res); + DoAssertEntry("3", 60, -1, 0.01258, res); + DoAssertEntry("32", 61, 60, 0.00490697, res); + DoAssertEntry("4", 62, -1, 0.00993158, res); + DoAssertEntry("5", 63, -1, 0.00617965, res); + DoAssertEntry("6", 64, -1, 0.00971088, res); + DoAssertEntry("7", 65, -1, 0.0101523, res); + DoAssertEntry("8", 66, -1, 0.00728316, res); + DoAssertEntry("9", 67, -1, 0.00728316, res); + DoAssertEntry(":", 68, -1, 0.000662106, res); + DoAssertEntry(";", 69, -1, 0.000882807, res); + DoAssertEntry("=", 71, -1, 0.01258, res); + DoAssertEntry("?", 73, -1, 0.00397263, res); + DoAssertEntry("A", 75, -1, 0.00264842, res); + DoAssertEntry("B", 76, -1, 0.00220702, res); + DoAssertEntry("C", 77, -1, 0.00353123, res); + DoAssertEntry("D", 78, -1, 0.00375193, res); + DoAssertEntry("E", 79, -1, 0.00286912, res); + DoAssertEntry("F", 80, -1, 0.00110351, res); + DoAssertEntry("G", 81, -1, 0.00110351, res); + DoAssertEntry("H", 82, -1, 0.000220702, res); + DoAssertEntry("I", 83, -1, 0.00198632, res); + DoAssertEntry("K", 85, -1, 0.000441404, res); + DoAssertEntry("L", 86, -1, 0.00198632, res); + DoAssertEntry("M", 87, -1, 0.00154491, res); + DoAssertEntry("N", 88, -1, 0.00154491, res); + DoAssertEntry("O", 89, -1, 0.00132421, res); + DoAssertEntry("P", 90, -1, 0.00308983, res); + DoAssertEntry("R", 92, -1, 0.000662106, res); + DoAssertEntry("S", 93, -1, 0.00264842, res); + DoAssertEntry("T", 94, -1, 0.00110351, res); + DoAssertEntry("U", 95, -1, 0.000220702, res); + DoAssertEntry("V", 96, -1, 0.000441404, res); + DoAssertEntry("W", 97, -1, 0.000441404, res); + DoAssertEntry("X", 98, -1, 0.000220702, res); + DoAssertEntry("Y", 99, -1, 0.000220702, res); + DoAssertEntry("_", 105, -1, 0.00904877, res); + DoAssertEntry("a", 107, -1, 0.0505407, res); + DoAssertEntry("an", 108, 107, 0.018273, res); + DoAssertEntry("ar", 109, 107, 0.0169385, res); + DoAssertEntry("b", 110, -1, 0.0156698, res); + DoAssertEntry("c", 111, -1, 0.018539, res); + DoAssertEntry("cat", 112, 111, 0.00846732, res); + DoAssertEntry("ch", 113, 111, 0.00644872, res); + DoAssertEntry("com", 114, 111, 0.00724235, res); + DoAssertEntry("ct", 115, 111, 0.00605729, res); + DoAssertEntry("d", 116, -1, 0.020746, res); + DoAssertEntry("di", 117, 116, 0.00730659, res); + DoAssertEntry("e", 118, -1, 0.0624586, res); + DoAssertEntry("en", 119, 118, 0.0108999, res); + DoAssertEntry("ent", 120, 119, 0.00616002, res); + DoAssertEntry("f", 121, -1, 0.00860737, res); + DoAssertEntry("fi", 122, 121, 0.00423196, res); + DoAssertEntry("g", 123, -1, 0.0180975, res); + DoAssertEntry("go", 124, 123, 0.00601862, res); + DoAssertEntry("h", 125, -1, 0.010373, res); + DoAssertEntry("ho", 126, 125, 0.00570298, res); + DoAssertEntry("http://", 127, 125, 0.0494372, res); + DoAssertEntry("http://www.", 128, 127, 0.0849702, res); + DoAssertEntry("http://www.booking.com/", 129, 128, 0.071066, res); + DoAssertEntry("http://www.booking.com/hotel/", 130, 129, 0.121607, res); + DoAssertEntry("i", 131, -1, 0.0258221, res); + DoAssertEntry("id=", 132, 131, 0.00725369, res); + DoAssertEntry("im", 133, 131, 0.00373318, res); + DoAssertEntry("in", 134, 131, 0.013625, res); + DoAssertEntry("ing", 135, 134, 0.00795491, res); + DoAssertEntry("ion", 136, 131, 0.00796149, res); + DoAssertEntry("it", 137, 131, 0.00953416, res); + DoAssertEntry("j", 138, -1, 0.00132421, res); + DoAssertEntry("k", 139, -1, 0.0134628, res); + DoAssertEntry("l", 140, -1, 0.0381814, res); + DoAssertEntry("m", 141, -1, 0.0174354, res); + DoAssertEntry("mer", 142, 141, 0.00711846, res); + DoAssertEntry("n", 143, -1, 0.0132421, res); + DoAssertEntry("o", 144, -1, 0.0302362, res); + DoAssertEntry("on", 145, 144, 0.00802271, res); + DoAssertEntry("ou", 146, 144, 0.00414545, res); + DoAssertEntry("p", 147, -1, 0.0225116, res); + DoAssertEntry("port", 148, 147, 0.0123532, res); + DoAssertEntry("q", 149, -1, 0.00176561, res); + DoAssertEntry("r", 150, -1, 0.0401677, res); + DoAssertEntry("ran", 151, 150, 0.00686918, res); + DoAssertEntry("s", 152, -1, 0.0487751, res); + DoAssertEntry("sho", 153, 152, 0.0113876, res); + DoAssertEntry("t", 154, -1, 0.0379607, res); + DoAssertEntry("u", 155, -1, 0.0211874, res); + DoAssertEntry("v", 156, -1, 0.00595895, res); + DoAssertEntry("vi", 157, 156, 0.00480673, res); + DoAssertEntry("w", 158, -1, 0.00816597, res); + DoAssertEntry("x", 159, -1, 0.00375193, res); + DoAssertEntry("y", 160, -1, 0.0130214, res); + DoAssertEntry("z", 161, -1, 0.00353123, res); + } +}; + +UNIT_TEST_SUITE_REGISTRATION(TGreedyDictTest); diff --git a/library/cpp/codecs/greedy_dict/ut/ya.make b/library/cpp/codecs/greedy_dict/ut/ya.make index e5d597a083..bd67d1a452 100644 --- a/library/cpp/codecs/greedy_dict/ut/ya.make +++ b/library/cpp/codecs/greedy_dict/ut/ya.make @@ -1,5 +1,5 @@ UNITTEST_FOR(library/cpp/codecs/greedy_dict) - + OWNER(velavokr) SRCS( diff --git a/library/cpp/codecs/greedy_dict/ya.make b/library/cpp/codecs/greedy_dict/ya.make index 6904a354de..2a57224f7e 100644 --- a/library/cpp/codecs/greedy_dict/ya.make +++ b/library/cpp/codecs/greedy_dict/ya.make @@ -1,15 +1,15 @@ OWNER(velavokr) -LIBRARY() - -SRCS( - gd_builder.cpp - gd_entry.cpp -) - -PEERDIR( +LIBRARY() + +SRCS( + gd_builder.cpp + gd_entry.cpp +) + +PEERDIR( library/cpp/containers/comptrie library/cpp/string_utils/relaxed_escaper -) - +) + END() diff --git a/library/cpp/codecs/huffman_codec.cpp b/library/cpp/codecs/huffman_codec.cpp index 391662fb0d..650fe7cdfd 100644 --- a/library/cpp/codecs/huffman_codec.cpp +++ b/library/cpp/codecs/huffman_codec.cpp @@ -1,14 +1,14 @@ -#include "huffman_codec.h" +#include "huffman_codec.h" #include <library/cpp/bit_io/bitinput.h> #include <library/cpp/bit_io/bitoutput.h> - -#include <util/generic/algorithm.h> + +#include <util/generic/algorithm.h> #include <util/generic/bitops.h> -#include <util/stream/buffer.h> -#include <util/stream/length.h> -#include <util/string/printf.h> - -namespace NCodecs { +#include <util/stream/buffer.h> +#include <util/stream/length.h> +#include <util/string/printf.h> + +namespace NCodecs { template <typename T> struct TCanonicalCmp { bool operator()(const T& a, const T& b) const { @@ -19,40 +19,40 @@ namespace NCodecs { } } }; - + template <typename T> struct TByCharCmp { bool operator()(const T& a, const T& b) const { - return a.Char < b.Char; - } + return a.Char < b.Char; + } }; - + struct TTreeEntry { static const ui32 InvalidBranch = (ui32)-1; - + ui64 Freq = 0; ui32 Branches[2]{InvalidBranch, InvalidBranch}; - + ui32 CodeLength = 0; ui8 Char = 0; bool Invalid = false; - + TTreeEntry() = default; - + static bool ByFreq(const TTreeEntry& a, const TTreeEntry& b) { return a.Freq < b.Freq; } - + static bool ByFreqRev(const TTreeEntry& a, const TTreeEntry& b) { return a.Freq > b.Freq; } }; - + using TCodeTree = TVector<TTreeEntry>; - + void InitTreeByFreqs(TCodeTree& tree, const ui64 freqs[256]) { tree.reserve(255 * 256 / 2); // worst case - balanced tree - + for (ui32 i = 0; i < 256; ++i) { tree.emplace_back(); tree.back().Char = i; @@ -72,24 +72,24 @@ namespace NCodecs { for (ui64 i = 0; i < r.size(); ++i) ++freqs[(ui8)r[i]]; } - + InitTreeByFreqs(tree, freqs); - } - + } + void CalculateCodeLengths(TCodeTree& tree) { Y_ENSURE(tree.size() == 256, " "); const ui32 firstbranch = tree.size(); - + ui32 curleaf = 0; ui32 curbranch = firstbranch; - + // building code tree. two priority queues are combined in one. while (firstbranch - curleaf + tree.size() - curbranch >= 2) { TTreeEntry e; - + for (auto& branche : e.Branches) { ui32 br; - + if (curleaf >= firstbranch) br = curbranch++; else if (curbranch >= tree.size()) @@ -98,84 +98,84 @@ namespace NCodecs { br = curleaf++; else br = curbranch++; - + Y_ENSURE(br < tree.size(), " "); branche = br; e.Freq += tree[br].Freq; } - + tree.push_back(e); PushHeap(tree.begin() + curbranch, tree.end(), TTreeEntry::ByFreqRev); - } - + } + // computing code lengths for (ui64 i = tree.size() - 1; i >= firstbranch; --i) { TTreeEntry e = tree[i]; - + for (auto branche : e.Branches) tree[branche].CodeLength = e.CodeLength + 1; } - + // chopping off the branches tree.resize(firstbranch); - + Sort(tree.begin(), tree.end(), TCanonicalCmp<TTreeEntry>()); - + // simplification: we are stripping codes longer than 64 bits while (!tree.empty() && tree.back().CodeLength > 64) tree.pop_back(); - + // will not compress if (tree.empty()) return; - + // special invalid code word tree.back().Invalid = true; } - + struct TEncoderEntry { ui64 Code = 0; - + ui8 CodeLength = 0; ui8 Char = 0; ui8 Invalid = true; - + explicit TEncoderEntry(TTreeEntry e) : CodeLength(e.CodeLength) , Char(e.Char) , Invalid(e.Invalid) { } - + TEncoderEntry() = default; }; - + struct TEncoderTable { TEncoderEntry Entries[256]; - + void Save(IOutputStream* out) const { ui16 nval = 0; - + for (auto entrie : Entries) nval += !entrie.Invalid; - + ::Save(out, nval); - + for (auto entrie : Entries) { if (!entrie.Invalid) { ::Save(out, entrie.Char); ::Save(out, entrie.CodeLength); } - } - } - + } + } + void Load(IInputStream* in) { ui16 nval = 0; ::Load(in, nval); - + for (ui32 i = 0; i < 256; ++i) Entries[i].Char = i; - + for (ui32 i = 0; i < nval; ++i) { ui8 ch = 0; ui8 len = 0; @@ -184,15 +184,15 @@ namespace NCodecs { Entries[ch].CodeLength = len; Entries[ch].Invalid = false; } - } + } }; - + struct TDecoderEntry { ui32 NextTable : 10; ui32 Char : 8; ui32 Invalid : 1; ui32 Bad : 1; - + TDecoderEntry() : NextTable() , Char() @@ -201,27 +201,27 @@ namespace NCodecs { { } }; - + struct TDecoderTable: public TIntrusiveListItem<TDecoderTable> { ui64 Length = 0; ui64 BaseCode = 0; - + TDecoderEntry Entries[256]; - + TDecoderTable() { Zero(Entries); } }; - + const int CACHE_BITS_COUNT = 16; class THuffmanCodec::TImpl: public TAtomicRefCount<TImpl> { TEncoderTable Encoder; TDecoderTable Decoder[256]; - + TEncoderEntry Invalid; - + ui32 SubTablesNum; - + class THuffmanCache { struct TCacheEntry { int EndOffset : 24; @@ -230,7 +230,7 @@ namespace NCodecs { TVector<char> DecodeCache; TVector<TCacheEntry> CacheEntries; const TImpl& Original; - + public: THuffmanCache(const THuffmanCodec::TImpl& encoder); @@ -252,51 +252,51 @@ namespace NCodecs { if (in.empty()) { return 0; } - + out.Reserve(in.size() * 2); - + { NBitIO::TBitOutputVector<TBuffer> bout(&out); TStringBuf tin = in; - + // data is under compression bout.Write(1, 1); - + for (auto t : tin) { const TEncoderEntry& ce = Encoder.Entries[(ui8)t]; - + bout.Write(ce.Code, ce.CodeLength); - + if (ce.Invalid) { bout.Write(t, 8); } } - + // in canonical huffman coding there cannot be a code having no 0 in the suffix // and shorter than 8 bits. bout.Write((ui64)-1, bout.GetByteReminder()); return bout.GetByteReminder(); - } - } - + } + } + void Decode(TStringBuf in, TBuffer& out) const { out.Clear(); - + if (in.empty()) { return; } - + NBitIO::TBitInput bin(in); ui64 f = 0; bin.ReadK<1>(f); - + // if data is uncompressed if (!f) { in.Skip(1); out.Append(in.data(), in.size()); } else { out.Reserve(in.size() * 8); - + if (Cache.Get()) { Cache->Decode(bin, out); } else { @@ -304,36 +304,36 @@ namespace NCodecs { } } } - } - + } + Y_FORCE_INLINE int ReadNextChar(NBitIO::TBitInput& bin, TBuffer& out) const { const TDecoderTable* table = Decoder; TDecoderEntry e; - + int bitsRead = 0; while (true) { ui64 code = 0; - + if (Y_UNLIKELY(!bin.Read(code, table->Length))) return 0; bitsRead += table->Length; - + if (Y_UNLIKELY(code < table->BaseCode)) return 0; - + code -= table->BaseCode; - + if (Y_UNLIKELY(code > 255)) return 0; - + e = table->Entries[code]; - + if (Y_UNLIKELY(e.Bad)) return 0; - + if (e.NextTable) { table = Decoder + e.NextTable; - } else { + } else { if (e.Invalid) { code = 0; bin.ReadK<8>(code); @@ -344,77 +344,77 @@ namespace NCodecs { } return bitsRead; - } + } } - + Y_ENSURE(false, " could not decode input"); return 0; - } - + } + void GenerateEncoder(TCodeTree& tree) { const ui64 sz = tree.size(); - + TEncoderEntry lastcode = Encoder.Entries[tree[0].Char] = TEncoderEntry(tree[0]); - + for (ui32 i = 1; i < sz; ++i) { const TTreeEntry& te = tree[i]; TEncoderEntry& e = Encoder.Entries[te.Char]; e = TEncoderEntry(te); - + e.Code = (lastcode.Code + 1) << (e.CodeLength - lastcode.CodeLength); lastcode = e; - + e.Code = ReverseBits(e.Code, e.CodeLength); - + if (e.Invalid) Invalid = e; } - + for (auto& e : Encoder.Entries) { if (e.Invalid) e = Invalid; Y_ENSURE(e.CodeLength, " "); } - } - + } + void RegenerateEncoder() { for (auto& entrie : Encoder.Entries) { if (entrie.Invalid) entrie.CodeLength = Invalid.CodeLength; } - + Sort(Encoder.Entries, Encoder.Entries + 256, TCanonicalCmp<TEncoderEntry>()); - + TEncoderEntry lastcode = Encoder.Entries[0]; - + for (ui32 i = 1; i < 256; ++i) { TEncoderEntry& e = Encoder.Entries[i]; e.Code = (lastcode.Code + 1) << (e.CodeLength - lastcode.CodeLength); lastcode = e; - + e.Code = ReverseBits(e.Code, e.CodeLength); } - + for (auto& entrie : Encoder.Entries) { if (entrie.Invalid) { Invalid = entrie; break; } } - + Sort(Encoder.Entries, Encoder.Entries + 256, TByCharCmp<TEncoderEntry>()); - + for (auto& entrie : Encoder.Entries) { if (entrie.Invalid) entrie = Invalid; - } - } - + } + } + void BuildDecoder() { TEncoderTable enc = Encoder; Sort(enc.Entries, enc.Entries + 256, TCanonicalCmp<TEncoderEntry>()); - + TEncoderEntry& e1 = enc.Entries[0]; Decoder[0].BaseCode = e1.Code; Decoder[0].Length = e1.CodeLength; @@ -423,22 +423,22 @@ namespace NCodecs { SetEntry(Decoder, e2.Code, e2.CodeLength, e2); } Cache.Reset(new THuffmanCache(*this)); - } - + } + void SetEntry(TDecoderTable* t, ui64 code, ui64 len, TEncoderEntry e) { Y_ENSURE(len >= t->Length, len << " < " << t->Length); - + ui64 idx = (code & MaskLowerBits(t->Length)) - t->BaseCode; TDecoderEntry& d = t->Entries[idx]; - + if (len == t->Length) { Y_ENSURE(!d.NextTable, " "); - + d.Char = e.Char; d.Invalid = e.Invalid; return; } - + if (!d.NextTable) { Y_ENSURE(SubTablesNum < Y_ARRAY_SIZE(Decoder), " "); d.NextTable = SubTablesNum++; @@ -446,10 +446,10 @@ namespace NCodecs { nt->Length = Min<ui64>(8, len - t->Length); nt->BaseCode = (code >> t->Length) & MaskLowerBits(nt->Length); } - + SetEntry(Decoder + d.NextTable, code >> t->Length, len - t->Length, e); - } - + } + void Learn(ISequenceReader* in) { { TCodeTree tree; @@ -459,11 +459,11 @@ namespace NCodecs { GenerateEncoder(tree); } BuildDecoder(); - } - + } + void LearnByFreqs(const TArrayRef<std::pair<char, ui64>>& freqs) { - TCodeTree tree; - + TCodeTree tree; + ui64 freqsArray[256]; Zero(freqsArray); @@ -491,7 +491,7 @@ namespace NCodecs { BuildDecoder(); } }; - + THuffmanCodec::TImpl::THuffmanCache::THuffmanCache(const THuffmanCodec::TImpl& codec) : Original(codec) { @@ -512,7 +512,7 @@ namespace NCodecs { CacheEntries[i] = e; break; } - + for (TBuffer::TConstIterator it = decoded.Begin(); it != decoded.End(); ++it) { DecodeCache.push_back(*it); } @@ -558,32 +558,32 @@ namespace NCodecs { MyTraits.SizeOnDecodeMultiplier = 8; MyTraits.RecommendedSampleSize = 1 << 21; } - + THuffmanCodec::~THuffmanCodec() = default; - + ui8 THuffmanCodec::Encode(TStringBuf in, TBuffer& bbb) const { if (Y_UNLIKELY(!Trained)) ythrow TCodecException() << " not trained"; - + return Impl->Encode(in, bbb); } - + void THuffmanCodec::Decode(TStringBuf in, TBuffer& bbb) const { Impl->Decode(in, bbb); } - + void THuffmanCodec::Save(IOutputStream* out) const { Impl->Save(out); } - + void THuffmanCodec::Load(IInputStream* in) { Impl->Load(in); } - + void THuffmanCodec::DoLearn(ISequenceReader& in) { Impl->Learn(&in); } - + void THuffmanCodec::LearnByFreqs(const TArrayRef<std::pair<char, ui64>>& freqs) { Impl->LearnByFreqs(freqs); Trained = true; diff --git a/library/cpp/codecs/huffman_codec.h b/library/cpp/codecs/huffman_codec.h index 1c00a80637..559545b90d 100644 --- a/library/cpp/codecs/huffman_codec.h +++ b/library/cpp/codecs/huffman_codec.h @@ -1,33 +1,33 @@ -#pragma once - -#include "codecs.h" - -#include <util/generic/ptr.h> +#pragma once + +#include "codecs.h" + +#include <util/generic/ptr.h> #include <util/string/cast.h> - -namespace NCodecs { + +namespace NCodecs { // for types greater than char, pipeline with TFreqCodec. - + class THuffmanCodec: public ICodec { class TImpl; TIntrusivePtr<TImpl> Impl; - + public: THuffmanCodec(); ~THuffmanCodec() override; - + static TStringBuf MyName() { return "huffman"; } - + TString GetName() const override { return ToString(MyName()); } - + ui8 Encode(TStringBuf in, TBuffer& bbb) const override; - + void Decode(TStringBuf in, TBuffer& bbb) const override; - + void LearnByFreqs(const TArrayRef<std::pair<char, ui64>>& freqs); protected: @@ -35,5 +35,5 @@ namespace NCodecs { void Save(IOutputStream* out) const override; void Load(IInputStream* in) override; }; - -} + +} diff --git a/library/cpp/codecs/pfor_codec.cpp b/library/cpp/codecs/pfor_codec.cpp index 3b51c99afa..f6b3b0920b 100644 --- a/library/cpp/codecs/pfor_codec.cpp +++ b/library/cpp/codecs/pfor_codec.cpp @@ -1,6 +1,6 @@ -#include "pfor_codec.h" - -namespace NCodecs { +#include "pfor_codec.h" + +namespace NCodecs { template <> TStringBuf TPForCodec<ui64, true>::MyName() { return "pfor-delta64-sorted"; @@ -9,7 +9,7 @@ namespace NCodecs { TStringBuf TPForCodec<ui32, true>::MyName() { return "pfor-delta32-sorted"; } - + template <> TStringBuf TPForCodec<ui64, false>::MyName() { return "pfor-ui64"; @@ -18,5 +18,5 @@ namespace NCodecs { TStringBuf TPForCodec<ui32, false>::MyName() { return "pfor-ui32"; } - -} + +} diff --git a/library/cpp/codecs/pfor_codec.h b/library/cpp/codecs/pfor_codec.h index b0207512ac..d7d4bb8bf4 100644 --- a/library/cpp/codecs/pfor_codec.h +++ b/library/cpp/codecs/pfor_codec.h @@ -1,48 +1,48 @@ -#pragma once - -#include "codecs.h" - -#include "delta_codec.h" -#include "tls_cache.h" - +#pragma once + +#include "codecs.h" + +#include "delta_codec.h" +#include "tls_cache.h" + #include <library/cpp/bit_io/bitinput.h> #include <library/cpp/bit_io/bitoutput.h> #include <util/string/cast.h> - -namespace NCodecs { + +namespace NCodecs { template <typename T, bool WithDelta = false> class TPForCodec: public ICodec { using TUnsigned = std::make_unsigned_t<T>; typedef TDeltaCodec<TUnsigned> TDCodec; - + typedef std::conditional_t<WithDelta, typename TDCodec::TDelta, T> TValue; static_assert(std::is_unsigned<TValue>::value, "expect std:is_unsigned<TValue>::value"); - + static const ui64 BitsInT = sizeof(TUnsigned) * 8; - + TDCodec DeltaCodec; - + public: static TStringBuf MyName(); - + TPForCodec() { MyTraits.AssumesStructuredInput = true; MyTraits.SizeOfInputElement = sizeof(T); MyTraits.SizeOnDecodeMultiplier = sizeof(T); } - + TString GetName() const override { return ToString(MyName()); } - + ui8 Encode(TStringBuf s, TBuffer& b) const override { b.Clear(); if (s.empty()) { return 0; } - + b.Reserve(2 * s.size() + b.Size()); - + if (WithDelta) { auto buffer = TBufferTlsCache::TlsInstance().Item(); TBuffer& db = buffer.Get(); @@ -51,50 +51,50 @@ namespace NCodecs { DeltaCodec.Encode(s, db); s = TStringBuf{db.data(), db.size()}; } - + TArrayRef<const TValue> tin{(const TValue*)s.data(), s.size() / sizeof(TValue)}; - + const ui64 sz = tin.size(); ui64 bitcounts[BitsInT + 1]; Zero(bitcounts); - + ui32 zeros = 0; - + for (const TValue* it = tin.begin(); it != tin.end(); ++it) { TUnsigned v = 1 + (TUnsigned)*it; ui64 l = MostSignificantBit(v) + 1; ++bitcounts[l]; - + if (!v) { ++zeros; } } - + // cumulative bit counts for (ui64 i = 0; i < BitsInT; ++i) { bitcounts[i + 1] += bitcounts[i]; - } - + } + bool hasexceptions = zeros; ui64 optimalbits = BitsInT; - + { ui64 excsize = 0; ui64 minsize = sz * BitsInT; - + for (ui64 current = BitsInT; current; --current) { ui64 size = bitcounts[current] * current + (sz - bitcounts[current]) * (current + 6 + excsize) + zeros * (current + 6); - + excsize += current * bitcounts[current]; - + if (size < minsize) { minsize = size; optimalbits = current; hasexceptions = zeros || sz - bitcounts[current]; } - } - } - + } + } + if (!optimalbits || BitsInT == optimalbits) { b.Append((ui8)-1); b.Append(s.data(), s.size()); @@ -104,7 +104,7 @@ namespace NCodecs { bout.Write(0, 1); bout.Write(hasexceptions, 1); bout.Write(optimalbits, 6); - + for (const TValue* it = tin.begin(); it != tin.end(); ++it) { TUnsigned word = 1 + (TUnsigned)*it; ui64 len = MostSignificantBit(word) + 1; @@ -116,29 +116,29 @@ namespace NCodecs { } else { bout.Write(word, optimalbits); } - } - + } + return bout.GetByteReminder(); } // the rest of the last byte is zero padded. BitsInT is always > 7. - } - + } + void Decode(TStringBuf s, TBuffer& b) const override { b.Clear(); if (s.empty()) { return; } - + b.Reserve(s.size() * sizeof(T) + b.Size()); - + ui64 isplain = 0; ui64 hasexceptions = 0; ui64 bits = 0; - + NBitIO::TBitInput bin(s); bin.ReadK<1>(isplain); bin.ReadK<1>(hasexceptions); bin.ReadK<6>(bits); - + if (Y_UNLIKELY(isplain)) { s.Skip(1); @@ -147,17 +147,17 @@ namespace NCodecs { } else { b.Append(s.data(), s.size()); } - } else { + } else { typename TDCodec::TDecoder decoder; - + if (hasexceptions) { ui64 word = 0; while (bin.Read(word, bits)) { if (word || (bin.ReadK<6>(word) && bin.Read(word, word))) { --word; - + TValue t = word; - + if (WithDelta) { if (decoder.Decode(t)) { TStringBuf r{(char*)&decoder.Result, sizeof(decoder.Result)}; @@ -166,46 +166,46 @@ namespace NCodecs { } else { TStringBuf r{(char*)&t, sizeof(t)}; b.Append(r.data(), r.size()); - } - } - } + } + } + } } else { ui64 word = 0; T outarr[256 / sizeof(T)]; ui32 cnt = 0; while (true) { ui64 v = bin.Read(word, bits); - + if ((!v) | (!word)) break; - + --word; TValue t = word; - + if (WithDelta) { if (decoder.Decode(t)) { outarr[cnt++] = decoder.Result; } } else { outarr[cnt++] = t; - } + } if (cnt == Y_ARRAY_SIZE(outarr)) { b.Append((const char*)outarr, sizeof(outarr)); cnt = 0; } - } - + } + if (cnt) { b.Append((const char*)outarr, cnt * sizeof(T)); - } - } - } - } - + } + } + } + } + protected: void DoLearn(ISequenceReader&) override { } }; - -} + +} diff --git a/library/cpp/codecs/sample.h b/library/cpp/codecs/sample.h index bce37e6a2c..15f03afcc5 100644 --- a/library/cpp/codecs/sample.h +++ b/library/cpp/codecs/sample.h @@ -1,89 +1,89 @@ -#pragma once - +#pragma once + #include <library/cpp/deprecated/accessors/accessors.h> - -#include <util/generic/buffer.h> -#include <util/generic/vector.h> -#include <util/random/fast.h> -#include <util/random/shuffle.h> - -#include <functional> -#include <type_traits> - -namespace NCodecs { - class ISequenceReader { - public: - virtual bool NextRegion(TStringBuf& s) = 0; - - virtual ~ISequenceReader() = default; - }; - - template <class TValue> - TStringBuf ValueToStringBuf(TValue&& t) { - return TStringBuf{NAccessors::Begin(t), NAccessors::End(t)}; - } - - template <class TIter> + +#include <util/generic/buffer.h> +#include <util/generic/vector.h> +#include <util/random/fast.h> +#include <util/random/shuffle.h> + +#include <functional> +#include <type_traits> + +namespace NCodecs { + class ISequenceReader { + public: + virtual bool NextRegion(TStringBuf& s) = 0; + + virtual ~ISequenceReader() = default; + }; + + template <class TValue> + TStringBuf ValueToStringBuf(TValue&& t) { + return TStringBuf{NAccessors::Begin(t), NAccessors::End(t)}; + } + + template <class TIter> TStringBuf IterToStringBuf(TIter iter) { - return ValueToStringBuf(*iter); - } - - template <class TItem> + return ValueToStringBuf(*iter); + } + + template <class TItem> class TSimpleSequenceReader: public ISequenceReader { const TVector<TItem>& Items; - size_t Idx = 0; - - public: + size_t Idx = 0; + + public: TSimpleSequenceReader(const TVector<TItem>& items) - : Items(items) + : Items(items) { } - - bool NextRegion(TStringBuf& s) override { - if (Idx >= Items.size()) { - return false; - } - - s = ValueToStringBuf(Items[Idx++]); - return true; - } - }; - - template <class TIter, class TGetter> - size_t GetInputSize(TIter begin, TIter end, TGetter getter) { - size_t totalBytes = 0; - for (TIter iter = begin; iter != end; ++iter) { - totalBytes += getter(iter).size(); - } - return totalBytes; - } - - template <class TIter> - size_t GetInputSize(TIter begin, TIter end) { - return GetInputSize(begin, end, IterToStringBuf<TIter>); - } - - template <class TIter, class TGetter> + + bool NextRegion(TStringBuf& s) override { + if (Idx >= Items.size()) { + return false; + } + + s = ValueToStringBuf(Items[Idx++]); + return true; + } + }; + + template <class TIter, class TGetter> + size_t GetInputSize(TIter begin, TIter end, TGetter getter) { + size_t totalBytes = 0; + for (TIter iter = begin; iter != end; ++iter) { + totalBytes += getter(iter).size(); + } + return totalBytes; + } + + template <class TIter> + size_t GetInputSize(TIter begin, TIter end) { + return GetInputSize(begin, end, IterToStringBuf<TIter>); + } + + template <class TIter, class TGetter> TVector<TBuffer> GetSample(TIter begin, TIter end, size_t sampleSizeBytes, TGetter getter) { - TFastRng64 rng{0x1ce1f2e507541a05, 0x07d45659, 0x7b8771030dd9917e, 0x2d6636ce}; - - size_t totalBytes = GetInputSize(begin, end, getter); - double sampleProb = (double)sampleSizeBytes / Max<size_t>(1, totalBytes); - + TFastRng64 rng{0x1ce1f2e507541a05, 0x07d45659, 0x7b8771030dd9917e, 0x2d6636ce}; + + size_t totalBytes = GetInputSize(begin, end, getter); + double sampleProb = (double)sampleSizeBytes / Max<size_t>(1, totalBytes); + TVector<TBuffer> result; - for (TIter iter = begin; iter != end; ++iter) { - if (sampleProb >= 1 || rng.GenRandReal1() < sampleProb) { - TStringBuf reg = getter(iter); + for (TIter iter = begin; iter != end; ++iter) { + if (sampleProb >= 1 || rng.GenRandReal1() < sampleProb) { + TStringBuf reg = getter(iter); result.emplace_back(reg.data(), reg.size()); - } - } - Shuffle(result.begin(), result.end(), rng); - return result; - } - - template <class TIter> + } + } + Shuffle(result.begin(), result.end(), rng); + return result; + } + + template <class TIter> TVector<TBuffer> GetSample(TIter begin, TIter end, size_t sampleSizeBytes) { - return GetSample(begin, end, sampleSizeBytes, IterToStringBuf<TIter>); - } - -} + return GetSample(begin, end, sampleSizeBytes, IterToStringBuf<TIter>); + } + +} diff --git a/library/cpp/codecs/solar_codec.cpp b/library/cpp/codecs/solar_codec.cpp index 088bdead19..d0692fe2a4 100644 --- a/library/cpp/codecs/solar_codec.cpp +++ b/library/cpp/codecs/solar_codec.cpp @@ -1,36 +1,36 @@ -#include "solar_codec.h" - +#include "solar_codec.h" + #include <library/cpp/codecs/greedy_dict/gd_builder.h> - + #include <library/cpp/containers/comptrie/comptrie_builder.h> #include <library/cpp/string_utils/relaxed_escaper/relaxed_escaper.h> -#include <util/stream/length.h> -#include <util/string/printf.h> -#include <util/ysaveload.h> - -namespace NCodecs { +#include <util/stream/length.h> +#include <util/string/printf.h> +#include <util/ysaveload.h> + +namespace NCodecs { static inline ui32 Append(TBuffer& pool, TStringBuf data) { pool.Append(data.data(), data.size()); return pool.Size(); } - + void TSolarCodec::DoLearn(ISequenceReader& r) { using namespace NGreedyDict; - + Decoder.clear(); Pool.Clear(); - + THolder<TEntrySet> set; - + { TMemoryPool pool(8112, TMemoryPool::TLinearGrow::Instance()); TStringBufs bufs; - + TStringBuf m; while (r.NextRegion(m)) { bufs.push_back(pool.AppendString(m)); } - + { TDictBuilder b(Settings); b.SetInput(bufs); @@ -38,66 +38,66 @@ namespace NCodecs { set = b.ReleaseEntrySet(); } - } - + } + set->SetScores(ES_LEN_COUNT); - { + { TVector<std::pair<float, TStringBuf>> tmp; tmp.reserve(set->size()); - + for (const auto& it : *set) { tmp.push_back(std::make_pair(-it.Score, TStringBuf(it.Str).Trunc(Max<ui32>() / Max<ui32>(MaxEntries, 1)))); } - + Sort(tmp.begin(), tmp.end()); - + Decoder.reserve(tmp.size() + 1); Decoder.push_back(0); - + for (const auto& it : tmp) { Y_ENSURE(Decoder.back() == Pool.Size(), "learning invariant failed"); ui32 endoff = Append(Pool, it.second); Decoder.push_back(endoff); } - } - + } + Pool.ShrinkToFit(); Decoder.shrink_to_fit(); - + TBufferOutput bout; - + { TVector<std::pair<TStringBuf, ui32>> tmp2; tmp2.reserve(Decoder.size()); - + for (ui32 i = 1, sz = Decoder.size(); i < sz; ++i) { TStringBuf s = DoDecode(i); tmp2.push_back(std::make_pair(s, i - 1)); Y_ENSURE(s.size() == (Decoder[i] - Decoder[i - 1]), "learning invariant failed"); } - + Sort(tmp2.begin(), tmp2.end()); - + { TEncoder::TBuilder builder(CTBF_PREFIX_GROUPED); for (const auto& it : tmp2) { builder.Add(it.first.data(), it.first.size(), it.second); } - + builder.Save(bout); - } - } - + } + } + Encoder.Init(TBlob::FromBuffer(bout.Buffer())); - } - + } + void TSolarCodec::Save(IOutputStream* out) const { TBlob b = Encoder.Data(); ::Save(out, (ui32)b.Size()); out->Write(b.Data(), b.Size()); } - + void TSolarCodec::Load(IInputStream* in) { ui32 sz; ::Load(in, sz); @@ -105,29 +105,29 @@ namespace NCodecs { Encoder.Init(TBlob::FromStream(lin)); Pool.Clear(); Decoder.clear(); - + TVector<std::pair<ui32, TString>> tmp; - + ui32 poolsz = 0; for (TEncoder::TConstIterator it = Encoder.Begin(); it != Encoder.End(); ++it) { const TString& s = it.GetKey(); tmp.push_back(std::make_pair(it.GetValue(), !s ? TString("\0", 1) : s)); poolsz += Max<ui32>(s.size(), 1); } - + Sort(tmp.begin(), tmp.end()); - + Pool.Reserve(poolsz); Decoder.reserve(tmp.size() + 1); Decoder.push_back(0); - + for (ui32 i = 0, sz2 = tmp.size(); i < sz2; ++i) { Y_ENSURE(i == tmp[i].first, "oops! " << i << " " << tmp[i].first); Decoder.push_back(Append(Pool, tmp[i].second)); } - + Pool.ShrinkToFit(); Decoder.shrink_to_fit(); - } - -} + } + +} diff --git a/library/cpp/codecs/solar_codec.h b/library/cpp/codecs/solar_codec.h index e6c0b891ad..7158ae7926 100644 --- a/library/cpp/codecs/solar_codec.h +++ b/library/cpp/codecs/solar_codec.h @@ -1,16 +1,16 @@ -#pragma once - -#include "codecs.h" +#pragma once + +#include "codecs.h" #include <library/cpp/containers/comptrie/comptrie_trie.h> #include <library/cpp/codecs/greedy_dict/gd_builder.h> - + #include <util/string/cast.h> -#include <util/string/escape.h> - -namespace NCodecs { - // TODO: Попробовать добавлять в словарь вместе с намайненными словами также их суффиксы. - // TODO: Возможно удастся, не слишком потеряв в сжатии, выиграть в робастности к небольшим изменениям в корпусе. - +#include <util/string/escape.h> + +namespace NCodecs { + // TODO: Попробовать добавлять в словарь вместе с намайненными словами также их суффиксы. + // TODO: Возможно удастся, не слишком потеряв в сжатии, выиграть в робастности к небольшим изменениям в корпусе. + struct TVarIntTraits { static const size_t MAX_VARINT32_BYTES = 5; @@ -52,7 +52,7 @@ namespace NCodecs { Y_FORCE_INLINE static void Read(TStringBuf& r, ui32& value) { ui32 result = static_cast<ui8>(r[0]); - r.Skip(1); + r.Skip(1); if (result >= 0x80) { Y_ENSURE_EX(!r.empty(), TCodecException() << "Bad data"); result = ((result << 8) & 0x7FFF) | static_cast<ui8>(r[0]); @@ -100,7 +100,7 @@ namespace NCodecs { static TStringBuf MyNameShortInt() { return TStringBuf("solar-si"); } - + explicit TSolarCodec(ui32 maxentries = 1 << 14, ui32 maxiter = 16, const NGreedyDict::TBuildSettings& s = NGreedyDict::TBuildSettings()) : Settings(s) , MaxEntries(maxentries) @@ -110,7 +110,7 @@ namespace NCodecs { MyTraits.SizeOnDecodeMultiplier = 2; MyTraits.RecommendedSampleSize = maxentries * s.GrowLimit * maxiter * 8; } - + ui8 /*free bits in last byte*/ Encode(TStringBuf r, TBuffer& b) const override { EncodeImpl<TVarIntTraits>(r, b); return 0; @@ -148,8 +148,8 @@ namespace NCodecs { TTraits::Write(val + 1, b); r.Skip(Max<size_t>(sz, 1)); } - } - + } + template <class TTraits> Y_FORCE_INLINE void DecodeImpl(TStringBuf r, TBuffer& b) const { b.Clear(); @@ -160,25 +160,25 @@ namespace NCodecs { TStringBuf s = DoDecode(v); b.Append(s.data(), s.size()); } - } - + } + inline bool CanUseShortInt() const { return Decoder.size() < TShortIntTraits::SHORTINT_SIZE_LIMIT; } - + private: typedef TCompactTrie<char, ui32> TEncoder; typedef TVector<ui32> TDecoder; - + TBuffer Pool; TEncoder Encoder; TDecoder Decoder; - + NGreedyDict::TBuildSettings Settings; ui32 MaxEntries; ui32 MaxIterations; }; - + // Uses varints or shortints depending on the decoder size class TAdaptiveSolarCodec: public TSolarCodec { public: @@ -186,7 +186,7 @@ namespace NCodecs { : TSolarCodec(maxentries, maxiter, s) { } - + ui8 /*free bits in last byte*/ Encode(TStringBuf r, TBuffer& b) const override { if (CanUseShortInt()) { EncodeImpl<TShortIntTraits>(r, b); @@ -225,7 +225,7 @@ namespace NCodecs { EncodeImpl<TShortIntTraits>(r, b); return 0; } - + void Decode(TStringBuf r, TBuffer& b) const override { DecodeImpl<TShortIntTraits>(r, b); } @@ -241,4 +241,4 @@ namespace NCodecs { } }; -} +} diff --git a/library/cpp/codecs/static/builder.cpp b/library/cpp/codecs/static/builder.cpp index 083f0fc6f6..93e34a3edb 100644 --- a/library/cpp/codecs/static/builder.cpp +++ b/library/cpp/codecs/static/builder.cpp @@ -1,39 +1,39 @@ -#include "builder.h" -#include "common.h" - +#include "builder.h" +#include "common.h" + #include <library/cpp/codecs/static/static_codec_info.pb.h> - + #include <library/cpp/codecs/codecs.h> - -#include <util/generic/yexception.h> -#include <util/string/subst.h> - -namespace NCodecs { + +#include <util/generic/yexception.h> +#include <util/string/subst.h> + +namespace NCodecs { TStaticCodecInfo BuildStaticCodec(const TVector<TString>& trainingData, const TCodecBuildInfo& info) { - TStaticCodecInfo result; - TCodecPtr codec = ICodec::GetInstance(info.CodecName); - Y_ENSURE_EX(codec, TCodecException() << "empty codec is not allowed"); - - codec->LearnX(trainingData.begin(), trainingData.end(), info.SampleSizeMultiplier); - { - TStringOutput sout{*result.MutableStoredCodec()}; - ICodec::Store(&sout, codec); - } - - auto& debugInfo = *result.MutableDebugInfo(); - debugInfo.SetStoredCodecHash(DataSignature(result.GetStoredCodec())); - debugInfo.SetCodecName(info.CodecName); - debugInfo.SetSampleSizeMultiplier(info.SampleSizeMultiplier); - debugInfo.SetTimestamp(info.Timestamp); - debugInfo.SetRevisionInfo(info.RevisionInfo); - debugInfo.SetTrainingSetComment(info.TrainingSetComment); - debugInfo.SetTrainingSetResId(info.TrainingSetResId); - return result; - } - + TStaticCodecInfo result; + TCodecPtr codec = ICodec::GetInstance(info.CodecName); + Y_ENSURE_EX(codec, TCodecException() << "empty codec is not allowed"); + + codec->LearnX(trainingData.begin(), trainingData.end(), info.SampleSizeMultiplier); + { + TStringOutput sout{*result.MutableStoredCodec()}; + ICodec::Store(&sout, codec); + } + + auto& debugInfo = *result.MutableDebugInfo(); + debugInfo.SetStoredCodecHash(DataSignature(result.GetStoredCodec())); + debugInfo.SetCodecName(info.CodecName); + debugInfo.SetSampleSizeMultiplier(info.SampleSizeMultiplier); + debugInfo.SetTimestamp(info.Timestamp); + debugInfo.SetRevisionInfo(info.RevisionInfo); + debugInfo.SetTrainingSetComment(info.TrainingSetComment); + debugInfo.SetTrainingSetResId(info.TrainingSetResId); + return result; + } + TString GetStandardFileName(const TStaticCodecInfo& info) { TString cName = info.GetDebugInfo().GetCodecName(); - SubstGlobal(cName, ':', '.'); - return TStringBuilder() << cName << "." << info.GetDebugInfo().GetTimestamp() << ".codec_info"; - } -} + SubstGlobal(cName, ':', '.'); + return TStringBuilder() << cName << "." << info.GetDebugInfo().GetTimestamp() << ".codec_info"; + } +} diff --git a/library/cpp/codecs/static/builder.h b/library/cpp/codecs/static/builder.h index 234ad42dff..d7533be4d5 100644 --- a/library/cpp/codecs/static/builder.h +++ b/library/cpp/codecs/static/builder.h @@ -1,29 +1,29 @@ -#pragma once - -#include "static.h" - +#pragma once + +#include "static.h" + #include <library/cpp/svnversion/svnversion.h> - -#include <util/datetime/base.h> + +#include <util/datetime/base.h> #include <util/generic/string.h> -#include <util/generic/vector.h> -#include <util/string/builder.h> - -namespace NCodecs { - struct TCodecBuildInfo { - // optimal values from SEARCH-1655 +#include <util/generic/vector.h> +#include <util/string/builder.h> + +namespace NCodecs { + struct TCodecBuildInfo { + // optimal values from SEARCH-1655 TString CodecName = "solar-8k-a:zstd08d-1"; - float SampleSizeMultiplier = 1; - - // debug info: - time_t Timestamp = TInstant::Now().TimeT(); + float SampleSizeMultiplier = 1; + + // debug info: + time_t Timestamp = TInstant::Now().TimeT(); TString RevisionInfo = (TStringBuilder() << "r" << ToString(GetProgramSvnRevision())); TString TrainingSetComment; // a human comment on the training data TString TrainingSetResId; // sandbox resid of the training set - }; - + }; + TStaticCodecInfo BuildStaticCodec(const TVector<TString>& trainingData, const TCodecBuildInfo&); - + TString GetStandardFileName(const TStaticCodecInfo&); - -} + +} diff --git a/library/cpp/codecs/static/common.h b/library/cpp/codecs/static/common.h index 84b0349d82..211de2a27d 100644 --- a/library/cpp/codecs/static/common.h +++ b/library/cpp/codecs/static/common.h @@ -1,32 +1,32 @@ -#pragma once - -#include <util/string/hex.h> -#include <util/digest/city.h> -#include <util/system/byteorder.h> - -namespace NCodecs { - template <class T> - ui64 DataSignature(const T& t) { - static_assert(!std::is_scalar<T>::value, "no scalars"); +#pragma once + +#include <util/string/hex.h> +#include <util/digest/city.h> +#include <util/system/byteorder.h> + +namespace NCodecs { + template <class T> + ui64 DataSignature(const T& t) { + static_assert(!std::is_scalar<T>::value, "no scalars"); return CityHash64(t.data(), t.size()); - } - - template <class T> + } + + template <class T> TString HexWriteScalar(T t) { - static_assert(std::is_scalar<T>::value, "scalars only"); - t = LittleToBig(t); + static_assert(std::is_scalar<T>::value, "scalars only"); + t = LittleToBig(t); TString res = HexEncode(&t, sizeof(t)); - res.to_lower(); - return res; - } - - template <class T> - T HexReadScalar(TStringBuf s) { - static_assert(std::is_scalar<T>::value, "scalars only"); - T t = 0; + res.to_lower(); + return res; + } + + template <class T> + T HexReadScalar(TStringBuf s) { + static_assert(std::is_scalar<T>::value, "scalars only"); + T t = 0; HexDecode(s.data(), Min(s.size(), sizeof(T)), &t); - t = BigToLittle(t); - return t; - } - -} + t = BigToLittle(t); + return t; + } + +} diff --git a/library/cpp/codecs/static/example/example.cpp b/library/cpp/codecs/static/example/example.cpp index 0c50a1a5be..5b750b717e 100644 --- a/library/cpp/codecs/static/example/example.cpp +++ b/library/cpp/codecs/static/example/example.cpp @@ -1,43 +1,43 @@ -#include "example.h" - +#include "example.h" + #include <library/cpp/codecs/static/static.h> - -#include <util/generic/yexception.h> - -extern "C" { + +#include <util/generic/yexception.h> + +extern "C" { extern const ui8 codec_info_huff_20160707[]; extern const ui32 codec_info_huff_20160707Size; extern const ui8 codec_info_sa_huff_20160707[]; extern const ui32 codec_info_sa_huff_20160707Size; -}; - -namespace NStaticCodecExample { - static const NCodecs::TCodecConstPtr CODECS[] = { - nullptr, - NCodecs::RestoreCodecFromArchive(codec_info_huff_20160707, codec_info_huff_20160707Size), - NCodecs::RestoreCodecFromArchive(codec_info_sa_huff_20160707, codec_info_sa_huff_20160707Size), - }; - - static_assert(Y_ARRAY_SIZE(CODECS) == DV_COUNT, "bad array size"); - - void Encode(TBuffer& out, TStringBuf in, EDictVersion dv) { - Y_ENSURE(dv > DV_NULL && dv < DV_COUNT, "invalid dict version: " << (int)dv); - out.Clear(); - if (!in) { - return; - } - CODECS[dv]->Encode(in, out); - out.Append((char)dv); - } - - void Decode(TBuffer& out, TStringBuf in) { - out.Clear(); - if (!in) { - return; - } - EDictVersion dv = (EDictVersion)in.back(); - Y_ENSURE(dv > DV_NULL && dv < DV_COUNT, "invalid dict version: " << (int)dv); - in.Chop(1); - CODECS[dv]->Decode(in, out); - } -} +}; + +namespace NStaticCodecExample { + static const NCodecs::TCodecConstPtr CODECS[] = { + nullptr, + NCodecs::RestoreCodecFromArchive(codec_info_huff_20160707, codec_info_huff_20160707Size), + NCodecs::RestoreCodecFromArchive(codec_info_sa_huff_20160707, codec_info_sa_huff_20160707Size), + }; + + static_assert(Y_ARRAY_SIZE(CODECS) == DV_COUNT, "bad array size"); + + void Encode(TBuffer& out, TStringBuf in, EDictVersion dv) { + Y_ENSURE(dv > DV_NULL && dv < DV_COUNT, "invalid dict version: " << (int)dv); + out.Clear(); + if (!in) { + return; + } + CODECS[dv]->Encode(in, out); + out.Append((char)dv); + } + + void Decode(TBuffer& out, TStringBuf in) { + out.Clear(); + if (!in) { + return; + } + EDictVersion dv = (EDictVersion)in.back(); + Y_ENSURE(dv > DV_NULL && dv < DV_COUNT, "invalid dict version: " << (int)dv); + in.Chop(1); + CODECS[dv]->Decode(in, out); + } +} diff --git a/library/cpp/codecs/static/example/example.h b/library/cpp/codecs/static/example/example.h index 070ca90f02..f9b3a7324b 100644 --- a/library/cpp/codecs/static/example/example.h +++ b/library/cpp/codecs/static/example/example.h @@ -1,17 +1,17 @@ -#pragma once - -#include <util/generic/strbuf.h> -#include <util/generic/buffer.h> - -namespace NStaticCodecExample { +#pragma once + +#include <util/generic/strbuf.h> +#include <util/generic/buffer.h> + +namespace NStaticCodecExample { enum EDictVersion : ui8 { DV_NULL = 0, DV_HUFF_20160707, DV_SA_HUFF_20160707, DV_COUNT - }; - - void Encode(TBuffer&, TStringBuf, EDictVersion dv = DV_SA_HUFF_20160707); - - void Decode(TBuffer&, TStringBuf); -} + }; + + void Encode(TBuffer&, TStringBuf, EDictVersion dv = DV_SA_HUFF_20160707); + + void Decode(TBuffer&, TStringBuf); +} diff --git a/library/cpp/codecs/static/example/ya.make b/library/cpp/codecs/static/example/ya.make index 85dc222624..ca6c5fd900 100644 --- a/library/cpp/codecs/static/example/ya.make +++ b/library/cpp/codecs/static/example/ya.make @@ -1,24 +1,24 @@ -LIBRARY() - -OWNER(velavokr) - -SRCS( - GLOBAL example.cpp -) - -PEERDIR( +LIBRARY() + +OWNER(velavokr) + +SRCS( + GLOBAL example.cpp +) + +PEERDIR( library/cpp/codecs library/cpp/codecs/static -) - -ARCHIVE_ASM( +) + +ARCHIVE_ASM( "solar-8k-a.huffman.1467494385.codec_info" NAME codec_info_sa_huff_20160707 -) - -ARCHIVE_ASM( +) + +ARCHIVE_ASM( "huffman.1467494385.codec_info" NAME codec_info_huff_20160707 -) - -END() +) + +END() diff --git a/library/cpp/codecs/static/static.cpp b/library/cpp/codecs/static/static.cpp index d2c99a15ee..44a07dd73a 100644 --- a/library/cpp/codecs/static/static.cpp +++ b/library/cpp/codecs/static/static.cpp @@ -1,98 +1,98 @@ -#include "static.h" -#include "common.h" - +#include "static.h" +#include "common.h" + #include <library/cpp/codecs/static/static_codec_info.pb.h> #include <library/cpp/archive/yarchive.h> - -#include <util/draft/datetime.h> - -#include <util/string/builder.h> -#include <util/stream/buffer.h> -#include <util/stream/mem.h> -#include <util/string/hex.h> -#include <util/ysaveload.h> - -namespace NCodecs { + +#include <util/draft/datetime.h> + +#include <util/string/builder.h> +#include <util/stream/buffer.h> +#include <util/stream/mem.h> +#include <util/string/hex.h> +#include <util/ysaveload.h> + +namespace NCodecs { static constexpr TStringBuf STATIC_CODEC_INFO_MAGIC = "CodecInf"; - - static TStringBuf GetStaticCodecInfoMagic() { + + static TStringBuf GetStaticCodecInfoMagic() { return STATIC_CODEC_INFO_MAGIC; - } - + } + void SaveCodecInfoToStream(IOutputStream& out, const TStaticCodecInfo& info) { - TBufferOutput bout; + TBufferOutput bout; info.SerializeToArcadiaStream(&bout); - ui64 hash = DataSignature(bout.Buffer()); - out.Write(GetStaticCodecInfoMagic()); - ::Save(&out, hash); - ::Save(&out, bout.Buffer()); - } - + ui64 hash = DataSignature(bout.Buffer()); + out.Write(GetStaticCodecInfoMagic()); + ::Save(&out, hash); + ::Save(&out, bout.Buffer()); + } + TStaticCodecInfo LoadCodecInfoFromStream(IInputStream& in) { - { - TBuffer magic; + { + TBuffer magic; magic.Resize(GetStaticCodecInfoMagic().size()); Y_ENSURE_EX(in.Read(magic.Data(), GetStaticCodecInfoMagic().size()) == GetStaticCodecInfoMagic().size(), - TCodecException() << "bad codec info"); + TCodecException() << "bad codec info"); Y_ENSURE_EX(TStringBuf(magic.data(), magic.size()) == GetStaticCodecInfoMagic(), - TCodecException() << "bad codec info"); - } - - ui64 hash; - ::Load(&in, hash); - TBuffer info; - ::Load(&in, info); - Y_ENSURE_EX(hash == DataSignature(info), TCodecException() << "bad codec info"); - - TStaticCodecInfo result; + TCodecException() << "bad codec info"); + } + + ui64 hash; + ::Load(&in, hash); + TBuffer info; + ::Load(&in, info); + Y_ENSURE_EX(hash == DataSignature(info), TCodecException() << "bad codec info"); + + TStaticCodecInfo result; Y_ENSURE_EX(result.ParseFromArray(info.data(), info.size()), TCodecException() << "bad codec info"); - - return result; - } - + + return result; + } + TString SaveCodecInfoToString(const TStaticCodecInfo& info) { - TStringStream s; - SaveCodecInfoToStream(s, info); - return s.Str(); - } - - TStaticCodecInfo LoadCodecInfoFromString(TStringBuf data) { + TStringStream s; + SaveCodecInfoToStream(s, info); + return s.Str(); + } + + TStaticCodecInfo LoadCodecInfoFromString(TStringBuf data) { TMemoryInput m{data.data(), data.size()}; - return LoadCodecInfoFromStream(m); - } - + return LoadCodecInfoFromStream(m); + } + TString FormatCodecInfo(const TStaticCodecInfo& ci) { - TStringBuilder s; - s << "codec name: " << ci.GetDebugInfo().GetCodecName() << Endl; - s << "codec hash: " << HexWriteScalar(ci.GetDebugInfo().GetStoredCodecHash()) << Endl; - s << "dict size: " << ci.GetStoredCodec().Size() << Endl; - s << "sample mult: " << ci.GetDebugInfo().GetSampleSizeMultiplier() << Endl; - s << "orig.compress: " << ci.GetDebugInfo().GetCompression() * 100 << " %" << Endl; - s << "timestamp: " << ci.GetDebugInfo().GetTimestamp() << " (" + TStringBuilder s; + s << "codec name: " << ci.GetDebugInfo().GetCodecName() << Endl; + s << "codec hash: " << HexWriteScalar(ci.GetDebugInfo().GetStoredCodecHash()) << Endl; + s << "dict size: " << ci.GetStoredCodec().Size() << Endl; + s << "sample mult: " << ci.GetDebugInfo().GetSampleSizeMultiplier() << Endl; + s << "orig.compress: " << ci.GetDebugInfo().GetCompression() * 100 << " %" << Endl; + s << "timestamp: " << ci.GetDebugInfo().GetTimestamp() << " (" << NDatetime::TSimpleTM::NewLocal(ci.GetDebugInfo().GetTimestamp()).ToString() << ")" << Endl; - s << "revision: " << ci.GetDebugInfo().GetRevisionInfo() << Endl; - s << "training set comment: " << ci.GetDebugInfo().GetTrainingSetComment() << Endl; - s << "training set resId: " << ci.GetDebugInfo().GetTrainingSetResId() << Endl; - return s; - } - + s << "revision: " << ci.GetDebugInfo().GetRevisionInfo() << Endl; + s << "training set comment: " << ci.GetDebugInfo().GetTrainingSetComment() << Endl; + s << "training set resId: " << ci.GetDebugInfo().GetTrainingSetResId() << Endl; + return s; + } + TString LoadStringFromArchive(const ui8* begin, size_t size) { - TArchiveReader ar(TBlob::NoCopy(begin, size)); - Y_VERIFY(ar.Count() == 1, "invalid number of entries"); - auto blob = ar.ObjectBlobByKey(ar.KeyByIndex(0)); + TArchiveReader ar(TBlob::NoCopy(begin, size)); + Y_VERIFY(ar.Count() == 1, "invalid number of entries"); + auto blob = ar.ObjectBlobByKey(ar.KeyByIndex(0)); return TString{blob.AsCharPtr(), blob.Size()}; - } - - TCodecConstPtr RestoreCodecFromCodecInfo(const TStaticCodecInfo& info) { - return NCodecs::ICodec::RestoreFromString(info.GetStoredCodec()); - } - - TCodecConstPtr RestoreCodecFromArchive(const ui8* begin, size_t size) { - const auto& data = LoadStringFromArchive(begin, size); - const auto& info = LoadCodecInfoFromString(data); - const auto& codec = RestoreCodecFromCodecInfo(info); - Y_ENSURE_EX(codec, TCodecException() << "null codec"); - return codec; - } -} + } + + TCodecConstPtr RestoreCodecFromCodecInfo(const TStaticCodecInfo& info) { + return NCodecs::ICodec::RestoreFromString(info.GetStoredCodec()); + } + + TCodecConstPtr RestoreCodecFromArchive(const ui8* begin, size_t size) { + const auto& data = LoadStringFromArchive(begin, size); + const auto& info = LoadCodecInfoFromString(data); + const auto& codec = RestoreCodecFromCodecInfo(info); + Y_ENSURE_EX(codec, TCodecException() << "null codec"); + return codec; + } +} diff --git a/library/cpp/codecs/static/static.h b/library/cpp/codecs/static/static.h index efa9c60c22..c1eaed2a74 100644 --- a/library/cpp/codecs/static/static.h +++ b/library/cpp/codecs/static/static.h @@ -1,34 +1,34 @@ -#pragma once - +#pragma once + #include <library/cpp/codecs/codecs.h> - -#include <util/generic/strbuf.h> + +#include <util/generic/strbuf.h> #include <util/generic/string.h> #include <util/stream/output.h> - -namespace NCodecs { - class TStaticCodecInfo; - - // load - - TCodecConstPtr RestoreCodecFromCodecInfo(const TStaticCodecInfo&); - - TStaticCodecInfo LoadCodecInfoFromString(TStringBuf data); - + +namespace NCodecs { + class TStaticCodecInfo; + + // load + + TCodecConstPtr RestoreCodecFromCodecInfo(const TStaticCodecInfo&); + + TStaticCodecInfo LoadCodecInfoFromString(TStringBuf data); + TString LoadStringFromArchive(const ui8* begin, size_t size); - - TCodecConstPtr RestoreCodecFromArchive(const ui8* begin, size_t size); - - // save - + + TCodecConstPtr RestoreCodecFromArchive(const ui8* begin, size_t size); + + // save + TString SaveCodecInfoToString(const TStaticCodecInfo&); - + void SaveCodecInfoToStream(IOutputStream& out, const TStaticCodecInfo&); - - // misc - + + // misc + TStaticCodecInfo LoadCodecInfoFromStream(IInputStream& in); - + TString FormatCodecInfo(const TStaticCodecInfo&); - -} + +} diff --git a/library/cpp/codecs/static/static_codec_info.proto b/library/cpp/codecs/static/static_codec_info.proto index 178459784b..362abb4dad 100644 --- a/library/cpp/codecs/static/static_codec_info.proto +++ b/library/cpp/codecs/static/static_codec_info.proto @@ -1,17 +1,17 @@ -package NCodecs; - -message TStaticCodecInfo { - message TDebugInfo { - optional string CodecName = 1; // the exact codec variant name - optional uint64 Timestamp = 2; // when the codec was built - optional string RevisionInfo = 3; // the arcadia revision info - optional float SampleSizeMultiplier = 4; // how the default sample size was modified to improve compression - optional float Compression = 5; // the compression on the training set ((raw_size - coded_size) / raw_size) - optional string TrainingSetComment = 6; // a human readable description of the training set - optional string TrainingSetResId = 7; // the training set sandbox resource id - optional uint64 StoredCodecHash = 8; // cityhash64(data) - } - - optional bytes StoredCodec = 1; // the data of the codec - optional TDebugInfo DebugInfo = 2; // misc debug info which could be useful in finding whereabouts later -} +package NCodecs; + +message TStaticCodecInfo { + message TDebugInfo { + optional string CodecName = 1; // the exact codec variant name + optional uint64 Timestamp = 2; // when the codec was built + optional string RevisionInfo = 3; // the arcadia revision info + optional float SampleSizeMultiplier = 4; // how the default sample size was modified to improve compression + optional float Compression = 5; // the compression on the training set ((raw_size - coded_size) / raw_size) + optional string TrainingSetComment = 6; // a human readable description of the training set + optional string TrainingSetResId = 7; // the training set sandbox resource id + optional uint64 StoredCodecHash = 8; // cityhash64(data) + } + + optional bytes StoredCodec = 1; // the data of the codec + optional TDebugInfo DebugInfo = 2; // misc debug info which could be useful in finding whereabouts later +} diff --git a/library/cpp/codecs/static/tools/common/ct_common.cpp b/library/cpp/codecs/static/tools/common/ct_common.cpp index cea40506e1..fe77691280 100644 --- a/library/cpp/codecs/static/tools/common/ct_common.cpp +++ b/library/cpp/codecs/static/tools/common/ct_common.cpp @@ -1,74 +1,74 @@ -#include "ct_common.h" - +#include "ct_common.h" + #include <library/cpp/codecs/codecs.h> #include <library/cpp/codecs/static/static_codec_info.pb.h> #include <library/cpp/string_utils/base64/base64.h> - + #include <util/stream/output.h> -#include <util/string/builder.h> -#include <util/system/hp_timer.h> - -namespace NCodecs { +#include <util/string/builder.h> +#include <util/system/hp_timer.h> + +namespace NCodecs { TString TComprStats::Format(const TStaticCodecInfo& info, bool checkMode) const { - TStringBuilder s; - s << "raw size/item: " << RawSizePerRecord() << Endl; - s << "enc.size/item: " << EncSizePerRecord() << Endl; - if (checkMode) { - s << "orig.enc.size/item: " << OldEncSizePerRecord(info.GetDebugInfo().GetCompression()) << Endl; - } - s << "enc time us/item: " << EncTimePerRecordUS() << Endl; - s << "dec time us/item: " << DecTimePerRecordUS() << Endl; - s << "dict size: " << info.GetStoredCodec().Size() << Endl; - s << "compression: " << AsPercent(Compression()) << " %" << Endl; - if (checkMode) { - s << "orig.compression: " << AsPercent(info.GetDebugInfo().GetCompression()) << " %" << Endl; - } - return s; - } - + TStringBuilder s; + s << "raw size/item: " << RawSizePerRecord() << Endl; + s << "enc.size/item: " << EncSizePerRecord() << Endl; + if (checkMode) { + s << "orig.enc.size/item: " << OldEncSizePerRecord(info.GetDebugInfo().GetCompression()) << Endl; + } + s << "enc time us/item: " << EncTimePerRecordUS() << Endl; + s << "dec time us/item: " << DecTimePerRecordUS() << Endl; + s << "dict size: " << info.GetStoredCodec().Size() << Endl; + s << "compression: " << AsPercent(Compression()) << " %" << Endl; + if (checkMode) { + s << "orig.compression: " << AsPercent(info.GetDebugInfo().GetCompression()) << " %" << Endl; + } + return s; + } + TComprStats TestCodec(const ICodec& c, const TVector<TString>& input) { - TComprStats stats; - - TBuffer encodeBuffer; - TBuffer decodeBuffer; - for (const auto& data : input) { - encodeBuffer.Clear(); - decodeBuffer.Clear(); - - stats.Records += 1; + TComprStats stats; + + TBuffer encodeBuffer; + TBuffer decodeBuffer; + for (const auto& data : input) { + encodeBuffer.Clear(); + decodeBuffer.Clear(); + + stats.Records += 1; stats.RawSize += data.size(); - - THPTimer timer; - c.Encode(data, encodeBuffer); + + THPTimer timer; + c.Encode(data, encodeBuffer); stats.EncSize += encodeBuffer.size(); - stats.EncSeconds += timer.PassedReset(); - + stats.EncSeconds += timer.PassedReset(); + c.Decode(TStringBuf{encodeBuffer.data(), encodeBuffer.size()}, decodeBuffer); - stats.DecSeconds += timer.PassedReset(); + stats.DecSeconds += timer.PassedReset(); Y_ENSURE(data == TStringBuf(decodeBuffer.data(), decodeBuffer.size()), "invalid encoding at record " << stats.Records); - } - - return stats; - } - + } + + return stats; + } + void ParseBlob(TVector<TString>& result, EDataStreamFormat fmt, const TBlob& blob) { TStringBuf bin(blob.AsCharPtr(), blob.Size()); - TStringBuf line; + TStringBuf line; TString buffer; - while (bin.ReadLine(line)) { - if (DSF_BASE64_LF == fmt) { - Base64Decode(line, buffer); - line = buffer; - } - if (!line) { - continue; - } + while (bin.ReadLine(line)) { + if (DSF_BASE64_LF == fmt) { + Base64Decode(line, buffer); + line = buffer; + } + if (!line) { + continue; + } result.emplace_back(line.data(), line.size()); - } - } - + } + } + TBlob GetInputBlob(const TString& dataFile) { - return dataFile && dataFile != "-" ? TBlob::FromFile(dataFile) : TBlob::FromStream(Cin); - } - -} + return dataFile && dataFile != "-" ? TBlob::FromFile(dataFile) : TBlob::FromStream(Cin); + } + +} diff --git a/library/cpp/codecs/static/tools/common/ct_common.h b/library/cpp/codecs/static/tools/common/ct_common.h index de531b27e6..9d3dcbda93 100644 --- a/library/cpp/codecs/static/tools/common/ct_common.h +++ b/library/cpp/codecs/static/tools/common/ct_common.h @@ -1,75 +1,75 @@ -#pragma once - +#pragma once + #include <util/generic/string.h> -#include <util/generic/vector.h> -#include <util/memory/blob.h> -#include <cmath> - -namespace NCodecs { - class TStaticCodecInfo; - class ICodec; - - struct TComprStats { - double EncSeconds = 0; - double DecSeconds = 0; - size_t Records = 0; - size_t RawSize = 0; - size_t EncSize = 0; - - static double Round(double n, size_t decPlaces = 2) { - double p = pow(10, decPlaces); - return round(n * p) / p; - } - - static double AsPercent(double n) { - return Round(n * 100); - } - - static double AsMicroSecond(double s) { - return s * 1000000; - } - - double PerRecord(double n) const { - return Round((double)(Records ? n / Records : 0)); - } - - double Compression() const { - return ((double)RawSize - (double)EncSize) / RawSize; - } - - double EncTimePerRecordUS() const { - return PerRecord(AsMicroSecond(EncSeconds)); - } - - double DecTimePerRecordUS() const { - return PerRecord(AsMicroSecond(DecSeconds)); - } - - double RawSizePerRecord() const { - return PerRecord(RawSize); - } - - double EncSizePerRecord() const { - return PerRecord(EncSize); - } - - double OldEncSizePerRecord(double compr) const { - return PerRecord((1 - compr) * RawSize); - } - +#include <util/generic/vector.h> +#include <util/memory/blob.h> +#include <cmath> + +namespace NCodecs { + class TStaticCodecInfo; + class ICodec; + + struct TComprStats { + double EncSeconds = 0; + double DecSeconds = 0; + size_t Records = 0; + size_t RawSize = 0; + size_t EncSize = 0; + + static double Round(double n, size_t decPlaces = 2) { + double p = pow(10, decPlaces); + return round(n * p) / p; + } + + static double AsPercent(double n) { + return Round(n * 100); + } + + static double AsMicroSecond(double s) { + return s * 1000000; + } + + double PerRecord(double n) const { + return Round((double)(Records ? n / Records : 0)); + } + + double Compression() const { + return ((double)RawSize - (double)EncSize) / RawSize; + } + + double EncTimePerRecordUS() const { + return PerRecord(AsMicroSecond(EncSeconds)); + } + + double DecTimePerRecordUS() const { + return PerRecord(AsMicroSecond(DecSeconds)); + } + + double RawSizePerRecord() const { + return PerRecord(RawSize); + } + + double EncSizePerRecord() const { + return PerRecord(EncSize); + } + + double OldEncSizePerRecord(double compr) const { + return PerRecord((1 - compr) * RawSize); + } + TString Format(const TStaticCodecInfo&, bool checkMode) const; - }; - + }; + TComprStats TestCodec(const ICodec&, const TVector<TString>& data); - - enum EDataStreamFormat { - DSF_NONE, - DSF_PLAIN_LF /* "plain" */, - DSF_BASE64_LF /* "base64" */, - }; - + + enum EDataStreamFormat { + DSF_NONE, + DSF_PLAIN_LF /* "plain" */, + DSF_BASE64_LF /* "base64" */, + }; + void ParseBlob(TVector<TString>&, EDataStreamFormat, const TBlob&); - + TBlob GetInputBlob(const TString& dataFile); - -} + +} diff --git a/library/cpp/codecs/static/tools/common/ya.make b/library/cpp/codecs/static/tools/common/ya.make index 5f575a2f28..d624222dad 100644 --- a/library/cpp/codecs/static/tools/common/ya.make +++ b/library/cpp/codecs/static/tools/common/ya.make @@ -1,19 +1,19 @@ -LIBRARY() - +LIBRARY() + OWNER(velavokr) - -SRCS( - ct_common.cpp -) - -PEERDIR( + +SRCS( + ct_common.cpp +) + +PEERDIR( library/cpp/codecs library/cpp/codecs/static library/cpp/getopt/small library/cpp/string_utils/base64 - util/draft -) - + util/draft +) + GENERATE_ENUM_SERIALIZATION(ct_common.h) - -END() + +END() diff --git a/library/cpp/codecs/static/tools/static_codec_checker/README b/library/cpp/codecs/static/tools/static_codec_checker/README index c66703227d..723a68300b 100644 --- a/library/cpp/codecs/static/tools/static_codec_checker/README +++ b/library/cpp/codecs/static/tools/static_codec_checker/README @@ -1,4 +1,4 @@ This is a viewer for generated codec and utility for verification of the compression quality on a new data. - + Usage: -static_codec_checker -t -c 029b29ff64a74927.codec_info -f plain samples.txt +static_codec_checker -t -c 029b29ff64a74927.codec_info -f plain samples.txt diff --git a/library/cpp/codecs/static/tools/static_codec_checker/static_codec_checker.cpp b/library/cpp/codecs/static/tools/static_codec_checker/static_codec_checker.cpp index 5ae901d8f8..9c8d568d82 100644 --- a/library/cpp/codecs/static/tools/static_codec_checker/static_codec_checker.cpp +++ b/library/cpp/codecs/static/tools/static_codec_checker/static_codec_checker.cpp @@ -3,25 +3,25 @@ #include <library/cpp/codecs/static/static_codec_info.pb.h> #include <library/cpp/codecs/codecs.h> #include <library/cpp/getopt/small/last_getopt.h> - -#include <util/digest/city.h> -#include <util/generic/yexception.h> -#include <util/stream/file.h> -#include <util/stream/buffer.h> -#include <util/stream/format.h> -#include <util/string/builder.h> - -int main(int argc, char** argv) { - NCodecs::TCodecPtr codecPtr; - NCodecs::EDataStreamFormat fmt = NCodecs::DSF_NONE; + +#include <util/digest/city.h> +#include <util/generic/yexception.h> +#include <util/stream/file.h> +#include <util/stream/buffer.h> +#include <util/stream/format.h> +#include <util/string/builder.h> + +int main(int argc, char** argv) { + NCodecs::TCodecPtr codecPtr; + NCodecs::EDataStreamFormat fmt = NCodecs::DSF_NONE; TString codecFile; - bool testCompression = false; - - auto opts = NLastGetopt::TOpts::Default(); - opts.SetTitle("Prints a .codec_info file and optionally checks its performance on new data. See also static_codec_generator."); - opts.SetCmdLineDescr("-c 9089f3e9b7a0f0d4.codec_info -t -f base64 qtrees.sample.txt"); - NCodecs::TStaticCodecInfo codec; - + bool testCompression = false; + + auto opts = NLastGetopt::TOpts::Default(); + opts.SetTitle("Prints a .codec_info file and optionally checks its performance on new data. See also static_codec_generator."); + opts.SetCmdLineDescr("-c 9089f3e9b7a0f0d4.codec_info -t -f base64 qtrees.sample.txt"); + NCodecs::TStaticCodecInfo codec; + opts.AddLongOption('c', "codec-info").RequiredArgument("codec_info").Handler1T<TString>([&codecFile, &codec, &codecPtr](TString name) { codecFile = name; codec.CopyFrom(NCodecs::LoadCodecInfoFromString(TUnbufferedFileInput(name).ReadAll())); @@ -29,45 +29,45 @@ int main(int argc, char** argv) { }) .Required() .Help(".codec_info file with serialized static data for codec"); - + opts.AddLongOption('t', "test").NoArgument().StoreValue(&testCompression, true).Optional().Help("test current performance"); - + opts.AddLongOption('f', "format").RequiredArgument(TStringBuilder() << "(" << NCodecs::DSF_PLAIN_LF << "|" << NCodecs::DSF_BASE64_LF << ")").StoreResult(&fmt).Optional().Help("test set input file format"); - - opts.SetFreeArgsMin(0); - opts.SetFreeArgTitle(0, "testing_set_input_file", "testing set input files"); - - NLastGetopt::TOptsParseResult res(&opts, argc, argv); - - Cout << codecFile << Endl; - Cout << NCodecs::FormatCodecInfo(codec) << Endl; - - if (testCompression) { - if (NCodecs::DSF_NONE == fmt) { - Cerr << "Specify format (-f|--format) for testing set input" << Endl; - exit(1); - } - - Cout << "Reading testing set data ... " << Flush; - + + opts.SetFreeArgsMin(0); + opts.SetFreeArgTitle(0, "testing_set_input_file", "testing set input files"); + + NLastGetopt::TOptsParseResult res(&opts, argc, argv); + + Cout << codecFile << Endl; + Cout << NCodecs::FormatCodecInfo(codec) << Endl; + + if (testCompression) { + if (NCodecs::DSF_NONE == fmt) { + Cerr << "Specify format (-f|--format) for testing set input" << Endl; + exit(1); + } + + Cout << "Reading testing set data ... " << Flush; + TVector<TString> allData; - for (const auto& freeArg : res.GetFreeArgs()) { - NCodecs::ParseBlob(allData, fmt, NCodecs::GetInputBlob(freeArg)); - } - - if (!res.GetFreeArgs()) { - NCodecs::ParseBlob(allData, fmt, NCodecs::GetInputBlob("-")); - } - - Cout << "Done" << Endl << Endl; - - Cout << "records: " << allData.size() << Endl; - Cout << "raw size: " << NCodecs::GetInputSize(allData.begin(), allData.end()) << " bytes" << Endl << Endl; - - Cout << "Testing compression ... " << Flush; - auto stats = NCodecs::TestCodec(*codecPtr, allData); - Cout << "Done" << Endl << Endl; - - Cout << stats.Format(codec, true) << Endl; - } -} + for (const auto& freeArg : res.GetFreeArgs()) { + NCodecs::ParseBlob(allData, fmt, NCodecs::GetInputBlob(freeArg)); + } + + if (!res.GetFreeArgs()) { + NCodecs::ParseBlob(allData, fmt, NCodecs::GetInputBlob("-")); + } + + Cout << "Done" << Endl << Endl; + + Cout << "records: " << allData.size() << Endl; + Cout << "raw size: " << NCodecs::GetInputSize(allData.begin(), allData.end()) << " bytes" << Endl << Endl; + + Cout << "Testing compression ... " << Flush; + auto stats = NCodecs::TestCodec(*codecPtr, allData); + Cout << "Done" << Endl << Endl; + + Cout << stats.Format(codec, true) << Endl; + } +} diff --git a/library/cpp/codecs/static/tools/static_codec_checker/ya.make b/library/cpp/codecs/static/tools/static_codec_checker/ya.make index 86b73dff6c..90e06ca448 100644 --- a/library/cpp/codecs/static/tools/static_codec_checker/ya.make +++ b/library/cpp/codecs/static/tools/static_codec_checker/ya.make @@ -1,16 +1,16 @@ -PROGRAM() - +PROGRAM() + OWNER(velavokr) - -SRCS( - static_codec_checker.cpp -) - -PEERDIR( + +SRCS( + static_codec_checker.cpp +) + +PEERDIR( library/cpp/codecs library/cpp/codecs/static library/cpp/codecs/static/tools/common library/cpp/getopt/small -) - -END() +) + +END() diff --git a/library/cpp/codecs/static/tools/static_codec_generator/README b/library/cpp/codecs/static/tools/static_codec_generator/README index f0fffd745a..e6bb52b959 100644 --- a/library/cpp/codecs/static/tools/static_codec_generator/README +++ b/library/cpp/codecs/static/tools/static_codec_generator/README @@ -1,4 +1,4 @@ This is a utility for reproducible teaching of a codec. And also for saving it into a file with a unique name for a static compilation as a resource. - + Usage: -static_codec_generator -t -m 'the training data description' -f plain samples.txt +static_codec_generator -t -m 'the training data description' -f plain samples.txt diff --git a/library/cpp/codecs/static/tools/static_codec_generator/static_codec_generator.cpp b/library/cpp/codecs/static/tools/static_codec_generator/static_codec_generator.cpp index b37a0f686d..45fdb5c5fe 100644 --- a/library/cpp/codecs/static/tools/static_codec_generator/static_codec_generator.cpp +++ b/library/cpp/codecs/static/tools/static_codec_generator/static_codec_generator.cpp @@ -2,81 +2,81 @@ #include <library/cpp/codecs/static/static_codec_info.pb.h> #include <library/cpp/codecs/static/builder.h> #include <library/cpp/codecs/codecs.h> - + #include <library/cpp/getopt/small/last_getopt.h> - -#include <util/generic/yexception.h> -#include <util/stream/file.h> -#include <util/string/builder.h> - -int main(int argc, char** argv) { - NCodecs::TCodecBuildInfo info; - NCodecs::EDataStreamFormat fmt = NCodecs::DSF_NONE; - - auto opts = NLastGetopt::TOpts::Default(); - opts.SetCmdLineDescr("-m 'Training set: 100000 qtrees taken from web mmeta logs' -f base64 qtrees.sample.txt"); - opts.SetTitle("Teaches the codec and serializes it as a file named CODECNAME.hash(CODECDATA).bin"); - + +#include <util/generic/yexception.h> +#include <util/stream/file.h> +#include <util/string/builder.h> + +int main(int argc, char** argv) { + NCodecs::TCodecBuildInfo info; + NCodecs::EDataStreamFormat fmt = NCodecs::DSF_NONE; + + auto opts = NLastGetopt::TOpts::Default(); + opts.SetCmdLineDescr("-m 'Training set: 100000 qtrees taken from web mmeta logs' -f base64 qtrees.sample.txt"); + opts.SetTitle("Teaches the codec and serializes it as a file named CODECNAME.hash(CODECDATA).bin"); + opts.AddLongOption('m', "message").RequiredArgument("training_set_comment").StoreResult(&info.TrainingSetComment).Required().Help("a human description for the training set"); - + opts.AddLongOption('r', "resource").RequiredArgument("training_set_res_id").StoreResult(&info.TrainingSetResId).Optional().Help("sandbox resource id for the training set"); - + opts.AddLongOption('c', "codec").RequiredArgument("codec_name").StoreResult(&info.CodecName).Optional().DefaultValue(info.CodecName); - + opts.AddLongOption('s', "sample-multiplier").RequiredArgument("multiplier").StoreResult(&info.SampleSizeMultiplier).Optional().DefaultValue(ToString(info.SampleSizeMultiplier)).Help("multiplier for default sample size"); - + opts.AddLongOption('f', "format").RequiredArgument(TStringBuilder() << "(" << NCodecs::DSF_PLAIN_LF << "|" << NCodecs::DSF_BASE64_LF << ")").StoreResult(&fmt).Required().Help("training set input file format"); - + opts.AddLongOption("list-codecs").NoArgument().Handler0([]() { Cout << JoinStrings(NCodecs::ICodec::GetCodecsList(), "\n") << Endl; exit(0); }) .Optional() .Help("list available codecs"); - + opts.AddLongOption("fake-revision").RequiredArgument("revision").StoreResult(&info.RevisionInfo).Optional().Hidden(); // replace static_codec_generator revision in debug info - + opts.AddLongOption("fake-timestamp").RequiredArgument("timestamp").StoreResult(&info.Timestamp).Optional().Hidden(); // replace generating timestamp in debug info - - opts.SetFreeArgsMin(0); - opts.SetFreeArgTitle(0, "training_set_input_file", "training set input files"); - - NLastGetopt::TOptsParseResult res(&opts, argc, argv); - - Cout << "Reading training set data ... " << Flush; + + opts.SetFreeArgsMin(0); + opts.SetFreeArgTitle(0, "training_set_input_file", "training set input files"); + + NLastGetopt::TOptsParseResult res(&opts, argc, argv); + + Cout << "Reading training set data ... " << Flush; TVector<TString> allData; - for (const auto& freeArg : res.GetFreeArgs()) { - NCodecs::ParseBlob(allData, fmt, NCodecs::GetInputBlob(freeArg)); - } - - if (!res.GetFreeArgs()) { - NCodecs::ParseBlob(allData, fmt, NCodecs::GetInputBlob("-")); - } - Cout << "Done" << Endl << Endl; - - Cout << "records: " << allData.size() << Endl; - Cout << "raw size: " << NCodecs::GetInputSize(allData.begin(), allData.end()) << " bytes" << Endl << Endl; - - Cout << "Training " << info.CodecName << " , sample size multiplier is " << info.SampleSizeMultiplier << " ... " << Flush; - auto codec = NCodecs::BuildStaticCodec(allData, info); - Cout << "Done" << Endl; - + for (const auto& freeArg : res.GetFreeArgs()) { + NCodecs::ParseBlob(allData, fmt, NCodecs::GetInputBlob(freeArg)); + } + + if (!res.GetFreeArgs()) { + NCodecs::ParseBlob(allData, fmt, NCodecs::GetInputBlob("-")); + } + Cout << "Done" << Endl << Endl; + + Cout << "records: " << allData.size() << Endl; + Cout << "raw size: " << NCodecs::GetInputSize(allData.begin(), allData.end()) << " bytes" << Endl << Endl; + + Cout << "Training " << info.CodecName << " , sample size multiplier is " << info.SampleSizeMultiplier << " ... " << Flush; + auto codec = NCodecs::BuildStaticCodec(allData, info); + Cout << "Done" << Endl; + TString codecName = NCodecs::GetStandardFileName(codec); - NCodecs::TCodecPtr codecPtr = NCodecs::ICodec::RestoreFromString(codec.GetStoredCodec()); - - Cout << "Testing compression ... " << Flush; - auto stats = NCodecs::TestCodec(*codecPtr, allData); - Cout << "Done" << Endl << Endl; - - codec.MutableDebugInfo()->SetCompression(stats.Compression()); - - Cout << stats.Format(codec, false) << Endl; - - Cout << "Saving as " << codecName << " ... " << Flush; - { + NCodecs::TCodecPtr codecPtr = NCodecs::ICodec::RestoreFromString(codec.GetStoredCodec()); + + Cout << "Testing compression ... " << Flush; + auto stats = NCodecs::TestCodec(*codecPtr, allData); + Cout << "Done" << Endl << Endl; + + codec.MutableDebugInfo()->SetCompression(stats.Compression()); + + Cout << stats.Format(codec, false) << Endl; + + Cout << "Saving as " << codecName << " ... " << Flush; + { TUnbufferedFileOutput fout{codecName}; - NCodecs::SaveCodecInfoToStream(fout, codec); - fout.Finish(); - } - Cout << "Done" << Endl << Endl; -} + NCodecs::SaveCodecInfoToStream(fout, codec); + fout.Finish(); + } + Cout << "Done" << Endl << Endl; +} diff --git a/library/cpp/codecs/static/tools/static_codec_generator/ya.make b/library/cpp/codecs/static/tools/static_codec_generator/ya.make index 21750dde49..efbc440dd1 100644 --- a/library/cpp/codecs/static/tools/static_codec_generator/ya.make +++ b/library/cpp/codecs/static/tools/static_codec_generator/ya.make @@ -1,17 +1,17 @@ -PROGRAM() - +PROGRAM() + OWNER(velavokr) - -SRCS( - static_codec_generator.cpp -) - -PEERDIR( + +SRCS( + static_codec_generator.cpp +) + +PEERDIR( library/cpp/codecs library/cpp/codecs/static library/cpp/codecs/static/tools/common library/cpp/digest/md5 library/cpp/getopt/small -) - -END() +) + +END() diff --git a/library/cpp/codecs/static/tools/tests/static_codec_tools.py b/library/cpp/codecs/static/tools/tests/static_codec_tools.py index a5baa262f7..db4140e370 100644 --- a/library/cpp/codecs/static/tools/tests/static_codec_tools.py +++ b/library/cpp/codecs/static/tools/tests/static_codec_tools.py @@ -1,18 +1,18 @@ -#!/usr/bin/env python - -import yatest.common as tt -import os.path as op - -def test_static_codec_tools(): +#!/usr/bin/env python + +import yatest.common as tt +import os.path as op + +def test_static_codec_tools(): tt.execute([tt.binary_path("library/cpp/codecs/static/tools/static_codec_generator/static_codec_generator")] - + ["-m", "test codec", "-r", "sbr://143310406", "-f", "plain", "-c", "solar-8k-a:huffman", "-s", "1", - "--fake-revision", "r2385905", "--fake-timestamp", "1467494385", "sample.txt"], - timeout=60) - assert(op.exists("solar-8k-a.huffman.1467494385.codec_info")) + + ["-m", "test codec", "-r", "sbr://143310406", "-f", "plain", "-c", "solar-8k-a:huffman", "-s", "1", + "--fake-revision", "r2385905", "--fake-timestamp", "1467494385", "sample.txt"], + timeout=60) + assert(op.exists("solar-8k-a.huffman.1467494385.codec_info")) tt.canonical_execute(tt.binary_path("library/cpp/codecs/static/tools/static_codec_checker/static_codec_checker"), - args=["-c", "solar-8k-a.huffman.1467494385.codec_info"], - timeout=60) + args=["-c", "solar-8k-a.huffman.1467494385.codec_info"], + timeout=60) tt.execute([tt.binary_path("library/cpp/codecs/static/tools/static_codec_checker/static_codec_checker")] - + ["-c", "solar-8k-a.huffman.1467494385.codec_info", "-f", "plain", "-t", "sample.txt"], - timeout=60) - return tt.canonical_file("solar-8k-a.huffman.1467494385.codec_info") + + ["-c", "solar-8k-a.huffman.1467494385.codec_info", "-f", "plain", "-t", "sample.txt"], + timeout=60) + return tt.canonical_file("solar-8k-a.huffman.1467494385.codec_info") diff --git a/library/cpp/codecs/static/tools/tests/ya.make b/library/cpp/codecs/static/tools/tests/ya.make index 5555d90cae..c5324eaf53 100644 --- a/library/cpp/codecs/static/tools/tests/ya.make +++ b/library/cpp/codecs/static/tools/tests/ya.make @@ -1,20 +1,20 @@ PY2TEST() - -OWNER(velavokr) - -TEST_SRCS(static_codec_tools.py) - -DATA(sbr://143310406) - -TIMEOUT(4200) - + +OWNER(velavokr) + +TEST_SRCS(static_codec_tools.py) + +DATA(sbr://143310406) + +TIMEOUT(4200) + TAG(ya:not_autocheck) -DEPENDS( +DEPENDS( library/cpp/codecs/static/tools/static_codec_checker library/cpp/codecs/static/tools/static_codec_generator -) - - +) + + -END() +END() diff --git a/library/cpp/codecs/static/tools/ya.make b/library/cpp/codecs/static/tools/ya.make index ab72769153..dd3e8437aa 100644 --- a/library/cpp/codecs/static/tools/ya.make +++ b/library/cpp/codecs/static/tools/ya.make @@ -1,5 +1,5 @@ -RECURSE( - common - static_codec_generator - static_codec_checker -) +RECURSE( + common + static_codec_generator + static_codec_checker +) diff --git a/library/cpp/codecs/static/ut/builder_ut.cpp b/library/cpp/codecs/static/ut/builder_ut.cpp index 48d5c98d5d..b47c279ed1 100644 --- a/library/cpp/codecs/static/ut/builder_ut.cpp +++ b/library/cpp/codecs/static/ut/builder_ut.cpp @@ -1,57 +1,57 @@ #include <library/cpp/testing/unittest/registar.h> #include <library/cpp/codecs/static/builder.h> #include <library/cpp/codecs/static/static_codec_info.pb.h> -#include <util/string/vector.h> - +#include <util/string/vector.h> + class TStaticCodecInfoBuilderTest: public NUnitTest::TTestBase { - UNIT_TEST_SUITE(TStaticCodecInfoBuilderTest) + UNIT_TEST_SUITE(TStaticCodecInfoBuilderTest) UNIT_TEST(TestBuild) - UNIT_TEST_SUITE_END(); + UNIT_TEST_SUITE_END(); -private: +private: TVector<TString> PrepareData() { TVector<TString> data; - for (ui32 i = 'a'; i <= 'z'; ++i) { + for (ui32 i = 'a'; i <= 'z'; ++i) { data.push_back(TString(1, (char)i)); - } - return data; - } - - void TestBuild() { + } + return data; + } + + void TestBuild() { TVector<TString> data; - NCodecs::TCodecBuildInfo info; - info.CodecName = "huffman"; - info.SampleSizeMultiplier = 2; - info.Timestamp = 1467494385; - info.RevisionInfo = "r2385905"; - info.TrainingSetComment = "some dummy data"; - info.TrainingSetResId = "sbr://1234"; - auto res = NCodecs::BuildStaticCodec(PrepareData(), info); - UNIT_ASSERT_VALUES_EQUAL(res.ShortUtf8DebugString(), - "StoredCodec: \"\\007\\000huffman@S\\000a" - "\\006b\\005c\\005d\\005e\\005f\\005g\\005h\\005i\\005j\\005k\\005l\\005m\\005n\\005o" - "\\005p\\005q\\005r\\005s\\005t\\005u\\004v\\004w\\004x\\004y\\004z\\004\xC7?\xC8>" - "\xC9=\xCA<\xCB;\xCC:\3159\3168\3177\3206\3215\3224\3233\3242\3251\3260\xD7/\xD8." - "\xD9-\xDA,\xDB+\xDC*\xDD)\xDE(\xDF\\'\xE0&\xE1%\xE2$\xE3#\xE4\\\"\xE5!\xE6 \xE7" - "\\037\xE8\\036\xE9\\035\xEA\\034\xEB\\033\xEC\\032\xED\\031\xEE\\030\xEF\\027\xF0" - "\\026\xF1\\025\xF2\\024\xF3\\023\xF4\\022\xF5\\021\xF6\\020\xF7\\017\xF8\\016\xF9" - "\\r\xFA\\014\xFB\\013\xFC\\n\xFD\\t\xFE\\010\xFF\\007\" " - "DebugInfo { " - "CodecName: \"huffman\" " - "Timestamp: 1467494385 " - "RevisionInfo: \"r2385905\" " - "SampleSizeMultiplier: 2 " - "TrainingSetComment: \"some dummy data\" " - "TrainingSetResId: \"sbr://1234\" " - "StoredCodecHash: 2509195835471488613 " - "}"); - - UNIT_ASSERT_VALUES_EQUAL(NCodecs::GetStandardFileName(res), "huffman.1467494385.codec_info"); - UNIT_ASSERT_VALUES_EQUAL(res.GetDebugInfo().GetStoredCodecHash(), 2509195835471488613ULL); - - auto res1 = NCodecs::LoadCodecInfoFromString(NCodecs::SaveCodecInfoToString(res)); - UNIT_ASSERT_VALUES_EQUAL(res1.ShortUtf8DebugString(), res.ShortUtf8DebugString()); - } -}; - -UNIT_TEST_SUITE_REGISTRATION(TStaticCodecInfoBuilderTest); + NCodecs::TCodecBuildInfo info; + info.CodecName = "huffman"; + info.SampleSizeMultiplier = 2; + info.Timestamp = 1467494385; + info.RevisionInfo = "r2385905"; + info.TrainingSetComment = "some dummy data"; + info.TrainingSetResId = "sbr://1234"; + auto res = NCodecs::BuildStaticCodec(PrepareData(), info); + UNIT_ASSERT_VALUES_EQUAL(res.ShortUtf8DebugString(), + "StoredCodec: \"\\007\\000huffman@S\\000a" + "\\006b\\005c\\005d\\005e\\005f\\005g\\005h\\005i\\005j\\005k\\005l\\005m\\005n\\005o" + "\\005p\\005q\\005r\\005s\\005t\\005u\\004v\\004w\\004x\\004y\\004z\\004\xC7?\xC8>" + "\xC9=\xCA<\xCB;\xCC:\3159\3168\3177\3206\3215\3224\3233\3242\3251\3260\xD7/\xD8." + "\xD9-\xDA,\xDB+\xDC*\xDD)\xDE(\xDF\\'\xE0&\xE1%\xE2$\xE3#\xE4\\\"\xE5!\xE6 \xE7" + "\\037\xE8\\036\xE9\\035\xEA\\034\xEB\\033\xEC\\032\xED\\031\xEE\\030\xEF\\027\xF0" + "\\026\xF1\\025\xF2\\024\xF3\\023\xF4\\022\xF5\\021\xF6\\020\xF7\\017\xF8\\016\xF9" + "\\r\xFA\\014\xFB\\013\xFC\\n\xFD\\t\xFE\\010\xFF\\007\" " + "DebugInfo { " + "CodecName: \"huffman\" " + "Timestamp: 1467494385 " + "RevisionInfo: \"r2385905\" " + "SampleSizeMultiplier: 2 " + "TrainingSetComment: \"some dummy data\" " + "TrainingSetResId: \"sbr://1234\" " + "StoredCodecHash: 2509195835471488613 " + "}"); + + UNIT_ASSERT_VALUES_EQUAL(NCodecs::GetStandardFileName(res), "huffman.1467494385.codec_info"); + UNIT_ASSERT_VALUES_EQUAL(res.GetDebugInfo().GetStoredCodecHash(), 2509195835471488613ULL); + + auto res1 = NCodecs::LoadCodecInfoFromString(NCodecs::SaveCodecInfoToString(res)); + UNIT_ASSERT_VALUES_EQUAL(res1.ShortUtf8DebugString(), res.ShortUtf8DebugString()); + } +}; + +UNIT_TEST_SUITE_REGISTRATION(TStaticCodecInfoBuilderTest); diff --git a/library/cpp/codecs/static/ut/static_ut.cpp b/library/cpp/codecs/static/ut/static_ut.cpp index fda9ffcccb..57e1e62887 100644 --- a/library/cpp/codecs/static/ut/static_ut.cpp +++ b/library/cpp/codecs/static/ut/static_ut.cpp @@ -1,27 +1,27 @@ #include <library/cpp/testing/unittest/registar.h> #include <library/cpp/codecs/static/example/example.h> - + class TStaticCodecUsageTest: public NUnitTest::TTestBase { - UNIT_TEST_SUITE(TStaticCodecUsageTest) + UNIT_TEST_SUITE(TStaticCodecUsageTest) UNIT_TEST(TestUsage) - UNIT_TEST_SUITE_END(); + UNIT_TEST_SUITE_END(); -private: - void DoTestUsage(NStaticCodecExample::EDictVersion dv, size_t expectedSize) { +private: + void DoTestUsage(NStaticCodecExample::EDictVersion dv, size_t expectedSize) { const TStringBuf letov = "Всё идёт по плану"; - - TBuffer outEnc, outDec; - NStaticCodecExample::Encode(outEnc, letov, dv); + + TBuffer outEnc, outDec; + NStaticCodecExample::Encode(outEnc, letov, dv); NStaticCodecExample::Decode(outDec, TStringBuf{outEnc.data(), outEnc.size()}); - - UNIT_ASSERT_VALUES_EQUAL(outEnc.Size(), expectedSize); + + UNIT_ASSERT_VALUES_EQUAL(outEnc.Size(), expectedSize); UNIT_ASSERT_EQUAL(TStringBuf(outDec.data(), outDec.size()), letov); - } - - void TestUsage() { - DoTestUsage(NStaticCodecExample::DV_HUFF_20160707, 18u); - DoTestUsage(NStaticCodecExample::DV_SA_HUFF_20160707, 22u); - } -}; - -UNIT_TEST_SUITE_REGISTRATION(TStaticCodecUsageTest) + } + + void TestUsage() { + DoTestUsage(NStaticCodecExample::DV_HUFF_20160707, 18u); + DoTestUsage(NStaticCodecExample::DV_SA_HUFF_20160707, 22u); + } +}; + +UNIT_TEST_SUITE_REGISTRATION(TStaticCodecUsageTest) diff --git a/library/cpp/codecs/static/ut/ya.make b/library/cpp/codecs/static/ut/ya.make index 5bb2017fac..b9116097d8 100644 --- a/library/cpp/codecs/static/ut/ya.make +++ b/library/cpp/codecs/static/ut/ya.make @@ -1,14 +1,14 @@ UNITTEST_FOR(library/cpp/codecs/static) - -OWNER(velavokr) - -SRCS( - builder_ut.cpp - static_ut.cpp -) - -PEERDIR( + +OWNER(velavokr) + +SRCS( + builder_ut.cpp + static_ut.cpp +) + +PEERDIR( library/cpp/codecs/static/example -) - -END() +) + +END() diff --git a/library/cpp/codecs/static/ya.make b/library/cpp/codecs/static/ya.make index a2698b9432..00e00fd8d4 100644 --- a/library/cpp/codecs/static/ya.make +++ b/library/cpp/codecs/static/ya.make @@ -1,18 +1,18 @@ -LIBRARY() - -OWNER(velavokr) - -SRCS( - builder.cpp - static_codec_info.proto - static.cpp -) - -PEERDIR( +LIBRARY() + +OWNER(velavokr) + +SRCS( + builder.cpp + static_codec_info.proto + static.cpp +) + +PEERDIR( library/cpp/codecs library/cpp/archive library/cpp/svnversion - util/draft -) - -END() + util/draft +) + +END() diff --git a/library/cpp/codecs/tls_cache.cpp b/library/cpp/codecs/tls_cache.cpp index d54339d869..0a1b32bda1 100644 --- a/library/cpp/codecs/tls_cache.cpp +++ b/library/cpp/codecs/tls_cache.cpp @@ -1,4 +1,4 @@ -#include "tls_cache.h" - -namespace NCodecs { -} +#include "tls_cache.h" + +namespace NCodecs { +} diff --git a/library/cpp/codecs/tls_cache.h b/library/cpp/codecs/tls_cache.h index fa166729c5..0184e4bb6c 100644 --- a/library/cpp/codecs/tls_cache.h +++ b/library/cpp/codecs/tls_cache.h @@ -1,100 +1,100 @@ -#pragma once - -#include <util/generic/buffer.h> -#include <util/generic/deque.h> -#include <util/generic/noncopyable.h> -#include <util/generic/strbuf.h> -#include <util/system/tls.h> -#include <util/thread/singleton.h> - -namespace NCodecs { - template <class TItem> - struct TClear { - void operator()(TItem& item) const { - item.Clear(); - } - }; - +#pragma once + +#include <util/generic/buffer.h> +#include <util/generic/deque.h> +#include <util/generic/noncopyable.h> +#include <util/generic/strbuf.h> +#include <util/system/tls.h> +#include <util/thread/singleton.h> + +namespace NCodecs { + template <class TItem> + struct TClear { + void operator()(TItem& item) const { + item.Clear(); + } + }; + template <class TItem, class TCleaner = TClear<TItem>> - class TTlsCache { - using TSelf = TTlsCache<TItem, TCleaner>; - + class TTlsCache { + using TSelf = TTlsCache<TItem, TCleaner>; + struct TItemHolder: public TIntrusiveListItem<TItemHolder> { - TItemHolder(TSelf& factory) - : Factory(factory) + TItemHolder(TSelf& factory) + : Factory(factory) { } - - void Release() { - Factory.Release(*this); - } - - TSelf& Factory; - TItem Item; - }; - - class TItemGuard { - public: - explicit TItemGuard(TSelf& fact) - : Holder(fact.Acquire()) + + void Release() { + Factory.Release(*this); + } + + TSelf& Factory; + TItem Item; + }; + + class TItemGuard { + public: + explicit TItemGuard(TSelf& fact) + : Holder(fact.Acquire()) { } - + TItemGuard(TItemGuard&& other) noexcept { - *this = std::move(other); - } - + *this = std::move(other); + } + TItemGuard& operator=(TItemGuard&& other) noexcept { - if (&other != this) { - std::swap(Holder, other.Holder); - } - return *this; - } - - ~TItemGuard() { - if (Holder) { - Holder->Release(); - } - } - - TItem& Get() & { - Y_ASSERT(Holder); - return Holder->Item; - } - - TItem& Get() && = delete; - - private: - TItemHolder* Holder = nullptr; - }; - - public: - TItemGuard Item() { - return TItemGuard(*this); - } - - static TSelf& TlsInstance() { - return *FastTlsSingleton<TSelf>(); - } - - private: - TItemHolder* Acquire() { - if (Free.Empty()) { - return new TItemHolder(*this); - } else { - return Free.PopBack(); - } - } - - void Release(TItemHolder& item) { - Cleaner(item.Item); - Free.PushBack(&item); - } - - private: - TIntrusiveListWithAutoDelete<TItemHolder, TDelete> Free; - TCleaner Cleaner; - }; - - using TBufferTlsCache = TTlsCache<TBuffer>; -} + if (&other != this) { + std::swap(Holder, other.Holder); + } + return *this; + } + + ~TItemGuard() { + if (Holder) { + Holder->Release(); + } + } + + TItem& Get() & { + Y_ASSERT(Holder); + return Holder->Item; + } + + TItem& Get() && = delete; + + private: + TItemHolder* Holder = nullptr; + }; + + public: + TItemGuard Item() { + return TItemGuard(*this); + } + + static TSelf& TlsInstance() { + return *FastTlsSingleton<TSelf>(); + } + + private: + TItemHolder* Acquire() { + if (Free.Empty()) { + return new TItemHolder(*this); + } else { + return Free.PopBack(); + } + } + + void Release(TItemHolder& item) { + Cleaner(item.Item); + Free.PushBack(&item); + } + + private: + TIntrusiveListWithAutoDelete<TItemHolder, TDelete> Free; + TCleaner Cleaner; + }; + + using TBufferTlsCache = TTlsCache<TBuffer>; +} diff --git a/library/cpp/codecs/ut/codecs_ut.cpp b/library/cpp/codecs/ut/codecs_ut.cpp index 1938202400..caf6089aef 100644 --- a/library/cpp/codecs/ut/codecs_ut.cpp +++ b/library/cpp/codecs/ut/codecs_ut.cpp @@ -4,15 +4,15 @@ #include <library/cpp/codecs/solar_codec.h> #include <library/cpp/codecs/zstd_dict_codec.h> #include <library/cpp/codecs/comptable_codec.h> - + #include <library/cpp/testing/unittest/registar.h> - -#include <util/generic/buffer.h> -#include <util/string/util.h> -#include <util/string/hex.h> + +#include <util/generic/buffer.h> +#include <util/string/util.h> +#include <util/string/hex.h> #include <library/cpp/string_utils/relaxed_escaper/relaxed_escaper.h> - -namespace { + +namespace { const char* TextValues[] = { "! сентября газета", "!(возмездие это)!", @@ -855,328 +855,328 @@ namespace { "lymphomatoid papulosis", "sez.com", }; -} - -class TCodecsTest: public TTestBase { +} + +class TCodecsTest: public TTestBase { UNIT_TEST_SUITE(TCodecsTest); - UNIT_TEST(TestPipeline) - UNIT_TEST(TestDelta) - UNIT_TEST(TestHuffman) - UNIT_TEST(TestZStdDict) - UNIT_TEST(TestCompTable) + UNIT_TEST(TestPipeline) + UNIT_TEST(TestDelta) + UNIT_TEST(TestHuffman) + UNIT_TEST(TestZStdDict) + UNIT_TEST(TestCompTable) UNIT_TEST(TestHuffmanLearnByFreqs) - UNIT_TEST(TestSolar) - UNIT_TEST(TestPFor) - UNIT_TEST(TestRegistry) - + UNIT_TEST(TestSolar) + UNIT_TEST(TestPFor) + UNIT_TEST(TestRegistry) + UNIT_TEST_SUITE_END(); - -private: + +private: TString PrintError(TStringBuf learn, TStringBuf test, TStringBuf codec, ui32 i) { TString s; - TStringOutput sout(s); + TStringOutput sout(s); sout << codec << ": " << i << ", " << "\n"; sout << HexEncode(learn.data(), learn.size()); //NEscJ::EscapeJ<true>(learn, sout); - sout << " != \n"; + sout << " != \n"; sout << HexEncode(test.data(), test.size()); //NEscJ::EscapeJ<true>(test, sout); - - if (s.Size() > 1536) { + + if (s.Size() > 1536) { TString res = s.substr(0, 512); - res.append("...<skipped ").append(ToString(s.size() - 1024)).append(">..."); - res.append(s.substr(s.size() - 512)); - } - - return s; - } - - TStringBuf AsStrBuf(const TBuffer& b) { + res.append("...<skipped ").append(ToString(s.size() - 1024)).append(">..."); + res.append(s.substr(s.size() - 512)); + } + + return s; + } + + TStringBuf AsStrBuf(const TBuffer& b) { return TStringBuf(b.data(), b.size()); - } - - template <typename TCodec, bool testsaveload> + } + + template <typename TCodec, bool testsaveload> void TestCodec(const TVector<TBuffer>& inlearn = TVector<TBuffer>(), const TVector<TBuffer>& in = TVector<TBuffer>(), NCodecs::TCodecPtr c = new TCodec) { - using namespace NCodecs; - - TBuffer buff; - - { + using namespace NCodecs; + + TBuffer buff; + + { TVector<TBuffer> out; - - c->Learn(inlearn.begin(), inlearn.end()); - - if (testsaveload) { - { - TBufferOutput bout(buff); - ICodec::Store(&bout, c); - } - - { - TBufferInput bin(buff); - c = ICodec::Restore(&bin); + + c->Learn(inlearn.begin(), inlearn.end()); + + if (testsaveload) { + { + TBufferOutput bout(buff); + ICodec::Store(&bout, c); + } + + { + TBufferInput bin(buff); + c = ICodec::Restore(&bin); UNIT_ASSERT(c->AlreadyTrained()); - } - } - - { - size_t insz = 0; - size_t outsz = buff.Size(); - - for (ui32 i = 0; i < inlearn.size(); ++i) { + } + } + + { + size_t insz = 0; + size_t outsz = buff.Size(); + + for (ui32 i = 0; i < inlearn.size(); ++i) { out.emplace_back(); - c->Encode(AsStrBuf(inlearn[i]), out[i]); - - insz += inlearn[i].Size(); - outsz += out[i].Size(); - } - - TBuffer vecl; - for (ui32 i = 0; i < out.size(); ++i) { - vecl.Clear(); - c->Decode(AsStrBuf(out[i]), vecl); - - UNIT_ASSERT_EQUAL_C(AsStrBuf(inlearn[i]), AsStrBuf(vecl), + c->Encode(AsStrBuf(inlearn[i]), out[i]); + + insz += inlearn[i].Size(); + outsz += out[i].Size(); + } + + TBuffer vecl; + for (ui32 i = 0; i < out.size(); ++i) { + vecl.Clear(); + c->Decode(AsStrBuf(out[i]), vecl); + + UNIT_ASSERT_EQUAL_C(AsStrBuf(inlearn[i]), AsStrBuf(vecl), PrintError(TStringBuf(inlearn[i].data(), inlearn[i].size()), TStringBuf(vecl.data(), vecl.size()), c->GetName(), i)); - } - } - } - - { - if (testsaveload) { - TBufferInput bin(buff); - c = ICodec::Restore(&bin); - } - - size_t insz = 0; - size_t outsz = buff.Size(); - - TBuffer out, in1; - for (ui32 i = 0; i < in.size(); ++i) { - out.Clear(); - in1.Clear(); - c->Encode(AsStrBuf(in[i]), out); - insz += in[i].Size(); - outsz += out.Size(); - c->Decode(AsStrBuf(out), in1); - UNIT_ASSERT_EQUAL_C(AsStrBuf(in[i]), AsStrBuf(in1), + } + } + } + + { + if (testsaveload) { + TBufferInput bin(buff); + c = ICodec::Restore(&bin); + } + + size_t insz = 0; + size_t outsz = buff.Size(); + + TBuffer out, in1; + for (ui32 i = 0; i < in.size(); ++i) { + out.Clear(); + in1.Clear(); + c->Encode(AsStrBuf(in[i]), out); + insz += in[i].Size(); + outsz += out.Size(); + c->Decode(AsStrBuf(out), in1); + UNIT_ASSERT_EQUAL_C(AsStrBuf(in[i]), AsStrBuf(in1), PrintError(TStringBuf(in[i].data(), in[i].size()), TStringBuf(in1.data(), in1.size()), c->GetName(), i)); - } - } - } - - template <class T> - void AppendTo(TBuffer& b, T t) { - b.Append((char*)&t, sizeof(t)); - } - - void TestDelta() { - using namespace NCodecs; + } + } + } + + template <class T> + void AppendTo(TBuffer& b, T t) { + b.Append((char*)&t, sizeof(t)); + } + + void TestDelta() { + using namespace NCodecs; TVector<TBuffer> d; - - // 1. common case + + // 1. common case d.emplace_back(); - AppendTo(d.back(), 1ULL); - AppendTo(d.back(), 10ULL); - AppendTo(d.back(), 100ULL); - AppendTo(d.back(), 1000ULL); - AppendTo(d.back(), 10000ULL); - AppendTo(d.back(), 100000ULL); - - // 2. delta overflow + AppendTo(d.back(), 1ULL); + AppendTo(d.back(), 10ULL); + AppendTo(d.back(), 100ULL); + AppendTo(d.back(), 1000ULL); + AppendTo(d.back(), 10000ULL); + AppendTo(d.back(), 100000ULL); + + // 2. delta overflow d.emplace_back(); - AppendTo(d.back(), 1ULL); - AppendTo(d.back(), 10ULL); - AppendTo(d.back(), 100ULL); - AppendTo(d.back(), 1000ULL); - AppendTo(d.back(), (ui64)-100LL); - AppendTo(d.back(), (ui64)-10ULL); - - // 3. bad sorting + AppendTo(d.back(), 1ULL); + AppendTo(d.back(), 10ULL); + AppendTo(d.back(), 100ULL); + AppendTo(d.back(), 1000ULL); + AppendTo(d.back(), (ui64)-100LL); + AppendTo(d.back(), (ui64)-10ULL); + + // 3. bad sorting d.emplace_back(); - AppendTo(d.back(), 1ULL); - AppendTo(d.back(), 10ULL); - AppendTo(d.back(), 1000ULL); - AppendTo(d.back(), 100ULL); - AppendTo(d.back(), 10000ULL); - AppendTo(d.back(), 100000ULL); - - // all bad + AppendTo(d.back(), 1ULL); + AppendTo(d.back(), 10ULL); + AppendTo(d.back(), 1000ULL); + AppendTo(d.back(), 100ULL); + AppendTo(d.back(), 10000ULL); + AppendTo(d.back(), 100000ULL); + + // all bad d.emplace_back(); - AppendTo(d.back(), -1LL); - AppendTo(d.back(), -1LL); - AppendTo(d.back(), -1LL); - AppendTo(d.back(), -1LL); - + AppendTo(d.back(), -1LL); + AppendTo(d.back(), -1LL); + AppendTo(d.back(), -1LL); + AppendTo(d.back(), -1LL); + TestCodec<TDeltaCodec<ui64, true>, false>(d); TestCodec<TDeltaCodec<ui64, false>, false>(d); - } - - void TestPFor() { - using namespace NCodecs; - { + } + + void TestPFor() { + using namespace NCodecs; + { TVector<TBuffer> d; d.emplace_back(); - AppendTo(d.back(), -1LL); - AppendTo(d.back(), -1LL); - AppendTo(d.back(), -1LL); - AppendTo(d.back(), -1LL); + AppendTo(d.back(), -1LL); + AppendTo(d.back(), -1LL); + AppendTo(d.back(), -1LL); + AppendTo(d.back(), -1LL); d.emplace_back(); - AppendTo(d.back(), 0LL); - AppendTo(d.back(), 1LL); - AppendTo(d.back(), 2LL); - AppendTo(d.back(), 1LL); - AppendTo(d.back(), 0LL); - AppendTo(d.back(), 1LL); - AppendTo(d.back(), 2LL); + AppendTo(d.back(), 0LL); + AppendTo(d.back(), 1LL); + AppendTo(d.back(), 2LL); + AppendTo(d.back(), 1LL); + AppendTo(d.back(), 0LL); + AppendTo(d.back(), 1LL); + AppendTo(d.back(), 2LL); d.emplace_back(); - AppendTo(d.back(), 0LL); - AppendTo(d.back(), 1LL); - AppendTo(d.back(), 2LL); - AppendTo(d.back(), 1LL); - AppendTo(d.back(), -1LL); - AppendTo(d.back(), 0LL); - AppendTo(d.back(), 1LL); - AppendTo(d.back(), 2LL); + AppendTo(d.back(), 0LL); + AppendTo(d.back(), 1LL); + AppendTo(d.back(), 2LL); + AppendTo(d.back(), 1LL); + AppendTo(d.back(), -1LL); + AppendTo(d.back(), 0LL); + AppendTo(d.back(), 1LL); + AppendTo(d.back(), 2LL); d.emplace_back(); - AppendTo(d.back(), 0LL); - AppendTo(d.back(), -1LL); - AppendTo(d.back(), -2LL); - AppendTo(d.back(), -1LL); - AppendTo(d.back(), -2LL); - AppendTo(d.back(), -1LL); - AppendTo(d.back(), 0LL); - AppendTo(d.back(), -1LL); - AppendTo(d.back(), -2LL); - + AppendTo(d.back(), 0LL); + AppendTo(d.back(), -1LL); + AppendTo(d.back(), -2LL); + AppendTo(d.back(), -1LL); + AppendTo(d.back(), -2LL); + AppendTo(d.back(), -1LL); + AppendTo(d.back(), 0LL); + AppendTo(d.back(), -1LL); + AppendTo(d.back(), -2LL); + TestCodec<TPForCodec<ui64>, false>(d); - TestCodec<TPForCodec<ui64, true>, true>(d); - } - { + TestCodec<TPForCodec<ui64, true>, true>(d); + } + { TVector<TBuffer> d; d.emplace_back(); - AppendTo(d.back(), -1); - AppendTo(d.back(), -1); - AppendTo(d.back(), -1); - AppendTo(d.back(), -1); + AppendTo(d.back(), -1); + AppendTo(d.back(), -1); + AppendTo(d.back(), -1); + AppendTo(d.back(), -1); d.emplace_back(); - AppendTo(d.back(), 0); - AppendTo(d.back(), 1); - AppendTo(d.back(), 2); - AppendTo(d.back(), 1); - AppendTo(d.back(), -1); - AppendTo(d.back(), 0); - AppendTo(d.back(), 1); - AppendTo(d.back(), 2); + AppendTo(d.back(), 0); + AppendTo(d.back(), 1); + AppendTo(d.back(), 2); + AppendTo(d.back(), 1); + AppendTo(d.back(), -1); + AppendTo(d.back(), 0); + AppendTo(d.back(), 1); + AppendTo(d.back(), 2); d.emplace_back(); - AppendTo(d.back(), 0); - AppendTo(d.back(), -1); - AppendTo(d.back(), -2); - AppendTo(d.back(), -1); - AppendTo(d.back(), -2); - AppendTo(d.back(), -1); - AppendTo(d.back(), 0); - AppendTo(d.back(), -1); - AppendTo(d.back(), -2); - + AppendTo(d.back(), 0); + AppendTo(d.back(), -1); + AppendTo(d.back(), -2); + AppendTo(d.back(), -1); + AppendTo(d.back(), -2); + AppendTo(d.back(), -1); + AppendTo(d.back(), 0); + AppendTo(d.back(), -1); + AppendTo(d.back(), -2); + TestCodec<TPForCodec<ui32>, false>(d); - TestCodec<TPForCodec<ui32, true>, false>(d); - } - { + TestCodec<TPForCodec<ui32, true>, false>(d); + } + { TVector<TBuffer> d; d.emplace_back(); - for (auto& textValue : TextValues) { - AppendTo(d.back(), (ui32)strlen(textValue)); - } - - TestCodec<TPForCodec<ui32>, false>(d); - TestCodec<TPForCodec<ui32, true>, false>(d); - } - { + for (auto& textValue : TextValues) { + AppendTo(d.back(), (ui32)strlen(textValue)); + } + + TestCodec<TPForCodec<ui32>, false>(d); + TestCodec<TPForCodec<ui32, true>, false>(d); + } + { TVector<TBuffer> d; d.emplace_back(); - for (auto& textValue : TextValues) { - AppendTo(d.back(), (ui64)strlen(textValue)); - } - - TestCodec<TPForCodec<ui64>, false>(d); - TestCodec<TPForCodec<ui64, true>, false>(d); - } - } - - template <class TCodec> - void DoTestSimpleCodec() { - using namespace NCodecs; - { + for (auto& textValue : TextValues) { + AppendTo(d.back(), (ui64)strlen(textValue)); + } + + TestCodec<TPForCodec<ui64>, false>(d); + TestCodec<TPForCodec<ui64, true>, false>(d); + } + } + + template <class TCodec> + void DoTestSimpleCodec() { + using namespace NCodecs; + { TVector<TBuffer> learn; - + for (auto& textValue : TextValues) { learn.emplace_back(textValue, strlen(textValue)); - } - - TestCodec<TCodec, true>(learn); - } - { - TestCodec<TCodec, true>(); - } - - { + } + + TestCodec<TCodec, true>(learn); + } + { + TestCodec<TCodec, true>(); + } + + { TVector<TBuffer> learn; learn.emplace_back(); - learn.back().Append('a'); - + learn.back().Append('a'); + TVector<TBuffer> test; test.emplace_back(); - for (ui32 i = 0; i < 256; ++i) { - test.back().Append((ui8)i); - } - - TestCodec<TCodec, true>(learn, test); - } - - { + for (ui32 i = 0; i < 256; ++i) { + test.back().Append((ui8)i); + } + + TestCodec<TCodec, true>(learn, test); + } + + { TVector<TBuffer> learn; learn.emplace_back(); - for (ui32 i = 0; i < 256; ++i) { - for (ui32 j = 0; j < i; ++j) { + for (ui32 i = 0; i < 256; ++i) { + for (ui32 j = 0; j < i; ++j) { learn.back().Append((ui8)i); - } - } - + } + } + TVector<TBuffer> test; test.emplace_back(); - for (ui32 i = 0; i < 256; ++i) { - test.back().Append((ui8)i); - } - - TestCodec<TCodec, true>(learn, test); - } - - { + for (ui32 i = 0; i < 256; ++i) { + test.back().Append((ui8)i); + } + + TestCodec<TCodec, true>(learn, test); + } + + { TVector<TBuffer> learn; learn.emplace_back(); - for (ui32 i = 0; i < 128; ++i) { - for (ui32 j = 0; j < i; ++j) { - learn.back().Append((ui8)i); - } - } - + for (ui32 i = 0; i < 128; ++i) { + for (ui32 j = 0; j < i; ++j) { + learn.back().Append((ui8)i); + } + } + TVector<TBuffer> test; test.emplace_back(); - for (ui32 i = 128; i < 256; ++i) { - test.back().Append((ui8)i); - } - - TestCodec<TCodec, true>(learn, test); - } - } - - void TestHuffman() { - DoTestSimpleCodec<NCodecs::THuffmanCodec>(); - } - - void TestZStdDict() { + for (ui32 i = 128; i < 256; ++i) { + test.back().Append((ui8)i); + } + + TestCodec<TCodec, true>(learn, test); + } + } + + void TestHuffman() { + DoTestSimpleCodec<NCodecs::THuffmanCodec>(); + } + + void TestZStdDict() { using namespace NCodecs; { TVector<TBuffer> learn; @@ -1188,12 +1188,12 @@ private: TestCodec<TZStdDictCodec, true>(learn); } - } - - void TestCompTable() { - DoTestSimpleCodec<NCodecs::TCompTableCodec>(); - } - + } + + void TestCompTable() { + DoTestSimpleCodec<NCodecs::TCompTableCodec>(); + } + void TestHuffmanLearnByFreqs() { using namespace NCodecs; @@ -1211,7 +1211,7 @@ private: for (ui32 i = 0; i < data.size(); ++i) { outLearn.emplace_back(); - codec.Encode(AsStrBuf(data[i]), outLearn[i]); + codec.Encode(AsStrBuf(data[i]), outLearn[i]); } } @@ -1228,133 +1228,133 @@ private: for (auto& textValue : TextValues) { size_t len = strlen(textValue); - for (size_t j = 0; j < len; ++j) { + for (size_t j = 0; j < len; ++j) { ++freqs[(ui32)(0xFF & textValue[j])].second; - } + } } codec.LearnByFreqs(TArrayRef<std::pair<char, ui64>>(freqs, Y_ARRAY_SIZE(freqs))); for (ui32 i = 0; i < data.size(); ++i) { outLearnByFreqs.emplace_back(); - codec.Encode(AsStrBuf(data[i]), outLearnByFreqs[i]); + codec.Encode(AsStrBuf(data[i]), outLearnByFreqs[i]); } } - UNIT_ASSERT_EQUAL(outLearn.size(), outLearnByFreqs.size()); - const size_t sz = outLearn.size(); - for (size_t n = 0; n < sz; ++n) { - UNIT_ASSERT_EQUAL(AsStrBuf(outLearn[n]), AsStrBuf(outLearnByFreqs[n])); - } + UNIT_ASSERT_EQUAL(outLearn.size(), outLearnByFreqs.size()); + const size_t sz = outLearn.size(); + for (size_t n = 0; n < sz; ++n) { + UNIT_ASSERT_EQUAL(AsStrBuf(outLearn[n]), AsStrBuf(outLearnByFreqs[n])); + } } - void TestSolar() { - using namespace NCodecs; - { + void TestSolar() { + using namespace NCodecs; + { TVector<TBuffer> learn; - + for (auto& textValue : TextValues) { learn.emplace_back(textValue, strlen(textValue)); - } - + } + TestCodec<TSolarCodec, true>(learn, TVector<TBuffer>(), new TSolarCodec(512, 8)); TestCodec<TAdaptiveSolarCodec, false>(learn, TVector<TBuffer>(), new TAdaptiveSolarCodec(512, 8)); TestCodec<TAdaptiveSolarCodec, true>(learn, TVector<TBuffer>(), new TAdaptiveSolarCodec(512, 8)); TestCodec<TSolarCodecShortInt, true>(learn, TVector<TBuffer>(), new TSolarCodecShortInt(512, 8)); - } - { + } + { TestCodec<TSolarCodec, true>(TVector<TBuffer>(), TVector<TBuffer>(), new TSolarCodec(512, 8)); TestCodec<TAdaptiveSolarCodec, false>(TVector<TBuffer>(), TVector<TBuffer>(), new TAdaptiveSolarCodec(512, 8)); TestCodec<TAdaptiveSolarCodec, true>(TVector<TBuffer>(), TVector<TBuffer>(), new TAdaptiveSolarCodec(512, 8)); TestCodec<TSolarCodecShortInt, true>(TVector<TBuffer>(), TVector<TBuffer>(), new TSolarCodecShortInt(512, 8)); - } - - { + } + + { TVector<TBuffer> learn; learn.emplace_back(); - learn.back().Append('a'); - + learn.back().Append('a'); + TVector<TBuffer> test; test.emplace_back(); - for (ui32 i = 0; i < 256; ++i) { - test.back().Append((ui8)i); - } - - TestCodec<TSolarCodec, true>(learn, test, new TSolarCodec(512, 8)); + for (ui32 i = 0; i < 256; ++i) { + test.back().Append((ui8)i); + } + + TestCodec<TSolarCodec, true>(learn, test, new TSolarCodec(512, 8)); TestCodec<TAdaptiveSolarCodec, false>(learn, test, new TAdaptiveSolarCodec(512, 8)); TestCodec<TAdaptiveSolarCodec, true>(learn, test, new TAdaptiveSolarCodec(512, 8)); TestCodec<TSolarCodecShortInt, true>(learn, test, new TSolarCodecShortInt(512, 8)); - } - - { + } + + { TVector<TBuffer> learn; learn.emplace_back(); - for (ui32 i = 0; i < 256; ++i) { - for (ui32 j = 0; j < i; ++j) { - learn.back().Append((ui8)i); - } - } - + for (ui32 i = 0; i < 256; ++i) { + for (ui32 j = 0; j < i; ++j) { + learn.back().Append((ui8)i); + } + } + TVector<TBuffer> test; test.emplace_back(); - for (ui32 i = 0; i < 256; ++i) { - test.back().Append((ui8)i); - } - - TestCodec<TSolarCodec, true>(learn, test, new TSolarCodec(512, 8)); + for (ui32 i = 0; i < 256; ++i) { + test.back().Append((ui8)i); + } + + TestCodec<TSolarCodec, true>(learn, test, new TSolarCodec(512, 8)); TestCodec<TAdaptiveSolarCodec, false>(learn, test, new TAdaptiveSolarCodec(512, 8)); TestCodec<TAdaptiveSolarCodec, true>(learn, test, new TAdaptiveSolarCodec(512, 8)); TestCodec<TSolarCodecShortInt, true>(learn, test, new TSolarCodecShortInt(512, 8)); - } - } - - void TestPipeline() { - using namespace NCodecs; - { + } + } + + void TestPipeline() { + using namespace NCodecs; + { TVector<TBuffer> learn; learn.emplace_back(); - for (ui32 i = 0; i < 256; ++i) { - for (i32 j = i; j >= 0; --j) { - learn.back().Append((ui8)j); - } - } - + for (ui32 i = 0; i < 256; ++i) { + for (i32 j = i; j >= 0; --j) { + learn.back().Append((ui8)j); + } + } + TVector<TBuffer> test; test.emplace_back(); - for (ui32 i = 0; i < 256; ++i) { - test.back().Append((ui8)i); - } - - TestCodec<TPipelineCodec, true>(learn, test, + for (ui32 i = 0; i < 256; ++i) { + test.back().Append((ui8)i); + } + + TestCodec<TPipelineCodec, true>(learn, test, new TPipelineCodec(new TSolarCodec(512, 8), new TSolarCodec(512, 8), new THuffmanCodec)); - } - { + } + { TVector<TBuffer> d; d.emplace_back(); - for (ui32 i = 0; i < 256; ++i) { - for (i32 j = i; j >= 0; --j) { - d.back().Append(i * i); - } - } - + for (ui32 i = 0; i < 256; ++i) { + for (i32 j = i; j >= 0; --j) { + d.back().Append(i * i); + } + } + TestCodec<TPipelineCodec, false>(d, TVector<TBuffer>(), new TPipelineCodec(new TDeltaCodec<ui32, false>, new TPForCodec<ui32>)); - } - } - - void TestRegistry() { - using namespace NCodecs; + } + } + + void TestRegistry() { + using namespace NCodecs; TVector<TString> vs = ICodec::GetCodecsList(); for (const auto& v : vs) { TCodecPtr p = ICodec::GetInstance(v); if (v == "none") { - UNIT_ASSERT(!p); - continue; - } + UNIT_ASSERT(!p); + continue; + } UNIT_ASSERT_C(!!p, v); UNIT_ASSERT_C(TStringBuf(v).Head(3) == TStringBuf(p->GetName()).Head(3), v + " " + p->GetName()); - } - } -}; - -UNIT_TEST_SUITE_REGISTRATION(TCodecsTest) + } + } +}; + +UNIT_TEST_SUITE_REGISTRATION(TCodecsTest) diff --git a/library/cpp/codecs/ut/tls_cache_ut.cpp b/library/cpp/codecs/ut/tls_cache_ut.cpp index 11dd5da53c..8101af761f 100644 --- a/library/cpp/codecs/ut/tls_cache_ut.cpp +++ b/library/cpp/codecs/ut/tls_cache_ut.cpp @@ -1,15 +1,15 @@ #include <library/cpp/testing/unittest/registar.h> #include <library/cpp/codecs/tls_cache.h> - + Y_UNIT_TEST_SUITE(CodecsBufferFactoryTest){ void AssignToBuffer(TBuffer & buf, TStringBuf val){ buf.Assign(val.data(), val.size()); } - + TStringBuf AsStringBuf(const TBuffer& b) { return TStringBuf(b.Data(), b.Size()); } - + Y_UNIT_TEST(TestAcquireReleaseReuse) { NCodecs::TBufferTlsCache factory; // acquiring the first buffer @@ -19,7 +19,7 @@ Y_UNIT_TEST(TestAcquireReleaseReuse) { // acquiring the second buffer auto buf2 = factory.Item(); AssignToBuffer(buf2.Get(), "Buffer_02"); - } + } // the first buffer should stay intact UNIT_ASSERT_EQUAL(AsStringBuf(buf1.Get()), "Buffer_01"); { diff --git a/library/cpp/codecs/ut/ya.make b/library/cpp/codecs/ut/ya.make index 0b53eba9e5..90841b05ef 100644 --- a/library/cpp/codecs/ut/ya.make +++ b/library/cpp/codecs/ut/ya.make @@ -12,7 +12,7 @@ PEERDIR( ) SRCS( - tls_cache_ut.cpp + tls_cache_ut.cpp codecs_ut.cpp float_huffman_ut.cpp ) diff --git a/library/cpp/codecs/ya.make b/library/cpp/codecs/ya.make index d105d6925e..7e76fb0c9a 100644 --- a/library/cpp/codecs/ya.make +++ b/library/cpp/codecs/ya.make @@ -1,24 +1,24 @@ -LIBRARY() - +LIBRARY() + OWNER( g:base velavokr ) -SRCS( - tls_cache.cpp - codecs.cpp - codecs_registry.cpp - comptable_codec.cpp - delta_codec.cpp +SRCS( + tls_cache.cpp + codecs.cpp + codecs_registry.cpp + comptable_codec.cpp + delta_codec.cpp float_huffman.cpp - huffman_codec.cpp - pfor_codec.cpp - solar_codec.cpp - zstd_dict_codec.cpp -) - -PEERDIR( + huffman_codec.cpp + pfor_codec.cpp + solar_codec.cpp + zstd_dict_codec.cpp +) + +PEERDIR( contrib/libs/zstd library/cpp/bit_io library/cpp/blockcodecs @@ -28,6 +28,6 @@ PEERDIR( library/cpp/deprecated/accessors library/cpp/packers library/cpp/string_utils/relaxed_escaper -) - +) + END() diff --git a/library/cpp/codecs/zstd_dict_codec.cpp b/library/cpp/codecs/zstd_dict_codec.cpp index d543736b3d..c42a2879e6 100644 --- a/library/cpp/codecs/zstd_dict_codec.cpp +++ b/library/cpp/codecs/zstd_dict_codec.cpp @@ -1,173 +1,173 @@ -#include "zstd_dict_codec.h" - +#include "zstd_dict_codec.h" + #include <library/cpp/packers/packers.h> - -#include <util/generic/ptr.h> -#include <util/generic/refcount.h> -#include <util/generic/noncopyable.h> -#include <util/string/builder.h> -#include <util/system/src_location.h> -#include <util/ysaveload.h> - -#define ZDICT_STATIC_LINKING_ONLY - + +#include <util/generic/ptr.h> +#include <util/generic/refcount.h> +#include <util/generic/noncopyable.h> +#include <util/string/builder.h> +#include <util/system/src_location.h> +#include <util/ysaveload.h> + +#define ZDICT_STATIC_LINKING_ONLY + #include <contrib/libs/zstd/include/zdict.h> #include <contrib/libs/zstd/include/zstd.h> #include <contrib/libs/zstd/include/zstd_errors.h> - -// See IGNIETFERRO-320 for possible bugs - -namespace NCodecs { - class TZStdDictCodec::TImpl: public TAtomicRefCount<TZStdDictCodec::TImpl> { - template <class T, size_t Deleter(T*)> - class TPtrHolder : TMoveOnly { - T* Ptr = nullptr; - - public: - TPtrHolder() = default; - - TPtrHolder(T* dict) - : Ptr(dict) + +// See IGNIETFERRO-320 for possible bugs + +namespace NCodecs { + class TZStdDictCodec::TImpl: public TAtomicRefCount<TZStdDictCodec::TImpl> { + template <class T, size_t Deleter(T*)> + class TPtrHolder : TMoveOnly { + T* Ptr = nullptr; + + public: + TPtrHolder() = default; + + TPtrHolder(T* dict) + : Ptr(dict) { } - - T* Get() { - return Ptr; - } - - const T* Get() const { - return Ptr; - } - - void Reset(T* dict) { - Dispose(); - Ptr = dict; - } - - void Dispose() { - if (Ptr) { - Deleter(Ptr); - Ptr = nullptr; - } - } - - ~TPtrHolder() { - Dispose(); - } - }; - - using TCDict = TPtrHolder<ZSTD_CDict, ZSTD_freeCDict>; - using TDDict = TPtrHolder<ZSTD_DDict, ZSTD_freeDDict>; - using TCCtx = TPtrHolder<ZSTD_CCtx, ZSTD_freeCCtx>; - using TDCtx = TPtrHolder<ZSTD_DCtx, ZSTD_freeDCtx>; - - using TSizePacker = NPackers::TPacker<ui64>; - - public: - static const ui32 SampleSize = (1 << 22) * 5; - - explicit TImpl(ui32 comprLevel) - : CompressionLevel(comprLevel) - { - const size_t zeroSz = TSizePacker().MeasureLeaf(0); - Zero.Resize(zeroSz); + + T* Get() { + return Ptr; + } + + const T* Get() const { + return Ptr; + } + + void Reset(T* dict) { + Dispose(); + Ptr = dict; + } + + void Dispose() { + if (Ptr) { + Deleter(Ptr); + Ptr = nullptr; + } + } + + ~TPtrHolder() { + Dispose(); + } + }; + + using TCDict = TPtrHolder<ZSTD_CDict, ZSTD_freeCDict>; + using TDDict = TPtrHolder<ZSTD_DDict, ZSTD_freeDDict>; + using TCCtx = TPtrHolder<ZSTD_CCtx, ZSTD_freeCCtx>; + using TDCtx = TPtrHolder<ZSTD_DCtx, ZSTD_freeDCtx>; + + using TSizePacker = NPackers::TPacker<ui64>; + + public: + static const ui32 SampleSize = (1 << 22) * 5; + + explicit TImpl(ui32 comprLevel) + : CompressionLevel(comprLevel) + { + const size_t zeroSz = TSizePacker().MeasureLeaf(0); + Zero.Resize(zeroSz); TSizePacker().PackLeaf(Zero.data(), 0, zeroSz); - } - - ui32 GetCompressionLevel() const { - return CompressionLevel; - } - - ui8 Encode(TStringBuf in, TBuffer& outbuf) const { - outbuf.Clear(); - + } + + ui32 GetCompressionLevel() const { + return CompressionLevel; + } + + ui8 Encode(TStringBuf in, TBuffer& outbuf) const { + outbuf.Clear(); + if (in.empty()) { - return 0; - } - - TSizePacker packer; - + return 0; + } + + TSizePacker packer; + const char* rawBeg = in.data(); const size_t rawSz = in.size(); - - const size_t szSz = packer.MeasureLeaf(rawSz); - const size_t maxDatSz = ZSTD_compressBound(rawSz); - - outbuf.Resize(szSz + maxDatSz); + + const size_t szSz = packer.MeasureLeaf(rawSz); + const size_t maxDatSz = ZSTD_compressBound(rawSz); + + outbuf.Resize(szSz + maxDatSz); packer.PackLeaf(outbuf.data(), rawSz, szSz); - - TCCtx ctx{CheckPtr(ZSTD_createCCtx(), __LOCATION__)}; - const size_t resSz = CheckSize(ZSTD_compress_usingCDict( + + TCCtx ctx{CheckPtr(ZSTD_createCCtx(), __LOCATION__)}; + const size_t resSz = CheckSize(ZSTD_compress_usingCDict( ctx.Get(), outbuf.data() + szSz, maxDatSz, rawBeg, rawSz, CDict.Get()), __LOCATION__); - - if (resSz < rawSz) { - outbuf.Resize(resSz + szSz); - } else { + + if (resSz < rawSz) { + outbuf.Resize(resSz + szSz); + } else { outbuf.Resize(Zero.size() + rawSz); memcpy(outbuf.data(), Zero.data(), Zero.size()); memcpy(outbuf.data() + Zero.size(), rawBeg, rawSz); - } - return 0; - } - - void Decode(TStringBuf in, TBuffer& outbuf) const { - outbuf.Clear(); - + } + return 0; + } + + void Decode(TStringBuf in, TBuffer& outbuf) const { + outbuf.Clear(); + if (in.empty()) { - return; - } - - TSizePacker packer; - + return; + } + + TSizePacker packer; + const char* rawBeg = in.data(); size_t rawSz = in.size(); - - const size_t szSz = packer.SkipLeaf(rawBeg); - ui64 datSz = 0; - packer.UnpackLeaf(rawBeg, datSz); - - rawBeg += szSz; - rawSz -= szSz; - - if (!datSz) { - outbuf.Resize(rawSz); + + const size_t szSz = packer.SkipLeaf(rawBeg); + ui64 datSz = 0; + packer.UnpackLeaf(rawBeg, datSz); + + rawBeg += szSz; + rawSz -= szSz; + + if (!datSz) { + outbuf.Resize(rawSz); memcpy(outbuf.data(), rawBeg, rawSz); - } else { + } else { // size_t zSz = ZSTD_getDecompressedSize(rawBeg, rawSz); // Y_ENSURE_EX(datSz == zSz, TCodecException() << datSz << " != " << zSz); - outbuf.Resize(datSz); - TDCtx ctx{CheckPtr(ZSTD_createDCtx(), __LOCATION__)}; - CheckSize(ZSTD_decompress_usingDDict( + outbuf.Resize(datSz); + TDCtx ctx{CheckPtr(ZSTD_createDCtx(), __LOCATION__)}; + CheckSize(ZSTD_decompress_usingDDict( ctx.Get(), outbuf.data(), outbuf.size(), rawBeg, rawSz, DDict.Get()), __LOCATION__); - outbuf.Resize(datSz); - } - } - + outbuf.Resize(datSz); + } + } + bool Learn(ISequenceReader& in, bool throwOnError) { - TBuffer data; + TBuffer data; TVector<size_t> lens; - - data.Reserve(2 * SampleSize); - TStringBuf r; - while (in.NextRegion(r)) { - if (!r) { - continue; - } + + data.Reserve(2 * SampleSize); + TStringBuf r; + while (in.NextRegion(r)) { + if (!r) { + continue; + } data.Append(r.data(), r.size()); lens.push_back(r.size()); - } - + } + ZDICT_legacy_params_t params; - memset(¶ms, 0, sizeof(params)); + memset(¶ms, 0, sizeof(params)); params.zParams.compressionLevel = 1; params.zParams.notificationLevel = 1; - Dict.Resize(Max<size_t>(1 << 20, data.Size() + 16 * lens.size())); - - if (!lens) { - Dict.Reset(); - } else { + Dict.Resize(Max<size_t>(1 << 20, data.Size() + 16 * lens.size())); + + if (!lens) { + Dict.Reset(); + } else { size_t trainResult = ZDICT_trainFromBuffer_legacy( Dict.data(), Dict.size(), data.Data(), const_cast<const size_t*>(&lens[0]), lens.size(), params); if (ZSTD_isError(trainResult)) { @@ -177,105 +177,105 @@ namespace NCodecs { CheckSize(trainResult, __LOCATION__); } Dict.Resize(trainResult); - Dict.ShrinkToFit(); - } - InitContexts(); + Dict.ShrinkToFit(); + } + InitContexts(); return true; - } - + } + void Save(IOutputStream* out) const { - ::Save(out, Dict); - } - + ::Save(out, Dict); + } + void Load(IInputStream* in) { - ::Load(in, Dict); - InitContexts(); - } - - void InitContexts() { + ::Load(in, Dict); + InitContexts(); + } + + void InitContexts() { CDict.Reset(CheckPtr(ZSTD_createCDict(Dict.data(), Dict.size(), CompressionLevel), __LOCATION__)); DDict.Reset(CheckPtr(ZSTD_createDDict(Dict.data(), Dict.size()), __LOCATION__)); - } - - static size_t CheckSize(size_t sz, TSourceLocation loc) { - if (ZSTD_isError(sz)) { - ythrow TCodecException() << loc << " " << ZSTD_getErrorName(sz) << " (code " << (int)ZSTD_getErrorCode(sz) << ")"; - } - return sz; - } - - template <class T> - static T* CheckPtr(T* t, TSourceLocation loc) { + } + + static size_t CheckSize(size_t sz, TSourceLocation loc) { + if (ZSTD_isError(sz)) { + ythrow TCodecException() << loc << " " << ZSTD_getErrorName(sz) << " (code " << (int)ZSTD_getErrorCode(sz) << ")"; + } + return sz; + } + + template <class T> + static T* CheckPtr(T* t, TSourceLocation loc) { Y_ENSURE_EX(t, TCodecException() << loc << " " << "unexpected nullptr"); - return t; - } - - private: - ui32 CompressionLevel = 1; - - TBuffer Zero; - TBuffer Dict; - - TCDict CDict; - TDDict DDict; - }; - - TZStdDictCodec::TZStdDictCodec(ui32 comprLevel) - : Impl(new TImpl(comprLevel)) - { - MyTraits.NeedsTraining = true; - MyTraits.SizeOnEncodeMultiplier = 2; - MyTraits.SizeOnDecodeMultiplier = 10; - MyTraits.RecommendedSampleSize = TImpl::SampleSize; // same as for solar - } - + return t; + } + + private: + ui32 CompressionLevel = 1; + + TBuffer Zero; + TBuffer Dict; + + TCDict CDict; + TDDict DDict; + }; + + TZStdDictCodec::TZStdDictCodec(ui32 comprLevel) + : Impl(new TImpl(comprLevel)) + { + MyTraits.NeedsTraining = true; + MyTraits.SizeOnEncodeMultiplier = 2; + MyTraits.SizeOnDecodeMultiplier = 10; + MyTraits.RecommendedSampleSize = TImpl::SampleSize; // same as for solar + } + TZStdDictCodec::~TZStdDictCodec() { } - + TString TZStdDictCodec::GetName() const { return TStringBuilder() << MyName() << "-" << Impl->GetCompressionLevel(); - } - - ui8 TZStdDictCodec::Encode(TStringBuf in, TBuffer& out) const { - return Impl->Encode(in, out); - } - - void TZStdDictCodec::Decode(TStringBuf in, TBuffer& out) const { - Impl->Decode(in, out); - } - - void TZStdDictCodec::DoLearn(ISequenceReader& in) { - Impl = new TImpl(Impl->GetCompressionLevel()); + } + + ui8 TZStdDictCodec::Encode(TStringBuf in, TBuffer& out) const { + return Impl->Encode(in, out); + } + + void TZStdDictCodec::Decode(TStringBuf in, TBuffer& out) const { + Impl->Decode(in, out); + } + + void TZStdDictCodec::DoLearn(ISequenceReader& in) { + Impl = new TImpl(Impl->GetCompressionLevel()); Impl->Learn(in, true/*throwOnError*/); - } - + } + bool TZStdDictCodec::DoTryToLearn(ISequenceReader& in) { Impl = new TImpl(Impl->GetCompressionLevel()); return Impl->Learn(in, false/*throwOnError*/); } void TZStdDictCodec::Save(IOutputStream* out) const { - Impl->Save(out); - } - + Impl->Save(out); + } + void TZStdDictCodec::Load(IInputStream* in) { - Impl->Load(in); - } - + Impl->Load(in); + } + TVector<TString> TZStdDictCodec::ListCompressionNames() { TVector<TString> res; - for (int i = 1; i <= ZSTD_maxCLevel(); ++i) { - res.emplace_back(TStringBuilder() << MyName() << "-" << i); - } - return res; - } - - int TZStdDictCodec::ParseCompressionName(TStringBuf name) { - int c = 0; - TryFromString(name.After('-'), c); - Y_ENSURE_EX(name.Before('-') == MyName() && c > 0 && c <= ZSTD_maxCLevel(), TCodecException() << "invald codec name" << name); - return c; - } - -} + for (int i = 1; i <= ZSTD_maxCLevel(); ++i) { + res.emplace_back(TStringBuilder() << MyName() << "-" << i); + } + return res; + } + + int TZStdDictCodec::ParseCompressionName(TStringBuf name) { + int c = 0; + TryFromString(name.After('-'), c); + Y_ENSURE_EX(name.Before('-') == MyName() && c > 0 && c <= ZSTD_maxCLevel(), TCodecException() << "invald codec name" << name); + return c; + } + +} diff --git a/library/cpp/codecs/zstd_dict_codec.h b/library/cpp/codecs/zstd_dict_codec.h index cdfc5c8285..59c1ad6c60 100644 --- a/library/cpp/codecs/zstd_dict_codec.h +++ b/library/cpp/codecs/zstd_dict_codec.h @@ -1,38 +1,38 @@ -#pragma once - -#include "codecs.h" - -#include <util/generic/ptr.h> - -namespace NCodecs { +#pragma once + +#include "codecs.h" + +#include <util/generic/ptr.h> + +namespace NCodecs { // benchmarks are here: https://st.yandex-team.ru/SEARCH-1655 - + class TZStdDictCodec: public ICodec { class TImpl; TIntrusivePtr<TImpl> Impl; - + public: explicit TZStdDictCodec(ui32 comprLevel = 1); ~TZStdDictCodec() override; - + static TStringBuf MyName() { return "zstd08d"; } - + TString GetName() const override; - + ui8 Encode(TStringBuf in, TBuffer& out) const override; - + void Decode(TStringBuf in, TBuffer& out) const override; - + static TVector<TString> ListCompressionNames(); static int ParseCompressionName(TStringBuf); - + protected: void DoLearn(ISequenceReader& in) override; bool DoTryToLearn(ISequenceReader& in) final; void Save(IOutputStream* out) const override; void Load(IInputStream* in) override; }; - -} + +} diff --git a/library/cpp/containers/atomizer/atomizer.h b/library/cpp/containers/atomizer/atomizer.h index f3dbbad788..5e40f47ab9 100644 --- a/library/cpp/containers/atomizer/atomizer.h +++ b/library/cpp/containers/atomizer/atomizer.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <library/cpp/containers/str_map/str_map.h> diff --git a/library/cpp/containers/compact_vector/compact_vector.h b/library/cpp/containers/compact_vector/compact_vector.h index e7f4bfc54d..dbe7473f0c 100644 --- a/library/cpp/containers/compact_vector/compact_vector.h +++ b/library/cpp/containers/compact_vector/compact_vector.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <util/generic/yexception.h> #include <util/generic/utility.h> diff --git a/library/cpp/containers/comptrie/array_with_size.h b/library/cpp/containers/comptrie/array_with_size.h index 267692b8bc..36e61c7410 100644 --- a/library/cpp/containers/comptrie/array_with_size.h +++ b/library/cpp/containers/comptrie/array_with_size.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <util/generic/ptr.h> #include <util/generic/noncopyable.h> diff --git a/library/cpp/containers/comptrie/comptrie.h b/library/cpp/containers/comptrie/comptrie.h index b9a4c65280..f77024327e 100644 --- a/library/cpp/containers/comptrie/comptrie.h +++ b/library/cpp/containers/comptrie/comptrie.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once -#include "comptrie_trie.h" -#include "comptrie_builder.h" +#include "comptrie_trie.h" +#include "comptrie_builder.h" diff --git a/library/cpp/containers/comptrie/comptrie_builder.h b/library/cpp/containers/comptrie/comptrie_builder.h index 8254c4f592..cf7d2e39a3 100644 --- a/library/cpp/containers/comptrie/comptrie_builder.h +++ b/library/cpp/containers/comptrie/comptrie_builder.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include "comptrie_packer.h" #include "minimize.h" @@ -17,7 +17,7 @@ // then all the keys that we add between these two also have the same prefix. // Actually in this mode the builder can accept even more freely ordered input, // but for input as above it is guaranteed to work. -enum ECompactTrieBuilderFlags { +enum ECompactTrieBuilderFlags { CTBF_NONE = 0, CTBF_PREFIX_GROUPED = 1 << 0, CTBF_VERBOSE = 1 << 1, @@ -25,7 +25,7 @@ enum ECompactTrieBuilderFlags { }; using TCompactTrieBuilderFlags = ECompactTrieBuilderFlags; - + inline TCompactTrieBuilderFlags operator|(TCompactTrieBuilderFlags first, TCompactTrieBuilderFlags second) { return static_cast<TCompactTrieBuilderFlags>(static_cast<int>(first) | second); } @@ -51,13 +51,13 @@ public: // All Add.. methods return true if it was a new key, false if the key already existed. bool Add(const TSymbol* key, size_t keylen, const TData& value); - bool Add(const TKeyBuf& key, const TData& value) { + bool Add(const TKeyBuf& key, const TData& value) { return Add(key.data(), key.size(), value); } // add already serialized data bool AddPtr(const TSymbol* key, size_t keylen, const char* data); - bool AddPtr(const TKeyBuf& key, const char* data) { + bool AddPtr(const TKeyBuf& key, const char* data) { return AddPtr(key.data(), key.size(), data); } @@ -88,8 +88,8 @@ public: return Save(out); } - void Clear(); // Returns all memory to the system and resets the builder state. - + void Clear(); // Returns all memory to the system and resets the builder state. + size_t GetEntryCount() const; size_t GetNodeCount() const; @@ -98,7 +98,7 @@ public: return Impl->MeasureByteSize(); } -protected: +protected: class TCompactTrieBuilderImpl; THolder<TCompactTrieBuilderImpl> Impl; }; @@ -133,12 +133,12 @@ size_t CompactTrieMinimize(IOutputStream& os, const TTrieBuilder& builder, bool // by Maxim Babenko and Ivan Puzyrevsky. The difference is that when we cut the tree into levels // two nodes connected by a forward link are put into the same level (because they usually lie near each other in the original tree). // The original paper (describing the layout in Section 2.1) is: -// Michael A. Bender, Erik D. Demaine, Martin Farach-Colton. Cache-Oblivious B-Trees -// SIAM Journal on Computing, volume 35, number 2, 2005, pages 341-358. +// Michael A. Bender, Erik D. Demaine, Martin Farach-Colton. Cache-Oblivious B-Trees +// SIAM Journal on Computing, volume 35, number 2, 2005, pages 341-358. // Available on the web: http://erikdemaine.org/papers/CacheObliviousBTrees_SICOMP/ -// Or: Michael A. Bender, Erik D. Demaine, and Martin Farach-Colton. Cache-Oblivious B-Trees -// Proceedings of the 41st Annual Symposium -// on Foundations of Computer Science (FOCS 2000), Redondo Beach, California, November 12-14, 2000, pages 399-409. +// Or: Michael A. Bender, Erik D. Demaine, and Martin Farach-Colton. Cache-Oblivious B-Trees +// Proceedings of the 41st Annual Symposium +// on Foundations of Computer Science (FOCS 2000), Redondo Beach, California, November 12-14, 2000, pages 399-409. // Available on the web: http://erikdemaine.org/papers/FOCS2000b/ // (there is not much difference between these papers, actually). // diff --git a/library/cpp/containers/comptrie/comptrie_builder.inl b/library/cpp/containers/comptrie/comptrie_builder.inl index ea151b946e..f273fa6571 100644 --- a/library/cpp/containers/comptrie/comptrie_builder.inl +++ b/library/cpp/containers/comptrie/comptrie_builder.inl @@ -1,6 +1,6 @@ -#pragma once +#pragma once -#include "comptrie_impl.h" +#include "comptrie_impl.h" #include "comptrie_trie.h" #include "make_fast_layout.h" #include "array_with_size.h" @@ -26,7 +26,7 @@ template <class T, class D, class S> class TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl { -protected: +protected: TMemoryPool Pool; size_t PayloadSize; THolder<TFixedSizeAllocator> NodeAllocator; @@ -45,7 +45,7 @@ protected: DATA_IN_MEMPOOL, }; -protected: +protected: void ConvertSymbolArrayToChar(const TSymbol* key, size_t keylen, TTempBuf& buf, size_t ckeylen) const; void NodeLinkTo(TNode* thiz, const TBlob& label, TNode* node); TNode* NodeForwardAdd(TNode* thiz, const char* label, size_t len, size_t& passed, size_t* nodeCount); @@ -335,10 +335,10 @@ public: union { char ArcsData[CONSTEXPR_MAX3(sizeof(TArcSet), sizeof(TBufferedSubtree), sizeof(TSubtreeInFile))]; - union { - void* Data1; - long long int Data2; - } Aligner; + union { + void* Data1; + long long int Data2; + } Aligner; }; inline ISubtree* Subtree() { @@ -451,7 +451,7 @@ bool TCompactTrieBuilder<T, D, S>::Find(const TSymbol* key, size_t keylen, TData } template <class T, class D, class S> -bool TCompactTrieBuilder<T, D, S>::FindLongestPrefix( +bool TCompactTrieBuilder<T, D, S>::FindLongestPrefix( const TSymbol* key, size_t keylen, size_t* prefixlen, TData* value) const { return Impl->FindLongestPrefix(key, keylen, prefixlen, value); } @@ -502,8 +502,8 @@ TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::~TCompactTrieBuilderImpl( } template <class T, class D, class S> -void TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::ConvertSymbolArrayToChar( - const TSymbol* key, size_t keylen, TTempBuf& buf, size_t buflen) const { +void TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::ConvertSymbolArrayToChar( + const TSymbol* key, size_t keylen, TTempBuf& buf, size_t buflen) const { char* ckeyptr = buf.Data(); for (size_t i = 0; i < keylen; ++i) { @@ -544,7 +544,7 @@ void TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::NodeReleasePayload(T template <class T, class D, class S> bool TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::AddEntry( - const TSymbol* key, size_t keylen, const TData& value) { + const TSymbol* key, size_t keylen, const TData& value) { size_t datalen = Packer.MeasureLeaf(value); bool isNewAddition = false; @@ -555,7 +555,7 @@ bool TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::AddEntry( template <class T, class D, class S> bool TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::AddEntryPtr( - const TSymbol* key, size_t keylen, const char* value) { + const TSymbol* key, size_t keylen, const char* value) { size_t datalen = Packer.SkipLeaf(value); bool isNewAddition = false; @@ -597,8 +597,8 @@ bool TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::AddSubtreeInBuffer( } template <class T, class D, class S> -typename TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::TNode* - TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::AddEntryForSomething( +typename TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::TNode* + TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::AddEntryForSomething( const TSymbol* key, size_t keylen, bool& isNewAddition) { using namespace NCompactTrie; @@ -684,7 +684,7 @@ bool TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::FindEntryImpl(const } template <class T, class D, class S> -bool TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::FindLongestPrefix( +bool TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::FindLongestPrefix( const TSymbol* key, size_t keylen, size_t* prefixlen, TData* value) const { using namespace NCompactTrie; @@ -778,8 +778,8 @@ size_t TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::GetNodeCount() con } template <class T, class D, class S> -typename TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::TNode* - TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::NodeForwardAdd( +typename TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::TNode* + TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::NodeForwardAdd( TNode* thiz, const char* label, size_t len, size_t& passed, size_t* nodeCount) { typename TNode::TArcSet* arcSet = dynamic_cast<typename TNode::TArcSet*>(thiz->Subtree()); if (!arcSet) @@ -892,8 +892,8 @@ TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::TArc::TArc(const TBlob& l {} template <class T, class D, class S> -ui64 TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::ArcMeasure( - const TArc* thiz, size_t leftsize, size_t rightsize) const { +ui64 TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::ArcMeasure( + const TArc* thiz, size_t leftsize, size_t rightsize) const { using namespace NCompactTrie; size_t coresize = 2 + NodeMeasureLeafValue(thiz->Node); // 2 == (char + flags) @@ -978,8 +978,8 @@ ui64 TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::ArcSaveAndDestroy(co // TCompactTrieBuilder::TCompactTrieBuilderImpl::TNode::TArcSet template <class T, class D, class S> -typename TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::TNode::TArcSet::iterator - TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::TNode::TArcSet::Find(char ch) { +typename TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::TNode::TArcSet::iterator + TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::TNode::TArcSet::Find(char ch) { using namespace NCompTriePrivate; iterator it = LowerBound(this->begin(), this->end(), ch, TCmp()); @@ -1083,12 +1083,12 @@ size_t CompactTrieMinimize(IOutputStream& os, const TTrieBuilder& builder, bool // by Maxim Babenko and Ivan Puzyrevsky. The difference is that when we cut the tree into levels // two nodes connected by a forward link are put into the same level (because they usually lie near each other in the original tree). // The original paper (describing the layout in Section 2.1) is: -// Michael A. Bender, Erik D. Demaine, Martin Farach-Colton. Cache-Oblivious B-Trees -// SIAM Journal on Computing, volume 35, number 2, 2005, pages 341-358. +// Michael A. Bender, Erik D. Demaine, Martin Farach-Colton. Cache-Oblivious B-Trees +// SIAM Journal on Computing, volume 35, number 2, 2005, pages 341-358. // Available on the web: http://erikdemaine.org/papers/CacheObliviousBTrees_SICOMP/ -// Or: Michael A. Bender, Erik D. Demaine, and Martin Farach-Colton. Cache-Oblivious B-Trees -// Proceedings of the 41st Annual Symposium -// on Foundations of Computer Science (FOCS 2000), Redondo Beach, California, November 12-14, 2000, pages 399-409. +// Or: Michael A. Bender, Erik D. Demaine, and Martin Farach-Colton. Cache-Oblivious B-Trees +// Proceedings of the 41st Annual Symposium +// on Foundations of Computer Science (FOCS 2000), Redondo Beach, California, November 12-14, 2000, pages 399-409. // Available on the web: http://erikdemaine.org/papers/FOCS2000b/ // (there is not much difference between these papers, actually). // diff --git a/library/cpp/containers/comptrie/comptrie_impl.h b/library/cpp/containers/comptrie/comptrie_impl.h index 37e5fa6221..f41c38311a 100644 --- a/library/cpp/containers/comptrie/comptrie_impl.h +++ b/library/cpp/containers/comptrie/comptrie_impl.h @@ -1,11 +1,11 @@ -#pragma once +#pragma once #include <util/stream/output.h> -#ifndef COMPTRIE_DATA_CHECK -#define COMPTRIE_DATA_CHECK 1 -#endif - +#ifndef COMPTRIE_DATA_CHECK +#define COMPTRIE_DATA_CHECK 1 +#endif + // NCompactTrie namespace NCompactTrie { diff --git a/library/cpp/containers/comptrie/comptrie_trie.h b/library/cpp/containers/comptrie/comptrie_trie.h index 5049be2831..40ec1e52b3 100644 --- a/library/cpp/containers/comptrie/comptrie_trie.h +++ b/library/cpp/containers/comptrie/comptrie_trie.h @@ -1,6 +1,6 @@ -#pragma once +#pragma once -#include "comptrie_impl.h" +#include "comptrie_impl.h" #include "comptrie_packer.h" #include "opaque_trie_iterator.h" #include "leaf_skipper.h" @@ -28,7 +28,7 @@ class TSearchIterator; template <class TTrie> class TPrefixIterator; -// in case of <char> specialization cannot distinguish between "" and "\0" keys +// in case of <char> specialization cannot distinguish between "" and "\0" keys template <class T = char, class D = ui64, class S = TCompactTriePacker<D>> class TCompactTrie { public: @@ -45,8 +45,8 @@ public: typedef TCompactTrieBuilder<T, D, S> TBuilder; -protected: - TBlob DataHolder; +protected: + TBlob DataHolder; const char* EmptyValue = nullptr; TPacker Packer; NCompactTrie::TPackerLeafSkipper<TPacker> Skipper = &Packer; // This should be true for every constructor. @@ -70,12 +70,12 @@ public: TCompactTrie& operator=(const TCompactTrie& other); TCompactTrie& operator=(TCompactTrie&& other) noexcept; - explicit operator bool() const { - return !IsEmpty(); - } - + explicit operator bool() const { + return !IsEmpty(); + } + void Init(const char* d, size_t len, TPacker packer = TPacker()); - void Init(const TBlob& data, TPacker packer = TPacker()); + void Init(const TBlob& data, TPacker packer = TPacker()); bool IsInitialized() const; bool IsEmpty() const; @@ -91,10 +91,10 @@ public: ythrow yexception() << "key " << TKey(key, keylen).Quote() << " not found in trie"; return value; } - TData Get(const TKeyBuf& key) const { + TData Get(const TKeyBuf& key) const { return Get(key.data(), key.size()); } - TData GetDefault(const TKeyBuf& key, const TData& def) const { + TData GetDefault(const TKeyBuf& key, const TData& def) const { TData value; if (!Find(key.data(), key.size(), &value)) return def; @@ -102,7 +102,7 @@ public: return value; } - const TBlob& Data() const { + const TBlob& Data() const { return DataHolder; }; @@ -119,7 +119,7 @@ public: } void FindPhrases(const TSymbol* key, size_t keylen, TPhraseMatchVector& matches, TSymbol separator = TSymbol(' ')) const; - void FindPhrases(const TKeyBuf& key, TPhraseMatchVector& matches, TSymbol separator = TSymbol(' ')) const { + void FindPhrases(const TKeyBuf& key, TPhraseMatchVector& matches, TSymbol separator = TSymbol(' ')) const { return FindPhrases(key.data(), key.size(), matches, separator); } bool FindLongestPrefix(const TSymbol* key, size_t keylen, size_t* prefixLen, TData* value = nullptr, bool* hasNext = nullptr) const; @@ -128,12 +128,12 @@ public: } // Return trie, containing all tails for the given key - inline TCompactTrie<T, D, S> FindTails(const TSymbol* key, size_t keylen) const; - TCompactTrie<T, D, S> FindTails(const TKeyBuf& key) const { + inline TCompactTrie<T, D, S> FindTails(const TSymbol* key, size_t keylen) const; + TCompactTrie<T, D, S> FindTails(const TKeyBuf& key) const { return FindTails(key.data(), key.size()); } - bool FindTails(const TSymbol* key, size_t keylen, TCompactTrie<T, D, S>& res) const; - bool FindTails(const TKeyBuf& key, TCompactTrie<T, D, S>& res) const { + bool FindTails(const TSymbol* key, size_t keylen, TCompactTrie<T, D, S>& res) const; + bool FindTails(const TKeyBuf& key, TCompactTrie<T, D, S>& res) const { return FindTails(key.data(), key.size(), res); } @@ -164,7 +164,7 @@ public: TValueType operator*(); TKey GetKey() const; - size_t GetKeySize() const; + size_t GetKeySize() const; TData GetValue() const; void GetValue(TData& data) const; const char* GetValuePtr() const; @@ -180,22 +180,22 @@ public: TConstIterator end() const; // Returns an iterator pointing to the smallest key in the trie >= the argument. - // TODO: misleading name. Should be called LowerBound for consistency with stl. + // TODO: misleading name. Should be called LowerBound for consistency with stl. // No. It is the STL that has a misleading name. // LowerBound of X cannot be greater than X. - TConstIterator UpperBound(const TKeyBuf& key) const; + TConstIterator UpperBound(const TKeyBuf& key) const; void Print(IOutputStream& os); - size_t Size() const; - + size_t Size() const; + friend class NCompactTrie::TFirstSymbolIterator<TCompactTrie>; friend class TSearchIterator<TCompactTrie>; friend class TPrefixIterator<TCompactTrie>; -protected: +protected: explicit TCompactTrie(const char* emptyValue); - TCompactTrie(const TBlob& data, const char* emptyValue, TPacker packer = TPacker()); + TCompactTrie(const TBlob& data, const char* emptyValue, TPacker packer = TPacker()); bool LookupLongestPrefix(const TSymbol* key, size_t keylen, size_t& prefixLen, const char*& valuepos, bool& hasNext) const; bool LookupLongestPrefix(const TSymbol* key, size_t keylen, size_t& prefixLen, const char*& valuepos) const { @@ -221,36 +221,36 @@ public: // TCompactTrie -template <class T, class D, class S> -TCompactTrie<T, D, S>::TCompactTrie(const TBlob& data, TPacker packer) +template <class T, class D, class S> +TCompactTrie<T, D, S>::TCompactTrie(const TBlob& data, TPacker packer) : DataHolder(data) , Packer(packer) { Init(data, packer); } -template <class T, class D, class S> -TCompactTrie<T, D, S>::TCompactTrie(const char* d, size_t len, TPacker packer) +template <class T, class D, class S> +TCompactTrie<T, D, S>::TCompactTrie(const char* d, size_t len, TPacker packer) : Packer(packer) { Init(d, len, packer); } -template <class T, class D, class S> -TCompactTrie<T, D, S>::TCompactTrie(const char* emptyValue) +template <class T, class D, class S> +TCompactTrie<T, D, S>::TCompactTrie(const char* emptyValue) : EmptyValue(emptyValue) { } -template <class T, class D, class S> -TCompactTrie<T, D, S>::TCompactTrie(const TBlob& data, const char* emptyValue, TPacker packer) +template <class T, class D, class S> +TCompactTrie<T, D, S>::TCompactTrie(const TBlob& data, const char* emptyValue, TPacker packer) : DataHolder(data) , EmptyValue(emptyValue) , Packer(packer) { } -template <class T, class D, class S> +template <class T, class D, class S> TCompactTrie<T, D, S>::TCompactTrie(const TCompactTrie& other) : DataHolder(other.DataHolder) , EmptyValue(other.EmptyValue) @@ -287,19 +287,19 @@ TCompactTrie<T, D, S>& TCompactTrie<T, D, S>::operator=(TCompactTrie&& other) no } template <class T, class D, class S> -void TCompactTrie<T, D, S>::Init(const char* d, size_t len, TPacker packer) { - Init(TBlob::NoCopy(d, len), packer); +void TCompactTrie<T, D, S>::Init(const char* d, size_t len, TPacker packer) { + Init(TBlob::NoCopy(d, len), packer); } -template <class T, class D, class S> -void TCompactTrie<T, D, S>::Init(const TBlob& data, TPacker packer) { +template <class T, class D, class S> +void TCompactTrie<T, D, S>::Init(const TBlob& data, TPacker packer) { using namespace NCompactTrie; DataHolder = data; Packer = packer; const char* datapos = DataHolder.AsCharPtr(); - size_t len = DataHolder.Length(); + size_t len = DataHolder.Length(); if (!len) return; @@ -313,17 +313,17 @@ void TCompactTrie<T, D, S>::Init(const TBlob& data, TPacker packer) { } } -template <class T, class D, class S> -bool TCompactTrie<T, D, S>::IsInitialized() const { +template <class T, class D, class S> +bool TCompactTrie<T, D, S>::IsInitialized() const { return DataHolder.Data() != nullptr; } -template <class T, class D, class S> -bool TCompactTrie<T, D, S>::IsEmpty() const { +template <class T, class D, class S> +bool TCompactTrie<T, D, S>::IsEmpty() const { return DataHolder.Size() == 0 && EmptyValue == nullptr; } -template <class T, class D, class S> +template <class T, class D, class S> bool TCompactTrie<T, D, S>::Find(const TSymbol* key, size_t keylen, TData* value) const { size_t prefixLen = 0; const char* valuepos = nullptr; @@ -335,20 +335,20 @@ bool TCompactTrie<T, D, S>::Find(const TSymbol* key, size_t keylen, TData* value return true; } -template <class T, class D, class S> -void TCompactTrie<T, D, S>::FindPhrases(const TSymbol* key, size_t keylen, TPhraseMatchVector& matches, TSymbol separator) const { +template <class T, class D, class S> +void TCompactTrie<T, D, S>::FindPhrases(const TSymbol* key, size_t keylen, TPhraseMatchVector& matches, TSymbol separator) const { LookupPhrases(DataHolder.AsCharPtr(), DataHolder.Length(), key, keylen, matches, separator); } -template <class T, class D, class S> -inline TCompactTrie<T, D, S> TCompactTrie<T, D, S>::FindTails(const TSymbol* key, size_t keylen) const { - TCompactTrie<T, D, S> ret; +template <class T, class D, class S> +inline TCompactTrie<T, D, S> TCompactTrie<T, D, S>::FindTails(const TSymbol* key, size_t keylen) const { + TCompactTrie<T, D, S> ret; FindTails(key, keylen, ret); return ret; } -template <class T, class D, class S> -bool TCompactTrie<T, D, S>::FindTails(const TSymbol* key, size_t keylen, TCompactTrie<T, D, S>& res) const { +template <class T, class D, class S> +bool TCompactTrie<T, D, S>::FindTails(const TSymbol* key, size_t keylen, TCompactTrie<T, D, S>& res) const { using namespace NCompactTrie; size_t len = DataHolder.Length(); @@ -376,7 +376,7 @@ bool TCompactTrie<T, D, S>::FindTails(const TSymbol* key, size_t keylen, TCompac if (key == keyend) { if (datapos) { Y_ASSERT(datapos >= datastart); - res = TCompactTrie<T, D, S>(TBlob::NoCopy(datapos, dataend - datapos), value); + res = TCompactTrie<T, D, S>(TBlob::NoCopy(datapos, dataend - datapos), value); } else { res = TCompactTrie<T, D, S>(value); } @@ -389,11 +389,11 @@ bool TCompactTrie<T, D, S>::FindTails(const TSymbol* key, size_t keylen, TCompac return false; } -template <class T, class D, class S> +template <class T, class D, class S> inline bool TCompactTrie<T, D, S>::FindTails(TSymbol label, TCompactTrie<T, D, S>& res) const { using namespace NCompactTrie; - const size_t len = DataHolder.Length(); + const size_t len = DataHolder.Length(); if (!len) return false; @@ -415,43 +415,43 @@ inline bool TCompactTrie<T, D, S>::FindTails(TSymbol label, TCompactTrie<T, D, S return true; } -template <class T, class D, class S> -typename TCompactTrie<T, D, S>::TConstIterator TCompactTrie<T, D, S>::Begin() const { +template <class T, class D, class S> +typename TCompactTrie<T, D, S>::TConstIterator TCompactTrie<T, D, S>::Begin() const { NCompactTrie::TOpaqueTrie self(DataHolder.AsCharPtr(), DataHolder.Length(), Skipper); return TConstIterator(self, EmptyValue, false, Packer); } -template <class T, class D, class S> +template <class T, class D, class S> typename TCompactTrie<T, D, S>::TConstIterator TCompactTrie<T, D, S>::begin() const { return Begin(); } template <class T, class D, class S> -typename TCompactTrie<T, D, S>::TConstIterator TCompactTrie<T, D, S>::End() const { +typename TCompactTrie<T, D, S>::TConstIterator TCompactTrie<T, D, S>::End() const { NCompactTrie::TOpaqueTrie self(DataHolder.AsCharPtr(), DataHolder.Length(), Skipper); return TConstIterator(self, EmptyValue, true, Packer); } -template <class T, class D, class S> +template <class T, class D, class S> typename TCompactTrie<T, D, S>::TConstIterator TCompactTrie<T, D, S>::end() const { return End(); } template <class T, class D, class S> -size_t TCompactTrie<T, D, S>::Size() const { - size_t res = 0; - for (TConstIterator it = Begin(); it != End(); ++it) - ++res; - return res; -} - -template <class T, class D, class S> -typename TCompactTrie<T, D, S>::TConstIterator TCompactTrie<T, D, S>::UpperBound(const TKeyBuf& key) const { +size_t TCompactTrie<T, D, S>::Size() const { + size_t res = 0; + for (TConstIterator it = Begin(); it != End(); ++it) + ++res; + return res; +} + +template <class T, class D, class S> +typename TCompactTrie<T, D, S>::TConstIterator TCompactTrie<T, D, S>::UpperBound(const TKeyBuf& key) const { NCompactTrie::TOpaqueTrie self(DataHolder.AsCharPtr(), DataHolder.Length(), Skipper); return TConstIterator(self, EmptyValue, key, Packer); } -template <class T, class D, class S> +template <class T, class D, class S> void TCompactTrie<T, D, S>::Print(IOutputStream& os) { typedef typename ::TCompactTrieKeySelector<T>::TKeyBuf TSBuffer; for (TConstIterator it = Begin(); it != End(); ++it) { @@ -459,7 +459,7 @@ void TCompactTrie<T, D, S>::Print(IOutputStream& os) { } } -template <class T, class D, class S> +template <class T, class D, class S> bool TCompactTrie<T, D, S>::FindLongestPrefix(const TSymbol* key, size_t keylen, size_t* prefixLen, TData* value, bool* hasNext) const { const char* valuepos = nullptr; size_t tempPrefixLen = 0; @@ -474,12 +474,12 @@ bool TCompactTrie<T, D, S>::FindLongestPrefix(const TSymbol* key, size_t keylen, return found; } -template <class T, class D, class S> +template <class T, class D, class S> bool TCompactTrie<T, D, S>::LookupLongestPrefix(const TSymbol* key, size_t keylen, size_t& prefixLen, const char*& valuepos, bool& hasNext) const { using namespace NCompactTrie; const char* datapos = DataHolder.AsCharPtr(); - size_t len = DataHolder.Length(); + size_t len = DataHolder.Length(); prefixLen = 0; hasNext = false; @@ -526,8 +526,8 @@ bool TCompactTrie<T, D, S>::LookupLongestPrefix(const TSymbol* key, size_t keyle return found; } -template <class T, class D, class S> -void TCompactTrie<T, D, S>::LookupPhrases( +template <class T, class D, class S> +void TCompactTrie<T, D, S>::LookupPhrases( const char* datapos, size_t len, const TSymbol* key, size_t keylen, TVector<TPhraseMatch>& matches, TSymbol separator) const { using namespace NCompactTrie; @@ -570,14 +570,14 @@ TCompactTrieHolder<T, D, S>::TCompactTrieHolder(IInputStream& is, size_t len) //---------------------------------------------------------------------------------------------------------------- // TCompactTrie::TConstIterator -template <class T, class D, class S> +template <class T, class D, class S> TCompactTrie<T, D, S>::TConstIterator::TConstIterator(const TOpaqueTrie& trie, const char* emptyValue, bool atend, TPacker packer) : Packer(packer) , Impl(new TOpaqueTrieIterator(trie, emptyValue, atend)) { } -template <class T, class D, class S> +template <class T, class D, class S> TCompactTrie<T, D, S>::TConstIterator::TConstIterator(const TOpaqueTrie& trie, const char* emptyValue, const TKeyBuf& key, TPacker packer) : Packer(packer) , Impl(new TOpaqueTrieIterator(trie, emptyValue, true)) @@ -585,7 +585,7 @@ TCompactTrie<T, D, S>::TConstIterator::TConstIterator(const TOpaqueTrie& trie, c Impl->UpperBound<TSymbol>(key); } -template <class T, class D, class S> +template <class T, class D, class S> bool TCompactTrie<T, D, S>::TConstIterator::operator==(const TConstIterator& other) const { if (!Impl) return !other.Impl; @@ -594,70 +594,70 @@ bool TCompactTrie<T, D, S>::TConstIterator::operator==(const TConstIterator& oth return *Impl == *other.Impl; } -template <class T, class D, class S> +template <class T, class D, class S> bool TCompactTrie<T, D, S>::TConstIterator::operator!=(const TConstIterator& other) const { return !operator==(other); } -template <class T, class D, class S> -typename TCompactTrie<T, D, S>::TConstIterator& TCompactTrie<T, D, S>::TConstIterator::operator++() { +template <class T, class D, class S> +typename TCompactTrie<T, D, S>::TConstIterator& TCompactTrie<T, D, S>::TConstIterator::operator++() { Impl->Forward(); return *this; } -template <class T, class D, class S> -typename TCompactTrie<T, D, S>::TConstIterator TCompactTrie<T, D, S>::TConstIterator::operator++(int /*unused*/) { +template <class T, class D, class S> +typename TCompactTrie<T, D, S>::TConstIterator TCompactTrie<T, D, S>::TConstIterator::operator++(int /*unused*/) { TConstIterator copy(*this); Impl->Forward(); return copy; } -template <class T, class D, class S> -typename TCompactTrie<T, D, S>::TConstIterator& TCompactTrie<T, D, S>::TConstIterator::operator--() { +template <class T, class D, class S> +typename TCompactTrie<T, D, S>::TConstIterator& TCompactTrie<T, D, S>::TConstIterator::operator--() { Impl->Backward(); return *this; } -template <class T, class D, class S> -typename TCompactTrie<T, D, S>::TConstIterator TCompactTrie<T, D, S>::TConstIterator::operator--(int /*unused*/) { +template <class T, class D, class S> +typename TCompactTrie<T, D, S>::TConstIterator TCompactTrie<T, D, S>::TConstIterator::operator--(int /*unused*/) { TConstIterator copy(*this); Impl->Backward(); return copy; } -template <class T, class D, class S> -typename TCompactTrie<T, D, S>::TValueType TCompactTrie<T, D, S>::TConstIterator::operator*() { +template <class T, class D, class S> +typename TCompactTrie<T, D, S>::TValueType TCompactTrie<T, D, S>::TConstIterator::operator*() { return TValueType(GetKey(), GetValue()); } -template <class T, class D, class S> +template <class T, class D, class S> typename TCompactTrie<T, D, S>::TKey TCompactTrie<T, D, S>::TConstIterator::GetKey() const { return Impl->GetKey<TSymbol>(); } -template <class T, class D, class S> +template <class T, class D, class S> size_t TCompactTrie<T, D, S>::TConstIterator::GetKeySize() const { return Impl->MeasureKey<TSymbol>(); -} - -template <class T, class D, class S> +} + +template <class T, class D, class S> const char* TCompactTrie<T, D, S>::TConstIterator::GetValuePtr() const { return Impl->GetValuePtr(); -} - -template <class T, class D, class S> +} + +template <class T, class D, class S> typename TCompactTrie<T, D, S>::TData TCompactTrie<T, D, S>::TConstIterator::GetValue() const { D data; GetValue(data); return data; } -template <class T, class D, class S> +template <class T, class D, class S> void TCompactTrie<T, D, S>::TConstIterator::GetValue(typename TCompactTrie<T, D, S>::TData& data) const { const char* ptr = GetValuePtr(); if (ptr) { Packer.UnpackLeaf(ptr, data); } else { - data = typename TCompactTrie<T, D, S>::TData(); + data = typename TCompactTrie<T, D, S>::TData(); } } diff --git a/library/cpp/containers/comptrie/comptrie_ut.cpp b/library/cpp/containers/comptrie/comptrie_ut.cpp index d73883afdc..74bee09b5d 100644 --- a/library/cpp/containers/comptrie/comptrie_ut.cpp +++ b/library/cpp/containers/comptrie/comptrie_ut.cpp @@ -36,7 +36,7 @@ private: UNIT_TEST(TestTrie8); UNIT_TEST(TestTrie16); UNIT_TEST(TestTrie32); - + UNIT_TEST(TestFastTrie8); UNIT_TEST(TestFastTrie16); UNIT_TEST(TestFastTrie32); @@ -44,7 +44,7 @@ private: UNIT_TEST(TestMinimizedTrie8); UNIT_TEST(TestMinimizedTrie16); UNIT_TEST(TestMinimizedTrie32); - + UNIT_TEST(TestFastMinimizedTrie8); UNIT_TEST(TestFastMinimizedTrie16); UNIT_TEST(TestFastMinimizedTrie32); @@ -52,11 +52,11 @@ private: UNIT_TEST(TestTrieIterator8); UNIT_TEST(TestTrieIterator16); UNIT_TEST(TestTrieIterator32); - + UNIT_TEST(TestMinimizedTrieIterator8); UNIT_TEST(TestMinimizedTrieIterator16); UNIT_TEST(TestMinimizedTrieIterator32); - + UNIT_TEST(TestPhraseSearch); UNIT_TEST(TestAddGet); UNIT_TEST(TestEmpty); @@ -169,11 +169,11 @@ private: public: void TestPackers(); - + void TestTrie8(); void TestTrie16(); void TestTrie32(); - + void TestFastTrie8(); void TestFastTrie16(); void TestFastTrie32(); @@ -181,7 +181,7 @@ public: void TestMinimizedTrie8(); void TestMinimizedTrie16(); void TestMinimizedTrie32(); - + void TestFastMinimizedTrie8(); void TestFastMinimizedTrie16(); void TestFastMinimizedTrie32(); @@ -189,11 +189,11 @@ public: void TestTrieIterator8(); void TestTrieIterator16(); void TestTrieIterator32(); - + void TestMinimizedTrieIterator8(); void TestMinimizedTrieIterator16(); void TestMinimizedTrieIterator32(); - + void TestPhraseSearch(); void TestAddGet(); void TestEmpty(); @@ -277,7 +277,7 @@ const char* TCompactTrieTest::SampleData[] = { }; template <class T> -typename TCompactTrie<T>::TKey MakeWideKey(const char* str, size_t len) { +typename TCompactTrie<T>::TKey MakeWideKey(const char* str, size_t len) { typename TCompactTrie<T>::TKey buffer; for (size_t i = 0; i < len; i++) { unsigned int ch = (str[i] & 0xFF); @@ -302,7 +302,7 @@ void TCompactTrieTest::CreateTrie(IOutputStream& out, bool minimize, bool useFas for (auto& i : SampleData) { size_t len = strlen(i); - + builder.Add(MakeWideKey<T>(i, len), len * 2); } @@ -362,7 +362,7 @@ void TCompactTrieTest::CheckData(const char* data, size_t datalen) { TCompactTrie<T> trie(data, datalen); UNIT_ASSERT_VALUES_EQUAL(Y_ARRAY_SIZE(SampleData), trie.Size()); - + for (auto& i : SampleData) { size_t len = strlen(i); ui64 value = 0; @@ -421,7 +421,7 @@ void TCompactTrieTest::CheckData(const char* data, size_t datalen) { template <class T> void TCompactTrieTest::CheckIterator(const char* data, size_t datalen) { - typedef typename TCompactTrie<T>::TKey TKey; + typedef typename TCompactTrie<T>::TKey TKey; typedef typename TCompactTrie<T>::TValueType TValue; TMap<TKey, ui64> stored; @@ -437,7 +437,7 @@ void TCompactTrieTest::CheckIterator(const char* data, size_t datalen) { size_t entry_count = 0; TMap<TKey, ui64> received; while (it != trie.End()) { - UNIT_ASSERT_VALUES_EQUAL(it.GetKeySize(), it.GetKey().size()); + UNIT_ASSERT_VALUES_EQUAL(it.GetKeySize(), it.GetKey().size()); received.insert(*it); items.push_back(*it); entry_count++; @@ -509,7 +509,7 @@ void TCompactTrieTest::TestTrie16() { void TCompactTrieTest::TestTrie32() { TestTrie<wchar32>(false, false); } - + void TCompactTrieTest::TestFastTrie8() { TestTrie<char>(false, true); } diff --git a/library/cpp/containers/comptrie/leaf_skipper.h b/library/cpp/containers/comptrie/leaf_skipper.h index e3aab8596f..3959258948 100644 --- a/library/cpp/containers/comptrie/leaf_skipper.h +++ b/library/cpp/containers/comptrie/leaf_skipper.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <cstddef> diff --git a/library/cpp/containers/comptrie/make_fast_layout.cpp b/library/cpp/containers/comptrie/make_fast_layout.cpp index 779146df1f..ade78d7899 100644 --- a/library/cpp/containers/comptrie/make_fast_layout.cpp +++ b/library/cpp/containers/comptrie/make_fast_layout.cpp @@ -4,7 +4,7 @@ #include "write_trie_backwards.h" #include "comptrie_impl.h" -#include <util/generic/hash.h> +#include <util/generic/hash.h> #include <util/generic/utility.h> // Lay the trie in memory in such a way that there are less cache misses when jumping from root to leaf. diff --git a/library/cpp/containers/comptrie/make_fast_layout.h b/library/cpp/containers/comptrie/make_fast_layout.h index f176de6b73..b8fab5d65b 100644 --- a/library/cpp/containers/comptrie/make_fast_layout.h +++ b/library/cpp/containers/comptrie/make_fast_layout.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include "leaf_skipper.h" #include <cstddef> diff --git a/library/cpp/containers/comptrie/minimize.cpp b/library/cpp/containers/comptrie/minimize.cpp index 7346668c57..80d0b25217 100644 --- a/library/cpp/containers/comptrie/minimize.cpp +++ b/library/cpp/containers/comptrie/minimize.cpp @@ -4,7 +4,7 @@ #include "write_trie_backwards.h" #include "comptrie_impl.h" -#include <util/generic/hash.h> +#include <util/generic/hash.h> #include <util/generic/algorithm.h> namespace NCompactTrie { @@ -42,7 +42,7 @@ namespace NCompactTrie { if (Data.size() <= hikey) Data.resize(hikey + 1); - + TSizePairVector& sublist = Data[hikey]; for (auto& it : sublist) { @@ -296,7 +296,7 @@ namespace NCompactTrie { if (iit == subtries.end()) continue; // fast forward to the next available length value - + TOffsetList& batch = iit->second; TPieceComparer comparer(trie.Data, curlen); Sort(batch.begin(), batch.end(), comparer); diff --git a/library/cpp/containers/comptrie/minimize.h b/library/cpp/containers/comptrie/minimize.h index 7b991584e9..baaa431d04 100644 --- a/library/cpp/containers/comptrie/minimize.h +++ b/library/cpp/containers/comptrie/minimize.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include "leaf_skipper.h" #include <cstddef> diff --git a/library/cpp/containers/comptrie/node.cpp b/library/cpp/containers/comptrie/node.cpp index 99a56fe83a..5fd22f15ec 100644 --- a/library/cpp/containers/comptrie/node.cpp +++ b/library/cpp/containers/comptrie/node.cpp @@ -1,6 +1,6 @@ #include "node.h" #include "leaf_skipper.h" -#include "comptrie_impl.h" +#include "comptrie_impl.h" #include <util/system/yassert.h> #include <util/generic/yexception.h> @@ -56,7 +56,7 @@ namespace NCompactTrie { Offsets[D_FINAL] = datapos - data; LeafLength = skipFunction.SkipLeaf(datapos); } - + CoreLength = 2 + leftsize + rightsize + LeafLength; if (flags & MT_NEXT) { size_t& forwardOffset = Offsets[D_NEXT]; diff --git a/library/cpp/containers/comptrie/node.h b/library/cpp/containers/comptrie/node.h index d4d90201ef..d6f4317db0 100644 --- a/library/cpp/containers/comptrie/node.h +++ b/library/cpp/containers/comptrie/node.h @@ -32,7 +32,7 @@ namespace NCompactTrie { size_t GetOffset() const { return Offset; } - + size_t GetLeafOffset() const { return Offsets[D_FINAL]; } diff --git a/library/cpp/containers/comptrie/opaque_trie_iterator.cpp b/library/cpp/containers/comptrie/opaque_trie_iterator.cpp index 1c812d9521..5fd3914be6 100644 --- a/library/cpp/containers/comptrie/opaque_trie_iterator.cpp +++ b/library/cpp/containers/comptrie/opaque_trie_iterator.cpp @@ -1,5 +1,5 @@ #include "opaque_trie_iterator.h" -#include "comptrie_impl.h" +#include "comptrie_impl.h" #include "node.h" namespace NCompactTrie { @@ -157,8 +157,8 @@ namespace NCompactTrie { } else { return Key.size() == 1 && Key[0] == '\0'; } - } - + } + size_t TForkStack::MeasureKey() const { size_t result = Key.size() + (TopHasLabelInKey() ? 1 : 0); if (result == 1 && HasEmptyKey()) { diff --git a/library/cpp/containers/comptrie/opaque_trie_iterator.h b/library/cpp/containers/comptrie/opaque_trie_iterator.h index e136270163..195da3c191 100644 --- a/library/cpp/containers/comptrie/opaque_trie_iterator.h +++ b/library/cpp/containers/comptrie/opaque_trie_iterator.h @@ -1,6 +1,6 @@ -#pragma once +#pragma once -#include "comptrie_impl.h" +#include "comptrie_impl.h" #include "node.h" #include "key_selector.h" #include "leaf_skipper.h" diff --git a/library/cpp/containers/comptrie/ut/ya.make b/library/cpp/containers/comptrie/ut/ya.make index ed3048f0e3..c4f4666009 100644 --- a/library/cpp/containers/comptrie/ut/ya.make +++ b/library/cpp/containers/comptrie/ut/ya.make @@ -1,7 +1,7 @@ UNITTEST_FOR(library/cpp/containers/comptrie) OWNER(alzobnin) - + SRCS( comptrie_ut.cpp ) diff --git a/library/cpp/containers/comptrie/ya.make b/library/cpp/containers/comptrie/ya.make index 4480a4f01b..81352da4b2 100644 --- a/library/cpp/containers/comptrie/ya.make +++ b/library/cpp/containers/comptrie/ya.make @@ -1,7 +1,7 @@ LIBRARY() OWNER(velavokr) - + SRCS( array_with_size.h chunked_helpers_trie.h @@ -29,7 +29,7 @@ PEERDIR( library/cpp/packers library/cpp/containers/compact_vector library/cpp/on_disk/chunks - util/draft + util/draft ) END() diff --git a/library/cpp/containers/intrusive_avl_tree/avltree.h b/library/cpp/containers/intrusive_avl_tree/avltree.h index 39fe191a20..a58c63b07c 100644 --- a/library/cpp/containers/intrusive_avl_tree/avltree.h +++ b/library/cpp/containers/intrusive_avl_tree/avltree.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <util/generic/noncopyable.h> diff --git a/library/cpp/containers/intrusive_rb_tree/rb_tree.h b/library/cpp/containers/intrusive_rb_tree/rb_tree.h index d595701bad..0259452a14 100644 --- a/library/cpp/containers/intrusive_rb_tree/rb_tree.h +++ b/library/cpp/containers/intrusive_rb_tree/rb_tree.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <util/generic/utility.h> #include <util/generic/yexception.h> diff --git a/library/cpp/containers/paged_vector/paged_vector.h b/library/cpp/containers/paged_vector/paged_vector.h index 9730b00670..6a3657d3ea 100644 --- a/library/cpp/containers/paged_vector/paged_vector.h +++ b/library/cpp/containers/paged_vector/paged_vector.h @@ -1,15 +1,15 @@ -#pragma once - -#include <util/generic/ptr.h> -#include <util/generic/vector.h> -#include <util/generic/yexception.h> - +#pragma once + +#include <util/generic/ptr.h> +#include <util/generic/vector.h> +#include <util/generic/yexception.h> + #include <iterator> -namespace NPagedVector { +namespace NPagedVector { template <class T, ui32 PageSize = 1u << 20, class A = std::allocator<T>> class TPagedVector; - + namespace NPrivate { template <class T, class TT, ui32 PageSize, class A> struct TPagedVectorIterator { @@ -19,119 +19,119 @@ namespace NPagedVector { typedef TPagedVectorIterator<T, TT, PageSize, A> TSelf; size_t Offset; TVec* Vector; - + template <class T1, class TT1, ui32 PageSize1, class A1> friend struct TPagedVectorIterator; - + public: TPagedVectorIterator() : Offset() , Vector() { } - + TPagedVectorIterator(TVec* vector, size_t offset) : Offset(offset) , Vector(vector) { } - + template <class T1, class TT1, ui32 PageSize1, class A1> TPagedVectorIterator(const TPagedVectorIterator<T1, TT1, PageSize1, A1>& it) : Offset(it.Offset) , Vector(it.Vector) { } - + T& operator*() const { return (*Vector)[Offset]; } - + T* operator->() const { return &(**this); } - + template <class T1, class TT1, ui32 PageSize1, class A1> bool operator==(const TPagedVectorIterator<T1, TT1, PageSize1, A1>& it) const { return Offset == it.Offset; } - + template <class T1, class TT1, ui32 PageSize1, class A1> bool operator!=(const TPagedVectorIterator<T1, TT1, PageSize1, A1>& it) const { return !(*this == it); } - + template <class T1, class TT1, ui32 PageSize1, class A1> bool operator<(const TPagedVectorIterator<T1, TT1, PageSize1, A1>& it) const { return Offset < it.Offset; } - + template <class T1, class TT1, ui32 PageSize1, class A1> bool operator<=(const TPagedVectorIterator<T1, TT1, PageSize1, A1>& it) const { return Offset <= it.Offset; } - + template <class T1, class TT1, ui32 PageSize1, class A1> bool operator>(const TPagedVectorIterator<T1, TT1, PageSize1, A1>& it) const { return !(*this <= it); } - + template <class T1, class TT1, ui32 PageSize1, class A1> bool operator>=(const TPagedVectorIterator<T1, TT1, PageSize1, A1>& it) const { return !(*this < it); } - + template <class T1, class TT1, ui32 PageSize1, class A1> ptrdiff_t operator-(const TPagedVectorIterator<T1, TT1, PageSize1, A1>& it) const { return Offset - it.Offset; } - + TSelf& operator+=(ptrdiff_t off) { Offset += off; return *this; } - + TSelf& operator-=(ptrdiff_t off) { return this->operator+=(-off); } - + TSelf& operator++() { return this->operator+=(1); } - + TSelf& operator--() { return this->operator+=(-1); } - + TSelf operator++(int) { TSelf it = *this; this->operator+=(1); return it; } - + TSelf operator--(int) { TSelf it = *this; this->operator+=(-1); return it; } - + TSelf operator+(ptrdiff_t off) { TSelf res = *this; res += off; return res; } - + TSelf operator-(ptrdiff_t off) { return this->operator+(-off); } - + size_t GetOffset() { return Offset; } }; - } -} - + } +} + namespace std { template <class T, class TT, ui32 PageSize, class A> struct iterator_traits<NPagedVector::NPrivate::TPagedVectorIterator<T, TT, PageSize, A>> { @@ -141,19 +141,19 @@ namespace std { typedef T& reference; typedef random_access_iterator_tag iterator_category; }; - -} - -namespace NPagedVector { + +} + +namespace NPagedVector { //2-level radix tree template <class T, ui32 PageSize, class A> class TPagedVector: private TVector<TSimpleSharedPtr<TVector<T, A>>, A> { static_assert(PageSize, "expect PageSize"); - + typedef TVector<T, A> TPage; typedef TVector<TSimpleSharedPtr<TPage>, A> TPages; typedef TPagedVector<T, PageSize, A> TSelf; - + public: typedef NPrivate::TPagedVectorIterator<T, T, PageSize, A> iterator; typedef NPrivate::TPagedVectorIterator<const T, T, PageSize, A> const_iterator; @@ -162,99 +162,99 @@ namespace NPagedVector { typedef T value_type; typedef value_type& reference; typedef const value_type& const_reference; - + TPagedVector() = default; - + template <typename TIter> TPagedVector(TIter b, TIter e) { append(b, e); } - + iterator begin() { return iterator(this, 0); } - + const_iterator begin() const { return const_iterator((TSelf*)this, 0); } - + iterator end() { return iterator(this, size()); } - + const_iterator end() const { return const_iterator((TSelf*)this, size()); } - + reverse_iterator rbegin() { return reverse_iterator(end()); } - + const_reverse_iterator rbegin() const { return const_reverse_iterator(end()); } - + reverse_iterator rend() { return reverse_iterator(begin()); } - + const_reverse_iterator rend() const { return const_reverse_iterator(begin()); } - + void swap(TSelf& v) { TPages::swap((TPages&)v); } - + private: static size_t PageNumber(size_t idx) { return idx / PageSize; } - + static size_t InPageIndex(size_t idx) { return idx % PageSize; } - + static size_t Index(size_t pnum, size_t poff) { return pnum * PageSize + poff; } - + TPage& PageAt(size_t pnum) const { return *TPages::at(pnum); } - + TPage& CurrentPage() const { return *TPages::back(); } - + size_t CurrentPageSize() const { return TPages::empty() ? 0 : CurrentPage().size(); } - + size_t NPages() const { return TPages::size(); } - + void AllocateNewPage() { TPages::push_back(new TPage()); CurrentPage().reserve(PageSize); } - + void MakeNewPage() { - AllocateNewPage(); + AllocateNewPage(); CurrentPage().resize(PageSize); } - + void PrepareAppend() { if (TPages::empty() || CurrentPage().size() + 1 > PageSize) AllocateNewPage(); } - + public: size_t size() const { return empty() ? 0 : (NPages() - 1) * PageSize + CurrentPage().size(); } - + bool empty() const { return TPages::empty() || 1 == NPages() && CurrentPage().empty(); } @@ -262,23 +262,23 @@ namespace NPagedVector { explicit operator bool() const noexcept { return !empty(); } - + void emplace_back() { PrepareAppend(); CurrentPage().emplace_back(); } - + void push_back(const_reference t) { - PrepareAppend(); + PrepareAppend(); CurrentPage().push_back(t); - } - + } + void pop_back() { if (CurrentPage().empty()) TPages::pop_back(); CurrentPage().pop_back(); - } - + } + template <typename TIter> void append(TIter b, TIter e) { size_t sz = e - b; @@ -303,15 +303,15 @@ namespace NPagedVector { TPage& p = CurrentPage(); p.insert(p.end(), b + sz1 + sz2 * PageSize, e); } - } - + } + iterator erase(iterator it) { size_t pnum = PageNumber(it.Offset); size_t pidx = InPageIndex(it.Offset); - + if (CurrentPage().empty()) TPages::pop_back(); - + for (size_t p = NPages() - 1; p > pnum; --p) { PageAt(p - 1).push_back(PageAt(p).front()); PageAt(p).erase(PageAt(p).begin()); @@ -319,114 +319,114 @@ namespace NPagedVector { PageAt(pnum).erase(PageAt(pnum).begin() + pidx); return it; - } - + } + iterator erase(iterator b, iterator e) { // todo : suboptimal! while (b != e) { b = erase(b); --e; } - + return b; - } - + } + iterator insert(iterator it, const value_type& v) { size_t pnum = PageNumber(it.Offset); size_t pidx = InPageIndex(it.Offset); - + PrepareAppend(); - + for (size_t p = NPages() - 1; p > pnum; --p) { PageAt(p).insert(PageAt(p).begin(), PageAt(p - 1).back()); PageAt(p - 1).pop_back(); } - + PageAt(pnum).insert(PageAt(pnum).begin() + pidx, v); return it; - } - + } + template <typename TIter> void insert(iterator it, TIter b, TIter e) { // todo : suboptimal! for (; b != e; ++b, ++it) it = insert(it, *b); } - + reference front() { return TPages::front()->front(); } - + const_reference front() const { return TPages::front()->front(); } - + reference back() { return CurrentPage().back(); } - + const_reference back() const { return CurrentPage().back(); } - + void clear() { TPages::clear(); } - + void resize(size_t sz) { if (sz == size()) return; - + const size_t npages = NPages(); const size_t newwholepages = sz / PageSize; const size_t pagepart = sz % PageSize; const size_t newpages = newwholepages + bool(pagepart); - + if (npages && newwholepages >= npages) CurrentPage().resize(PageSize); - + if (newpages < npages) TPages::resize(newpages); else for (size_t i = npages; i < newpages; ++i) MakeNewPage(); - + if (pagepart) CurrentPage().resize(pagepart); - + Y_VERIFY(sz == size(), "%" PRIu64 " %" PRIu64, (ui64)sz, (ui64)size()); } - + reference at(size_t idx) { return TPages::at(PageNumber(idx))->at(InPageIndex(idx)); } - + const_reference at(size_t idx) const { return TPages::at(PageNumber(idx))->at(InPageIndex(idx)); } - + reference operator[](size_t idx) { return TPages::operator[](PageNumber(idx))->operator[](InPageIndex(idx)); } - + const_reference operator[](size_t idx) const { return TPages::operator[](PageNumber(idx))->operator[](InPageIndex(idx)); } - + friend bool operator==(const TSelf& a, const TSelf& b) { return a.size() == b.size() && std::equal(a.begin(), a.end(), b.begin()); } - + friend bool operator<(const TSelf& a, const TSelf& b) { return std::lexicographical_compare(a.begin(), a.end(), b.begin(), b.end()); } }; - + namespace NPrivate { typedef std::is_same<std::random_access_iterator_tag, std::iterator_traits< TPagedVector<ui32>::iterator>::iterator_category> TIteratorCheck; static_assert(TIteratorCheck::value, "expect TIteratorCheck::Result"); - } - -} + } + +} diff --git a/library/cpp/containers/paged_vector/ut/paged_vector_ut.cpp b/library/cpp/containers/paged_vector/ut/paged_vector_ut.cpp index ff6d86648c..e867808ee4 100644 --- a/library/cpp/containers/paged_vector/ut/paged_vector_ut.cpp +++ b/library/cpp/containers/paged_vector/ut/paged_vector_ut.cpp @@ -2,8 +2,8 @@ #include <library/cpp/testing/unittest/registar.h> #include <stdexcept> - -class TPagedVectorTest: public TTestBase { + +class TPagedVectorTest: public TTestBase { UNIT_TEST_SUITE(TPagedVectorTest); UNIT_TEST(Test0) UNIT_TEST(Test1) @@ -18,7 +18,7 @@ class TPagedVectorTest: public TTestBase { UNIT_TEST(TestIterators) //UNIT_TEST(TestEbo) UNIT_TEST_SUITE_END(); - + private: // Copy-paste of STLPort tests void Test0() { @@ -27,33 +27,33 @@ private: UNIT_ASSERT(v1.empty() == true); UNIT_ASSERT(v1.size() == 0); - + for (size_t i = 0; i < 256; ++i) { v1.resize(i + 1); UNIT_ASSERT_VALUES_EQUAL(v1.size(), i + 1); } - + for (size_t i = 256; i-- > 0;) { v1.resize(i); UNIT_ASSERT_VALUES_EQUAL(v1.size(), i); - } + } } - + void Test1() { using NPagedVector::TPagedVector; TPagedVector<int, 3> v1; // Empty vector of integers. - + UNIT_ASSERT(v1.empty() == true); UNIT_ASSERT(v1.size() == 0); - + // UNIT_ASSERT(v1.max_size() == INT_MAX / sizeof(int)); // cout << "max_size = " << v1.max_size() << endl; v1.push_back(42); // Add an integer to the vector. - + UNIT_ASSERT(v1.size() == 1); - + UNIT_ASSERT(v1[0] == 42); - + { TPagedVector<TPagedVector<int, 3>, 3> vect; vect.resize(10); @@ -63,10 +63,10 @@ private: UNIT_ASSERT((*it).empty()); UNIT_ASSERT((*it).size() == 0); UNIT_ASSERT((*it).begin() == (*it).end()); - } - } + } + } } - + void Test2() { using NPagedVector::TPagedVector; TPagedVector<double, 3> v1; // Empty vector of doubles. @@ -76,109 +76,109 @@ private: v1.push_back(33.4); TPagedVector<double, 3> v2; // Another empty vector of doubles. v2.push_back(3.56); - + UNIT_ASSERT(v1.size() == 4); UNIT_ASSERT(v1[0] == 32.1); UNIT_ASSERT(v1[1] == 40.5); UNIT_ASSERT(v1[2] == 45.5); UNIT_ASSERT(v1[3] == 33.4); - + UNIT_ASSERT(v2.size() == 1); UNIT_ASSERT(v2[0] == 3.56); v1.swap(v2); // Swap the vector's contents. - + UNIT_ASSERT(v1.size() == 1); UNIT_ASSERT(v1[0] == 3.56); - + UNIT_ASSERT(v2.size() == 4); UNIT_ASSERT(v2[0] == 32.1); UNIT_ASSERT(v2[1] == 40.5); UNIT_ASSERT(v2[2] == 45.5); UNIT_ASSERT(v2[3] == 33.4); - + v2 = v1; // Assign one vector to another. - + UNIT_ASSERT(v2.size() == 1); UNIT_ASSERT(v2[0] == 3.56); - + v2.pop_back(); UNIT_ASSERT(v2.size() == 0); UNIT_ASSERT(v2.empty()); } - + void Test3() { using NPagedVector::TPagedVector; TPagedVector<char, 1> v1; - + v1.push_back('h'); v1.push_back('i'); - + UNIT_ASSERT(v1.size() == 2); UNIT_ASSERT(v1[0] == 'h'); UNIT_ASSERT(v1[1] == 'i'); - + TPagedVector<char, 1> v2; v2.resize(v1.size()); - + for (size_t i = 0; i < v1.size(); ++i) v2[i] = v1[i]; - + v2[1] = 'o'; // Replace second character. - + UNIT_ASSERT(v2.size() == 2); UNIT_ASSERT(v2[0] == 'h'); UNIT_ASSERT(v2[1] == 'o'); - + UNIT_ASSERT((v1 == v2) == false); - + UNIT_ASSERT((v1 < v2) == true); } - + void Test4() { using NPagedVector::TPagedVector; TPagedVector<int, 3> v; v.resize(4); - + v[0] = 1; v[1] = 4; v[2] = 9; v[3] = 16; - + UNIT_ASSERT(v.front() == 1); UNIT_ASSERT(v.back() == 16); - + v.push_back(25); - + UNIT_ASSERT(v.back() == 25); UNIT_ASSERT(v.size() == 5); - + v.pop_back(); - + UNIT_ASSERT(v.back() == 16); UNIT_ASSERT(v.size() == 4); } - + void Test5() { int array[] = {1, 4, 9, 16}; - + typedef NPagedVector::TPagedVector<int, 3> TVectorType; TVectorType v(array, array + 4); - + UNIT_ASSERT(v.size() == 4); - + UNIT_ASSERT(v[0] == 1); UNIT_ASSERT(v[1] == 4); UNIT_ASSERT(v[2] == 9); UNIT_ASSERT(v[3] == 16); } - + void Test6() { int array[] = {1, 4, 9, 16, 25, 36}; - + typedef NPagedVector::TPagedVector<int, 3> TVectorType; TVectorType v(array, array + 6); TVectorType::iterator vit; - + UNIT_ASSERT_VALUES_EQUAL(v.size(), 6u); UNIT_ASSERT(v[0] == 1); UNIT_ASSERT(v[1] == 4); @@ -186,59 +186,59 @@ private: UNIT_ASSERT(v[3] == 16); UNIT_ASSERT(v[4] == 25); UNIT_ASSERT(v[5] == 36); - + vit = v.erase(v.begin()); // Erase first element. UNIT_ASSERT(*vit == 4); - + UNIT_ASSERT(v.size() == 5); UNIT_ASSERT(v[0] == 4); UNIT_ASSERT(v[1] == 9); UNIT_ASSERT(v[2] == 16); UNIT_ASSERT(v[3] == 25); UNIT_ASSERT(v[4] == 36); - + vit = v.erase(v.end() - 1); // Erase last element. UNIT_ASSERT(vit == v.end()); - + UNIT_ASSERT(v.size() == 4); UNIT_ASSERT(v[0] == 4); UNIT_ASSERT(v[1] == 9); UNIT_ASSERT(v[2] == 16); UNIT_ASSERT(v[3] == 25); - + v.erase(v.begin() + 1, v.end() - 1); // Erase all but first and last. - + UNIT_ASSERT(v.size() == 2); UNIT_ASSERT(v[0] == 4); UNIT_ASSERT(v[1] == 25); } - + void Test7() { int array1[] = {1, 4, 25}; int array2[] = {9, 16}; - + typedef NPagedVector::TPagedVector<int, 3> TVectorType; - + TVectorType v(array1, array1 + 3); TVectorType::iterator vit; vit = v.insert(v.begin(), 0); // Insert before first element. UNIT_ASSERT_VALUES_EQUAL(*vit, 0); - + vit = v.insert(v.end(), 36); // Insert after last element. UNIT_ASSERT(*vit == 36); - + UNIT_ASSERT(v.size() == 5); UNIT_ASSERT(v[0] == 0); UNIT_ASSERT(v[1] == 1); UNIT_ASSERT(v[2] == 4); UNIT_ASSERT(v[3] == 25); UNIT_ASSERT(v[4] == 36); - + // Insert contents of array2 before fourth element. v.insert(v.begin() + 3, array2, array2 + 2); - + UNIT_ASSERT(v.size() == 7); - + UNIT_ASSERT(v[0] == 0); UNIT_ASSERT(v[1] == 1); UNIT_ASSERT(v[2] == 4); @@ -246,21 +246,21 @@ private: UNIT_ASSERT(v[4] == 16); UNIT_ASSERT(v[5] == 25); UNIT_ASSERT(v[6] == 36); - + v.clear(); UNIT_ASSERT(v.empty()); } - + void TestAt() { using NPagedVector::TPagedVector; TPagedVector<int, 3> v; TPagedVector<int, 3> const& cv = v; - + v.push_back(10); UNIT_ASSERT(v.at(0) == 10); v.at(0) = 20; UNIT_ASSERT(cv.at(0) == 20); - + for (;;) { try { v.at(1) = 20; @@ -269,10 +269,10 @@ private: return; } catch (...) { UNIT_ASSERT(false); - } - } + } + } } - + void TestAutoRef() { using NPagedVector::TPagedVector; typedef TPagedVector<int, 3> TVec; @@ -280,7 +280,7 @@ private: for (int i = 0; i < 5; ++i) { ref.push_back(i); } - + TPagedVector<TVec, 3> v_v_int; v_v_int.push_back(ref); v_v_int.push_back(v_v_int[0]); @@ -288,17 +288,17 @@ private: v_v_int.push_back(v_v_int[0]); v_v_int.push_back(v_v_int[0]); v_v_int.push_back(ref); - + TPagedVector<TVec, 3>::iterator vvit(v_v_int.begin()), vvitEnd(v_v_int.end()); for (; vvit != vvitEnd; ++vvit) { UNIT_ASSERT(*vvit == ref); - } + } } - + struct Point { int x, y; }; - + struct PointEx: public Point { PointEx() : builtFromBase(false) @@ -308,42 +308,42 @@ private: : builtFromBase(true) { } - + bool builtFromBase; }; - + void TestIterators() { using NPagedVector::TPagedVector; TPagedVector<int, 3> vint; vint.resize(10); TPagedVector<int, 3> const& crvint = vint; - + UNIT_ASSERT(vint.begin() == vint.begin()); UNIT_ASSERT(crvint.begin() == vint.begin()); UNIT_ASSERT(vint.begin() == crvint.begin()); UNIT_ASSERT(crvint.begin() == crvint.begin()); - + UNIT_ASSERT(vint.begin() != vint.end()); UNIT_ASSERT(crvint.begin() != vint.end()); UNIT_ASSERT(vint.begin() != crvint.end()); UNIT_ASSERT(crvint.begin() != crvint.end()); - + UNIT_ASSERT(vint.rbegin() == vint.rbegin()); // Not Standard: //UNIT_ASSERT(vint.rbegin() == crvint.rbegin()); //UNIT_ASSERT(crvint.rbegin() == vint.rbegin()); UNIT_ASSERT(crvint.rbegin() == crvint.rbegin()); - + UNIT_ASSERT(vint.rbegin() != vint.rend()); // Not Standard: //UNIT_ASSERT(vint.rbegin() != crvint.rend()); //UNIT_ASSERT(crvint.rbegin() != vint.rend()); UNIT_ASSERT(crvint.rbegin() != crvint.rend()); } - + /* This test check a potential issue with empty base class - * optimization. Some compilers (VC6) do not implement it - * correctly resulting ina wrong behavior. */ + * optimization. Some compilers (VC6) do not implement it + * correctly resulting ina wrong behavior. */ void TestEbo() { using NPagedVector::TPagedVector; // We use heap memory as test failure can corrupt vector internal @@ -352,27 +352,27 @@ private: // by size or capacity checks. typedef TPagedVector<int, 3> V; V* pv1 = new V; - + pv1->resize(1); pv1->at(0) = 1; - + V* pv2 = new V; - + pv2->resize(10); for (int i = 0; i < 10; ++i) pv2->at(i) = 2; - + pv1->swap(*pv2); - + UNIT_ASSERT(pv1->size() == 10); UNIT_ASSERT((*pv1)[5] == 2); - + UNIT_ASSERT(pv2->size() == 1); UNIT_ASSERT((*pv2)[0] == 1); - + delete pv2; delete pv1; } -}; - -UNIT_TEST_SUITE_REGISTRATION(TPagedVectorTest); +}; + +UNIT_TEST_SUITE_REGISTRATION(TPagedVectorTest); diff --git a/library/cpp/containers/str_map/str_map.h b/library/cpp/containers/str_map/str_map.h index f734e74550..31b00d1b99 100644 --- a/library/cpp/containers/str_map/str_map.h +++ b/library/cpp/containers/str_map/str_map.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <util/memory/segmented_string_pool.h> #include <util/generic/map.h> diff --git a/library/cpp/coroutine/engine/callbacks.h b/library/cpp/coroutine/engine/callbacks.h index 640e6c25d7..e81b17344f 100644 --- a/library/cpp/coroutine/engine/callbacks.h +++ b/library/cpp/coroutine/engine/callbacks.h @@ -1,18 +1,18 @@ -#pragma once - -class TCont; -class TContExecutor; - -namespace NCoro { - class IScheduleCallback { - public: +#pragma once + +class TCont; +class TContExecutor; + +namespace NCoro { + class IScheduleCallback { + public: virtual void OnSchedule(TContExecutor&, TCont&) = 0; virtual void OnUnschedule(TContExecutor&) = 0; }; - + class IEnterPollerCallback { public: virtual void OnEnterPoller() = 0; virtual void OnExitPoller() = 0; - }; -} + }; +} diff --git a/library/cpp/coroutine/engine/condvar.h b/library/cpp/coroutine/engine/condvar.h index ecfeab5b73..ffceede6fa 100644 --- a/library/cpp/coroutine/engine/condvar.h +++ b/library/cpp/coroutine/engine/condvar.h @@ -1,11 +1,11 @@ -#pragma once +#pragma once + +#include "events.h" +#include "mutex.h" -#include "events.h" -#include "mutex.h" - class TContCondVar { public: - int WaitD(TCont* current, TContMutex* mutex, TInstant deadline) { + int WaitD(TCont* current, TContMutex* mutex, TInstant deadline) { mutex->UnLock(); const int ret = WaitQueue_.WaitD(current, deadline); @@ -17,19 +17,19 @@ public: return mutex->LockD(current, deadline); } - int WaitT(TCont* current, TContMutex* mutex, TDuration timeout) { + int WaitT(TCont* current, TContMutex* mutex, TDuration timeout) { return WaitD(current, mutex, timeout.ToDeadLine()); } - int WaitI(TCont* current, TContMutex* mutex) { + int WaitI(TCont* current, TContMutex* mutex) { return WaitD(current, mutex, TInstant::Max()); } - void Signal() noexcept { + void Signal() noexcept { WaitQueue_.Signal(); } - void BroadCast() noexcept { + void BroadCast() noexcept { WaitQueue_.BroadCast(); } diff --git a/library/cpp/coroutine/engine/cont_poller.cpp b/library/cpp/coroutine/engine/cont_poller.cpp index d801fe617f..76593d4e9b 100644 --- a/library/cpp/coroutine/engine/cont_poller.cpp +++ b/library/cpp/coroutine/engine/cont_poller.cpp @@ -1,70 +1,70 @@ -#include "cont_poller.h" +#include "cont_poller.h" #include "impl.h" -namespace NCoro { - namespace { - template <class T> - int DoExecuteEvent(T* event) noexcept { - auto* cont = event->Cont(); - - if (cont->Cancelled()) { - return ECANCELED; - } - - cont->Executor()->ScheduleIoWait(event); +namespace NCoro { + namespace { + template <class T> + int DoExecuteEvent(T* event) noexcept { + auto* cont = event->Cont(); + + if (cont->Cancelled()) { + return ECANCELED; + } + + cont->Executor()->ScheduleIoWait(event); cont->Switch(); - - if (cont->Cancelled()) { - return ECANCELED; - } - - return event->Status(); - } + + if (cont->Cancelled()) { + return ECANCELED; + } + + return event->Status(); + } + } + + void TContPollEvent::Wake() noexcept { + UnLink(); + Cont()->ReSchedule(); } - void TContPollEvent::Wake() noexcept { - UnLink(); - Cont()->ReSchedule(); - } - - - TInstant TEventWaitQueue::WakeTimedout(TInstant now) noexcept { - TIoWait::TIterator it = IoWait_.Begin(); - - if (it != IoWait_.End()) { - if (it->DeadLine() > now) { - return it->DeadLine(); - } - - do { - (it++)->Wake(ETIMEDOUT); - } while (it != IoWait_.End() && it->DeadLine() <= now); - } - - return now; - } - - void TEventWaitQueue::Register(NCoro::TContPollEvent* event) { - IoWait_.Insert(event); - event->Cont()->Unlink(); - } - - void TEventWaitQueue::Abort() noexcept { - auto visitor = [](TContPollEvent& e) { - e.Cont()->Cancel(); - }; - IoWait_.ForEach(visitor); - } + + TInstant TEventWaitQueue::WakeTimedout(TInstant now) noexcept { + TIoWait::TIterator it = IoWait_.Begin(); + + if (it != IoWait_.End()) { + if (it->DeadLine() > now) { + return it->DeadLine(); + } + + do { + (it++)->Wake(ETIMEDOUT); + } while (it != IoWait_.End() && it->DeadLine() <= now); + } + + return now; + } + + void TEventWaitQueue::Register(NCoro::TContPollEvent* event) { + IoWait_.Insert(event); + event->Cont()->Unlink(); + } + + void TEventWaitQueue::Abort() noexcept { + auto visitor = [](TContPollEvent& e) { + e.Cont()->Cancel(); + }; + IoWait_.ForEach(visitor); + } +} + +void TFdEvent::RemoveFromIOWait() noexcept { + this->Cont()->Executor()->Poller()->Remove(this); +} + +int ExecuteEvent(TFdEvent* event) noexcept { + return NCoro::DoExecuteEvent(event); +} + +int ExecuteEvent(TTimerEvent* event) noexcept { + return NCoro::DoExecuteEvent(event); } - -void TFdEvent::RemoveFromIOWait() noexcept { - this->Cont()->Executor()->Poller()->Remove(this); -} - -int ExecuteEvent(TFdEvent* event) noexcept { - return NCoro::DoExecuteEvent(event); -} - -int ExecuteEvent(TTimerEvent* event) noexcept { - return NCoro::DoExecuteEvent(event); -} diff --git a/library/cpp/coroutine/engine/cont_poller.h b/library/cpp/coroutine/engine/cont_poller.h index 824b05489a..b638b2df1a 100644 --- a/library/cpp/coroutine/engine/cont_poller.h +++ b/library/cpp/coroutine/engine/cont_poller.h @@ -1,245 +1,245 @@ -#pragma once +#pragma once #include "poller.h" -#include "sockmap.h" +#include "sockmap.h" #include <library/cpp/containers/intrusive_rb_tree/rb_tree.h> - -#include <util/datetime/base.h> -#include <util/memory/pool.h> + +#include <util/datetime/base.h> +#include <util/memory/pool.h> #include <util/memory/smallobj.h> -#include <util/network/init.h> +#include <util/network/init.h> + +#include <cerrno> + -#include <cerrno> - - class TCont; class TContExecutor; -class TFdEvent; +class TFdEvent; -namespace NCoro { - - class IPollEvent; +namespace NCoro { + class IPollEvent; - struct TContPollEventCompare { - template <class T> - static inline bool Compare(const T& l, const T& r) noexcept { - return l.DeadLine() < r.DeadLine() || (l.DeadLine() == r.DeadLine() && &l < &r); - } - }; - - class TContPollEvent : public TRbTreeItem<TContPollEvent, TContPollEventCompare> { - public: - TContPollEvent(TCont* cont, TInstant deadLine) noexcept - : Cont_(cont) - , DeadLine_(deadLine) - {} + struct TContPollEventCompare { + template <class T> + static inline bool Compare(const T& l, const T& r) noexcept { + return l.DeadLine() < r.DeadLine() || (l.DeadLine() == r.DeadLine() && &l < &r); + } + }; - static bool Compare(const TContPollEvent& l, const TContPollEvent& r) noexcept { - return l.DeadLine() < r.DeadLine() || (l.DeadLine() == r.DeadLine() && &l < &r); - } - int Status() const noexcept { - return Status_; - } + class TContPollEvent : public TRbTreeItem<TContPollEvent, TContPollEventCompare> { + public: + TContPollEvent(TCont* cont, TInstant deadLine) noexcept + : Cont_(cont) + , DeadLine_(deadLine) + {} - void SetStatus(int status) noexcept { - Status_ = status; - } + static bool Compare(const TContPollEvent& l, const TContPollEvent& r) noexcept { + return l.DeadLine() < r.DeadLine() || (l.DeadLine() == r.DeadLine() && &l < &r); + } - TCont* Cont() noexcept { - return Cont_; - } + int Status() const noexcept { + return Status_; + } + + void SetStatus(int status) noexcept { + Status_ = status; + } + + TCont* Cont() noexcept { + return Cont_; + } - TInstant DeadLine() const noexcept { - return DeadLine_; - } + TInstant DeadLine() const noexcept { + return DeadLine_; + } - void Wake(int status) noexcept { - SetStatus(status); - Wake(); + void Wake(int status) noexcept { + SetStatus(status); + Wake(); } - private: - void Wake() noexcept; + private: + void Wake() noexcept; - private: - TCont* Cont_; - TInstant DeadLine_; - int Status_ = EINPROGRESS; + private: + TCont* Cont_; + TInstant DeadLine_; + int Status_ = EINPROGRESS; }; - class IPollEvent: public TIntrusiveListItem<IPollEvent> { - public: - IPollEvent(SOCKET fd, ui16 what) noexcept - : Fd_(fd) - , What_(what) - {} + class IPollEvent: public TIntrusiveListItem<IPollEvent> { + public: + IPollEvent(SOCKET fd, ui16 what) noexcept + : Fd_(fd) + , What_(what) + {} + + virtual ~IPollEvent() {} - virtual ~IPollEvent() {} + SOCKET Fd() const noexcept { + return Fd_; + } - SOCKET Fd() const noexcept { - return Fd_; - } + int What() const noexcept { + return What_; + } - int What() const noexcept { - return What_; - } + virtual void OnPollEvent(int status) noexcept = 0; - virtual void OnPollEvent(int status) noexcept = 0; + private: + SOCKET Fd_; + ui16 What_; + }; - private: - SOCKET Fd_; - ui16 What_; - }; + template <class T> + class TBigArray { + struct TValue: public T, public TObjectFromPool<TValue> { + TValue() {} + }; - template <class T> - class TBigArray { - struct TValue: public T, public TObjectFromPool<TValue> { - TValue() {} - }; + public: + TBigArray() + : Pool_(TMemoryPool::TExpGrow::Instance(), TDefaultAllocator::Instance()) + {} - public: - TBigArray() - : Pool_(TMemoryPool::TExpGrow::Instance(), TDefaultAllocator::Instance()) - {} - - T* Get(size_t index) { - TRef& ret = Lst_.Get(index); - if (!ret) { + T* Get(size_t index) { + TRef& ret = Lst_.Get(index); + if (!ret) { ret = TRef(new (&Pool_) TValue()); - } - return ret.Get(); - } - - private: - using TRef = THolder<TValue>; - typename TValue::TPool Pool_; - TSocketMap<TRef> Lst_; - }; - - - using TPollEventList = TIntrusiveList<IPollEvent>; - - class TContPoller { - public: - using TEvent = IPollerFace::TEvent; - using TEvents = IPollerFace::TEvents; - - TContPoller() - : P_(IPollerFace::Default()) - { - } - - explicit TContPoller(THolder<IPollerFace> poller) - : P_(std::move(poller)) - {} - - void Schedule(IPollEvent* event) { - auto* lst = Lists_.Get(event->Fd()); - const ui16 oldFlags = Flags(*lst); - lst->PushFront(event); + } + return ret.Get(); + } + + private: + using TRef = THolder<TValue>; + typename TValue::TPool Pool_; + TSocketMap<TRef> Lst_; + }; + + + using TPollEventList = TIntrusiveList<IPollEvent>; + + class TContPoller { + public: + using TEvent = IPollerFace::TEvent; + using TEvents = IPollerFace::TEvents; + + TContPoller() + : P_(IPollerFace::Default()) + { + } + + explicit TContPoller(THolder<IPollerFace> poller) + : P_(std::move(poller)) + {} + + void Schedule(IPollEvent* event) { + auto* lst = Lists_.Get(event->Fd()); + const ui16 oldFlags = Flags(*lst); + lst->PushFront(event); ui16 newFlags = Flags(*lst); - if (newFlags != oldFlags) { + if (newFlags != oldFlags) { if (oldFlags) { newFlags |= CONT_POLL_MODIFY; } - P_->Set(lst, event->Fd(), newFlags); - } + P_->Set(lst, event->Fd(), newFlags); + } } - void Remove(IPollEvent* event) noexcept { - auto* lst = Lists_.Get(event->Fd()); - const ui16 oldFlags = Flags(*lst); - event->Unlink(); + void Remove(IPollEvent* event) noexcept { + auto* lst = Lists_.Get(event->Fd()); + const ui16 oldFlags = Flags(*lst); + event->Unlink(); ui16 newFlags = Flags(*lst); - if (newFlags != oldFlags) { + if (newFlags != oldFlags) { if (newFlags) { newFlags |= CONT_POLL_MODIFY; } - P_->Set(lst, event->Fd(), newFlags); - } + P_->Set(lst, event->Fd(), newFlags); + } } - void Wait(TEvents& events, TInstant deadLine) { - events.clear(); - P_->Wait(events, deadLine); + void Wait(TEvents& events, TInstant deadLine) { + events.clear(); + P_->Wait(events, deadLine); } EContPoller PollEngine() const { return P_->PollEngine(); } - private: - static ui16 Flags(TIntrusiveList<IPollEvent>& lst) noexcept { - ui16 ret = 0; - for (auto&& item : lst) { - ret |= item.What(); + private: + static ui16 Flags(TIntrusiveList<IPollEvent>& lst) noexcept { + ui16 ret = 0; + for (auto&& item : lst) { + ret |= item.What(); } - return ret; + return ret; } - private: - TBigArray<TPollEventList> Lists_; - THolder<IPollerFace> P_; + private: + TBigArray<TPollEventList> Lists_; + THolder<IPollerFace> P_; }; - - - class TEventWaitQueue { - using TIoWait = TRbTree<NCoro::TContPollEvent, NCoro::TContPollEventCompare>; - - public: - void Register(NCoro::TContPollEvent* event); - - bool Empty() const noexcept { - return IoWait_.Empty(); - } - - void Abort() noexcept; - - TInstant WakeTimedout(TInstant now) noexcept; - - private: - TIoWait IoWait_; - }; -} - -class TFdEvent final: - public NCoro::TContPollEvent, - public NCoro::IPollEvent -{ + + + class TEventWaitQueue { + using TIoWait = TRbTree<NCoro::TContPollEvent, NCoro::TContPollEventCompare>; + + public: + void Register(NCoro::TContPollEvent* event); + + bool Empty() const noexcept { + return IoWait_.Empty(); + } + + void Abort() noexcept; + + TInstant WakeTimedout(TInstant now) noexcept; + + private: + TIoWait IoWait_; + }; +} + +class TFdEvent final: + public NCoro::TContPollEvent, + public NCoro::IPollEvent +{ public: - TFdEvent(TCont* cont, SOCKET fd, ui16 what, TInstant deadLine) noexcept - : TContPollEvent(cont, deadLine) - , IPollEvent(fd, what) - {} - - ~TFdEvent() { - RemoveFromIOWait(); + TFdEvent(TCont* cont, SOCKET fd, ui16 what, TInstant deadLine) noexcept + : TContPollEvent(cont, deadLine) + , IPollEvent(fd, what) + {} + + ~TFdEvent() { + RemoveFromIOWait(); } - void RemoveFromIOWait() noexcept; + void RemoveFromIOWait() noexcept; - void OnPollEvent(int status) noexcept override { - Wake(status); + void OnPollEvent(int status) noexcept override { + Wake(status); } }; - -class TTimerEvent: public NCoro::TContPollEvent { + +class TTimerEvent: public NCoro::TContPollEvent { public: - TTimerEvent(TCont* cont, TInstant deadLine) noexcept - : TContPollEvent(cont, deadLine) - {} + TTimerEvent(TCont* cont, TInstant deadLine) noexcept + : TContPollEvent(cont, deadLine) + {} }; - -int ExecuteEvent(TFdEvent* event) noexcept; - -int ExecuteEvent(TTimerEvent* event) noexcept; + +int ExecuteEvent(TFdEvent* event) noexcept; + +int ExecuteEvent(TTimerEvent* event) noexcept; diff --git a/library/cpp/coroutine/engine/coroutine_ut.cpp b/library/cpp/coroutine/engine/coroutine_ut.cpp index 1ba31245b0..8b372496a2 100644 --- a/library/cpp/coroutine/engine/coroutine_ut.cpp +++ b/library/cpp/coroutine/engine/coroutine_ut.cpp @@ -1,19 +1,19 @@ #include "impl.h" -#include "condvar.h" -#include "network.h" +#include "condvar.h" +#include "network.h" #include <library/cpp/testing/unittest/registar.h> #include <util/string/cast.h> #include <util/system/pipe.h> -#include <util/system/env.h> -#include <util/system/info.h> +#include <util/system/env.h> +#include <util/system/info.h> #include <util/system/thread.h> -#include <util/generic/xrange.h> +#include <util/generic/xrange.h> #include <util/generic/serialized_enum.h> -// TODO (velavokr): BALANCER-1345 add more tests on pollers - +// TODO (velavokr): BALANCER-1345 add more tests on pollers + class TCoroTest: public TTestBase { UNIT_TEST_SUITE(TCoroTest); UNIT_TEST(TestSimpleX1); @@ -23,23 +23,23 @@ class TCoroTest: public TTestBase { UNIT_TEST(TestMemFun); UNIT_TEST(TestMutex); UNIT_TEST(TestCondVar); - UNIT_TEST(TestJoinDefault); - UNIT_TEST(TestJoinEpoll); - UNIT_TEST(TestJoinKqueue); - UNIT_TEST(TestJoinPoll); - UNIT_TEST(TestJoinSelect); + UNIT_TEST(TestJoinDefault); + UNIT_TEST(TestJoinEpoll); + UNIT_TEST(TestJoinKqueue); + UNIT_TEST(TestJoinPoll); + UNIT_TEST(TestJoinSelect); UNIT_TEST(TestException); - UNIT_TEST(TestJoinCancelExitRaceBug); - UNIT_TEST(TestWaitWakeLivelockBug); - UNIT_TEST(TestFastPathWakeDefault) - // TODO (velavokr): BALANCER-1338 our epoll wrapper cannot handle pipe eofs -// UNIT_TEST(TestFastPathWakeEpoll) - UNIT_TEST(TestFastPathWakeKqueue) - UNIT_TEST(TestFastPathWakePoll) + UNIT_TEST(TestJoinCancelExitRaceBug); + UNIT_TEST(TestWaitWakeLivelockBug); + UNIT_TEST(TestFastPathWakeDefault) + // TODO (velavokr): BALANCER-1338 our epoll wrapper cannot handle pipe eofs +// UNIT_TEST(TestFastPathWakeEpoll) + UNIT_TEST(TestFastPathWakeKqueue) + UNIT_TEST(TestFastPathWakePoll) UNIT_TEST(TestFastPathWakeSelect) - UNIT_TEST(TestLegacyCancelYieldRaceBug) - UNIT_TEST(TestJoinRescheduleBug); - UNIT_TEST(TestEventQueue) + UNIT_TEST(TestLegacyCancelYieldRaceBug) + UNIT_TEST(TestJoinRescheduleBug); + UNIT_TEST(TestEventQueue) UNIT_TEST(TestNestedExecutor) UNIT_TEST(TestComputeCoroutineYield) UNIT_TEST(TestPollEngines); @@ -57,21 +57,21 @@ public: void TestMemFun(); void TestMutex(); void TestCondVar(); - void TestJoinDefault(); - void TestJoinEpoll(); - void TestJoinKqueue(); - void TestJoinPoll(); - void TestJoinSelect(); - void TestJoinCancelExitRaceBug(); - void TestWaitWakeLivelockBug(); - void TestFastPathWakeDefault(); - void TestFastPathWakeEpoll(); - void TestFastPathWakeKqueue(); - void TestFastPathWakePoll(); - void TestFastPathWakeSelect(); - void TestLegacyCancelYieldRaceBug(); - void TestJoinRescheduleBug(); - void TestEventQueue(); + void TestJoinDefault(); + void TestJoinEpoll(); + void TestJoinKqueue(); + void TestJoinPoll(); + void TestJoinSelect(); + void TestJoinCancelExitRaceBug(); + void TestWaitWakeLivelockBug(); + void TestFastPathWakeDefault(); + void TestFastPathWakeEpoll(); + void TestFastPathWakeKqueue(); + void TestFastPathWakePoll(); + void TestFastPathWakeSelect(); + void TestLegacyCancelYieldRaceBug(); + void TestJoinRescheduleBug(); + void TestEventQueue(); void TestNestedExecutor(); void TestComputeCoroutineYield(); void TestPollEngines(); @@ -88,7 +88,7 @@ void TCoroTest::TestException() { auto f1 = [&f2run](TCont* c) { struct TCtx { ~TCtx() { - Y_VERIFY(!*F2); + Y_VERIFY(!*F2); C->Yield(); } @@ -144,18 +144,18 @@ static void CoMain(TCont* c, void* /*arg*/) { } } -void TCoroTest::TestSimpleX1() { - i0 = 0; - TContExecutor e(32000); +void TCoroTest::TestSimpleX1() { + i0 = 0; + TContExecutor e(32000); UNIT_ASSERT(RunningCont() == nullptr); - e.Execute(CoMain); - UNIT_ASSERT_VALUES_EQUAL(i0, 100000); + e.Execute(CoMain); + UNIT_ASSERT_VALUES_EQUAL(i0, 100000); UNIT_ASSERT(RunningCont() == nullptr); -} - +} + void TCoroTest::TestSimpleX1MultiThread() { TVector<THolder<TThread>> threads; const size_t nThreads = 0; @@ -179,10 +179,10 @@ void TCoroTest::TestSimpleX1MultiThread() { } struct TTestObject { - int i = 0; - int j = 0; + int i = 0; + int j = 0; -public: +public: void RunTask1(TCont*) { i = 1; } @@ -325,7 +325,7 @@ void TCoroTest::TestCondVar() { res.clear(); } -namespace NCoroTestJoin { +namespace NCoroTestJoin { struct TSleepCont { const TInstant Deadline; int Result; @@ -342,7 +342,7 @@ namespace NCoroTestJoin { inline void operator()(TCont* c) { char buf = 0; - Result = NCoro::ReadD(c, Sock, &buf, sizeof(buf), Deadline).Status(); + Result = NCoro::ReadD(c, Sock, &buf, sizeof(buf), Deadline).Status(); } }; @@ -356,513 +356,513 @@ namespace NCoroTestJoin { } }; - void DoTestJoin(EContPoller pollerType) { - auto poller = IPollerFace::Construct(pollerType); - - if (!poller) { - return; - } - - TContExecutor e(32000, std::move(poller)); - - TPipe in, out; - TPipe::Pipe(in, out); - SetNonBlock(in.GetHandle()); - - { - TSleepCont sc = {TInstant::Max(), 0}; - TJoinCont jc = {TDuration::MilliSeconds(100).ToDeadLine(), e.Create(sc, "sc"), true}; - - e.Execute(jc); - - UNIT_ASSERT_EQUAL(sc.Result, ECANCELED); - UNIT_ASSERT_EQUAL(jc.Result, false); - } - - { - TSleepCont sc = {TDuration::MilliSeconds(100).ToDeadLine(), 0}; - TJoinCont jc = {TDuration::MilliSeconds(200).ToDeadLine(), e.Create(sc, "sc"), false}; - - e.Execute(jc); - - UNIT_ASSERT_EQUAL(sc.Result, ETIMEDOUT); - UNIT_ASSERT_EQUAL(jc.Result, true); - } - - { - TSleepCont sc = {TDuration::MilliSeconds(200).ToDeadLine(), 0}; - TJoinCont jc = {TDuration::MilliSeconds(100).ToDeadLine(), e.Create(sc, "sc"), true}; - - e.Execute(jc); - - UNIT_ASSERT_EQUAL(sc.Result, ECANCELED); - UNIT_ASSERT_EQUAL(jc.Result, false); - } - - { - TReadCont rc = {TInstant::Max(), in.GetHandle(), 0}; - TJoinCont jc = {TDuration::MilliSeconds(100).ToDeadLine(), e.Create(rc, "rc"), true}; - - e.Execute(jc); - - UNIT_ASSERT_EQUAL(rc.Result, ECANCELED); - UNIT_ASSERT_EQUAL(jc.Result, false); - } - - { - TReadCont rc = {TDuration::MilliSeconds(100).ToDeadLine(), in.GetHandle(), 0}; - TJoinCont jc = {TDuration::MilliSeconds(200).ToDeadLine(), e.Create(rc, "rc"), false}; - - e.Execute(jc); - - UNIT_ASSERT_EQUAL(rc.Result, ETIMEDOUT); - UNIT_ASSERT_EQUAL(jc.Result, true); - } - - { - TReadCont rc = {TDuration::MilliSeconds(200).ToDeadLine(), in.GetHandle(), 0}; - TJoinCont jc = {TDuration::MilliSeconds(100).ToDeadLine(), e.Create(rc, "rc"), true}; - - e.Execute(jc); - - UNIT_ASSERT_EQUAL(rc.Result, ECANCELED); - UNIT_ASSERT_EQUAL(jc.Result, false); - } - } -} - -void TCoroTest::TestJoinDefault() { - NCoroTestJoin::DoTestJoin(EContPoller::Default); -} - -void TCoroTest::TestJoinEpoll() { - NCoroTestJoin::DoTestJoin(EContPoller::Epoll); -} - -void TCoroTest::TestJoinKqueue() { - NCoroTestJoin::DoTestJoin(EContPoller::Kqueue); -} - -void TCoroTest::TestJoinPoll() { - NCoroTestJoin::DoTestJoin(EContPoller::Poll); -} - -void TCoroTest::TestJoinSelect() { - NCoroTestJoin::DoTestJoin(EContPoller::Select); -} - -namespace NCoroJoinCancelExitRaceBug { - struct TState { - TCont* Sub = nullptr; - }; - - static void DoAux(TCont*, void* argPtr) noexcept { - TState& state = *(TState*)(argPtr); - - // 06.{Ready:[Sub2]} > {Ready:[Sub2,Sub]} - state.Sub->Cancel(); - } - - static void DoSub2(TCont*, void*) noexcept { - // 07.{Ready:[Sub]} > Exit > {Ready:[Sub],ToDelete:[Sub2]} - // 08.{Ready:[Sub],ToDelete:[Sub2]} > Release(Sub2) > {Ready:[Sub],Deleted:[Sub2]} - } - - static void DoSub(TCont* cont, void* argPtr) noexcept { - TState& state = *(TState*)(argPtr); - state.Sub = cont; - - // 04.{Ready:[Aux]} > {Ready:[Aux,Sub2]} - auto* sub2 = cont->Executor()->Create(DoSub2, argPtr, "Sub2"); - - // 05.{Ready:[Aux,Sub2]} > SwitchTo(Aux) - // 09.{Ready:[],Deleted:[Sub2]} > Cancel(Sub2) > {Ready:[Sub2],Deleted:[Sub2]} - // 10.{Ready:[Sub2],Deleted:[Sub2]} > SwitchTo(Sub2) > FAIL: can not return from exit - cont->Join(sub2); - - state.Sub = nullptr; - } - - static void DoMain(TCont* cont) noexcept { - TState state; - - // 01.{Ready:[]} > {Ready:[Sub]} - auto* sub = cont->Executor()->Create(DoSub, &state, "Sub"); - - // 02.{Ready:[Sub]} > {Ready:[Sub,Aux]} - cont->Executor()->Create(DoAux, &state, "Aux"); - - // 03.{Ready:[Sub,Aux]} > SwitchTo(Sub) - cont->Join(sub); - } -} - -void TCoroTest::TestJoinCancelExitRaceBug() { - TContExecutor exec(20000); - exec.SetFailOnError(true); - exec.Execute(NCoroJoinCancelExitRaceBug::DoMain); -} - -namespace NCoroWaitWakeLivelockBug { - struct TState; - - struct TSubState { - TSubState(TState& parent, ui32 self) - : Parent(parent) + void DoTestJoin(EContPoller pollerType) { + auto poller = IPollerFace::Construct(pollerType); + + if (!poller) { + return; + } + + TContExecutor e(32000, std::move(poller)); + + TPipe in, out; + TPipe::Pipe(in, out); + SetNonBlock(in.GetHandle()); + + { + TSleepCont sc = {TInstant::Max(), 0}; + TJoinCont jc = {TDuration::MilliSeconds(100).ToDeadLine(), e.Create(sc, "sc"), true}; + + e.Execute(jc); + + UNIT_ASSERT_EQUAL(sc.Result, ECANCELED); + UNIT_ASSERT_EQUAL(jc.Result, false); + } + + { + TSleepCont sc = {TDuration::MilliSeconds(100).ToDeadLine(), 0}; + TJoinCont jc = {TDuration::MilliSeconds(200).ToDeadLine(), e.Create(sc, "sc"), false}; + + e.Execute(jc); + + UNIT_ASSERT_EQUAL(sc.Result, ETIMEDOUT); + UNIT_ASSERT_EQUAL(jc.Result, true); + } + + { + TSleepCont sc = {TDuration::MilliSeconds(200).ToDeadLine(), 0}; + TJoinCont jc = {TDuration::MilliSeconds(100).ToDeadLine(), e.Create(sc, "sc"), true}; + + e.Execute(jc); + + UNIT_ASSERT_EQUAL(sc.Result, ECANCELED); + UNIT_ASSERT_EQUAL(jc.Result, false); + } + + { + TReadCont rc = {TInstant::Max(), in.GetHandle(), 0}; + TJoinCont jc = {TDuration::MilliSeconds(100).ToDeadLine(), e.Create(rc, "rc"), true}; + + e.Execute(jc); + + UNIT_ASSERT_EQUAL(rc.Result, ECANCELED); + UNIT_ASSERT_EQUAL(jc.Result, false); + } + + { + TReadCont rc = {TDuration::MilliSeconds(100).ToDeadLine(), in.GetHandle(), 0}; + TJoinCont jc = {TDuration::MilliSeconds(200).ToDeadLine(), e.Create(rc, "rc"), false}; + + e.Execute(jc); + + UNIT_ASSERT_EQUAL(rc.Result, ETIMEDOUT); + UNIT_ASSERT_EQUAL(jc.Result, true); + } + + { + TReadCont rc = {TDuration::MilliSeconds(200).ToDeadLine(), in.GetHandle(), 0}; + TJoinCont jc = {TDuration::MilliSeconds(100).ToDeadLine(), e.Create(rc, "rc"), true}; + + e.Execute(jc); + + UNIT_ASSERT_EQUAL(rc.Result, ECANCELED); + UNIT_ASSERT_EQUAL(jc.Result, false); + } + } +} + +void TCoroTest::TestJoinDefault() { + NCoroTestJoin::DoTestJoin(EContPoller::Default); +} + +void TCoroTest::TestJoinEpoll() { + NCoroTestJoin::DoTestJoin(EContPoller::Epoll); +} + +void TCoroTest::TestJoinKqueue() { + NCoroTestJoin::DoTestJoin(EContPoller::Kqueue); +} + +void TCoroTest::TestJoinPoll() { + NCoroTestJoin::DoTestJoin(EContPoller::Poll); +} + +void TCoroTest::TestJoinSelect() { + NCoroTestJoin::DoTestJoin(EContPoller::Select); +} + +namespace NCoroJoinCancelExitRaceBug { + struct TState { + TCont* Sub = nullptr; + }; + + static void DoAux(TCont*, void* argPtr) noexcept { + TState& state = *(TState*)(argPtr); + + // 06.{Ready:[Sub2]} > {Ready:[Sub2,Sub]} + state.Sub->Cancel(); + } + + static void DoSub2(TCont*, void*) noexcept { + // 07.{Ready:[Sub]} > Exit > {Ready:[Sub],ToDelete:[Sub2]} + // 08.{Ready:[Sub],ToDelete:[Sub2]} > Release(Sub2) > {Ready:[Sub],Deleted:[Sub2]} + } + + static void DoSub(TCont* cont, void* argPtr) noexcept { + TState& state = *(TState*)(argPtr); + state.Sub = cont; + + // 04.{Ready:[Aux]} > {Ready:[Aux,Sub2]} + auto* sub2 = cont->Executor()->Create(DoSub2, argPtr, "Sub2"); + + // 05.{Ready:[Aux,Sub2]} > SwitchTo(Aux) + // 09.{Ready:[],Deleted:[Sub2]} > Cancel(Sub2) > {Ready:[Sub2],Deleted:[Sub2]} + // 10.{Ready:[Sub2],Deleted:[Sub2]} > SwitchTo(Sub2) > FAIL: can not return from exit + cont->Join(sub2); + + state.Sub = nullptr; + } + + static void DoMain(TCont* cont) noexcept { + TState state; + + // 01.{Ready:[]} > {Ready:[Sub]} + auto* sub = cont->Executor()->Create(DoSub, &state, "Sub"); + + // 02.{Ready:[Sub]} > {Ready:[Sub,Aux]} + cont->Executor()->Create(DoAux, &state, "Aux"); + + // 03.{Ready:[Sub,Aux]} > SwitchTo(Sub) + cont->Join(sub); + } +} + +void TCoroTest::TestJoinCancelExitRaceBug() { + TContExecutor exec(20000); + exec.SetFailOnError(true); + exec.Execute(NCoroJoinCancelExitRaceBug::DoMain); +} + +namespace NCoroWaitWakeLivelockBug { + struct TState; + + struct TSubState { + TSubState(TState& parent, ui32 self) + : Parent(parent) , Name(TStringBuilder() << "Sub" << self) - , Self(self) - { - UNIT_ASSERT(self < 2); - } - - TSubState& OtherState(); - - TState& Parent; - TTimerEvent* Event = nullptr; - TCont* Cont = nullptr; - TString Name; - ui32 Self = -1; - }; - - struct TState { - TState() - : Subs{{*this, 0}, {*this, 1}} - {} - - TSubState Subs[2]; - bool Stop = false; - }; - - TSubState& TSubState::OtherState() { - return Parent.Subs[1 - Self]; - } - - static void DoStop(TCont* cont, void* argPtr) { - TState& state = *(TState*)(argPtr); - - TTimerEvent event(cont, TInstant::Now()); - ExecuteEvent(&event); - state.Stop = true; - for (auto& sub: state.Subs) { - if (sub.Event) { - sub.Event->Wake(EWAKEDUP); - } - } - } - - static void DoSub(TCont* cont, void* argPtr) { - TSubState& state = *(TSubState*)(argPtr); - - while (!state.Parent.Stop) { - TTimerEvent event(cont, TInstant::Max()); - if (state.OtherState().Event) { - state.OtherState().Event->Wake(EWAKEDUP); - } - state.Event = &event; - ExecuteEvent(&event); - state.Event = nullptr; - } - - state.Cont = nullptr; - } - - static void DoMain(TCont* cont) noexcept { - TState state; - - for (auto& subState : state.Subs) { - subState.Cont = cont->Executor()->Create(DoSub, &subState, subState.Name.data()); - } - - cont->Join( - cont->Executor()->Create(DoStop, &state, "Stop") - ); - - for (auto& subState : state.Subs) { - if (subState.Cont) { - cont->Join(subState.Cont); - } - } - } -} - -void TCoroTest::TestWaitWakeLivelockBug() { - TContExecutor exec(20000); - exec.SetFailOnError(true); - exec.Execute(NCoroWaitWakeLivelockBug::DoMain); -} - -namespace NCoroTestFastPathWake { - struct TState; - - struct TSubState { - TSubState(TState& parent, ui32 self) - : Parent(parent) - , Name(TStringBuilder() << "Sub" << self) - {} - - TState& Parent; - TInstant Finish; - TTimerEvent* Event = nullptr; - TCont* Cont = nullptr; - TString Name; - }; - - struct TState { - TState() - : Subs{{*this, 0}, {*this, 1}} - { - TPipe::Pipe(In, Out); - SetNonBlock(In.GetHandle()); - } - - TSubState Subs[2]; - TPipe In, Out; - bool IoSleepRunning = false; - }; - - static void DoIoSleep(TCont* cont, void* argPtr) noexcept { - try { - TState& state = *(TState*) (argPtr); - state.IoSleepRunning = true; - - TTempBuf tmp; - // Wait for the event from io - auto res = NCoro::ReadD(cont, state.In.GetHandle(), tmp.Data(), 1, TDuration::Seconds(10).ToDeadLine()); - UNIT_ASSERT_VALUES_EQUAL(res.Checked(), 0); - state.IoSleepRunning = false; - } catch (const NUnitTest::TAssertException& ex) { - Cerr << ex.AsStrBuf() << Endl; - ex.BackTrace()->PrintTo(Cerr); - throw; - } catch (...) { - Cerr << CurrentExceptionMessage() << Endl; - throw; - } - } - - static void DoSub(TCont* cont, void* argPtr) noexcept { - TSubState& state = *(TSubState*)(argPtr); - - TTimerEvent event(cont, TInstant::Max()); - state.Event = &event; - ExecuteEvent(&event); - state.Event = nullptr; - state.Cont = nullptr; - state.Finish = TInstant::Now(); - } - - static void DoMain(TCont* cont) noexcept { - try { - TState state; - TInstant start = TInstant::Now(); - - // This guy sleeps on io - auto sleeper = cont->Executor()->Create(DoIoSleep, &state, "io_sleeper"); - - // These guys are to be woken up right away - for (auto& subState : state.Subs) { - subState.Cont = cont->Executor()->Create(DoSub, &subState, subState.Name.data()); - } - - // Give way - cont->Yield(); - - // Check everyone has started, wake those to be woken - UNIT_ASSERT(state.IoSleepRunning); - - for (auto& subState : state.Subs) { - UNIT_ASSERT(subState.Event); - subState.Event->Wake(EWAKEDUP); - } - - // Give way again - cont->Yield(); - - // Check the woken guys have finished and quite soon - for (auto& subState : state.Subs) { - UNIT_ASSERT(subState.Finish - start < TDuration::MilliSeconds(100)); - UNIT_ASSERT(!subState.Cont); - } - - // Wake the io guy and finish - state.Out.Close(); - - if (state.IoSleepRunning) { - cont->Join(sleeper); - } - - // Check everything has ended sooner than the timeout - UNIT_ASSERT(TInstant::Now() - start < TDuration::Seconds(1)); - } catch (const NUnitTest::TAssertException& ex) { - Cerr << ex.AsStrBuf() << Endl; - ex.BackTrace()->PrintTo(Cerr); - throw; - } catch (...) { - Cerr << CurrentExceptionMessage() << Endl; - throw; - } - } - - static void DoTestFastPathWake(EContPoller pollerType) { - if (auto poller = IPollerFace::Construct(pollerType)) { - TContExecutor exec(20000, std::move(poller)); - exec.SetFailOnError(true); - exec.Execute(NCoroTestFastPathWake::DoMain); - } - } -} - -void TCoroTest::TestFastPathWakeDefault() { - NCoroTestFastPathWake::DoTestFastPathWake(EContPoller::Default); -} - -void TCoroTest::TestFastPathWakeEpoll() { - NCoroTestFastPathWake::DoTestFastPathWake(EContPoller::Epoll); -} - -void TCoroTest::TestFastPathWakeKqueue() { - NCoroTestFastPathWake::DoTestFastPathWake(EContPoller::Kqueue); -} - -void TCoroTest::TestFastPathWakePoll() { - NCoroTestFastPathWake::DoTestFastPathWake(EContPoller::Poll); -} - -void TCoroTest::TestFastPathWakeSelect() { - NCoroTestFastPathWake::DoTestFastPathWake(EContPoller::Select); -} - -namespace NCoroTestLegacyCancelYieldRaceBug { - enum class EState { - Idle, Running, Finished, - }; - - struct TState { - EState SubState = EState::Idle; - }; - - static void DoSub(TCont* cont, void* argPtr) { - TState& state = *(TState*)argPtr; - state.SubState = EState::Running; - cont->Yield(); - cont->Yield(); - state.SubState = EState::Finished; - } - - static void DoMain(TCont* cont, void* argPtr) { - TState& state = *(TState*)argPtr; - TCont* sub = cont->Executor()->Create(DoSub, argPtr, "Sub"); - sub->Cancel(); - cont->Yield(); - UNIT_ASSERT_EQUAL(state.SubState, EState::Finished); - } -} - -void TCoroTest::TestLegacyCancelYieldRaceBug() { - NCoroTestLegacyCancelYieldRaceBug::TState state; - TContExecutor exec(20000); - exec.SetFailOnError(true); - exec.Execute(NCoroTestLegacyCancelYieldRaceBug::DoMain, &state); -} - -namespace NCoroTestJoinRescheduleBug { - enum class EState { - Idle, Running, Finished, - }; - - struct TState { - TCont* volatile SubA = nullptr; - volatile EState SubAState = EState::Idle; - volatile EState SubBState = EState::Idle; - volatile EState SubCState = EState::Idle; - }; - - static void DoSubC(TCont* cont, void* argPtr) { - TState& state = *(TState*)argPtr; - state.SubCState = EState::Running; - while (state.SubBState != EState::Running) { - cont->Yield(); - } - while (cont->SleepD(TInstant::Max()) != ECANCELED) { - } - state.SubCState = EState::Finished; - } - - static void DoSubB(TCont* cont, void* argPtr) { - TState& state = *(TState*)argPtr; - state.SubBState = EState::Running; - while (state.SubAState != EState::Running && state.SubCState != EState::Running) { - cont->Yield(); - } - for (auto i : xrange(100)) { - Y_UNUSED(i); - if (!state.SubA) { - break; - } - state.SubA->ReSchedule(); - cont->Yield(); - } - state.SubBState = EState::Finished; - } - - static void DoSubA(TCont* cont, void* argPtr) { - TState& state = *(TState*)argPtr; - state.SubAState = EState::Running; - TCont* subC = cont->Executor()->Create(DoSubC, argPtr, "SubC"); - while (state.SubBState != EState::Running && state.SubCState != EState::Running) { - cont->Yield(); - } - cont->Join(subC); - UNIT_ASSERT_EQUAL(state.SubCState, EState::Finished); - state.SubA = nullptr; - state.SubAState = EState::Finished; - } - - static void DoMain(TCont* cont, void* argPtr) { - TState& state = *(TState*)argPtr; - TCont* subA = cont->Executor()->Create(DoSubA, argPtr, "SubA"); - state.SubA = subA; - cont->Join(cont->Executor()->Create(DoSubB, argPtr, "SubB")); - - if (state.SubA) { - subA->Cancel(); - cont->Join(subA); - } - } -} - -void TCoroTest::TestJoinRescheduleBug() { - using namespace NCoroTestJoinRescheduleBug; - TState state; - { - TContExecutor exec(20000); - exec.Execute(DoMain, &state); - } - UNIT_ASSERT_EQUAL(state.SubAState, EState::Finished); - UNIT_ASSERT_EQUAL(state.SubBState, EState::Finished); - UNIT_ASSERT_EQUAL(state.SubCState, EState::Finished); -} - -void TCoroTest::TestEventQueue() { - NCoro::TEventWaitQueue queue; - UNIT_ASSERT(queue.Empty()); - UNIT_ASSERT_VALUES_EQUAL(queue.WakeTimedout(TInstant()), TInstant()); - TContExecutor exec(32000); - exec.Execute([](TCont* cont, void* arg) { - NCoro::TEventWaitQueue* q = (NCoro::TEventWaitQueue*)arg; - TTimerEvent ev(cont, TInstant::Max()); - TTimerEvent ev2(cont, TInstant::Seconds(12345)); - q->Register(&ev); - UNIT_ASSERT_VALUES_EQUAL(q->WakeTimedout(TInstant::Seconds(12344)), TInstant::Max()); - UNIT_ASSERT_VALUES_EQUAL(q->WakeTimedout(TInstant::Seconds(12344)), TInstant::Max()); - q->Register(&ev2); - UNIT_ASSERT_VALUES_EQUAL(q->WakeTimedout(TInstant::Seconds(12344)), TInstant::Seconds(12345)); - UNIT_ASSERT_VALUES_EQUAL(q->WakeTimedout(TInstant::Seconds(12344)), TInstant::Seconds(12345)); - UNIT_ASSERT_VALUES_EQUAL(q->WakeTimedout(TInstant::Seconds(12345)), TInstant::Seconds(12345)); - UNIT_ASSERT_VALUES_EQUAL(q->WakeTimedout(TInstant::Seconds(12345)), TInstant::Max()); - }, &queue); -} - + , Self(self) + { + UNIT_ASSERT(self < 2); + } + + TSubState& OtherState(); + + TState& Parent; + TTimerEvent* Event = nullptr; + TCont* Cont = nullptr; + TString Name; + ui32 Self = -1; + }; + + struct TState { + TState() + : Subs{{*this, 0}, {*this, 1}} + {} + + TSubState Subs[2]; + bool Stop = false; + }; + + TSubState& TSubState::OtherState() { + return Parent.Subs[1 - Self]; + } + + static void DoStop(TCont* cont, void* argPtr) { + TState& state = *(TState*)(argPtr); + + TTimerEvent event(cont, TInstant::Now()); + ExecuteEvent(&event); + state.Stop = true; + for (auto& sub: state.Subs) { + if (sub.Event) { + sub.Event->Wake(EWAKEDUP); + } + } + } + + static void DoSub(TCont* cont, void* argPtr) { + TSubState& state = *(TSubState*)(argPtr); + + while (!state.Parent.Stop) { + TTimerEvent event(cont, TInstant::Max()); + if (state.OtherState().Event) { + state.OtherState().Event->Wake(EWAKEDUP); + } + state.Event = &event; + ExecuteEvent(&event); + state.Event = nullptr; + } + + state.Cont = nullptr; + } + + static void DoMain(TCont* cont) noexcept { + TState state; + + for (auto& subState : state.Subs) { + subState.Cont = cont->Executor()->Create(DoSub, &subState, subState.Name.data()); + } + + cont->Join( + cont->Executor()->Create(DoStop, &state, "Stop") + ); + + for (auto& subState : state.Subs) { + if (subState.Cont) { + cont->Join(subState.Cont); + } + } + } +} + +void TCoroTest::TestWaitWakeLivelockBug() { + TContExecutor exec(20000); + exec.SetFailOnError(true); + exec.Execute(NCoroWaitWakeLivelockBug::DoMain); +} + +namespace NCoroTestFastPathWake { + struct TState; + + struct TSubState { + TSubState(TState& parent, ui32 self) + : Parent(parent) + , Name(TStringBuilder() << "Sub" << self) + {} + + TState& Parent; + TInstant Finish; + TTimerEvent* Event = nullptr; + TCont* Cont = nullptr; + TString Name; + }; + + struct TState { + TState() + : Subs{{*this, 0}, {*this, 1}} + { + TPipe::Pipe(In, Out); + SetNonBlock(In.GetHandle()); + } + + TSubState Subs[2]; + TPipe In, Out; + bool IoSleepRunning = false; + }; + + static void DoIoSleep(TCont* cont, void* argPtr) noexcept { + try { + TState& state = *(TState*) (argPtr); + state.IoSleepRunning = true; + + TTempBuf tmp; + // Wait for the event from io + auto res = NCoro::ReadD(cont, state.In.GetHandle(), tmp.Data(), 1, TDuration::Seconds(10).ToDeadLine()); + UNIT_ASSERT_VALUES_EQUAL(res.Checked(), 0); + state.IoSleepRunning = false; + } catch (const NUnitTest::TAssertException& ex) { + Cerr << ex.AsStrBuf() << Endl; + ex.BackTrace()->PrintTo(Cerr); + throw; + } catch (...) { + Cerr << CurrentExceptionMessage() << Endl; + throw; + } + } + + static void DoSub(TCont* cont, void* argPtr) noexcept { + TSubState& state = *(TSubState*)(argPtr); + + TTimerEvent event(cont, TInstant::Max()); + state.Event = &event; + ExecuteEvent(&event); + state.Event = nullptr; + state.Cont = nullptr; + state.Finish = TInstant::Now(); + } + + static void DoMain(TCont* cont) noexcept { + try { + TState state; + TInstant start = TInstant::Now(); + + // This guy sleeps on io + auto sleeper = cont->Executor()->Create(DoIoSleep, &state, "io_sleeper"); + + // These guys are to be woken up right away + for (auto& subState : state.Subs) { + subState.Cont = cont->Executor()->Create(DoSub, &subState, subState.Name.data()); + } + + // Give way + cont->Yield(); + + // Check everyone has started, wake those to be woken + UNIT_ASSERT(state.IoSleepRunning); + + for (auto& subState : state.Subs) { + UNIT_ASSERT(subState.Event); + subState.Event->Wake(EWAKEDUP); + } + + // Give way again + cont->Yield(); + + // Check the woken guys have finished and quite soon + for (auto& subState : state.Subs) { + UNIT_ASSERT(subState.Finish - start < TDuration::MilliSeconds(100)); + UNIT_ASSERT(!subState.Cont); + } + + // Wake the io guy and finish + state.Out.Close(); + + if (state.IoSleepRunning) { + cont->Join(sleeper); + } + + // Check everything has ended sooner than the timeout + UNIT_ASSERT(TInstant::Now() - start < TDuration::Seconds(1)); + } catch (const NUnitTest::TAssertException& ex) { + Cerr << ex.AsStrBuf() << Endl; + ex.BackTrace()->PrintTo(Cerr); + throw; + } catch (...) { + Cerr << CurrentExceptionMessage() << Endl; + throw; + } + } + + static void DoTestFastPathWake(EContPoller pollerType) { + if (auto poller = IPollerFace::Construct(pollerType)) { + TContExecutor exec(20000, std::move(poller)); + exec.SetFailOnError(true); + exec.Execute(NCoroTestFastPathWake::DoMain); + } + } +} + +void TCoroTest::TestFastPathWakeDefault() { + NCoroTestFastPathWake::DoTestFastPathWake(EContPoller::Default); +} + +void TCoroTest::TestFastPathWakeEpoll() { + NCoroTestFastPathWake::DoTestFastPathWake(EContPoller::Epoll); +} + +void TCoroTest::TestFastPathWakeKqueue() { + NCoroTestFastPathWake::DoTestFastPathWake(EContPoller::Kqueue); +} + +void TCoroTest::TestFastPathWakePoll() { + NCoroTestFastPathWake::DoTestFastPathWake(EContPoller::Poll); +} + +void TCoroTest::TestFastPathWakeSelect() { + NCoroTestFastPathWake::DoTestFastPathWake(EContPoller::Select); +} + +namespace NCoroTestLegacyCancelYieldRaceBug { + enum class EState { + Idle, Running, Finished, + }; + + struct TState { + EState SubState = EState::Idle; + }; + + static void DoSub(TCont* cont, void* argPtr) { + TState& state = *(TState*)argPtr; + state.SubState = EState::Running; + cont->Yield(); + cont->Yield(); + state.SubState = EState::Finished; + } + + static void DoMain(TCont* cont, void* argPtr) { + TState& state = *(TState*)argPtr; + TCont* sub = cont->Executor()->Create(DoSub, argPtr, "Sub"); + sub->Cancel(); + cont->Yield(); + UNIT_ASSERT_EQUAL(state.SubState, EState::Finished); + } +} + +void TCoroTest::TestLegacyCancelYieldRaceBug() { + NCoroTestLegacyCancelYieldRaceBug::TState state; + TContExecutor exec(20000); + exec.SetFailOnError(true); + exec.Execute(NCoroTestLegacyCancelYieldRaceBug::DoMain, &state); +} + +namespace NCoroTestJoinRescheduleBug { + enum class EState { + Idle, Running, Finished, + }; + + struct TState { + TCont* volatile SubA = nullptr; + volatile EState SubAState = EState::Idle; + volatile EState SubBState = EState::Idle; + volatile EState SubCState = EState::Idle; + }; + + static void DoSubC(TCont* cont, void* argPtr) { + TState& state = *(TState*)argPtr; + state.SubCState = EState::Running; + while (state.SubBState != EState::Running) { + cont->Yield(); + } + while (cont->SleepD(TInstant::Max()) != ECANCELED) { + } + state.SubCState = EState::Finished; + } + + static void DoSubB(TCont* cont, void* argPtr) { + TState& state = *(TState*)argPtr; + state.SubBState = EState::Running; + while (state.SubAState != EState::Running && state.SubCState != EState::Running) { + cont->Yield(); + } + for (auto i : xrange(100)) { + Y_UNUSED(i); + if (!state.SubA) { + break; + } + state.SubA->ReSchedule(); + cont->Yield(); + } + state.SubBState = EState::Finished; + } + + static void DoSubA(TCont* cont, void* argPtr) { + TState& state = *(TState*)argPtr; + state.SubAState = EState::Running; + TCont* subC = cont->Executor()->Create(DoSubC, argPtr, "SubC"); + while (state.SubBState != EState::Running && state.SubCState != EState::Running) { + cont->Yield(); + } + cont->Join(subC); + UNIT_ASSERT_EQUAL(state.SubCState, EState::Finished); + state.SubA = nullptr; + state.SubAState = EState::Finished; + } + + static void DoMain(TCont* cont, void* argPtr) { + TState& state = *(TState*)argPtr; + TCont* subA = cont->Executor()->Create(DoSubA, argPtr, "SubA"); + state.SubA = subA; + cont->Join(cont->Executor()->Create(DoSubB, argPtr, "SubB")); + + if (state.SubA) { + subA->Cancel(); + cont->Join(subA); + } + } +} + +void TCoroTest::TestJoinRescheduleBug() { + using namespace NCoroTestJoinRescheduleBug; + TState state; + { + TContExecutor exec(20000); + exec.Execute(DoMain, &state); + } + UNIT_ASSERT_EQUAL(state.SubAState, EState::Finished); + UNIT_ASSERT_EQUAL(state.SubBState, EState::Finished); + UNIT_ASSERT_EQUAL(state.SubCState, EState::Finished); +} + +void TCoroTest::TestEventQueue() { + NCoro::TEventWaitQueue queue; + UNIT_ASSERT(queue.Empty()); + UNIT_ASSERT_VALUES_EQUAL(queue.WakeTimedout(TInstant()), TInstant()); + TContExecutor exec(32000); + exec.Execute([](TCont* cont, void* arg) { + NCoro::TEventWaitQueue* q = (NCoro::TEventWaitQueue*)arg; + TTimerEvent ev(cont, TInstant::Max()); + TTimerEvent ev2(cont, TInstant::Seconds(12345)); + q->Register(&ev); + UNIT_ASSERT_VALUES_EQUAL(q->WakeTimedout(TInstant::Seconds(12344)), TInstant::Max()); + UNIT_ASSERT_VALUES_EQUAL(q->WakeTimedout(TInstant::Seconds(12344)), TInstant::Max()); + q->Register(&ev2); + UNIT_ASSERT_VALUES_EQUAL(q->WakeTimedout(TInstant::Seconds(12344)), TInstant::Seconds(12345)); + UNIT_ASSERT_VALUES_EQUAL(q->WakeTimedout(TInstant::Seconds(12344)), TInstant::Seconds(12345)); + UNIT_ASSERT_VALUES_EQUAL(q->WakeTimedout(TInstant::Seconds(12345)), TInstant::Seconds(12345)); + UNIT_ASSERT_VALUES_EQUAL(q->WakeTimedout(TInstant::Seconds(12345)), TInstant::Max()); + }, &queue); +} + void TCoroTest::TestNestedExecutor() { #ifndef _win_ //nested executors actually don't work correctly, but anyway shouldn't break RunningCont() ptr diff --git a/library/cpp/coroutine/engine/events.h b/library/cpp/coroutine/engine/events.h index e58d9d239b..07cc4d25e8 100644 --- a/library/cpp/coroutine/engine/events.h +++ b/library/cpp/coroutine/engine/events.h @@ -1,49 +1,49 @@ -#pragma once +#pragma once + +#include "impl.h" + +#include <util/datetime/base.h> -#include "impl.h" - -#include <util/datetime/base.h> - class TContEvent { public: - TContEvent(TCont* current) noexcept + TContEvent(TCont* current) noexcept : Cont_(current) , Status_(0) { } - ~TContEvent() { + ~TContEvent() { } - int WaitD(TInstant deadline) { + int WaitD(TInstant deadline) { Status_ = 0; const int ret = Cont_->SleepD(deadline); return Status_ ? Status_ : ret; } - int WaitT(TDuration timeout) { + int WaitT(TDuration timeout) { return WaitD(timeout.ToDeadLine()); } - int WaitI() { + int WaitI() { return WaitD(TInstant::Max()); } - void Wake() noexcept { + void Wake() noexcept { SetStatus(EWAKEDUP); Cont_->ReSchedule(); } - TCont* Cont() noexcept { + TCont* Cont() noexcept { return Cont_; } - int Status() const noexcept { + int Status() const noexcept { return Status_; } - void SetStatus(int status) noexcept { + void SetStatus(int status) noexcept { Status_ = status; } @@ -55,24 +55,24 @@ private: class TContWaitQueue { class TWaiter: public TContEvent, public TIntrusiveListItem<TWaiter> { public: - TWaiter(TCont* current) noexcept + TWaiter(TCont* current) noexcept : TContEvent(current) { } - ~TWaiter() { + ~TWaiter() { } }; public: - TContWaitQueue() noexcept { + TContWaitQueue() noexcept { } - ~TContWaitQueue() { + ~TContWaitQueue() { Y_ASSERT(Waiters_.Empty()); } - int WaitD(TCont* current, TInstant deadline) { + int WaitD(TCont* current, TInstant deadline) { TWaiter waiter(current); Waiters_.PushBack(&waiter); @@ -80,27 +80,27 @@ public: return waiter.WaitD(deadline); } - int WaitT(TCont* current, TDuration timeout) { + int WaitT(TCont* current, TDuration timeout) { return WaitD(current, timeout.ToDeadLine()); } - int WaitI(TCont* current) { + int WaitI(TCont* current) { return WaitD(current, TInstant::Max()); } - void Signal() noexcept { + void Signal() noexcept { if (!Waiters_.Empty()) { Waiters_.PopFront()->Wake(); } } - void BroadCast() noexcept { + void BroadCast() noexcept { while (!Waiters_.Empty()) { Waiters_.PopFront()->Wake(); } } - void BroadCast(size_t number) noexcept { + void BroadCast(size_t number) noexcept { for (size_t i = 0; i < number && !Waiters_.Empty(); ++i) { Waiters_.PopFront()->Wake(); } @@ -110,35 +110,35 @@ private: TIntrusiveList<TWaiter> Waiters_; }; - + class TContSimpleEvent { public: - TContSimpleEvent(TContExecutor* e) + TContSimpleEvent(TContExecutor* e) : E_(e) { } - TContExecutor* Executor() const noexcept { + TContExecutor* Executor() const noexcept { return E_; } - void Signal() noexcept { + void Signal() noexcept { Q_.Signal(); } - void BroadCast() noexcept { + void BroadCast() noexcept { Q_.BroadCast(); } - int WaitD(TInstant deadLine) noexcept { - return Q_.WaitD(E_->Running(), deadLine); + int WaitD(TInstant deadLine) noexcept { + return Q_.WaitD(E_->Running(), deadLine); } - int WaitT(TDuration timeout) noexcept { + int WaitT(TDuration timeout) noexcept { return WaitD(timeout.ToDeadLine()); } - int WaitI() noexcept { + int WaitI() noexcept { return WaitD(TInstant::Max()); } diff --git a/library/cpp/coroutine/engine/impl.cpp b/library/cpp/coroutine/engine/impl.cpp index 6bf8b03348..7ae6f74051 100644 --- a/library/cpp/coroutine/engine/impl.cpp +++ b/library/cpp/coroutine/engine/impl.cpp @@ -1,162 +1,162 @@ -#include "impl.h" +#include "impl.h" #include "stack/stack_allocator.h" #include "stack/stack_guards.h" #include <util/generic/scope.h> #include <util/thread/singleton.h> -#include <util/stream/format.h> +#include <util/stream/format.h> #include <util/stream/output.h> #include <util/system/yassert.h> -TCont::TJoinWait::TJoinWait(TCont& c) noexcept - : Cont_(c) -{} - -void TCont::TJoinWait::Wake() noexcept { - Cont_.ReSchedule(); -} - +TCont::TJoinWait::TJoinWait(TCont& c) noexcept + : Cont_(c) +{} + +void TCont::TJoinWait::Wake() noexcept { + Cont_.ReSchedule(); +} + TCont::TCont(NCoro::NStack::IAllocator& allocator, uint32_t stackSize, TContExecutor& executor, NCoro::TTrampoline::TFunc func, const char* name) noexcept - : Executor_(executor) + : Executor_(executor) , Name_(name) - , Trampoline_( + , Trampoline_( allocator, - stackSize, + stackSize, std::move(func), this - ) -{} - - + ) +{} + + void TCont::PrintMe(IOutputStream& out) const noexcept { out << "cont(" - << "name = " << Name_ << ", " - << "addr = " << Hex((size_t)this) + << "name = " << Name_ << ", " + << "addr = " << Hex((size_t)this) << ")"; } -bool TCont::Join(TCont* c, TInstant deadLine) noexcept { - TJoinWait ev(*this); - c->Waiters_.PushBack(&ev); - - do { - if (SleepD(deadLine) == ETIMEDOUT || Cancelled()) { - if (!ev.Empty()) { - c->Cancel(); - - do { +bool TCont::Join(TCont* c, TInstant deadLine) noexcept { + TJoinWait ev(*this); + c->Waiters_.PushBack(&ev); + + do { + if (SleepD(deadLine) == ETIMEDOUT || Cancelled()) { + if (!ev.Empty()) { + c->Cancel(); + + do { Switch(); - } while (!ev.Empty()); - } - - return false; - } - } while (!ev.Empty()); - - return true; -} - -int TCont::SleepD(TInstant deadline) noexcept { - TTimerEvent event(this, deadline); - - return ExecuteEvent(&event); -} - + } while (!ev.Empty()); + } + + return false; + } + } while (!ev.Empty()); + + return true; +} + +int TCont::SleepD(TInstant deadline) noexcept { + TTimerEvent event(this, deadline); + + return ExecuteEvent(&event); +} + void TCont::Switch() noexcept { Executor()->RunScheduler(); } -void TCont::Yield() noexcept { - if (SleepD(TInstant::Zero())) { - ReScheduleAndSwitch(); - } -} - -void TCont::ReScheduleAndSwitch() noexcept { - ReSchedule(); +void TCont::Yield() noexcept { + if (SleepD(TInstant::Zero())) { + ReScheduleAndSwitch(); + } +} + +void TCont::ReScheduleAndSwitch() noexcept { + ReSchedule(); Switch(); -} - -void TCont::Terminate() { - while (!Waiters_.Empty()) { - Waiters_.PopFront()->Wake(); - } - Executor()->Exit(this); -} - -bool TCont::IAmRunning() const noexcept { - return this == Executor()->Running(); -} - -void TCont::Cancel() noexcept { - if (Cancelled()) { - return; - } - - Cancelled_ = true; - - if (!IAmRunning()) { - ReSchedule(); - } -} - -void TCont::ReSchedule() noexcept { - if (Cancelled()) { - // Legacy code may expect a Cancelled coroutine to be scheduled without delay. - Executor()->ScheduleExecutionNow(this); - } else { - Executor()->ScheduleExecution(this); - } -} - - -TContExecutor::TContExecutor( +} + +void TCont::Terminate() { + while (!Waiters_.Empty()) { + Waiters_.PopFront()->Wake(); + } + Executor()->Exit(this); +} + +bool TCont::IAmRunning() const noexcept { + return this == Executor()->Running(); +} + +void TCont::Cancel() noexcept { + if (Cancelled()) { + return; + } + + Cancelled_ = true; + + if (!IAmRunning()) { + ReSchedule(); + } +} + +void TCont::ReSchedule() noexcept { + if (Cancelled()) { + // Legacy code may expect a Cancelled coroutine to be scheduled without delay. + Executor()->ScheduleExecutionNow(this); + } else { + Executor()->ScheduleExecution(this); + } +} + + +TContExecutor::TContExecutor( uint32_t defaultStackSize, - THolder<IPollerFace> poller, + THolder<IPollerFace> poller, NCoro::IScheduleCallback* scheduleCallback, NCoro::IEnterPollerCallback* enterPollerCallback, NCoro::NStack::EGuard defaultGuard, TMaybe<NCoro::NStack::TPoolAllocatorSettings> poolSettings, NCoro::ITime* time -) +) : ScheduleCallback_(scheduleCallback) , EnterPollerCallback_(enterPollerCallback) - , DefaultStackSize_(defaultStackSize) - , Poller_(std::move(poller)) + , DefaultStackSize_(defaultStackSize) + , Poller_(std::move(poller)) , Time_(time) { StackAllocator_ = NCoro::NStack::GetAllocator(poolSettings, defaultGuard); } - -TContExecutor::~TContExecutor() { - Y_VERIFY(Allocated_ == 0, "leaked %u coroutines", (ui32)Allocated_); -} - -void TContExecutor::Execute() noexcept { - auto nop = [](void*){}; - Execute(nop); -} - -void TContExecutor::Execute(TContFunc func, void* arg) noexcept { + +TContExecutor::~TContExecutor() { + Y_VERIFY(Allocated_ == 0, "leaked %u coroutines", (ui32)Allocated_); +} + +void TContExecutor::Execute() noexcept { + auto nop = [](void*){}; + Execute(nop); +} + +void TContExecutor::Execute(TContFunc func, void* arg) noexcept { CreateOwned([=](TCont* cont) { func(cont, arg); }, "sys_main"); - RunScheduler(); + RunScheduler(); } -void TContExecutor::WaitForIO() { - while (Ready_.Empty() && !WaitQueue_.Empty()) { +void TContExecutor::WaitForIO() { + while (Ready_.Empty() && !WaitQueue_.Empty()) { const auto now = Now(); - - // Waking a coroutine puts it into ReadyNext_ list - const auto next = WaitQueue_.WakeTimedout(now); - + + // Waking a coroutine puts it into ReadyNext_ list + const auto next = WaitQueue_.WakeTimedout(now); + if (!UserEvents_.Empty()) { TIntrusiveList<IUserEvent> userEvents; userEvents.Swap(UserEvents_); @@ -165,11 +165,11 @@ void TContExecutor::WaitForIO() { } while (!userEvents.Empty()); } - // Polling will return as soon as there is an event to process or a timeout. - // If there are woken coroutines we do not want to sleep in the poller - // yet still we want to check for new io - // to prevent ourselves from locking out of io by constantly waking coroutines. - + // Polling will return as soon as there is an event to process or a timeout. + // If there are woken coroutines we do not want to sleep in the poller + // yet still we want to check for new io + // to prevent ourselves from locking out of io by constantly waking coroutines. + if (ReadyNext_.Empty()) { if (EnterPollerCallback_) { EnterPollerCallback_->OnEnterPoller(); @@ -187,9 +187,9 @@ void TContExecutor::WaitForIO() { EnterPollerCallback_->OnExitPoller(); } } - - Ready_.Append(ReadyNext_); - } + + Ready_.Append(ReadyNext_); + } } void TContExecutor::Poll(TInstant deadline) { @@ -198,42 +198,42 @@ void TContExecutor::Poll(TInstant deadline) { // Waking a coroutine puts it into ReadyNext_ list for (auto event : PollerEvents_) { - auto* lst = (NCoro::TPollEventList*)event.Data; - const int status = event.Status; - - if (status) { - for (auto it = lst->Begin(); it != lst->End();) { - (it++)->OnPollEvent(status); - } - } else { - const ui16 filter = event.Filter; - - for (auto it = lst->Begin(); it != lst->End();) { - if (it->What() & filter) { - (it++)->OnPollEvent(0); - } else { - ++it; - } - } - } - } -} - -void TContExecutor::Abort() noexcept { - WaitQueue_.Abort(); - auto visitor = [](TCont* c) { - c->Cancel(); - }; - Ready_.ForEach(visitor); - ReadyNext_.ForEach(visitor); -} - -TCont* TContExecutor::Create( - TContFunc func, - void* arg, - const char* name, - TMaybe<ui32> customStackSize -) noexcept { + auto* lst = (NCoro::TPollEventList*)event.Data; + const int status = event.Status; + + if (status) { + for (auto it = lst->Begin(); it != lst->End();) { + (it++)->OnPollEvent(status); + } + } else { + const ui16 filter = event.Filter; + + for (auto it = lst->Begin(); it != lst->End();) { + if (it->What() & filter) { + (it++)->OnPollEvent(0); + } else { + ++it; + } + } + } + } +} + +void TContExecutor::Abort() noexcept { + WaitQueue_.Abort(); + auto visitor = [](TCont* c) { + c->Cancel(); + }; + Ready_.ForEach(visitor); + ReadyNext_.ForEach(visitor); +} + +TCont* TContExecutor::Create( + TContFunc func, + void* arg, + const char* name, + TMaybe<ui32> customStackSize +) noexcept { return CreateOwned([=](TCont* cont) { func(cont, arg); }, name, customStackSize); @@ -244,38 +244,38 @@ TCont* TContExecutor::CreateOwned( const char* name, TMaybe<ui32> customStackSize ) noexcept { - Allocated_ += 1; - if (!customStackSize) { - customStackSize = DefaultStackSize_; - } + Allocated_ += 1; + if (!customStackSize) { + customStackSize = DefaultStackSize_; + } auto* cont = new TCont(*StackAllocator_, *customStackSize, *this, std::move(func), name); - ScheduleExecution(cont); - return cont; -} - + ScheduleExecution(cont); + return cont; +} + NCoro::NStack::TAllocatorStats TContExecutor::GetAllocatorStats() const noexcept { return StackAllocator_->GetStackStats(); } -void TContExecutor::Release(TCont* cont) noexcept { - delete cont; - Allocated_ -= 1; -} - -void TContExecutor::ScheduleToDelete(TCont* cont) noexcept { - ToDelete_.PushBack(cont); -} - -void TContExecutor::ScheduleExecution(TCont* cont) noexcept { - cont->Scheduled_ = true; - ReadyNext_.PushBack(cont); -} - -void TContExecutor::ScheduleExecutionNow(TCont* cont) noexcept { - cont->Scheduled_ = true; - Ready_.PushBack(cont); -} - +void TContExecutor::Release(TCont* cont) noexcept { + delete cont; + Allocated_ -= 1; +} + +void TContExecutor::ScheduleToDelete(TCont* cont) noexcept { + ToDelete_.PushBack(cont); +} + +void TContExecutor::ScheduleExecution(TCont* cont) noexcept { + cont->Scheduled_ = true; + ReadyNext_.PushBack(cont); +} + +void TContExecutor::ScheduleExecutionNow(TCont* cont) noexcept { + cont->Scheduled_ = true; + Ready_.PushBack(cont); +} + namespace { inline TContExecutor*& ThisThreadExecutor() { struct TThisThreadExecutorHolder { @@ -285,12 +285,12 @@ namespace { } } -void TContExecutor::DeleteScheduled() noexcept { - ToDelete_.ForEach([this](TCont* c) { - Release(c); - }); -} - +void TContExecutor::DeleteScheduled() noexcept { + ToDelete_.ForEach([this](TCont* c) { + Release(c); + }); +} + TCont* RunningCont() { TContExecutor* thisThreadExecutor = ThisThreadExecutor(); return thisThreadExecutor ? thisThreadExecutor->Running() : nullptr; @@ -314,7 +314,7 @@ void TContExecutor::RunScheduler() noexcept { WaitForIO(); DeleteScheduled(); Ready_.Append(ReadyNext_); - + if (Ready_.Empty()) { Current_ = nullptr; if (caller) { @@ -322,18 +322,18 @@ void TContExecutor::RunScheduler() noexcept { } break; } - - TCont* cont = Ready_.PopFront(); + + TCont* cont = Ready_.PopFront(); if (ScheduleCallback_) { ScheduleCallback_->OnSchedule(*this, *cont); - } + } Current_ = cont; cont->Scheduled_ = false; if (cont == caller) { break; - } + } context->SwitchTo(cont->Trampoline_.Context()); if (Paused_) { Paused_ = false; @@ -346,9 +346,9 @@ void TContExecutor::RunScheduler() noexcept { } } catch (...) { TBackTrace::FromCurrentException().PrintTo(Cerr); - Y_FAIL("Uncaught exception in the scheduler: %s", CurrentExceptionMessage().c_str()); + Y_FAIL("Uncaught exception in the scheduler: %s", CurrentExceptionMessage().c_str()); } -} +} void TContExecutor::Pause() { if (auto cont = Running()) { @@ -358,17 +358,17 @@ void TContExecutor::Pause() { } } -void TContExecutor::Exit(TCont* cont) noexcept { - ScheduleToDelete(cont); - cont->SwitchTo(&SchedContext_); - Y_FAIL("can not return from exit"); -} - +void TContExecutor::Exit(TCont* cont) noexcept { + ScheduleToDelete(cont); + cont->SwitchTo(&SchedContext_); + Y_FAIL("can not return from exit"); +} + TInstant TContExecutor::Now() { return Y_LIKELY(Time_ == nullptr) ? TInstant::Now() : Time_->Now(); } -template <> -void Out<TCont>(IOutputStream& out, const TCont& c) { - c.PrintMe(out); +template <> +void Out<TCont>(IOutputStream& out, const TCont& c) { + c.PrintMe(out); } diff --git a/library/cpp/coroutine/engine/impl.h b/library/cpp/coroutine/engine/impl.h index 4713b61efe..283a96ecf1 100644 --- a/library/cpp/coroutine/engine/impl.h +++ b/library/cpp/coroutine/engine/impl.h @@ -1,25 +1,25 @@ -#pragma once +#pragma once #include "callbacks.h" -#include "cont_poller.h" -#include "iostatus.h" +#include "cont_poller.h" +#include "iostatus.h" #include "poller.h" #include "stack/stack_common.h" -#include "trampoline.h" +#include "trampoline.h" #include "custom_time.h" #include <library/cpp/containers/intrusive_rb_tree/rb_tree.h> - + #include <util/system/error.h> #include <util/generic/ptr.h> #include <util/generic/intrlist.h> #include <util/datetime/base.h> -#include <util/generic/maybe.h> +#include <util/generic/maybe.h> #include <util/generic/function.h> - -#define EWAKEDUP 34567 - + +#define EWAKEDUP 34567 + class TCont; struct TContRep; class TContExecutor; @@ -28,97 +28,97 @@ class TContPollEvent; namespace NCoro::NStack { class IAllocator; } - -class TCont : private TIntrusiveListItem<TCont> { - struct TJoinWait: public TIntrusiveListItem<TJoinWait> { - TJoinWait(TCont& c) noexcept; - void Wake() noexcept; +class TCont : private TIntrusiveListItem<TCont> { + struct TJoinWait: public TIntrusiveListItem<TJoinWait> { + TJoinWait(TCont& c) noexcept; + + void Wake() noexcept; - public: - TCont& Cont_; - }; + public: + TCont& Cont_; + }; - friend class TContExecutor; - friend class TIntrusiveListItem<TCont>; - friend class NCoro::TEventWaitQueue; - friend class NCoro::TTrampoline; + friend class TContExecutor; + friend class TIntrusiveListItem<TCont>; + friend class NCoro::TEventWaitQueue; + friend class NCoro::TTrampoline; -private: - TCont( +private: + TCont( NCoro::NStack::IAllocator& allocator, uint32_t stackSize, - TContExecutor& executor, + TContExecutor& executor, NCoro::TTrampoline::TFunc func, - const char* name - ) noexcept; + const char* name + ) noexcept; -public: - TContExecutor* Executor() noexcept { - return &Executor_; +public: + TContExecutor* Executor() noexcept { + return &Executor_; } - const TContExecutor* Executor() const noexcept { - return &Executor_; + const TContExecutor* Executor() const noexcept { + return &Executor_; } - const char* Name() const noexcept { + const char* Name() const noexcept { return Name_; } void PrintMe(IOutputStream& out) const noexcept; - void Yield() noexcept; + void Yield() noexcept; - void ReScheduleAndSwitch() noexcept; + void ReScheduleAndSwitch() noexcept; /// @return ETIMEDOUT on success - int SleepD(TInstant deadline) noexcept; + int SleepD(TInstant deadline) noexcept; - int SleepT(TDuration timeout) noexcept { + int SleepT(TDuration timeout) noexcept { return SleepD(timeout.ToDeadLine()); } - int SleepI() noexcept { + int SleepI() noexcept { return SleepD(TInstant::Max()); } - bool IAmRunning() const noexcept; + bool IAmRunning() const noexcept; - void Cancel() noexcept; + void Cancel() noexcept; - bool Cancelled() const noexcept { + bool Cancelled() const noexcept { return Cancelled_; } - bool Scheduled() const noexcept { + bool Scheduled() const noexcept { return Scheduled_; } - bool Join(TCont* c, TInstant deadLine = TInstant::Max()) noexcept; + bool Join(TCont* c, TInstant deadLine = TInstant::Max()) noexcept; - void ReSchedule() noexcept; + void ReSchedule() noexcept; void Switch() noexcept; - void SwitchTo(TExceptionSafeContext* ctx) { - Trampoline_.SwitchTo(ctx); - } + void SwitchTo(TExceptionSafeContext* ctx) { + Trampoline_.SwitchTo(ctx); + } private: - void Terminate(); - + void Terminate(); + private: - TContExecutor& Executor_; - - // TODO(velavokr): allow name storage owning (for generated names backed by TString) - const char* Name_ = nullptr; + TContExecutor& Executor_; + + // TODO(velavokr): allow name storage owning (for generated names backed by TString) + const char* Name_ = nullptr; NCoro::TTrampoline Trampoline_; TIntrusiveList<TJoinWait> Waiters_; - bool Cancelled_ = false; - bool Scheduled_ = false; + bool Cancelled_ = false; + bool Scheduled_ = false; }; TCont* RunningCont(); @@ -147,60 +147,60 @@ public: /// Note, coroutines are single-threaded, and all methods must be called from the single thread class TContExecutor { friend class TCont; - using TContList = TIntrusiveList<TCont>; + using TContList = TIntrusiveList<TCont>; public: - TContExecutor( + TContExecutor( uint32_t defaultStackSize, - THolder<IPollerFace> poller = IPollerFace::Default(), - NCoro::IScheduleCallback* = nullptr, + THolder<IPollerFace> poller = IPollerFace::Default(), + NCoro::IScheduleCallback* = nullptr, NCoro::IEnterPollerCallback* = nullptr, NCoro::NStack::EGuard stackGuard = NCoro::NStack::EGuard::Canary, TMaybe<NCoro::NStack::TPoolAllocatorSettings> poolSettings = Nothing(), NCoro::ITime* time = nullptr - ); + ); ~TContExecutor(); - // if we already have a coroutine to run - void Execute() noexcept; + // if we already have a coroutine to run + void Execute() noexcept; - void Execute(TContFunc func, void* arg = nullptr) noexcept; + void Execute(TContFunc func, void* arg = nullptr) noexcept; template <class Functor> - void Execute(Functor& f) noexcept { + void Execute(Functor& f) noexcept { Execute((TContFunc)ContHelperFunc<Functor>, (void*)&f); } template <typename T, void (T::*M)(TCont*)> - void Execute(T* obj) noexcept { + void Execute(T* obj) noexcept { Execute(ContHelperMemberFunc<T, M>, obj); } template <class Functor> - TCont* Create( - Functor& f, - const char* name, - TMaybe<ui32> customStackSize = Nothing() - ) noexcept { - return Create((TContFunc)ContHelperFunc<Functor>, (void*)&f, name, customStackSize); + TCont* Create( + Functor& f, + const char* name, + TMaybe<ui32> customStackSize = Nothing() + ) noexcept { + return Create((TContFunc)ContHelperFunc<Functor>, (void*)&f, name, customStackSize); } template <typename T, void (T::*M)(TCont*)> - TCont* Create( - T* obj, - const char* name, - TMaybe<ui32> customStackSize = Nothing() - ) noexcept { - return Create(ContHelperMemberFunc<T, M>, obj, name, customStackSize); + TCont* Create( + T* obj, + const char* name, + TMaybe<ui32> customStackSize = Nothing() + ) noexcept { + return Create(ContHelperMemberFunc<T, M>, obj, name, customStackSize); } - TCont* Create( - TContFunc func, - void* arg, - const char* name, - TMaybe<ui32> customStackSize = Nothing() - ) noexcept; + TCont* Create( + TContFunc func, + void* arg, + const char* name, + TMaybe<ui32> customStackSize = Nothing() + ) noexcept; TCont* CreateOwned( NCoro::TTrampoline::TFunc func, @@ -208,44 +208,44 @@ public: TMaybe<ui32> customStackSize = Nothing() ) noexcept; - NCoro::TContPoller* Poller() noexcept { + NCoro::TContPoller* Poller() noexcept { return &Poller_; } - TCont* Running() noexcept { + TCont* Running() noexcept { return Current_; } - const TCont* Running() const noexcept { + const TCont* Running() const noexcept { return Current_; } - size_t TotalReadyConts() const noexcept { - return Ready_.Size() + TotalScheduledConts(); - } - - size_t TotalScheduledConts() const noexcept { - return ReadyNext_.Size(); - } - - size_t TotalConts() const noexcept { - return Allocated_; - } - - size_t TotalWaitingConts() const noexcept { - return TotalConts() - TotalReadyConts(); - } - + size_t TotalReadyConts() const noexcept { + return Ready_.Size() + TotalScheduledConts(); + } + + size_t TotalScheduledConts() const noexcept { + return ReadyNext_.Size(); + } + + size_t TotalConts() const noexcept { + return Allocated_; + } + + size_t TotalWaitingConts() const noexcept { + return TotalConts() - TotalReadyConts(); + } + NCoro::NStack::TAllocatorStats GetAllocatorStats() const noexcept; - // TODO(velavokr): rename, it is just CancelAll actually - void Abort() noexcept; + // TODO(velavokr): rename, it is just CancelAll actually + void Abort() noexcept; - void SetFailOnError(bool fail) noexcept { + void SetFailOnError(bool fail) noexcept { FailOnError_ = fail; } - bool FailOnError() const noexcept { + bool FailOnError() const noexcept { return FailOnError_; } @@ -253,12 +253,12 @@ public: WaitQueue_.Register(event); } - void ScheduleIoWait(TFdEvent* event) { + void ScheduleIoWait(TFdEvent* event) { RegisterInWaitQueue(event); Poller_.Schedule(event); } - void ScheduleIoWait(TTimerEvent* event) noexcept { + void ScheduleIoWait(TTimerEvent* event) noexcept { RegisterInWaitQueue(event); } @@ -269,45 +269,45 @@ public: void Pause(); TInstant Now(); private: - void Release(TCont* cont) noexcept; + void Release(TCont* cont) noexcept; - void Exit(TCont* cont) noexcept; + void Exit(TCont* cont) noexcept; void RunScheduler() noexcept; - void ScheduleToDelete(TCont* cont) noexcept; + void ScheduleToDelete(TCont* cont) noexcept; - void ScheduleExecution(TCont* cont) noexcept; + void ScheduleExecution(TCont* cont) noexcept; - void ScheduleExecutionNow(TCont* cont) noexcept; - - void DeleteScheduled() noexcept; + void ScheduleExecutionNow(TCont* cont) noexcept; + + void DeleteScheduled() noexcept; void WaitForIO(); void Poll(TInstant deadline); - + private: NCoro::IScheduleCallback* const ScheduleCallback_ = nullptr; NCoro::IEnterPollerCallback* const EnterPollerCallback_ = nullptr; const uint32_t DefaultStackSize_; THolder<NCoro::NStack::IAllocator> StackAllocator_; - - TExceptionSafeContext SchedContext_; - + + TExceptionSafeContext SchedContext_; + TContList ToDelete_; TContList Ready_; - TContList ReadyNext_; - NCoro::TEventWaitQueue WaitQueue_; - NCoro::TContPoller Poller_; + TContList ReadyNext_; + NCoro::TEventWaitQueue WaitQueue_; + NCoro::TContPoller Poller_; NCoro::TContPoller::TEvents PollerEvents_; TInstant LastPoll_; - + TIntrusiveList<IUserEvent> UserEvents_; - size_t Allocated_ = 0; - TCont* Current_ = nullptr; - bool FailOnError_ = false; + size_t Allocated_ = 0; + TCont* Current_ = nullptr; + bool FailOnError_ = false; bool Paused_ = false; NCoro::ITime* Time_ = nullptr; }; diff --git a/library/cpp/coroutine/engine/iostatus.h b/library/cpp/coroutine/engine/iostatus.h index 201cc77825..bf6036805d 100644 --- a/library/cpp/coroutine/engine/iostatus.h +++ b/library/cpp/coroutine/engine/iostatus.h @@ -1,41 +1,41 @@ -#pragma once +#pragma once #include <util/generic/yexception.h> class TIOStatus { public: - TIOStatus(int status) noexcept + TIOStatus(int status) noexcept : Status_(status) { } - static TIOStatus Error(int status) noexcept { + static TIOStatus Error(int status) noexcept { return TIOStatus(status); } - static TIOStatus Error() noexcept { + static TIOStatus Error() noexcept { return TIOStatus(LastSystemError()); } - static TIOStatus Success() noexcept { + static TIOStatus Success() noexcept { return TIOStatus(0); } - void Check() const { + void Check() const { if (Status_) { ythrow TSystemError(Status_) << "io error"; } } - bool Failed() const noexcept { + bool Failed() const noexcept { return (bool)Status_; } - bool Succeed() const noexcept { + bool Succeed() const noexcept { return !Failed(); } - int Status() const noexcept { + int Status() const noexcept { return Status_; } @@ -43,43 +43,43 @@ private: int Status_; }; - + class TContIOStatus { public: - TContIOStatus(size_t processed, TIOStatus status) noexcept + TContIOStatus(size_t processed, TIOStatus status) noexcept : Processed_(processed) , Status_(status) { } - static TContIOStatus Error(TIOStatus status) noexcept { + static TContIOStatus Error(TIOStatus status) noexcept { return TContIOStatus(0, status); } - static TContIOStatus Error() noexcept { + static TContIOStatus Error() noexcept { return TContIOStatus(0, TIOStatus::Error()); } - static TContIOStatus Success(size_t processed) noexcept { + static TContIOStatus Success(size_t processed) noexcept { return TContIOStatus(processed, TIOStatus::Success()); } - static TContIOStatus Eof() noexcept { + static TContIOStatus Eof() noexcept { return Success(0); } - ~TContIOStatus() { + ~TContIOStatus() { } - size_t Processed() const noexcept { + size_t Processed() const noexcept { return Processed_; } - int Status() const noexcept { + int Status() const noexcept { return Status_.Status(); } - size_t Checked() const { + size_t Checked() const { Status_.Check(); return Processed_; diff --git a/library/cpp/coroutine/engine/mutex.h b/library/cpp/coroutine/engine/mutex.h index 5a73e13c2c..93e9119503 100644 --- a/library/cpp/coroutine/engine/mutex.h +++ b/library/cpp/coroutine/engine/mutex.h @@ -1,20 +1,20 @@ -#pragma once +#pragma once + +#include "impl.h" +#include "events.h" -#include "impl.h" -#include "events.h" - class TContMutex { public: - TContMutex() noexcept + TContMutex() noexcept : Token_(true) { } - ~TContMutex() { + ~TContMutex() { Y_ASSERT(Token_); } - int LockD(TCont* current, TInstant deadline) { + int LockD(TCont* current, TInstant deadline) { while (!Token_) { const int ret = WaitQueue_.WaitD(current, deadline); @@ -28,15 +28,15 @@ public: return 0; } - int LockT(TCont* current, TDuration timeout) { + int LockT(TCont* current, TDuration timeout) { return LockD(current, timeout.ToDeadLine()); } - int LockI(TCont* current) { + int LockI(TCont* current) { return LockD(current, TInstant::Max()); } - void UnLock() noexcept { + void UnLock() noexcept { Y_ASSERT(!Token_); Token_ = true; diff --git a/library/cpp/coroutine/engine/network.cpp b/library/cpp/coroutine/engine/network.cpp index 46100a8023..85b647d210 100644 --- a/library/cpp/coroutine/engine/network.cpp +++ b/library/cpp/coroutine/engine/network.cpp @@ -1,325 +1,325 @@ -#include "impl.h" -#include "network.h" +#include "impl.h" +#include "network.h" -#include <util/generic/scope.h> -#include <util/generic/xrange.h> +#include <util/generic/scope.h> +#include <util/generic/xrange.h> #include <sys/uio.h> -#if defined(_bionic_) -# define IOV_MAX 1024 -#endif +#if defined(_bionic_) +# define IOV_MAX 1024 +#endif -namespace NCoro { - namespace { - bool IsBlocked(int lasterr) noexcept { - return lasterr == EAGAIN || lasterr == EWOULDBLOCK; - } +namespace NCoro { + namespace { + bool IsBlocked(int lasterr) noexcept { + return lasterr == EAGAIN || lasterr == EWOULDBLOCK; + } - ssize_t DoReadVector(SOCKET fd, TContIOVector* vec) noexcept { + ssize_t DoReadVector(SOCKET fd, TContIOVector* vec) noexcept { return readv(fd, (const iovec*) vec->Parts(), Min(IOV_MAX, (int) vec->Count())); - } + } - ssize_t DoWriteVector(SOCKET fd, TContIOVector* vec) noexcept { + ssize_t DoWriteVector(SOCKET fd, TContIOVector* vec) noexcept { return writev(fd, (const iovec*) vec->Parts(), Min(IOV_MAX, (int) vec->Count())); - } - } - - - int SelectD(TCont* cont, SOCKET fds[], int what[], size_t nfds, SOCKET* outfd, TInstant deadline) noexcept { - if (cont->Cancelled()) { - return ECANCELED; - } - - if (nfds == 0) { - return 0; - } - - TTempArray<TFdEvent> events(nfds); - - for (auto i : xrange(nfds)) { - new(events.Data() + i) TFdEvent(cont, fds[i], (ui16) what[i], deadline); - } - - Y_DEFER { - for (auto i : xrange(nfds)) { - (events.Data() + i)->~TFdEvent(); - } - }; - - for (auto i : xrange(nfds)) { - cont->Executor()->ScheduleIoWait(events.Data() + i); - } + } + } + + + int SelectD(TCont* cont, SOCKET fds[], int what[], size_t nfds, SOCKET* outfd, TInstant deadline) noexcept { + if (cont->Cancelled()) { + return ECANCELED; + } + + if (nfds == 0) { + return 0; + } + + TTempArray<TFdEvent> events(nfds); + + for (auto i : xrange(nfds)) { + new(events.Data() + i) TFdEvent(cont, fds[i], (ui16) what[i], deadline); + } + + Y_DEFER { + for (auto i : xrange(nfds)) { + (events.Data() + i)->~TFdEvent(); + } + }; + + for (auto i : xrange(nfds)) { + cont->Executor()->ScheduleIoWait(events.Data() + i); + } cont->Switch(); - - if (cont->Cancelled()) { - return ECANCELED; - } - - TFdEvent* ret = nullptr; - int status = EINPROGRESS; - - for (auto i : xrange(nfds)) { - auto& ev = *(events.Data() + i); - switch (ev.Status()) { - case EINPROGRESS: - break; - case ETIMEDOUT: - if (status != EINPROGRESS) { - break; + + if (cont->Cancelled()) { + return ECANCELED; + } + + TFdEvent* ret = nullptr; + int status = EINPROGRESS; + + for (auto i : xrange(nfds)) { + auto& ev = *(events.Data() + i); + switch (ev.Status()) { + case EINPROGRESS: + break; + case ETIMEDOUT: + if (status != EINPROGRESS) { + break; } [[fallthrough]]; - default: - status = ev.Status(); - ret = &ev; - } - } - - if (ret) { - if (outfd) { - *outfd = ret->Fd(); + default: + status = ev.Status(); + ret = &ev; } - return ret->Status(); } - return EINPROGRESS; + if (ret) { + if (outfd) { + *outfd = ret->Fd(); + } + return ret->Status(); + } + + return EINPROGRESS; + } + + int SelectT(TCont* cont, SOCKET fds[], int what[], size_t nfds, SOCKET* outfd, TDuration timeout) noexcept { + return SelectD(cont, fds, what, nfds, outfd, timeout.ToDeadLine()); } - int SelectT(TCont* cont, SOCKET fds[], int what[], size_t nfds, SOCKET* outfd, TDuration timeout) noexcept { - return SelectD(cont, fds, what, nfds, outfd, timeout.ToDeadLine()); + int SelectI(TCont* cont, SOCKET fds[], int what[], size_t nfds, SOCKET* outfd) { + return SelectD(cont, fds, what, nfds, outfd, TInstant::Max()); } - int SelectI(TCont* cont, SOCKET fds[], int what[], size_t nfds, SOCKET* outfd) { - return SelectD(cont, fds, what, nfds, outfd, TInstant::Max()); - } - - int PollD(TCont* cont, SOCKET fd, int what, TInstant deadline) noexcept { - TFdEvent event(cont, fd, (ui16)what, deadline); - return ExecuteEvent(&event); + int PollD(TCont* cont, SOCKET fd, int what, TInstant deadline) noexcept { + TFdEvent event(cont, fd, (ui16)what, deadline); + return ExecuteEvent(&event); } - int PollT(TCont* cont, SOCKET fd, int what, TDuration timeout) noexcept { - return PollD(cont, fd, what, timeout.ToDeadLine()); - } - - int PollI(TCont* cont, SOCKET fd, int what) noexcept { - return PollD(cont, fd, what, TInstant::Max()); + int PollT(TCont* cont, SOCKET fd, int what, TDuration timeout) noexcept { + return PollD(cont, fd, what, timeout.ToDeadLine()); + } + + int PollI(TCont* cont, SOCKET fd, int what) noexcept { + return PollD(cont, fd, what, TInstant::Max()); } - TContIOStatus ReadVectorD(TCont* cont, SOCKET fd, TContIOVector* vec, TInstant deadline) noexcept { - while (true) { - ssize_t res = DoReadVector(fd, vec); + TContIOStatus ReadVectorD(TCont* cont, SOCKET fd, TContIOVector* vec, TInstant deadline) noexcept { + while (true) { + ssize_t res = DoReadVector(fd, vec); - if (res >= 0) { - return TContIOStatus::Success((size_t) res); + if (res >= 0) { + return TContIOStatus::Success((size_t) res); } { const int err = LastSystemError(); if (!IsBlocked(err)) { - return TContIOStatus::Error(err); + return TContIOStatus::Error(err); } } - if ((res = PollD(cont, fd, CONT_POLL_READ, deadline)) != 0) { - return TContIOStatus::Error((int) res); + if ((res = PollD(cont, fd, CONT_POLL_READ, deadline)) != 0) { + return TContIOStatus::Error((int) res); } } } - TContIOStatus ReadVectorT(TCont* cont, SOCKET fd, TContIOVector* vec, TDuration timeOut) noexcept { - return ReadVectorD(cont, fd, vec, timeOut.ToDeadLine()); - } + TContIOStatus ReadVectorT(TCont* cont, SOCKET fd, TContIOVector* vec, TDuration timeOut) noexcept { + return ReadVectorD(cont, fd, vec, timeOut.ToDeadLine()); + } - TContIOStatus ReadVectorI(TCont* cont, SOCKET fd, TContIOVector* vec) noexcept { - return ReadVectorD(cont, fd, vec, TInstant::Max()); - } + TContIOStatus ReadVectorI(TCont* cont, SOCKET fd, TContIOVector* vec) noexcept { + return ReadVectorD(cont, fd, vec, TInstant::Max()); + } - TContIOStatus ReadD(TCont* cont, SOCKET fd, void* buf, size_t len, TInstant deadline) noexcept { - IOutputStream::TPart part(buf, len); - TContIOVector vec(&part, 1); - return ReadVectorD(cont, fd, &vec, deadline); - } + TContIOStatus ReadD(TCont* cont, SOCKET fd, void* buf, size_t len, TInstant deadline) noexcept { + IOutputStream::TPart part(buf, len); + TContIOVector vec(&part, 1); + return ReadVectorD(cont, fd, &vec, deadline); + } - TContIOStatus ReadT(TCont* cont, SOCKET fd, void* buf, size_t len, TDuration timeout) noexcept { - return ReadD(cont, fd, buf, len, timeout.ToDeadLine()); - } + TContIOStatus ReadT(TCont* cont, SOCKET fd, void* buf, size_t len, TDuration timeout) noexcept { + return ReadD(cont, fd, buf, len, timeout.ToDeadLine()); + } - TContIOStatus ReadI(TCont* cont, SOCKET fd, void* buf, size_t len) noexcept { - return ReadD(cont, fd, buf, len, TInstant::Max()); + TContIOStatus ReadI(TCont* cont, SOCKET fd, void* buf, size_t len) noexcept { + return ReadD(cont, fd, buf, len, TInstant::Max()); } - TContIOStatus WriteVectorD(TCont* cont, SOCKET fd, TContIOVector* vec, TInstant deadline) noexcept { - size_t written = 0; + TContIOStatus WriteVectorD(TCont* cont, SOCKET fd, TContIOVector* vec, TInstant deadline) noexcept { + size_t written = 0; - while (!vec->Complete()) { - ssize_t res = DoWriteVector(fd, vec); + while (!vec->Complete()) { + ssize_t res = DoWriteVector(fd, vec); - if (res >= 0) { - written += res; + if (res >= 0) { + written += res; - vec->Proceed((size_t) res); - } else { - { - const int err = LastSystemError(); + vec->Proceed((size_t) res); + } else { + { + const int err = LastSystemError(); - if (!IsBlocked(err)) { - return TContIOStatus(written, err); - } - } + if (!IsBlocked(err)) { + return TContIOStatus(written, err); + } + } - if ((res = PollD(cont, fd, CONT_POLL_WRITE, deadline)) != 0) { - return TContIOStatus(written, (int) res); - } - } + if ((res = PollD(cont, fd, CONT_POLL_WRITE, deadline)) != 0) { + return TContIOStatus(written, (int) res); + } + } } - return TContIOStatus::Success(written); + return TContIOStatus::Success(written); } - TContIOStatus WriteVectorT(TCont* cont, SOCKET fd, TContIOVector* vec, TDuration timeOut) noexcept { - return WriteVectorD(cont, fd, vec, timeOut.ToDeadLine()); - } + TContIOStatus WriteVectorT(TCont* cont, SOCKET fd, TContIOVector* vec, TDuration timeOut) noexcept { + return WriteVectorD(cont, fd, vec, timeOut.ToDeadLine()); + } - TContIOStatus WriteVectorI(TCont* cont, SOCKET fd, TContIOVector* vec) noexcept { - return WriteVectorD(cont, fd, vec, TInstant::Max()); - } + TContIOStatus WriteVectorI(TCont* cont, SOCKET fd, TContIOVector* vec) noexcept { + return WriteVectorD(cont, fd, vec, TInstant::Max()); + } - TContIOStatus WriteD(TCont* cont, SOCKET fd, const void* buf, size_t len, TInstant deadline) noexcept { - IOutputStream::TPart part(buf, len); - TContIOVector vec(&part, 1); - return WriteVectorD(cont, fd, &vec, deadline); - } + TContIOStatus WriteD(TCont* cont, SOCKET fd, const void* buf, size_t len, TInstant deadline) noexcept { + IOutputStream::TPart part(buf, len); + TContIOVector vec(&part, 1); + return WriteVectorD(cont, fd, &vec, deadline); + } - TContIOStatus WriteT(TCont* cont, SOCKET fd, const void* buf, size_t len, TDuration timeout) noexcept { - return WriteD(cont, fd, buf, len, timeout.ToDeadLine()); - } + TContIOStatus WriteT(TCont* cont, SOCKET fd, const void* buf, size_t len, TDuration timeout) noexcept { + return WriteD(cont, fd, buf, len, timeout.ToDeadLine()); + } - TContIOStatus WriteI(TCont* cont, SOCKET fd, const void* buf, size_t len) noexcept { - return WriteD(cont, fd, buf, len, TInstant::Max()); + TContIOStatus WriteI(TCont* cont, SOCKET fd, const void* buf, size_t len) noexcept { + return WriteD(cont, fd, buf, len, TInstant::Max()); } - int ConnectD(TCont* cont, TSocketHolder& s, const struct addrinfo& ai, TInstant deadline) noexcept { - TSocketHolder res(Socket(ai)); + int ConnectD(TCont* cont, TSocketHolder& s, const struct addrinfo& ai, TInstant deadline) noexcept { + TSocketHolder res(Socket(ai)); - if (res.Closed()) { - return LastSystemError(); - } + if (res.Closed()) { + return LastSystemError(); + } - const int ret = ConnectD(cont, res, ai.ai_addr, (socklen_t) ai.ai_addrlen, deadline); + const int ret = ConnectD(cont, res, ai.ai_addr, (socklen_t) ai.ai_addrlen, deadline); - if (!ret) { - s.Swap(res); - } - - return ret; + if (!ret) { + s.Swap(res); + } + + return ret; } - int ConnectD(TCont* cont, TSocketHolder& s, const TNetworkAddress& addr, TInstant deadline) noexcept { - int ret = EHOSTUNREACH; + int ConnectD(TCont* cont, TSocketHolder& s, const TNetworkAddress& addr, TInstant deadline) noexcept { + int ret = EHOSTUNREACH; - for (auto it = addr.Begin(); it != addr.End(); ++it) { - ret = ConnectD(cont, s, *it, deadline); + for (auto it = addr.Begin(); it != addr.End(); ++it) { + ret = ConnectD(cont, s, *it, deadline); - if (ret == 0 || ret == ETIMEDOUT) { - return ret; - } - } + if (ret == 0 || ret == ETIMEDOUT) { + return ret; + } + } - return ret; + return ret; } - int ConnectT(TCont* cont, TSocketHolder& s, const TNetworkAddress& addr, TDuration timeout) noexcept { - return ConnectD(cont, s, addr, timeout.ToDeadLine()); - } + int ConnectT(TCont* cont, TSocketHolder& s, const TNetworkAddress& addr, TDuration timeout) noexcept { + return ConnectD(cont, s, addr, timeout.ToDeadLine()); + } - int ConnectI(TCont* cont, TSocketHolder& s, const TNetworkAddress& addr) noexcept { - return ConnectD(cont, s, addr, TInstant::Max()); + int ConnectI(TCont* cont, TSocketHolder& s, const TNetworkAddress& addr) noexcept { + return ConnectD(cont, s, addr, TInstant::Max()); } - int ConnectD(TCont* cont, SOCKET s, const struct sockaddr* name, socklen_t namelen, TInstant deadline) noexcept { - if (connect(s, name, namelen)) { - const int err = LastSystemError(); - - if (!IsBlocked(err) && err != EINPROGRESS) { - return err; - } - - int ret = PollD(cont, s, CONT_POLL_WRITE, deadline); - - if (ret) { - return ret; - } - - // check if we really connected - // FIXME: Unportable ?? - int serr = 0; - socklen_t slen = sizeof(serr); - - ret = getsockopt(s, SOL_SOCKET, SO_ERROR, (char*) &serr, &slen); - - if (ret) { - return LastSystemError(); - } - - if (serr) { - return serr; - } - } - - return 0; - } - - int ConnectT(TCont* cont, SOCKET s, const struct sockaddr* name, socklen_t namelen, TDuration timeout) noexcept { - return ConnectD(cont, s, name, namelen, timeout.ToDeadLine()); - } - - int ConnectI(TCont* cont, SOCKET s, const struct sockaddr* name, socklen_t namelen) noexcept { - return ConnectD(cont, s, name, namelen, TInstant::Max()); - } - - - int AcceptD(TCont* cont, SOCKET s, struct sockaddr* addr, socklen_t* addrlen, TInstant deadline) noexcept { - SOCKET ret; - - while ((ret = Accept4(s, addr, addrlen)) == INVALID_SOCKET) { - int err = LastSystemError(); - - if (!IsBlocked(err)) { - return -err; + int ConnectD(TCont* cont, SOCKET s, const struct sockaddr* name, socklen_t namelen, TInstant deadline) noexcept { + if (connect(s, name, namelen)) { + const int err = LastSystemError(); + + if (!IsBlocked(err) && err != EINPROGRESS) { + return err; } - - err = PollD(cont, s, CONT_POLL_READ, deadline); - if (err) { - return -err; - } - } + int ret = PollD(cont, s, CONT_POLL_WRITE, deadline); + + if (ret) { + return ret; + } + + // check if we really connected + // FIXME: Unportable ?? + int serr = 0; + socklen_t slen = sizeof(serr); + + ret = getsockopt(s, SOL_SOCKET, SO_ERROR, (char*) &serr, &slen); + + if (ret) { + return LastSystemError(); + } + + if (serr) { + return serr; + } + } + + return 0; + } + + int ConnectT(TCont* cont, SOCKET s, const struct sockaddr* name, socklen_t namelen, TDuration timeout) noexcept { + return ConnectD(cont, s, name, namelen, timeout.ToDeadLine()); + } - return (int) ret; + int ConnectI(TCont* cont, SOCKET s, const struct sockaddr* name, socklen_t namelen) noexcept { + return ConnectD(cont, s, name, namelen, TInstant::Max()); } - int AcceptT(TCont* cont, SOCKET s, struct sockaddr* addr, socklen_t* addrlen, TDuration timeout) noexcept { - return AcceptD(cont, s, addr, addrlen, timeout.ToDeadLine()); - } - - int AcceptI(TCont* cont, SOCKET s, struct sockaddr* addr, socklen_t* addrlen) noexcept { - return AcceptD(cont, s, addr, addrlen, TInstant::Max()); - } - - SOCKET Socket(int domain, int type, int protocol) noexcept { - return Socket4(domain, type, protocol); - } - - SOCKET Socket(const struct addrinfo& ai) noexcept { - return Socket(ai.ai_family, ai.ai_socktype, ai.ai_protocol); - } -} + + int AcceptD(TCont* cont, SOCKET s, struct sockaddr* addr, socklen_t* addrlen, TInstant deadline) noexcept { + SOCKET ret; + + while ((ret = Accept4(s, addr, addrlen)) == INVALID_SOCKET) { + int err = LastSystemError(); + + if (!IsBlocked(err)) { + return -err; + } + + err = PollD(cont, s, CONT_POLL_READ, deadline); + + if (err) { + return -err; + } + } + + return (int) ret; + } + + int AcceptT(TCont* cont, SOCKET s, struct sockaddr* addr, socklen_t* addrlen, TDuration timeout) noexcept { + return AcceptD(cont, s, addr, addrlen, timeout.ToDeadLine()); + } + + int AcceptI(TCont* cont, SOCKET s, struct sockaddr* addr, socklen_t* addrlen) noexcept { + return AcceptD(cont, s, addr, addrlen, TInstant::Max()); + } + + SOCKET Socket(int domain, int type, int protocol) noexcept { + return Socket4(domain, type, protocol); + } + + SOCKET Socket(const struct addrinfo& ai) noexcept { + return Socket(ai.ai_family, ai.ai_socktype, ai.ai_protocol); + } +} diff --git a/library/cpp/coroutine/engine/network.h b/library/cpp/coroutine/engine/network.h index 79462be586..f2c9afe4f8 100644 --- a/library/cpp/coroutine/engine/network.h +++ b/library/cpp/coroutine/engine/network.h @@ -1,55 +1,55 @@ -#pragma once +#pragma once -#include "iostatus.h" +#include "iostatus.h" -#include <util/datetime/base.h> -#include <util/network/init.h> +#include <util/datetime/base.h> +#include <util/network/init.h> #include <util/network/iovec.h> -#include <util/network/nonblock.h> +#include <util/network/nonblock.h> #include <util/network/socket.h> class TCont; -namespace NCoro { - - int SelectD(TCont* cont, SOCKET fds[], int what[], size_t nfds, SOCKET* outfd, TInstant deadline) noexcept; - int SelectT(TCont* cont, SOCKET fds[], int what[], size_t nfds, SOCKET* outfd, TDuration timeout) noexcept; - int SelectI(TCont* cont, SOCKET fds[], int what[], size_t nfds, SOCKET* outfd); +namespace NCoro { - int PollD(TCont* cont, SOCKET fd, int what, TInstant deadline) noexcept; - int PollT(TCont* cont, SOCKET fd, int what, TDuration timeout) noexcept; - int PollI(TCont* cont, SOCKET fd, int what) noexcept; + int SelectD(TCont* cont, SOCKET fds[], int what[], size_t nfds, SOCKET* outfd, TInstant deadline) noexcept; + int SelectT(TCont* cont, SOCKET fds[], int what[], size_t nfds, SOCKET* outfd, TDuration timeout) noexcept; + int SelectI(TCont* cont, SOCKET fds[], int what[], size_t nfds, SOCKET* outfd); - TContIOStatus ReadVectorD(TCont* cont, SOCKET fd, TContIOVector* vec, TInstant deadline) noexcept; - TContIOStatus ReadVectorT(TCont* cont, SOCKET fd, TContIOVector* vec, TDuration timeOut) noexcept; - TContIOStatus ReadVectorI(TCont* cont, SOCKET fd, TContIOVector* vec) noexcept; + int PollD(TCont* cont, SOCKET fd, int what, TInstant deadline) noexcept; + int PollT(TCont* cont, SOCKET fd, int what, TDuration timeout) noexcept; + int PollI(TCont* cont, SOCKET fd, int what) noexcept; - TContIOStatus ReadD(TCont* cont, SOCKET fd, void* buf, size_t len, TInstant deadline) noexcept; - TContIOStatus ReadT(TCont* cont, SOCKET fd, void* buf, size_t len, TDuration timeout) noexcept; - TContIOStatus ReadI(TCont* cont, SOCKET fd, void* buf, size_t len) noexcept; + TContIOStatus ReadVectorD(TCont* cont, SOCKET fd, TContIOVector* vec, TInstant deadline) noexcept; + TContIOStatus ReadVectorT(TCont* cont, SOCKET fd, TContIOVector* vec, TDuration timeOut) noexcept; + TContIOStatus ReadVectorI(TCont* cont, SOCKET fd, TContIOVector* vec) noexcept; - TContIOStatus WriteVectorD(TCont* cont, SOCKET fd, TContIOVector* vec, TInstant deadline) noexcept; - TContIOStatus WriteVectorT(TCont* cont, SOCKET fd, TContIOVector* vec, TDuration timeOut) noexcept; - TContIOStatus WriteVectorI(TCont* cont, SOCKET fd, TContIOVector* vec) noexcept; + TContIOStatus ReadD(TCont* cont, SOCKET fd, void* buf, size_t len, TInstant deadline) noexcept; + TContIOStatus ReadT(TCont* cont, SOCKET fd, void* buf, size_t len, TDuration timeout) noexcept; + TContIOStatus ReadI(TCont* cont, SOCKET fd, void* buf, size_t len) noexcept; - TContIOStatus WriteD(TCont* cont, SOCKET fd, const void* buf, size_t len, TInstant deadline) noexcept; - TContIOStatus WriteT(TCont* cont, SOCKET fd, const void* buf, size_t len, TDuration timeout) noexcept; - TContIOStatus WriteI(TCont* cont, SOCKET fd, const void* buf, size_t len) noexcept; + TContIOStatus WriteVectorD(TCont* cont, SOCKET fd, TContIOVector* vec, TInstant deadline) noexcept; + TContIOStatus WriteVectorT(TCont* cont, SOCKET fd, TContIOVector* vec, TDuration timeOut) noexcept; + TContIOStatus WriteVectorI(TCont* cont, SOCKET fd, TContIOVector* vec) noexcept; - int ConnectD(TCont* cont, TSocketHolder& s, const struct addrinfo& ai, TInstant deadline) noexcept; + TContIOStatus WriteD(TCont* cont, SOCKET fd, const void* buf, size_t len, TInstant deadline) noexcept; + TContIOStatus WriteT(TCont* cont, SOCKET fd, const void* buf, size_t len, TDuration timeout) noexcept; + TContIOStatus WriteI(TCont* cont, SOCKET fd, const void* buf, size_t len) noexcept; - int ConnectD(TCont* cont, TSocketHolder& s, const TNetworkAddress& addr, TInstant deadline) noexcept; - int ConnectT(TCont* cont, TSocketHolder& s, const TNetworkAddress& addr, TDuration timeout) noexcept; - int ConnectI(TCont* cont, TSocketHolder& s, const TNetworkAddress& addr) noexcept; + int ConnectD(TCont* cont, TSocketHolder& s, const struct addrinfo& ai, TInstant deadline) noexcept; - int ConnectD(TCont* cont, SOCKET s, const struct sockaddr* name, socklen_t namelen, TInstant deadline) noexcept; - int ConnectT(TCont* cont, SOCKET s, const struct sockaddr* name, socklen_t namelen, TDuration timeout) noexcept; - int ConnectI(TCont* cont, SOCKET s, const struct sockaddr* name, socklen_t namelen) noexcept; + int ConnectD(TCont* cont, TSocketHolder& s, const TNetworkAddress& addr, TInstant deadline) noexcept; + int ConnectT(TCont* cont, TSocketHolder& s, const TNetworkAddress& addr, TDuration timeout) noexcept; + int ConnectI(TCont* cont, TSocketHolder& s, const TNetworkAddress& addr) noexcept; - int AcceptD(TCont* cont, SOCKET s, struct sockaddr* addr, socklen_t* addrlen, TInstant deadline) noexcept; - int AcceptT(TCont* cont, SOCKET s, struct sockaddr* addr, socklen_t* addrlen, TDuration timeout) noexcept; - int AcceptI(TCont* cont, SOCKET s, struct sockaddr* addr, socklen_t* addrlen) noexcept; + int ConnectD(TCont* cont, SOCKET s, const struct sockaddr* name, socklen_t namelen, TInstant deadline) noexcept; + int ConnectT(TCont* cont, SOCKET s, const struct sockaddr* name, socklen_t namelen, TDuration timeout) noexcept; + int ConnectI(TCont* cont, SOCKET s, const struct sockaddr* name, socklen_t namelen) noexcept; - SOCKET Socket(int domain, int type, int protocol) noexcept; - SOCKET Socket(const struct addrinfo& ai) noexcept; + int AcceptD(TCont* cont, SOCKET s, struct sockaddr* addr, socklen_t* addrlen, TInstant deadline) noexcept; + int AcceptT(TCont* cont, SOCKET s, struct sockaddr* addr, socklen_t* addrlen, TDuration timeout) noexcept; + int AcceptI(TCont* cont, SOCKET s, struct sockaddr* addr, socklen_t* addrlen) noexcept; + + SOCKET Socket(int domain, int type, int protocol) noexcept; + SOCKET Socket(const struct addrinfo& ai) noexcept; } diff --git a/library/cpp/coroutine/engine/poller.cpp b/library/cpp/coroutine/engine/poller.cpp index 4efabd0c7e..61164fa56b 100644 --- a/library/cpp/coroutine/engine/poller.cpp +++ b/library/cpp/coroutine/engine/poller.cpp @@ -5,30 +5,30 @@ #include <util/generic/intrlist.h> #include <util/generic/singleton.h> #include <util/system/env.h> -#include <util/string/cast.h> +#include <util/string/cast.h> namespace { - using TChange = IPollerFace::TChange; - using TEvent = IPollerFace::TEvent; - using TEvents = IPollerFace::TEvents; + using TChange = IPollerFace::TChange; + using TEvent = IPollerFace::TEvent; + using TEvents = IPollerFace::TEvents; template <class T> class TUnsafeBuf { public: - TUnsafeBuf() noexcept + TUnsafeBuf() noexcept : L_(0) { } - T* operator~() const noexcept { + T* operator~() const noexcept { return B_.Get(); } - size_t operator+() const noexcept { + size_t operator+() const noexcept { return L_; } - void Reserve(size_t len) { + void Reserve(size_t len) { len = FastClp2(len); if (len > L_) { @@ -42,7 +42,7 @@ namespace { size_t L_; }; - + template <class T> class TVirtualize: public IPollerFace { public: @@ -67,25 +67,25 @@ namespace { const EContPoller PollerEngine_; }; - + template <class T> class TPoller { - using TInternalEvent = typename T::TEvent; + using TInternalEvent = typename T::TEvent; public: - TPoller() { + TPoller() { E_.Reserve(1); } - void Set(const TChange& c) { + void Set(const TChange& c) { P_.Set(c.Data, c.Fd, c.Flags); } - void Reserve(size_t size) { + void Reserve(size_t size) { E_.Reserve(size); } - void Wait(TEvents& events, TInstant deadLine) { + void Wait(TEvents& events, TInstant deadLine) { const size_t ret = P_.WaitD(~E_, +E_, deadLine); events.reserve(ret); @@ -110,21 +110,21 @@ namespace { TUnsafeBuf<TInternalEvent> E_; }; - + template <class T> class TIndexedArray { - struct TVal: - public T, - public TIntrusiveListItem<TVal>, - public TObjectFromPool<TVal> - { + struct TVal: + public T, + public TIntrusiveListItem<TVal>, + public TObjectFromPool<TVal> + { // NOTE Constructor must be user-defined (and not =default) here // because TVal objects are created in the UB-capable placement // TObjectFromPool::new operator that stores data in a memory // allocated for the object. Without user defined constructor // zero-initialization takes place in TVal() expression and the // data is overwritten. - TVal() { + TVal() { } }; @@ -134,32 +134,32 @@ namespace { typedef typename TListType::TIterator TIterator; typedef typename TListType::TConstIterator TConstIterator; - TIndexedArray() + TIndexedArray() : P_(TMemoryPool::TExpGrow::Instance(), TDefaultAllocator::Instance()) { } - TIterator Begin() noexcept { + TIterator Begin() noexcept { return I_.Begin(); } - TIterator End() noexcept { + TIterator End() noexcept { return I_.End(); } - TConstIterator Begin() const noexcept { + TConstIterator Begin() const noexcept { return I_.Begin(); } - TConstIterator End() const noexcept { + TConstIterator End() const noexcept { return I_.End(); } - T& operator[](size_t i) { + T& operator[](size_t i) { return *Get(i); } - T* Get(size_t i) { + T* Get(size_t i) { TValRef& v = V_.Get(i); if (Y_UNLIKELY(!v)) { @@ -172,22 +172,22 @@ namespace { return v.Get(); } - void Erase(size_t i) noexcept { + void Erase(size_t i) noexcept { V_.Get(i).Destroy(); } - size_t Size() const noexcept { + size_t Size() const noexcept { return I_.Size(); } private: - using TValRef = THolder<TVal>; + using TValRef = THolder<TVal>; typename TVal::TPool P_; TSocketMap<TValRef> V_; TListType I_; }; - + inline short PollFlags(ui16 flags) noexcept { short ret = 0; @@ -208,15 +208,15 @@ namespace { return ret; } - + class TPollPoller { public: - size_t Size() const noexcept { + size_t Size() const noexcept { return S_.Size(); } template <class T> - void Build(T& t) const { + void Build(T& t) const { for (TFds::TConstIterator it = S_.Begin(); it != S_.End(); ++it) { t.Set(*it); } @@ -224,7 +224,7 @@ namespace { t.Reserve(Size()); } - void Set(const TChange& c) { + void Set(const TChange& c) { if (c.Flags) { S_[c.Fd] = c; } else { @@ -232,7 +232,7 @@ namespace { } } - void Wait(TEvents& events, TInstant deadLine) { + void Wait(TEvents& events, TInstant deadLine) { T_.clear(); T_.reserve(Size()); @@ -265,8 +265,8 @@ namespace { int status = 0; ui16 filter = 0; - // We are perfectly fine with an EOF while reading a pipe or a unix socket - if ((ev & POLLIN) || (ev & POLLHUP) && (pfd.events & POLLIN)) { + // We are perfectly fine with an EOF while reading a pipe or a unix socket + if ((ev & POLLIN) || (ev & POLLHUP) && (pfd.events & POLLIN)) { filter |= CONT_POLL_READ; } @@ -310,16 +310,16 @@ namespace { TPollVec T_; }; - + class TCombinedPoller { typedef TPoller<TPollerImpl<TWithoutLocking>> TDefaultPoller; public: - TCombinedPoller() { + TCombinedPoller() { P_.Reset(new TPollPoller()); } - void Set(const TChange& c) { + void Set(const TChange& c) { if (!P_) { D_->Set(c); } else { @@ -327,7 +327,7 @@ namespace { } } - void Wait(TEvents& events, TInstant deadLine) { + void Wait(TEvents& events, TInstant deadLine) { if (!P_) { D_->Wait(events, deadLine); } else { @@ -343,48 +343,48 @@ namespace { } private: - THolder<TPollPoller> P_; - THolder<TDefaultPoller> D_; + THolder<TPollPoller> P_; + THolder<TDefaultPoller> D_; }; struct TUserPoller: public TString { - TUserPoller() + TUserPoller() : TString(GetEnv("USER_POLLER")) { } }; } -THolder<IPollerFace> IPollerFace::Default() { +THolder<IPollerFace> IPollerFace::Default() { return Construct(*SingletonWithPriority<TUserPoller, 0>()); } -THolder<IPollerFace> IPollerFace::Construct(TStringBuf name) { - return Construct(name ? FromString<EContPoller>(name) : EContPoller::Default); -} +THolder<IPollerFace> IPollerFace::Construct(TStringBuf name) { + return Construct(name ? FromString<EContPoller>(name) : EContPoller::Default); +} -THolder<IPollerFace> IPollerFace::Construct(EContPoller poller) { - switch (poller) { - case EContPoller::Default: +THolder<IPollerFace> IPollerFace::Construct(EContPoller poller) { + switch (poller) { + case EContPoller::Default: case EContPoller::Combined: return MakeHolder<TVirtualize<TCombinedPoller>>(EContPoller::Combined); - case EContPoller::Select: + case EContPoller::Select: return MakeHolder<TVirtualize<TPoller<TGenericPoller<TSelectPoller<TWithoutLocking>>>>>(poller); - case EContPoller::Poll: + case EContPoller::Poll: return MakeHolder<TVirtualize<TPollPoller>>(poller); - case EContPoller::Epoll: + case EContPoller::Epoll: #if defined(HAVE_EPOLL_POLLER) return MakeHolder<TVirtualize<TPoller<TGenericPoller<TEpollPoller<TWithoutLocking>>>>>(poller); -#else - return nullptr; +#else + return nullptr; #endif - case EContPoller::Kqueue: -#if defined(HAVE_KQUEUE_POLLER) + case EContPoller::Kqueue: +#if defined(HAVE_KQUEUE_POLLER) return MakeHolder<TVirtualize<TPoller<TGenericPoller<TKqueuePoller<TWithoutLocking>>>>>(poller); -#else - return nullptr; +#else + return nullptr; #endif - default: - Y_FAIL("bad poller type"); + default: + Y_FAIL("bad poller type"); } } diff --git a/library/cpp/coroutine/engine/poller.h b/library/cpp/coroutine/engine/poller.h index 73d482cfaf..8ea012c0fc 100644 --- a/library/cpp/coroutine/engine/poller.h +++ b/library/cpp/coroutine/engine/poller.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <util/generic/ptr.h> #include <util/generic/vector.h> @@ -6,15 +6,15 @@ #include <util/network/pollerimpl.h> #include <util/datetime/base.h> -enum class EContPoller { - Default /* "default" */, +enum class EContPoller { + Default /* "default" */, Combined /* "combined" */, - Select /* "select" */, - Poll /* "poll" */, - Epoll /* "epoll" */, - Kqueue /* "kqueue" */ -}; - + Select /* "select" */, + Poll /* "poll" */, + Epoll /* "epoll" */, + Kqueue /* "kqueue" */ +}; + class IPollerFace { public: struct TChange { @@ -29,12 +29,12 @@ public: ui16 Filter; }; - using TEvents = TVector<TEvent>; + using TEvents = TVector<TEvent>; virtual ~IPollerFace() { } - void Set(void* ptr, SOCKET fd, ui16 flags) { + void Set(void* ptr, SOCKET fd, ui16 flags) { const TChange c = {fd, ptr, flags}; Set(c); @@ -44,7 +44,7 @@ public: virtual void Wait(TEvents& events, TInstant deadLine) = 0; virtual EContPoller PollEngine() const = 0; - static THolder<IPollerFace> Default(); - static THolder<IPollerFace> Construct(TStringBuf name); - static THolder<IPollerFace> Construct(EContPoller poller); + static THolder<IPollerFace> Default(); + static THolder<IPollerFace> Construct(TStringBuf name); + static THolder<IPollerFace> Construct(EContPoller poller); }; diff --git a/library/cpp/coroutine/engine/sockmap.h b/library/cpp/coroutine/engine/sockmap.h index b4a3af73f6..fd189e1774 100644 --- a/library/cpp/coroutine/engine/sockmap.h +++ b/library/cpp/coroutine/engine/sockmap.h @@ -6,7 +6,7 @@ template <class T> class TSocketMap { public: - T& Get(size_t idx) { + T& Get(size_t idx) { if (idx < 128000) { if (V_.size() <= idx) { V_.resize(idx + 1); diff --git a/library/cpp/coroutine/engine/sockpool.cpp b/library/cpp/coroutine/engine/sockpool.cpp index 39b97ecd85..b9482e780f 100644 --- a/library/cpp/coroutine/engine/sockpool.cpp +++ b/library/cpp/coroutine/engine/sockpool.cpp @@ -31,7 +31,7 @@ TPooledSocket TSocketPool::AllocateMore(TConnectData* conn) { TCont* cont = conn->Cont; while (true) { - TSocketHolder s(NCoro::Socket(Addr_->Addr()->sa_family, SOCK_STREAM, 0)); + TSocketHolder s(NCoro::Socket(Addr_->Addr()->sa_family, SOCK_STREAM, 0)); if (s == INVALID_SOCKET) { ythrow TSystemError(errno) << TStringBuf("can not create socket"); @@ -40,7 +40,7 @@ TPooledSocket TSocketPool::AllocateMore(TConnectData* conn) { SetCommonSockOpts(s, Addr_->Addr()); SetZeroLinger(s); - const int ret = NCoro::ConnectD(cont, s, Addr_->Addr(), Addr_->Len(), conn->DeadLine); + const int ret = NCoro::ConnectD(cont, s, Addr_->Addr(), Addr_->Len(), conn->DeadLine); if (ret == EINTR) { continue; diff --git a/library/cpp/coroutine/engine/sockpool.h b/library/cpp/coroutine/engine/sockpool.h index b343b65fb6..1ebb7e7b38 100644 --- a/library/cpp/coroutine/engine/sockpool.h +++ b/library/cpp/coroutine/engine/sockpool.h @@ -1,7 +1,7 @@ -#pragma once +#pragma once #include "impl.h" -#include "network.h" +#include "network.h" #include <util/network/address.h> #include <util/network/socket.h> @@ -14,7 +14,7 @@ class TSocketPool; class TPooledSocket { class TImpl: public TIntrusiveListItem<TImpl>, public TSimpleRefCount<TImpl, TImpl> { public: - TImpl(SOCKET fd, TSocketPool* pool) noexcept + TImpl(SOCKET fd, TSocketPool* pool) noexcept : Pool_(pool) , IsKeepAlive_(false) , Fd_(fd) @@ -22,11 +22,11 @@ class TPooledSocket { Touch(); } - static void Destroy(TImpl* impl) noexcept { + static void Destroy(TImpl* impl) noexcept { impl->DoDestroy(); } - void DoDestroy() noexcept { + void DoDestroy() noexcept { if (!Closed() && IsKeepAlive() && IsInGoodState()) { ReturnToPool(); } else { @@ -34,28 +34,28 @@ class TPooledSocket { } } - bool IsKeepAlive() const noexcept { + bool IsKeepAlive() const noexcept { return IsKeepAlive_; } - void SetKeepAlive(bool ka) { + void SetKeepAlive(bool ka) { ::SetKeepAlive(Fd_, ka); IsKeepAlive_ = ka; } - SOCKET Socket() const noexcept { + SOCKET Socket() const noexcept { return Fd_; } - bool Closed() const noexcept { + bool Closed() const noexcept { return Fd_.Closed(); } - void Close() noexcept { + void Close() noexcept { Fd_.Close(); } - bool IsInGoodState() const noexcept { + bool IsInGoodState() const noexcept { int err = 0; socklen_t len = sizeof(err); @@ -64,15 +64,15 @@ class TPooledSocket { return !err; } - bool IsOpen() const noexcept { - return IsInGoodState() && IsNotSocketClosedByOtherSide(Fd_); + bool IsOpen() const noexcept { + return IsInGoodState() && IsNotSocketClosedByOtherSide(Fd_); } - void Touch() noexcept { + void Touch() noexcept { TouchTime_ = TInstant::Now(); } - const TInstant& LastTouch() const noexcept { + const TInstant& LastTouch() const noexcept { return TouchTime_; } @@ -89,31 +89,31 @@ class TPooledSocket { friend class TSocketPool; public: - TPooledSocket() + TPooledSocket() : Impl_(nullptr) { } - TPooledSocket(TImpl* impl) + TPooledSocket(TImpl* impl) : Impl_(impl) { } - ~TPooledSocket() { + ~TPooledSocket() { if (UncaughtException() && !!Impl_) { Close(); } } - operator SOCKET() const noexcept { + operator SOCKET() const noexcept { return Impl_->Socket(); } - void SetKeepAlive(bool ka) { + void SetKeepAlive(bool ka) { Impl_->SetKeepAlive(ka); } - void Close() noexcept { + void Close() noexcept { Impl_->Close(); } @@ -122,13 +122,13 @@ private: }; struct TConnectData { - TConnectData(TCont* cont, const TInstant& deadLine) + TConnectData(TCont* cont, const TInstant& deadLine) : Cont(cont) , DeadLine(deadLine) { } - TConnectData(TCont* cont, const TDuration& timeOut) + TConnectData(TCont* cont, const TDuration& timeOut) : Cont(cont) , DeadLine(TInstant::Now() + timeOut) { @@ -144,17 +144,17 @@ class TSocketPool { public: typedef TAtomicSharedPtr<NAddr::IRemoteAddr> TAddrRef; - TSocketPool(int ip, int port) + TSocketPool(int ip, int port) : Addr_(new NAddr::TIPv4Addr(TIpAddress((ui32)ip, (ui16)port))) { } - TSocketPool(const TAddrRef& addr) + TSocketPool(const TAddrRef& addr) : Addr_(addr) { } - void EraseStale(const TInstant& maxAge) noexcept { + void EraseStale(const TInstant& maxAge) noexcept { TSockets toDelete; { @@ -170,7 +170,7 @@ public: } } - TPooledSocket Get(TConnectData* conn) { + TPooledSocket Get(TConnectData* conn) { TPooledSocket ret; if (TPooledSocket::TImpl* alive = GetImpl()) { @@ -184,7 +184,7 @@ public: return ret; } - bool GetAlive(TPooledSocket& socket) { + bool GetAlive(TPooledSocket& socket) { if (TPooledSocket::TImpl* alive = GetImpl()) { alive->Touch(); socket = TPooledSocket(alive); @@ -194,7 +194,7 @@ public: } private: - TPooledSocket::TImpl* GetImpl() { + TPooledSocket::TImpl* GetImpl() { TGuard<TMutex> guard(Mutex_); while (!Pool_.Empty()) { @@ -207,7 +207,7 @@ private: return nullptr; } - void Release(TPooledSocket::TImpl* impl) noexcept { + void Release(TPooledSocket::TImpl* impl) noexcept { TGuard<TMutex> guard(Mutex_); Pool_.PushFront(impl); @@ -217,7 +217,7 @@ private: private: TAddrRef Addr_; - using TSockets = TIntrusiveListWithAutoDelete<TPooledSocket::TImpl, TDelete>; + using TSockets = TIntrusiveListWithAutoDelete<TPooledSocket::TImpl, TDelete>; TSockets Pool_; TMutex Mutex_; }; @@ -226,24 +226,24 @@ inline void TPooledSocket::TImpl::ReturnToPool() noexcept { Pool_->Release(this); } - + class TContIO: public IInputStream, public IOutputStream { public: - TContIO(SOCKET fd, TCont* cont) + TContIO(SOCKET fd, TCont* cont) : Fd_(fd) , Cont_(cont) { } void DoWrite(const void* buf, size_t len) override { - NCoro::WriteI(Cont_, Fd_, buf, len).Checked(); + NCoro::WriteI(Cont_, Fd_, buf, len).Checked(); } size_t DoRead(void* buf, size_t len) override { - return NCoro::ReadI(Cont_, Fd_, buf, len).Checked(); + return NCoro::ReadI(Cont_, Fd_, buf, len).Checked(); } - SOCKET Fd() const noexcept { + SOCKET Fd() const noexcept { return Fd_; } diff --git a/library/cpp/coroutine/engine/trampoline.cpp b/library/cpp/coroutine/engine/trampoline.cpp index 38e3951a51..10ea69ddc3 100644 --- a/library/cpp/coroutine/engine/trampoline.cpp +++ b/library/cpp/coroutine/engine/trampoline.cpp @@ -1,50 +1,50 @@ -#include "impl.h" -#include "trampoline.h" +#include "impl.h" +#include "trampoline.h" #include "stack/stack_allocator.h" -#include <util/system/info.h> -#include <util/system/protect.h> -#include <util/system/valgrind.h> +#include <util/system/info.h> +#include <util/system/protect.h> +#include <util/system/valgrind.h> #include <util/system/yassert.h> -#include <cstdlib> -#include <util/stream/format.h> +#include <cstdlib> +#include <util/stream/format.h> -namespace NCoro { - +namespace NCoro { + TTrampoline::TTrampoline(NStack::IAllocator& allocator, ui32 stackSize, TFunc f, TCont* cont) noexcept : Stack_(allocator, stackSize, cont->Name()) , Clo_{this, Stack_.Get(), cont->Name()} - , Ctx_(Clo_) + , Ctx_(Clo_) , Func_(std::move(f)) - , Cont_(cont) - {} - - void TTrampoline::DoRun() { + , Cont_(cont) + {} + + void TTrampoline::DoRun() { if (Cont_->Executor()->FailOnError()) { Func_(Cont_); } else { try { Func_(Cont_); } catch (...) {} - } - - Cont_->Terminate(); - } - - TArrayRef<char> TTrampoline::Stack() noexcept { - return Stack_.Get(); - } - + } + + Cont_->Terminate(); + } + + TArrayRef<char> TTrampoline::Stack() noexcept { + return Stack_.Get(); + } + const char* TTrampoline::ContName() const noexcept { return Cont_->Name(); } - + void TTrampoline::DoRunNaked() { DoRun(); abort(); } -} +} diff --git a/library/cpp/coroutine/engine/trampoline.h b/library/cpp/coroutine/engine/trampoline.h index 5ece7873b0..37b61cf015 100644 --- a/library/cpp/coroutine/engine/trampoline.h +++ b/library/cpp/coroutine/engine/trampoline.h @@ -1,60 +1,60 @@ -#pragma once +#pragma once #include "stack/stack_common.h" #include "stack/stack.h" -#include <util/generic/noncopyable.h> -#include <util/generic/ptr.h> +#include <util/generic/noncopyable.h> +#include <util/generic/ptr.h> #include <util/system/context.h> #include <util/system/defaults.h> -#if !defined(STACK_GROW_DOWN) -# error "unsupported" -#endif - +#if !defined(STACK_GROW_DOWN) +# error "unsupported" +#endif + class TCont; -typedef void (*TContFunc)(TCont*, void*); +typedef void (*TContFunc)(TCont*, void*); + +namespace NCoro { -namespace NCoro { - namespace NStack { class IAllocator; - } - - class TTrampoline : public ITrampoLine, TNonCopyable { - public: + } + + class TTrampoline : public ITrampoLine, TNonCopyable { + public: typedef std::function<void (TCont*)> TFunc; - TTrampoline( + TTrampoline( NCoro::NStack::IAllocator& allocator, uint32_t stackSize, TFunc f, TCont* cont - ) noexcept; + ) noexcept; + + TArrayRef<char> Stack() noexcept; - TArrayRef<char> Stack() noexcept; + TExceptionSafeContext* Context() noexcept { + return &Ctx_; + } - TExceptionSafeContext* Context() noexcept { - return &Ctx_; - } - - void SwitchTo(TExceptionSafeContext* ctx) noexcept { + void SwitchTo(TExceptionSafeContext* ctx) noexcept { Y_VERIFY(Stack_.LowerCanaryOk(), "Stack overflow (%s)", ContName()); Y_VERIFY(Stack_.UpperCanaryOk(), "Stack override (%s)", ContName()); - Ctx_.SwitchTo(ctx); - } - + Ctx_.SwitchTo(ctx); + } + void DoRun() override; - + void DoRunNaked() override; - private: + private: const char* ContName() const noexcept; private: NStack::TStackHolder Stack_; - const TContClosure Clo_; - TExceptionSafeContext Ctx_; + const TContClosure Clo_; + TExceptionSafeContext Ctx_; TFunc Func_; - TCont* const Cont_; + TCont* const Cont_; }; } diff --git a/library/cpp/coroutine/engine/ya.make b/library/cpp/coroutine/engine/ya.make index 2ea3c237c7..8c20b9afc3 100644 --- a/library/cpp/coroutine/engine/ya.make +++ b/library/cpp/coroutine/engine/ya.make @@ -1,24 +1,24 @@ LIBRARY() -OWNER( - pg - g:balancer -) +OWNER( + pg + g:balancer +) GENERATE_ENUM_SERIALIZATION(poller.h) GENERATE_ENUM_SERIALIZATION(stack/stack_common.h) - + PEERDIR( contrib/libs/libc_compat library/cpp/containers/intrusive_rb_tree ) SRCS( - cont_poller.cpp + cont_poller.cpp helper.cpp impl.cpp iostatus.cpp - network.cpp + network.cpp poller.cpp sockpool.cpp stack/stack.cpp @@ -26,7 +26,7 @@ SRCS( stack/stack_guards.cpp stack/stack_storage.cpp stack/stack_utils.cpp - trampoline.cpp + trampoline.cpp ) END() diff --git a/library/cpp/coroutine/listener/listen.cpp b/library/cpp/coroutine/listener/listen.cpp index 02441c879b..3d4e711d1d 100644 --- a/library/cpp/coroutine/listener/listen.cpp +++ b/library/cpp/coroutine/listener/listen.cpp @@ -107,7 +107,7 @@ private: SetDeferAccept(ListenSocket_); } - C_ = Parent_->E_->Create<TOneSocketListener, &TOneSocketListener::Run>(this, "listen_job"); + C_ = Parent_->E_->Create<TOneSocketListener, &TOneSocketListener::Run>(this, "listen_job"); } } @@ -120,7 +120,7 @@ private: C_->Cancel(); while (C_) { - Parent_->E_->Running()->Yield(); + Parent_->E_->Running()->Yield(); } } } @@ -130,7 +130,7 @@ private: while (!C_->Cancelled()) { try { TOpaqueAddr remote; - const int res = NCoro::AcceptI(C_, ListenSocket_, remote.MutableAddr(), remote.LenPtr()); + const int res = NCoro::AcceptI(C_, ListenSocket_, remote.MutableAddr(), remote.LenPtr()); if (res < 0) { const int err = -res; @@ -277,16 +277,16 @@ TContListener::TContListener(ICallBack* cb, TContExecutor* e, const TOptions& op TContListener::~TContListener() { } -namespace { - template <class T> - static inline T&& CheckImpl(T&& impl) { - Y_ENSURE_EX(impl, yexception() << "not running"); - return std::forward<T>(impl); +namespace { + template <class T> + static inline T&& CheckImpl(T&& impl) { + Y_ENSURE_EX(impl, yexception() << "not running"); + return std::forward<T>(impl); } -} +} void TContListener::Listen(const IRemoteAddr& addr) { - CheckImpl(Impl_)->Listen(addr); + CheckImpl(Impl_)->Listen(addr); } void TContListener::Listen(const TIpAddress& addr) { @@ -300,11 +300,11 @@ void TContListener::Listen(const TNetworkAddress& addr) { } void TContListener::Listen() { - CheckImpl(Impl_)->Listen(); + CheckImpl(Impl_)->Listen(); } void TContListener::Bind(const IRemoteAddr& addr) { - CheckImpl(Impl_)->Bind(addr); + CheckImpl(Impl_)->Bind(addr); } void TContListener::Bind(const TIpAddress& addr) { @@ -312,7 +312,7 @@ void TContListener::Bind(const TIpAddress& addr) { } void TContListener::Bind(const TNetworkAddress& addr) { - CheckImpl(Impl_)->Bind(addr); + CheckImpl(Impl_)->Bind(addr); } void TContListener::Stop() noexcept { @@ -320,7 +320,7 @@ void TContListener::Stop() noexcept { } void TContListener::StopListenAddr(const IRemoteAddr& addr) { - CheckImpl(Impl_)->StopListenAddr(addr); + CheckImpl(Impl_)->StopListenAddr(addr); } void TContListener::StopListenAddr(const TIpAddress& addr) { diff --git a/library/cpp/coroutine/listener/listen.h b/library/cpp/coroutine/listener/listen.h index 998284ec95..3a89cd3ecc 100644 --- a/library/cpp/coroutine/listener/listen.h +++ b/library/cpp/coroutine/listener/listen.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <util/generic/ptr.h> #include <util/generic/ylimits.h> diff --git a/library/cpp/deprecated/accessors/accessors.h b/library/cpp/deprecated/accessors/accessors.h index 3ecdc2aab5..6d4b1da3ad 100644 --- a/library/cpp/deprecated/accessors/accessors.h +++ b/library/cpp/deprecated/accessors/accessors.h @@ -1,38 +1,38 @@ -#pragma once - -#include "accessors_impl.h" - -namespace NAccessors { +#pragma once + +#include "accessors_impl.h" + +namespace NAccessors { /* - * Adds API compatibility between different types representing memory regions. - * - * i.e. this will work: - * + * Adds API compatibility between different types representing memory regions. + * + * i.e. this will work: + * * TString t; - * const char* beg = NAccessors::Begin(t); // t.begin() - * const char* end = NAccessors::End(t); // t.end() - * size_t sz = NAccessors::Size(t); // t.size() - * - * as well as this: - * - * ui64 t; - * const ui64* beg = NAccessors::Begin(t); // &t - * const ui64* end = NAccessors::End(t); // &t + 1 - * size_t sz = NAccessors::Size(t); // 1 - * - * Both will give you begin, end and size of the underlying memory region. - */ - + * const char* beg = NAccessors::Begin(t); // t.begin() + * const char* end = NAccessors::End(t); // t.end() + * size_t sz = NAccessors::Size(t); // t.size() + * + * as well as this: + * + * ui64 t; + * const ui64* beg = NAccessors::Begin(t); // &t + * const ui64* end = NAccessors::End(t); // &t + 1 + * size_t sz = NAccessors::Size(t); // 1 + * + * Both will give you begin, end and size of the underlying memory region. + */ + template <typename T> inline const typename TMemoryTraits<T>::TElementType* Begin(const T& t) { return NPrivate::TBegin<T>::Get(t); } - + template <typename T> inline const typename TMemoryTraits<T>::TElementType* End(const T& t) { return NPrivate::TEnd<T>::Get(t); } - + template <typename T> inline size_t Size(const T& t) { return End(t) - Begin(t); @@ -80,4 +80,4 @@ namespace NAccessors { const typename TMemoryTraits<T>::TElementType* end) { NPrivate::TAssign<T>::Do(t, beg, end); } -} +} diff --git a/library/cpp/deprecated/accessors/accessors_impl.h b/library/cpp/deprecated/accessors/accessors_impl.h index f73d0ad577..6b2b987351 100644 --- a/library/cpp/deprecated/accessors/accessors_impl.h +++ b/library/cpp/deprecated/accessors/accessors_impl.h @@ -1,8 +1,8 @@ -#pragma once - -#include "memory_traits.h" - -namespace NAccessors { +#pragma once + +#include "memory_traits.h" + +namespace NAccessors { namespace NPrivate { template <typename Ta> struct TMemoryAccessorBase { @@ -10,33 +10,33 @@ namespace NAccessors { SimpleMemory = TMemoryTraits<Ta>::SimpleMemory, ContinuousMemory = TMemoryTraits<Ta>::ContinuousMemory, }; - + struct TBadAccessor; }; - + template <typename Ta> struct TBegin: public TMemoryAccessorBase<Ta> { using TElementType = typename TMemoryTraits<Ta>::TElementType; - + template <typename Tb> struct TNoMemoryIndirectionBegin { static const TElementType* Get(const Tb& b) { return (const TElementType*)&b; } }; - + template <typename Tb> struct TIndirectMemoryRegionBegin { Y_HAS_MEMBER(Begin); Y_HAS_MEMBER(begin); - + template <typename Tc> struct TByBegin { static const TElementType* Get(const Tc& b) { return (const TElementType*)b.Begin(); } }; - + template <typename Tc> struct TBybegin { static const TElementType* Get(const Tc& b) { @@ -61,46 +61,46 @@ namespace NAccessors { static const TElementType* Get(const Ta& b) { return TGet::Get(b); - } - }; - + } + }; + template <typename Ta> struct TEnd: public TMemoryAccessorBase<Ta> { using TElementType = typename TMemoryTraits<Ta>::TElementType; - + template <typename Tb> struct TNoMemoryIndirectionEnd { static const TElementType* Get(const Tb& b) { return (const TElementType*)(&b + 1); } }; - + template <typename Tb> struct TIndirectMemoryRegionEnd { Y_HAS_MEMBER(End); Y_HAS_MEMBER(end); - + template <typename Tc> struct TByEnd { static const TElementType* Get(const Tc& b) { return (const TElementType*)b.End(); } }; - + template <typename Tc> struct TByend { static const TElementType* Get(const Tc& b) { return (const TElementType*)b.end(); } }; - + using TGet = std::conditional_t<THasEnd<Tb>::value, TByEnd<Tb>, TByend<Tb>>; - + static const TElementType* Get(const Tb& b) { return TGet::Get(b); } }; - + using TGet = std::conditional_t< TMemoryAccessorBase<Ta>::SimpleMemory, TNoMemoryIndirectionEnd<Ta>, @@ -108,11 +108,11 @@ namespace NAccessors { TMemoryAccessorBase<Ta>::ContinuousMemory, TIndirectMemoryRegionEnd<Ta>, typename TMemoryAccessorBase<Ta>::TBadAccessor>>; - + static const TElementType* Get(const Ta& b) { return TGet::Get(b); - } - }; + } + }; template <typename Ta, bool Init> struct TClear: public TMemoryAccessorBase<Ta> { @@ -346,39 +346,39 @@ namespace NAccessors { template <typename Ta> struct TAssign: public TMemoryAccessorBase<Ta> { using TElementType = typename TMemoryTraits<Ta>::TElementType; - + template <typename Tb> struct TNoMemoryIndirectionAssign { static void Do(Tb& b, const TElementType* beg, const TElementType* end) { - if (sizeof(Tb) == sizeof(TElementType) && end - beg > 0) { - memcpy(&b, beg, sizeof(Tb)); - } else if (end - beg > 0) { - memcpy(&b, beg, Min<size_t>((end - beg) * sizeof(TElementType), sizeof(Tb))); - } else { - Zero(b); - } + if (sizeof(Tb) == sizeof(TElementType) && end - beg > 0) { + memcpy(&b, beg, sizeof(Tb)); + } else if (end - beg > 0) { + memcpy(&b, beg, Min<size_t>((end - beg) * sizeof(TElementType), sizeof(Tb))); + } else { + Zero(b); + } } }; - + template <typename Tb> struct TIndirectMemoryRegionAssign { Y_HAS_MEMBER(Assign); Y_HAS_MEMBER(assign); - + template <typename Tc> struct TByAssign { static void Do(Tc& b, const TElementType* beg, const TElementType* end) { b.Assign(beg, end); } }; - + template <typename Tc> struct TByassign { static void Do(Tc& b, const TElementType* beg, const TElementType* end) { b.assign(beg, end); } }; - + template <typename Tc> struct TByClearAppend { static void Do(Tc& b, const TElementType* beg, const TElementType* end) { @@ -386,14 +386,14 @@ namespace NAccessors { TAppendRegion<Tc>::Do(b, beg, end); } }; - + template <typename Tc> struct TByConstruction { static void Do(Tc& b, const TElementType* beg, const TElementType* end) { b = Tc(beg, end); } }; - + using TDo = std::conditional_t< THasAssign<Tb>::value, TByAssign<Tb>, @@ -404,17 +404,17 @@ namespace NAccessors { TMemoryTraits<Tb>::OwnsMemory, TByClearAppend<Tb>, TByConstruction<Tb>>>>; - + static void Do(Tb& b, const TElementType* beg, const TElementType* end) { TDo::Do(b, beg, end); } }; - + using TDo = std::conditional_t<TMemoryAccessorBase<Ta>::SimpleMemory, TNoMemoryIndirectionAssign<Ta>, TIndirectMemoryRegionAssign<Ta>>; - + static void Do(Ta& b, const TElementType* beg, const TElementType* end) { TDo::Do(b, beg, end); - } - }; - } -} + } + }; + } +} diff --git a/library/cpp/deprecated/accessors/accessors_ut.cpp b/library/cpp/deprecated/accessors/accessors_ut.cpp index 80453964fd..a9bdc9fcc4 100644 --- a/library/cpp/deprecated/accessors/accessors_ut.cpp +++ b/library/cpp/deprecated/accessors/accessors_ut.cpp @@ -1,27 +1,27 @@ -#include "accessors.h" - +#include "accessors.h" + #include <library/cpp/testing/unittest/registar.h> -#include <util/generic/buffer.h> -#include <util/generic/vector.h> - +#include <util/generic/buffer.h> +#include <util/generic/vector.h> + #include <array> -class TAccessorsTest: public TTestBase { +class TAccessorsTest: public TTestBase { UNIT_TEST_SUITE(TAccessorsTest); UNIT_TEST(TestAccessors); UNIT_TEST_SUITE_END(); -private: - template <typename T> - void TestRead(const T& t, const char* comm) { - const char* beg = (const char*)NAccessors::Begin(t); - const char* end = (const char*)NAccessors::End(t); +private: + template <typename T> + void TestRead(const T& t, const char* comm) { + const char* beg = (const char*)NAccessors::Begin(t); + const char* end = (const char*)NAccessors::End(t); long sz = NAccessors::Size(t) * sizeof(typename TMemoryTraits<T>::TElementType); - - UNIT_ASSERT_VALUES_EQUAL_C(end - beg, sz, comm); - } - + + UNIT_ASSERT_VALUES_EQUAL_C(end - beg, sz, comm); + } + template <typename T> void TestWrite(const char* comm) { typename TMemoryTraits<T>::TElementType val[4] = {'t', 'e', 's', 't'}; @@ -46,32 +46,32 @@ private: UNIT_ASSERT_VALUES_EQUAL_C(0u, sz, comm); } - void TestAccessors() { - TestRead('a', "char"); - TestRead(1, "int"); - + void TestAccessors() { + TestRead('a', "char"); + TestRead(1, "int"); + int t[4] = {0, 1, 2, 3}; - - TestRead(t, "int[4]"); - - TStringBuf sbuf = "test"; - - TestRead(sbuf, "TStringBuf"); - + + TestRead(t, "int[4]"); + + TStringBuf sbuf = "test"; + + TestRead(sbuf, "TStringBuf"); + TUtf16String wtr; - wtr.resize(10, 1024); - + wtr.resize(10, 1024); + TestRead(wtr, "TUtf16String"); - - TBuffer buf; - buf.Resize(30); - - TestRead(buf, "TBuffer"); - + + TBuffer buf; + buf.Resize(30); + + TestRead(buf, "TBuffer"); + TVector<ui64> vec(10, 100); - + TestRead(vec, "TVector<ui64>"); - + TestWrite<TString>("TString"); TestWrite<TVector<char>>("TVector<char>"); TestWrite<TBuffer>("TBuffer"); @@ -86,7 +86,7 @@ private: NAccessors::Init(carr); NAccessors::Clear(carr); TestRead(carr, "std::array<char, 10>"); - } -}; - -UNIT_TEST_SUITE_REGISTRATION(TAccessorsTest) + } +}; + +UNIT_TEST_SUITE_REGISTRATION(TAccessorsTest) diff --git a/library/cpp/deprecated/accessors/memory_traits.h b/library/cpp/deprecated/accessors/memory_traits.h index 1034113211..aa837705d3 100644 --- a/library/cpp/deprecated/accessors/memory_traits.h +++ b/library/cpp/deprecated/accessors/memory_traits.h @@ -1,168 +1,168 @@ -#pragma once - +#pragma once + #include <util/generic/array_ref.h> #include <util/memory/blob.h> #include <util/memory/tempbuf.h> -#include <util/generic/buffer.h> -#include <util/generic/strbuf.h> +#include <util/generic/buffer.h> +#include <util/generic/strbuf.h> #include <util/generic/string.h> #include <util/generic/vector.h> #include <util/generic/typetraits.h> - + #include <array> -#include <string> +#include <string> #include <utility> - -template <typename T> -struct TMemoryTraits { - enum { + +template <typename T> +struct TMemoryTraits { + enum { SimpleMemory = std::is_arithmetic<T>::value, - ContinuousMemory = SimpleMemory, - OwnsMemory = SimpleMemory, - }; - + ContinuousMemory = SimpleMemory, + OwnsMemory = SimpleMemory, + }; + using TElementType = T; -}; - -template <typename T, size_t n> -struct TMemoryTraits<T[n]> { - enum { - SimpleMemory = TMemoryTraits<T>::SimpleMemory, - ContinuousMemory = SimpleMemory, - OwnsMemory = SimpleMemory, - }; - +}; + +template <typename T, size_t n> +struct TMemoryTraits<T[n]> { + enum { + SimpleMemory = TMemoryTraits<T>::SimpleMemory, + ContinuousMemory = SimpleMemory, + OwnsMemory = SimpleMemory, + }; + using TElementType = T; -}; - -template <typename T, size_t n> +}; + +template <typename T, size_t n> struct TMemoryTraits<std::array<T, n>> { - enum { - SimpleMemory = TMemoryTraits<T>::SimpleMemory, - ContinuousMemory = SimpleMemory, - OwnsMemory = SimpleMemory, - }; - + enum { + SimpleMemory = TMemoryTraits<T>::SimpleMemory, + ContinuousMemory = SimpleMemory, + OwnsMemory = SimpleMemory, + }; + using TElementType = T; -}; - -template <typename A, typename B> +}; + +template <typename A, typename B> struct TMemoryTraits<std::pair<A, B>> { - enum { - SimpleMemory = TMemoryTraits<A>::SimpleMemory && TMemoryTraits<B>::SimpleMemory, - ContinuousMemory = SimpleMemory, - OwnsMemory = SimpleMemory, - }; - + enum { + SimpleMemory = TMemoryTraits<A>::SimpleMemory && TMemoryTraits<B>::SimpleMemory, + ContinuousMemory = SimpleMemory, + OwnsMemory = SimpleMemory, + }; + using TElementType = std::pair<A, B>; -}; - -template <> -struct TMemoryTraits<TBuffer> { - enum { - SimpleMemory = false, - ContinuousMemory = true, - OwnsMemory = true, - }; - +}; + +template <> +struct TMemoryTraits<TBuffer> { + enum { + SimpleMemory = false, + ContinuousMemory = true, + OwnsMemory = true, + }; + using TElementType = char; -}; - -template <> -struct TMemoryTraits<TTempBuf> { - enum { - SimpleMemory = false, - ContinuousMemory = true, - OwnsMemory = true, - }; - +}; + +template <> +struct TMemoryTraits<TTempBuf> { + enum { + SimpleMemory = false, + ContinuousMemory = true, + OwnsMemory = true, + }; + using TElementType = char; -}; - -template <> +}; + +template <> struct TMemoryTraits< ::TBlob> { - enum { - SimpleMemory = false, - ContinuousMemory = true, - OwnsMemory = true, - }; - + enum { + SimpleMemory = false, + ContinuousMemory = true, + OwnsMemory = true, + }; + using TElementType = char; -}; - -template <typename T> -struct TElementDependentMemoryTraits { - enum { - SimpleMemory = false, - ContinuousMemory = TMemoryTraits<T>::SimpleMemory, - }; - +}; + +template <typename T> +struct TElementDependentMemoryTraits { + enum { + SimpleMemory = false, + ContinuousMemory = TMemoryTraits<T>::SimpleMemory, + }; + using TElementType = T; -}; - -template <typename T, typename TAlloc> +}; + +template <typename T, typename TAlloc> struct TMemoryTraits<std::vector<T, TAlloc>>: public TElementDependentMemoryTraits<T> { - enum { - OwnsMemory = TMemoryTraits<T>::OwnsMemory - }; -}; - -template <typename T, typename TAlloc> + enum { + OwnsMemory = TMemoryTraits<T>::OwnsMemory + }; +}; + +template <typename T, typename TAlloc> struct TMemoryTraits<TVector<T, TAlloc>>: public TMemoryTraits<std::vector<T, TAlloc>> { -}; - -template <typename T> +}; + +template <typename T> struct TMemoryTraits<TTempArray<T>>: public TElementDependentMemoryTraits<T> { - enum { - OwnsMemory = TMemoryTraits<T>::OwnsMemory - }; -}; - -template <typename T, typename TCharTraits, typename TAlloc> + enum { + OwnsMemory = TMemoryTraits<T>::OwnsMemory + }; +}; + +template <typename T, typename TCharTraits, typename TAlloc> struct TMemoryTraits<std::basic_string<T, TCharTraits, TAlloc>>: public TElementDependentMemoryTraits<T> { - enum { - OwnsMemory = TMemoryTraits<T>::OwnsMemory - }; -}; - -template <> + enum { + OwnsMemory = TMemoryTraits<T>::OwnsMemory + }; +}; + +template <> struct TMemoryTraits<TString>: public TElementDependentMemoryTraits<char> { - enum { - OwnsMemory = true - }; -}; - -template <> + enum { + OwnsMemory = true + }; +}; + +template <> struct TMemoryTraits<TUtf16String>: public TElementDependentMemoryTraits<wchar16> { - enum { - OwnsMemory = true - }; -}; - -template <typename T> + enum { + OwnsMemory = true + }; +}; + +template <typename T> struct TMemoryTraits<TArrayRef<T>>: public TElementDependentMemoryTraits<T> { - enum { - OwnsMemory = false - }; -}; - -template <typename TCharType, typename TCharTraits> + enum { + OwnsMemory = false + }; +}; + +template <typename TCharType, typename TCharTraits> struct TMemoryTraits<TBasicStringBuf<TCharType, TCharTraits>>: public TElementDependentMemoryTraits<TCharType> { - enum { - OwnsMemory = false - }; -}; - -template <> + enum { + OwnsMemory = false + }; +}; + +template <> struct TMemoryTraits<TStringBuf>: public TElementDependentMemoryTraits<char> { - enum { - OwnsMemory = false - }; -}; - -template <> + enum { + OwnsMemory = false + }; +}; + +template <> struct TMemoryTraits<TWtringBuf>: public TElementDependentMemoryTraits<wchar16> { - enum { - OwnsMemory = false - }; -}; + enum { + OwnsMemory = false + }; +}; diff --git a/library/cpp/deprecated/kmp/kmp.h b/library/cpp/deprecated/kmp/kmp.h index c983129f3d..a7f72eece6 100644 --- a/library/cpp/deprecated/kmp/kmp.h +++ b/library/cpp/deprecated/kmp/kmp.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <util/generic/ptr.h> #include <util/generic/string.h> diff --git a/library/cpp/deprecated/mapped_file/mapped_file.h b/library/cpp/deprecated/mapped_file/mapped_file.h index 8a88742164..45859ed65a 100644 --- a/library/cpp/deprecated/mapped_file/mapped_file.h +++ b/library/cpp/deprecated/mapped_file/mapped_file.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <util/generic/flags.h> #include <util/generic/ptr.h> diff --git a/library/cpp/deprecated/split/delim_string_iter.h b/library/cpp/deprecated/split/delim_string_iter.h index 0309b915e4..8e4ca171a0 100644 --- a/library/cpp/deprecated/split/delim_string_iter.h +++ b/library/cpp/deprecated/split/delim_string_iter.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <util/generic/algorithm.h> #include <util/generic/strbuf.h> diff --git a/library/cpp/deprecated/split/split_iterator.h b/library/cpp/deprecated/split/split_iterator.h index 26e5865ef6..0eacc29228 100644 --- a/library/cpp/deprecated/split/split_iterator.h +++ b/library/cpp/deprecated/split/split_iterator.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <library/cpp/deprecated/kmp/kmp.h> #include <util/string/cast.h> diff --git a/library/cpp/digest/md5/md5.h b/library/cpp/digest/md5/md5.h index 1055540e1f..2c17aa0518 100644 --- a/library/cpp/digest/md5/md5.h +++ b/library/cpp/digest/md5/md5.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <util/generic/array_ref.h> #include <util/generic/strbuf.h> diff --git a/library/cpp/digest/old_crc/crc.h b/library/cpp/digest/old_crc/crc.h index 6d878566c8..4a3ce6d05e 100644 --- a/library/cpp/digest/old_crc/crc.h +++ b/library/cpp/digest/old_crc/crc.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <util/system/defaults.h> diff --git a/library/cpp/digest/sfh/sfh.h b/library/cpp/digest/sfh/sfh.h index 07300c6427..372938654c 100644 --- a/library/cpp/digest/sfh/sfh.h +++ b/library/cpp/digest/sfh/sfh.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <util/system/defaults.h> #include <util/system/unaligned_mem.h> diff --git a/library/cpp/execprofile/profile.h b/library/cpp/execprofile/profile.h index 349a1e92ac..ccb8866656 100644 --- a/library/cpp/execprofile/profile.h +++ b/library/cpp/execprofile/profile.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <stdio.h> diff --git a/library/cpp/getopt/small/last_getopt.h b/library/cpp/getopt/small/last_getopt.h index 9691e85ebf..07687bc914 100644 --- a/library/cpp/getopt/small/last_getopt.h +++ b/library/cpp/getopt/small/last_getopt.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include "last_getopt_opts.h" #include "last_getopt_easy_setup.h" diff --git a/library/cpp/getopt/small/last_getopt_support.h b/library/cpp/getopt/small/last_getopt_support.h index 5e1850620f..17bed3e614 100644 --- a/library/cpp/getopt/small/last_getopt_support.h +++ b/library/cpp/getopt/small/last_getopt_support.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <util/string/cast.h> #include <util/generic/string.h> diff --git a/library/cpp/getopt/small/opt.h b/library/cpp/getopt/small/opt.h index d301e50b55..ecb57439bc 100644 --- a/library/cpp/getopt/small/opt.h +++ b/library/cpp/getopt/small/opt.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include "last_getopt.h" diff --git a/library/cpp/getopt/small/opt2.h b/library/cpp/getopt/small/opt2.h index 4ac692ef44..4d9d943237 100644 --- a/library/cpp/getopt/small/opt2.h +++ b/library/cpp/getopt/small/opt2.h @@ -1,5 +1,5 @@ -#pragma once - +#pragma once + #include <util/system/defaults.h> #include <util/generic/string.h> #include <util/generic/vector.h> diff --git a/library/cpp/getopt/small/posix_getopt.h b/library/cpp/getopt/small/posix_getopt.h index f4b91a23d7..e6af1e0284 100644 --- a/library/cpp/getopt/small/posix_getopt.h +++ b/library/cpp/getopt/small/posix_getopt.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once // implementation of posix getopt using last getopt for demonstration purposes diff --git a/library/cpp/getopt/small/ygetopt.h b/library/cpp/getopt/small/ygetopt.h index e004a25a5a..615d3dd18e 100644 --- a/library/cpp/getopt/small/ygetopt.h +++ b/library/cpp/getopt/small/ygetopt.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <util/generic/fwd.h> #include <util/generic/ptr.h> diff --git a/library/cpp/http/fetch/exthttpcodes.h b/library/cpp/http/fetch/exthttpcodes.h index 3d268b0301..6b525052cd 100644 --- a/library/cpp/http/fetch/exthttpcodes.h +++ b/library/cpp/http/fetch/exthttpcodes.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <util/system/defaults.h> #include <library/cpp/http/misc/httpcodes.h> diff --git a/library/cpp/http/fetch/http_digest.h b/library/cpp/http/fetch/http_digest.h index cfedc233a6..3b1872d70b 100644 --- a/library/cpp/http/fetch/http_digest.h +++ b/library/cpp/http/fetch/http_digest.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include "httpheader.h" diff --git a/library/cpp/http/fetch/httpfetcher.h b/library/cpp/http/fetch/httpfetcher.h index 1b9896ffb8..7fc251afd2 100644 --- a/library/cpp/http/fetch/httpfetcher.h +++ b/library/cpp/http/fetch/httpfetcher.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #ifdef _MSC_VER #include <io.h> diff --git a/library/cpp/http/fetch/httpfsm.h b/library/cpp/http/fetch/httpfsm.h index 351dfc7e30..c4abdcd0d2 100644 --- a/library/cpp/http/fetch/httpfsm.h +++ b/library/cpp/http/fetch/httpfsm.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include "httpheader.h" diff --git a/library/cpp/http/fetch/httpheader.h b/library/cpp/http/fetch/httpheader.h index 6542fc89d3..b2810bbd41 100644 --- a/library/cpp/http/fetch/httpheader.h +++ b/library/cpp/http/fetch/httpheader.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include "exthttpcodes.h" diff --git a/library/cpp/http/fetch/httpload.h b/library/cpp/http/fetch/httpload.h index 5258eb3e27..e22e4b809e 100644 --- a/library/cpp/http/fetch/httpload.h +++ b/library/cpp/http/fetch/httpload.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include "httpagent.h" #include "httpparser.h" diff --git a/library/cpp/http/fetch/httpparser.h b/library/cpp/http/fetch/httpparser.h index 88650e8968..769828e4ae 100644 --- a/library/cpp/http/fetch/httpparser.h +++ b/library/cpp/http/fetch/httpparser.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include "httpfsm.h" #include "httpheader.h" diff --git a/library/cpp/http/fetch/httpzreader.h b/library/cpp/http/fetch/httpzreader.h index 69d3ccaf19..68eb00853d 100644 --- a/library/cpp/http/fetch/httpzreader.h +++ b/library/cpp/http/fetch/httpzreader.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include "httpheader.h" #include "httpparser.h" diff --git a/library/cpp/http/io/chunk.h b/library/cpp/http/io/chunk.h index f5815b89ec..88d89fafda 100644 --- a/library/cpp/http/io/chunk.h +++ b/library/cpp/http/io/chunk.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <util/stream/output.h> #include <util/generic/maybe.h> diff --git a/library/cpp/http/io/compression.h b/library/cpp/http/io/compression.h index 60e5a39f15..f16c4a18eb 100644 --- a/library/cpp/http/io/compression.h +++ b/library/cpp/http/io/compression.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include "stream.h" diff --git a/library/cpp/http/io/headers.h b/library/cpp/http/io/headers.h index 4c5e2f896c..a71793d1c6 100644 --- a/library/cpp/http/io/headers.h +++ b/library/cpp/http/io/headers.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <util/generic/string.h> #include <util/generic/strbuf.h> diff --git a/library/cpp/http/io/stream.h b/library/cpp/http/io/stream.h index c0058a1c57..78ca4fc814 100644 --- a/library/cpp/http/io/stream.h +++ b/library/cpp/http/io/stream.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include "headers.h" diff --git a/library/cpp/http/misc/httpcodes.h b/library/cpp/http/misc/httpcodes.h index 8469ab2159..cbfbaa1188 100644 --- a/library/cpp/http/misc/httpcodes.h +++ b/library/cpp/http/misc/httpcodes.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <util/generic/strbuf.h> diff --git a/library/cpp/http/misc/httpdate.h b/library/cpp/http/misc/httpdate.h index f82ab000cd..04876f38fe 100644 --- a/library/cpp/http/misc/httpdate.h +++ b/library/cpp/http/misc/httpdate.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <util/datetime/base.h> #include <util/generic/string.h> diff --git a/library/cpp/http/misc/httpreqdata.h b/library/cpp/http/misc/httpreqdata.h index defc326542..16e59c4d78 100644 --- a/library/cpp/http/misc/httpreqdata.h +++ b/library/cpp/http/misc/httpreqdata.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <library/cpp/digest/lower_case/hash_ops.h> diff --git a/library/cpp/http/misc/parsed_request.h b/library/cpp/http/misc/parsed_request.h index 2fddabf4fa..d4df705495 100644 --- a/library/cpp/http/misc/parsed_request.h +++ b/library/cpp/http/misc/parsed_request.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <util/generic/strbuf.h> diff --git a/library/cpp/http/server/http.h b/library/cpp/http/server/http.h index 197cba2d60..b292d38f27 100644 --- a/library/cpp/http/server/http.h +++ b/library/cpp/http/server/http.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include "conn.h" #include "options.h" diff --git a/library/cpp/http/server/options.h b/library/cpp/http/server/options.h index 344941479e..38eda0e5e7 100644 --- a/library/cpp/http/server/options.h +++ b/library/cpp/http/server/options.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <util/network/ip.h> #include <util/network/init.h> diff --git a/library/cpp/json/easy_parse/json_easy_parser_impl.h b/library/cpp/json/easy_parse/json_easy_parser_impl.h index f40cbfa2e2..ec55d838b3 100644 --- a/library/cpp/json/easy_parse/json_easy_parser_impl.h +++ b/library/cpp/json/easy_parse/json_easy_parser_impl.h @@ -17,16 +17,16 @@ namespace NJson { int ArrayCounter; TPathElemImpl(NImpl::EType type) - : Type(type) - , ArrayCounter() - { + : Type(type) + , ArrayCounter() + { } TPathElemImpl(const TStringType& key) : Type(NImpl::MAP_KEY) - , Key(key) - , ArrayCounter() - { + , Key(key) + , ArrayCounter() + { } TPathElemImpl(int arrayCounter) diff --git a/library/cpp/json/fast_sax/parser.rl6 b/library/cpp/json/fast_sax/parser.rl6 index b7b7900071..edb4e9ee1b 100644 --- a/library/cpp/json/fast_sax/parser.rl6 +++ b/library/cpp/json/fast_sax/parser.rl6 @@ -1,215 +1,215 @@ #include <library/cpp/json/fast_sax/unescape.h> #include <library/cpp/json/fast_sax/parser.h> - -#include <util/string/cast.h> + +#include <util/string/cast.h> #include <util/generic/buffer.h> -#include <util/generic/strbuf.h> -#include <util/generic/ymath.h> - -namespace NJson { - -enum EStoredStr { - SS_NONE = 0, SS_NOCOPY, SS_MUSTCOPY -}; - -struct TParserCtx { - TJsonCallbacks& Hndl; - +#include <util/generic/strbuf.h> +#include <util/generic/ymath.h> + +namespace NJson { + +enum EStoredStr { + SS_NONE = 0, SS_NOCOPY, SS_MUSTCOPY +}; + +struct TParserCtx { + TJsonCallbacks& Hndl; + TBuffer Buffer; - TStringBuf String; - EStoredStr Stored = SS_NONE; - bool ExpectValue = true; - - const char* p0 = nullptr; - const char* p = nullptr; - const char* pe = nullptr; - const char* eof = nullptr; - const char* ts = nullptr; - const char* te = nullptr; - int cs = 0; - int act = 0; - - TParserCtx(TJsonCallbacks& h, TStringBuf data) - : Hndl(h) + TStringBuf String; + EStoredStr Stored = SS_NONE; + bool ExpectValue = true; + + const char* p0 = nullptr; + const char* p = nullptr; + const char* pe = nullptr; + const char* eof = nullptr; + const char* ts = nullptr; + const char* te = nullptr; + int cs = 0; + int act = 0; + + TParserCtx(TJsonCallbacks& h, TStringBuf data) + : Hndl(h) , p0(data.data()) , p(data.data()) - , pe(data.end()) - , eof(data.end()) - {} - - static inline bool GoodPtrs(const char* b, const char* e) { - return b && e && b <= e; - } - + , pe(data.end()) + , eof(data.end()) + {} + + static inline bool GoodPtrs(const char* b, const char* e) { + return b && e && b <= e; + } + bool OnError(TStringBuf reason = TStringBuf(""), bool end = false) const { - size_t off = 0; - TStringBuf token; - - if (GoodPtrs(p0, ts)) { - off = ts - p0; - } else if (end && GoodPtrs(p0, pe)) { - off = pe - p0; - } - - if (GoodPtrs(ts, te)) { - token = TStringBuf(ts, te); - } - - if (!token) { - Hndl.OnError(off, reason); - } else { + size_t off = 0; + TStringBuf token; + + if (GoodPtrs(p0, ts)) { + off = ts - p0; + } else if (end && GoodPtrs(p0, pe)) { + off = pe - p0; + } + + if (GoodPtrs(ts, te)) { + token = TStringBuf(ts, te); + } + + if (!token) { + Hndl.OnError(off, reason); + } else { Hndl.OnError(off, TString::Join(reason, " at token: '", token, "'")); - } - - return false; - } - - bool OnVal() { - if (Y_UNLIKELY(!ExpectValue)) { - return false; - } - ExpectValue = false; - return true; - } - - bool OnNull() { - return Y_LIKELY(OnVal()) - && Hndl.OnNull(); - } - - bool OnTrue() { - return Y_LIKELY(OnVal()) - && Hndl.OnBoolean(true); - } - - bool OnFalse() { - return Y_LIKELY(OnVal()) - && Hndl.OnBoolean(false); - } - - bool OnPInt() { - unsigned long long res = 0; - return Y_LIKELY(OnVal()) - && TryFromString<unsigned long long>(TStringBuf(ts, te), res) - && Hndl.OnUInteger(res); - } - - bool OnNInt() { - long long res = 0; - return Y_LIKELY(OnVal()) - && TryFromString<long long>(TStringBuf(ts, te), res) - && Hndl.OnInteger(res); - } - - bool OnFlt() { - double res = 0; - return Y_LIKELY(OnVal()) - && TryFromString<double>(TStringBuf(ts, te), res) - && IsFinite(res) - && Hndl.OnDouble(res); - } - - bool OnMapOpen() { - bool res = Y_LIKELY(OnVal()) - && Hndl.OnOpenMap(); - ExpectValue = true; - return res; - } - - bool OnArrOpen() { - bool res = Y_LIKELY(OnVal()) - && Hndl.OnOpenArray(); - ExpectValue = true; - return res; - } - - bool OnString(TStringBuf s, EStoredStr t) { + } + + return false; + } + + bool OnVal() { + if (Y_UNLIKELY(!ExpectValue)) { + return false; + } + ExpectValue = false; + return true; + } + + bool OnNull() { + return Y_LIKELY(OnVal()) + && Hndl.OnNull(); + } + + bool OnTrue() { + return Y_LIKELY(OnVal()) + && Hndl.OnBoolean(true); + } + + bool OnFalse() { + return Y_LIKELY(OnVal()) + && Hndl.OnBoolean(false); + } + + bool OnPInt() { + unsigned long long res = 0; + return Y_LIKELY(OnVal()) + && TryFromString<unsigned long long>(TStringBuf(ts, te), res) + && Hndl.OnUInteger(res); + } + + bool OnNInt() { + long long res = 0; + return Y_LIKELY(OnVal()) + && TryFromString<long long>(TStringBuf(ts, te), res) + && Hndl.OnInteger(res); + } + + bool OnFlt() { + double res = 0; + return Y_LIKELY(OnVal()) + && TryFromString<double>(TStringBuf(ts, te), res) + && IsFinite(res) + && Hndl.OnDouble(res); + } + + bool OnMapOpen() { + bool res = Y_LIKELY(OnVal()) + && Hndl.OnOpenMap(); + ExpectValue = true; + return res; + } + + bool OnArrOpen() { + bool res = Y_LIKELY(OnVal()) + && Hndl.OnOpenArray(); + ExpectValue = true; + return res; + } + + bool OnString(TStringBuf s, EStoredStr t) { if (Y_LIKELY(OnVal())) { - String = s; - Stored = t; - return true; - } else { - return false; - } - } - - bool OnStrU() { - return OnString(TStringBuf(ts, te), SS_NOCOPY); - } - - bool OnStrQ() { - return OnString(TStringBuf(ts + 1, te - 1), SS_NOCOPY); - } - - bool OnStrE() { + String = s; + Stored = t; + return true; + } else { + return false; + } + } + + bool OnStrU() { + return OnString(TStringBuf(ts, te), SS_NOCOPY); + } + + bool OnStrQ() { + return OnString(TStringBuf(ts + 1, te - 1), SS_NOCOPY); + } + + bool OnStrE() { Buffer.Clear(); Buffer.Reserve(2 * (te - ts)); return OnString(UnescapeJsonUnicode(TStringBuf(ts + 1, te - ts - 2), Buffer.data()), SS_MUSTCOPY); - } - - bool OnMapClose() { - ExpectValue = false; - return Y_LIKELY(OnAfterVal()) - && Hndl.OnCloseMap(); - } - - bool OnArrClose() { - ExpectValue = false; - return Y_LIKELY(OnAfterVal()) - && Hndl.OnCloseArray(); - } - - bool OnColon() { - if (ExpectValue) { - return false; - } - - ExpectValue = true; - const auto stored = Stored; - Stored = SS_NONE; - - switch (stored) { + } + + bool OnMapClose() { + ExpectValue = false; + return Y_LIKELY(OnAfterVal()) + && Hndl.OnCloseMap(); + } + + bool OnArrClose() { + ExpectValue = false; + return Y_LIKELY(OnAfterVal()) + && Hndl.OnCloseArray(); + } + + bool OnColon() { + if (ExpectValue) { + return false; + } + + ExpectValue = true; + const auto stored = Stored; + Stored = SS_NONE; + + switch (stored) { default: return false; - case SS_NOCOPY: - return Hndl.OnMapKeyNoCopy(String); - case SS_MUSTCOPY: - return Hndl.OnMapKey(String); - } - } - - bool OnAfterVal() { - const auto stored = Stored; - Stored = SS_NONE; - - switch (stored) { - default: - return true; - case SS_NOCOPY: - return Hndl.OnStringNoCopy(String); - case SS_MUSTCOPY: - return Hndl.OnString(String); - } - } - - bool OnComma() { - if (Y_UNLIKELY(ExpectValue)) { - return false; - } - ExpectValue = true; - return OnAfterVal(); - } - - bool Parse(); -}; - -#if 0 -%%{ -machine fastjson; - -alphtype char; - + case SS_NOCOPY: + return Hndl.OnMapKeyNoCopy(String); + case SS_MUSTCOPY: + return Hndl.OnMapKey(String); + } + } + + bool OnAfterVal() { + const auto stored = Stored; + Stored = SS_NONE; + + switch (stored) { + default: + return true; + case SS_NOCOPY: + return Hndl.OnStringNoCopy(String); + case SS_MUSTCOPY: + return Hndl.OnString(String); + } + } + + bool OnComma() { + if (Y_UNLIKELY(ExpectValue)) { + return false; + } + ExpectValue = true; + return OnAfterVal(); + } + + bool Parse(); +}; + +#if 0 +%%{ +machine fastjson; + +alphtype char; + action OnNull { if (Y_UNLIKELY(!OnNull())) goto TOKEN_ERROR; } action OnTrue { if (Y_UNLIKELY(!OnTrue())) goto TOKEN_ERROR; } action OnFalse { if (Y_UNLIKELY(!OnFalse())) goto TOKEN_ERROR; } @@ -225,90 +225,90 @@ action OnArrO { if (Y_UNLIKELY(!OnArrOpen())) goto TOKEN_ERROR; } action OnArrC { if (Y_UNLIKELY(!OnArrClose())) goto TOKEN_ERROR; } action OnComma { if (Y_UNLIKELY(!OnComma())) goto TOKEN_ERROR; } action OnColon { if (Y_UNLIKELY(!OnColon())) goto TOKEN_ERROR; } -action OnError { goto TOKEN_ERROR; } - -comment1 = "/*" (any* -- "*/") "*/"; - -pint = [0-9]+; -nint = '-'[0-9]+; -flt = '-'?[0-9.][0-9.eE+\-]+; - -uchar0 = [a-zA-Z_@$] | (0x80 .. 0xFF); -uchar = uchar0 | digit | [.\-]; - -qchar = [^'\\]; #'; -dchar = [^"\\]; #"; - -echar = "\\" any; - -qechar = qchar | echar; -dechar = dchar | echar; - -strq = "'" qchar* "'"; -strd = '"' dchar* '"'; - -strqe = "'" qechar* "'"; -strde = '"' dechar* '"'; - -strU = uchar0 uchar*; -strQ = strq | strd; -strE = strqe | strde; - -ws = (0x00 .. 0x20) | 0x7F; -sp = ws+; - -main := |* - 'null' => OnNull; - 'true' => OnTrue; - 'false' => OnFalse; - - pint => OnPInt; - nint => OnNInt; - flt => OnFlt; - - strU => OnStrU; - strQ => OnStrQ; - strE => OnStrE; - - ',' => OnComma; - ':' => OnColon; - - '{' => OnDictO; - '}' => OnDictC; - '[' => OnArrO; - ']' => OnArrC; - - sp; - comment1; - - (flt | pint | nint) (any - (ws | ',' | ':' | '{' | '}' | '[' | ']')) => OnError; - - any => OnError; - *|; -}%% -#endif - -bool TParserCtx::Parse() { - try { - %%{ - write data noerror nofinal; - write init; - write exec; - }%% - ; +action OnError { goto TOKEN_ERROR; } + +comment1 = "/*" (any* -- "*/") "*/"; + +pint = [0-9]+; +nint = '-'[0-9]+; +flt = '-'?[0-9.][0-9.eE+\-]+; + +uchar0 = [a-zA-Z_@$] | (0x80 .. 0xFF); +uchar = uchar0 | digit | [.\-]; + +qchar = [^'\\]; #'; +dchar = [^"\\]; #"; + +echar = "\\" any; + +qechar = qchar | echar; +dechar = dchar | echar; + +strq = "'" qchar* "'"; +strd = '"' dchar* '"'; + +strqe = "'" qechar* "'"; +strde = '"' dechar* '"'; + +strU = uchar0 uchar*; +strQ = strq | strd; +strE = strqe | strde; + +ws = (0x00 .. 0x20) | 0x7F; +sp = ws+; + +main := |* + 'null' => OnNull; + 'true' => OnTrue; + 'false' => OnFalse; + + pint => OnPInt; + nint => OnNInt; + flt => OnFlt; + + strU => OnStrU; + strQ => OnStrQ; + strE => OnStrE; + + ',' => OnComma; + ':' => OnColon; + + '{' => OnDictO; + '}' => OnDictC; + '[' => OnArrO; + ']' => OnArrC; + + sp; + comment1; + + (flt | pint | nint) (any - (ws | ',' | ':' | '{' | '}' | '[' | ']')) => OnError; + + any => OnError; + *|; +}%% +#endif + +bool TParserCtx::Parse() { + try { + %%{ + write data noerror nofinal; + write init; + write exec; + }%% + ; Y_UNUSED(fastjson_en_main); - } catch (const TFromStringException& e) { - return OnError(e.what()); - } - - return OnAfterVal() && Hndl.OnEnd() || OnError("invalid or truncated", true); - - TOKEN_ERROR: - return OnError("invalid syntax"); -} - -bool ReadJsonFast(TStringBuf data, TJsonCallbacks* h) { - return TParserCtx(*h, data).Parse(); -} - -} + } catch (const TFromStringException& e) { + return OnError(e.what()); + } + + return OnAfterVal() && Hndl.OnEnd() || OnError("invalid or truncated", true); + + TOKEN_ERROR: + return OnError("invalid syntax"); +} + +bool ReadJsonFast(TStringBuf data, TJsonCallbacks* h) { + return TParserCtx(*h, data).Parse(); +} + +} diff --git a/library/cpp/json/json_prettifier.cpp b/library/cpp/json/json_prettifier.cpp index 2fc85420bc..bb16aab44e 100644 --- a/library/cpp/json/json_prettifier.cpp +++ b/library/cpp/json/json_prettifier.cpp @@ -1,61 +1,61 @@ -#include "json_prettifier.h" - -#include <util/generic/deque.h> +#include "json_prettifier.h" + +#include <util/generic/deque.h> #include <util/generic/algorithm.h> -#include <util/memory/pool.h> -#include <util/string/util.h> - +#include <util/memory/pool.h> +#include <util/string/util.h> + #include <library/cpp/string_utils/relaxed_escaper/relaxed_escaper.h> - -namespace NJson { + +namespace NJson { struct TRewritableOut { IOutputStream& Slave; - + char Last = 0; bool Dirty = false; - + TRewritableOut(IOutputStream& sl) : Slave(sl) { } - + template <typename T> void Write(const T& t) { Flush(); Slave << t; } - + void Hold(char c) { if (Dirty) Flush(); Last = c; Dirty = true; } - + void Flush() { if (Dirty) { Slave << Last; Dirty = false; } } - + void Revert() { - Dirty = false; - } + Dirty = false; + } }; - + struct TSpaces { char S[256]; - + TSpaces() { memset(&S, ' ', sizeof(S)); } - + TStringBuf Get(ui8 sz) const { return TStringBuf(S, sz); } }; - + bool TJsonPrettifier::MayUnquoteNew(TStringBuf s) { static str_spn alpha("a-zA-Z_@$", true); static str_spn alnum("a-zA-Z_@$0-9.-", true); @@ -64,8 +64,8 @@ namespace NJson { static TStringBuf null0("null"); return !!s && alpha.chars_table[(ui8)s[0]] && alnum.cbrk(s.begin() + 1, s.end()) == s.end() && !EqualToOneOf(s, null0, true0, false0); - } - + } + // to keep arcadia tests happy bool TJsonPrettifier::MayUnquoteOld(TStringBuf s) { static str_spn alpha("a-zA-Z_@$", true); @@ -77,24 +77,24 @@ namespace NJson { static TStringBuf true2("da"); static TStringBuf false2("net"); static TStringBuf null0("null"); - + return !!s && alpha.chars_table[(ui8)s[0]] && alnum.cbrk(s.begin() + 1, s.end()) == s.end() && !EqualToOneOf(s, null0, true0, false0, true1, false1, true2, false2); } - + class TPrettifier: public TJsonCallbacks { TRewritableOut Out; TStringBuf Spaces; TStringBuf Quote; TStringBuf Unsafe; TStringBuf Safe; - + ui32 Level = 0; ui32 MaxPaddingLevel; - + bool Unquote = false; bool Compactify = false; bool NewUnquote = false; - + public: TPrettifier(IOutputStream& out, const TJsonPrettifier& p) : Out(out) @@ -113,7 +113,7 @@ namespace NJson { Safe = "'"; } } - + void Pad(bool close = false) { if (Compactify) { Out.Flush(); @@ -130,23 +130,23 @@ namespace NJson { Out.Write(Spaces); } } - + void WriteSpace(char sp) { if (Compactify) { Out.Flush(); return; } - + Out.Write(sp); - } - + } + void OnVal() { if (Out.Dirty && ':' == Out.Last) { WriteSpace(' '); } else { Pad(); } - } + } void AfterVal() { Out.Hold(','); @@ -158,28 +158,28 @@ namespace NJson { Out.Write(t); AfterVal(); return true; - } + } bool OnNull() override { return WriteVal(TStringBuf("null")); - } - + } + bool OnBoolean(bool v) override { return WriteVal(v ? TStringBuf("true") : TStringBuf("false")); - } - + } + bool OnInteger(long long i) override { return WriteVal(i); } - + bool OnUInteger(unsigned long long i) override { return WriteVal(i); - } - + } + bool OnDouble(double d) override { return WriteVal(d); } - + void WriteString(TStringBuf s) { if (Unquote && (NewUnquote ? TJsonPrettifier::MayUnquoteNew(s) : TJsonPrettifier::MayUnquoteOld(s))) { Out.Slave << s; @@ -189,53 +189,53 @@ namespace NJson { Out.Slave << Quote; } } - + bool OnString(const TStringBuf& s) override { OnVal(); WriteString(s); AfterVal(); return true; } - + bool OnOpen(char c) { OnVal(); Level++; Out.Hold(c); return true; } - + bool OnOpenMap() override { return OnOpen('{'); } - + bool OnOpenArray() override { return OnOpen('['); } - + bool OnMapKey(const TStringBuf& k) override { OnVal(); WriteString(k); WriteSpace(' '); Out.Hold(':'); return true; - } - + } + bool OnClose(char c) { if (!Level) return false; - + Level--; - + if (Out.Dirty && c == Out.Last) { WriteSpace(' '); } else { Out.Revert(); Pad(true); } - + return true; } - + bool OnCloseMap() override { if (!OnClose('{')) return false; @@ -243,7 +243,7 @@ namespace NJson { AfterVal(); return true; } - + bool OnCloseArray() override { if (!OnClose('[')) return false; @@ -251,27 +251,27 @@ namespace NJson { AfterVal(); return true; } - + bool OnEnd() override { return !Level; } }; - + bool TJsonPrettifier::Prettify(TStringBuf in, IOutputStream& out) const { TPrettifier p(out, *this); if (Strict) { TMemoryInput mIn(in.data(), in.size()); return ReadJson(&mIn, &p); - } else { + } else { return ReadJsonFast(in, &p); - } - } - + } + } + TString TJsonPrettifier::Prettify(TStringBuf in) const { TStringStream s; if (Prettify(in, s)) return s.Str(); return TString(); - } - -} + } + +} diff --git a/library/cpp/json/json_prettifier.h b/library/cpp/json/json_prettifier.h index 70473ce0e7..27d611b0b4 100644 --- a/library/cpp/json/json_prettifier.h +++ b/library/cpp/json/json_prettifier.h @@ -1,10 +1,10 @@ -#pragma once - -#include "json_reader.h" - +#pragma once + +#include "json_reader.h" + #include <util/generic/ylimits.h> -namespace NJson { +namespace NJson { struct TJsonPrettifier { bool Unquote = false; ui8 Padding = 4; @@ -13,7 +13,7 @@ namespace NJson { bool Strict = false; bool NewUnquote = false; // use new unquote, may break old tests ui32 MaxPaddingLevel = Max<ui32>(); - + static TJsonPrettifier Prettifier(bool unquote = false, ui8 padding = 4, bool singlequotes = false) { TJsonPrettifier p; p.Unquote = unquote; @@ -21,7 +21,7 @@ namespace NJson { p.SingleQuotes = singlequotes; return p; } - + static TJsonPrettifier Compactifier(bool unquote = false, bool singlequote = false) { TJsonPrettifier p; p.Unquote = unquote; @@ -30,29 +30,29 @@ namespace NJson { p.SingleQuotes = singlequote; return p; } - + bool Prettify(TStringBuf in, IOutputStream& out) const; - + TString Prettify(TStringBuf in) const; - + static bool MayUnquoteNew(TStringBuf in); static bool MayUnquoteOld(TStringBuf in); }; - + inline TString PrettifyJson(TStringBuf in, bool unquote = false, ui8 padding = 4, bool sq = false) { return TJsonPrettifier::Prettifier(unquote, padding, sq).Prettify(in); } - + inline bool PrettifyJson(TStringBuf in, IOutputStream& out, bool unquote = false, ui8 padding = 4, bool sq = false) { return TJsonPrettifier::Prettifier(unquote, padding, sq).Prettify(in, out); } - + inline bool CompactifyJson(TStringBuf in, IOutputStream& out, bool unquote = false, bool sq = false) { return TJsonPrettifier::Compactifier(unquote, sq).Prettify(in, out); } - + inline TString CompactifyJson(TStringBuf in, bool unquote = false, bool sq = false) { return TJsonPrettifier::Compactifier(unquote, sq).Prettify(in); } - -} + +} diff --git a/library/cpp/json/json_reader.cpp b/library/cpp/json/json_reader.cpp index 5162280d62..072c8deafe 100644 --- a/library/cpp/json/json_reader.cpp +++ b/library/cpp/json/json_reader.cpp @@ -2,24 +2,24 @@ #include "rapidjson_helpers.h" -#include <contrib/libs/rapidjson/include/rapidjson/error/en.h> -#include <contrib/libs/rapidjson/include/rapidjson/error/error.h> +#include <contrib/libs/rapidjson/include/rapidjson/error/en.h> +#include <contrib/libs/rapidjson/include/rapidjson/error/error.h> #include <contrib/libs/rapidjson/include/rapidjson/reader.h> #include <util/generic/stack.h> #include <util/string/cast.h> #include <util/system/yassert.h> -#include <util/string/builder.h> +#include <util/string/builder.h> namespace NJson { - namespace { - TString PrintError(const rapidjson::ParseResult& result) { + namespace { + TString PrintError(const rapidjson::ParseResult& result) { return TStringBuilder() << TStringBuf("Offset: ") << result.Offset() << TStringBuf(", Code: ") << (int)result.Code() << TStringBuf(", Error: ") << GetParseError_En(result.Code()); - } - } - + } + } + static const size_t DEFAULT_BUFFER_LEN = 65536; bool TParserCallbacks::OpenComplexValue(EJsonValueType type) { @@ -355,7 +355,7 @@ namespace NJson { if (result.IsError()) { if (throwOnError) { - ythrow TJsonException() << PrintError(result); + ythrow TJsonException() << PrintError(result); } else { return false; } @@ -538,7 +538,7 @@ namespace NJson { auto result = Read(*config, reader, is, wrapper); if (result.IsError()) { - cbs->OnError(result.Offset(), PrintError(result)); + cbs->OnError(result.Offset(), PrintError(result)); return false; } diff --git a/library/cpp/json/json_reader.h b/library/cpp/json/json_reader.h index 18bfe05982..b673788330 100644 --- a/library/cpp/json/json_reader.h +++ b/library/cpp/json/json_reader.h @@ -1,15 +1,15 @@ #pragma once -#include "json_value.h" - +#include "json_value.h" + #include <library/cpp/json/common/defs.h> #include <library/cpp/json/fast_sax/parser.h> -#include <util/generic/yexception.h> - +#include <util/generic/yexception.h> + #include <util/stream/input.h> -#include <util/stream/str.h> -#include <util/stream/mem.h> +#include <util/stream/str.h> +#include <util/stream/mem.h> namespace NJson { struct TJsonReaderConfig { @@ -22,7 +22,7 @@ namespace NJson { void SetBufferSize(size_t bufferSize); size_t GetBufferSize() const; - + private: size_t BufferSize; }; @@ -34,7 +34,7 @@ namespace NJson { bool ReadJsonTree(IInputStream* in, TJsonValue* out, bool throwOnError = false); bool ReadJsonTree(IInputStream* in, bool allowComments, TJsonValue* out, bool throwOnError = false); bool ReadJsonTree(IInputStream* in, const TJsonReaderConfig* config, TJsonValue* out, bool throwOnError = false); - + TJsonValue ReadJsonTree(IInputStream* in, bool throwOnError = false); TJsonValue ReadJsonTree(IInputStream* in, bool allowComments, bool throwOnError); TJsonValue ReadJsonTree(IInputStream* in, const TJsonReaderConfig* config, bool throwOnError = false); @@ -73,11 +73,11 @@ namespace NJson { inline bool ValidateJsonThrow(IInputStream* in, const TJsonReaderConfig* config) { return ValidateJson(in, config, true); } - + inline bool ValidateJsonThrow(TStringBuf in, const TJsonReaderConfig& config = TJsonReaderConfig()) { return ValidateJson(in, config, true); } - + class TParserCallbacks: public TJsonCallbacks { public: TParserCallbacks(TJsonValue& value, bool throwOnError = false, bool notClosedBracketIsError = false); @@ -93,13 +93,13 @@ namespace NJson { bool OnCloseMap() override; bool OnMapKey(const TStringBuf& val) override; bool OnEnd() override; - + protected: TJsonValue& Value; TString Key; TVector<TJsonValue*> ValuesStack; bool NotClosedBracketIsError; - + enum { START, AFTER_MAP_KEY, @@ -137,4 +137,4 @@ namespace NJson { //// relaxed json, used in library/cpp/scheme bool ReadJsonFastTree(TStringBuf in, TJsonValue* out, bool throwOnError = false, bool notClosedBracketIsError = false); TJsonValue ReadJsonFastTree(TStringBuf in, bool notClosedBracketIsError = false); -} +} diff --git a/library/cpp/json/json_writer.cpp b/library/cpp/json/json_writer.cpp index 121ee3a335..3d058bae36 100644 --- a/library/cpp/json/json_writer.cpp +++ b/library/cpp/json/json_writer.cpp @@ -1,7 +1,7 @@ #include "json_writer.h" #include <util/charset/utf8.h> -#include <util/generic/algorithm.h> +#include <util/generic/algorithm.h> #include <util/string/cast.h> #include <util/system/yassert.h> @@ -85,7 +85,7 @@ namespace NJson { void TJsonWriter::WriteNull() { Buf.WriteNull(); } - + void TJsonWriter::Write(float value) { Buf.WriteFloat(value, FloatToStringMode, FloatNDigits); } @@ -113,11 +113,11 @@ namespace NJson { } }; } - + void TJsonWriter::Write(const TJsonValue* v) { Buf.WriteJsonValue(v, SortKeys, FloatToStringMode, DoubleNDigits); } - + void TJsonWriter::Write(const TJsonValue& v) { Buf.WriteJsonValue(&v, SortKeys, FloatToStringMode, DoubleNDigits); } @@ -139,7 +139,7 @@ namespace NJson { w.Write(val); w.Flush(); } - + void WriteJson(IOutputStream* out, const TJsonValue* val, const TJsonWriterConfig& config) { TJsonWriter w(out, config, true); w.Write(val); diff --git a/library/cpp/json/json_writer.h b/library/cpp/json/json_writer.h index 59398094cc..c7f5c9499a 100644 --- a/library/cpp/json/json_writer.h +++ b/library/cpp/json/json_writer.h @@ -2,8 +2,8 @@ // Deprecated. Use library/cpp/json/writer in new code. -#include "json_value.h" - +#include "json_value.h" + #include <library/cpp/json/writer/json.h> #include <util/stream/output.h> @@ -86,7 +86,7 @@ namespace NJson { void Write(bool value); void Write(const TJsonValue* value); void Write(const TJsonValue& value); - + // must use all variations of integer types since long // and long long are different types but with same size void Write(long long value); @@ -151,7 +151,7 @@ namespace NJson { Write(key, *value); } } - + void WriteOptional(const TStringBuf&, const TNothing&) { // nothing to do } diff --git a/library/cpp/json/ut/json_prettifier_ut.cpp b/library/cpp/json/ut/json_prettifier_ut.cpp index 10946de93a..ae5f8dd81a 100644 --- a/library/cpp/json/ut/json_prettifier_ut.cpp +++ b/library/cpp/json/ut/json_prettifier_ut.cpp @@ -1,25 +1,25 @@ #include <library/cpp/json/json_prettifier.h> #include <library/cpp/testing/unittest/registar.h> - + Y_UNIT_TEST_SUITE(JsonPrettifier) { Y_UNIT_TEST(PrettifyJsonShort) { - UNIT_ASSERT_STRINGS_EQUAL(NJson::PrettifyJson(""), ""); - UNIT_ASSERT_STRINGS_EQUAL(NJson::PrettifyJson("null"), "null"); - UNIT_ASSERT_STRINGS_EQUAL(NJson::PrettifyJson("true"), "true"); - UNIT_ASSERT_STRINGS_EQUAL(NJson::PrettifyJson("false"), "false"); - UNIT_ASSERT_STRINGS_EQUAL(NJson::PrettifyJson("1.5"), "1.5"); - UNIT_ASSERT_STRINGS_EQUAL(NJson::PrettifyJson("test", false, 2, true), "'test'"); - - UNIT_ASSERT_STRINGS_EQUAL(NJson::PrettifyJson("[]"), "[ ]"); - UNIT_ASSERT_STRINGS_EQUAL(NJson::PrettifyJson("[a]", false, 2), "[\n \"a\"\n]"); - UNIT_ASSERT_STRINGS_EQUAL(NJson::PrettifyJson("[a,b]", false, 2, true), "[\n 'a',\n 'b'\n]"); - UNIT_ASSERT_STRINGS_EQUAL(NJson::PrettifyJson("[{},b]", false, 2, true), "[\n { },\n 'b'\n]"); - UNIT_ASSERT_STRINGS_EQUAL(NJson::PrettifyJson("[a,{}]", false, 2, true), "[\n 'a',\n { }\n]"); - UNIT_ASSERT_STRINGS_EQUAL(NJson::PrettifyJson("[{},{}]"), "[\n { },\n { }\n]"); - UNIT_ASSERT_STRINGS_EQUAL(NJson::PrettifyJson("{}"), "{ }"); - UNIT_ASSERT_STRINGS_EQUAL(NJson::PrettifyJson("{}"), "{ }"); - UNIT_ASSERT_STRINGS_EQUAL(NJson::PrettifyJson("{k:v}", false, 2, true), "{\n 'k' : 'v'\n}"); + UNIT_ASSERT_STRINGS_EQUAL(NJson::PrettifyJson(""), ""); + UNIT_ASSERT_STRINGS_EQUAL(NJson::PrettifyJson("null"), "null"); + UNIT_ASSERT_STRINGS_EQUAL(NJson::PrettifyJson("true"), "true"); + UNIT_ASSERT_STRINGS_EQUAL(NJson::PrettifyJson("false"), "false"); + UNIT_ASSERT_STRINGS_EQUAL(NJson::PrettifyJson("1.5"), "1.5"); + UNIT_ASSERT_STRINGS_EQUAL(NJson::PrettifyJson("test", false, 2, true), "'test'"); + + UNIT_ASSERT_STRINGS_EQUAL(NJson::PrettifyJson("[]"), "[ ]"); + UNIT_ASSERT_STRINGS_EQUAL(NJson::PrettifyJson("[a]", false, 2), "[\n \"a\"\n]"); + UNIT_ASSERT_STRINGS_EQUAL(NJson::PrettifyJson("[a,b]", false, 2, true), "[\n 'a',\n 'b'\n]"); + UNIT_ASSERT_STRINGS_EQUAL(NJson::PrettifyJson("[{},b]", false, 2, true), "[\n { },\n 'b'\n]"); + UNIT_ASSERT_STRINGS_EQUAL(NJson::PrettifyJson("[a,{}]", false, 2, true), "[\n 'a',\n { }\n]"); + UNIT_ASSERT_STRINGS_EQUAL(NJson::PrettifyJson("[{},{}]"), "[\n { },\n { }\n]"); + UNIT_ASSERT_STRINGS_EQUAL(NJson::PrettifyJson("{}"), "{ }"); + UNIT_ASSERT_STRINGS_EQUAL(NJson::PrettifyJson("{}"), "{ }"); + UNIT_ASSERT_STRINGS_EQUAL(NJson::PrettifyJson("{k:v}", false, 2, true), "{\n 'k' : 'v'\n}"); UNIT_ASSERT_STRINGS_EQUAL(NJson::PrettifyJson("Test545", true, 2), "Test545"); UNIT_ASSERT_STRINGS_EQUAL(NJson::PrettifyJson("'null'", true, 2, true), "'null'"); @@ -45,7 +45,7 @@ Y_UNIT_TEST_SUITE(JsonPrettifier) { " }\n" "]"); - UNIT_ASSERT_STRINGS_EQUAL(NJson::PrettifyJson("{k:v,a:b,x:[1,2,3]}", false, 2, true), + UNIT_ASSERT_STRINGS_EQUAL(NJson::PrettifyJson("{k:v,a:b,x:[1,2,3]}", false, 2, true), "{\n" " 'k' : 'v',\n" " 'a' : 'b',\n" @@ -56,7 +56,7 @@ Y_UNIT_TEST_SUITE(JsonPrettifier) { " ]\n" "}"); - UNIT_ASSERT_STRINGS_EQUAL(NJson::PrettifyJson("{k:v,a:b,x:[1,{f:b},3],m:n}", false, 2, true), + UNIT_ASSERT_STRINGS_EQUAL(NJson::PrettifyJson("{k:v,a:b,x:[1,{f:b},3],m:n}", false, 2, true), "{\n" " 'k' : 'v',\n" " 'a' : 'b',\n" @@ -69,7 +69,7 @@ Y_UNIT_TEST_SUITE(JsonPrettifier) { " ],\n" " 'm' : 'n'\n" "}"); - + NJson::TJsonPrettifier prettifierMaxLevel1 = NJson::TJsonPrettifier::Prettifier(false, 2, true); prettifierMaxLevel1.MaxPaddingLevel = 1; UNIT_ASSERT_STRINGS_EQUAL(prettifierMaxLevel1.Prettify("{k:v,a:b,x:[1,{f:b},3],m:n}"), @@ -79,8 +79,8 @@ Y_UNIT_TEST_SUITE(JsonPrettifier) { " 'x' : [ 1, { 'f' : 'b' }, 3 ],\n" " 'm' : 'n'\n" "}"); - - UNIT_ASSERT_STRINGS_EQUAL(NJson::PrettifyJson("{g:{x:{a:{b:c,e:f},q:{x:y}},y:fff}}", true, 2), + + UNIT_ASSERT_STRINGS_EQUAL(NJson::PrettifyJson("{g:{x:{a:{b:c,e:f},q:{x:y}},y:fff}}", true, 2), "{\n" " g : {\n" " x : {\n" @@ -108,8 +108,8 @@ Y_UNIT_TEST_SUITE(JsonPrettifier) { " y : fff\n" " }\n" "}"); - } - + } + Y_UNIT_TEST(PrettifyJsonInvalid) { UNIT_ASSERT_STRINGS_EQUAL(NJson::PrettifyJson("}"), ""); UNIT_ASSERT_STRINGS_EQUAL(NJson::PrettifyJson("}}"), ""); @@ -124,26 +124,26 @@ Y_UNIT_TEST_SUITE(JsonPrettifier) { } Y_UNIT_TEST(CompactifyJsonShort) { - UNIT_ASSERT_STRINGS_EQUAL(NJson::CompactifyJson(""), ""); - UNIT_ASSERT_STRINGS_EQUAL(NJson::CompactifyJson("null"), "null"); - UNIT_ASSERT_STRINGS_EQUAL(NJson::CompactifyJson("true"), "true"); - UNIT_ASSERT_STRINGS_EQUAL(NJson::CompactifyJson("false"), "false"); - UNIT_ASSERT_STRINGS_EQUAL(NJson::CompactifyJson("1.5"), "1.5"); - UNIT_ASSERT_STRINGS_EQUAL(NJson::CompactifyJson("test", true), "test"); - UNIT_ASSERT_STRINGS_EQUAL(NJson::CompactifyJson("test", false), "\"test\""); - - UNIT_ASSERT_STRINGS_EQUAL(NJson::CompactifyJson("[ ]"), "[]"); - UNIT_ASSERT_STRINGS_EQUAL(NJson::CompactifyJson("[\n \"a\"\n]", true), "[a]"); - UNIT_ASSERT_STRINGS_EQUAL(NJson::CompactifyJson("[\n 'a',\n 'b'\n]", true), "[a,b]"); - UNIT_ASSERT_STRINGS_EQUAL(NJson::CompactifyJson("[\n { },\n 'b'\n]", true), "[{},b]"); - UNIT_ASSERT_STRINGS_EQUAL(NJson::CompactifyJson("[\n 'a',\n { }\n]", true), "[a,{}]"); - UNIT_ASSERT_STRINGS_EQUAL(NJson::CompactifyJson("[\n { },\n { }\n]", true), "[{},{}]"); - UNIT_ASSERT_STRINGS_EQUAL(NJson::CompactifyJson("{ }"), "{}"); - UNIT_ASSERT_STRINGS_EQUAL(NJson::CompactifyJson("{\n 'k' : 'v'\n}", true), "{k:v}"); + UNIT_ASSERT_STRINGS_EQUAL(NJson::CompactifyJson(""), ""); + UNIT_ASSERT_STRINGS_EQUAL(NJson::CompactifyJson("null"), "null"); + UNIT_ASSERT_STRINGS_EQUAL(NJson::CompactifyJson("true"), "true"); + UNIT_ASSERT_STRINGS_EQUAL(NJson::CompactifyJson("false"), "false"); + UNIT_ASSERT_STRINGS_EQUAL(NJson::CompactifyJson("1.5"), "1.5"); + UNIT_ASSERT_STRINGS_EQUAL(NJson::CompactifyJson("test", true), "test"); + UNIT_ASSERT_STRINGS_EQUAL(NJson::CompactifyJson("test", false), "\"test\""); + + UNIT_ASSERT_STRINGS_EQUAL(NJson::CompactifyJson("[ ]"), "[]"); + UNIT_ASSERT_STRINGS_EQUAL(NJson::CompactifyJson("[\n \"a\"\n]", true), "[a]"); + UNIT_ASSERT_STRINGS_EQUAL(NJson::CompactifyJson("[\n 'a',\n 'b'\n]", true), "[a,b]"); + UNIT_ASSERT_STRINGS_EQUAL(NJson::CompactifyJson("[\n { },\n 'b'\n]", true), "[{},b]"); + UNIT_ASSERT_STRINGS_EQUAL(NJson::CompactifyJson("[\n 'a',\n { }\n]", true), "[a,{}]"); + UNIT_ASSERT_STRINGS_EQUAL(NJson::CompactifyJson("[\n { },\n { }\n]", true), "[{},{}]"); + UNIT_ASSERT_STRINGS_EQUAL(NJson::CompactifyJson("{ }"), "{}"); + UNIT_ASSERT_STRINGS_EQUAL(NJson::CompactifyJson("{\n 'k' : 'v'\n}", true), "{k:v}"); } Y_UNIT_TEST(CompactifyJsonLong) { - UNIT_ASSERT_STRINGS_EQUAL(NJson::CompactifyJson( + UNIT_ASSERT_STRINGS_EQUAL(NJson::CompactifyJson( "[\n" " {\n" " 'k' : 'v'\n" @@ -166,8 +166,8 @@ Y_UNIT_TEST_SUITE(JsonPrettifier) { "}", true), "{k:v,a:b,x:[1,2,3]}"); - - UNIT_ASSERT_STRINGS_EQUAL(NJson::CompactifyJson( + + UNIT_ASSERT_STRINGS_EQUAL(NJson::CompactifyJson( "{\n" " 'k' : 'v',\n" " 'a' : 'b',\n" @@ -182,8 +182,8 @@ Y_UNIT_TEST_SUITE(JsonPrettifier) { "}", true), "{k:v,a:b,x:[1,{f:b},3],m:n}"); - - UNIT_ASSERT_STRINGS_EQUAL(NJson::CompactifyJson( + + UNIT_ASSERT_STRINGS_EQUAL(NJson::CompactifyJson( "{\n" " g : {\n" " x : {\n" @@ -200,5 +200,5 @@ Y_UNIT_TEST_SUITE(JsonPrettifier) { "}", true), "{g:{x:{a:{b:c,e:f},q:{x:y}},y:fff}}"); - } + } } diff --git a/library/cpp/json/ut/json_reader_fast_ut.cpp b/library/cpp/json/ut/json_reader_fast_ut.cpp index bc1a84b3a6..60dffc91c7 100644 --- a/library/cpp/json/ut/json_reader_fast_ut.cpp +++ b/library/cpp/json/ut/json_reader_fast_ut.cpp @@ -1,12 +1,12 @@ #include <library/cpp/json/json_reader.h> #include <library/cpp/json/json_prettifier.h> #include <library/cpp/testing/unittest/registar.h> - + #include <library/cpp/string_utils/relaxed_escaper/relaxed_escaper.h> #include <util/string/cast.h> -#include <util/string/printf.h> - -namespace NJson { +#include <util/string/printf.h> + +namespace NJson { namespace NTest { enum ETestEvent { E_NO_EVENT = 0, @@ -23,37 +23,37 @@ namespace NJson { E_STR, E_KEY }; - + struct TEvent { ETestEvent Type = E_NO_EVENT; - + i64 INum = 0; double DNum = 0; TString Str; - + TEvent(ETestEvent e = E_NO_EVENT) : Type(e) { } - + TEvent(double v, ETestEvent e) : Type(e) , DNum(v) { } - + TEvent(i64 v, ETestEvent e) : Type(e) , INum(v) { } - + TEvent(TStringBuf t, ETestEvent e) : Type(e) , Str(NEscJ::EscapeJ<true, false>(t)) { } - + TString ToString() const { switch (Type) { default: @@ -83,74 +83,74 @@ namespace NJson { } } }; - + using TEvents = TVector<TEvent>; - + struct TTestHandler : TJsonCallbacks { TEvents Events; - + bool OnOpenMap() override { Events.push_back(E_DICT_OPEN); return true; } - + bool OnCloseMap() override { Events.push_back(E_DICT_CLOSE); return true; } - + bool OnOpenArray() override { Events.push_back(E_ARR_OPEN); return true; } - + bool OnCloseArray() override { Events.push_back(E_ARR_CLOSE); return true; } - + bool OnNull() override { Events.push_back(E_NULL); return true; } - + bool OnBoolean(bool v) override { Events.push_back(TEvent((i64)v, E_BOOL)); return true; } - + bool OnInteger(long long v) override { Events.push_back(TEvent((i64)v, E_INT)); return true; } - + bool OnUInteger(unsigned long long v) override { return OnInteger(v); } - + bool OnDouble(double v) override { Events.push_back(TEvent(v, E_FLT)); return true; } - + bool OnString(const TStringBuf& v) override { Events.push_back(TEvent(v, E_STR)); return true; } - + bool OnMapKey(const TStringBuf& v) override { Events.push_back(TEvent(v, E_KEY)); return true; } - + void OnError(size_t, TStringBuf token) override { Events.push_back(TEvent(token, E_ERROR)); } - + void Assert(const TEvents& e, TString str) { try { UNIT_ASSERT_VALUES_EQUAL_C(e.size(), Events.size(), str); - + for (ui32 i = 0, sz = e.size(); i < sz; ++i) { UNIT_ASSERT_VALUES_EQUAL_C((int)e[i].Type, (int)Events[i].Type, Sprintf("'%s' %u", str.data(), i)); UNIT_ASSERT_VALUES_EQUAL_C(e[i].INum, Events[i].INum, Sprintf("'%s' %u", str.data(), i)); @@ -162,32 +162,32 @@ namespace NJson { for (const auto& event : Events) { Clog << event.ToString() << Endl; } - + throw; } - } + } }; - } -} - + } +} + class TFastJsonTest: public TTestBase { UNIT_TEST_SUITE(TFastJsonTest) - UNIT_TEST(TestParse) + UNIT_TEST(TestParse) UNIT_TEST(TestReadJsonFastTree) - UNIT_TEST(TestNoInlineComment) + UNIT_TEST(TestNoInlineComment) UNIT_TEST_SUITE_END(); -public: - template <bool accept> - void DoTestParse(TStringBuf json, ui32 amount, ...) { - using namespace NJson::NTest; - TEvents evs; - va_list vl; - va_start(vl, amount); - for (ui32 i = 0; i < amount; i++) { - ETestEvent e = (ETestEvent)va_arg(vl, int); - - switch ((int)e) { +public: + template <bool accept> + void DoTestParse(TStringBuf json, ui32 amount, ...) { + using namespace NJson::NTest; + TEvents evs; + va_list vl; + va_start(vl, amount); + for (ui32 i = 0; i < amount; i++) { + ETestEvent e = (ETestEvent)va_arg(vl, int); + + switch ((int)e) { case E_NO_EVENT: case E_DICT_OPEN: case E_DICT_CLOSE: @@ -227,40 +227,40 @@ public: evs.push_back(TEvent(TStringBuf(s), e)); break; } - } - } - va_end(vl); - - TTestHandler h; + } + } + va_end(vl); + + TTestHandler h; const bool res = ReadJsonFast(json, &h); UNIT_ASSERT_VALUES_EQUAL_C(res, accept, Sprintf("%s (%s)", ToString(json).data(), h.Events.back().Str.data())); h.Assert(evs, ToString(json)); - } - - void TestParse() { - using namespace NJson::NTest; - - DoTestParse<true>("", 0); - DoTestParse<true>(" \t \t ", 0); - DoTestParse<true>("a-b-c@аб_вгд909AБ", 1, E_STR, "a-b-c@аб_вгд909AБ"); - DoTestParse<true>("'я тестовая строка'", 1, E_STR, "я тестовая строка"); - DoTestParse<true>("\"я тестовая строка\"", 1, E_STR, "я тестовая строка"); - DoTestParse<true>("'\\xA\\xA\\xA'", 1, E_STR, "\n\n\n"); - DoTestParse<true>("12.15", 1, E_FLT, 12.15); - DoTestParse<true>("null", 1, E_NULL); - DoTestParse<true>("true", 1, E_BOOL, true); - DoTestParse<true>("false", 1, E_BOOL, false); - DoTestParse<true>("[]", 2, E_ARR_OPEN, E_ARR_CLOSE); - DoTestParse<true>("[ a ]", 3, E_ARR_OPEN, E_STR, "a", E_ARR_CLOSE); - DoTestParse<true>("[ a, b ]", 4, E_ARR_OPEN, E_STR, "a", E_STR, "b", E_ARR_CLOSE); - DoTestParse<true>("[a,b]", 4, E_ARR_OPEN, E_STR, "a", E_STR, "b", E_ARR_CLOSE); - DoTestParse<false>("[a,b][a,b]", 5, E_ARR_OPEN, E_STR, "a", E_STR, "b", E_ARR_CLOSE, E_ERROR, "invalid syntax at token: '['"); - DoTestParse<false>("[a,,b]", 3, E_ARR_OPEN, E_STR, "a", E_ERROR, "invalid syntax at token: ','"); - DoTestParse<true>("{ k : v }", 4, E_DICT_OPEN, E_KEY, "k", E_STR, "v", E_DICT_CLOSE); - DoTestParse<true>("{a:'\\b'/*comment*/, k /*comment*/\n : v }", 6, E_DICT_OPEN, E_KEY, "a", E_STR, "\b", E_KEY, "k", E_STR, "v", E_DICT_CLOSE); + } + + void TestParse() { + using namespace NJson::NTest; + + DoTestParse<true>("", 0); + DoTestParse<true>(" \t \t ", 0); + DoTestParse<true>("a-b-c@аб_вгд909AБ", 1, E_STR, "a-b-c@аб_вгд909AБ"); + DoTestParse<true>("'я тестовая строка'", 1, E_STR, "я тестовая строка"); + DoTestParse<true>("\"я тестовая строка\"", 1, E_STR, "я тестовая строка"); + DoTestParse<true>("'\\xA\\xA\\xA'", 1, E_STR, "\n\n\n"); + DoTestParse<true>("12.15", 1, E_FLT, 12.15); + DoTestParse<true>("null", 1, E_NULL); + DoTestParse<true>("true", 1, E_BOOL, true); + DoTestParse<true>("false", 1, E_BOOL, false); + DoTestParse<true>("[]", 2, E_ARR_OPEN, E_ARR_CLOSE); + DoTestParse<true>("[ a ]", 3, E_ARR_OPEN, E_STR, "a", E_ARR_CLOSE); + DoTestParse<true>("[ a, b ]", 4, E_ARR_OPEN, E_STR, "a", E_STR, "b", E_ARR_CLOSE); + DoTestParse<true>("[a,b]", 4, E_ARR_OPEN, E_STR, "a", E_STR, "b", E_ARR_CLOSE); + DoTestParse<false>("[a,b][a,b]", 5, E_ARR_OPEN, E_STR, "a", E_STR, "b", E_ARR_CLOSE, E_ERROR, "invalid syntax at token: '['"); + DoTestParse<false>("[a,,b]", 3, E_ARR_OPEN, E_STR, "a", E_ERROR, "invalid syntax at token: ','"); + DoTestParse<true>("{ k : v }", 4, E_DICT_OPEN, E_KEY, "k", E_STR, "v", E_DICT_CLOSE); + DoTestParse<true>("{a:'\\b'/*comment*/, k /*comment*/\n : v }", 6, E_DICT_OPEN, E_KEY, "a", E_STR, "\b", E_KEY, "k", E_STR, "v", E_DICT_CLOSE); DoTestParse<true>("{a:.15, k : v }", 6, E_DICT_OPEN, E_KEY, "a", E_FLT, .15, E_KEY, "k", E_STR, "v", E_DICT_CLOSE); DoTestParse<true>("[ a, -.1e+5, 1E-7]", 5, E_ARR_OPEN, E_STR, "a", E_FLT, -.1e+5, E_FLT, 1e-7, E_ARR_CLOSE); - DoTestParse<true>("{}", 2, E_DICT_OPEN, E_DICT_CLOSE); + DoTestParse<true>("{}", 2, E_DICT_OPEN, E_DICT_CLOSE); DoTestParse<true>("{ a : x, b : [ c, d, ] }", 9, E_DICT_OPEN, E_KEY, "a", E_STR, "x", E_KEY, "b", E_ARR_OPEN, E_STR, "c", E_STR, "d", E_ARR_CLOSE, E_DICT_CLOSE); DoTestParse<false>("{ a : x, b : [ c, d,, ] }", 8, E_DICT_OPEN, E_KEY, "a", E_STR, "x", E_KEY, "b", E_ARR_OPEN, E_STR, "c", E_STR, "d", E_ERROR, "invalid syntax at token: ','"); // DoTestParse<false>("{ a : x : y }", 4, E_DICT_OPEN @@ -271,14 +271,14 @@ public: // , E_KEY, "queries", E_DICT_OPEN // , E_KEY, "ref", E_ARR_OPEN, E_ARR_CLOSE // , E_DICT_CLOSE, E_ERROR, ""); - DoTestParse<true>("'100x00'", 1, E_STR, "100x00"); - DoTestParse<true>("-1", 1, E_INT, -1); - DoTestParse<true>("-9223372036854775808", 1, E_LONG_LONG, (long long)Min<i64>()); - DoTestParse<false>("100x00", 1, E_ERROR, "invalid syntax at token: '100x'"); - DoTestParse<false>("100 200", 2, E_INT, 100, E_ERROR, "invalid syntax at token: '200'"); + DoTestParse<true>("'100x00'", 1, E_STR, "100x00"); + DoTestParse<true>("-1", 1, E_INT, -1); + DoTestParse<true>("-9223372036854775808", 1, E_LONG_LONG, (long long)Min<i64>()); + DoTestParse<false>("100x00", 1, E_ERROR, "invalid syntax at token: '100x'"); + DoTestParse<false>("100 200", 2, E_INT, 100, E_ERROR, "invalid syntax at token: '200'"); DoTestParse<true>("{g:{x:{a:{b:c,e:f},q:{x:y}},y:fff}}", 22, E_DICT_OPEN, E_KEY, "g", E_DICT_OPEN, E_KEY, "x", E_DICT_OPEN, E_KEY, "a", E_DICT_OPEN, E_KEY, "b", E_STR, "c", E_KEY, "e", E_STR, "f", E_DICT_CLOSE, E_KEY, "q", E_DICT_OPEN, E_KEY, "x", E_STR, "y", E_DICT_CLOSE, E_DICT_CLOSE, E_KEY, "y", E_STR, "fff", E_DICT_CLOSE, E_DICT_CLOSE); - } - + } + void TestReadJsonFastTree() { const TString json = R"( { @@ -290,15 +290,15 @@ public: NJson::TJsonValue value; UNIT_ASSERT(!ReadJsonFastTree(json, &value)); } - - void TestNoInlineComment() { - using namespace NJson::NTest; - DoTestParse<false>("{\"a\":1}//d{\"b\":2}", 5, E_DICT_OPEN, E_KEY, "a", E_INT, 1, E_DICT_CLOSE, E_ERROR, "invalid syntax at token: '/'"); - DoTestParse<false>("{\"a\":1}//d{\"b\":2}\n", 5, E_DICT_OPEN, E_KEY, "a", E_INT, 1, E_DICT_CLOSE, E_ERROR, "invalid syntax at token: '/'"); - DoTestParse<false>("{\"a\":{//d{\"b\":2}\n}}", 4, E_DICT_OPEN, E_KEY, "a", E_DICT_OPEN, E_ERROR, "invalid syntax at token: '/'"); - DoTestParse<false>("{\"a\":{//d{\"b\":2}}}\n", 4, E_DICT_OPEN, E_KEY, "a", E_DICT_OPEN, E_ERROR, "invalid syntax at token: '/'"); - DoTestParse<false>("{\"a\":{//d{\"b\":2}}}", 4, E_DICT_OPEN, E_KEY, "a", E_DICT_OPEN, E_ERROR, "invalid syntax at token: '/'"); - } -}; - -UNIT_TEST_SUITE_REGISTRATION(TFastJsonTest) + + void TestNoInlineComment() { + using namespace NJson::NTest; + DoTestParse<false>("{\"a\":1}//d{\"b\":2}", 5, E_DICT_OPEN, E_KEY, "a", E_INT, 1, E_DICT_CLOSE, E_ERROR, "invalid syntax at token: '/'"); + DoTestParse<false>("{\"a\":1}//d{\"b\":2}\n", 5, E_DICT_OPEN, E_KEY, "a", E_INT, 1, E_DICT_CLOSE, E_ERROR, "invalid syntax at token: '/'"); + DoTestParse<false>("{\"a\":{//d{\"b\":2}\n}}", 4, E_DICT_OPEN, E_KEY, "a", E_DICT_OPEN, E_ERROR, "invalid syntax at token: '/'"); + DoTestParse<false>("{\"a\":{//d{\"b\":2}}}\n", 4, E_DICT_OPEN, E_KEY, "a", E_DICT_OPEN, E_ERROR, "invalid syntax at token: '/'"); + DoTestParse<false>("{\"a\":{//d{\"b\":2}}}", 4, E_DICT_OPEN, E_KEY, "a", E_DICT_OPEN, E_ERROR, "invalid syntax at token: '/'"); + } +}; + +UNIT_TEST_SUITE_REGISTRATION(TFastJsonTest) diff --git a/library/cpp/json/ut/json_reader_ut.cpp b/library/cpp/json/ut/json_reader_ut.cpp index 92435e52a4..cd31afa0b8 100644 --- a/library/cpp/json/ut/json_reader_ut.cpp +++ b/library/cpp/json/ut/json_reader_ut.cpp @@ -230,7 +230,7 @@ Y_UNIT_TEST_SUITE(TJsonReaderTest) { UNIT_ASSERT(value.Has("test")); UNIT_ASSERT(!value["test"].IsInteger()); UNIT_ASSERT(value["test"].IsUInteger()); - UNIT_ASSERT_EQUAL(value["test"].GetIntegerRobust(), (i64)(Max<i64>() + 1ull)); + UNIT_ASSERT_EQUAL(value["test"].GetIntegerRobust(), (i64)(Max<i64>() + 1ull)); } // Max<i64>() + 1 } diff --git a/library/cpp/json/ut/json_writer_ut.cpp b/library/cpp/json/ut/json_writer_ut.cpp index 9bbd4e3f77..ca11d34dad 100644 --- a/library/cpp/json/ut/json_writer_ut.cpp +++ b/library/cpp/json/ut/json_writer_ut.cpp @@ -11,48 +11,48 @@ Y_UNIT_TEST_SUITE(TJsonWriterTest) { TString expected2 = expected1 + ",\"array\":[\"stroka\",false]"; TString expected3 = expected2 + "}"; - TStringStream out; + TStringStream out; - TJsonWriter json(&out, false); - json.OpenMap(); - json.Write("key1", (ui16)1); + TJsonWriter json(&out, false); + json.OpenMap(); + json.Write("key1", (ui16)1); json.WriteKey("key2"); json.Write((i32)2); - json.Write("key3", (ui64)3); + json.Write("key3", (ui64)3); UNIT_ASSERT(out.Empty()); - json.Flush(); - UNIT_ASSERT_VALUES_EQUAL(out.Str(), expected1); + json.Flush(); + UNIT_ASSERT_VALUES_EQUAL(out.Str(), expected1); - json.Write("array"); - json.OpenArray(); - json.Write("stroka"); - json.Write(false); - json.CloseArray(); + json.Write("array"); + json.OpenArray(); + json.Write("stroka"); + json.Write(false); + json.CloseArray(); - UNIT_ASSERT_VALUES_EQUAL(out.Str(), expected1); - json.Flush(); - UNIT_ASSERT_VALUES_EQUAL(out.Str(), expected2); + UNIT_ASSERT_VALUES_EQUAL(out.Str(), expected1); + json.Flush(); + UNIT_ASSERT_VALUES_EQUAL(out.Str(), expected2); - json.CloseMap(); + json.CloseMap(); - UNIT_ASSERT_VALUES_EQUAL(out.Str(), expected2); - json.Flush(); - UNIT_ASSERT_VALUES_EQUAL(out.Str(), expected3); - } + UNIT_ASSERT_VALUES_EQUAL(out.Str(), expected2); + json.Flush(); + UNIT_ASSERT_VALUES_EQUAL(out.Str(), expected3); + } Y_UNIT_TEST(SimpleWriteValueTest) { TString expected = "{\"key1\":null,\"key2\":{\"subkey1\":[1,{\"subsubkey\":\"test2\"},null,true],\"subkey2\":\"test\"}}"; - TJsonValue v; - v["key1"] = JSON_NULL; - v["key2"]["subkey1"].AppendValue(1); - v["key2"]["subkey1"].AppendValue(JSON_MAP)["subsubkey"] = "test2"; - v["key2"]["subkey1"].AppendValue(JSON_NULL); - v["key2"]["subkey1"].AppendValue(true); - v["key2"]["subkey2"] = "test"; - TStringStream out; - WriteJson(&out, &v); - UNIT_ASSERT_VALUES_EQUAL(out.Str(), expected); + TJsonValue v; + v["key1"] = JSON_NULL; + v["key2"]["subkey1"].AppendValue(1); + v["key2"]["subkey1"].AppendValue(JSON_MAP)["subsubkey"] = "test2"; + v["key2"]["subkey1"].AppendValue(JSON_NULL); + v["key2"]["subkey1"].AppendValue(true); + v["key2"]["subkey2"] = "test"; + TStringStream out; + WriteJson(&out, &v); + UNIT_ASSERT_VALUES_EQUAL(out.Str(), expected); } Y_UNIT_TEST(FormatOutput) { diff --git a/library/cpp/json/writer/json.h b/library/cpp/json/writer/json.h index e8639713f6..0aae2531b9 100644 --- a/library/cpp/json/writer/json.h +++ b/library/cpp/json/writer/json.h @@ -148,8 +148,8 @@ namespace NJsonWriter { IOutputStream* Stream; THolder<TStringStream> StringStream; typedef TVector<const TString*> TKeys; - TKeys Keys; - + TKeys Keys; + TVector<EJsonEntity> Stack; bool NeedComma; bool NeedNewline; diff --git a/library/cpp/json/writer/json_value.cpp b/library/cpp/json/writer/json_value.cpp index 64ba408bce..c61e8d1dc4 100644 --- a/library/cpp/json/writer/json_value.cpp +++ b/library/cpp/json/writer/json_value.cpp @@ -5,15 +5,15 @@ #include <util/generic/ylimits.h> #include <util/generic/utility.h> #include <util/generic/singleton.h> -#include <util/stream/str.h> +#include <util/stream/str.h> #include <util/stream/output.h> -#include <util/string/cast.h> -#include <util/string/type.h> +#include <util/string/cast.h> +#include <util/string/type.h> #include <util/string/vector.h> #include <util/system/yassert.h> #include <util/ysaveload.h> #include <util/generic/bt_exception.h> - + static bool AreJsonMapsEqual(const NJson::TJsonValue& lhs, const NJson::TJsonValue& rhs) { using namespace NJson; @@ -120,7 +120,7 @@ namespace NJson { tmp.Swap(*this); return *this; } - + TJsonValue::TJsonValue(const bool value) noexcept { SetType(JSON_BOOLEAN); Value.Boolean = value; @@ -160,7 +160,7 @@ namespace NJson { SetType(JSON_DOUBLE); Value.Double = value; } - + TJsonValue::TJsonValue(TString value) { SetType(JSON_STRING); Value.String = std::move(value); @@ -170,12 +170,12 @@ namespace NJson { SetType(JSON_STRING); Value.String = value; } - + TJsonValue::TJsonValue(const char* value) { SetType(JSON_STRING); Value.String = value; } - + EJsonValueType TJsonValue::GetType() const noexcept { return Type; } @@ -183,7 +183,7 @@ namespace NJson { TJsonValue& TJsonValue::SetType(const EJsonValueType type) { if (Type == type) return *this; - + Clear(); Type = type; @@ -237,7 +237,7 @@ namespace NJson { SetType(JSON_MAP); return (*Value.Map)[key] = std::move(value); } - + TJsonValue& TJsonValue::InsertValue(const TStringBuf key, TJsonValue&& value) { SetType(JSON_MAP); return (*Value.Map)[key] = std::move(value); @@ -351,7 +351,7 @@ namespace NJson { bool TJsonValue::GetBoolean() const { return Type != JSON_BOOLEAN ? false : Value.Boolean; } - + long long TJsonValue::GetInteger() const { if (!IsInteger()) return 0; @@ -371,7 +371,7 @@ namespace NJson { return 0; } } - + unsigned long long TJsonValue::GetUInteger() const { if (!IsUInteger()) return 0; @@ -411,19 +411,19 @@ namespace NJson { return 0.0; } } - + const TString& TJsonValue::GetString() const { return Type != JSON_STRING ? Singleton<TDefaultsHolder>()->String : Value.String; } - + const TJsonValue::TMapType& TJsonValue::GetMap() const { return Type != JSON_MAP ? Singleton<TDefaultsHolder>()->Map : *Value.Map; } - + const TJsonValue::TArray& TJsonValue::GetArray() const { return (Type != JSON_ARRAY) ? Singleton<TDefaultsHolder>()->Array : *Value.Array; } - + bool TJsonValue::GetBooleanSafe() const { if (Type != JSON_BOOLEAN) ythrow TJsonException() << "Not a boolean"; @@ -535,8 +535,8 @@ namespace NJson { case JSON_BOOLEAN: return Value.Boolean; } - } - + } + long long TJsonValue::GetIntegerRobust() const noexcept { switch (Type) { case JSON_ARRAY: @@ -564,8 +564,8 @@ namespace NJson { case JSON_UINTEGER: return Value.Integer; } - } - + } + unsigned long long TJsonValue::GetUIntegerRobust() const noexcept { switch (Type) { case JSON_ARRAY: @@ -623,8 +623,8 @@ namespace NJson { case JSON_DOUBLE: return Value.Double; } - } - + } + TString TJsonValue::GetStringRobust() const { switch (Type) { case JSON_ARRAY: @@ -643,12 +643,12 @@ namespace NJson { case JSON_STRING: return Value.String; } - } - + } + bool TJsonValue::GetBoolean(bool* value) const noexcept { if (Type != JSON_BOOLEAN) return false; - + *value = Value.Boolean; return true; } @@ -656,7 +656,7 @@ namespace NJson { bool TJsonValue::GetInteger(long long* value) const noexcept { if (!IsInteger()) return false; - + *value = GetInteger(); return true; } @@ -672,7 +672,7 @@ namespace NJson { bool TJsonValue::GetDouble(double* value) const noexcept { if (!IsDouble()) return false; - + *value = GetDouble(); return true; } @@ -680,7 +680,7 @@ namespace NJson { bool TJsonValue::GetString(TString* value) const { if (Type != JSON_STRING) return false; - + *value = Value.String; return true; } @@ -688,7 +688,7 @@ namespace NJson { bool TJsonValue::GetMap(TJsonValue::TMapType* value) const { if (Type != JSON_MAP) return false; - + *value = *Value.Map; return true; } @@ -696,7 +696,7 @@ namespace NJson { bool TJsonValue::GetArray(TJsonValue::TArray* value) const { if (Type != JSON_ARRAY) return false; - + *value = *Value.Array; return true; } @@ -712,7 +712,7 @@ namespace NJson { bool TJsonValue::GetArrayPointer(const TJsonValue::TArray** value) const noexcept { if (Type != JSON_ARRAY) return false; - + *value = Value.Array; return true; } @@ -806,10 +806,10 @@ namespace NJson { return true; case JSON_INTEGER: - return (1ll << std::numeric_limits<double>::digits) >= Abs(Value.Integer); + return (1ll << std::numeric_limits<double>::digits) >= Abs(Value.Integer); case JSON_UINTEGER: - return (1ull << std::numeric_limits<double>::digits) >= Value.UInteger; + return (1ull << std::numeric_limits<double>::digits) >= Value.UInteger; default: return false; @@ -1052,12 +1052,12 @@ namespace NJson { } //**************************************************************** - + bool GetMapPointer(const TJsonValue& jv, const size_t index, const TJsonValue::TMapType** value) { const TJsonValue* v; if (!jv.GetValuePointer(index, &v) || !v->IsMap()) return false; - + *value = &v->GetMap(); return true; } @@ -1066,7 +1066,7 @@ namespace NJson { const TJsonValue* v; if (!jv.GetValuePointer(index, &v) || !v->IsArray()) return false; - + *value = &v->GetArray(); return true; } @@ -1075,11 +1075,11 @@ namespace NJson { const TJsonValue* v; if (!jv.GetValuePointer(key, &v) || !v->IsMap()) return false; - + *value = &v->GetMap(); return true; } - + bool GetArrayPointer(const TJsonValue& jv, const TStringBuf key, const TJsonValue::TArray** value) { const TJsonValue* v; if (!jv.GetValuePointer(key, &v) || !v->IsArray()) @@ -1088,7 +1088,7 @@ namespace NJson { *value = &v->GetArray(); return true; } - + void TJsonValue::BackChecks() const { if (Type != JSON_ARRAY) ythrow TJsonException() << "Not an array"; diff --git a/library/cpp/json/writer/json_value.h b/library/cpp/json/writer/json_value.h index 1646f4297a..3f0f50bc4c 100644 --- a/library/cpp/json/writer/json_value.h +++ b/library/cpp/json/writer/json_value.h @@ -68,7 +68,7 @@ namespace NJson { ~TJsonValue() { Clear(); } - + EJsonValueType GetType() const noexcept; TJsonValue& SetType(EJsonValueType type); @@ -143,7 +143,7 @@ namespace NJson { bool GetDouble(double* value) const noexcept; bool GetMapPointer(const TMapType** value) const noexcept; bool GetArrayPointer(const TArray** value) const noexcept; - + bool GetString(TString* value) const; bool GetMap(TMapType* value) const; bool GetArray(TArray* value) const; diff --git a/library/cpp/json/ya.make b/library/cpp/json/ya.make index df9deb651c..d58eead8ec 100644 --- a/library/cpp/json/ya.make +++ b/library/cpp/json/ya.make @@ -4,7 +4,7 @@ OWNER( pg velavokr ) - + SRCS( json_writer.cpp json_reader.cpp diff --git a/library/cpp/lcs/README.md b/library/cpp/lcs/README.md index d58334e776..7ed3d946d6 100644 --- a/library/cpp/lcs/README.md +++ b/library/cpp/lcs/README.md @@ -1,11 +1,11 @@ A set of algorithms for approximate string matching. ==================================================== - + **Lcs_via_lis.h** Combinatorial algorithm LCS (Longest common subsequence) through LIS (Longest increasing subsequence) for rows S1 and S2 of lengths n and m respectively (we assume that n < m). Complexity is O(r log n) by time and O(r) of additional memory, where r is the number of pairs (i, j) for which S1[i] = S2[j]. Thus for the uniform distribution of letters of an alphabet of s elements the estimate is O(nm / s * log n). - + Effective for large alphabets s = O(n) (like hashes of words in a text). If only the LCS length is needed, the LCS itself will not be built. See Gusfield, Algorithms on Strings, Trees and Sequences, 1997, Chapt.12.5 Or here: http://www.cs.ucf.edu/courses/cap5937/fall2004/Longest%20common%20subsequence.pdf @@ -23,7 +23,7 @@ Denote by: It is easy to prove the theorem that the dimensions of SC and LIS are the same, and it is possible to construct LIS from SC. Next, let's suppose we have 2 strings of S1 and S2. It can be shown that if for each symbol in S1 we take the list of its appearances in S2 in the reverse order, and concatenate all such lists keeping order, then any IS in the resulting list will be equivalent to some common subsequence S1 and S2 of the same length. And, consequently, LIS will be equivalent to LCS. - + The idea of the algorithm for constructing DS: - Going along the original sequence, for the current member x in the list of its DS we find the leftmost, such that its last term is not less than x. - If there is one, then add x to the end. diff --git a/library/cpp/lcs/lcs_via_lis.h b/library/cpp/lcs/lcs_via_lis.h index 43e1c7ccd5..d26733d94e 100644 --- a/library/cpp/lcs/lcs_via_lis.h +++ b/library/cpp/lcs/lcs_via_lis.h @@ -1,34 +1,34 @@ -#pragma once - +#pragma once + #include <library/cpp/containers/paged_vector/paged_vector.h> -#include <util/generic/ptr.h> -#include <util/generic/hash.h> -#include <util/generic/vector.h> +#include <util/generic/ptr.h> +#include <util/generic/hash.h> +#include <util/generic/vector.h> #include <util/generic/algorithm.h> -#include <util/memory/pool.h> - -namespace NLCS { +#include <util/memory/pool.h> + +namespace NLCS { template <typename TVal> struct TLCSCtx { typedef TVector<ui32> TSubsequence; typedef THashMap<TVal, TSubsequence, THash<TVal>, TEqualTo<TVal>, ::TPoolAllocator> TEncounterIndex; typedef TVector<std::pair<ui32, ui32>> TLastIndex; typedef NPagedVector::TPagedVector<TSubsequence, 4096> TCover; - + TMemoryPool Pool; THolder<TEncounterIndex> Encounters; TLastIndex LastIndex; TCover Cover; - + TSubsequence ResultBuffer; - + TLCSCtx() : Pool(16 * 1024 - 64, TMemoryPool::TExpGrow::Instance()) { Reset(); } - + void Reset() { Encounters.Reset(nullptr); Pool.Clear(); @@ -38,17 +38,17 @@ namespace NLCS { ResultBuffer.clear(); } }; - + namespace NPrivate { template <typename TIt, typename TVl> struct TSequence { typedef TIt TIter; typedef TVl TVal; - + const TIter Begin; const TIter End; const size_t Size; - + TSequence(TIter beg, TIter end) : Begin(beg) , End(end) @@ -56,45 +56,45 @@ namespace NLCS { { } }; - + template <typename TVal, typename TSequence, typename TResult> size_t MakeLCS(TSequence s1, TSequence s2, TResult* res = nullptr, TLCSCtx<TVal>* ctx = nullptr) { typedef TLCSCtx<TVal> TCtx; - + THolder<TCtx> ctxhld; - + if (!ctx) { ctxhld.Reset(new TCtx()); ctx = ctxhld.Get(); } else { ctx->Reset(); } - + size_t maxsize = Max(s1.Size, s2.Size); auto& index = *(ctx->Encounters); - + for (auto it = s1.Begin; it != s1.End; ++it) { index[*it]; } - + for (auto it = s2.Begin; it != s2.End; ++it) { auto hit = index.find(*it); - + if (hit != index.end()) { hit->second.push_back(it - s2.Begin); } } - + if (!res) { auto& lastindex = ctx->ResultBuffer; lastindex.reserve(maxsize); for (auto it1 = s1.Begin; it1 != s1.End; ++it1) { const auto& sub2 = index[*it1]; - + for (auto it2 = sub2.rbegin(); it2 != sub2.rend(); ++it2) { ui32 x = *it2; - + auto lit = LowerBound(lastindex.begin(), lastindex.end(), x); if (lit == lastindex.end()) { @@ -104,22 +104,22 @@ namespace NLCS { } } } - + return lastindex.size(); } else { auto& lastindex = ctx->LastIndex; auto& cover = ctx->Cover; lastindex.reserve(maxsize); - + for (auto it1 = s1.Begin; it1 != s1.End; ++it1) { const auto& sub2 = index[*it1]; - + for (auto it2 = sub2.rbegin(); it2 != sub2.rend(); ++it2) { ui32 x = *it2; - + auto lit = LowerBound(lastindex.begin(), lastindex.end(), std::make_pair(x, (ui32)0u)); - + if (lit == lastindex.end()) { lastindex.push_back(std::make_pair(x, cover.size())); cover.emplace_back(); @@ -129,16 +129,16 @@ namespace NLCS { cover[lit->second].push_back(x); } } - } - + } + if (cover.empty()) { return 0; } - + std::reverse(cover.begin(), cover.end()); - + auto& resbuf = ctx->ResultBuffer; - + resbuf.push_back(cover.front().front()); for (auto it = cover.begin() + 1; it != cover.end(); ++it) { @@ -148,9 +148,9 @@ namespace NLCS { resbuf.push_back(*pit); } - + std::reverse(resbuf.begin(), resbuf.end()); - + for (auto it = resbuf.begin(); it != resbuf.end(); ++it) { res->push_back(*(s2.Begin + *it)); } @@ -158,36 +158,36 @@ namespace NLCS { return lastindex.size(); } } - } - + } + template <typename TVal, typename TIter, typename TResult> size_t MakeLCS(TIter beg1, TIter end1, TIter beg2, TIter end2, TResult* res = nullptr, TLCSCtx<TVal>* ctx = nullptr) { typedef NPrivate::TSequence<TIter, TVal> TSeq; - + size_t sz1 = end1 - beg1; size_t sz2 = end2 - beg2; - + if (sz2 > sz1) { DoSwap(beg1, beg2); DoSwap(end1, end2); DoSwap(sz1, sz2); } - + return NPrivate::MakeLCS<TVal>(TSeq(beg1, end1), TSeq(beg2, end2), res, ctx); - } - + } + template <typename TVal, typename TColl, typename TRes> size_t MakeLCS(const TColl& coll1, const TColl& coll2, TRes* res = nullptr, TLCSCtx<TVal>* ctx = nullptr) { return MakeLCS<TVal>(coll1.begin(), coll1.end(), coll2.begin(), coll2.end(), res, ctx); } - + template <typename TVal, typename TIter> size_t MeasureLCS(TIter beg1, TIter end1, TIter beg2, TIter end2, TLCSCtx<TVal>* ctx = nullptr) { return MakeLCS<TVal>(beg1, end1, beg2, end2, (TVector<TVal>*)nullptr, ctx); } - + template <typename TVal, typename TColl> size_t MeasureLCS(const TColl& coll1, const TColl& coll2, TLCSCtx<TVal>* ctx = nullptr) { return MeasureLCS<TVal>(coll1.begin(), coll1.end(), coll2.begin(), coll2.end(), ctx); } -} +} diff --git a/library/cpp/lcs/lcs_via_lis_ut.cpp b/library/cpp/lcs/lcs_via_lis_ut.cpp index 4d90dd400e..f6ad5152b6 100644 --- a/library/cpp/lcs/lcs_via_lis_ut.cpp +++ b/library/cpp/lcs/lcs_via_lis_ut.cpp @@ -1,64 +1,64 @@ -#include <util/generic/utility.h> -#include <util/string/util.h> +#include <util/generic/utility.h> +#include <util/string/util.h> #include <library/cpp/testing/unittest/registar.h> -#include "lcs_via_lis.h" - -class TLCSTest: public TTestBase { +#include "lcs_via_lis.h" + +class TLCSTest: public TTestBase { UNIT_TEST_SUITE(TLCSTest); UNIT_TEST(LCSTest); UNIT_TEST_SUITE_END(); -private: - size_t Length(TStringBuf s1, TStringBuf s2) { +private: + size_t Length(TStringBuf s1, TStringBuf s2) { TVector<TVector<size_t>> c; - c.resize(s1.size() + 1); - - for (size_t i = 0; i < c.size(); ++i) { - c[i].resize(s2.size() + 1); - } - - for (size_t i = 0; i < s1.size(); ++i) { - for (size_t j = 0; j < s2.size(); ++j) { - if (s1[i] == s2[j]) - c[i + 1][j + 1] = c[i][j] + 1; - else - c[i + 1][j + 1] = Max(c[i + 1][j], c[i][j + 1]); - } - } - - return c[s1.size()][s2.size()]; - } - - void CheckLCSLength(TStringBuf s1, TStringBuf s2, size_t size) { - size_t len = NLCS::MeasureLCS<char>(s1, s2); - - UNIT_ASSERT_VALUES_EQUAL(len, Length(s1, s2)); - UNIT_ASSERT_VALUES_EQUAL(len, size); - } - - void CheckLCSString(TStringBuf s1, TStringBuf s2, TStringBuf reflcs) { + c.resize(s1.size() + 1); + + for (size_t i = 0; i < c.size(); ++i) { + c[i].resize(s2.size() + 1); + } + + for (size_t i = 0; i < s1.size(); ++i) { + for (size_t j = 0; j < s2.size(); ++j) { + if (s1[i] == s2[j]) + c[i + 1][j + 1] = c[i][j] + 1; + else + c[i + 1][j + 1] = Max(c[i + 1][j], c[i][j + 1]); + } + } + + return c[s1.size()][s2.size()]; + } + + void CheckLCSLength(TStringBuf s1, TStringBuf s2, size_t size) { + size_t len = NLCS::MeasureLCS<char>(s1, s2); + + UNIT_ASSERT_VALUES_EQUAL(len, Length(s1, s2)); + UNIT_ASSERT_VALUES_EQUAL(len, size); + } + + void CheckLCSString(TStringBuf s1, TStringBuf s2, TStringBuf reflcs) { TString lcs; - size_t len = NLCS::MakeLCS<char>(s1, s2, &lcs); + size_t len = NLCS::MakeLCS<char>(s1, s2, &lcs); const char* comment = Sprintf("%s & %s = %s", s1.data(), s2.data(), reflcs.data()).c_str(); - - UNIT_ASSERT_VALUES_EQUAL_C(Length(s1, s2), len, comment); + + UNIT_ASSERT_VALUES_EQUAL_C(Length(s1, s2), len, comment); UNIT_ASSERT_VALUES_EQUAL_C(lcs.size(), len, comment); - UNIT_ASSERT_VALUES_EQUAL_C(NLCS::MeasureLCS<char>(s1, s2), len, comment); - UNIT_ASSERT_VALUES_EQUAL_C(reflcs, TStringBuf(lcs), comment); - } - - void LCSTest() { - CheckLCSString("abacx", "baabca", "bac"); + UNIT_ASSERT_VALUES_EQUAL_C(NLCS::MeasureLCS<char>(s1, s2), len, comment); + UNIT_ASSERT_VALUES_EQUAL_C(reflcs, TStringBuf(lcs), comment); + } + + void LCSTest() { + CheckLCSString("abacx", "baabca", "bac"); const char* m = "mama_myla_ramu"; const char* n = "papa_lubil_mamu"; const char* s = "aa_l_amu"; - CheckLCSString(m, n, s); - CheckLCSString(n, m, s); - CheckLCSString(m, m, m); - CheckLCSString(m, "", ""); - CheckLCSString("", m, ""); - CheckLCSString("", "", ""); - { + CheckLCSString(m, n, s); + CheckLCSString(n, m, s); + CheckLCSString(m, m, m); + CheckLCSString(m, "", ""); + CheckLCSString("", m, ""); + CheckLCSString("", "", ""); + { const char* s1 = "atmwuaoccmgirveexxtbkkmioclarskvicyesddirlrgflnietcagkswbvsdnxksipfndmusnuee" "tojygyjyobdfiutsbruuspvibmywhokxsarwbyirsqqnxxnbtkmucmdafaogwmobuuhejspgurpe" @@ -83,9 +83,9 @@ private: "ctsnyjleqgttcgpnhlnagxenuknpxiramgeshhjyoesupkcfcvvpwyweuvcwrawsgvfshppijuug" "hdnujdqjtcdissmlnjgibdljjxntxrgytxlbgvsrrusatqelspeoyvndjifjqxqrpduwbyojjbhi" "tmondbbnuuhpkglmfykeheddwkxjyapfniqoic"; - CheckLCSLength(s1, s2, 247); - } - { + CheckLCSLength(s1, s2, 247); + } + { const char* s1 = "ssyrtvllktbhijkyckviokukaodexiykxcvlvninoasblpuujnqnilnhmacsulaulskphkccnfop" "jlhejqhemdpnihasouinuceugkfkaifdepvffntbkfivsddtxyslnlwiyfbbpnrwlkryetncahih" @@ -113,8 +113,8 @@ private: "xmfwokjskummmkaksbeoowfgjwiumpbkdujexectnghqjjuotofwuwvcgtluephymdkrscbfiaeg" "aaypnclkstfqimisanikjxocmhcrotkntprwjbuudswuyuujfawjucwgifhqgepxeidmvcwqsqrv" "karuvpnxhmrvdcocidgtuxasdqkwsdvijmnpmayhfiva"; - CheckLCSLength(s1, s2, 288); - } - } -}; -UNIT_TEST_SUITE_REGISTRATION(TLCSTest) + CheckLCSLength(s1, s2, 288); + } + } +}; +UNIT_TEST_SUITE_REGISTRATION(TLCSTest) diff --git a/library/cpp/lcs/ya.make b/library/cpp/lcs/ya.make index 50455f1243..3a7caafc88 100644 --- a/library/cpp/lcs/ya.make +++ b/library/cpp/lcs/ya.make @@ -1,5 +1,5 @@ -LIBRARY() - +LIBRARY() + OWNER(velavokr) PEERDIR( diff --git a/library/cpp/lfalloc/ya.make b/library/cpp/lfalloc/ya.make index 04c81047f0..cace05f9d8 100644 --- a/library/cpp/lfalloc/ya.make +++ b/library/cpp/lfalloc/ya.make @@ -5,7 +5,7 @@ OWNER(gulin) NO_UTIL() NO_COMPILER_WARNINGS() - + IF (ARCH_AARCH64) PEERDIR( contrib/libs/jemalloc diff --git a/library/cpp/lfalloc/yt/ya.make b/library/cpp/lfalloc/yt/ya.make index 5c644a4c64..8c1a4f8a72 100644 --- a/library/cpp/lfalloc/yt/ya.make +++ b/library/cpp/lfalloc/yt/ya.make @@ -5,7 +5,7 @@ OWNER(a-romanov) NO_UTIL() NO_COMPILER_WARNINGS() - + IF (ARCH_AARCH64) PEERDIR( contrib/libs/jemalloc diff --git a/library/cpp/logger/all.h b/library/cpp/logger/all.h index 228ec5e339..ee1666844e 100644 --- a/library/cpp/logger/all.h +++ b/library/cpp/logger/all.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include "log.h" #include "null.h" diff --git a/library/cpp/logger/backend.h b/library/cpp/logger/backend.h index d150fa6739..d088726d6d 100644 --- a/library/cpp/logger/backend.h +++ b/library/cpp/logger/backend.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include "priority.h" diff --git a/library/cpp/logger/element.h b/library/cpp/logger/element.h index c8811d2922..fc9bff851f 100644 --- a/library/cpp/logger/element.h +++ b/library/cpp/logger/element.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include "priority.h" diff --git a/library/cpp/logger/file.h b/library/cpp/logger/file.h index f76966ea4a..10b4cd0c20 100644 --- a/library/cpp/logger/file.h +++ b/library/cpp/logger/file.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include "backend.h" diff --git a/library/cpp/logger/log.h b/library/cpp/logger/log.h index f0091cfdff..8be984ccc8 100644 --- a/library/cpp/logger/log.h +++ b/library/cpp/logger/log.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include "backend.h" #include "element.h" diff --git a/library/cpp/logger/null.h b/library/cpp/logger/null.h index 1089d6c75d..a02f250b00 100644 --- a/library/cpp/logger/null.h +++ b/library/cpp/logger/null.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include "backend.h" diff --git a/library/cpp/logger/priority.h b/library/cpp/logger/priority.h index de7cd4657f..d2a9fa0a07 100644 --- a/library/cpp/logger/priority.h +++ b/library/cpp/logger/priority.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once enum ELogPriority { TLOG_EMERG = 0 /* "EMERG" */, diff --git a/library/cpp/logger/record.h b/library/cpp/logger/record.h index 6f402851b1..c28a7785fd 100644 --- a/library/cpp/logger/record.h +++ b/library/cpp/logger/record.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include "priority.h" diff --git a/library/cpp/logger/stream.h b/library/cpp/logger/stream.h index 00f953fcaa..feb240afcb 100644 --- a/library/cpp/logger/stream.h +++ b/library/cpp/logger/stream.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include "backend.h" diff --git a/library/cpp/logger/system.h b/library/cpp/logger/system.h index 3b8a20e9ee..b8c60b3023 100644 --- a/library/cpp/logger/system.h +++ b/library/cpp/logger/system.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include "log.h" #include "backend.h" diff --git a/library/cpp/logger/thread.h b/library/cpp/logger/thread.h index ba80ab4223..65f7a88e87 100644 --- a/library/cpp/logger/thread.h +++ b/library/cpp/logger/thread.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include "backend.h" diff --git a/library/cpp/messagebus/base.h b/library/cpp/messagebus/base.h index 06f5a1066d..79fccc312e 100644 --- a/library/cpp/messagebus/base.h +++ b/library/cpp/messagebus/base.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <util/system/defaults.h> diff --git a/library/cpp/messagebus/coreconn.h b/library/cpp/messagebus/coreconn.h index 76b489a4c6..fca228d82e 100644 --- a/library/cpp/messagebus/coreconn.h +++ b/library/cpp/messagebus/coreconn.h @@ -1,5 +1,5 @@ -#pragma once - +#pragma once + ////////////////////////////////////////////////////////////// /// \file /// \brief Definitions for asynchonous connection queue diff --git a/library/cpp/messagebus/oldmodule/startsession.h b/library/cpp/messagebus/oldmodule/startsession.h index aee8ee6167..5e26e7e1e5 100644 --- a/library/cpp/messagebus/oldmodule/startsession.h +++ b/library/cpp/messagebus/oldmodule/startsession.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <library/cpp/messagebus/ybus.h> diff --git a/library/cpp/messagebus/protobuf/ybusbuf.h b/library/cpp/messagebus/protobuf/ybusbuf.h index 0fa4f7797c..57b4267ea5 100644 --- a/library/cpp/messagebus/protobuf/ybusbuf.h +++ b/library/cpp/messagebus/protobuf/ybusbuf.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <library/cpp/messagebus/ybus.h> diff --git a/library/cpp/messagebus/rain_check/core/coro_stack.h b/library/cpp/messagebus/rain_check/core/coro_stack.h index da77f85ace..2f3520e6e4 100644 --- a/library/cpp/messagebus/rain_check/core/coro_stack.h +++ b/library/cpp/messagebus/rain_check/core/coro_stack.h @@ -1,7 +1,7 @@ #pragma once -#include <util/generic/array_ref.h> -#include <util/generic/ptr.h> +#include <util/generic/array_ref.h> +#include <util/generic/ptr.h> #include <util/system/valgrind.h> namespace NRainCheck { diff --git a/library/cpp/messagebus/test/helper/example.h b/library/cpp/messagebus/test/helper/example.h index 5c6e5773ef..26b7475308 100644 --- a/library/cpp/messagebus/test/helper/example.h +++ b/library/cpp/messagebus/test/helper/example.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <library/cpp/testing/unittest/registar.h> diff --git a/library/cpp/messagebus/test/perftest/perftest.cpp b/library/cpp/messagebus/test/perftest/perftest.cpp index e31e8e3d82..8489319278 100644 --- a/library/cpp/messagebus/test/perftest/perftest.cpp +++ b/library/cpp/messagebus/test/perftest/perftest.cpp @@ -20,7 +20,7 @@ #include <util/generic/string.h> #include <util/generic/vector.h> #include <util/generic/yexception.h> -#include <util/random/random.h> +#include <util/random/random.h> #include <util/stream/file.h> #include <util/stream/output.h> #include <util/stream/str.h> diff --git a/library/cpp/messagebus/test/ut/moduletest.h b/library/cpp/messagebus/test/ut/moduletest.h index 2ed5df7315..d5da72c0cb 100644 --- a/library/cpp/messagebus/test/ut/moduletest.h +++ b/library/cpp/messagebus/test/ut/moduletest.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once /////////////////////////////////////////////////////////////////// /// \file diff --git a/library/cpp/mime/types/mime.h b/library/cpp/mime/types/mime.h index 6469250797..05da389ea9 100644 --- a/library/cpp/mime/types/mime.h +++ b/library/cpp/mime/types/mime.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <util/system/defaults.h> #include <util/generic/strbuf.h> diff --git a/library/cpp/monlib/counters/counters.h b/library/cpp/monlib/counters/counters.h index 0ab4195081..038b55f0c8 100644 --- a/library/cpp/monlib/counters/counters.h +++ b/library/cpp/monlib/counters/counters.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <util/datetime/base.h> #include <util/generic/algorithm.h> diff --git a/library/cpp/monlib/service/service.cpp b/library/cpp/monlib/service/service.cpp index 50445f2e54..929efbf816 100644 --- a/library/cpp/monlib/service/service.cpp +++ b/library/cpp/monlib/service/service.cpp @@ -6,7 +6,7 @@ #include <library/cpp/http/fetch/httpfsm.h> #include <library/cpp/uri/http_url.h> -#include <util/generic/buffer.h> +#include <util/generic/buffer.h> #include <util/stream/str.h> #include <util/stream/buffer.h> #include <util/stream/zerocopy.h> diff --git a/library/cpp/monlib/service/service.h b/library/cpp/monlib/service/service.h index 42aed0d2fa..2f66dddaf8 100644 --- a/library/cpp/monlib/service/service.h +++ b/library/cpp/monlib/service/service.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <library/cpp/coroutine/engine/impl.h> #include <library/cpp/coroutine/listener/listen.h> diff --git a/library/cpp/object_factory/object_factory.cpp b/library/cpp/object_factory/object_factory.cpp index af986a94c4..3ef5bd9d18 100644 --- a/library/cpp/object_factory/object_factory.cpp +++ b/library/cpp/object_factory/object_factory.cpp @@ -1 +1 @@ -#include "object_factory.h" +#include "object_factory.h" diff --git a/library/cpp/object_factory/object_factory.h b/library/cpp/object_factory/object_factory.h index 9b99a92529..96cc11bcfd 100644 --- a/library/cpp/object_factory/object_factory.h +++ b/library/cpp/object_factory/object_factory.h @@ -7,7 +7,7 @@ #include <util/generic/singleton.h> #include <util/generic/yexception.h> -namespace NObjectFactory { +namespace NObjectFactory { template <class TProduct, class... TArgs> class IFactoryObjectCreator { public: @@ -15,7 +15,7 @@ namespace NObjectFactory { virtual ~IFactoryObjectCreator() { } }; - + template <class TProduct> class IFactoryObjectCreator<TProduct, void> { public: @@ -239,5 +239,5 @@ namespace NObjectFactory { } }; }; - -} + +} diff --git a/library/cpp/object_factory/ya.make b/library/cpp/object_factory/ya.make index 4ac809462e..bb93f75c23 100644 --- a/library/cpp/object_factory/ya.make +++ b/library/cpp/object_factory/ya.make @@ -1,9 +1,9 @@ -LIBRARY() - -OWNER(ivanmorozov) - -SRCS( - object_factory.cpp -) - -END() +LIBRARY() + +OWNER(ivanmorozov) + +SRCS( + object_factory.cpp +) + +END() diff --git a/library/cpp/on_disk/chunks/chunked_helpers.cpp b/library/cpp/on_disk/chunks/chunked_helpers.cpp index 36ad4237d6..b7adba2753 100644 --- a/library/cpp/on_disk/chunks/chunked_helpers.cpp +++ b/library/cpp/on_disk/chunks/chunked_helpers.cpp @@ -5,7 +5,7 @@ TBlob GetBlock(const TBlob& blob, size_t index) { TChunkedDataReader reader(blob); if (index >= reader.GetBlocksCount()) - ythrow yexception() << "index " << index << " is >= than block count " << reader.GetBlocksCount(); + ythrow yexception() << "index " << index << " is >= than block count " << reader.GetBlocksCount(); size_t begin = (const char*)reader.GetBlock(index) - (const char*)blob.Data(); return blob.SubBlob(begin, begin + reader.GetBlockLen(index)); } diff --git a/library/cpp/on_disk/chunks/chunked_helpers.h b/library/cpp/on_disk/chunks/chunked_helpers.h index ab2da0aacd..5fa96afdca 100644 --- a/library/cpp/on_disk/chunks/chunked_helpers.h +++ b/library/cpp/on_disk/chunks/chunked_helpers.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <util/generic/vector.h> #include <util/generic/buffer.h> diff --git a/library/cpp/on_disk/chunks/chunks_ut.cpp b/library/cpp/on_disk/chunks/chunks_ut.cpp index 1244ded8e1..f727647f7f 100644 --- a/library/cpp/on_disk/chunks/chunks_ut.cpp +++ b/library/cpp/on_disk/chunks/chunks_ut.cpp @@ -53,7 +53,7 @@ public: { TBlob temp = TBlob::FromStreamSingleThreaded(stream); TPlainHash<ui64, ui16> reader(temp); - ui16 value = 0; + ui16 value = 0; UNIT_ASSERT(reader.Find(5, &value)); UNIT_ASSERT_EQUAL(7, value); UNIT_ASSERT(!reader.Find(6, &value)); @@ -202,7 +202,7 @@ public: void TestStrings() { const TString FILENAME = "chunked_helpers_test.bin"; TTempFileHandle file(FILENAME.c_str()); - + { TFixedBufferFileOutput fOut(FILENAME); TStringsVectorWriter stringsWriter; @@ -274,56 +274,56 @@ public: UNIT_TEST_SUITE_REGISTRATION(TChunkedHelpersTest); -class TChunkedDataTest: public TTestBase { -private: - UNIT_TEST_SUITE(TChunkedDataTest); +class TChunkedDataTest: public TTestBase { +private: + UNIT_TEST_SUITE(TChunkedDataTest); UNIT_TEST(Test) UNIT_TEST(TestEmpty) - UNIT_TEST_SUITE_END(); - - void Test() { - TBuffer buffer; - { - TBufferOutput out(buffer); - TChunkedDataWriter writer(out); - writer.NewBlock(); - writer << "test"; - writer.NewBlock(); - writer << 4; - writer.NewBlock(); - writer.NewBlock(); - writer << 1; - writer << 2; - writer.WriteFooter(); - } - { + UNIT_TEST_SUITE_END(); + + void Test() { + TBuffer buffer; + { + TBufferOutput out(buffer); + TChunkedDataWriter writer(out); + writer.NewBlock(); + writer << "test"; + writer.NewBlock(); + writer << 4; + writer.NewBlock(); + writer.NewBlock(); + writer << 1; + writer << 2; + writer.WriteFooter(); + } + { TBlob blob = TBlob::FromBufferSingleThreaded(buffer); - TChunkedDataReader data(blob); - // printf("%d\n", (int)data.GetBlockLen(3)); - UNIT_ASSERT_EQUAL(4, data.GetBlockLen(0)); - UNIT_ASSERT_EQUAL(1, data.GetBlockLen(1)); - UNIT_ASSERT_EQUAL(0, data.GetBlockLen(2)); - UNIT_ASSERT_EQUAL(2, data.GetBlockLen(3)); - } - } - - void TestEmpty() { - TBuffer buffer; - { - TBufferOutput out(buffer); - TChunkedDataWriter writer(out); - writer.NewBlock(); - writer.NewBlock(); - writer.WriteFooter(); - } - { + TChunkedDataReader data(blob); + // printf("%d\n", (int)data.GetBlockLen(3)); + UNIT_ASSERT_EQUAL(4, data.GetBlockLen(0)); + UNIT_ASSERT_EQUAL(1, data.GetBlockLen(1)); + UNIT_ASSERT_EQUAL(0, data.GetBlockLen(2)); + UNIT_ASSERT_EQUAL(2, data.GetBlockLen(3)); + } + } + + void TestEmpty() { + TBuffer buffer; + { + TBufferOutput out(buffer); + TChunkedDataWriter writer(out); + writer.NewBlock(); + writer.NewBlock(); + writer.WriteFooter(); + } + { TBlob blob = TBlob::FromBufferSingleThreaded(buffer); - TChunkedDataReader data(blob); - // printf("%d\n", (int)data.GetBlockLen(1)); - UNIT_ASSERT_EQUAL(0, data.GetBlockLen(0)); - UNIT_ASSERT_EQUAL(0, data.GetBlockLen(1)); - } - } -}; - -UNIT_TEST_SUITE_REGISTRATION(TChunkedDataTest); + TChunkedDataReader data(blob); + // printf("%d\n", (int)data.GetBlockLen(1)); + UNIT_ASSERT_EQUAL(0, data.GetBlockLen(0)); + UNIT_ASSERT_EQUAL(0, data.GetBlockLen(1)); + } + } +}; + +UNIT_TEST_SUITE_REGISTRATION(TChunkedDataTest); diff --git a/library/cpp/packedtypes/fixed_point.h b/library/cpp/packedtypes/fixed_point.h index bb7ca0a643..b36b80876e 100644 --- a/library/cpp/packedtypes/fixed_point.h +++ b/library/cpp/packedtypes/fixed_point.h @@ -1,71 +1,71 @@ -#pragma once - -namespace NFixedPoint { +#pragma once + +namespace NFixedPoint { template <ui64 FracMult> class TFixedPoint { typedef TFixedPoint<FracMult> TSelf; ui64 Rep; - + public: TFixedPoint() : Rep() { } - + template <typename T> explicit TFixedPoint(T t) : Rep(ui64(t * FracMult)) { } - + explicit TFixedPoint(ui64 i, ui64 f) : Rep(i * FracMult + f % FracMult) { } - + template <typename T> TSelf& operator=(T t) { Rep = t * FracMult; return *this; } - + operator double() const { return Int() + double(Frac()) / FracMult; } - + operator ui64() const { return Int(); } - + template <typename T> TSelf& operator/=(T t) { Rep = ui64(Rep / t); return *this; } - + template <typename T> TSelf operator/(T t) const { TSelf r = *this; return r /= t; } - + ui64 Frac() const { return Rep % FracMult; } - + ui64 Int() const { return Rep / FracMult; } - + static ui64 Mult() { return FracMult; } - + static TSelf Make(ui64 rep) { TFixedPoint<FracMult> fp; fp.Rep = rep; return fp; } }; - -} + +} diff --git a/library/cpp/packedtypes/longs.h b/library/cpp/packedtypes/longs.h index 221d2561d0..084098d705 100644 --- a/library/cpp/packedtypes/longs.h +++ b/library/cpp/packedtypes/longs.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <util/system/defaults.h> // _BIDSCLASS _EXPCLASS #include <util/system/yassert.h> diff --git a/library/cpp/packedtypes/longs_ut.cpp b/library/cpp/packedtypes/longs_ut.cpp index 88e19a960d..8b06c934d2 100644 --- a/library/cpp/packedtypes/longs_ut.cpp +++ b/library/cpp/packedtypes/longs_ut.cpp @@ -55,9 +55,9 @@ Y_UNIT_TEST_SUITE(TLongsTest) { template <typename TSignedInt> void TestCornerCasesImpl(int maxPow) { for (int i = 0; i <= maxPow; ++i) { - TestOneValue<TSignedInt>((TSignedInt)(1ull << i)); - TestOneValue<TSignedInt>((TSignedInt)((1ull << i) - 1)); - TestOneValue<TSignedInt>((TSignedInt)((1ull << i) + 1)); + TestOneValue<TSignedInt>((TSignedInt)(1ull << i)); + TestOneValue<TSignedInt>((TSignedInt)((1ull << i) - 1)); + TestOneValue<TSignedInt>((TSignedInt)((1ull << i) + 1)); } } diff --git a/library/cpp/packedtypes/packed.h b/library/cpp/packedtypes/packed.h index 9abc1c2adb..88cff26ae2 100644 --- a/library/cpp/packedtypes/packed.h +++ b/library/cpp/packedtypes/packed.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <library/cpp/streams/zc_memory_input/zc_memory_input.h> diff --git a/library/cpp/packedtypes/packedfloat.h b/library/cpp/packedtypes/packedfloat.h index 9cb17a8dc1..f178912ed3 100644 --- a/library/cpp/packedtypes/packedfloat.h +++ b/library/cpp/packedtypes/packedfloat.h @@ -1,65 +1,65 @@ -#pragma once - +#pragma once + #include <util/generic/cast.h> #include <util/generic/ylimits.h> #include <util/system/hi_lo.h> -#include <cmath> -#include <cfloat> +#include <cmath> +#include <cfloat> #include <limits> -#include <algorithm> +#include <algorithm> #include <cassert> - -namespace NPackedFloat { + +namespace NPackedFloat { /* - Exponent Mantissa zero Mantissa non-zero Equation - 0x00 zero denormal (-1)^sign * 2^-126 * 0.mantissa - 0x01–0xfe normalized value (-1)^sign * 2^(exponent - 127) * 1.mantissa - 0xff infinity NaN - * */ - + Exponent Mantissa zero Mantissa non-zero Equation + 0x00 zero denormal (-1)^sign * 2^-126 * 0.mantissa + 0x01–0xfe normalized value (-1)^sign * 2^(exponent - 127) * 1.mantissa + 0xff infinity NaN + * */ + //fast 16 bit floats by melkov template <ui8 SIGNED> struct float16 { private: typedef float16<SIGNED> self; - + public: ui16 val; - + explicit float16(ui16 v = 0) : val(v) { } - + self& operator=(float t) { assert(SIGNED == 1 || SIGNED == 0 && t >= 0.); val = BitCast<ui32>(t) >> (15 + SIGNED); return *this; } - + operator float() const { return BitCast<float>((ui32)val << (15 + SIGNED)); } - + static self New(float v) { self f; return f = v; } - + static self denorm_min() { return self(0x0001); } - + static self min() { return self(SIGNED ? 0x0080 : 0x0100); } - + static self max() { return self(SIGNED ? 0x7f7f : 0xfeff); } - }; - + }; + //fast 8 bit floats template <ui8 SIGNED, ui8 DENORM = 0> struct float8 { @@ -70,23 +70,23 @@ namespace NPackedFloat { FMaxExp = SIGNED ? 0x83 : 0x87, MaxExp = SIGNED ? 0x70 : 0xf0, }; - + public: ui8 val; - + explicit float8(ui8 v = 0) : val(v) { } - + self& operator=(float t) { assert(SIGNED == 1 || SIGNED == 0 && t >= 0.); ui16 hi16 = Hi16(t); - + ui8 sign = SIGNED ? Hi8(hi16) & 0x80 : 0; - + hi16 <<= 1; - + ui8 fexp = Hi8(hi16); ui8 exp; ui8 frac = (Lo8(hi16) & 0xf0) >> 4; @@ -103,15 +103,15 @@ namespace NPackedFloat { val = sign | exp | frac; return *this; - } - + } + operator float() const { ui32 v = 0; - + v |= SIGNED ? (val & 0x80) << 24 : 0; ui8 frac = val & 0x0f; ui8 exp = val & MaxExp; - + if (exp) { v |= ((exp >> 4) + FMinExp) << 23 | frac << 19; } else if (DENORM && val & 0x0f) { @@ -119,33 +119,33 @@ namespace NPackedFloat { frac <<= 1; ++exp; } - + v |= (FMinExp - exp + 1) << 23 | (frac & 0x0f) << 19; } else v |= 0; - + return BitCast<float>(v); } - + static self New(float v) { self f; return f = v; } - + static self denorm_min() { return self(0x01); } - + static self min() { return self(0x10); } - + static self max() { return self(SIGNED ? 0x7f : 0xff); } }; -} - +} + using f64 = double; using f32 = float; static_assert(sizeof(f32) == 4, "expect sizeof(f32) == 4"); @@ -156,7 +156,7 @@ using f8 = NPackedFloat::float8<1>; using uf8 = NPackedFloat::float8<0>; using f8d = NPackedFloat::float8<1, 1>; using uf8d = NPackedFloat::float8<0, 1>; - + // [0,1) value in 1/255s. using frac8 = ui8; diff --git a/library/cpp/packedtypes/packedfloat_ut.cpp b/library/cpp/packedtypes/packedfloat_ut.cpp index fb33883c1c..f61e49014c 100644 --- a/library/cpp/packedtypes/packedfloat_ut.cpp +++ b/library/cpp/packedtypes/packedfloat_ut.cpp @@ -1,10 +1,10 @@ #include "packedfloat.h" #include <library/cpp/testing/unittest/registar.h> - + #include <util/generic/ylimits.h> -class TPackedFloatTest: public TTestBase { +class TPackedFloatTest: public TTestBase { UNIT_TEST_SUITE(TPackedFloatTest); UNIT_TEST(F16Test); UNIT_TEST(Uf16Test); @@ -14,117 +14,117 @@ class TPackedFloatTest: public TTestBase { UNIT_TEST(Uf8Test); UNIT_TEST_SUITE_END(); -private: +private: template <typename F> - void TestF(float f, float rf) { - UNIT_ASSERT(F::New(f) == rf); - } - - void TestF16(float f, float rf) { - TestF<f16>(f, rf); - } - - void F16Test() { - TestF16(f16(), 0.); - TestF16(0, 0); - TestF16(1.5, 1.5); - TestF16(-1.5, -1.5); - TestF16(f16::max(), f16::max()); - TestF16(f16::min(), f16::min()); - TestF16(2.0f * f16::max(), std::numeric_limits<float>::infinity()); - TestF16(0.5 * f16::min(), 0.5 * f16::min()); - TestF16(0.5 * f16::denorm_min(), 0); + void TestF(float f, float rf) { + UNIT_ASSERT(F::New(f) == rf); + } + + void TestF16(float f, float rf) { + TestF<f16>(f, rf); + } + + void F16Test() { + TestF16(f16(), 0.); + TestF16(0, 0); + TestF16(1.5, 1.5); + TestF16(-1.5, -1.5); + TestF16(f16::max(), f16::max()); + TestF16(f16::min(), f16::min()); + TestF16(2.0f * f16::max(), std::numeric_limits<float>::infinity()); + TestF16(0.5 * f16::min(), 0.5 * f16::min()); + TestF16(0.5 * f16::denorm_min(), 0); TestF16(-0.5 * f16::denorm_min(), 0); - TestF16(FLT_MIN, FLT_MIN); - TestF16(FLT_MAX, f16::max()); - TestF16(f16::min(), FLT_MIN); - TestF16(f16::denorm_min(), (FLT_MIN / (1 << 23)) * (1 << 16)); - } - - void TestUf16(float f, float rf) { - TestF<uf16>(f, rf); - } - - void Uf16Test() { - UNIT_ASSERT(uf16() == 0.); - - TestUf16(0, 0); - TestUf16(1.5, 1.5); - TestUf16(uf16::max(), uf16::max()); - TestUf16(uf16::min(), uf16::min()); - TestUf16(2.0f * uf16::max(), std::numeric_limits<float>::infinity()); - TestUf16(0.5 * uf16::min(), 0.5 * uf16::min()); - TestUf16(0.5 * uf16::denorm_min(), 0); - TestUf16(FLT_MIN, FLT_MIN); - TestUf16(FLT_MAX, uf16::max()); - TestUf16(uf16::min(), FLT_MIN); - TestUf16(uf16::denorm_min(), (FLT_MIN / (1 << 23)) * (1 << 15)); - } - - void TestF8d(float f, float rf) { - TestF<f8d>(f, rf); - } - - void F8dTest() { - UNIT_ASSERT(f8d() == 0.); - - TestF8d(0, 0); - TestF8d(1.5, 1.5); - TestF8d(-1.5, -1.5); - TestF8d(f8d::max(), f8d::max()); - TestF8d(f8d::min(), f8d::min()); - TestF8d(f8d::denorm_min(), f8d::denorm_min()); - TestF8d(2.0 * f8d::max(), f8d::max()); - TestF8d(0.5 * f8d::min(), 0.5 * f8d::min()); - TestF8d(0.5 * f8d::denorm_min(), 0); - } - - void TestF8(float f, float rf) { - TestF<f8>(f, rf); - } - - void F8Test() { - UNIT_ASSERT(f8() == 0.); - - TestF8(0, 0); - TestF8(1.5, 1.5); - TestF8(-1.5, -1.5); - TestF8(f8::max(), f8::max()); - TestF8(f8::min(), f8::min()); - TestF8(2.0 * f8::max(), f8::max()); - TestF8(0.5 * f8::min(), 0); - } - - void TestUf8d(float f, float rf) { - TestF<uf8d>(f, rf); - } - - void Uf8dTest() { - UNIT_ASSERT(uf8d() == 0.); - - TestUf8d(0, 0); - TestUf8d(1.5, 1.5); - TestUf8d(uf8d::max(), uf8d::max()); - TestUf8d(uf8d::min(), uf8d::min()); - TestUf8d(uf8d::denorm_min(), uf8d::denorm_min()); - TestUf8d(2.0 * uf8d::max(), uf8d::max()); - TestUf8d(0.5 * uf8d::min(), 0.5 * uf8d::min()); - TestUf8d(0.5 * uf8d::denorm_min(), 0); - } - - void TestUf8(float f, float rf) { - TestF<uf8>(f, rf); - } - - void Uf8Test() { - UNIT_ASSERT(uf8() == 0.); - - TestUf8(0, 0); - TestUf8(1.5, 1.5); - TestUf8(uf8::max(), uf8::max()); - TestUf8(uf8::min(), uf8::min()); - TestUf8(2.0 * uf8::max(), uf8::max()); - TestUf8(0.5 * uf8::min(), 0); + TestF16(FLT_MIN, FLT_MIN); + TestF16(FLT_MAX, f16::max()); + TestF16(f16::min(), FLT_MIN); + TestF16(f16::denorm_min(), (FLT_MIN / (1 << 23)) * (1 << 16)); + } + + void TestUf16(float f, float rf) { + TestF<uf16>(f, rf); + } + + void Uf16Test() { + UNIT_ASSERT(uf16() == 0.); + + TestUf16(0, 0); + TestUf16(1.5, 1.5); + TestUf16(uf16::max(), uf16::max()); + TestUf16(uf16::min(), uf16::min()); + TestUf16(2.0f * uf16::max(), std::numeric_limits<float>::infinity()); + TestUf16(0.5 * uf16::min(), 0.5 * uf16::min()); + TestUf16(0.5 * uf16::denorm_min(), 0); + TestUf16(FLT_MIN, FLT_MIN); + TestUf16(FLT_MAX, uf16::max()); + TestUf16(uf16::min(), FLT_MIN); + TestUf16(uf16::denorm_min(), (FLT_MIN / (1 << 23)) * (1 << 15)); + } + + void TestF8d(float f, float rf) { + TestF<f8d>(f, rf); + } + + void F8dTest() { + UNIT_ASSERT(f8d() == 0.); + + TestF8d(0, 0); + TestF8d(1.5, 1.5); + TestF8d(-1.5, -1.5); + TestF8d(f8d::max(), f8d::max()); + TestF8d(f8d::min(), f8d::min()); + TestF8d(f8d::denorm_min(), f8d::denorm_min()); + TestF8d(2.0 * f8d::max(), f8d::max()); + TestF8d(0.5 * f8d::min(), 0.5 * f8d::min()); + TestF8d(0.5 * f8d::denorm_min(), 0); + } + + void TestF8(float f, float rf) { + TestF<f8>(f, rf); + } + + void F8Test() { + UNIT_ASSERT(f8() == 0.); + + TestF8(0, 0); + TestF8(1.5, 1.5); + TestF8(-1.5, -1.5); + TestF8(f8::max(), f8::max()); + TestF8(f8::min(), f8::min()); + TestF8(2.0 * f8::max(), f8::max()); + TestF8(0.5 * f8::min(), 0); + } + + void TestUf8d(float f, float rf) { + TestF<uf8d>(f, rf); + } + + void Uf8dTest() { + UNIT_ASSERT(uf8d() == 0.); + + TestUf8d(0, 0); + TestUf8d(1.5, 1.5); + TestUf8d(uf8d::max(), uf8d::max()); + TestUf8d(uf8d::min(), uf8d::min()); + TestUf8d(uf8d::denorm_min(), uf8d::denorm_min()); + TestUf8d(2.0 * uf8d::max(), uf8d::max()); + TestUf8d(0.5 * uf8d::min(), 0.5 * uf8d::min()); + TestUf8d(0.5 * uf8d::denorm_min(), 0); + } + + void TestUf8(float f, float rf) { + TestF<uf8>(f, rf); + } + + void Uf8Test() { + UNIT_ASSERT(uf8() == 0.); + + TestUf8(0, 0); + TestUf8(1.5, 1.5); + TestUf8(uf8::max(), uf8::max()); + TestUf8(uf8::min(), uf8::min()); + TestUf8(2.0 * uf8::max(), uf8::max()); + TestUf8(0.5 * uf8::min(), 0); } }; UNIT_TEST_SUITE_REGISTRATION(TPackedFloatTest); diff --git a/library/cpp/packers/packers.cpp b/library/cpp/packers/packers.cpp index 5b2e938573..54615a9e7f 100644 --- a/library/cpp/packers/packers.cpp +++ b/library/cpp/packers/packers.cpp @@ -1,20 +1,20 @@ -#include "packers.h" -#include "region_packer.h" - -namespace NPackers { -#define _X_4(X) X, X, X, X -#define _X_8(X) _X_4(X), _X_4(X) -#define _X_16(X) _X_8(X), _X_8(X) -#define _X_32(X) _X_16(X), _X_16(X) -#define _X_64(X) _X_32(X), _X_32(X) -#define _X_128(X) _X_64(X), _X_64(X) - +#include "packers.h" +#include "region_packer.h" + +namespace NPackers { +#define _X_4(X) X, X, X, X +#define _X_8(X) _X_4(X), _X_4(X) +#define _X_16(X) _X_8(X), _X_8(X) +#define _X_32(X) _X_16(X), _X_16(X) +#define _X_64(X) _X_32(X), _X_32(X) +#define _X_128(X) _X_64(X), _X_64(X) + const ui8 SkipTable[256] = {_X_128(1), _X_64(2), _X_32(3), _X_16(4), _X_8(5), _X_4(6), 7, 7, 8, 9}; - -#undef _X_4 -#undef _X_8 -#undef _X_16 -#undef _X_32 -#undef _X_64 -#undef _X_128 -} + +#undef _X_4 +#undef _X_8 +#undef _X_16 +#undef _X_32 +#undef _X_64 +#undef _X_128 +} diff --git a/library/cpp/packers/packers.h b/library/cpp/packers/packers.h index fe3fd633c4..1bde1b59aa 100644 --- a/library/cpp/packers/packers.h +++ b/library/cpp/packers/packers.h @@ -1,7 +1,7 @@ -#pragma once +#pragma once #include <util/generic/string.h> -#include <util/generic/strbuf.h> +#include <util/generic/strbuf.h> #include <util/generic/set.h> #include <util/generic/list.h> #include <util/generic/vector.h> @@ -51,7 +51,7 @@ public: // Implementation -namespace NPackers { +namespace NPackers { template <class T> inline ui64 ConvertIntegral(const T& data); @@ -131,19 +131,19 @@ namespace NPackers { buffer[0] = (char)(lenmask | value); } - extern const ui8 SkipTable[]; + extern const ui8 SkipTable[]; - template <> - inline void TIntegralPacker<ui64>::UnpackLeaf(const char* p, ui64& result) const { - unsigned char ch = *(p++); - size_t taillen = SkipTable[ch] - 1; + template <> + inline void TIntegralPacker<ui64>::UnpackLeaf(const char* p, ui64& result) const { + unsigned char ch = *(p++); + size_t taillen = SkipTable[ch] - 1; + + result = (ch & (0x7F >> taillen)); - result = (ch & (0x7F >> taillen)); + while (taillen--) + result = ((result << 8) | (*(p++) & 0xFF)); + } - while (taillen--) - result = ((result << 8) | (*(p++) & 0xFF)); - } - template <> inline size_t TIntegralPacker<ui64>::SkipLeaf(const char* p) const { return SkipTable[(ui8)*p]; @@ -197,58 +197,58 @@ namespace NPackers { } //------------------------------------------- - // TFPPacker --- for float/double - namespace NImpl { - template <class TFloat, class TUInt> - class TFPPackerBase { - protected: - typedef TIntegralPacker<TUInt> TPacker; - - union THelper { - TFloat F; - TUInt U; - }; - - TFloat FromUInt(TUInt u) const { - THelper h; + // TFPPacker --- for float/double + namespace NImpl { + template <class TFloat, class TUInt> + class TFPPackerBase { + protected: + typedef TIntegralPacker<TUInt> TPacker; + + union THelper { + TFloat F; + TUInt U; + }; + + TFloat FromUInt(TUInt u) const { + THelper h; h.U = ReverseBytes(u); - return h.F; - } - - TUInt ToUInt(TFloat f) const { - THelper h; - h.F = f; + return h.F; + } + + TUInt ToUInt(TFloat f) const { + THelper h; + h.F = f; return ReverseBytes(h.U); - } - - public: - void UnpackLeaf(const char* c, TFloat& t) const { - TUInt u = 0; - TPacker().UnpackLeaf(c, u); - t = FromUInt(u); - } - - void PackLeaf(char* c, const TFloat& t, size_t sz) const { - TPacker().PackLeaf(c, ToUInt(t), sz); - } - - size_t MeasureLeaf(const TFloat& t) const { - return TPacker().MeasureLeaf(ToUInt(t)); - } - - size_t SkipLeaf(const char* c) const { - return TPacker().SkipLeaf(c); - } - }; - } - + } + + public: + void UnpackLeaf(const char* c, TFloat& t) const { + TUInt u = 0; + TPacker().UnpackLeaf(c, u); + t = FromUInt(u); + } + + void PackLeaf(char* c, const TFloat& t, size_t sz) const { + TPacker().PackLeaf(c, ToUInt(t), sz); + } + + size_t MeasureLeaf(const TFloat& t) const { + return TPacker().MeasureLeaf(ToUInt(t)); + } + + size_t SkipLeaf(const char* c) const { + return TPacker().SkipLeaf(c); + } + }; + } + class TFloatPacker: public NImpl::TFPPackerBase<float, ui32> { - }; - + }; + class TDoublePacker: public NImpl::TFPPackerBase<double, ui64> { - }; - - //------------------------------------------- + }; + + //------------------------------------------- // TStringPacker --- for TString/TUtf16String and TStringBuf. template <class TStringType> @@ -480,7 +480,7 @@ namespace NPackers { } template <class T1, class T2, class TPacker1, class TPacker2> - inline size_t TPairPacker<T1, T2, TPacker1, TPacker2>::SkipLeaf(const char* buffer) const { + inline size_t TPairPacker<T1, T2, TPacker1, TPacker2>::SkipLeaf(const char* buffer) const { size_t size1 = TPacker1().SkipLeaf(buffer); size_t size2 = TPacker2().SkipLeaf(buffer + size1); return size1 + size2; @@ -539,10 +539,10 @@ namespace NPackers { //------------------------------------ // TPacker --- the generic packer. - + template <class T, bool IsIntegral> class TPackerImpl; - + template <class T> class TPackerImpl<T, true>: public TIntegralPacker<T> { }; diff --git a/library/cpp/packers/region_packer.h b/library/cpp/packers/region_packer.h index 35ec9f3359..2c661cb5bc 100644 --- a/library/cpp/packers/region_packer.h +++ b/library/cpp/packers/region_packer.h @@ -1,7 +1,7 @@ #pragma once -#include "packers.h" - +#include "packers.h" + #include <util/generic/array_ref.h> // Stores an array of PODs in the trie (copying them with memcpy). @@ -14,8 +14,8 @@ public: void UnpackLeaf(const char* p, TRecords& result) const { size_t len; - NPackers::TIntegralPacker<size_t>().UnpackLeaf(p, len); - size_t start = NPackers::TIntegralPacker<size_t>().SkipLeaf(p); + NPackers::TIntegralPacker<size_t>().UnpackLeaf(p, len); + size_t start = NPackers::TIntegralPacker<size_t>().SkipLeaf(p); result = TRecords((TRecord*)(p + start), len); } @@ -23,19 +23,19 @@ public: size_t len = data.size(); size_t lenChar = len * sizeof(TRecord); size_t start = computedSize - lenChar; - NPackers::TIntegralPacker<size_t>().PackLeaf(buf, len, NPackers::TIntegralPacker<size_t>().MeasureLeaf(len)); + NPackers::TIntegralPacker<size_t>().PackLeaf(buf, len, NPackers::TIntegralPacker<size_t>().MeasureLeaf(len)); memcpy(buf + start, data.data(), lenChar); } size_t MeasureLeaf(const TRecords& data) const { size_t len = data.size(); - return NPackers::TIntegralPacker<size_t>().MeasureLeaf(len) + len * sizeof(TRecord); + return NPackers::TIntegralPacker<size_t>().MeasureLeaf(len) + len * sizeof(TRecord); } size_t SkipLeaf(const char* p) const { - size_t result = NPackers::TIntegralPacker<size_t>().SkipLeaf(p); + size_t result = NPackers::TIntegralPacker<size_t>().SkipLeaf(p); size_t len; - NPackers::TIntegralPacker<size_t>().UnpackLeaf(p, len); + NPackers::TIntegralPacker<size_t>().UnpackLeaf(p, len); result += len * sizeof(TRecord); return result; } diff --git a/library/cpp/packers/ut/packers_ut.cpp b/library/cpp/packers/ut/packers_ut.cpp index 331ff7f995..18ce2150d1 100644 --- a/library/cpp/packers/ut/packers_ut.cpp +++ b/library/cpp/packers/ut/packers_ut.cpp @@ -17,14 +17,14 @@ #include <util/string/hex.h> -#include "packers.h" +#include "packers.h" #include <array> #include <iterator> class TPackersTest: public TTestBase { private: - UNIT_TEST_SUITE(TPackersTest); + UNIT_TEST_SUITE(TPackersTest); UNIT_TEST(TestPackers); UNIT_TEST_SUITE_END(); @@ -38,10 +38,10 @@ public: void TestPackers(); }; -UNIT_TEST_SUITE_REGISTRATION(TPackersTest); +UNIT_TEST_SUITE_REGISTRATION(TPackersTest); template <class TData, class TPacker> -void TPackersTest::TestPacker(const TData& data) { +void TPackersTest::TestPacker(const TData& data) { size_t len = TPacker().MeasureLeaf(data); size_t bufLen = len * 3; @@ -58,13 +58,13 @@ void TPackersTest::TestPacker(const TData& data) { } template <class TData, class TPacker> -void TPackersTest::TestPacker(const TData* test, size_t size) { +void TPackersTest::TestPacker(const TData* test, size_t size) { for (size_t i = 0; i < size; ++i) { TestPacker<TData, TPacker>(test[i]); } } -void TPackersTest::TestPackers() { +void TPackersTest::TestPackers() { { const TString test[] = {"", "a", "b", "c", "d", @@ -91,20 +91,20 @@ void TPackersTest::TestPackers() { TestPacker<int, NPackers::TPacker<int>>(test, Y_ARRAY_SIZE(test)); } - { - const float test[] = { - 2.f, 3.f, 4.f, 0.f, -0.f, 1.f, -1.f, 1.1f, -1.1f, + { + const float test[] = { + 2.f, 3.f, 4.f, 0.f, -0.f, 1.f, -1.f, 1.1f, -1.1f, std::numeric_limits<float>::min(), -std::numeric_limits<float>::min(), std::numeric_limits<float>::max(), -std::numeric_limits<float>::max()}; - - TestPacker<float, NPackers::TFloatPacker>(test, Y_ARRAY_SIZE(test)); - } - { - const double test[] = { - 0., -0., 1., -1., 1.1, -1.1, + + TestPacker<float, NPackers::TFloatPacker>(test, Y_ARRAY_SIZE(test)); + } + { + const double test[] = { + 0., -0., 1., -1., 1.1, -1.1, std::numeric_limits<double>::min(), -std::numeric_limits<double>::min(), std::numeric_limits<double>::max(), -std::numeric_limits<double>::max()}; - - TestPacker<double, NPackers::TDoublePacker>(test, Y_ARRAY_SIZE(test)); - } + + TestPacker<double, NPackers::TDoublePacker>(test, Y_ARRAY_SIZE(test)); + } } diff --git a/library/cpp/packers/ut/ya.make b/library/cpp/packers/ut/ya.make index 7767faff47..1c024ffd94 100644 --- a/library/cpp/packers/ut/ya.make +++ b/library/cpp/packers/ut/ya.make @@ -1,12 +1,12 @@ UNITTEST_FOR(library/cpp/packers) - + OWNER(velavokr) - -SRCS( - packers_ut.cpp + +SRCS( + packers_ut.cpp proto_packer_ut.cpp - region_packer_ut.cpp + region_packer_ut.cpp test.proto -) - -END() +) + +END() diff --git a/library/cpp/packers/ya.make b/library/cpp/packers/ya.make index e2f08f922f..e1ec4972ed 100644 --- a/library/cpp/packers/ya.make +++ b/library/cpp/packers/ya.make @@ -1,11 +1,11 @@ -LIBRARY() - +LIBRARY() + OWNER(velavokr) - -SRCS( - packers.cpp + +SRCS( + packers.cpp proto_packer.cpp region_packer.cpp -) - -END() +) + +END() diff --git a/library/cpp/protobuf/json/json2proto.cpp b/library/cpp/protobuf/json/json2proto.cpp index 70605275e6..640c10f5a5 100644 --- a/library/cpp/protobuf/json/json2proto.cpp +++ b/library/cpp/protobuf/json/json2proto.cpp @@ -6,7 +6,7 @@ #include <google/protobuf/message.h> #include <google/protobuf/descriptor.h> -#include <util/generic/hash.h> +#include <util/generic/hash.h> #include <util/generic/maybe.h> #include <util/string/ascii.h> #include <util/string/cast.h> @@ -378,10 +378,10 @@ namespace NProtobufJson { } Y_ENSURE(json.IsMap(), "expected json map"); - + const google::protobuf::Descriptor* descriptor = proto.GetDescriptor(); Y_ASSERT(!!descriptor); - + for (int f = 0, endF = descriptor->field_count(); f < endF; ++f) { const google::protobuf::FieldDescriptor* field = descriptor->field(f); Y_ASSERT(!!field); @@ -391,8 +391,8 @@ namespace NProtobufJson { } else { Json2SingleField(json, proto, *field, config); } - } - + } + if (!config.AllowUnknownFields) { THashMap<TString, bool> knownFields; for (int f = 0, endF = descriptor->field_count(); f < endF; ++f) { @@ -402,8 +402,8 @@ namespace NProtobufJson { for (const auto& f : json.GetMap()) { Y_ENSURE(knownFields.contains(f.first), "unknown field " << f.first); } - } - } + } + } void MergeJson2Proto(const TStringBuf& json, google::protobuf::Message& proto, const TJson2ProtoConfig& config) { NJson::TJsonReaderConfig jsonCfg; diff --git a/library/cpp/protobuf/json/ut/json2proto_ut.cpp b/library/cpp/protobuf/json/ut/json2proto_ut.cpp index 97333a8986..0dfe57bc7a 100644 --- a/library/cpp/protobuf/json/ut/json2proto_ut.cpp +++ b/library/cpp/protobuf/json/ut/json2proto_ut.cpp @@ -506,14 +506,14 @@ class TBytesTransform: public IStringTransform { public: int GetType() const override { return 0; - } + } void Transform(TString&) const override { } void TransformBytes(TString& str) const override { str = "transformed_bytes"; } }; - + Y_UNIT_TEST(TestInvalidJson) { NJson::TJsonValue val{"bad value"}; TFlatOptional proto; diff --git a/library/cpp/regex/pcre/regexp.h b/library/cpp/regex/pcre/regexp.h index 70ead8c494..bc610bd2f3 100644 --- a/library/cpp/regex/pcre/regexp.h +++ b/library/cpp/regex/pcre/regexp.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <sys/types.h> diff --git a/library/cpp/regex/pire/regexp.h b/library/cpp/regex/pire/regexp.h index 7e4e3bd9ca..94bba4064b 100644 --- a/library/cpp/regex/pire/regexp.h +++ b/library/cpp/regex/pire/regexp.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include "pire.h" @@ -35,16 +35,16 @@ namespace NRegExp { return *this; } - inline TOptions& SetAndNotSupport(bool andNotSupport) noexcept { - AndNotSupport = andNotSupport; - return *this; - } - - bool CaseInsensitive = false; - bool Surround = false; + inline TOptions& SetAndNotSupport(bool andNotSupport) noexcept { + AndNotSupport = andNotSupport; + return *this; + } + + bool CaseInsensitive = false; + bool Surround = false; TMaybe<size_t> CapturePos; - ECharset Charset = CODES_UNKNOWN; - bool AndNotSupport = false; + ECharset Charset = CODES_UNKNOWN; + bool AndNotSupport = false; }; static inline NPire::TFsm Parse(const TStringBuf& regexp, @@ -74,10 +74,10 @@ namespace NRegExp { lexer.AddFeature(NPire::NFeatures::Capture(*opts.CapturePos)); } - if (opts.AndNotSupport) { - lexer.AddFeature(NPire::NFeatures::AndNotSupport()); - } - + if (opts.AndNotSupport) { + lexer.AddFeature(NPire::NFeatures::AndNotSupport()); + } + switch (opts.Charset) { case CODES_UNKNOWN: break; diff --git a/library/cpp/regex/pire/ut/regexp_ut.cpp b/library/cpp/regex/pire/ut/regexp_ut.cpp index 96b92bbc38..e7206de9ad 100644 --- a/library/cpp/regex/pire/ut/regexp_ut.cpp +++ b/library/cpp/regex/pire/ut/regexp_ut.cpp @@ -52,37 +52,37 @@ Y_UNIT_TEST_SUITE(TRegExp) { UNIT_ASSERT(!TMatcher(TFsm("чзн", opts)).Match("чзх").Final()); } - Y_UNIT_TEST(AndNot) { - NRegExp::TFsmBase::TOptions opts; - opts.AndNotSupport = true; - { - NRegExp::TFsm fsm(".*&~([0-9]*)", opts); - UNIT_ASSERT(TMatcher(fsm).Match("a2").Final()); - UNIT_ASSERT(TMatcher(fsm).Match("ab").Final()); - UNIT_ASSERT(TMatcher(fsm).Match("1a").Final()); - UNIT_ASSERT(!TMatcher(fsm).Match("12").Final()); - } - { - NRegExp::TFsm fsm(".*&~(.*[0-9].*)", opts); - UNIT_ASSERT(TMatcher(fsm).Match("ab").Final()); - UNIT_ASSERT(!TMatcher(fsm).Match("a2").Final()); - UNIT_ASSERT(!TMatcher(fsm).Match("1a").Final()); - UNIT_ASSERT(!TMatcher(fsm).Match("12").Final()); - } - { - NRegExp::TFsm fsm( - "((([a-z0-9_\\-]+[.])*[a-z0-9_\\-]+)" - "&~(\\d+[.]\\d+[.]\\d+[.]\\d+))(:\\d+)?", - TFsm::TOptions().SetCaseInsensitive(true).SetAndNotSupport(true) - ); - UNIT_ASSERT(TMatcher(fsm).Match("yandex.ru").Final()); - UNIT_ASSERT(TMatcher(fsm).Match("yandex").Final()); - UNIT_ASSERT(TMatcher(fsm).Match("yandex:80").Final()); - UNIT_ASSERT(!TMatcher(fsm).Match("127.0.0.1").Final()); - UNIT_ASSERT(!TMatcher(fsm).Match("127.0.0.1:8080").Final()); - } - } - + Y_UNIT_TEST(AndNot) { + NRegExp::TFsmBase::TOptions opts; + opts.AndNotSupport = true; + { + NRegExp::TFsm fsm(".*&~([0-9]*)", opts); + UNIT_ASSERT(TMatcher(fsm).Match("a2").Final()); + UNIT_ASSERT(TMatcher(fsm).Match("ab").Final()); + UNIT_ASSERT(TMatcher(fsm).Match("1a").Final()); + UNIT_ASSERT(!TMatcher(fsm).Match("12").Final()); + } + { + NRegExp::TFsm fsm(".*&~(.*[0-9].*)", opts); + UNIT_ASSERT(TMatcher(fsm).Match("ab").Final()); + UNIT_ASSERT(!TMatcher(fsm).Match("a2").Final()); + UNIT_ASSERT(!TMatcher(fsm).Match("1a").Final()); + UNIT_ASSERT(!TMatcher(fsm).Match("12").Final()); + } + { + NRegExp::TFsm fsm( + "((([a-z0-9_\\-]+[.])*[a-z0-9_\\-]+)" + "&~(\\d+[.]\\d+[.]\\d+[.]\\d+))(:\\d+)?", + TFsm::TOptions().SetCaseInsensitive(true).SetAndNotSupport(true) + ); + UNIT_ASSERT(TMatcher(fsm).Match("yandex.ru").Final()); + UNIT_ASSERT(TMatcher(fsm).Match("yandex").Final()); + UNIT_ASSERT(TMatcher(fsm).Match("yandex:80").Final()); + UNIT_ASSERT(!TMatcher(fsm).Match("127.0.0.1").Final()); + UNIT_ASSERT(!TMatcher(fsm).Match("127.0.0.1:8080").Final()); + } + } + Y_UNIT_TEST(Glue) { TFsm glued = TFsm("qw", TFsm::TOptions().SetCaseInsensitive(true)) | diff --git a/library/cpp/scheme/README b/library/cpp/scheme/README index b85325adda..117c81979b 100644 --- a/library/cpp/scheme/README +++ b/library/cpp/scheme/README @@ -1 +1 @@ -see http://wiki.yandex-team.ru/JandeksPoisk/KachestvoPoiska/Rearrange/RealizacijaFormata
\ No newline at end of file +see http://wiki.yandex-team.ru/JandeksPoisk/KachestvoPoiska/Rearrange/RealizacijaFormata
\ No newline at end of file diff --git a/library/cpp/scheme/scheme.cpp b/library/cpp/scheme/scheme.cpp index fefca57c81..3efd116d4f 100644 --- a/library/cpp/scheme/scheme.cpp +++ b/library/cpp/scheme/scheme.cpp @@ -1,181 +1,181 @@ -#include "scheme.h" -#include "scimpl_private.h" - -#include <util/generic/algorithm.h> +#include "scheme.h" +#include "scimpl_private.h" + +#include <util/generic/algorithm.h> #include <util/string/cast.h> - -namespace NSc { - TStringBufs& TValue::DictKeys(TStringBufs& vs, bool sorted) const { - if (!IsDict()) { - return vs; - } - - const ::NSc::TDict& dict = GetDict(); - vs.reserve(vs.size() + dict.size()); - for (const auto& it : dict) - vs.push_back(it.first); - - if (sorted) { - Sort(vs.begin(), vs.end()); - } - - return vs; - } - - TStringBufs TValue::DictKeys(bool sorted) const { - TStringBufs bufs; - DictKeys(bufs, sorted); - return bufs; - } - - TValue& TValue::MergeUpdate(const TValue& delta) { - return DoMerge(delta, false); - } - - TValue& TValue::ReverseMerge(const TValue& delta) { - return DoMerge(delta, true); - } - - TValue& TValue::MergeUpdateJson(TStringBuf data) { - return MergeUpdate(FromJson(data)); - } - - TValue& TValue::ReverseMergeJson(TStringBuf data) { - return ReverseMerge(FromJson(data)); - } - - bool TValue::MergeUpdateJson(TValue& v, TStringBuf data) { - NSc::TValue m; - if (!FromJson(m, data)) { - return false; - } - - v.MergeUpdate(m); - return true; - } - - bool TValue::ReverseMergeJson(TValue& v, TStringBuf data) { - NSc::TValue m; - if (!FromJson(m, data)) { - return false; - } - - v.ReverseMerge(m); - return true; - } - - TValue TValue::Clone() const { + +namespace NSc { + TStringBufs& TValue::DictKeys(TStringBufs& vs, bool sorted) const { + if (!IsDict()) { + return vs; + } + + const ::NSc::TDict& dict = GetDict(); + vs.reserve(vs.size() + dict.size()); + for (const auto& it : dict) + vs.push_back(it.first); + + if (sorted) { + Sort(vs.begin(), vs.end()); + } + + return vs; + } + + TStringBufs TValue::DictKeys(bool sorted) const { + TStringBufs bufs; + DictKeys(bufs, sorted); + return bufs; + } + + TValue& TValue::MergeUpdate(const TValue& delta) { + return DoMerge(delta, false); + } + + TValue& TValue::ReverseMerge(const TValue& delta) { + return DoMerge(delta, true); + } + + TValue& TValue::MergeUpdateJson(TStringBuf data) { + return MergeUpdate(FromJson(data)); + } + + TValue& TValue::ReverseMergeJson(TStringBuf data) { + return ReverseMerge(FromJson(data)); + } + + bool TValue::MergeUpdateJson(TValue& v, TStringBuf data) { + NSc::TValue m; + if (!FromJson(m, data)) { + return false; + } + + v.MergeUpdate(m); + return true; + } + + bool TValue::ReverseMergeJson(TValue& v, TStringBuf data) { + NSc::TValue m; + if (!FromJson(m, data)) { + return false; + } + + v.ReverseMerge(m); + return true; + } + + TValue TValue::Clone() const { return TValue().CopyFrom(*this); - } - - double TValue::ForceNumber(double deflt) const { - const TScCore& core = Core(); - if (core.IsNumber()) { - return core.GetNumber(deflt); - } - - if (TStringBuf str = core.GetString(TStringBuf())) { - { - double result = 0; - if (TryFromString<double>(str, result)) { - return result; - } - } - { - i64 result = 0; - if (TryFromString<i64>(str, result)) { - return result; - } - } - { - ui64 result = 0; - if (TryFromString<ui64>(str, result)) { - return result; - } - } - } - - return deflt; - } - - i64 TValue::ForceIntNumber(i64 deflt) const { - const TScCore& core = Core(); - if (core.IsNumber()) { - return core.GetIntNumber(deflt); - } - - if (TStringBuf str = core.GetString(TStringBuf())) { - { - i64 result = 0; - if (TryFromString<i64>(str, result)) { - return result; - } - } - { - ui64 result = 0; - if (TryFromString<ui64>(str, result)) { - return result; - } - } - { - double result = 0; - if (TryFromString<double>(str, result)) { - return result; - } - } - } - - return deflt; - } - + } + + double TValue::ForceNumber(double deflt) const { + const TScCore& core = Core(); + if (core.IsNumber()) { + return core.GetNumber(deflt); + } + + if (TStringBuf str = core.GetString(TStringBuf())) { + { + double result = 0; + if (TryFromString<double>(str, result)) { + return result; + } + } + { + i64 result = 0; + if (TryFromString<i64>(str, result)) { + return result; + } + } + { + ui64 result = 0; + if (TryFromString<ui64>(str, result)) { + return result; + } + } + } + + return deflt; + } + + i64 TValue::ForceIntNumber(i64 deflt) const { + const TScCore& core = Core(); + if (core.IsNumber()) { + return core.GetIntNumber(deflt); + } + + if (TStringBuf str = core.GetString(TStringBuf())) { + { + i64 result = 0; + if (TryFromString<i64>(str, result)) { + return result; + } + } + { + ui64 result = 0; + if (TryFromString<ui64>(str, result)) { + return result; + } + } + { + double result = 0; + if (TryFromString<double>(str, result)) { + return result; + } + } + } + + return deflt; + } + TString TValue::ForceString(const TString& deflt) const { - const TScCore& core = Core(); - if (core.IsString()) { - return ToString(core.GetString(TStringBuf())); - } - - if (core.IsIntNumber()) { - return ToString(core.GetIntNumber(0)); - } - - if (core.IsNumber()) { - return ToString(core.GetNumber(0)); - } - - return deflt; - } - - TValue& /*this*/ TValue::CopyFrom(const TValue& other) { - if (Same(*this, other)) { - return *this; - } - - using namespace NImpl; - return DoCopyFromImpl(other, GetTlsInstance<TSelfLoopContext>(), GetTlsInstance<TSelfOverrideContext>()); - } - - TValue& TValue::DoCopyFromImpl(const TValue& other, - NImpl::TSelfLoopContext& otherLoopCtx, + const TScCore& core = Core(); + if (core.IsString()) { + return ToString(core.GetString(TStringBuf())); + } + + if (core.IsIntNumber()) { + return ToString(core.GetIntNumber(0)); + } + + if (core.IsNumber()) { + return ToString(core.GetNumber(0)); + } + + return deflt; + } + + TValue& /*this*/ TValue::CopyFrom(const TValue& other) { + if (Same(*this, other)) { + return *this; + } + + using namespace NImpl; + return DoCopyFromImpl(other, GetTlsInstance<TSelfLoopContext>(), GetTlsInstance<TSelfOverrideContext>()); + } + + TValue& TValue::DoCopyFromImpl(const TValue& other, + NImpl::TSelfLoopContext& otherLoopCtx, NImpl::TSelfOverrideContext& selfOverrideCtx) { - if (Same(*this, other)) { - return *this; - } - + if (Same(*this, other)) { + return *this; + } + CoreMutableForSet(); // trigger COW - - TScCore& selfCore = *TheCore; - const TScCore& otherCore = other.Core(); - - NImpl::TSelfLoopContext::TGuard loopCheck(otherLoopCtx, otherCore); - NImpl::TSelfOverrideContext::TGuard overrideGuard(selfOverrideCtx, selfCore); - - selfCore.SetNull(); - - if (!loopCheck.Ok) { - return *this; // a loop encountered (and asserted), skip the back reference - } - - switch (otherCore.ValueType) { + + TScCore& selfCore = *TheCore; + const TScCore& otherCore = other.Core(); + + NImpl::TSelfLoopContext::TGuard loopCheck(otherLoopCtx, otherCore); + NImpl::TSelfOverrideContext::TGuard overrideGuard(selfOverrideCtx, selfCore); + + selfCore.SetNull(); + + if (!loopCheck.Ok) { + return *this; // a loop encountered (and asserted), skip the back reference + } + + switch (otherCore.ValueType) { default: Y_ASSERT(false); [[fallthrough]]; @@ -214,136 +214,136 @@ namespace NSc { } TheCore = std::move(tmp); break; - } - } - - return *this; - } - - TValue& TValue::Swap(TValue& v) { - DoSwap(TheCore, v.TheCore); - DoSwap(CopyOnWrite, v.CopyOnWrite); - return *this; - } - - bool TValue::Same(const TValue& a, const TValue& b) { - return a.TheCore.Get() == b.TheCore.Get(); - } - - bool TValue::SamePool(const TValue& a, const TValue& b) { + } + } + + return *this; + } + + TValue& TValue::Swap(TValue& v) { + DoSwap(TheCore, v.TheCore); + DoSwap(CopyOnWrite, v.CopyOnWrite); + return *this; + } + + bool TValue::Same(const TValue& a, const TValue& b) { + return a.TheCore.Get() == b.TheCore.Get(); + } + + bool TValue::SamePool(const TValue& a, const TValue& b) { return Same(a, b) || a.TheCore->Pool.Get() == b.TheCore->Pool.Get(); - } - - bool TValue::Equal(const TValue& a, const TValue& b) { - if (Same(a, b)) { - return true; - } - - const NSc::TValue::TScCore& coreA = a.Core(); - const NSc::TValue::TScCore& coreB = b.Core(); - - if (coreA.IsNumber() && coreB.IsNumber()) { - return coreA.GetIntNumber(0) == coreB.GetIntNumber(0) && coreA.GetNumber(0) == coreB.GetNumber(0); - } - - if (coreA.ValueType != coreB.ValueType) { - return false; - } - - if (coreA.IsString()) { + } + + bool TValue::Equal(const TValue& a, const TValue& b) { + if (Same(a, b)) { + return true; + } + + const NSc::TValue::TScCore& coreA = a.Core(); + const NSc::TValue::TScCore& coreB = b.Core(); + + if (coreA.IsNumber() && coreB.IsNumber()) { + return coreA.GetIntNumber(0) == coreB.GetIntNumber(0) && coreA.GetNumber(0) == coreB.GetNumber(0); + } + + if (coreA.ValueType != coreB.ValueType) { + return false; + } + + if (coreA.IsString()) { std::string_view strA = coreA.String; std::string_view strB = coreB.String; - + if (strA != strB) { - return false; - } - } else if (coreA.IsArray()) { - const TArray& arrA = coreA.Array; - const TArray& arrB = coreB.Array; - - if (arrA.size() != arrB.size()) { - return false; - } - - for (size_t i = 0; i < arrA.size(); ++i) { - if (!Equal(arrA[i], arrB[i])) { - return false; - } - } - } else if (coreA.IsDict()) { - const ::NSc::TDict& dictA = coreA.Dict; - const ::NSc::TDict& dictB = coreB.Dict; - - if (dictA.size() != dictB.size()) { - return false; - } - - for (const auto& ita : dictA) { - ::NSc::TDict::const_iterator itb = dictB.find(ita.first); - - if (itb == dictB.end() || !Equal(ita.second, itb->second)) { - return false; + return false; + } + } else if (coreA.IsArray()) { + const TArray& arrA = coreA.Array; + const TArray& arrB = coreB.Array; + + if (arrA.size() != arrB.size()) { + return false; + } + + for (size_t i = 0; i < arrA.size(); ++i) { + if (!Equal(arrA[i], arrB[i])) { + return false; + } + } + } else if (coreA.IsDict()) { + const ::NSc::TDict& dictA = coreA.Dict; + const ::NSc::TDict& dictB = coreB.Dict; + + if (dictA.size() != dictB.size()) { + return false; + } + + for (const auto& ita : dictA) { + ::NSc::TDict::const_iterator itb = dictB.find(ita.first); + + if (itb == dictB.end() || !Equal(ita.second, itb->second)) { + return false; } - } - } - - return true; - } - - TValue& TValue::DoMerge(const TValue& delta, bool lowPriorityDelta) { - if (Same(*this, delta)) { - return *this; - } - - using namespace NImpl; - return DoMergeImpl(delta, lowPriorityDelta, GetTlsInstance<TSelfLoopContext>(), GetTlsInstance<TSelfOverrideContext>()); - } - - TValue& TValue::DoMergeImpl(const TValue& delta, bool lowPriorityDelta, - NImpl::TSelfLoopContext& otherLoopCtx, + } + } + + return true; + } + + TValue& TValue::DoMerge(const TValue& delta, bool lowPriorityDelta) { + if (Same(*this, delta)) { + return *this; + } + + using namespace NImpl; + return DoMergeImpl(delta, lowPriorityDelta, GetTlsInstance<TSelfLoopContext>(), GetTlsInstance<TSelfOverrideContext>()); + } + + TValue& TValue::DoMergeImpl(const TValue& delta, bool lowPriorityDelta, + NImpl::TSelfLoopContext& otherLoopCtx, NImpl::TSelfOverrideContext& selfOverrideGuard) { - if (Same(*this, delta)) { - return *this; - } - - if (delta.IsDict() && (!lowPriorityDelta || IsDict() || IsNull())) { - TScCore& core = CoreMutable(); - const TScCore& deltaCore = delta.Core(); - - NImpl::TSelfLoopContext::TGuard loopCheck(otherLoopCtx, deltaCore); - - if (!loopCheck.Ok) { - return *this; // a loop encountered (and asserted), skip the back reference - } - - if (!lowPriorityDelta || IsNull()) { - SetDict(); - } - - const TDict& ddelta = deltaCore.Dict; - - for (const auto& dit : ddelta) { + if (Same(*this, delta)) { + return *this; + } + + if (delta.IsDict() && (!lowPriorityDelta || IsDict() || IsNull())) { + TScCore& core = CoreMutable(); + const TScCore& deltaCore = delta.Core(); + + NImpl::TSelfLoopContext::TGuard loopCheck(otherLoopCtx, deltaCore); + + if (!loopCheck.Ok) { + return *this; // a loop encountered (and asserted), skip the back reference + } + + if (!lowPriorityDelta || IsNull()) { + SetDict(); + } + + const TDict& ddelta = deltaCore.Dict; + + for (const auto& dit : ddelta) { core.GetOrAdd(dit.first).DoMergeImpl(dit.second, lowPriorityDelta, otherLoopCtx, selfOverrideGuard); - } - } else if (!delta.IsNull() && (!lowPriorityDelta || IsNull())) { - DoCopyFromImpl(delta, otherLoopCtx, selfOverrideGuard); - } - - return *this; - } - - NJson::TJsonValue TValue::ToJsonValue() const { - using namespace NImpl; - return ToJsonValueImpl(GetTlsInstance<TSelfLoopContext>()); - } - - NJson::TJsonValue TValue::ToJsonValueImpl(NImpl::TSelfLoopContext& loopCtx) const { - const TScCore& core = Core(); - - switch (core.ValueType) { + } + } else if (!delta.IsNull() && (!lowPriorityDelta || IsNull())) { + DoCopyFromImpl(delta, otherLoopCtx, selfOverrideGuard); + } + + return *this; + } + + NJson::TJsonValue TValue::ToJsonValue() const { + using namespace NImpl; + return ToJsonValueImpl(GetTlsInstance<TSelfLoopContext>()); + } + + NJson::TJsonValue TValue::ToJsonValueImpl(NImpl::TSelfLoopContext& loopCtx) const { + const TScCore& core = Core(); + + switch (core.ValueType) { default: case EType::Null: - return NJson::TJsonValue(NJson::JSON_NULL); + return NJson::TJsonValue(NJson::JSON_NULL); case EType::Bool: return NJson::TJsonValue(core.GetBool()); case EType::IntNumber: @@ -354,27 +354,27 @@ namespace NSc { return NJson::TJsonValue(core.String); case EType::Array: { NImpl::TSelfLoopContext::TGuard loopGuard(loopCtx, core); - + if (!loopGuard.Ok) { return NJson::TJsonValue(NJson::JSON_NULL); } - + NJson::TJsonValue result(NJson::JSON_ARRAY); const TArray& arr = core.Array; - + for (const auto& item : arr) { result.AppendValue(NJson::TJsonValue::UNDEFINED) = item.ToJsonValueImpl(loopCtx); } - + return result; - } + } case EType::Dict: { NImpl::TSelfLoopContext::TGuard loopGuard(loopCtx, core); - + if (!loopGuard.Ok) { return NJson::TJsonValue(NJson::JSON_NULL); } - + NJson::TJsonValue result(NJson::JSON_MAP); const TDict& dict = core.Dict; @@ -383,21 +383,21 @@ namespace NSc { } return result; - } - } - } - - TValue TValue::FromJsonValue(const NJson::TJsonValue& val) { - TValue result; - FromJsonValue(result, val); - return result; - } - - TValue& TValue::FromJsonValue(TValue& res, const NJson::TJsonValue& val) { + } + } + } + + TValue TValue::FromJsonValue(const NJson::TJsonValue& val) { + TValue result; + FromJsonValue(result, val); + return result; + } + + TValue& TValue::FromJsonValue(TValue& res, const NJson::TJsonValue& val) { TScCore& core = res.CoreMutableForSet(); - core.SetNull(); - - switch (val.GetType()) { + core.SetNull(); + + switch (val.GetType()) { default: case NJson::JSON_UNDEFINED: case NJson::JSON_NULL: @@ -423,114 +423,114 @@ namespace NSc { FromJsonValue(core.Push(), item); } break; - } + } case NJson::JSON_MAP: { core.SetDict(); for (const auto& item : val.GetMap()) { FromJsonValue(core.Add(item.first), item.second); } break; - } - } - - return res; - } - - struct TDefaults { - TValue::TPoolPtr Pool = MakeIntrusive<NDefinitions::TPool>(); - TValue::TScCore Core{Pool}; - }; - - const TValue::TScCore& TValue::DefaultCore() { - return Default<TDefaults>().Core; - } - - const TArray& TValue::DefaultArray() { - return Default<TDefaults>().Core.Array; - } - + } + } + + return res; + } + + struct TDefaults { + TValue::TPoolPtr Pool = MakeIntrusive<NDefinitions::TPool>(); + TValue::TScCore Core{Pool}; + }; + + const TValue::TScCore& TValue::DefaultCore() { + return Default<TDefaults>().Core; + } + + const TArray& TValue::DefaultArray() { + return Default<TDefaults>().Core.Array; + } + const TDict& TValue::DefaultDict() { - return Default<TDefaults>().Core.Dict; - } - - const TValue& TValue::DefaultValue() { - return *FastTlsSingleton<TValue>(); - } - - bool TValue::IsSameOrAncestorOf(const TValue& other) const { - using namespace NImpl; - return IsSameOrAncestorOfImpl(other.Core(), GetTlsInstance<TSelfLoopContext>()); - } - - bool TValue::IsSameOrAncestorOfImpl(const TScCore& other, NImpl::TSelfLoopContext& loopCtx) const { - const TScCore& core = Core(); - - if (&core == &other) { - return true; - } - - switch (core.ValueType) { - default: - return false; - case EType::Array: { - NImpl::TSelfLoopContext::TGuard loopGuard(loopCtx, core); - - if (!loopGuard.Ok) { - return false; - } - - for (const auto& item : core.Array) { - if (item.IsSameOrAncestorOfImpl(other, loopCtx)) { - return true; - } - } - - return false; - } - case EType::Dict: { - NImpl::TSelfLoopContext::TGuard loopGuard(loopCtx, core); - - if (!loopGuard.Ok) { - return false; - } - - for (const auto& item : core.Dict) { - if (item.second.IsSameOrAncestorOfImpl(other, loopCtx)) { - return true; - } - } - - return false; - } - } - } - - namespace NPrivate { + return Default<TDefaults>().Core.Dict; + } + + const TValue& TValue::DefaultValue() { + return *FastTlsSingleton<TValue>(); + } + + bool TValue::IsSameOrAncestorOf(const TValue& other) const { + using namespace NImpl; + return IsSameOrAncestorOfImpl(other.Core(), GetTlsInstance<TSelfLoopContext>()); + } + + bool TValue::IsSameOrAncestorOfImpl(const TScCore& other, NImpl::TSelfLoopContext& loopCtx) const { + const TScCore& core = Core(); + + if (&core == &other) { + return true; + } + + switch (core.ValueType) { + default: + return false; + case EType::Array: { + NImpl::TSelfLoopContext::TGuard loopGuard(loopCtx, core); + + if (!loopGuard.Ok) { + return false; + } + + for (const auto& item : core.Array) { + if (item.IsSameOrAncestorOfImpl(other, loopCtx)) { + return true; + } + } + + return false; + } + case EType::Dict: { + NImpl::TSelfLoopContext::TGuard loopGuard(loopCtx, core); + + if (!loopGuard.Ok) { + return false; + } + + for (const auto& item : core.Dict) { + if (item.second.IsSameOrAncestorOfImpl(other, loopCtx)) { + return true; + } + } + + return false; + } + } + } + + namespace NPrivate { int CompareStr(const NSc::TValue& a, TStringBuf b) { return a.GetString().compare(b); } - + int CompareInt(const NSc::TValue& a, i64 r) { i64 l = a.GetIntNumber(); return l < r ? -1 : l > r ? 1 : 0; } - + int CompareFloat(const NSc::TValue& a, double r) { double l = a.GetNumber(); return l < r ? -1 : l > r ? 1 : 0; } - - } - + + } + bool operator==(const NSc::TValue& a, const NSc::TValue& b) { - return NSc::TValue::Equal(a, b); - } - + return NSc::TValue::Equal(a, b); + } + bool operator!=(const NSc::TValue& a, const NSc::TValue& b) { - return !NSc::TValue::Equal(a, b); - } - -#define LIBRARY_SCHEME_DECLARE_TVALUE_OPS_IMPL(T, Impl) \ + return !NSc::TValue::Equal(a, b); + } + +#define LIBRARY_SCHEME_DECLARE_TVALUE_OPS_IMPL(T, Impl) \ bool operator==(const NSc::TValue& a, T b) { \ return NPrivate::Impl(a, b) == 0; \ } \ @@ -567,32 +567,32 @@ namespace NSc { bool operator>(T b, const NSc::TValue& a) { \ return NPrivate::Impl(a, b) < 0; \ } - + #define LIBRARY_SCHEME_DECLARE_TVALUE_INT_OPS_IMPL(T) \ - LIBRARY_SCHEME_DECLARE_TVALUE_OPS_IMPL(signed T, CompareInt) \ - LIBRARY_SCHEME_DECLARE_TVALUE_OPS_IMPL(unsigned T, CompareInt) - - //LIBRARY_SCHEME_DECLARE_TVALUE_OPS_IMPL(bool, CompareInt) - LIBRARY_SCHEME_DECLARE_TVALUE_OPS_IMPL(char, CompareInt) - LIBRARY_SCHEME_DECLARE_TVALUE_INT_OPS_IMPL(char) - LIBRARY_SCHEME_DECLARE_TVALUE_INT_OPS_IMPL(short) - LIBRARY_SCHEME_DECLARE_TVALUE_INT_OPS_IMPL(int) - LIBRARY_SCHEME_DECLARE_TVALUE_INT_OPS_IMPL(long) - LIBRARY_SCHEME_DECLARE_TVALUE_INT_OPS_IMPL(long long) - - LIBRARY_SCHEME_DECLARE_TVALUE_OPS_IMPL(float, CompareFloat) - LIBRARY_SCHEME_DECLARE_TVALUE_OPS_IMPL(double, CompareFloat) - - LIBRARY_SCHEME_DECLARE_TVALUE_OPS_IMPL(TStringBuf, CompareStr) + LIBRARY_SCHEME_DECLARE_TVALUE_OPS_IMPL(signed T, CompareInt) \ + LIBRARY_SCHEME_DECLARE_TVALUE_OPS_IMPL(unsigned T, CompareInt) + + //LIBRARY_SCHEME_DECLARE_TVALUE_OPS_IMPL(bool, CompareInt) + LIBRARY_SCHEME_DECLARE_TVALUE_OPS_IMPL(char, CompareInt) + LIBRARY_SCHEME_DECLARE_TVALUE_INT_OPS_IMPL(char) + LIBRARY_SCHEME_DECLARE_TVALUE_INT_OPS_IMPL(short) + LIBRARY_SCHEME_DECLARE_TVALUE_INT_OPS_IMPL(int) + LIBRARY_SCHEME_DECLARE_TVALUE_INT_OPS_IMPL(long) + LIBRARY_SCHEME_DECLARE_TVALUE_INT_OPS_IMPL(long long) + + LIBRARY_SCHEME_DECLARE_TVALUE_OPS_IMPL(float, CompareFloat) + LIBRARY_SCHEME_DECLARE_TVALUE_OPS_IMPL(double, CompareFloat) + + LIBRARY_SCHEME_DECLARE_TVALUE_OPS_IMPL(TStringBuf, CompareStr) LIBRARY_SCHEME_DECLARE_TVALUE_OPS_IMPL(const TString&, CompareStr) - LIBRARY_SCHEME_DECLARE_TVALUE_OPS_IMPL(const char* const, CompareStr) - -#undef LIBRARY_SCHEME_DECLARE_TVALUE_OPS_IMPL -#undef LIBRARY_SCHEME_DECLARE_TVALUE_INT_OPS_IMPL - -} - -template <> + LIBRARY_SCHEME_DECLARE_TVALUE_OPS_IMPL(const char* const, CompareStr) + +#undef LIBRARY_SCHEME_DECLARE_TVALUE_OPS_IMPL +#undef LIBRARY_SCHEME_DECLARE_TVALUE_INT_OPS_IMPL + +} + +template <> void Out<NSc::TValue>(IOutputStream& o, TTypeTraits<NSc::TValue>::TFuncParam v) { - o.Write(v.ToJson(true)); -} + o.Write(v.ToJson(true)); +} diff --git a/library/cpp/scheme/scheme.h b/library/cpp/scheme/scheme.h index 1ebef0c316..3d7c59f3c9 100644 --- a/library/cpp/scheme/scheme.h +++ b/library/cpp/scheme/scheme.h @@ -1,20 +1,20 @@ -#pragma once - -#include "scimpl_defs.h" - +#pragma once + +#include "scimpl_defs.h" + #include "fwd.h" -#include <iterator> -#include <utility> - -namespace NSc { +#include <iterator> +#include <utility> + +namespace NSc { #ifdef _MSC_VER #pragma warning(disable : 4521 4522) #endif - // todo: try to remove some rarely used methods - class TValue { - public: + // todo: try to remove some rarely used methods + class TValue { + public: enum class EType { Null = 0 /* "Null" */, Bool /* "Bool" */, @@ -23,514 +23,514 @@ namespace NSc { String /* "String" */, Array /* "Array" */, Dict /* "Dict" */ - }; - - struct TScCore; - using TCorePtr = TIntrusivePtr<TScCore>; - using TPoolPtr = TIntrusivePtr<NDefinitions::TPool>; - - using TArray = ::NSc::TArray; - using TDict = ::NSc::TDict; - + }; + + struct TScCore; + using TCorePtr = TIntrusivePtr<TScCore>; + using TPoolPtr = TIntrusivePtr<NDefinitions::TPool>; + + using TArray = ::NSc::TArray; + using TDict = ::NSc::TDict; + private: // A TValue instance has only these 3 fields mutable TCorePtr TheCore; // a pointer to a refcounted (kind of) variant - bool CopyOnWrite = false; // a flag that thevalue is a COW shallow copy and should produce a deep copy once modified - - // Thus all copies of a TValue are by default shallow. Use TValue::Clone to force a deep copy. - // A COW copy will see changes in its parent, but no change in the COW copy will propagate to its parent. - - public: + bool CopyOnWrite = false; // a flag that thevalue is a COW shallow copy and should produce a deep copy once modified + + // Thus all copies of a TValue are by default shallow. Use TValue::Clone to force a deep copy. + // A COW copy will see changes in its parent, but no change in the COW copy will propagate to its parent. + + public: inline TValue(); - inline TValue(TValue& v); - inline TValue(const TValue& v); - inline TValue(TValue&& v) noexcept; - - public: // Operators - inline TValue(double t); - inline TValue(unsigned long long t); - inline TValue(unsigned long t); - inline TValue(unsigned t); - inline TValue(long long t); - inline TValue(long t); - inline TValue(int t); + inline TValue(TValue& v); + inline TValue(const TValue& v); + inline TValue(TValue&& v) noexcept; + + public: // Operators + inline TValue(double t); + inline TValue(unsigned long long t); + inline TValue(unsigned long t); + inline TValue(unsigned t); + inline TValue(long long t); + inline TValue(long t); + inline TValue(int t); // inline TValue(bool b); - - inline TValue(TStringBuf t); - inline TValue(const char*); - - inline operator double() const; - inline operator float() const; - inline operator long long() const; - inline operator long() const; - inline operator int() const; - inline operator short() const; - inline operator char() const; - inline operator unsigned long long() const; - inline operator unsigned long() const; - inline operator unsigned() const; - inline operator unsigned short() const; - inline operator unsigned char() const; - inline operator signed char() const; - - inline operator TStringBuf() const; - - inline operator const ::NSc::TArray&() const; - inline operator const ::NSc::TDict&() const; - - inline TValue& operator=(double t); - - inline TValue& operator=(unsigned long long t); - inline TValue& operator=(unsigned long t); - inline TValue& operator=(unsigned t); - - inline TValue& operator=(long long t); - inline TValue& operator=(long t); - inline TValue& operator=(int t); + + inline TValue(TStringBuf t); + inline TValue(const char*); + + inline operator double() const; + inline operator float() const; + inline operator long long() const; + inline operator long() const; + inline operator int() const; + inline operator short() const; + inline operator char() const; + inline operator unsigned long long() const; + inline operator unsigned long() const; + inline operator unsigned() const; + inline operator unsigned short() const; + inline operator unsigned char() const; + inline operator signed char() const; + + inline operator TStringBuf() const; + + inline operator const ::NSc::TArray&() const; + inline operator const ::NSc::TDict&() const; + + inline TValue& operator=(double t); + + inline TValue& operator=(unsigned long long t); + inline TValue& operator=(unsigned long t); + inline TValue& operator=(unsigned t); + + inline TValue& operator=(long long t); + inline TValue& operator=(long t); + inline TValue& operator=(int t); // inline TValue& operator=(bool t); - - inline TValue& operator=(TStringBuf t); - inline TValue& operator=(const char* t); - + + inline TValue& operator=(TStringBuf t); + inline TValue& operator=(const char* t); + inline TValue& operator=(TValue& v) &; inline TValue& operator=(const TValue& v) &; inline TValue& operator=(TValue&& v) & noexcept; - + inline TValue& operator=(TValue& v) && = delete; inline TValue& operator=(const TValue& v) && = delete; inline TValue& operator=(TValue&& v) && = delete; - public: - template <class T> // ui16 or TStringBuf + public: + template <class T> // ui16 or TStringBuf inline TValue& operator[](const T& idx) { return GetOrAdd(idx); } - - template <class T> // ui16 or TStringBuf + + template <class T> // ui16 or TStringBuf inline const TValue& operator[](const T& idx) const { return Get(idx); } - - public: // Data methods /////////////////////////////////////////////////////////// - inline EType GetType() const; - - inline bool IsNull() const; - - inline TValue& SetNull(); // returns self, will set type to Null - + + public: // Data methods /////////////////////////////////////////////////////////// + inline EType GetType() const; + + inline bool IsNull() const; + + inline TValue& SetNull(); // returns self, will set type to Null + TValue& Clear() { return ClearArray().ClearDict().SetNull(); } - - public: // Number methods ///////////////////////////////////////////////////////// - // Bool, IntNumber and FloatNumber are all compatible. - // If a TValue node has one of the types it may as well be used as another. - // FloatNumber methods. Forces FloatNumber representation. Compatible with IntNumber and Bool - - inline bool IsNumber() const; // true if any of FloatNumber, IntNumber, Bool - - inline double GetNumber(double defaultval = 0) const; // Compatible with Bool, IntNumber and FloatNumber types - - inline double& GetNumberMutable(double defaultval = 0); // Will switch the type to FloatNumber - - inline TValue& SetNumber(double val = 0); // returns self, will switch the type to FloatNumber - - double ForceNumber(double deflt = 0) const; // Best-effort cast to double (will do TryFromString if applicable) - - // IntNumber methods. Forces integer representation. Compatible with FloatNumber and Bool types. - // Note: if you don't care about distinguishing bools, ints and doubles, use *Number methods above - - inline bool IsIntNumber() const; // true only if IntNumber or Bool - - inline i64 GetIntNumber(i64 defaultval = 0) const; // Compatible with Bool, IntNumber and FloatNumber types - - inline i64& GetIntNumberMutable(i64 defaultval = 0); // Will switch the type to IntNumber - - inline TValue& SetIntNumber(i64 val = 0); // returns self, will switch the type to IntNumber - - i64 ForceIntNumber(i64 deflt = 0) const; // Best-effort cast to i64 (will do TryFromString for String) - - // Bool methods. Forces bool representation. Compatible with Float Number and Int Number methods above. - // Note: if you don't care about distinguishing Bool, IntNumber and FloatNumber, use *Number methods above - - inline bool IsBool() const; // true only if Bool - - inline bool GetBool(bool defaultval = false) const; // Compatible with Bool, IntNumber and FloatNumber types - - inline TValue& SetBool(bool val = false); // returns self, will switch the type to Bool - - public: // Arcadia-specific boolean representation support - // Tests for explicit True, also checks for arcadia-specific boolean representation - bool IsTrue() const { - return IsNumber() ? GetNumber() : ::IsTrue(GetString()); - } - - // Tests for explicit False, also checks for arcadia-specific boolean representation - bool IsExplicitFalse() const { - return IsNumber() ? !GetNumber() : IsFalse(GetString()); - } - - public: // String methods ///////////////////////////////////////////////////////// - inline bool IsString() const; - - inline TStringBuf GetString(TStringBuf defaultval = TStringBuf()) const; - - inline TValue& SetString(TStringBuf val = TStringBuf()); // returns self - + + public: // Number methods ///////////////////////////////////////////////////////// + // Bool, IntNumber and FloatNumber are all compatible. + // If a TValue node has one of the types it may as well be used as another. + // FloatNumber methods. Forces FloatNumber representation. Compatible with IntNumber and Bool + + inline bool IsNumber() const; // true if any of FloatNumber, IntNumber, Bool + + inline double GetNumber(double defaultval = 0) const; // Compatible with Bool, IntNumber and FloatNumber types + + inline double& GetNumberMutable(double defaultval = 0); // Will switch the type to FloatNumber + + inline TValue& SetNumber(double val = 0); // returns self, will switch the type to FloatNumber + + double ForceNumber(double deflt = 0) const; // Best-effort cast to double (will do TryFromString if applicable) + + // IntNumber methods. Forces integer representation. Compatible with FloatNumber and Bool types. + // Note: if you don't care about distinguishing bools, ints and doubles, use *Number methods above + + inline bool IsIntNumber() const; // true only if IntNumber or Bool + + inline i64 GetIntNumber(i64 defaultval = 0) const; // Compatible with Bool, IntNumber and FloatNumber types + + inline i64& GetIntNumberMutable(i64 defaultval = 0); // Will switch the type to IntNumber + + inline TValue& SetIntNumber(i64 val = 0); // returns self, will switch the type to IntNumber + + i64 ForceIntNumber(i64 deflt = 0) const; // Best-effort cast to i64 (will do TryFromString for String) + + // Bool methods. Forces bool representation. Compatible with Float Number and Int Number methods above. + // Note: if you don't care about distinguishing Bool, IntNumber and FloatNumber, use *Number methods above + + inline bool IsBool() const; // true only if Bool + + inline bool GetBool(bool defaultval = false) const; // Compatible with Bool, IntNumber and FloatNumber types + + inline TValue& SetBool(bool val = false); // returns self, will switch the type to Bool + + public: // Arcadia-specific boolean representation support + // Tests for explicit True, also checks for arcadia-specific boolean representation + bool IsTrue() const { + return IsNumber() ? GetNumber() : ::IsTrue(GetString()); + } + + // Tests for explicit False, also checks for arcadia-specific boolean representation + bool IsExplicitFalse() const { + return IsNumber() ? !GetNumber() : IsFalse(GetString()); + } + + public: // String methods ///////////////////////////////////////////////////////// + inline bool IsString() const; + + inline TStringBuf GetString(TStringBuf defaultval = TStringBuf()) const; + + inline TValue& SetString(TStringBuf val = TStringBuf()); // returns self + TString ForceString(const TString& deflt = TString()) const; // Best-effort cast to TString (will do ToString for numeric types) - - // todo: remove + + // todo: remove inline bool StringEmpty() const; - inline size_t StringSize() const; - - public: // Array methods ////////////////////////////////////////////////////////// - inline bool IsArray() const; - - inline const TArray& GetArray() const; - inline TArray& GetArrayMutable(); - inline TValue& SetArray(); // turns into array if needed, returns self - inline TValue& ClearArray(); - - inline bool Has(size_t idx) const; - - inline const TValue& Get(size_t idx) const; // returns child or default + inline size_t StringSize() const; + + public: // Array methods ////////////////////////////////////////////////////////// + inline bool IsArray() const; + + inline const TArray& GetArray() const; + inline TArray& GetArrayMutable(); + inline TValue& SetArray(); // turns into array if needed, returns self + inline TValue& ClearArray(); + + inline bool Has(size_t idx) const; + + inline const TValue& Get(size_t idx) const; // returns child or default inline TValue* GetNoAdd(size_t idx); // returns link to existing child or nullptr - - inline TValue& Push(); // returns new child - - template <class T> - TValue& Push(T&& t) { - return Push() = std::forward<T>(t); - } // returns new child - + + inline TValue& Push(); // returns new child + + template <class T> + TValue& Push(T&& t) { + return Push() = std::forward<T>(t); + } // returns new child + TValue& Insert(ui16 idx) { return InsertUnsafe(idx); } // creates missing values, returns new child - - template <class T> - TValue& Insert(ui16 idx, T&& v) { - return InsertUnsafe(idx, std::forward<T>(v)); + + template <class T> + TValue& Insert(ui16 idx, T&& v) { + return InsertUnsafe(idx, std::forward<T>(v)); } // creates missing values, returns new child - - template <class TIt> - inline TValue& AppendAll(TIt begin, TIt end); // Append(vec.begin(), vec.end()) - - template <class TColl> + + template <class TIt> + inline TValue& AppendAll(TIt begin, TIt end); // Append(vec.begin(), vec.end()) + + template <class TColl> inline TValue& AppendAll(TColl&& coll); // Append(vec) - - inline TValue& AppendAll(std::initializer_list<TValue> coll); - + + inline TValue& AppendAll(std::initializer_list<TValue> coll); + TValue& GetOrAdd(ui16 idx) { return GetOrAddUnsafe(idx); } // creates missing values, returns new child - - inline TValue& InsertUnsafe(size_t idx); // creates missing values, returns new child - - template <class T> - TValue& InsertUnsafe(size_t idx, T&& t) { - return InsertUnsafe(idx) = std::forward<T>(t); - } // creates missing values, returns new child - + + inline TValue& InsertUnsafe(size_t idx); // creates missing values, returns new child + + template <class T> + TValue& InsertUnsafe(size_t idx, T&& t) { + return InsertUnsafe(idx) = std::forward<T>(t); + } // creates missing values, returns new child + inline TValue& GetOrAddUnsafe(size_t idx); // creates missing values, returns new child - + inline TValue Pop(); // returns popped value - inline TValue Delete(size_t idx); // returns deleted value if it existed, NSc::Null() otherwise - + inline TValue Delete(size_t idx); // returns deleted value if it existed, NSc::Null() otherwise + inline TValue& Front() { return GetOrAdd(0); } // creates missing value, returns child inline const TValue& Front() const { return Get(0); } // returns child or default - - inline TValue& Back(); // creates missing value, returns child - inline const TValue& Back() const; // returns child or default - - // todo: remove - inline bool ArrayEmpty() const; - inline size_t ArraySize() const; - - public: // Dict methods - inline bool IsDict() const; - - inline const TDict& GetDict() const; + + inline TValue& Back(); // creates missing value, returns child + inline const TValue& Back() const; // returns child or default + + // todo: remove + inline bool ArrayEmpty() const; + inline size_t ArraySize() const; + + public: // Dict methods + inline bool IsDict() const; + + inline const TDict& GetDict() const; inline TDict& GetDictMutable(); - inline TValue& SetDict(); // turns into dict if not one, returns self - inline TValue& ClearDict(); - - inline bool Has(TStringBuf idx) const; - - inline const TValue& Get(TStringBuf idx) const; + inline TValue& SetDict(); // turns into dict if not one, returns self + inline TValue& ClearDict(); + + inline bool Has(TStringBuf idx) const; + + inline const TValue& Get(TStringBuf idx) const; inline TValue* GetNoAdd(TStringBuf idx); // returns link to existing child or nullptr - + TValue& Add(TStringBuf idx) { return GetOrAdd(idx); } - - template <class T> - TValue& Add(TStringBuf idx, T&& t) { - return Add(idx) = std::forward<T>(t); + + template <class T> + TValue& Add(TStringBuf idx, T&& t) { + return Add(idx) = std::forward<T>(t); } - - inline TValue& GetOrAdd(TStringBuf idx); // creates missing value, returns child - - inline TValue Delete(TStringBuf idx); // returns deleted value - - inline TValue& AddAll(std::initializer_list<std::pair<TStringBuf, TValue>> t); - - TStringBufs DictKeys(bool sorted = true) const; - TStringBufs& DictKeys(TStringBufs&, bool sorted = true) const; - - // todo: remove - inline bool DictEmpty() const; - inline size_t DictSize() const; - - public: // Json methods //////////////////////////////////////////////// - using TJsonOpts = NSc::TJsonOpts; - using EJsonOpts = TJsonOpts::EJsonOpts; - static const EJsonOpts JO_DEFAULT = TJsonOpts::JO_DEFAULT; - static const EJsonOpts JO_SORT_KEYS = TJsonOpts::JO_SORT_KEYS; - static const EJsonOpts JO_SKIP_UNSAFE = TJsonOpts::JO_SKIP_UNSAFE; // skip non-utf8 strings - static const EJsonOpts JO_PRETTY = TJsonOpts::JO_PRETTY; + + inline TValue& GetOrAdd(TStringBuf idx); // creates missing value, returns child + + inline TValue Delete(TStringBuf idx); // returns deleted value + + inline TValue& AddAll(std::initializer_list<std::pair<TStringBuf, TValue>> t); + + TStringBufs DictKeys(bool sorted = true) const; + TStringBufs& DictKeys(TStringBufs&, bool sorted = true) const; + + // todo: remove + inline bool DictEmpty() const; + inline size_t DictSize() const; + + public: // Json methods //////////////////////////////////////////////// + using TJsonOpts = NSc::TJsonOpts; + using EJsonOpts = TJsonOpts::EJsonOpts; + static const EJsonOpts JO_DEFAULT = TJsonOpts::JO_DEFAULT; + static const EJsonOpts JO_SORT_KEYS = TJsonOpts::JO_SORT_KEYS; + static const EJsonOpts JO_SKIP_UNSAFE = TJsonOpts::JO_SKIP_UNSAFE; // skip non-utf8 strings + static const EJsonOpts JO_PRETTY = TJsonOpts::JO_PRETTY; static const EJsonOpts JO_SAFE = TJsonOpts::JO_SAFE; // JO_SORT_KEYS | JO_SKIP_UNSAFE - static const EJsonOpts JO_PARSER_STRICT_WITH_COMMENTS = TJsonOpts::JO_PARSER_STRICT_WITH_COMMENTS; // strict json + strict utf8 + static const EJsonOpts JO_PARSER_STRICT_WITH_COMMENTS = TJsonOpts::JO_PARSER_STRICT_WITH_COMMENTS; // strict json + strict utf8 static const EJsonOpts JO_PARSER_STRICT = TJsonOpts::JO_PARSER_STRICT; // strict json + strict utf8 + comments are disallowed static const EJsonOpts JO_PARSER_DISALLOW_DUPLICATE_KEYS = TJsonOpts::JO_PARSER_DISALLOW_DUPLICATE_KEYS; - - static TValue FromJson(TStringBuf, const TJsonOpts& = TJsonOpts()); - static TValue FromJsonThrow(TStringBuf, const TJsonOpts& = TJsonOpts()); - static bool FromJson(TValue&, TStringBuf, const TJsonOpts& = TJsonOpts()); - - // TODO: Переименовать ToJson в ToJsonUnsafe, а ToJsonSafe в ToJson + + static TValue FromJson(TStringBuf, const TJsonOpts& = TJsonOpts()); + static TValue FromJsonThrow(TStringBuf, const TJsonOpts& = TJsonOpts()); + static bool FromJson(TValue&, TStringBuf, const TJsonOpts& = TJsonOpts()); + + // TODO: Переименовать ToJson в ToJsonUnsafe, а ToJsonSafe в ToJson TString ToJson(const TJsonOpts& = TJsonOpts()) const; const TValue& ToJson(IOutputStream&, const TJsonOpts& = TJsonOpts()) const; // returns self - - // ToJson(JO_SORT_KEYS | JO_SKIP_UNSAFE) + + // ToJson(JO_SORT_KEYS | JO_SKIP_UNSAFE) TString ToJsonSafe(const TJsonOpts& = TJsonOpts()) const; const TValue& ToJsonSafe(IOutputStream&, const TJsonOpts& = TJsonOpts()) const; - - // ToJson(JO_SORT_KEYS | JO_PRETTY | JO_SKIP_UNSAFE) - TString ToJsonPretty(const TJsonOpts& = TJsonOpts()) const; + + // ToJson(JO_SORT_KEYS | JO_PRETTY | JO_SKIP_UNSAFE) + TString ToJsonPretty(const TJsonOpts& = TJsonOpts()) const; const TValue& ToJsonPretty(IOutputStream&, const TJsonOpts& = TJsonOpts()) const; - - NJson::TJsonValue ToJsonValue() const; - - static TValue FromJsonValue(const NJson::TJsonValue&); - static TValue& FromJsonValue(TValue&, const NJson::TJsonValue&); // returns self - - static TJsonOpts MakeOptsSafeForSerializer(TJsonOpts = TJsonOpts()); - static TJsonOpts MakeOptsPrettyForSerializer(TJsonOpts = TJsonOpts()); - - public: // Merge methods //////////////////////////////////////////////// + + NJson::TJsonValue ToJsonValue() const; + + static TValue FromJsonValue(const NJson::TJsonValue&); + static TValue& FromJsonValue(TValue&, const NJson::TJsonValue&); // returns self + + static TJsonOpts MakeOptsSafeForSerializer(TJsonOpts = TJsonOpts()); + static TJsonOpts MakeOptsPrettyForSerializer(TJsonOpts = TJsonOpts()); + + public: // Merge methods //////////////////////////////////////////////// /* - * LHS.MergeUpdate(RHS): - * 1. Dict <- Dict: - * - Copy all nonconflicting key-value pairs from RHS to LHS. - * - For every pair of conflicting values apply LHS[key].MergeUpdate(RHS[key]). - * 2. Anything <- Null: - * - Do nothing. - * 3. Other conflicts: - * - Copy RHS over LHS. - * - * LHS.ReverseMerge(RHS): - * 1. Dict <- Dict: - * - Copy all nonconflicting key-value pairs from RHS to LHS. - * - For every pair of conflicting values apply LHS[key].ReverseMerge(RHS[key]). - * 2. Null <- Anything: - * - Copy RHS over LHS. - * 3. Other conflicts: - * - Do nothing. - */ - - TValue& MergeUpdateJson(TStringBuf json); // returns self - TValue& ReverseMergeJson(TStringBuf json); // returns self - - static bool MergeUpdateJson(TValue&, TStringBuf json); // returns true unless failed to parse the json - static bool ReverseMergeJson(TValue&, TStringBuf json); // returns true unless failed to parse the json - + * LHS.MergeUpdate(RHS): + * 1. Dict <- Dict: + * - Copy all nonconflicting key-value pairs from RHS to LHS. + * - For every pair of conflicting values apply LHS[key].MergeUpdate(RHS[key]). + * 2. Anything <- Null: + * - Do nothing. + * 3. Other conflicts: + * - Copy RHS over LHS. + * + * LHS.ReverseMerge(RHS): + * 1. Dict <- Dict: + * - Copy all nonconflicting key-value pairs from RHS to LHS. + * - For every pair of conflicting values apply LHS[key].ReverseMerge(RHS[key]). + * 2. Null <- Anything: + * - Copy RHS over LHS. + * 3. Other conflicts: + * - Do nothing. + */ + + TValue& MergeUpdateJson(TStringBuf json); // returns self + TValue& ReverseMergeJson(TStringBuf json); // returns self + + static bool MergeUpdateJson(TValue&, TStringBuf json); // returns true unless failed to parse the json + static bool ReverseMergeJson(TValue&, TStringBuf json); // returns true unless failed to parse the json + TValue& MergeUpdate(const TValue& delta); // return self - TValue& ReverseMerge(const TValue& delta); // return self - - public: // Path methods ///////////////////////////////////////////////////////// - // TODO: add throwing variants - // make sure to properly escape the tokens - + TValue& ReverseMerge(const TValue& delta); // return self + + public: // Path methods ///////////////////////////////////////////////////////// + // TODO: add throwing variants + // make sure to properly escape the tokens + static TString EscapeForPath(TStringBuf rawKey); // converts a raw dict key into a valid token for a selector path - + static bool PathValid(TStringBuf path); // returns true if the path is syntactically valid - + bool PathExists(TStringBuf path) const; // returns true if the path is syntactically valid and the target value exists - - const TValue& TrySelect(TStringBuf path) const; // returns the target value - // if the path is syntactically valid and the target value exists - // otherwise returns NSc::Null() - + + const TValue& TrySelect(TStringBuf path) const; // returns the target value + // if the path is syntactically valid and the target value exists + // otherwise returns NSc::Null() + TValue* TrySelectOrAdd(TStringBuf path); // returns the target value if it exists or creates if not - // if the path is syntactically valid - // otherwise returns NSc::Null() - - TValue TrySelectAndDelete(TStringBuf path); // deletes and returns the target value - // if the path is syntactically valid and the target value existed - // otherwise returns NSc::Null() - + // if the path is syntactically valid + // otherwise returns NSc::Null() + + TValue TrySelectAndDelete(TStringBuf path); // deletes and returns the target value + // if the path is syntactically valid and the target value existed + // otherwise returns NSc::Null() + public: // Copy methods ///////////////////////////////////////////////////////// TValue Clone() const; // returns deep copy of self (on the separate pool) - TValue& CopyFrom(const TValue& other); // deep copy other value into self, returns self - - TValue& Swap(TValue& v); - + TValue& CopyFrom(const TValue& other); // deep copy other value into self, returns self + + TValue& Swap(TValue& v); + static bool Same(const TValue&, const TValue&); // point to the same core static bool Equal(const TValue&, const TValue&); // recursively equal - static bool SamePool(const TValue&, const TValue&); // share arena - - public: - // very specific methods useful in very specific corner cases + static bool SamePool(const TValue&, const TValue&); // share arena + + public: + // very specific methods useful in very specific corner cases static TValue From(const ::google::protobuf::Message&, bool mapAsDict = false); - + void To(::google::protobuf::Message&, const TProtoOpts& opts = {}) const; - public: - inline explicit TValue(TPoolPtr&); - - static const TScCore& DefaultCore(); - static const TArray& DefaultArray(); + public: + inline explicit TValue(TPoolPtr&); + + static const TScCore& DefaultCore(); + static const TArray& DefaultArray(); static const TDict& DefaultDict(); - static const TValue& DefaultValue(); + static const TValue& DefaultValue(); static const TValue& Null() { return DefaultValue(); } - + void DoWriteJsonImpl(IOutputStream&, const TJsonOpts&, NImpl::TKeySortContext&, NImpl::TSelfLoopContext&) const; - - bool IsSameOrAncestorOf(const TValue& other) const; - - private: - TValue& DoMerge(const TValue& delta, bool olddelta); - TValue& DoMergeImpl(const TValue& delta, bool olddelta, NImpl::TSelfLoopContext&, NImpl::TSelfOverrideContext&); - TValue& DoCopyFromImpl(const TValue& other, NImpl::TSelfLoopContext&, NImpl::TSelfOverrideContext&); - NJson::TJsonValue ToJsonValueImpl(NImpl::TSelfLoopContext&) const; - - bool IsSameOrAncestorOfImpl(const TScCore& other, NImpl::TSelfLoopContext& loopCtx) const; - - inline TScCore& CoreMutable(); + + bool IsSameOrAncestorOf(const TValue& other) const; + + private: + TValue& DoMerge(const TValue& delta, bool olddelta); + TValue& DoMergeImpl(const TValue& delta, bool olddelta, NImpl::TSelfLoopContext&, NImpl::TSelfOverrideContext&); + TValue& DoCopyFromImpl(const TValue& other, NImpl::TSelfLoopContext&, NImpl::TSelfOverrideContext&); + NJson::TJsonValue ToJsonValueImpl(NImpl::TSelfLoopContext&) const; + + bool IsSameOrAncestorOfImpl(const TScCore& other, NImpl::TSelfLoopContext& loopCtx) const; + + inline TScCore& CoreMutable(); inline TScCore& CoreMutableForSet(); - inline const TScCore& Core() const; - - static inline TScCore* NewCore(TPoolPtr&); - - static TValue FromField(const ::google::protobuf::Message&, const ::google::protobuf::FieldDescriptor*); - static TValue FromRepeatedField(const ::google::protobuf::Message&, const ::google::protobuf::FieldDescriptor*, int index); + inline const TScCore& Core() const; + + static inline TScCore* NewCore(TPoolPtr&); + + static TValue FromField(const ::google::protobuf::Message&, const ::google::protobuf::FieldDescriptor*); + static TValue FromRepeatedField(const ::google::protobuf::Message&, const ::google::protobuf::FieldDescriptor*, int index); void ValueToField(const TValue& value, ::google::protobuf::Message&, const ::google::protobuf::FieldDescriptor*, const TProtoOpts& opts) const; void ToField(::google::protobuf::Message&, const ::google::protobuf::FieldDescriptor*, const TProtoOpts& opts) const; void ToEnumField(::google::protobuf::Message&, const ::google::protobuf::FieldDescriptor*, const TProtoOpts& opts) const; void ToRepeatedField(::google::protobuf::Message&, const ::google::protobuf::FieldDescriptor*, const TProtoOpts& opts) const; void ToMapField(::google::protobuf::Message&, const ::google::protobuf::FieldDescriptor*, const TProtoOpts& opts) const; - }; - - - inline const TValue& Null() { - return TValue::DefaultValue(); - } - - + }; + + + inline const TValue& Null() { + return TValue::DefaultValue(); + } + + class TArray: public TDeque<TValue, TPoolAllocator>, TNonCopyable { using TParent = TDeque<TValue, TPoolAllocator>; - - public: - TArray(TMemoryPool* p) - : TParent(p) + + public: + TArray(TMemoryPool* p) + : TParent(p) { } - - template <class TIt> - void AppendAll(TIt begin, TIt end) { - TParent::insert(TParent::end(), begin, end); - } - - template <class TColl> - void AppendAll(TColl&& coll) { - AppendAll(std::begin(coll), std::end(coll)); - } - - void AppendAll(std::initializer_list<TValue> coll) { - AppendAll(coll.begin(), coll.end()); - } - - const TValue& operator[](size_t i) const { - return EnsureIndex(i); - } - - TValue& operator[](size_t i) { - return EnsureIndex(i); - } - - const TValue& front() const { - return EnsureIndex(0); - } - - TValue& front() { - return EnsureIndex(0); - } - - const TValue& back() const { - return EnsureIndex(LastIndex()); - } - - TValue& back() { - return EnsureIndex(LastIndex()); - } - - void pop_back() { - if (empty()) - return; - TParent::pop_back(); - } - - void pop_front() { - if (empty()) - return; - TParent::pop_front(); - } - - private: - size_t LastIndex() const { - return ::Max<size_t>(size(), 1) - 1; - } - - TValue& EnsureIndex(size_t i) { - if (i >= size()) - resize(::Min<size_t>(i + 1, ::Max<ui16>()), TValue::DefaultValue()); - return TParent::operator[](i); - } - - const TValue& EnsureIndex(size_t i) const { + + template <class TIt> + void AppendAll(TIt begin, TIt end) { + TParent::insert(TParent::end(), begin, end); + } + + template <class TColl> + void AppendAll(TColl&& coll) { + AppendAll(std::begin(coll), std::end(coll)); + } + + void AppendAll(std::initializer_list<TValue> coll) { + AppendAll(coll.begin(), coll.end()); + } + + const TValue& operator[](size_t i) const { + return EnsureIndex(i); + } + + TValue& operator[](size_t i) { + return EnsureIndex(i); + } + + const TValue& front() const { + return EnsureIndex(0); + } + + TValue& front() { + return EnsureIndex(0); + } + + const TValue& back() const { + return EnsureIndex(LastIndex()); + } + + TValue& back() { + return EnsureIndex(LastIndex()); + } + + void pop_back() { + if (empty()) + return; + TParent::pop_back(); + } + + void pop_front() { + if (empty()) + return; + TParent::pop_front(); + } + + private: + size_t LastIndex() const { + return ::Max<size_t>(size(), 1) - 1; + } + + TValue& EnsureIndex(size_t i) { + if (i >= size()) + resize(::Min<size_t>(i + 1, ::Max<ui16>()), TValue::DefaultValue()); + return TParent::operator[](i); + } + + const TValue& EnsureIndex(size_t i) const { return i < size() ? TParent::operator[](i) : TValue::DefaultValue(); - } - }; - - - // todo: densehashtable - // todo: allow insertions - // todo: make TDict methods safe + } + }; + + + // todo: densehashtable + // todo: allow insertions + // todo: make TDict methods safe class TDict: public THashMap<TStringBuf, TValue, THash<TStringBuf>, TEqualTo<TStringBuf>, TPoolAllocator>, TNonCopyable { using TParent = THashMap<TStringBuf, TValue, THash<TStringBuf>, TEqualTo<TStringBuf>, TPoolAllocator>; - public: - TDict(TMemoryPool* p) - : TParent(p) + public: + TDict(TMemoryPool* p) + : TParent(p) { } - - template <class TStr> - const TValue& Get(const TStr& key) const { - const_iterator it = find(key); - return it != end() ? it->second : TValue::DefaultValue(); - } - }; -} - -#include "scimpl.h" + + template <class TStr> + const TValue& Get(const TStr& key) const { + const_iterator it = find(key); + return it != end() ? it->second : TValue::DefaultValue(); + } + }; +} + +#include "scimpl.h" #include "scheme_cast.h" #ifdef _MSC_VER diff --git a/library/cpp/scheme/scimpl.h b/library/cpp/scheme/scimpl.h index 076d743793..4f68f16290 100644 --- a/library/cpp/scheme/scimpl.h +++ b/library/cpp/scheme/scimpl.h @@ -1,95 +1,95 @@ -#pragma once - -#include "scheme.h" - -#include <util/stream/output.h> - -namespace NSc { - struct TValue::TScCore : TAtomicRefCount<TScCore, TDestructor>, TNonCopyable { +#pragma once + +#include "scheme.h" + +#include <util/stream/output.h> + +namespace NSc { + struct TValue::TScCore : TAtomicRefCount<TScCore, TDestructor>, TNonCopyable { TPoolPtr Pool; - double FloatNumber = 0; - i64 IntNumber = 0; - TStringBuf String; - TDict Dict; - TArray Array; + double FloatNumber = 0; + i64 IntNumber = 0; + TStringBuf String; + TDict Dict; + TArray Array; TValue::EType ValueType = TValue::EType::Null; - - TScCore(TPoolPtr& p) + + TScCore(TPoolPtr& p) : Pool(p) , Dict(Pool->Get()) , Array(Pool->Get()) { } - - bool IsNull() const { + + bool IsNull() const { return TValue::EType::Null == ValueType; - } - - bool IsBool() const { + } + + bool IsBool() const { return TValue::EType::Bool == ValueType; - } - - bool IsIntNumber() const { + } + + bool IsIntNumber() const { return TValue::EType::IntNumber == ValueType || IsBool(); - } - - bool IsNumber() const { + } + + bool IsNumber() const { return TValue::EType::FloatNumber == ValueType || IsIntNumber(); - } - - bool IsString() const { + } + + bool IsString() const { return TValue::EType::String == ValueType; - } - - bool IsArray() const { + } + + bool IsArray() const { return TValue::EType::Array == ValueType; - } - - bool IsDict() const { + } + + bool IsDict() const { return TValue::EType::Dict == ValueType; - } - - bool HasChildren() const { - return GetDict().size() + GetArray().size(); - } - - void SetNull() { + } + + bool HasChildren() const { + return GetDict().size() + GetArray().size(); + } + + void SetNull() { ValueType = TValue::EType::Null; - } - - void SetArray() { - if (Y_LIKELY(IsArray())) { - return; - } - + } + + void SetArray() { + if (Y_LIKELY(IsArray())) { + return; + } + ValueType = TValue::EType::Array; - Array.clear(); - } - - void SetDict() { - if (Y_LIKELY(IsDict())) { - return; - } - + Array.clear(); + } + + void SetDict() { + if (Y_LIKELY(IsDict())) { + return; + } + ValueType = TValue::EType::Dict; - Dict.clear(); - } - - void SetNumber(double n) { + Dict.clear(); + } + + void SetNumber(double n) { ValueType = TValue::EType::FloatNumber; - FloatNumber = n; - } - - void SetIntNumber(i64 n) { + FloatNumber = n; + } + + void SetIntNumber(i64 n) { ValueType = TValue::EType::IntNumber; - IntNumber = n; - } - - void SetBool(bool b) { + IntNumber = n; + } + + void SetBool(bool b) { ValueType = TValue::EType::Bool; - IntNumber = b; - } - + IntNumber = b; + } + void SetString(TStringBuf s) { SetOwnedString(Pool->AppendBuf(s)); } @@ -97,10 +97,10 @@ namespace NSc { void SetOwnedString(TStringBuf s) { ValueType = TValue::EType::String; String = s; - } - - double& GetNumberMutable(double defaultnum) { - switch (ValueType) { + } + + double& GetNumberMutable(double defaultnum) { + switch (ValueType) { case TValue::EType::Bool: SetNumber(bool(IntNumber)); break; @@ -112,13 +112,13 @@ namespace NSc { default: SetNumber(defaultnum); break; - } - - return FloatNumber; - } - - i64& GetIntNumberMutable(i64 defaultnum) { - switch (ValueType) { + } + + return FloatNumber; + } + + i64& GetIntNumberMutable(i64 defaultnum) { + switch (ValueType) { case TValue::EType::Bool: SetIntNumber(bool(IntNumber)); break; @@ -130,29 +130,29 @@ namespace NSc { default: SetIntNumber(defaultnum); break; - } - - return IntNumber; - } - - void ClearArray() { - if (!IsArray()) { - return; - } - - Array.clear(); - } - - void ClearDict() { - if (!IsDict()) { - return; - } - - Dict.clear(); - } - - double GetNumber(double d = 0) const { - switch (ValueType) { + } + + return IntNumber; + } + + void ClearArray() { + if (!IsArray()) { + return; + } + + Array.clear(); + } + + void ClearDict() { + if (!IsDict()) { + return; + } + + Dict.clear(); + } + + double GetNumber(double d = 0) const { + switch (ValueType) { case TValue::EType::Bool: return (bool)IntNumber; case TValue::EType::IntNumber: @@ -161,11 +161,11 @@ namespace NSc { return FloatNumber; default: return d; - } - } - - i64 GetIntNumber(i64 n = 0) const { - switch (ValueType) { + } + } + + i64 GetIntNumber(i64 n = 0) const { + switch (ValueType) { case TValue::EType::Bool: return (bool)IntNumber; case TValue::EType::IntNumber: @@ -174,615 +174,615 @@ namespace NSc { return FloatNumber; default: return n; - } - } - - bool GetBool(bool b = false) const { - return GetIntNumber(b); - } - - TStringBuf GetString(TStringBuf s = TStringBuf()) const { - return IsString() ? String : s; - } - - const TArray& GetArray() const { - return IsArray() ? Array : TValue::DefaultArray(); - } - - TArray& GetArrayMutable() { - SetArray(); - return Array; - } - + } + } + + bool GetBool(bool b = false) const { + return GetIntNumber(b); + } + + TStringBuf GetString(TStringBuf s = TStringBuf()) const { + return IsString() ? String : s; + } + + const TArray& GetArray() const { + return IsArray() ? Array : TValue::DefaultArray(); + } + + TArray& GetArrayMutable() { + SetArray(); + return Array; + } + TDict& GetDictMutable() { SetDict(); return Dict; } - const TDict& GetDict() const { - return IsDict() ? Dict : TValue::DefaultDict(); - } - - static void DoPush(TPoolPtr& p, TArray& a) { - a.push_back(TValue(p)); - a.back().CopyOnWrite = false; - } - + const TDict& GetDict() const { + return IsDict() ? Dict : TValue::DefaultDict(); + } + + static void DoPush(TPoolPtr& p, TArray& a) { + a.push_back(TValue(p)); + a.back().CopyOnWrite = false; + } + TValue& Push() { - SetArray(); + SetArray(); DoPush(Pool, Array); - return Array.back(); - } - - TValue Pop() { - if (!IsArray() || Array.empty()) { - return TValue::DefaultValue(); - } - - TValue v = Array.back(); - Array.pop_back(); - return v; - } - - const TValue& Get(size_t key) const { - return IsArray() && Array.size() > key ? Array[key] : TValue::DefaultValue(); - } - + return Array.back(); + } + + TValue Pop() { + if (!IsArray() || Array.empty()) { + return TValue::DefaultValue(); + } + + TValue v = Array.back(); + Array.pop_back(); + return v; + } + + const TValue& Get(size_t key) const { + return IsArray() && Array.size() > key ? Array[key] : TValue::DefaultValue(); + } + TValue* GetNoAdd(size_t key) { return IsArray() && Array.size() > key ? &Array[key] : nullptr; - } - + } + TValue& GetOrAdd(size_t key) { - SetArray(); - for (size_t i = Array.size(); i <= key; ++i) { + SetArray(); + for (size_t i = Array.size(); i <= key; ++i) { DoPush(Pool, Array); - } - - return Array[key]; - } - + } + + return Array[key]; + } + TValue& Back() { - SetArray(); - - if (Array.empty()) { + SetArray(); + + if (Array.empty()) { DoPush(Pool, Array); - } - - return Array.back(); - } - + } + + return Array.back(); + } + TValue& Insert(size_t key) { - SetArray(); - - if (Array.size() <= key) { + SetArray(); + + if (Array.size() <= key) { return GetOrAdd(key); - } else { + } else { Array.insert(Array.begin() + key, TValue(Pool)); - Array[key].CopyOnWrite = false; - return Array[key]; - } - } - - TValue Delete(size_t key) { - if (!IsArray() || Array.size() <= key) { - return TValue::DefaultValue(); - } - - TValue v = Array[key]; - Array.erase(Array.begin() + key); - return v; - } - - const TValue& Get(TStringBuf key) const { - if (!IsDict()) { - return TValue::DefaultValue(); - } - - TDict::const_iterator it = Dict.find(key); - return it != Dict.end() ? it->second : TValue::DefaultValue(); + Array[key].CopyOnWrite = false; + return Array[key]; + } + } + + TValue Delete(size_t key) { + if (!IsArray() || Array.size() <= key) { + return TValue::DefaultValue(); + } + + TValue v = Array[key]; + Array.erase(Array.begin() + key); + return v; + } + + const TValue& Get(TStringBuf key) const { + if (!IsDict()) { + return TValue::DefaultValue(); + } + + TDict::const_iterator it = Dict.find(key); + return it != Dict.end() ? it->second : TValue::DefaultValue(); } TValue* GetNoAdd(TStringBuf key) { - if (!IsDict()) { + if (!IsDict()) { return nullptr; - } + } return Dict.FindPtr(key); - } - + } + TValue& Add(TStringBuf key) { - SetDict(); + SetDict(); TDict::iterator it = Dict.insert(std::make_pair(Pool->AppendBuf(key), TValue(Pool))).first; - it->second.CopyOnWrite = false; - return it->second; - } - + it->second.CopyOnWrite = false; + return it->second; + } + TValue& GetOrAdd(TStringBuf key) { - SetDict(); - TDict::insert_ctx ctx; - TDict::iterator it = Dict.find(key, ctx); - - if (it == Dict.end()) { + SetDict(); + TDict::insert_ctx ctx; + TDict::iterator it = Dict.find(key, ctx); + + if (it == Dict.end()) { it = Dict.insert_direct(std::make_pair(Pool->AppendBuf(key), TValue(Pool)), ctx); - it->second.CopyOnWrite = false; - } - - return it->second; - } - - TValue Delete(TStringBuf key) { - if (!IsDict()) { - return TValue::DefaultValue(); - } - - TDict::iterator it = Dict.find(key); - - if (it == Dict.end()) { - return TValue::DefaultValue(); - } - - TValue v = it->second; - Dict.erase(key); - return v; - } - }; - - TValue::TScCore* TValue::NewCore(TPoolPtr& p) { - return new (p->Pool.Allocate<TScCore>()) TScCore(p); - } - + it->second.CopyOnWrite = false; + } + + return it->second; + } + + TValue Delete(TStringBuf key) { + if (!IsDict()) { + return TValue::DefaultValue(); + } + + TDict::iterator it = Dict.find(key); + + if (it == Dict.end()) { + return TValue::DefaultValue(); + } + + TValue v = it->second; + Dict.erase(key); + return v; + } + }; + + TValue::TScCore* TValue::NewCore(TPoolPtr& p) { + return new (p->Pool.Allocate<TScCore>()) TScCore(p); + } + TValue::TValue() { auto p = TPoolPtr(new NDefinitions::TPool); TheCore = NewCore(p); } - TValue::TValue(double t) + TValue::TValue(double t) : TValue() - { - SetNumber(t); - } - - TValue::TValue(unsigned long long t) + { + SetNumber(t); + } + + TValue::TValue(unsigned long long t) : TValue() - { - SetIntNumber(t); - } - - TValue::TValue(unsigned long t) + { + SetIntNumber(t); + } + + TValue::TValue(unsigned long t) : TValue() - { - SetIntNumber(t); - } - - TValue::TValue(unsigned t) + { + SetIntNumber(t); + } + + TValue::TValue(unsigned t) : TValue() - { - SetIntNumber(t); - } - - TValue::TValue(long long t) + { + SetIntNumber(t); + } + + TValue::TValue(long long t) : TValue() - { - SetIntNumber(t); - } - - TValue::TValue(long t) + { + SetIntNumber(t); + } + + TValue::TValue(long t) : TValue() - { - SetIntNumber(t); - } - - TValue::TValue(int t) + { + SetIntNumber(t); + } + + TValue::TValue(int t) : TValue() - { - SetIntNumber(t); - } - - //TValue::TValue(bool t) - // : TValue() - //{ - // SetBool(t); - //} - - TValue::TValue(TStringBuf t) + { + SetIntNumber(t); + } + + //TValue::TValue(bool t) + // : TValue() + //{ + // SetBool(t); + //} + + TValue::TValue(TStringBuf t) : TValue() - { - SetString(t); - } - - TValue::TValue(const char* t) + { + SetString(t); + } + + TValue::TValue(const char* t) : TValue() - { - SetString(t); - } - - TValue::TValue(TValue& v) + { + SetString(t); + } + + TValue::TValue(TValue& v) : TheCore(v.TheCore) , CopyOnWrite(v.CopyOnWrite) { } - - TValue::TValue(const TValue& v) + + TValue::TValue(const TValue& v) : TheCore(v.TheCore) - , CopyOnWrite(true) + , CopyOnWrite(true) { } - - TValue::TValue(TValue&& v) noexcept + + TValue::TValue(TValue&& v) noexcept : TheCore(std::move(v.TheCore)) - , CopyOnWrite(v.CopyOnWrite) + , CopyOnWrite(v.CopyOnWrite) {} - - TValue::operator double() const { - return GetNumber(); - } - - TValue::operator float() const { - return GetNumber(); - } - - TValue::operator long long() const { - return GetIntNumber(); - } - - TValue::operator long() const { - return GetIntNumber(); - } - - TValue::operator int() const { - return GetIntNumber(); - } - - TValue::operator short() const { - return GetIntNumber(); - } - - TValue::operator char() const { - return GetIntNumber(); - } - - TValue::operator unsigned long long() const { - return GetIntNumber(); - } - - TValue::operator unsigned long() const { - return GetIntNumber(); - } - - TValue::operator unsigned() const { - return GetIntNumber(); - } - - TValue::operator unsigned short() const { - return GetIntNumber(); - } - - TValue::operator unsigned char() const { - return GetIntNumber(); - } - - TValue::operator signed char() const { - return GetIntNumber(); - } - - TValue::operator TStringBuf() const { - return GetString(); - } - - TValue::operator const ::NSc::TArray&() const { - return GetArray(); - } - - TValue::operator const ::NSc::TDict&() const { - return GetDict(); - } - - TValue& TValue::operator=(double t) { - return SetNumber(t); - } - - TValue& TValue::operator=(unsigned long long t) { - return SetIntNumber(t); - } - - TValue& TValue::operator=(unsigned long t) { - return SetIntNumber(t); - } - - TValue& TValue::operator=(unsigned t) { - return SetIntNumber(t); - } - - TValue& TValue::operator=(long long t) { - return SetIntNumber(t); - } - - TValue& TValue::operator=(long t) { - return SetIntNumber(t); - } - - TValue& TValue::operator=(int t) { - return SetIntNumber(t); - } - - //TValue& TValue::operator=(bool t) { - // return SetBool(t); - //} - - TValue& TValue::operator=(TStringBuf t) { - return SetString(t); - } - - TValue& TValue::operator=(const char* t) { - return SetString(t); - } - + + TValue::operator double() const { + return GetNumber(); + } + + TValue::operator float() const { + return GetNumber(); + } + + TValue::operator long long() const { + return GetIntNumber(); + } + + TValue::operator long() const { + return GetIntNumber(); + } + + TValue::operator int() const { + return GetIntNumber(); + } + + TValue::operator short() const { + return GetIntNumber(); + } + + TValue::operator char() const { + return GetIntNumber(); + } + + TValue::operator unsigned long long() const { + return GetIntNumber(); + } + + TValue::operator unsigned long() const { + return GetIntNumber(); + } + + TValue::operator unsigned() const { + return GetIntNumber(); + } + + TValue::operator unsigned short() const { + return GetIntNumber(); + } + + TValue::operator unsigned char() const { + return GetIntNumber(); + } + + TValue::operator signed char() const { + return GetIntNumber(); + } + + TValue::operator TStringBuf() const { + return GetString(); + } + + TValue::operator const ::NSc::TArray&() const { + return GetArray(); + } + + TValue::operator const ::NSc::TDict&() const { + return GetDict(); + } + + TValue& TValue::operator=(double t) { + return SetNumber(t); + } + + TValue& TValue::operator=(unsigned long long t) { + return SetIntNumber(t); + } + + TValue& TValue::operator=(unsigned long t) { + return SetIntNumber(t); + } + + TValue& TValue::operator=(unsigned t) { + return SetIntNumber(t); + } + + TValue& TValue::operator=(long long t) { + return SetIntNumber(t); + } + + TValue& TValue::operator=(long t) { + return SetIntNumber(t); + } + + TValue& TValue::operator=(int t) { + return SetIntNumber(t); + } + + //TValue& TValue::operator=(bool t) { + // return SetBool(t); + //} + + TValue& TValue::operator=(TStringBuf t) { + return SetString(t); + } + + TValue& TValue::operator=(const char* t) { + return SetString(t); + } + TValue& TValue::operator=(TValue& v) & { if (!Same(*this, v)) { - //Extend TheCore lifetime not to trigger possible v deletion via parent-child chain - auto tmpCore = TheCore; - TheCore = v.TheCore; + //Extend TheCore lifetime not to trigger possible v deletion via parent-child chain + auto tmpCore = TheCore; + TheCore = v.TheCore; CopyOnWrite = v.CopyOnWrite; - } - return *this; - } - + } + return *this; + } + TValue& TValue::operator=(const TValue& v) & { - if (!Same(*this, v)) { + if (!Same(*this, v)) { //Extend TheCore lifetime not to trigger possible v deletion via parent-child chain auto tmpCore = TheCore; - TheCore = v.TheCore; - CopyOnWrite = true; - } - return *this; - } - + TheCore = v.TheCore; + CopyOnWrite = true; + } + return *this; + } + TValue& TValue::operator=(TValue&& v) & noexcept { - if (!Same(*this, v)) { + if (!Same(*this, v)) { //Extend TheCore lifetime not to trigger possible v deletion via parent-child chain auto tmpCore = TheCore; - TheCore = std::move(v.TheCore); - CopyOnWrite = v.CopyOnWrite; - } - return *this; - } - - bool TValue::Has(size_t idx) const { - return IsArray() && GetArray().size() > idx; - } - - bool TValue::Has(TStringBuf s) const { + TheCore = std::move(v.TheCore); + CopyOnWrite = v.CopyOnWrite; + } + return *this; + } + + bool TValue::Has(size_t idx) const { + return IsArray() && GetArray().size() > idx; + } + + bool TValue::Has(TStringBuf s) const { return GetDict().contains(s); - } - - TValue& TValue::GetOrAddUnsafe(size_t idx) { + } + + TValue& TValue::GetOrAddUnsafe(size_t idx) { return CoreMutable().GetOrAdd(idx); - } - - TValue& TValue::GetOrAdd(TStringBuf idx) { + } + + TValue& TValue::GetOrAdd(TStringBuf idx) { return CoreMutable().GetOrAdd(idx); - } - - const TValue& TValue::Get(size_t idx) const { - return Core().Get(idx); - } - + } + + const TValue& TValue::Get(size_t idx) const { + return Core().Get(idx); + } + TValue* TValue::GetNoAdd(size_t idx) { - return CoreMutable().GetNoAdd(idx); - } - - const TValue& TValue::Get(TStringBuf idx) const { - return Core().Get(idx); - } - + return CoreMutable().GetNoAdd(idx); + } + + const TValue& TValue::Get(TStringBuf idx) const { + return Core().Get(idx); + } + TValue* TValue::GetNoAdd(TStringBuf key) { - return CoreMutable().GetNoAdd(key); - } - - TValue& TValue::Back() { + return CoreMutable().GetNoAdd(key); + } + + TValue& TValue::Back() { return CoreMutable().Back(); - } - - const TValue& TValue::Back() const { - const TArray& arr = GetArray(); - return arr.empty() ? DefaultValue() : arr.back(); - } - - TValue TValue::Delete(size_t idx) { - return CoreMutable().Delete(idx); - } - - TValue TValue::Delete(TStringBuf idx) { - return CoreMutable().Delete(idx); - } - - TValue& TValue::AddAll(std::initializer_list<std::pair<TStringBuf, TValue>> t) { - for (const auto& el : t) { - Add(el.first) = el.second; - } - return *this; - } - - TValue& TValue::InsertUnsafe(size_t idx) { + } + + const TValue& TValue::Back() const { + const TArray& arr = GetArray(); + return arr.empty() ? DefaultValue() : arr.back(); + } + + TValue TValue::Delete(size_t idx) { + return CoreMutable().Delete(idx); + } + + TValue TValue::Delete(TStringBuf idx) { + return CoreMutable().Delete(idx); + } + + TValue& TValue::AddAll(std::initializer_list<std::pair<TStringBuf, TValue>> t) { + for (const auto& el : t) { + Add(el.first) = el.second; + } + return *this; + } + + TValue& TValue::InsertUnsafe(size_t idx) { return CoreMutable().Insert(idx); - } - - template <class TIt> - TValue& TValue::AppendAll(TIt begin, TIt end) { + } + + template <class TIt> + TValue& TValue::AppendAll(TIt begin, TIt end) { GetArrayMutable().AppendAll(begin, end); return *this; - } - - template <class TColl> - TValue& TValue::AppendAll(TColl&& coll) { + } + + template <class TColl> + TValue& TValue::AppendAll(TColl&& coll) { return AppendAll(std::begin(coll), std::end(coll)); - } - - TValue& TValue::AppendAll(std::initializer_list<TValue> coll) { - return AppendAll(coll.begin(), coll.end()); - } - - TValue& TValue::Push() { + } + + TValue& TValue::AppendAll(std::initializer_list<TValue> coll) { + return AppendAll(coll.begin(), coll.end()); + } + + TValue& TValue::Push() { return CoreMutable().Push(); - } - - TValue TValue::Pop() { - return CoreMutable().Pop(); - } - - TValue::EType TValue::GetType() const { - return Core().ValueType; - } - - bool TValue::IsNull() const { - return Core().IsNull(); - } - - bool TValue::IsNumber() const { - return Core().IsNumber(); - } - - bool TValue::IsIntNumber() const { - return Core().IsIntNumber(); - } - - bool TValue::IsBool() const { - return Core().IsBool(); - } - - bool TValue::IsString() const { - return Core().IsString(); - } - - bool TValue::IsArray() const { - return Core().IsArray(); - } - - bool TValue::IsDict() const { - return Core().IsDict(); - } - - TValue& TValue::SetNumber(double i) { + } + + TValue TValue::Pop() { + return CoreMutable().Pop(); + } + + TValue::EType TValue::GetType() const { + return Core().ValueType; + } + + bool TValue::IsNull() const { + return Core().IsNull(); + } + + bool TValue::IsNumber() const { + return Core().IsNumber(); + } + + bool TValue::IsIntNumber() const { + return Core().IsIntNumber(); + } + + bool TValue::IsBool() const { + return Core().IsBool(); + } + + bool TValue::IsString() const { + return Core().IsString(); + } + + bool TValue::IsArray() const { + return Core().IsArray(); + } + + bool TValue::IsDict() const { + return Core().IsDict(); + } + + TValue& TValue::SetNumber(double i) { CoreMutableForSet().SetNumber(i); - return *this; - } - - TValue& TValue::SetIntNumber(i64 n) { + return *this; + } + + TValue& TValue::SetIntNumber(i64 n) { CoreMutableForSet().SetIntNumber(n); - return *this; - } - - TValue& TValue::SetBool(bool val) { + return *this; + } + + TValue& TValue::SetBool(bool val) { CoreMutableForSet().SetBool(val); - return *this; - } - - TValue& TValue::SetString(TStringBuf s) { + return *this; + } + + TValue& TValue::SetString(TStringBuf s) { CoreMutableForSet().SetString(s); - return *this; - } - - double TValue::GetNumber(double d) const { - return Core().GetNumber(d); - } - - i64 TValue::GetIntNumber(i64 n) const { - return Core().GetIntNumber(n); - } - - bool TValue::GetBool(bool b) const { - return Core().GetBool(b); - } - - double& TValue::GetNumberMutable(double defaultval) { - return CoreMutable().GetNumberMutable(defaultval); - } - - i64& TValue::GetIntNumberMutable(i64 defaultval) { - return CoreMutable().GetIntNumberMutable(defaultval); - } - - TStringBuf TValue::GetString(TStringBuf d) const { - return Core().GetString(d); - } - - TValue& TValue::SetArray() { - CoreMutable().SetArray(); - return *this; - } - - TValue& TValue::ClearArray() { - CoreMutable().ClearArray(); - return *this; - } - - TValue& TValue::SetDict() { - CoreMutable().SetDict(); - return *this; - } - - TValue& TValue::ClearDict() { - CoreMutable().ClearDict(); - return *this; - } - - const TArray& TValue::GetArray() const { - return Core().GetArray(); - } - - TArray& TValue::GetArrayMutable() { - return CoreMutable().GetArrayMutable(); - } - - const TDict& TValue::GetDict() const { - return Core().GetDict(); - } - + return *this; + } + + double TValue::GetNumber(double d) const { + return Core().GetNumber(d); + } + + i64 TValue::GetIntNumber(i64 n) const { + return Core().GetIntNumber(n); + } + + bool TValue::GetBool(bool b) const { + return Core().GetBool(b); + } + + double& TValue::GetNumberMutable(double defaultval) { + return CoreMutable().GetNumberMutable(defaultval); + } + + i64& TValue::GetIntNumberMutable(i64 defaultval) { + return CoreMutable().GetIntNumberMutable(defaultval); + } + + TStringBuf TValue::GetString(TStringBuf d) const { + return Core().GetString(d); + } + + TValue& TValue::SetArray() { + CoreMutable().SetArray(); + return *this; + } + + TValue& TValue::ClearArray() { + CoreMutable().ClearArray(); + return *this; + } + + TValue& TValue::SetDict() { + CoreMutable().SetDict(); + return *this; + } + + TValue& TValue::ClearDict() { + CoreMutable().ClearDict(); + return *this; + } + + const TArray& TValue::GetArray() const { + return Core().GetArray(); + } + + TArray& TValue::GetArrayMutable() { + return CoreMutable().GetArrayMutable(); + } + + const TDict& TValue::GetDict() const { + return Core().GetDict(); + } + TDict& TValue::GetDictMutable() { return CoreMutable().GetDictMutable(); } - size_t TValue::StringSize() const { - return GetString().size(); - } - - size_t TValue::ArraySize() const { - return GetArray().size(); - } - - size_t TValue::DictSize() const { - return GetDict().size(); - } - - bool TValue::StringEmpty() const { - return GetString().empty(); - } - - bool TValue::ArrayEmpty() const { - return GetArray().empty(); - } - - bool TValue::DictEmpty() const { - return GetDict().empty(); - } - - TValue::TValue(TPoolPtr& p) + size_t TValue::StringSize() const { + return GetString().size(); + } + + size_t TValue::ArraySize() const { + return GetArray().size(); + } + + size_t TValue::DictSize() const { + return GetDict().size(); + } + + bool TValue::StringEmpty() const { + return GetString().empty(); + } + + bool TValue::ArrayEmpty() const { + return GetArray().empty(); + } + + bool TValue::DictEmpty() const { + return GetDict().empty(); + } + + TValue::TValue(TPoolPtr& p) : TheCore(NewCore(p)) { - } - - TValue::TScCore& TValue::CoreMutable() { - if (Y_UNLIKELY(!TheCore)) { + } + + TValue::TScCore& TValue::CoreMutable() { + if (Y_UNLIKELY(!TheCore)) { *this = TValue(); - } else if (Y_UNLIKELY(CopyOnWrite) && Y_UNLIKELY(TheCore->RefCount() > 1)) { - *this = Clone(); - } - - CopyOnWrite = false; - - return *TheCore; - } - + } else if (Y_UNLIKELY(CopyOnWrite) && Y_UNLIKELY(TheCore->RefCount() > 1)) { + *this = Clone(); + } + + CopyOnWrite = false; + + return *TheCore; + } + TValue::TScCore& TValue::CoreMutableForSet() { if (Y_UNLIKELY(!TheCore) || Y_UNLIKELY(CopyOnWrite) && Y_UNLIKELY(TheCore->RefCount() > 1)) { *this = TValue(); @@ -793,33 +793,33 @@ namespace NSc { return *TheCore; } - const TValue::TScCore& TValue::Core() const { - return TheCore ? *TheCore : DefaultCore(); - } - - TValue& TValue::SetNull() { + const TValue::TScCore& TValue::Core() const { + return TheCore ? *TheCore : DefaultCore(); + } + + TValue& TValue::SetNull() { CoreMutableForSet().SetNull(); - return *this; - } - - namespace NPrivate { - int CompareStr(const NSc::TValue& a, TStringBuf b); - - int CompareInt(const NSc::TValue& a, i64 r); - - int CompareFloat(const NSc::TValue& a, double r); - } - + return *this; + } + + namespace NPrivate { + int CompareStr(const NSc::TValue& a, TStringBuf b); + + int CompareInt(const NSc::TValue& a, i64 r); + + int CompareFloat(const NSc::TValue& a, double r); + } + bool operator==(const TValue& a, const TValue& b); - + bool operator!=(const TValue& a, const TValue& b); - + bool operator<=(const TValue&, const TValue&) = delete; bool operator>=(const TValue&, const TValue&) = delete; bool operator<(const TValue&, const TValue&) = delete; bool operator>(const TValue&, const TValue&) = delete; - -#define LIBRARY_SCHEME_DECLARE_TVALUE_OPS(T, Impl) \ + +#define LIBRARY_SCHEME_DECLARE_TVALUE_OPS(T, Impl) \ bool operator==(const NSc::TValue& a, T b); \ bool operator==(T b, const NSc::TValue& a); \ bool operator!=(const NSc::TValue& a, T b); \ @@ -832,27 +832,27 @@ namespace NSc { bool operator<(T b, const NSc::TValue& a); \ bool operator>(const NSc::TValue& a, T b); \ bool operator>(T b, const NSc::TValue& a); - + #define LIBRARY_SCHEME_DECLARE_TVALUE_INT_OPS(T) \ - LIBRARY_SCHEME_DECLARE_TVALUE_OPS(signed T, CompareInt) \ - LIBRARY_SCHEME_DECLARE_TVALUE_OPS(unsigned T, CompareInt) - - //LIBRARY_SCHEME_DECLARE_TVALUE_OPS(bool, CompareInt) - LIBRARY_SCHEME_DECLARE_TVALUE_OPS(char, CompareInt) - LIBRARY_SCHEME_DECLARE_TVALUE_INT_OPS(char) - LIBRARY_SCHEME_DECLARE_TVALUE_INT_OPS(short) - LIBRARY_SCHEME_DECLARE_TVALUE_INT_OPS(int) - LIBRARY_SCHEME_DECLARE_TVALUE_INT_OPS(long) - LIBRARY_SCHEME_DECLARE_TVALUE_INT_OPS(long long) - - LIBRARY_SCHEME_DECLARE_TVALUE_OPS(float, CompareFloat) - LIBRARY_SCHEME_DECLARE_TVALUE_OPS(double, CompareFloat) - - LIBRARY_SCHEME_DECLARE_TVALUE_OPS(TStringBuf, CompareStr) + LIBRARY_SCHEME_DECLARE_TVALUE_OPS(signed T, CompareInt) \ + LIBRARY_SCHEME_DECLARE_TVALUE_OPS(unsigned T, CompareInt) + + //LIBRARY_SCHEME_DECLARE_TVALUE_OPS(bool, CompareInt) + LIBRARY_SCHEME_DECLARE_TVALUE_OPS(char, CompareInt) + LIBRARY_SCHEME_DECLARE_TVALUE_INT_OPS(char) + LIBRARY_SCHEME_DECLARE_TVALUE_INT_OPS(short) + LIBRARY_SCHEME_DECLARE_TVALUE_INT_OPS(int) + LIBRARY_SCHEME_DECLARE_TVALUE_INT_OPS(long) + LIBRARY_SCHEME_DECLARE_TVALUE_INT_OPS(long long) + + LIBRARY_SCHEME_DECLARE_TVALUE_OPS(float, CompareFloat) + LIBRARY_SCHEME_DECLARE_TVALUE_OPS(double, CompareFloat) + + LIBRARY_SCHEME_DECLARE_TVALUE_OPS(TStringBuf, CompareStr) LIBRARY_SCHEME_DECLARE_TVALUE_OPS(const TString&, CompareStr) - LIBRARY_SCHEME_DECLARE_TVALUE_OPS(const char* const, CompareStr) - -#undef LIBRARY_SCHEME_DECLARE_TVALUE_OPS -#undef LIBRARY_SCHEME_DECLARE_TVALUE_INT_OPS - -} + LIBRARY_SCHEME_DECLARE_TVALUE_OPS(const char* const, CompareStr) + +#undef LIBRARY_SCHEME_DECLARE_TVALUE_OPS +#undef LIBRARY_SCHEME_DECLARE_TVALUE_INT_OPS + +} diff --git a/library/cpp/scheme/scimpl_defs.h b/library/cpp/scheme/scimpl_defs.h index 4720cd35cd..f3dd66b437 100644 --- a/library/cpp/scheme/scimpl_defs.h +++ b/library/cpp/scheme/scimpl_defs.h @@ -1,122 +1,122 @@ -#pragma once - +#pragma once + #include <library/cpp/json/json_reader.h> #include <library/cpp/json/json_value.h> - -#include <util/system/types.h> -#include <util/memory/pool.h> - -#include <util/generic/deque.h> -#include <util/generic/hash.h> -#include <util/generic/ptr.h> -#include <util/generic/strbuf.h> + +#include <util/system/types.h> +#include <util/memory/pool.h> + +#include <util/generic/deque.h> +#include <util/generic/hash.h> +#include <util/generic/ptr.h> +#include <util/generic/strbuf.h> #include <util/generic/string.h> -#include <util/generic/vector.h> +#include <util/generic/vector.h> #include <functional> #include <util/string/vector.h> -#include <util/string/type.h> - -namespace NSc { - namespace NDefinitions { - const size_t POOL_BLOCK_SIZE = 4000; // leave 96 bytes for overhead - +#include <util/string/type.h> + +namespace NSc { + namespace NDefinitions { + const size_t POOL_BLOCK_SIZE = 4000; // leave 96 bytes for overhead + struct TPool: public TAtomicRefCount<TPool> { - TMemoryPool Pool; - - TPool(size_t blsz = POOL_BLOCK_SIZE, TMemoryPool::IGrowPolicy* grow = TMemoryPool::TExpGrow::Instance()) - : Pool(blsz, grow) + TMemoryPool Pool; + + TPool(size_t blsz = POOL_BLOCK_SIZE, TMemoryPool::IGrowPolicy* grow = TMemoryPool::TExpGrow::Instance()) + : Pool(blsz, grow) { } - + TMemoryPool* Get() { return &Pool; } - - TStringBuf AppendBuf(const TStringBuf& strb) { - return Pool.AppendCString(strb); - } - - template <typename T> - T* NewWithPool() { - return new (Pool.Allocate<T>()) T(&Pool); - } - }; - } - + + TStringBuf AppendBuf(const TStringBuf& strb) { + return Pool.AppendCString(strb); + } + + template <typename T> + T* NewWithPool() { + return new (Pool.Allocate<T>()) T(&Pool); + } + }; + } + using TStringBufs = TVector<TStringBuf>; - - class TSchemeException : public yexception { - }; - - class TSchemeParseException : public TSchemeException { - public: - size_t Offset = 0; + + class TSchemeException : public yexception { + }; + + class TSchemeParseException : public TSchemeException { + public: + size_t Offset = 0; TString Reason; - - public: - TSchemeParseException() = default; - + + public: + TSchemeParseException() = default; + TSchemeParseException(size_t off, const TString& reason) - : Offset(off) - , Reason(reason) + : Offset(off) + , Reason(reason) { } - }; - + }; + struct TJsonOpts: public NJson::TJsonReaderConfig { - enum EJsonOpts { - JO_DEFAULT = 0, // just dump json, used to be default, actually - JO_SORT_KEYS = 1, // sort dict keys to make output more predictable - JO_SKIP_UNSAFE = 2, // skip nonunicode data to make external json parsers happy - // will skip nonunicode dict keys and replace nonunicode values with nulls - JO_FORMAT = 8, // format json - + enum EJsonOpts { + JO_DEFAULT = 0, // just dump json, used to be default, actually + JO_SORT_KEYS = 1, // sort dict keys to make output more predictable + JO_SKIP_UNSAFE = 2, // skip nonunicode data to make external json parsers happy + // will skip nonunicode dict keys and replace nonunicode values with nulls + JO_FORMAT = 8, // format json + JO_PARSER_STRICT_JSON = 16, // strict standard json JO_PARSER_STRICT_UTF8 = 32, // strict utf8 - JO_PARSER_DISALLOW_COMMENTS = 64, + JO_PARSER_DISALLOW_COMMENTS = 64, JO_PARSER_DISALLOW_DUPLICATE_KEYS = 128, - - JO_PRETTY = JO_FORMAT | JO_SORT_KEYS, // pretty print json - JO_SAFE = JO_SKIP_UNSAFE | JO_SORT_KEYS, // ensure standard parser-safe json - - JO_PARSER_STRICT_WITH_COMMENTS = JO_PARSER_STRICT_JSON | JO_PARSER_STRICT_UTF8, - JO_PARSER_STRICT = JO_PARSER_STRICT_JSON | JO_PARSER_STRICT_UTF8 | JO_PARSER_DISALLOW_COMMENTS, - }; - - public: - TJsonOpts(int opts = JO_SORT_KEYS) + + JO_PRETTY = JO_FORMAT | JO_SORT_KEYS, // pretty print json + JO_SAFE = JO_SKIP_UNSAFE | JO_SORT_KEYS, // ensure standard parser-safe json + + JO_PARSER_STRICT_WITH_COMMENTS = JO_PARSER_STRICT_JSON | JO_PARSER_STRICT_UTF8, + JO_PARSER_STRICT = JO_PARSER_STRICT_JSON | JO_PARSER_STRICT_UTF8 | JO_PARSER_DISALLOW_COMMENTS, + }; + + public: + TJsonOpts(int opts = JO_SORT_KEYS) : Opts(opts) , SortKeys(opts & JO_SORT_KEYS) - , FormatJson(opts & JO_FORMAT) - , StringPolicy((opts & JO_SKIP_UNSAFE) ? StringPolicySafe : StringPolicyUnsafe) - { - AllowComments = !(opts & JO_PARSER_DISALLOW_COMMENTS); - RelaxedJson = !(opts & JO_PARSER_STRICT_JSON); - DontValidateUtf8 = !(opts & JO_PARSER_STRICT_UTF8); - } - - public: + , FormatJson(opts & JO_FORMAT) + , StringPolicy((opts & JO_SKIP_UNSAFE) ? StringPolicySafe : StringPolicyUnsafe) + { + AllowComments = !(opts & JO_PARSER_DISALLOW_COMMENTS); + RelaxedJson = !(opts & JO_PARSER_STRICT_JSON); + DontValidateUtf8 = !(opts & JO_PARSER_STRICT_UTF8); + } + + public: bool RelaxedJson = false; int Opts = 0; - bool SortKeys = true; - bool FormatJson = false; - - // return true to proceed with output, false to skip, optionally modify value - std::function<bool(double&)> NumberPolicy = NumberPolicySafe; - std::function<bool(TStringBuf&)> StringPolicy = StringPolicyUnsafe; - - public: - static bool NumberPolicySafe(double&); // skip if nan or inf + bool SortKeys = true; + bool FormatJson = false; + + // return true to proceed with output, false to skip, optionally modify value + std::function<bool(double&)> NumberPolicy = NumberPolicySafe; + std::function<bool(TStringBuf&)> StringPolicy = StringPolicyUnsafe; + + public: + static bool NumberPolicySafe(double&); // skip if nan or inf static bool NumberPolicyUnsafe(double&) { return true; } - - static bool StringPolicySafe(TStringBuf&); // skip if not utf8 + + static bool StringPolicySafe(TStringBuf&); // skip if not utf8 static bool StringPolicyUnsafe(TStringBuf&) { return true; } - }; - + }; + struct TProtoOpts { // Serialization throws on unknown enum value if not set, else use default value bool UnknownEnumValueIsDefault = false; @@ -125,16 +125,16 @@ namespace NSc { bool SkipTypeMismatch = false; }; - namespace NImpl { - class TKeySortContext; - class TSelfLoopContext; - class TSelfOverrideContext; - } -} + namespace NImpl { + class TKeySortContext; + class TSelfLoopContext; + class TSelfOverrideContext; + } +} namespace google { - namespace protobuf { - class Message; - class FieldDescriptor; - } + namespace protobuf { + class Message; + class FieldDescriptor; + } } diff --git a/library/cpp/scheme/scimpl_json_read.cpp b/library/cpp/scheme/scimpl_json_read.cpp index 64e6cc8d59..8a29cc7739 100644 --- a/library/cpp/scheme/scimpl_json_read.cpp +++ b/library/cpp/scheme/scimpl_json_read.cpp @@ -1,229 +1,229 @@ -#include "scimpl.h" - +#include "scimpl.h" + #include <library/cpp/json/json_reader.h> -#include <util/stream/output.h> -#include <util/generic/maybe.h> - -namespace NSc { - struct TJsonError { - size_t Offset = 0; +#include <util/stream/output.h> +#include <util/generic/maybe.h> + +namespace NSc { + struct TJsonError { + size_t Offset = 0; TMaybe<TString> Reason; - }; - - struct TJsonDeserializer : NJson::TJsonCallbacks { - struct TContainer { - TValue* Container = nullptr; - TValue* LastValue = nullptr; - bool ExpectKey = false; - - TContainer(TValue& v) - : Container(&v) - , ExpectKey(v.IsDict()) + }; + + struct TJsonDeserializer : NJson::TJsonCallbacks { + struct TContainer { + TValue* Container = nullptr; + TValue* LastValue = nullptr; + bool ExpectKey = false; + + TContainer(TValue& v) + : Container(&v) + , ExpectKey(v.IsDict()) { } - + bool Add(TStringBuf v, bool allowDuplicated) { if (!ExpectKey || Y_UNLIKELY(!Container->IsDict())) - return false; - + return false; + if (!allowDuplicated && Y_UNLIKELY(Container->Has(v))) return false; - LastValue = &Container->GetOrAdd(v); - ExpectKey = false; - return true; - } - - void Push() { - LastValue = &Container->Push(); - } - - TValue& NextValue() { - if (Container->IsArray()) { - Push(); - } else if (Container->IsDict()) { - ExpectKey = true; - } - - return *(LastValue ? LastValue : Container); - } - - bool IsArray() const { - return !!Container && Container->IsArray(); - } - - bool IsDict() const { - return !!Container && Container->IsDict(); - } - }; - + LastValue = &Container->GetOrAdd(v); + ExpectKey = false; + return true; + } + + void Push() { + LastValue = &Container->Push(); + } + + TValue& NextValue() { + if (Container->IsArray()) { + Push(); + } else if (Container->IsDict()) { + ExpectKey = true; + } + + return *(LastValue ? LastValue : Container); + } + + bool IsArray() const { + return !!Container && Container->IsArray(); + } + + bool IsDict() const { + return !!Container && Container->IsDict(); + } + }; + typedef TVector<TContainer> TStackType; - - public: - TValue& Root; - TJsonError& Error; + + public: + TValue& Root; + TJsonError& Error; const TJsonOpts& Cfg; - + TStackType Stack; - bool Virgin = true; - - public: + bool Virgin = true; + + public: TJsonDeserializer(TValue& root, TJsonError& err, const TJsonOpts& cfg) - : Root(root) - , Error(err) + : Root(root) + , Error(err) , Cfg(cfg) - { - Root.SetNull(); - Stack.reserve(10); - } - - bool HasNextValue() const { - return Virgin | !Stack.empty(); - } - - TValue& NextValue() { - Virgin = false; - return Stack.empty() ? Root : Stack.back().NextValue(); - } - - bool OnNull() override { - if (Y_UNLIKELY(!HasNextValue())) - return false; - - NextValue().SetNull(); - return true; - } - - bool OnEnd() override { - return Stack.empty(); - } - - template <typename T> - bool OnValue(T v) { - if (Y_UNLIKELY(!HasNextValue())) - return false; - - NextValue() = v; - return true; - } - - template <typename T> - bool OnIntValue(T v) { - if (Y_UNLIKELY(!HasNextValue())) - return false; - - NextValue().SetIntNumber(v); - return true; - } - - bool OnBoolean(bool v) override { - if (Y_UNLIKELY(!HasNextValue())) - return false; - - NextValue().SetBool(v); - return true; - } - - bool OnInteger(long long v) override { - return OnIntValue(v); - } - - bool OnUInteger(unsigned long long v) override { - return OnIntValue(v); - } - - bool OnDouble(double v) override { - return OnValue(v); - } - - bool OnString(const TStringBuf& v) override { - return OnValue(v); - } - - bool OnMapKey(const TStringBuf& k) override { - if (Y_UNLIKELY(Stack.empty())) - return false; + { + Root.SetNull(); + Stack.reserve(10); + } + + bool HasNextValue() const { + return Virgin | !Stack.empty(); + } + + TValue& NextValue() { + Virgin = false; + return Stack.empty() ? Root : Stack.back().NextValue(); + } + + bool OnNull() override { + if (Y_UNLIKELY(!HasNextValue())) + return false; + + NextValue().SetNull(); + return true; + } + + bool OnEnd() override { + return Stack.empty(); + } + + template <typename T> + bool OnValue(T v) { + if (Y_UNLIKELY(!HasNextValue())) + return false; + + NextValue() = v; + return true; + } + + template <typename T> + bool OnIntValue(T v) { + if (Y_UNLIKELY(!HasNextValue())) + return false; + + NextValue().SetIntNumber(v); + return true; + } + + bool OnBoolean(bool v) override { + if (Y_UNLIKELY(!HasNextValue())) + return false; + + NextValue().SetBool(v); + return true; + } + + bool OnInteger(long long v) override { + return OnIntValue(v); + } + + bool OnUInteger(unsigned long long v) override { + return OnIntValue(v); + } + + bool OnDouble(double v) override { + return OnValue(v); + } + + bool OnString(const TStringBuf& v) override { + return OnValue(v); + } + + bool OnMapKey(const TStringBuf& k) override { + if (Y_UNLIKELY(Stack.empty())) + return false; return Stack.back().Add(k, !(Cfg.Opts & TJsonOpts::JO_PARSER_DISALLOW_DUPLICATE_KEYS)); - } - - bool OnOpenMap() override { - if (Y_UNLIKELY(!HasNextValue())) - return false; - Stack.push_back(TContainer(NextValue().SetDict())); - return true; - } - - bool OnCloseMap() override { - if (Y_UNLIKELY(Stack.empty() || !Stack.back().IsDict())) - return false; - Stack.pop_back(); - return true; - } - - bool OnOpenArray() override { - if (Y_UNLIKELY(!HasNextValue())) - return false; - Stack.push_back(TContainer(NextValue().SetArray())); - return true; - } - - bool OnCloseArray() override { - if (Y_UNLIKELY(Stack.empty() || !Stack.back().IsArray())) - return false; - Stack.pop_back(); - return true; - } - - void OnError(size_t off, TStringBuf reason) override { - Error.Offset = off; + } + + bool OnOpenMap() override { + if (Y_UNLIKELY(!HasNextValue())) + return false; + Stack.push_back(TContainer(NextValue().SetDict())); + return true; + } + + bool OnCloseMap() override { + if (Y_UNLIKELY(Stack.empty() || !Stack.back().IsDict())) + return false; + Stack.pop_back(); + return true; + } + + bool OnOpenArray() override { + if (Y_UNLIKELY(!HasNextValue())) + return false; + Stack.push_back(TContainer(NextValue().SetArray())); + return true; + } + + bool OnCloseArray() override { + if (Y_UNLIKELY(Stack.empty() || !Stack.back().IsArray())) + return false; + Stack.pop_back(); + return true; + } + + void OnError(size_t off, TStringBuf reason) override { + Error.Offset = off; Error.Reason = reason; - } - }; - + } + }; + static bool DoParseFromJson(TValue& res, TJsonError& err, TStringBuf json, const TJsonOpts& cfg) { TJsonDeserializer d(res, err, cfg); - - if (cfg.RelaxedJson) { - return NJson::ReadJsonFast(json, &d); - } else { + + if (cfg.RelaxedJson) { + return NJson::ReadJsonFast(json, &d); + } else { TMemoryInput min(json.data(), json.size()); - return NJson::ReadJson(&min, &cfg, &d); - } - } - + return NJson::ReadJson(&min, &cfg, &d); + } + } + static bool DoParseFromJson(TValue& res, TStringBuf json, const TJsonOpts& cfg) { - TJsonError err; - return DoParseFromJson(res, err, json, cfg); - } - - TValue TValue::FromJson(TStringBuf v, const TJsonOpts& cfg) { - TValue res; - if (FromJson(res, v, cfg)) { - return res; - } else { - return DefaultValue(); - } - } - - TValue TValue::FromJsonThrow(TStringBuf json, const TJsonOpts& cfg) { - TValue res; - TJsonError err; - - if (DoParseFromJson(res, err, json, cfg)) { - return res; - } - + TJsonError err; + return DoParseFromJson(res, err, json, cfg); + } + + TValue TValue::FromJson(TStringBuf v, const TJsonOpts& cfg) { + TValue res; + if (FromJson(res, v, cfg)) { + return res; + } else { + return DefaultValue(); + } + } + + TValue TValue::FromJsonThrow(TStringBuf json, const TJsonOpts& cfg) { + TValue res; + TJsonError err; + + if (DoParseFromJson(res, err, json, cfg)) { + return res; + } + TString reason = err.Reason.Empty() ? "NULL" : *err.Reason; - ythrow TSchemeParseException(err.Offset, reason) << "JSON error at offset " << err.Offset << " (" << reason << ")"; - } - - bool TValue::FromJson(TValue& res, TStringBuf json, const TJsonOpts& cfg) { - if (DoParseFromJson(res, json, cfg)) { - return true; - } - - res.SetNull(); - return false; - } - -} + ythrow TSchemeParseException(err.Offset, reason) << "JSON error at offset " << err.Offset << " (" << reason << ")"; + } + + bool TValue::FromJson(TValue& res, TStringBuf json, const TJsonOpts& cfg) { + if (DoParseFromJson(res, json, cfg)) { + return true; + } + + res.SetNull(); + return false; + } + +} diff --git a/library/cpp/scheme/scimpl_json_write.cpp b/library/cpp/scheme/scimpl_json_write.cpp index 541fb8d2cb..aadd7e6cd5 100644 --- a/library/cpp/scheme/scimpl_json_write.cpp +++ b/library/cpp/scheme/scimpl_json_write.cpp @@ -1,88 +1,88 @@ -#include "scimpl.h" -#include "scimpl_private.h" - +#include "scimpl.h" +#include "scimpl_private.h" + #include <library/cpp/json/json_prettifier.h> #include <library/cpp/string_utils/relaxed_escaper/relaxed_escaper.h> - + #include <util/charset/utf8.h> -#include <util/generic/algorithm.h> -#include <util/generic/ymath.h> -#include <util/system/tls.h> - -namespace NSc { - bool TJsonOpts::StringPolicySafe(TStringBuf& s) { - return IsUtf(s); - } - - bool TJsonOpts::NumberPolicySafe(double& d) { - return IsFinite(d); - } - +#include <util/generic/algorithm.h> +#include <util/generic/ymath.h> +#include <util/system/tls.h> + +namespace NSc { + bool TJsonOpts::StringPolicySafe(TStringBuf& s) { + return IsUtf(s); + } + + bool TJsonOpts::NumberPolicySafe(double& d) { + return IsFinite(d); + } + static inline void WriteString(IOutputStream& out, TStringBuf s) { - NEscJ::EscapeJ<true, true>(s, out); - } - - static inline const NSc::TValue& GetValue(size_t, TStringBuf key, const TDict& dict) { - return dict.find(key)->second; - } - - static inline const NSc::TValue& GetValue(TDict::const_iterator it, TStringBuf, const TDict&) { - return it->second; - } - - static inline TStringBuf GetKey(size_t it, const NImpl::TKeySortContext::TGuard& keys) { - return keys.GetVector()[it]; - } - - static inline TStringBuf GetKey(TDict::const_iterator it, const TDict&) { - return it->first; - } - - template <typename TDictKeys> + NEscJ::EscapeJ<true, true>(s, out); + } + + static inline const NSc::TValue& GetValue(size_t, TStringBuf key, const TDict& dict) { + return dict.find(key)->second; + } + + static inline const NSc::TValue& GetValue(TDict::const_iterator it, TStringBuf, const TDict&) { + return it->second; + } + + static inline TStringBuf GetKey(size_t it, const NImpl::TKeySortContext::TGuard& keys) { + return keys.GetVector()[it]; + } + + static inline TStringBuf GetKey(TDict::const_iterator it, const TDict&) { + return it->first; + } + + template <typename TDictKeys> static inline void WriteDict(IOutputStream& out, const TDictKeys& keys, const TDict& dict, const TJsonOpts& jopts, NImpl::TKeySortContext& sortCtx, NImpl::TSelfLoopContext& loopCtx) { - using const_iterator = typename TDictKeys::const_iterator; - const_iterator begin = keys.begin(); - const_iterator end = keys.end(); - for (const_iterator it = begin; it != end; ++it) { - TStringBuf key = GetKey(it, keys); - - if (jopts.StringPolicy && !jopts.StringPolicy(key)) { - ++begin; - continue; - } - - if (it != begin) { - out << ','; - } - - NEscJ::EscapeJ<true, true>(key, out); - out << ':'; - - GetValue(it, key, dict).DoWriteJsonImpl(out, jopts, sortCtx, loopCtx); - } - } - + using const_iterator = typename TDictKeys::const_iterator; + const_iterator begin = keys.begin(); + const_iterator end = keys.end(); + for (const_iterator it = begin; it != end; ++it) { + TStringBuf key = GetKey(it, keys); + + if (jopts.StringPolicy && !jopts.StringPolicy(key)) { + ++begin; + continue; + } + + if (it != begin) { + out << ','; + } + + NEscJ::EscapeJ<true, true>(key, out); + out << ':'; + + GetValue(it, key, dict).DoWriteJsonImpl(out, jopts, sortCtx, loopCtx); + } + } + void TValue::DoWriteJsonImpl(IOutputStream& out, const TJsonOpts& jopts, NImpl::TKeySortContext& sortCtx, NImpl::TSelfLoopContext& loopCtx) const { - const TScCore& core = Core(); - - NImpl::TSelfLoopContext::TGuard loopCheck(loopCtx, core); - - if (!loopCheck.Ok) { + const TScCore& core = Core(); + + NImpl::TSelfLoopContext::TGuard loopCheck(loopCtx, core); + + if (!loopCheck.Ok) { out << TStringBuf("null"); // a loop encountered (and asserted), skip the back reference - return; - } - - switch (core.ValueType) { + return; + } + + switch (core.ValueType) { default: { Y_ASSERT(false); [[fallthrough]]; /* no break */ - } + } case EType::Null: { out << TStringBuf("null"); break; - } + } case EType::Bool: { out << (core.IntNumber ? TStringBuf("true") : TStringBuf("false")); break; @@ -97,7 +97,7 @@ namespace NSc { out << d; } else { out << TStringBuf("null"); - } + } break; } case EType::String: { @@ -116,17 +116,17 @@ namespace NSc { if (it != a.begin()) { out << ','; } - + it->DoWriteJsonImpl(out, jopts, sortCtx, loopCtx); } out << ']'; break; - } + } case EType::Dict: { out << '{'; - + const TDict& dict = core.GetDict(); - + if (jopts.SortKeys) { NImpl::TKeySortContext::TGuard keys(sortCtx, dict); WriteDict(out, keys, dict, jopts, sortCtx, loopCtx); @@ -136,58 +136,58 @@ namespace NSc { out << '}'; break; - } - } - } - + } + } + } + const TValue& TValue::ToJson(IOutputStream& out, const TJsonOpts& jopts) const { - using namespace NImpl; - - if (jopts.FormatJson) { - TStringStream str; - DoWriteJsonImpl(str, jopts, GetTlsInstance<TKeySortContext>(), GetTlsInstance<TSelfLoopContext>()); - NJson::PrettifyJson(str.Str(), out); - } else { - DoWriteJsonImpl(out, jopts, GetTlsInstance<TKeySortContext>(), GetTlsInstance<TSelfLoopContext>()); - } - - return *this; - } - + using namespace NImpl; + + if (jopts.FormatJson) { + TStringStream str; + DoWriteJsonImpl(str, jopts, GetTlsInstance<TKeySortContext>(), GetTlsInstance<TSelfLoopContext>()); + NJson::PrettifyJson(str.Str(), out); + } else { + DoWriteJsonImpl(out, jopts, GetTlsInstance<TKeySortContext>(), GetTlsInstance<TSelfLoopContext>()); + } + + return *this; + } + TString TValue::ToJson(const TJsonOpts& jopts) const { TString s; - { - TStringOutput out(s); - ToJson(out, jopts); - } - return s; - } - - TJsonOpts TValue::MakeOptsSafeForSerializer(TJsonOpts opts) { - opts.SortKeys = true; - opts.StringPolicy = TJsonOpts::StringPolicySafe; - opts.NumberPolicy = TJsonOpts::NumberPolicySafe; - return opts; - } - - TJsonOpts TValue::MakeOptsPrettyForSerializer(TJsonOpts opts) { - opts.FormatJson = true; - return MakeOptsSafeForSerializer(opts); - } - + { + TStringOutput out(s); + ToJson(out, jopts); + } + return s; + } + + TJsonOpts TValue::MakeOptsSafeForSerializer(TJsonOpts opts) { + opts.SortKeys = true; + opts.StringPolicy = TJsonOpts::StringPolicySafe; + opts.NumberPolicy = TJsonOpts::NumberPolicySafe; + return opts; + } + + TJsonOpts TValue::MakeOptsPrettyForSerializer(TJsonOpts opts) { + opts.FormatJson = true; + return MakeOptsSafeForSerializer(opts); + } + TString TValue::ToJsonSafe(const TJsonOpts& jopts) const { - return ToJson(MakeOptsSafeForSerializer(jopts)); - } - + return ToJson(MakeOptsSafeForSerializer(jopts)); + } + const TValue& TValue::ToJsonSafe(IOutputStream& out, const TJsonOpts& jopts) const { - return ToJson(out, MakeOptsSafeForSerializer(jopts)); - } - - TString TValue::ToJsonPretty(const TJsonOpts& jopts) const { - return ToJson(MakeOptsPrettyForSerializer(jopts)); - } - + return ToJson(out, MakeOptsSafeForSerializer(jopts)); + } + + TString TValue::ToJsonPretty(const TJsonOpts& jopts) const { + return ToJson(MakeOptsPrettyForSerializer(jopts)); + } + const TValue& TValue::ToJsonPretty(IOutputStream& out, const TJsonOpts& jopts) const { - return ToJson(out, MakeOptsPrettyForSerializer(jopts)); - } -} + return ToJson(out, MakeOptsPrettyForSerializer(jopts)); + } +} diff --git a/library/cpp/scheme/scimpl_private.cpp b/library/cpp/scheme/scimpl_private.cpp index c2c1e6b6f3..024bf8cc3b 100644 --- a/library/cpp/scheme/scimpl_private.cpp +++ b/library/cpp/scheme/scimpl_private.cpp @@ -1,74 +1,74 @@ -#include "scimpl_private.h" - -#include <util/generic/algorithm.h> +#include "scimpl_private.h" + +#include <util/generic/algorithm.h> #include <utility> - -namespace NSc { - namespace NImpl { - struct TGetKey { - static inline TStringBuf Do(const TDict::value_type& v) { - return v.first; - } - }; - - struct TMoveValue { - static inline TValue&& Do(TDict::value_type& v) { - return std::move(v.second); - } - - static inline TValue&& Do(TArray::value_type& v) { - return std::move(v); - } - }; - - template <typename TAction, typename TElement, typename TColl> + +namespace NSc { + namespace NImpl { + struct TGetKey { + static inline TStringBuf Do(const TDict::value_type& v) { + return v.first; + } + }; + + struct TMoveValue { + static inline TValue&& Do(TDict::value_type& v) { + return std::move(v.second); + } + + static inline TValue&& Do(TArray::value_type& v) { + return std::move(v); + } + }; + + template <typename TAction, typename TElement, typename TColl> static inline void PutToVector(TVector<TElement>& vector, TColl& coll) { - size_t i = vector.size(); - vector.resize(vector.size() + coll.size()); - - for (auto& item : coll) { - vector[i++] = TAction::Do(item); - } - } - - bool TKeySortContext::Process(const TDict& self) { - size_t oldSz = Vector.size(); - PutToVector<TGetKey>(Vector, self); - Sort(Vector.begin() + oldSz, Vector.end()); - return true; - } - - bool TSelfOverrideContext::Process(TValue::TScCore& self) { - if (self.GetDict().size()) { - PutToVector<TMoveValue>(Vector, self.Dict); - } else if (self.GetArray().size()) { - PutToVector<TMoveValue>(Vector, self.Array); - } - return true; - } - - bool TSelfLoopContext::Process(const TValue::TScCore& self) { - const bool ok = (Vector.end() == Find(Vector.begin(), Vector.end(), &self)); - - if (!ok) { - switch (ReportingMode) { - case EMode::Assert: - Y_ASSERT(false); // make sure the debug build sees this - break; - case EMode::Throw: - ythrow TSchemeException() << "REFERENCE LOOP DETECTED"; - case EMode::Abort: - Y_FAIL("REFERENCE LOOP DETECTED"); - break; - case EMode::Stderr: + size_t i = vector.size(); + vector.resize(vector.size() + coll.size()); + + for (auto& item : coll) { + vector[i++] = TAction::Do(item); + } + } + + bool TKeySortContext::Process(const TDict& self) { + size_t oldSz = Vector.size(); + PutToVector<TGetKey>(Vector, self); + Sort(Vector.begin() + oldSz, Vector.end()); + return true; + } + + bool TSelfOverrideContext::Process(TValue::TScCore& self) { + if (self.GetDict().size()) { + PutToVector<TMoveValue>(Vector, self.Dict); + } else if (self.GetArray().size()) { + PutToVector<TMoveValue>(Vector, self.Array); + } + return true; + } + + bool TSelfLoopContext::Process(const TValue::TScCore& self) { + const bool ok = (Vector.end() == Find(Vector.begin(), Vector.end(), &self)); + + if (!ok) { + switch (ReportingMode) { + case EMode::Assert: + Y_ASSERT(false); // make sure the debug build sees this + break; + case EMode::Throw: + ythrow TSchemeException() << "REFERENCE LOOP DETECTED"; + case EMode::Abort: + Y_FAIL("REFERENCE LOOP DETECTED"); + break; + case EMode::Stderr: Cerr << "REFERENCE LOOP DETECTED: " << JoinStrings(Vector.begin(), Vector.end(), ", ") << " AND " << ToString((const void*)&self) << Endl; - break; - } - } - - Vector.push_back(&self); - return ok; - } - } -} + break; + } + } + + Vector.push_back(&self); + return ok; + } + } +} diff --git a/library/cpp/scheme/scimpl_private.h b/library/cpp/scheme/scimpl_private.h index 3527e0911c..b92badabde 100644 --- a/library/cpp/scheme/scimpl_private.h +++ b/library/cpp/scheme/scimpl_private.h @@ -1,114 +1,114 @@ -#pragma once - -#include "scheme.h" - -#include <util/thread/singleton.h> - -namespace NSc { - namespace NImpl { - template <typename TContext> - static inline TContext& GetTlsInstance() { - return *FastTlsSingleton<TContext>(); - } - - template <typename TContext> - class TContextGuard : TNonCopyable { - using TElement = typename TContext::TElement; - using TTarget = typename TContext::TTarget; +#pragma once + +#include "scheme.h" + +#include <util/thread/singleton.h> + +namespace NSc { + namespace NImpl { + template <typename TContext> + static inline TContext& GetTlsInstance() { + return *FastTlsSingleton<TContext>(); + } + + template <typename TContext> + class TContextGuard : TNonCopyable { + using TElement = typename TContext::TElement; + using TTarget = typename TContext::TTarget; using TVectorType = TVector<TElement>; - public: - TContextGuard(TContext& ctx, TTarget& target) - : Ctx(ctx) - , Active(TContext::Needed(target)) - { - if (Active) { - Begin = Ctx.Vector.size(); - Ok = Ctx.Process(target); - End = Ctx.Vector.size(); - } - } - - ~TContextGuard() noexcept { - if (Active) { - Ctx.Vector.resize(Begin); - } - } - + public: + TContextGuard(TContext& ctx, TTarget& target) + : Ctx(ctx) + , Active(TContext::Needed(target)) + { + if (Active) { + Begin = Ctx.Vector.size(); + Ok = Ctx.Process(target); + End = Ctx.Vector.size(); + } + } + + ~TContextGuard() noexcept { + if (Active) { + Ctx.Vector.resize(Begin); + } + } + const TVectorType& GetVector() const { - return Ctx.Vector; - } - - using const_iterator = size_t; - - size_t begin() const { - return Begin; - } - - size_t end() const { - return End; - } - - bool Ok = true; - - private: - TContext& Ctx; - size_t Begin = 0; - size_t End = 0; - bool Active = false; - }; - - template <typename TElem, typename TTgt> - class TBasicContext { - public: - using TElement = TElem; - using TTarget = TTgt; - + return Ctx.Vector; + } + + using const_iterator = size_t; + + size_t begin() const { + return Begin; + } + + size_t end() const { + return End; + } + + bool Ok = true; + + private: + TContext& Ctx; + size_t Begin = 0; + size_t End = 0; + bool Active = false; + }; + + template <typename TElem, typename TTgt> + class TBasicContext { + public: + using TElement = TElem; + using TTarget = TTgt; + TBasicContext() { - Vector.reserve(64); - } - + Vector.reserve(64); + } + TVector<TElement> Vector; - }; - + }; + class TKeySortContext: public TBasicContext<TStringBuf, const TDict> { - public: - using TGuard = TContextGuard<TKeySortContext>; - - bool Process(const TDict& self); - - static bool Needed(const TDict& self) { - return self.size(); - } - }; - + public: + using TGuard = TContextGuard<TKeySortContext>; + + bool Process(const TDict& self); + + static bool Needed(const TDict& self) { + return self.size(); + } + }; + class TSelfOverrideContext: public TBasicContext<TValue, TValue::TScCore> { - public: - using TGuard = TContextGuard<TSelfOverrideContext>; - - bool Process(TValue::TScCore& self); - - static bool Needed(const TValue::TScCore& self) { - return self.HasChildren(); - } - }; - + public: + using TGuard = TContextGuard<TSelfOverrideContext>; + + bool Process(TValue::TScCore& self); + + static bool Needed(const TValue::TScCore& self) { + return self.HasChildren(); + } + }; + class TSelfLoopContext: public TBasicContext<const void*, const TValue::TScCore> { - public: - enum class EMode { - Assert, Throw, Abort, Stderr - }; - - using TGuard = TContextGuard<TSelfLoopContext>; - - bool Process(const TValue::TScCore& self); - - static bool Needed(const TValue::TScCore& self) { - return self.HasChildren(); - } - - public: - EMode ReportingMode = EMode::Assert; - }; - } -} + public: + enum class EMode { + Assert, Throw, Abort, Stderr + }; + + using TGuard = TContextGuard<TSelfLoopContext>; + + bool Process(const TValue::TScCore& self); + + static bool Needed(const TValue::TScCore& self) { + return self.HasChildren(); + } + + public: + EMode ReportingMode = EMode::Assert; + }; + } +} diff --git a/library/cpp/scheme/scimpl_protobuf.cpp b/library/cpp/scheme/scimpl_protobuf.cpp index 256541178d..0c99122c69 100644 --- a/library/cpp/scheme/scimpl_protobuf.cpp +++ b/library/cpp/scheme/scimpl_protobuf.cpp @@ -11,17 +11,17 @@ using namespace google::protobuf; namespace NSc { TValue TValue::From(const Message& msg, bool mapAsDict) { - TValue v; - const Reflection* r = msg.GetReflection(); + TValue v; + const Reflection* r = msg.GetReflection(); TVector<const FieldDescriptor*> fields; TVector<const FieldDescriptor*>::iterator it; int i1; - r->ListFields(msg, &fields); + r->ListFields(msg, &fields); for (it = fields.begin(), i1 = 0; it != fields.end(); ++it, ++i1) { - const FieldDescriptor* field = *it; - try { - if (field->is_repeated()) { + const FieldDescriptor* field = *it; + try { + if (field->is_repeated()) { if (field->is_map() && mapAsDict) { auto& elem = v[field->name()]; for (int i2 = 0; i2 < r->FieldSize(msg, field); ++i2) { @@ -34,22 +34,22 @@ namespace NSc { for (int i2 = 0; i2 < r->FieldSize(msg, field); ++i2) v[field->name()][i2] = FromRepeatedField(msg, field, i2); } - } else { - v[field->name()] = FromField(msg, field); - } - } catch (...) { - /* conversion failed, skip this field */ + } else { + v[field->name()] = FromField(msg, field); + } + } catch (...) { + /* conversion failed, skip this field */ } } - - return v; + + return v; } - TValue TValue::FromField(const Message& msg, const FieldDescriptor* field) { - TValue v; - const Reflection* r = msg.GetReflection(); + TValue TValue::FromField(const Message& msg, const FieldDescriptor* field) { + TValue v; + const Reflection* r = msg.GetReflection(); - switch (field->cpp_type()) { + switch (field->cpp_type()) { case FieldDescriptor::CPPTYPE_INT32: v = r->GetInt32(msg, field); break; @@ -82,16 +82,16 @@ namespace NSc { break; default: ythrow TSchemeException() << "field " << field->full_name() << " unexpected type " << (int)field->cpp_type(); - } + } - return v; + return v; } - TValue TValue::FromRepeatedField(const Message& msg, const FieldDescriptor* field, int index) { - TValue v; - const Reflection* r = msg.GetReflection(); + TValue TValue::FromRepeatedField(const Message& msg, const FieldDescriptor* field, int index) { + TValue v; + const Reflection* r = msg.GetReflection(); - switch (field->cpp_type()) { + switch (field->cpp_type()) { case FieldDescriptor::CPPTYPE_INT32: v = r->GetRepeatedInt32(msg, field, index); break; @@ -124,9 +124,9 @@ namespace NSc { break; default: ythrow TSchemeException() << "field " << field->full_name() << " unexpected type " << (int)field->cpp_type(); - } + } - return v; + return v; } void TValue::To(Message& msg, const TProtoOpts& opts) const { diff --git a/library/cpp/scheme/scimpl_select.rl6 b/library/cpp/scheme/scimpl_select.rl6 index d9518460c5..11aa549b78 100644 --- a/library/cpp/scheme/scimpl_select.rl6 +++ b/library/cpp/scheme/scimpl_select.rl6 @@ -1,270 +1,270 @@ #include <library/cpp/scheme/scimpl.h> - -#include <util/string/cast.h> -#include <util/string/escape.h> + +#include <util/string/cast.h> +#include <util/string/escape.h> #include <library/cpp/string_utils/relaxed_escaper/relaxed_escaper.h> -#include <util/generic/is_in.h> -#include <util/string/util.h> - -namespace NSc { - - template <typename TValue, typename TGetter> - struct TSelector { - TValue& Root; - TValue* Current = nullptr; - TValue* Parent = nullptr; - TStringBuf CurrentDictKey; - size_t CurrentArrayKey = 0; - size_t Depth = 0; - bool HasArray = false; - bool HasError = false; - - TSelector(TValue& root) - : Root(root) - , Current(&Root) - {} - - template <typename T> - bool Next(T k) { - Depth += 1; - Parent = Current; - Current = TGetter::Next(Current, k); - return Current != nullptr; - } - - bool NextDict(TStringBuf k) { - return Next(CurrentDictKey = k); - } - - bool NextArray(size_t k) { - HasArray = true; - return Next(CurrentArrayKey = k); - } - - bool Error() { - Parent = nullptr; - Current = nullptr; - CurrentArrayKey = 0; - CurrentDictKey = TStringBuf(); - HasError = true; - return false; - } - }; - - template <typename TSelector> - struct TSelectorCtx { - TSelector Selector; +#include <util/generic/is_in.h> +#include <util/string/util.h> + +namespace NSc { + + template <typename TValue, typename TGetter> + struct TSelector { + TValue& Root; + TValue* Current = nullptr; + TValue* Parent = nullptr; + TStringBuf CurrentDictKey; + size_t CurrentArrayKey = 0; + size_t Depth = 0; + bool HasArray = false; + bool HasError = false; + + TSelector(TValue& root) + : Root(root) + , Current(&Root) + {} + + template <typename T> + bool Next(T k) { + Depth += 1; + Parent = Current; + Current = TGetter::Next(Current, k); + return Current != nullptr; + } + + bool NextDict(TStringBuf k) { + return Next(CurrentDictKey = k); + } + + bool NextArray(size_t k) { + HasArray = true; + return Next(CurrentArrayKey = k); + } + + bool Error() { + Parent = nullptr; + Current = nullptr; + CurrentArrayKey = 0; + CurrentDictKey = TStringBuf(); + HasError = true; + return false; + } + }; + + template <typename TSelector> + struct TSelectorCtx { + TSelector Selector; TString Buffer; - - const char* p0 = nullptr; - const char* p = nullptr; - const char* pe = nullptr; - const char* eof = nullptr; - const char* ts = nullptr; - const char* te = nullptr; - int cs = 0; - int act = 0; - - TSelectorCtx(TSelector sel, TStringBuf data) - : Selector(sel) + + const char* p0 = nullptr; + const char* p = nullptr; + const char* pe = nullptr; + const char* eof = nullptr; + const char* ts = nullptr; + const char* te = nullptr; + int cs = 0; + int act = 0; + + TSelectorCtx(TSelector sel, TStringBuf data) + : Selector(sel) , p0(data.data()) , p(data.data()) - , pe(data.end()) - , eof(data.end()) - {} - - bool OnString(TStringBuf s) { - return Selector.NextDict(s); - } - - bool OnInt(size_t k) { - return Selector.NextArray(k); - } - - bool OnStrU() { - return OnString(TStringBuf(ts, te)); - } - - bool OnStrQ() { - return OnString(TStringBuf(ts + 1, te - 1)); - } - - bool OnStrE() { - Buffer.clear(); - Buffer.reserve(te - ts); - UnescapeC(ts + 1, te - ts - 2, Buffer); - return OnString(Buffer); - } - - bool OnIntU() { - return OnInt(FromString<ui32>(TStringBuf(ts, te))); - } - - bool OnIntQ() { - return OnInt(FromString<ui32>(TStringBuf(ts + 1, te - 1))); - } - - bool OnError() { - Selector.Error(); - return false; - } - - bool SelectPath(); - }; - -#if 0 - %%{ - machine schemeselect; - - alphtype char; - - action OnIntU { if (Y_UNLIKELY(!OnIntU())) goto TOKEN_ERROR; } - action OnIntQ { if (Y_UNLIKELY(!OnIntQ())) goto TOKEN_ERROR; } - action OnStrU { if (Y_UNLIKELY(!OnStrU())) goto TOKEN_ERROR; } - action OnStrQ { if (Y_UNLIKELY(!OnStrQ())) goto TOKEN_ERROR; } - action OnStrE { if (Y_UNLIKELY(!OnStrE())) goto TOKEN_ERROR; } - action OnError { goto TOKEN_ERROR; } - - intu = [0-9]+; - intq = '[' intu ']'; - - uchar0 = [a-zA-Z_@$] | (0x80 .. 0xFF); - uchar = uchar0 | digit | [.\-]; - - qchar = [^'\\]; #'; - dchar = [^"\\]; #"; - bchar = [^\]\\]; - - echar = "\\" any; - - qechar = qchar | echar; - dechar = dchar | echar; - bechar = bchar | echar; - - strq = "'" qchar* "'"; - strd = '"' dchar* '"'; - strb = '[' bchar* ']'; - - strqe = "'" qechar* "'"; - strde = '"' dechar* '"'; - strbe = '[' bechar* ']'; - - strU = uchar0 uchar*; - strQ = strq | strd | strb; - strE = strqe | strde | strbe; - - main := |* - intu => OnIntU; - intq => OnIntQ; - - strU => OnStrU; - strQ => OnStrQ; - strE => OnStrE; - - '/'; - - (intu) (any - ('/' | '[' )) => OnError; - - any => OnError; - *|; - }%% -#endif - - template <typename TSelector> - bool TSelectorCtx<TSelector>::SelectPath() { - try { - %%{ - write data noerror nofinal; - write init; - write exec; - }%% - ; - Y_UNUSED(schemeselect_en_main); - } catch (const TFromStringException&) { - return OnError(); - } - - return Selector.Current; - - TOKEN_ERROR: - return OnError(); - } - - template <bool CheckHas> - struct TGetNext { - template <typename TValue, typename TIdx> - static TValue* Next(TValue* val, TIdx idx) { - if (val) { - if (CheckHas && !val->Has(idx)) { - return nullptr; - } else { - return &(*val)[idx]; - } - } else { - return nullptr; - } - } - }; - - const TValue& TValue::TrySelect(TStringBuf path) const { - TSelectorCtx<TSelector<const TValue, TGetNext<true> > > ctx(*this, path); - - if (ctx.SelectPath()) { - return *ctx.Selector.Current; - } - - return DefaultValue(); - } - + , pe(data.end()) + , eof(data.end()) + {} + + bool OnString(TStringBuf s) { + return Selector.NextDict(s); + } + + bool OnInt(size_t k) { + return Selector.NextArray(k); + } + + bool OnStrU() { + return OnString(TStringBuf(ts, te)); + } + + bool OnStrQ() { + return OnString(TStringBuf(ts + 1, te - 1)); + } + + bool OnStrE() { + Buffer.clear(); + Buffer.reserve(te - ts); + UnescapeC(ts + 1, te - ts - 2, Buffer); + return OnString(Buffer); + } + + bool OnIntU() { + return OnInt(FromString<ui32>(TStringBuf(ts, te))); + } + + bool OnIntQ() { + return OnInt(FromString<ui32>(TStringBuf(ts + 1, te - 1))); + } + + bool OnError() { + Selector.Error(); + return false; + } + + bool SelectPath(); + }; + +#if 0 + %%{ + machine schemeselect; + + alphtype char; + + action OnIntU { if (Y_UNLIKELY(!OnIntU())) goto TOKEN_ERROR; } + action OnIntQ { if (Y_UNLIKELY(!OnIntQ())) goto TOKEN_ERROR; } + action OnStrU { if (Y_UNLIKELY(!OnStrU())) goto TOKEN_ERROR; } + action OnStrQ { if (Y_UNLIKELY(!OnStrQ())) goto TOKEN_ERROR; } + action OnStrE { if (Y_UNLIKELY(!OnStrE())) goto TOKEN_ERROR; } + action OnError { goto TOKEN_ERROR; } + + intu = [0-9]+; + intq = '[' intu ']'; + + uchar0 = [a-zA-Z_@$] | (0x80 .. 0xFF); + uchar = uchar0 | digit | [.\-]; + + qchar = [^'\\]; #'; + dchar = [^"\\]; #"; + bchar = [^\]\\]; + + echar = "\\" any; + + qechar = qchar | echar; + dechar = dchar | echar; + bechar = bchar | echar; + + strq = "'" qchar* "'"; + strd = '"' dchar* '"'; + strb = '[' bchar* ']'; + + strqe = "'" qechar* "'"; + strde = '"' dechar* '"'; + strbe = '[' bechar* ']'; + + strU = uchar0 uchar*; + strQ = strq | strd | strb; + strE = strqe | strde | strbe; + + main := |* + intu => OnIntU; + intq => OnIntQ; + + strU => OnStrU; + strQ => OnStrQ; + strE => OnStrE; + + '/'; + + (intu) (any - ('/' | '[' )) => OnError; + + any => OnError; + *|; + }%% +#endif + + template <typename TSelector> + bool TSelectorCtx<TSelector>::SelectPath() { + try { + %%{ + write data noerror nofinal; + write init; + write exec; + }%% + ; + Y_UNUSED(schemeselect_en_main); + } catch (const TFromStringException&) { + return OnError(); + } + + return Selector.Current; + + TOKEN_ERROR: + return OnError(); + } + + template <bool CheckHas> + struct TGetNext { + template <typename TValue, typename TIdx> + static TValue* Next(TValue* val, TIdx idx) { + if (val) { + if (CheckHas && !val->Has(idx)) { + return nullptr; + } else { + return &(*val)[idx]; + } + } else { + return nullptr; + } + } + }; + + const TValue& TValue::TrySelect(TStringBuf path) const { + TSelectorCtx<TSelector<const TValue, TGetNext<true> > > ctx(*this, path); + + if (ctx.SelectPath()) { + return *ctx.Selector.Current; + } + + return DefaultValue(); + } + TValue* TValue::TrySelectOrAdd(TStringBuf path) { - TSelectorCtx<TSelector<TValue, TGetNext<false> > > ctx(*this, path); - - if (ctx.SelectPath()) { + TSelectorCtx<TSelector<TValue, TGetNext<false> > > ctx(*this, path); + + if (ctx.SelectPath()) { return ctx.Selector.Current; - } else { + } else { return nullptr; - } - } - - TValue TValue::TrySelectAndDelete(TStringBuf path) { - TSelectorCtx<TSelector<TValue, TGetNext<true> > > ctx(*this, path); - - if (ctx.SelectPath() && ctx.Selector.Parent) { - if (ctx.Selector.Parent->IsArray()) { - return ctx.Selector.Parent->Delete(ctx.Selector.CurrentArrayKey); - } else if (ctx.Selector.Parent->IsDict()) { - return ctx.Selector.Parent->Delete(ctx.Selector.CurrentDictKey); - } else { - Y_ASSERT(false); - return DefaultValue(); - } - } else { - return DefaultValue(); - } - } - - bool TValue::PathExists(TStringBuf path) const { - return TSelectorCtx<TSelector<const TValue, TGetNext<true>>>(*this, path).SelectPath(); - } - - bool TValue::PathValid(TStringBuf path) { - TSelectorCtx<TSelector<const TValue, TGetNext<false>>> ctx(DefaultValue(), path); - return ctx.SelectPath() || !ctx.Selector.HasError; - } - + } + } + + TValue TValue::TrySelectAndDelete(TStringBuf path) { + TSelectorCtx<TSelector<TValue, TGetNext<true> > > ctx(*this, path); + + if (ctx.SelectPath() && ctx.Selector.Parent) { + if (ctx.Selector.Parent->IsArray()) { + return ctx.Selector.Parent->Delete(ctx.Selector.CurrentArrayKey); + } else if (ctx.Selector.Parent->IsDict()) { + return ctx.Selector.Parent->Delete(ctx.Selector.CurrentDictKey); + } else { + Y_ASSERT(false); + return DefaultValue(); + } + } else { + return DefaultValue(); + } + } + + bool TValue::PathExists(TStringBuf path) const { + return TSelectorCtx<TSelector<const TValue, TGetNext<true>>>(*this, path).SelectPath(); + } + + bool TValue::PathValid(TStringBuf path) { + TSelectorCtx<TSelector<const TValue, TGetNext<false>>> ctx(DefaultValue(), path); + return ctx.SelectPath() || !ctx.Selector.HasError; + } + TString TValue::EscapeForPath(TStringBuf rawKey) { - static const str_spn danger{"/[]"}; - if (!rawKey || danger.brk(rawKey.begin(), rawKey.end()) != rawKey.end()) { - return NEscJ::EscapeJ<true>(rawKey); - } - - TSelectorCtx<TSelector<const TValue, TGetNext<false>>> ctx(DefaultValue(), rawKey); - ctx.SelectPath(); - if (ctx.Selector.HasError || ctx.Selector.Depth > 1 || ctx.Selector.HasArray) { - return NEscJ::EscapeJ<true>(rawKey); - } else { + static const str_spn danger{"/[]"}; + if (!rawKey || danger.brk(rawKey.begin(), rawKey.end()) != rawKey.end()) { + return NEscJ::EscapeJ<true>(rawKey); + } + + TSelectorCtx<TSelector<const TValue, TGetNext<false>>> ctx(DefaultValue(), rawKey); + ctx.SelectPath(); + if (ctx.Selector.HasError || ctx.Selector.Depth > 1 || ctx.Selector.HasArray) { + return NEscJ::EscapeJ<true>(rawKey); + } else { return ToString(rawKey); - } - } - -} + } + } + +} diff --git a/library/cpp/scheme/tests/fuzz_json/fuzz_json.cpp b/library/cpp/scheme/tests/fuzz_json/fuzz_json.cpp index 5e93e027c5..8d4c0fa8a0 100644 --- a/library/cpp/scheme/tests/fuzz_json/fuzz_json.cpp +++ b/library/cpp/scheme/tests/fuzz_json/fuzz_json.cpp @@ -1,6 +1,6 @@ #include <library/cpp/scheme/tests/fuzz_json/lib/fuzz_json.h> - -extern "C" int LLVMFuzzerTestOneInput(const ui8* wireData, const size_t wireSize) { - NSc::NUt::FuzzJson({(const char*)wireData, wireSize}); - return 0; -} + +extern "C" int LLVMFuzzerTestOneInput(const ui8* wireData, const size_t wireSize) { + NSc::NUt::FuzzJson({(const char*)wireData, wireSize}); + return 0; +} diff --git a/library/cpp/scheme/tests/fuzz_json/lib/fuzz_json.cpp b/library/cpp/scheme/tests/fuzz_json/lib/fuzz_json.cpp index efaf3d28ef..7c16527c23 100644 --- a/library/cpp/scheme/tests/fuzz_json/lib/fuzz_json.cpp +++ b/library/cpp/scheme/tests/fuzz_json/lib/fuzz_json.cpp @@ -1,9 +1,9 @@ -#include "fuzz_json.h" +#include "fuzz_json.h" #include "util/generic/fwd.h" - + #include <library/cpp/scheme/scheme.h> -#include <util/stream/null.h> - +#include <util/stream/null.h> + namespace { static constexpr size_t MAX_DEPTH = 4; static constexpr size_t MAX_PATH_LEN = 256; @@ -75,25 +75,25 @@ namespace { } } -namespace NSc::NUt { +namespace NSc::NUt { - void FuzzJson(TStringBuf wire) { + void FuzzJson(TStringBuf wire) { if (wire.size() < 2) { - return; - } - + return; + } + ProcessPath("[123][1234][12][2134][12312312][1][12]"); - ui8 len1 = wire[0]; - ui8 len2 = wire[1]; - wire.Skip(2); - auto json1 = wire.NextTokAt(len1); - auto json2 = wire.NextTokAt(len2); - NSc::TValue val1 = NSc::TValue::FromJson(json1); - NSc::TValue val2 = NSc::TValue::FromJson(json2); - NSc::TValue val3; - val3.MergeUpdate(val1); + ui8 len1 = wire[0]; + ui8 len2 = wire[1]; + wire.Skip(2); + auto json1 = wire.NextTokAt(len1); + auto json2 = wire.NextTokAt(len2); + NSc::TValue val1 = NSc::TValue::FromJson(json1); + NSc::TValue val2 = NSc::TValue::FromJson(json2); + NSc::TValue val3; + val3.MergeUpdate(val1); size_t i = 0; while (!wire.empty()) { @@ -110,6 +110,6 @@ namespace NSc::NUt { val3 = NSc::TValue(); } } - Cnull << val3.ToJson(); - } -} + Cnull << val3.ToJson(); + } +} diff --git a/library/cpp/scheme/tests/fuzz_json/lib/fuzz_json.h b/library/cpp/scheme/tests/fuzz_json/lib/fuzz_json.h index f61a0c20ef..f8cf7a4770 100644 --- a/library/cpp/scheme/tests/fuzz_json/lib/fuzz_json.h +++ b/library/cpp/scheme/tests/fuzz_json/lib/fuzz_json.h @@ -1,7 +1,7 @@ -#pragma once - -#include <util/generic/strbuf.h> - -namespace NSc::NUt { - void FuzzJson(TStringBuf wire); -} +#pragma once + +#include <util/generic/strbuf.h> + +namespace NSc::NUt { + void FuzzJson(TStringBuf wire); +} diff --git a/library/cpp/scheme/tests/fuzz_json/lib/ya.make b/library/cpp/scheme/tests/fuzz_json/lib/ya.make index fb49e26b9d..b30a6c9350 100644 --- a/library/cpp/scheme/tests/fuzz_json/lib/ya.make +++ b/library/cpp/scheme/tests/fuzz_json/lib/ya.make @@ -1,18 +1,18 @@ -LIBRARY() - -OWNER( - g:blender - g:middle - g:upper - velavokr -) - -SRCS( - fuzz_json.cpp -) - -PEERDIR( +LIBRARY() + +OWNER( + g:blender + g:middle + g:upper + velavokr +) + +SRCS( + fuzz_json.cpp +) + +PEERDIR( library/cpp/scheme -) - -END() +) + +END() diff --git a/library/cpp/scheme/tests/fuzz_json/ya.make b/library/cpp/scheme/tests/fuzz_json/ya.make index 0b682d1b50..0d91c70585 100644 --- a/library/cpp/scheme/tests/fuzz_json/ya.make +++ b/library/cpp/scheme/tests/fuzz_json/ya.make @@ -1,20 +1,20 @@ -FUZZ() - -OWNER( - g:blender - g:middle - g:upper - velavokr -) - +FUZZ() + +OWNER( + g:blender + g:middle + g:upper + velavokr +) + SIZE(MEDIUM) -SRCS( - fuzz_json.cpp -) - -PEERDIR( +SRCS( + fuzz_json.cpp +) + +PEERDIR( library/cpp/scheme/tests/fuzz_json/lib -) - -END() +) + +END() diff --git a/library/cpp/scheme/tests/fuzz_ops/fuzz_ops.cpp b/library/cpp/scheme/tests/fuzz_ops/fuzz_ops.cpp index f912290b0a..facde50f5a 100644 --- a/library/cpp/scheme/tests/fuzz_ops/fuzz_ops.cpp +++ b/library/cpp/scheme/tests/fuzz_ops/fuzz_ops.cpp @@ -1,6 +1,6 @@ #include <library/cpp/scheme/tests/fuzz_ops/lib/fuzz_ops.h> - -extern "C" int LLVMFuzzerTestOneInput(const ui8* wireData, const size_t wireSize) { - NSc::NUt::FuzzOps({(const char*)wireData, wireSize}, false); - return 0; -} + +extern "C" int LLVMFuzzerTestOneInput(const ui8* wireData, const size_t wireSize) { + NSc::NUt::FuzzOps({(const char*)wireData, wireSize}, false); + return 0; +} diff --git a/library/cpp/scheme/tests/fuzz_ops/lib/fuzz_ops.cpp b/library/cpp/scheme/tests/fuzz_ops/lib/fuzz_ops.cpp index 30f6a3a619..8a7facba24 100644 --- a/library/cpp/scheme/tests/fuzz_ops/lib/fuzz_ops.cpp +++ b/library/cpp/scheme/tests/fuzz_ops/lib/fuzz_ops.cpp @@ -1,37 +1,37 @@ -#include "fuzz_ops.h" -#include "vm_apply.h" -#include "vm_defs.h" -#include "vm_parse.h" - +#include "fuzz_ops.h" +#include "vm_apply.h" +#include "vm_defs.h" +#include "vm_parse.h" + #include <library/cpp/bit_io/bitinput.h> - + #include <library/cpp/scheme/scheme.h> #include <library/cpp/scheme/scimpl_private.h> - -#include <util/generic/maybe.h> - -namespace NSc::NUt { - - void FuzzOps(TStringBuf wire, bool log) { - if (log) { - NImpl::GetTlsInstance<NImpl::TSelfLoopContext>().ReportingMode = NImpl::TSelfLoopContext::EMode::Stderr; - } - - // We start with a single TValue node - TVMState st {wire, 1, 0}; - - while (auto act = ParseNextAction(st)) { - if (log) { - Cerr << " STATE: " << st.ToString() << Endl; - Cerr << "ACTION: " << (act ? act->ToString() : TString("(empty)")) << Endl; - } - - if (!ApplyNextAction(st, *act)) { - break; - } + +#include <util/generic/maybe.h> + +namespace NSc::NUt { + + void FuzzOps(TStringBuf wire, bool log) { + if (log) { + NImpl::GetTlsInstance<NImpl::TSelfLoopContext>().ReportingMode = NImpl::TSelfLoopContext::EMode::Stderr; + } + + // We start with a single TValue node + TVMState st {wire, 1, 0}; + + while (auto act = ParseNextAction(st)) { + if (log) { + Cerr << " STATE: " << st.ToString() << Endl; + Cerr << "ACTION: " << (act ? act->ToString() : TString("(empty)")) << Endl; + } + + if (!ApplyNextAction(st, *act)) { + break; + } if (!NSc::TValue::DefaultValue().IsNull()) { std::terminate(); } - } - } -} + } + } +} diff --git a/library/cpp/scheme/tests/fuzz_ops/lib/fuzz_ops.h b/library/cpp/scheme/tests/fuzz_ops/lib/fuzz_ops.h index a15383c0a9..26ba42eaef 100644 --- a/library/cpp/scheme/tests/fuzz_ops/lib/fuzz_ops.h +++ b/library/cpp/scheme/tests/fuzz_ops/lib/fuzz_ops.h @@ -1,7 +1,7 @@ -#pragma once - -#include <util/generic/strbuf.h> - -namespace NSc::NUt { - void FuzzOps(TStringBuf wire, bool log); -} +#pragma once + +#include <util/generic/strbuf.h> + +namespace NSc::NUt { + void FuzzOps(TStringBuf wire, bool log); +} diff --git a/library/cpp/scheme/tests/fuzz_ops/lib/vm_apply.cpp b/library/cpp/scheme/tests/fuzz_ops/lib/vm_apply.cpp index 16a6329963..ada7b8854f 100644 --- a/library/cpp/scheme/tests/fuzz_ops/lib/vm_apply.cpp +++ b/library/cpp/scheme/tests/fuzz_ops/lib/vm_apply.cpp @@ -1,82 +1,82 @@ -#include "vm_apply.h" - -namespace NSc::NUt { - -#define Y_GEN_TRY_OP(op) \ - if (!op) { return false; } - -#define Y_GEN_SRC_OP(op, arg, st, act) \ - switch (act.GetSrc(arg).Type) { \ - case TSrc::T_LREF__POS: \ - op(st.LRef(act.GetSrc(arg).Pos)); \ - break; \ - case TSrc::T_CREF__POS: \ - op(st.CRef(act.GetSrc(arg).Pos)); \ - break; \ - case TSrc::T_RREF__POS: \ - op(st.RRef(act.GetSrc(arg).Pos)); \ - break; \ - default: \ - Y_FAIL(); \ - } - - -#define Y_GEN_SRC_TRY_OP(op, arg, st, act) \ - if (st.CRef(act.GetSrc(arg).Pos).IsSameOrAncestorOf(st.Current())) { \ - return false; \ - } \ - Y_GEN_SRC_OP(op, arg, st, act); - - -#define Y_GEN_DST_OP(op, arg, st, act) \ - switch (act.GetDst(arg).Type) { \ - case TDst::T_CREATE_BACK_LREF: \ - Y_GEN_TRY_OP(st.TryPushBack(op)) \ - break; \ - case TDst::T_CREATE_BACK_CREF: \ - Y_GEN_TRY_OP(st.TryPushBack(std::as_const(op))) \ - break; \ - case TDst::T_CREATE_BACK_RREF: \ - Y_GEN_TRY_OP(st.TryPushBack(std::move(op))) \ - break; \ - case TDst::T_CREATE_FRONT_LREF: \ - Y_GEN_TRY_OP(st.TryPushFront(op)) \ - break; \ - case TDst::T_CREATE_FRONT_CREF: \ - Y_GEN_TRY_OP(st.TryPushFront(std::as_const(op))) \ - break; \ - case TDst::T_CREATE_FRONT_RREF: \ - Y_GEN_TRY_OP(st.TryPushFront(std::move(op))) \ - break; \ - case TDst::T_LREF__POS: \ - st.LRef(act.GetDst(arg).Pos) = op; \ - break; \ - case TDst::T_CREF__POS: \ - st.LRef(act.GetDst(arg).Pos) = std::as_const(op); \ - break; \ - case TDst::T_RREF__POS: \ - st.LRef(act.GetDst(arg).Pos) = std::move(op); \ - break; \ - default: \ - Y_FAIL(); \ - } - - -#define Y_GEN_REF_OP(op, arg, st, act) \ - switch (act.GetRef(arg).Type) { \ - case TRef::T_CREATE_BACK: \ - Y_GEN_TRY_OP(st.TryPushBack(op)) \ - break; \ - case TRef::T_CREATE_FRONT: \ - Y_GEN_TRY_OP(st.TryPushFront(op)) \ - break; \ - case TRef::T_REF__POS: \ - st.LRef(act.GetRef(arg).Pos) = op; \ - break; \ - default: \ - Y_FAIL(); \ - } - +#include "vm_apply.h" + +namespace NSc::NUt { + +#define Y_GEN_TRY_OP(op) \ + if (!op) { return false; } + +#define Y_GEN_SRC_OP(op, arg, st, act) \ + switch (act.GetSrc(arg).Type) { \ + case TSrc::T_LREF__POS: \ + op(st.LRef(act.GetSrc(arg).Pos)); \ + break; \ + case TSrc::T_CREF__POS: \ + op(st.CRef(act.GetSrc(arg).Pos)); \ + break; \ + case TSrc::T_RREF__POS: \ + op(st.RRef(act.GetSrc(arg).Pos)); \ + break; \ + default: \ + Y_FAIL(); \ + } + + +#define Y_GEN_SRC_TRY_OP(op, arg, st, act) \ + if (st.CRef(act.GetSrc(arg).Pos).IsSameOrAncestorOf(st.Current())) { \ + return false; \ + } \ + Y_GEN_SRC_OP(op, arg, st, act); + + +#define Y_GEN_DST_OP(op, arg, st, act) \ + switch (act.GetDst(arg).Type) { \ + case TDst::T_CREATE_BACK_LREF: \ + Y_GEN_TRY_OP(st.TryPushBack(op)) \ + break; \ + case TDst::T_CREATE_BACK_CREF: \ + Y_GEN_TRY_OP(st.TryPushBack(std::as_const(op))) \ + break; \ + case TDst::T_CREATE_BACK_RREF: \ + Y_GEN_TRY_OP(st.TryPushBack(std::move(op))) \ + break; \ + case TDst::T_CREATE_FRONT_LREF: \ + Y_GEN_TRY_OP(st.TryPushFront(op)) \ + break; \ + case TDst::T_CREATE_FRONT_CREF: \ + Y_GEN_TRY_OP(st.TryPushFront(std::as_const(op))) \ + break; \ + case TDst::T_CREATE_FRONT_RREF: \ + Y_GEN_TRY_OP(st.TryPushFront(std::move(op))) \ + break; \ + case TDst::T_LREF__POS: \ + st.LRef(act.GetDst(arg).Pos) = op; \ + break; \ + case TDst::T_CREF__POS: \ + st.LRef(act.GetDst(arg).Pos) = std::as_const(op); \ + break; \ + case TDst::T_RREF__POS: \ + st.LRef(act.GetDst(arg).Pos) = std::move(op); \ + break; \ + default: \ + Y_FAIL(); \ + } + + +#define Y_GEN_REF_OP(op, arg, st, act) \ + switch (act.GetRef(arg).Type) { \ + case TRef::T_CREATE_BACK: \ + Y_GEN_TRY_OP(st.TryPushBack(op)) \ + break; \ + case TRef::T_CREATE_FRONT: \ + Y_GEN_TRY_OP(st.TryPushFront(op)) \ + break; \ + case TRef::T_REF__POS: \ + st.LRef(act.GetRef(arg).Pos) = op; \ + break; \ + default: \ + Y_FAIL(); \ + } + #define Y_GEN_PTR_OP(op, arg, st, act) \ if (auto* r = (op)) { \ switch (act.GetRef(arg).Type) { \ @@ -93,210 +93,210 @@ namespace NSc::NUt { Y_FAIL(); \ } \ } - - bool ApplyNextAction(TVMState& st, TVMAction act) { - switch (act.Type) { - case VMA_JMP__POS: - st.Pos = act.GetPos(0); - return true; - - case VMA_CREATE_BACK: - Y_GEN_TRY_OP(st.TryPushBack()) - return true; - - case VMA_CREATE_BACK__SRC: - switch (act.GetSrc(0).Type) { - case TSrc::T_LREF__POS: - Y_GEN_TRY_OP(st.TryPushBack(st.LRef(act.GetSrc(0).Pos))) - break; - case TSrc::T_CREF__POS: - Y_GEN_TRY_OP(st.TryPushBack(st.CRef(act.GetSrc(0).Pos))) - break; - case TSrc::T_RREF__POS: - Y_GEN_TRY_OP(st.TryPushBack(st.RRef(act.GetSrc(0).Pos))) - break; - default: - Y_FAIL(); - } - - return true; - - case VMA_CREATE_FRONT: - Y_GEN_TRY_OP(st.TryPushFront()) - return true; - - case VMA_CREATE_FRONT__SRC: - switch (act.GetSrc(0).Type) { - case TSrc::T_LREF__POS: - Y_GEN_TRY_OP(st.TryPushFront(st.LRef(act.GetSrc(0).Pos))) - break; - case TSrc::T_CREF__POS: - Y_GEN_TRY_OP(st.TryPushFront(st.CRef(act.GetSrc(0).Pos))) - break; - case TSrc::T_RREF__POS: - Y_GEN_TRY_OP(st.TryPushFront(st.RRef(act.GetSrc(0).Pos))) - break; - default: - Y_FAIL(); - } - return true; - - case VMA_DESTROY_BACK: - return st.TryPopBack(); - - case VMA_DESTROY_FRONT: - return st.TryPopFront(); - - case VMA_ASSIGN__SRC: - Y_GEN_SRC_OP(st.Current() = , 0, st, act); - return true; - - case VMA_SET_STRING__IDX: - st.Current().SetString(act.GetString(0)); - return true; - - case VMA_SET_INT_NUMBER__IDX: - st.Current().SetIntNumber(act.GetIntNumber(0)); - return true; - - case VMA_SET_DICT: - st.Current().SetDict(); - return true; - - case VMA_SET_ARRAY: - st.Current().SetArray(); - return true; - - case VMA_SET_NULL: - st.Current().SetNull(); - return true; - - case VMA_GET_JSON: - st.Current().ToJson(); - return true; - - case VMA_ARRAY_CLEAR: - st.Current().ClearArray(); - return true; - - case VMA_ARRAY_PUSH: - st.Current().Push(); - return true; - - case VMA_ARRAY_PUSH__SRC: - Y_GEN_SRC_TRY_OP(st.Current().Push, 0, st, act); - return true; - - case VMA_ARRAY_PUSH__DST: - Y_GEN_DST_OP(st.Current().Push(), 0, st, act); - return true; - - case VMA_ARRAY_POP__REF: - Y_GEN_REF_OP(st.Current().Pop(), 0, st, act); - return true; - - case VMA_ARRAY_INSERT__IDX: - st.Current().Insert(act.GetIdx(0)); - return true; - - case VMA_ARRAY_INSERT__IDX_SRC: - Y_GEN_SRC_TRY_OP(st.Current().Insert(act.GetIdx(0)) = , 1, st, act); - return true; - - case VMA_ARRAY_INSERT__IDX_DST: - Y_GEN_DST_OP(st.Current().Insert(act.GetIdx(0)), 1, st, act); - return true; - - case VMA_ARRAY_DELETE__IDX_REF: - Y_GEN_REF_OP(st.Current().Delete(act.GetIdx(0)), 1, st, act); - return true; - - case VMA_ARRAY_GET__IDX_REF: - Y_GEN_REF_OP(st.Current().Get(act.GetIdx(0)), 1, st, act); - return true; - - case VMA_ARRAY_GET_OR_ADD__IDX: - st.Current().GetOrAdd(act.GetIdx(0)); - return true; - - case VMA_ARRAY_GET_OR_ADD__IDX_SRC: - Y_GEN_SRC_TRY_OP(st.Current().GetOrAdd(act.GetIdx(0)) = , 1, st, act); - return true; - - case VMA_ARRAY_GET_OR_ADD__IDX_DST: - Y_GEN_DST_OP(st.Current().GetOrAdd(act.GetIdx(0)), 1, st, act); - return true; - - case VMA_ARRAY_GET_NO_ADD__IDX_REF: + + bool ApplyNextAction(TVMState& st, TVMAction act) { + switch (act.Type) { + case VMA_JMP__POS: + st.Pos = act.GetPos(0); + return true; + + case VMA_CREATE_BACK: + Y_GEN_TRY_OP(st.TryPushBack()) + return true; + + case VMA_CREATE_BACK__SRC: + switch (act.GetSrc(0).Type) { + case TSrc::T_LREF__POS: + Y_GEN_TRY_OP(st.TryPushBack(st.LRef(act.GetSrc(0).Pos))) + break; + case TSrc::T_CREF__POS: + Y_GEN_TRY_OP(st.TryPushBack(st.CRef(act.GetSrc(0).Pos))) + break; + case TSrc::T_RREF__POS: + Y_GEN_TRY_OP(st.TryPushBack(st.RRef(act.GetSrc(0).Pos))) + break; + default: + Y_FAIL(); + } + + return true; + + case VMA_CREATE_FRONT: + Y_GEN_TRY_OP(st.TryPushFront()) + return true; + + case VMA_CREATE_FRONT__SRC: + switch (act.GetSrc(0).Type) { + case TSrc::T_LREF__POS: + Y_GEN_TRY_OP(st.TryPushFront(st.LRef(act.GetSrc(0).Pos))) + break; + case TSrc::T_CREF__POS: + Y_GEN_TRY_OP(st.TryPushFront(st.CRef(act.GetSrc(0).Pos))) + break; + case TSrc::T_RREF__POS: + Y_GEN_TRY_OP(st.TryPushFront(st.RRef(act.GetSrc(0).Pos))) + break; + default: + Y_FAIL(); + } + return true; + + case VMA_DESTROY_BACK: + return st.TryPopBack(); + + case VMA_DESTROY_FRONT: + return st.TryPopFront(); + + case VMA_ASSIGN__SRC: + Y_GEN_SRC_OP(st.Current() = , 0, st, act); + return true; + + case VMA_SET_STRING__IDX: + st.Current().SetString(act.GetString(0)); + return true; + + case VMA_SET_INT_NUMBER__IDX: + st.Current().SetIntNumber(act.GetIntNumber(0)); + return true; + + case VMA_SET_DICT: + st.Current().SetDict(); + return true; + + case VMA_SET_ARRAY: + st.Current().SetArray(); + return true; + + case VMA_SET_NULL: + st.Current().SetNull(); + return true; + + case VMA_GET_JSON: + st.Current().ToJson(); + return true; + + case VMA_ARRAY_CLEAR: + st.Current().ClearArray(); + return true; + + case VMA_ARRAY_PUSH: + st.Current().Push(); + return true; + + case VMA_ARRAY_PUSH__SRC: + Y_GEN_SRC_TRY_OP(st.Current().Push, 0, st, act); + return true; + + case VMA_ARRAY_PUSH__DST: + Y_GEN_DST_OP(st.Current().Push(), 0, st, act); + return true; + + case VMA_ARRAY_POP__REF: + Y_GEN_REF_OP(st.Current().Pop(), 0, st, act); + return true; + + case VMA_ARRAY_INSERT__IDX: + st.Current().Insert(act.GetIdx(0)); + return true; + + case VMA_ARRAY_INSERT__IDX_SRC: + Y_GEN_SRC_TRY_OP(st.Current().Insert(act.GetIdx(0)) = , 1, st, act); + return true; + + case VMA_ARRAY_INSERT__IDX_DST: + Y_GEN_DST_OP(st.Current().Insert(act.GetIdx(0)), 1, st, act); + return true; + + case VMA_ARRAY_DELETE__IDX_REF: + Y_GEN_REF_OP(st.Current().Delete(act.GetIdx(0)), 1, st, act); + return true; + + case VMA_ARRAY_GET__IDX_REF: + Y_GEN_REF_OP(st.Current().Get(act.GetIdx(0)), 1, st, act); + return true; + + case VMA_ARRAY_GET_OR_ADD__IDX: + st.Current().GetOrAdd(act.GetIdx(0)); + return true; + + case VMA_ARRAY_GET_OR_ADD__IDX_SRC: + Y_GEN_SRC_TRY_OP(st.Current().GetOrAdd(act.GetIdx(0)) = , 1, st, act); + return true; + + case VMA_ARRAY_GET_OR_ADD__IDX_DST: + Y_GEN_DST_OP(st.Current().GetOrAdd(act.GetIdx(0)), 1, st, act); + return true; + + case VMA_ARRAY_GET_NO_ADD__IDX_REF: Y_GEN_PTR_OP(st.Current().GetNoAdd(act.GetIdx(0)), 1, st, act); - return true; - - case VMA_DICT_CLEAR: - st.Current().ClearDict(); - return true; - - case VMA_DICT_DELETE__IDX: - st.Current().Delete(act.GetKey(0)); - return true; - - case VMA_DICT_DELETE__IDX_REF: - Y_GEN_REF_OP(st.Current().Delete(act.GetKey(0)), 1, st, act); - return true; - - case VMA_DICT_GET__IDX_REF: - Y_GEN_REF_OP(st.Current().Get(act.GetKey(0)), 1, st, act); - return true; - - case VMA_DICT_GET_OR_ADD__IDX: - st.Current().GetOrAdd(act.GetKey(0)); - return true; - - case VMA_DICT_GET_OR_ADD__IDX_SRC: - Y_GEN_SRC_TRY_OP(st.Current().GetOrAdd(act.GetKey(0)) = , 1, st, act); - return true; - - case VMA_DICT_GET_OR_ADD__IDX_DST: - Y_GEN_DST_OP(st.Current().GetOrAdd(act.GetKey(0)), 1, st, act); - return true; - - case VMA_DICT_GET_NO_ADD__IDX_REF: + return true; + + case VMA_DICT_CLEAR: + st.Current().ClearDict(); + return true; + + case VMA_DICT_DELETE__IDX: + st.Current().Delete(act.GetKey(0)); + return true; + + case VMA_DICT_DELETE__IDX_REF: + Y_GEN_REF_OP(st.Current().Delete(act.GetKey(0)), 1, st, act); + return true; + + case VMA_DICT_GET__IDX_REF: + Y_GEN_REF_OP(st.Current().Get(act.GetKey(0)), 1, st, act); + return true; + + case VMA_DICT_GET_OR_ADD__IDX: + st.Current().GetOrAdd(act.GetKey(0)); + return true; + + case VMA_DICT_GET_OR_ADD__IDX_SRC: + Y_GEN_SRC_TRY_OP(st.Current().GetOrAdd(act.GetKey(0)) = , 1, st, act); + return true; + + case VMA_DICT_GET_OR_ADD__IDX_DST: + Y_GEN_DST_OP(st.Current().GetOrAdd(act.GetKey(0)), 1, st, act); + return true; + + case VMA_DICT_GET_NO_ADD__IDX_REF: Y_GEN_PTR_OP(st.Current().GetNoAdd(act.GetKey(0)), 1, st, act); - return true; - - case VMA_MERGE_UPDATE__POS: - st.Current().MergeUpdate(st.LRef(act.GetPos(0))); - return true; - - case VMA_MERGE_REVERSE__POS: - st.Current().ReverseMerge(st.LRef(act.GetPos(0))); - return true; - - case VMA_MERGE_COPY_FROM__POS: - st.Current().CopyFrom(st.LRef(act.GetPos(0))); - return true; - - case VMA_SWAP__POS: - st.Current().Swap(st.LRef(act.GetPos(0))); - return true; - - case VMA_EQUAL__POS_POS: - TValue::Equal(st.CRef(act.GetPos(0)), st.CRef(act.GetPos(1))); - return true; - - case VMA_SELECT_NO_ADD__PATH_REF: - Y_GEN_REF_OP(st.Current().TrySelect(act.GetPath(0)), 1, st, act); - return true; - - case VMA_SELECT_OR_ADD__PATH_REF: + return true; + + case VMA_MERGE_UPDATE__POS: + st.Current().MergeUpdate(st.LRef(act.GetPos(0))); + return true; + + case VMA_MERGE_REVERSE__POS: + st.Current().ReverseMerge(st.LRef(act.GetPos(0))); + return true; + + case VMA_MERGE_COPY_FROM__POS: + st.Current().CopyFrom(st.LRef(act.GetPos(0))); + return true; + + case VMA_SWAP__POS: + st.Current().Swap(st.LRef(act.GetPos(0))); + return true; + + case VMA_EQUAL__POS_POS: + TValue::Equal(st.CRef(act.GetPos(0)), st.CRef(act.GetPos(1))); + return true; + + case VMA_SELECT_NO_ADD__PATH_REF: + Y_GEN_REF_OP(st.Current().TrySelect(act.GetPath(0)), 1, st, act); + return true; + + case VMA_SELECT_OR_ADD__PATH_REF: Y_GEN_PTR_OP(st.Current().TrySelectOrAdd(act.GetPath(0)), 1, st, act); - return true; - - case VMA_SELECT_AND_DELETE__PATH_REF: - Y_GEN_REF_OP(st.Current().TrySelectAndDelete(act.GetPath(0)), 1, st, act); - return true; - - default: - Y_FAIL(); - } - } -} + return true; + + case VMA_SELECT_AND_DELETE__PATH_REF: + Y_GEN_REF_OP(st.Current().TrySelectAndDelete(act.GetPath(0)), 1, st, act); + return true; + + default: + Y_FAIL(); + } + } +} diff --git a/library/cpp/scheme/tests/fuzz_ops/lib/vm_apply.h b/library/cpp/scheme/tests/fuzz_ops/lib/vm_apply.h index 89477e98f2..82906b53fb 100644 --- a/library/cpp/scheme/tests/fuzz_ops/lib/vm_apply.h +++ b/library/cpp/scheme/tests/fuzz_ops/lib/vm_apply.h @@ -1,9 +1,9 @@ -#pragma once - -#include "vm_defs.h" - -namespace NSc::NUt { - - bool ApplyNextAction(TVMState& st, TVMAction act); - -} +#pragma once + +#include "vm_defs.h" + +namespace NSc::NUt { + + bool ApplyNextAction(TVMState& st, TVMAction act); + +} diff --git a/library/cpp/scheme/tests/fuzz_ops/lib/vm_defs.cpp b/library/cpp/scheme/tests/fuzz_ops/lib/vm_defs.cpp index dbaa3f7a42..55a971d9e4 100644 --- a/library/cpp/scheme/tests/fuzz_ops/lib/vm_defs.cpp +++ b/library/cpp/scheme/tests/fuzz_ops/lib/vm_defs.cpp @@ -1,168 +1,168 @@ -#include "vm_defs.h" - -#include <util/generic/xrange.h> -#include <util/string/builder.h> - -namespace NSc::NUt { - namespace { - TStringBuf GetStringByIdx(ui32 idx) { - static const TStringBuf strings[TIdx::ValueCount] {{}, "a", "aa", "aaa"}; - return strings[idx % TIdx::ValueCount]; - } - } - - - TVMState::TVMState(TStringBuf wire, ui32 memSz, ui32 pos) - : Input(wire) - , Memory(memSz) - , Pos(pos) - {} - - bool TVMState::TryPushBack() { - if (MayAddMoreMemory()) { - Memory.emplace_back(); - return true; - } else { - return false; - } - } - - bool TVMState::TryPushFront() { - if (MayAddMoreMemory()) { - Pos += 1; - Memory.emplace_front(); - return true; - } else { - return false; - } - } - - bool TVMState::TryPopBack() { - if (Memory.size() > 1 && Pos < Memory.size() - 1) { - Memory.pop_back(); - return true; - } else { - return false; - } - } - - bool TVMState::TryPopFront() { - if (Memory.size() > 1 && Pos > 0) { - Memory.pop_front(); - Pos -= 1; - return true; - } else { - return false; - } - } - - TString TVMState::ToString() const { - TStringBuilder b; - b << "pos=" << Pos << ";"; - for (const auto i : xrange(Memory.size())) { - b << " " << i << (i == Pos ? "@" : ".") << Memory[i].ToJson().Quote(); - } - - return b; - } - - TString TIdx::ToString() const { - return TStringBuilder() << "IDX:" << Idx; - } - - TString TPos::ToString() const { - return TStringBuilder() << "POS:" << Pos; - } - - template <class T> - TString RenderToString(TStringBuf name, T type, ui32 pos) { - TStringBuilder b; - b << name << ":" << type; - if ((ui32)-1 != pos) { - b << "," << pos; - } - return b; - } - - TString TRef::ToString() const { - return RenderToString("REF", Type, Pos); - } - - TString TSrc::ToString() const { - return RenderToString("SRC", Type, Pos); - } - - TString TDst::ToString() const { - return RenderToString("DST", Type, Pos); - } - - TString TPath::ToString() const { - return TStringBuilder() << "PATH:" << Path.Quote(); - } - - - TRef TVMAction::GetRef(ui32 arg) const noexcept { +#include "vm_defs.h" + +#include <util/generic/xrange.h> +#include <util/string/builder.h> + +namespace NSc::NUt { + namespace { + TStringBuf GetStringByIdx(ui32 idx) { + static const TStringBuf strings[TIdx::ValueCount] {{}, "a", "aa", "aaa"}; + return strings[idx % TIdx::ValueCount]; + } + } + + + TVMState::TVMState(TStringBuf wire, ui32 memSz, ui32 pos) + : Input(wire) + , Memory(memSz) + , Pos(pos) + {} + + bool TVMState::TryPushBack() { + if (MayAddMoreMemory()) { + Memory.emplace_back(); + return true; + } else { + return false; + } + } + + bool TVMState::TryPushFront() { + if (MayAddMoreMemory()) { + Pos += 1; + Memory.emplace_front(); + return true; + } else { + return false; + } + } + + bool TVMState::TryPopBack() { + if (Memory.size() > 1 && Pos < Memory.size() - 1) { + Memory.pop_back(); + return true; + } else { + return false; + } + } + + bool TVMState::TryPopFront() { + if (Memory.size() > 1 && Pos > 0) { + Memory.pop_front(); + Pos -= 1; + return true; + } else { + return false; + } + } + + TString TVMState::ToString() const { + TStringBuilder b; + b << "pos=" << Pos << ";"; + for (const auto i : xrange(Memory.size())) { + b << " " << i << (i == Pos ? "@" : ".") << Memory[i].ToJson().Quote(); + } + + return b; + } + + TString TIdx::ToString() const { + return TStringBuilder() << "IDX:" << Idx; + } + + TString TPos::ToString() const { + return TStringBuilder() << "POS:" << Pos; + } + + template <class T> + TString RenderToString(TStringBuf name, T type, ui32 pos) { + TStringBuilder b; + b << name << ":" << type; + if ((ui32)-1 != pos) { + b << "," << pos; + } + return b; + } + + TString TRef::ToString() const { + return RenderToString("REF", Type, Pos); + } + + TString TSrc::ToString() const { + return RenderToString("SRC", Type, Pos); + } + + TString TDst::ToString() const { + return RenderToString("DST", Type, Pos); + } + + TString TPath::ToString() const { + return TStringBuilder() << "PATH:" << Path.Quote(); + } + + + TRef TVMAction::GetRef(ui32 arg) const noexcept { return std::get<TRef>(Arg[arg]); - } - - TSrc TVMAction::GetSrc(ui32 arg) const noexcept { + } + + TSrc TVMAction::GetSrc(ui32 arg) const noexcept { return std::get<TSrc>(Arg[arg]); - } - - TDst TVMAction::GetDst(ui32 arg) const noexcept { + } + + TDst TVMAction::GetDst(ui32 arg) const noexcept { return std::get<TDst>(Arg[arg]); - } - - ui32 TVMAction::GetPos(ui32 arg) const noexcept { + } + + ui32 TVMAction::GetPos(ui32 arg) const noexcept { return std::get<TPos>(Arg[arg]).Pos; - } - - ui32 TVMAction::GetIdx(ui32 arg) const noexcept { + } + + ui32 TVMAction::GetIdx(ui32 arg) const noexcept { return std::get<TIdx>(Arg[arg]).Idx; - } - - TStringBuf TVMAction::GetKey(ui32 arg) const noexcept { - return GetString(arg); - } - - TStringBuf TVMAction::GetString(ui32 arg) const noexcept { - return GetStringByIdx(GetIdx(arg)); - } - - i64 TVMAction::GetIntNumber(ui32 arg) const noexcept { - return GetIdx(arg); - } - - TStringBuf TVMAction::GetPath(ui32 arg) const noexcept { + } + + TStringBuf TVMAction::GetKey(ui32 arg) const noexcept { + return GetString(arg); + } + + TStringBuf TVMAction::GetString(ui32 arg) const noexcept { + return GetStringByIdx(GetIdx(arg)); + } + + i64 TVMAction::GetIntNumber(ui32 arg) const noexcept { + return GetIdx(arg); + } + + TStringBuf TVMAction::GetPath(ui32 arg) const noexcept { return std::get<TPath>(Arg[arg]).Path; - } - - - struct TActionPrinter { - TActionPrinter(IOutputStream& out) - : Out(out) - {} - + } + + + struct TActionPrinter { + TActionPrinter(IOutputStream& out) + : Out(out) + {} + bool operator()(const std::monostate&) const { - return true; - } - //TIdx, TPos, TRef, TSrc, TDst, TPath - template <class T> - bool operator()(const T& t) const { - Out << "; " << t.ToString(); - return false; - } - IOutputStream& Out; - }; - - TString TVMAction::ToString() const { - TStringBuilder out; - out << Type; - for (const auto& arg : Arg) { + return true; + } + //TIdx, TPos, TRef, TSrc, TDst, TPath + template <class T> + bool operator()(const T& t) const { + Out << "; " << t.ToString(); + return false; + } + IOutputStream& Out; + }; + + TString TVMAction::ToString() const { + TStringBuilder out; + out << Type; + for (const auto& arg : Arg) { if (std::visit(TActionPrinter(out.Out), arg)) { - break; - } - } - return out; - } -} + break; + } + } + return out; + } +} diff --git a/library/cpp/scheme/tests/fuzz_ops/lib/vm_defs.h b/library/cpp/scheme/tests/fuzz_ops/lib/vm_defs.h index 0ec7a37ceb..9a0ddf7351 100644 --- a/library/cpp/scheme/tests/fuzz_ops/lib/vm_defs.h +++ b/library/cpp/scheme/tests/fuzz_ops/lib/vm_defs.h @@ -1,277 +1,277 @@ -#pragma once - +#pragma once + #include <library/cpp/bit_io/bitinput.h> #include <library/cpp/scheme/scheme.h> - -#include <util/generic/deque.h> -#include <util/generic/maybe.h> -#include <util/generic/variant.h> -#include <util/string/builder.h> - -namespace NSc::NUt { - - enum EVMAction { - VMA_JMP__POS /* "pos=POS" */, - - VMA_CREATE_BACK /* "TValue()->emplace_back" */, - VMA_CREATE_BACK__SRC /* "TValue(SRC)->emplace_back" */, - VMA_CREATE_FRONT /* "TValue()->emplace_front,pos+=1" */, - VMA_CREATE_FRONT__SRC /* "TValue(SRC)->emplace_front,pos+=1" */, - - VMA_DESTROY_BACK /* "pop_back" */, - VMA_DESTROY_FRONT /* "pop_front,pos-=1" */, - - VMA_ASSIGN__SRC /* "operator=(SRC)" */, - - VMA_SET_STRING__IDX /* "SetString(str[IDX])" */, - VMA_SET_INT_NUMBER__IDX /* "SetIntNumber(IDX)" */, - VMA_SET_DICT /* "SetDict()" */, - VMA_SET_ARRAY /* "SetArray()" */, - VMA_SET_NULL /* "SetNull()" */, - - VMA_GET_JSON /* "ToJson()" */, - - VMA_ARRAY_CLEAR /* "ClearArray()" */, - VMA_ARRAY_PUSH /* "Push()" */, - VMA_ARRAY_PUSH__SRC /* "Push()=SRC" */, - VMA_ARRAY_PUSH__DST /* "Push()->DST" */, - VMA_ARRAY_POP /* "Pop()" */, - VMA_ARRAY_POP__REF /* "Pop()->REF" */, - VMA_ARRAY_INSERT__IDX /* "Insert(IDX)" */, - VMA_ARRAY_INSERT__IDX_SRC /* "Insert(IDX)=SRC" */, - VMA_ARRAY_INSERT__IDX_DST /* "Insert(IDX)->DST" */, - VMA_ARRAY_DELETE__IDX /* "Delete(IDX)" */, - VMA_ARRAY_DELETE__IDX_REF /* "Delete(IDX)->REF" */, - VMA_ARRAY_GET__IDX_REF /* "Get(IDX)->REF" */, - VMA_ARRAY_GET_OR_ADD__IDX /* "GetOrAdd(IDX)" */, - VMA_ARRAY_GET_OR_ADD__IDX_SRC /* "GetOrAdd(IDX)=SRC" */, - VMA_ARRAY_GET_OR_ADD__IDX_DST /* "GetOrAdd(IDX)->DST" */, - VMA_ARRAY_GET_NO_ADD__IDX_REF /* "GetNoAdd(IDX)->REF" */, - - VMA_DICT_CLEAR /* "ClearDict()" */, - VMA_DICT_DELETE__IDX /* "Delete(str[IDX])" */, - VMA_DICT_DELETE__IDX_REF /* "Delete(str[IDX])->REF" */, - VMA_DICT_GET__IDX_REF /* "Get(str[IDX])->REF" */, - VMA_DICT_GET_OR_ADD__IDX /* "GetOrAdd(str[IDX])" */, - VMA_DICT_GET_OR_ADD__IDX_SRC /* "GetOrAdd(str[IDX])=SRC" */, - VMA_DICT_GET_OR_ADD__IDX_DST /* "GetOrAdd(str[IDX])->DST" */, - VMA_DICT_GET_NO_ADD__IDX_REF /* "GetNoAdd(str[IDX])->REF" */, - - VMA_MERGE_UPDATE__POS /* "MergeUpdate(POS)" */, - VMA_MERGE_REVERSE__POS /* "ReverseMerge(POS)" */, - VMA_MERGE_COPY_FROM__POS /* "CopyFrom(POS)" */, - - VMA_SWAP__POS /* "Swap(POS)" */, - VMA_EQUAL__POS_POS /* "Equal(POS, POS)" */, - - VMA_SELECT_NO_ADD__PATH_REF /* "TrySelect(PATH)->REF" */, - VMA_SELECT_OR_ADD__PATH_REF /* "TrySelectOrAdd(PATH)->REF" */, - VMA_SELECT_AND_DELETE__PATH_REF /* "TrySelectAndDelete(PATH)->REF" */, - - VMA_COUNT, - }; - - - struct TVMState { - NBitIO::TBitInput Input; - TDeque<TValue> Memory { 1 }; - ui32 Pos = 0; - - static const ui32 MaxMemory = 16; - - public: - explicit TVMState(TStringBuf wire, ui32 memSz, ui32 pos); - - TValue& Current() { - return Memory[Pos]; - } - - TValue& LRef(ui32 pos) { - return Memory[pos]; - } - - const TValue& CRef(ui32 pos) { - return Memory[pos]; - } - - TValue&& RRef(ui32 pos) { - return std::move(Memory[pos]); - } - + +#include <util/generic/deque.h> +#include <util/generic/maybe.h> +#include <util/generic/variant.h> +#include <util/string/builder.h> + +namespace NSc::NUt { + + enum EVMAction { + VMA_JMP__POS /* "pos=POS" */, + + VMA_CREATE_BACK /* "TValue()->emplace_back" */, + VMA_CREATE_BACK__SRC /* "TValue(SRC)->emplace_back" */, + VMA_CREATE_FRONT /* "TValue()->emplace_front,pos+=1" */, + VMA_CREATE_FRONT__SRC /* "TValue(SRC)->emplace_front,pos+=1" */, + + VMA_DESTROY_BACK /* "pop_back" */, + VMA_DESTROY_FRONT /* "pop_front,pos-=1" */, + + VMA_ASSIGN__SRC /* "operator=(SRC)" */, + + VMA_SET_STRING__IDX /* "SetString(str[IDX])" */, + VMA_SET_INT_NUMBER__IDX /* "SetIntNumber(IDX)" */, + VMA_SET_DICT /* "SetDict()" */, + VMA_SET_ARRAY /* "SetArray()" */, + VMA_SET_NULL /* "SetNull()" */, + + VMA_GET_JSON /* "ToJson()" */, + + VMA_ARRAY_CLEAR /* "ClearArray()" */, + VMA_ARRAY_PUSH /* "Push()" */, + VMA_ARRAY_PUSH__SRC /* "Push()=SRC" */, + VMA_ARRAY_PUSH__DST /* "Push()->DST" */, + VMA_ARRAY_POP /* "Pop()" */, + VMA_ARRAY_POP__REF /* "Pop()->REF" */, + VMA_ARRAY_INSERT__IDX /* "Insert(IDX)" */, + VMA_ARRAY_INSERT__IDX_SRC /* "Insert(IDX)=SRC" */, + VMA_ARRAY_INSERT__IDX_DST /* "Insert(IDX)->DST" */, + VMA_ARRAY_DELETE__IDX /* "Delete(IDX)" */, + VMA_ARRAY_DELETE__IDX_REF /* "Delete(IDX)->REF" */, + VMA_ARRAY_GET__IDX_REF /* "Get(IDX)->REF" */, + VMA_ARRAY_GET_OR_ADD__IDX /* "GetOrAdd(IDX)" */, + VMA_ARRAY_GET_OR_ADD__IDX_SRC /* "GetOrAdd(IDX)=SRC" */, + VMA_ARRAY_GET_OR_ADD__IDX_DST /* "GetOrAdd(IDX)->DST" */, + VMA_ARRAY_GET_NO_ADD__IDX_REF /* "GetNoAdd(IDX)->REF" */, + + VMA_DICT_CLEAR /* "ClearDict()" */, + VMA_DICT_DELETE__IDX /* "Delete(str[IDX])" */, + VMA_DICT_DELETE__IDX_REF /* "Delete(str[IDX])->REF" */, + VMA_DICT_GET__IDX_REF /* "Get(str[IDX])->REF" */, + VMA_DICT_GET_OR_ADD__IDX /* "GetOrAdd(str[IDX])" */, + VMA_DICT_GET_OR_ADD__IDX_SRC /* "GetOrAdd(str[IDX])=SRC" */, + VMA_DICT_GET_OR_ADD__IDX_DST /* "GetOrAdd(str[IDX])->DST" */, + VMA_DICT_GET_NO_ADD__IDX_REF /* "GetNoAdd(str[IDX])->REF" */, + + VMA_MERGE_UPDATE__POS /* "MergeUpdate(POS)" */, + VMA_MERGE_REVERSE__POS /* "ReverseMerge(POS)" */, + VMA_MERGE_COPY_FROM__POS /* "CopyFrom(POS)" */, + + VMA_SWAP__POS /* "Swap(POS)" */, + VMA_EQUAL__POS_POS /* "Equal(POS, POS)" */, + + VMA_SELECT_NO_ADD__PATH_REF /* "TrySelect(PATH)->REF" */, + VMA_SELECT_OR_ADD__PATH_REF /* "TrySelectOrAdd(PATH)->REF" */, + VMA_SELECT_AND_DELETE__PATH_REF /* "TrySelectAndDelete(PATH)->REF" */, + + VMA_COUNT, + }; + + + struct TVMState { + NBitIO::TBitInput Input; + TDeque<TValue> Memory { 1 }; + ui32 Pos = 0; + + static const ui32 MaxMemory = 16; + + public: + explicit TVMState(TStringBuf wire, ui32 memSz, ui32 pos); + + TValue& Current() { + return Memory[Pos]; + } + + TValue& LRef(ui32 pos) { + return Memory[pos]; + } + + const TValue& CRef(ui32 pos) { + return Memory[pos]; + } + + TValue&& RRef(ui32 pos) { + return std::move(Memory[pos]); + } + [[nodiscard]] - bool TryPushBack(); - - template <class T> + bool TryPushBack(); + + template <class T> [[nodiscard]] - bool TryPushBack(T&& t) { - if (MayAddMoreMemory()) { - Memory.emplace_back(std::forward<T>(t)); - return true; - } else { - return false; - } - } - + bool TryPushBack(T&& t) { + if (MayAddMoreMemory()) { + Memory.emplace_back(std::forward<T>(t)); + return true; + } else { + return false; + } + } + [[nodiscard]] - bool TryPushFront(); - - template <class T> + bool TryPushFront(); + + template <class T> [[nodiscard]] - bool TryPushFront(T&& t) { - if (MayAddMoreMemory()) { - Pos += 1; - Memory.emplace_front(std::forward<T>(t)); - return true; - } else { - return false; - } - } - + bool TryPushFront(T&& t) { + if (MayAddMoreMemory()) { + Pos += 1; + Memory.emplace_front(std::forward<T>(t)); + return true; + } else { + return false; + } + } + [[nodiscard]] - bool TryPopBack(); - + bool TryPopBack(); + [[nodiscard]] - bool TryPopFront(); - - TString ToString() const; - - private: + bool TryPopFront(); + + TString ToString() const; + + private: [[nodiscard]] - bool MayAddMoreMemory() const noexcept { - return Memory.size() < MaxMemory; - } - }; - - - struct TIdx { - ui32 Idx = -1; - static const ui32 ValueCount = 4; - - public: - TString ToString() const; - }; - - - struct TPos { - ui32 Pos = -1; - static const ui32 ValueCount = TVMState::MaxMemory; - - public: - TString ToString() const; - }; - - - struct TRef : public TPos { - enum EType { - T_CREATE_FRONT /* "emplace_front(RES),pos+=1" */, - T_CREATE_BACK /* "emplace_back(RES)" */, - T_REF__POS /* "pos@(RES)" */, - T_COUNT - }; - - EType Type = T_COUNT; - static const ui32 TypeCount = T_COUNT; - - public: - TString ToString() const; - }; - - - struct TSrc : public TPos { - enum EType { - T_LREF__POS /* "pos@(TValue&)" */, - T_CREF__POS /* "pos@(const TValue&)" */, - T_RREF__POS /* "pos@(TValue&&)" */, - T_COUNT - }; - - EType Type = T_COUNT; - static const ui32 TypeCount = T_COUNT; - - public: - TString ToString() const; - }; - - - struct TDst : public TPos { - enum EType { - T_CREATE_FRONT_LREF /* "emplace_front(TValue&),pos+=1" */, - T_CREATE_FRONT_CREF /* "emplace_front(const TValue&),pos+=1" */, - T_CREATE_FRONT_RREF /* "emplace_front(TValue&&),pos+=1" */, - T_CREATE_BACK_LREF /* "emplace_back(TValue&)" */, - T_CREATE_BACK_CREF /* "emplace_back(TValue&),pos+=1" */, - T_CREATE_BACK_RREF /* "emplace_back(TValue&),pos+=1" */, - T_LREF__POS /* "pos@(TValue&)" */, - T_CREF__POS /* "pos@(const TValue&)" */, - T_RREF__POS /* "pos@(TValue&&)" */, - T_COUNT - }; - - EType Type = T_COUNT; - static const ui32 TypeCount = T_COUNT; - - public: - TString ToString() const; - }; - - - struct TPath { - TString Path; - static const ui32 MaxLength = 32; - - public: - TString ToString() const; - }; - + bool MayAddMoreMemory() const noexcept { + return Memory.size() < MaxMemory; + } + }; + + + struct TIdx { + ui32 Idx = -1; + static const ui32 ValueCount = 4; + + public: + TString ToString() const; + }; + + + struct TPos { + ui32 Pos = -1; + static const ui32 ValueCount = TVMState::MaxMemory; + + public: + TString ToString() const; + }; + + + struct TRef : public TPos { + enum EType { + T_CREATE_FRONT /* "emplace_front(RES),pos+=1" */, + T_CREATE_BACK /* "emplace_back(RES)" */, + T_REF__POS /* "pos@(RES)" */, + T_COUNT + }; + + EType Type = T_COUNT; + static const ui32 TypeCount = T_COUNT; + + public: + TString ToString() const; + }; + + + struct TSrc : public TPos { + enum EType { + T_LREF__POS /* "pos@(TValue&)" */, + T_CREF__POS /* "pos@(const TValue&)" */, + T_RREF__POS /* "pos@(TValue&&)" */, + T_COUNT + }; + + EType Type = T_COUNT; + static const ui32 TypeCount = T_COUNT; + + public: + TString ToString() const; + }; + + + struct TDst : public TPos { + enum EType { + T_CREATE_FRONT_LREF /* "emplace_front(TValue&),pos+=1" */, + T_CREATE_FRONT_CREF /* "emplace_front(const TValue&),pos+=1" */, + T_CREATE_FRONT_RREF /* "emplace_front(TValue&&),pos+=1" */, + T_CREATE_BACK_LREF /* "emplace_back(TValue&)" */, + T_CREATE_BACK_CREF /* "emplace_back(TValue&),pos+=1" */, + T_CREATE_BACK_RREF /* "emplace_back(TValue&),pos+=1" */, + T_LREF__POS /* "pos@(TValue&)" */, + T_CREF__POS /* "pos@(const TValue&)" */, + T_RREF__POS /* "pos@(TValue&&)" */, + T_COUNT + }; + + EType Type = T_COUNT; + static const ui32 TypeCount = T_COUNT; + + public: + TString ToString() const; + }; + + + struct TPath { + TString Path; + static const ui32 MaxLength = 32; + + public: + TString ToString() const; + }; + using TArg = std::variant<std::monostate, TIdx, TPos, TRef, TSrc, TDst, TPath>; - - struct TVMAction { - using EType = EVMAction; - EVMAction Type = VMA_COUNT; - static const ui32 TypeCount = VMA_COUNT; - - public: - template <class T> - bool SetArg(TMaybe<T> arg) { - Y_VERIFY(CurrArg < Y_ARRAY_SIZE(Arg)); - if (arg) { - Arg[CurrArg++] = *arg; - return true; - } else { - return false; - } - } - - public: - TRef GetRef(ui32 arg) const noexcept; - - TSrc GetSrc(ui32 arg) const noexcept; - - TDst GetDst(ui32 arg) const noexcept; - - - ui32 GetPos(ui32 arg) const noexcept; - - ui32 GetIdx(ui32 arg) const noexcept; - - TStringBuf GetKey(ui32 arg) const noexcept; - - TStringBuf GetString(ui32 arg) const noexcept; - - i64 GetIntNumber(ui32 arg) const noexcept; - - TStringBuf GetPath(ui32 arg) const noexcept; - - TString ToString() const; - - private: - TArg Arg[2]; - ui32 CurrArg = 0; - }; - + + struct TVMAction { + using EType = EVMAction; + EVMAction Type = VMA_COUNT; + static const ui32 TypeCount = VMA_COUNT; + + public: + template <class T> + bool SetArg(TMaybe<T> arg) { + Y_VERIFY(CurrArg < Y_ARRAY_SIZE(Arg)); + if (arg) { + Arg[CurrArg++] = *arg; + return true; + } else { + return false; + } + } + + public: + TRef GetRef(ui32 arg) const noexcept; + + TSrc GetSrc(ui32 arg) const noexcept; + + TDst GetDst(ui32 arg) const noexcept; + + + ui32 GetPos(ui32 arg) const noexcept; + + ui32 GetIdx(ui32 arg) const noexcept; + + TStringBuf GetKey(ui32 arg) const noexcept; + + TStringBuf GetString(ui32 arg) const noexcept; + + i64 GetIntNumber(ui32 arg) const noexcept; + + TStringBuf GetPath(ui32 arg) const noexcept; + + TString ToString() const; + + private: + TArg Arg[2]; + ui32 CurrArg = 0; + }; + [[nodiscard]] - inline ui32 GetCountWidth(ui32 cnt) { - return MostSignificantBit(cnt - 1) + 1; - } - -} + inline ui32 GetCountWidth(ui32 cnt) { + return MostSignificantBit(cnt - 1) + 1; + } + +} diff --git a/library/cpp/scheme/tests/fuzz_ops/lib/vm_parse.cpp b/library/cpp/scheme/tests/fuzz_ops/lib/vm_parse.cpp index db9b98abba..a03f5aef53 100644 --- a/library/cpp/scheme/tests/fuzz_ops/lib/vm_parse.cpp +++ b/library/cpp/scheme/tests/fuzz_ops/lib/vm_parse.cpp @@ -1,265 +1,265 @@ -#include "vm_parse.h" - -namespace NSc::NUt { -#define Y_TRY_READ_BITS(state, out, bits, res) do { if (!state.Input.Read(out, bits)) { return res; } } while (false) -#define Y_TRY_READ_BITS_BOOL(state, out, bits) Y_TRY_READ_BITS(state, out, bits, false) -#define Y_TRY_READ_BITS_MAYBE(state, out, bits) Y_TRY_READ_BITS(state, out, bits, Nothing()) - - TMaybe<TIdx> ParseIdx(TVMState& st) { - static_assert(IsPowerOf2(TIdx::ValueCount)); - static const auto bits = GetCountWidth(TIdx::ValueCount); - - TIdx idx; - if (!st.Input.Read(idx.Idx, bits)) { - return Nothing(); - } - return idx; - } - - namespace { - bool DoParsePos(TVMState& st, TPos& pos) { - static const auto bits = GetCountWidth(TPos::ValueCount); - const ui32 sz = st.Memory.size(); - - ui32 neg = -1; - Y_TRY_READ_BITS_BOOL(st, neg, 1); - - ui32 delta = -1; - Y_TRY_READ_BITS_BOOL(st, delta, bits); - - if (neg) { - if (st.Pos < delta) { - return false; - } - pos.Pos = st.Pos - delta; - } else { - if (st.Pos + delta >= sz) { - return false; - } - pos.Pos = st.Pos + delta; - } - - return true; - } - - template <class T> - bool DoParseType(TVMState& state, T& res) { - static const auto bits = GetCountWidth(T::TypeCount); - - ui32 type = -1; - if (state.Input.Read(type, bits) && type < T::TypeCount) { - res.Type = (typename T::EType)(type); - return true; - } else { - return false; - } - } - } - - TMaybe<TPos> ParsePos(TVMState& state) { - TPos res; - if (DoParsePos(state, res)) { - return res; - } - return Nothing(); - } - - TMaybe<TRef> ParseRef(TVMState& state) { - TRef ref; - if (!DoParseType(state, ref)) { - return Nothing(); - } - - switch (ref.Type) { - case TRef::T_REF__POS: - if (!DoParsePos(state, ref)) { - return Nothing(); - } +#include "vm_parse.h" + +namespace NSc::NUt { +#define Y_TRY_READ_BITS(state, out, bits, res) do { if (!state.Input.Read(out, bits)) { return res; } } while (false) +#define Y_TRY_READ_BITS_BOOL(state, out, bits) Y_TRY_READ_BITS(state, out, bits, false) +#define Y_TRY_READ_BITS_MAYBE(state, out, bits) Y_TRY_READ_BITS(state, out, bits, Nothing()) + + TMaybe<TIdx> ParseIdx(TVMState& st) { + static_assert(IsPowerOf2(TIdx::ValueCount)); + static const auto bits = GetCountWidth(TIdx::ValueCount); + + TIdx idx; + if (!st.Input.Read(idx.Idx, bits)) { + return Nothing(); + } + return idx; + } + + namespace { + bool DoParsePos(TVMState& st, TPos& pos) { + static const auto bits = GetCountWidth(TPos::ValueCount); + const ui32 sz = st.Memory.size(); + + ui32 neg = -1; + Y_TRY_READ_BITS_BOOL(st, neg, 1); + + ui32 delta = -1; + Y_TRY_READ_BITS_BOOL(st, delta, bits); + + if (neg) { + if (st.Pos < delta) { + return false; + } + pos.Pos = st.Pos - delta; + } else { + if (st.Pos + delta >= sz) { + return false; + } + pos.Pos = st.Pos + delta; + } + + return true; + } + + template <class T> + bool DoParseType(TVMState& state, T& res) { + static const auto bits = GetCountWidth(T::TypeCount); + + ui32 type = -1; + if (state.Input.Read(type, bits) && type < T::TypeCount) { + res.Type = (typename T::EType)(type); + return true; + } else { + return false; + } + } + } + + TMaybe<TPos> ParsePos(TVMState& state) { + TPos res; + if (DoParsePos(state, res)) { + return res; + } + return Nothing(); + } + + TMaybe<TRef> ParseRef(TVMState& state) { + TRef ref; + if (!DoParseType(state, ref)) { + return Nothing(); + } + + switch (ref.Type) { + case TRef::T_REF__POS: + if (!DoParsePos(state, ref)) { + return Nothing(); + } [[fallthrough]]; - case TRef::T_CREATE_FRONT: - case TRef::T_CREATE_BACK: - return ref; - default: - Y_FAIL(); - } - } - - TMaybe<TSrc> ParseSrc(TVMState& state) { - TSrc src; - if (!DoParseType(state, src)) { - return Nothing(); - } - - switch (src.Type) { - case TSrc::T_LREF__POS: - case TSrc::T_CREF__POS: - case TSrc::T_RREF__POS: - if (!DoParsePos(state, src)) { - return Nothing(); - } - return src; - default: - Y_FAIL(); - } - } - - TMaybe<TDst> ParseDst(TVMState& state) { - TDst dst; - if (!DoParseType(state, dst)) { - return Nothing(); - } - - switch (dst.Type) { - case TDst::T_LREF__POS: - case TDst::T_CREF__POS: - case TDst::T_RREF__POS: - if (!DoParsePos(state, dst)) { - return Nothing(); - } + case TRef::T_CREATE_FRONT: + case TRef::T_CREATE_BACK: + return ref; + default: + Y_FAIL(); + } + } + + TMaybe<TSrc> ParseSrc(TVMState& state) { + TSrc src; + if (!DoParseType(state, src)) { + return Nothing(); + } + + switch (src.Type) { + case TSrc::T_LREF__POS: + case TSrc::T_CREF__POS: + case TSrc::T_RREF__POS: + if (!DoParsePos(state, src)) { + return Nothing(); + } + return src; + default: + Y_FAIL(); + } + } + + TMaybe<TDst> ParseDst(TVMState& state) { + TDst dst; + if (!DoParseType(state, dst)) { + return Nothing(); + } + + switch (dst.Type) { + case TDst::T_LREF__POS: + case TDst::T_CREF__POS: + case TDst::T_RREF__POS: + if (!DoParsePos(state, dst)) { + return Nothing(); + } [[fallthrough]]; - case TDst::T_CREATE_FRONT_LREF: - case TDst::T_CREATE_FRONT_CREF: - case TDst::T_CREATE_FRONT_RREF: - case TDst::T_CREATE_BACK_LREF: - case TDst::T_CREATE_BACK_CREF: - case TDst::T_CREATE_BACK_RREF: - return dst; - default: - Y_FAIL(); - } - } - - TMaybe<TPath> ParsePath(TVMState& state) { - static const ui32 bits = GetCountWidth(TPath::MaxLength); - TPath path; - - ui32 len = -1; - Y_TRY_READ_BITS_MAYBE(state, len, bits); - while (len--) { - ui8 c; - Y_TRY_READ_BITS_MAYBE(state, c, 8); - path.Path.push_back(c); - } - return path; - } - - TMaybe<TVMAction> ParseNextAction(TVMState& state) { - TVMAction res; - - if (!DoParseType(state, res)) { - return Nothing(); - } - - switch (res.Type) { - case VMA_CREATE_BACK: - case VMA_CREATE_FRONT: - case VMA_DESTROY_BACK: - case VMA_DESTROY_FRONT: - - case VMA_SET_DICT: - case VMA_SET_ARRAY: - case VMA_SET_NULL: - case VMA_GET_JSON: - - case VMA_ARRAY_CLEAR: - case VMA_ARRAY_PUSH: - case VMA_DICT_CLEAR: - return res; - - case VMA_JMP__POS: - case VMA_MERGE_UPDATE__POS: - case VMA_MERGE_REVERSE__POS: - case VMA_MERGE_COPY_FROM__POS: - case VMA_SWAP__POS: - if (res.SetArg(ParsePos(state))) { - return res; - } else { - return Nothing(); - } - - case VMA_ARRAY_POP__REF: - if (res.SetArg(ParseRef(state))) { - return res; - } else { - return Nothing(); - } - - case VMA_SET_STRING__IDX: - case VMA_SET_INT_NUMBER__IDX: - case VMA_ARRAY_INSERT__IDX: - case VMA_ARRAY_GET_OR_ADD__IDX: - case VMA_DICT_GET_OR_ADD__IDX: - if (res.SetArg(ParseIdx(state))) { - return res; - } else { - return Nothing(); - } - - case VMA_CREATE_BACK__SRC: - case VMA_CREATE_FRONT__SRC: - case VMA_ASSIGN__SRC: - case VMA_ARRAY_PUSH__SRC: - if (res.SetArg(ParseSrc(state))) { - return res; - } else { - return Nothing(); - } - - case VMA_ARRAY_PUSH__DST: - if (res.SetArg(ParseDst(state))) { - return res; - } else { - return Nothing(); - } - - case VMA_ARRAY_DELETE__IDX_REF: - case VMA_ARRAY_GET__IDX_REF: - case VMA_ARRAY_GET_NO_ADD__IDX_REF: - case VMA_DICT_DELETE__IDX_REF: - case VMA_DICT_GET__IDX_REF: - case VMA_DICT_GET_NO_ADD__IDX_REF: - if (res.SetArg(ParseIdx(state)) && res.SetArg(ParseRef(state))) { - return res; - } else { - return Nothing(); - } - - case VMA_ARRAY_INSERT__IDX_SRC: - case VMA_ARRAY_GET_OR_ADD__IDX_SRC: - case VMA_DICT_GET_OR_ADD__IDX_SRC: - if (res.SetArg(ParseIdx(state)) && res.SetArg(ParseSrc(state))) { - return res; - } else { - return Nothing(); - } - - case VMA_ARRAY_INSERT__IDX_DST: - case VMA_ARRAY_GET_OR_ADD__IDX_DST: - case VMA_DICT_GET_OR_ADD__IDX_DST: - if (res.SetArg(ParseIdx(state)) && res.SetArg(ParseDst(state))) { - return res; - } else { - return Nothing(); - } - - case VMA_EQUAL__POS_POS: - if (res.SetArg(ParsePos(state)) && res.SetArg(ParsePos(state))) { - return res; - } else { - return Nothing(); - } - - case VMA_SELECT_NO_ADD__PATH_REF: - case VMA_SELECT_OR_ADD__PATH_REF: - case VMA_SELECT_AND_DELETE__PATH_REF: - if (res.SetArg(ParsePath(state)) && res.SetArg(ParseRef(state))) { - return res; - } else { - return Nothing(); - } - - default: - return Nothing(); - } - } -} + case TDst::T_CREATE_FRONT_LREF: + case TDst::T_CREATE_FRONT_CREF: + case TDst::T_CREATE_FRONT_RREF: + case TDst::T_CREATE_BACK_LREF: + case TDst::T_CREATE_BACK_CREF: + case TDst::T_CREATE_BACK_RREF: + return dst; + default: + Y_FAIL(); + } + } + + TMaybe<TPath> ParsePath(TVMState& state) { + static const ui32 bits = GetCountWidth(TPath::MaxLength); + TPath path; + + ui32 len = -1; + Y_TRY_READ_BITS_MAYBE(state, len, bits); + while (len--) { + ui8 c; + Y_TRY_READ_BITS_MAYBE(state, c, 8); + path.Path.push_back(c); + } + return path; + } + + TMaybe<TVMAction> ParseNextAction(TVMState& state) { + TVMAction res; + + if (!DoParseType(state, res)) { + return Nothing(); + } + + switch (res.Type) { + case VMA_CREATE_BACK: + case VMA_CREATE_FRONT: + case VMA_DESTROY_BACK: + case VMA_DESTROY_FRONT: + + case VMA_SET_DICT: + case VMA_SET_ARRAY: + case VMA_SET_NULL: + case VMA_GET_JSON: + + case VMA_ARRAY_CLEAR: + case VMA_ARRAY_PUSH: + case VMA_DICT_CLEAR: + return res; + + case VMA_JMP__POS: + case VMA_MERGE_UPDATE__POS: + case VMA_MERGE_REVERSE__POS: + case VMA_MERGE_COPY_FROM__POS: + case VMA_SWAP__POS: + if (res.SetArg(ParsePos(state))) { + return res; + } else { + return Nothing(); + } + + case VMA_ARRAY_POP__REF: + if (res.SetArg(ParseRef(state))) { + return res; + } else { + return Nothing(); + } + + case VMA_SET_STRING__IDX: + case VMA_SET_INT_NUMBER__IDX: + case VMA_ARRAY_INSERT__IDX: + case VMA_ARRAY_GET_OR_ADD__IDX: + case VMA_DICT_GET_OR_ADD__IDX: + if (res.SetArg(ParseIdx(state))) { + return res; + } else { + return Nothing(); + } + + case VMA_CREATE_BACK__SRC: + case VMA_CREATE_FRONT__SRC: + case VMA_ASSIGN__SRC: + case VMA_ARRAY_PUSH__SRC: + if (res.SetArg(ParseSrc(state))) { + return res; + } else { + return Nothing(); + } + + case VMA_ARRAY_PUSH__DST: + if (res.SetArg(ParseDst(state))) { + return res; + } else { + return Nothing(); + } + + case VMA_ARRAY_DELETE__IDX_REF: + case VMA_ARRAY_GET__IDX_REF: + case VMA_ARRAY_GET_NO_ADD__IDX_REF: + case VMA_DICT_DELETE__IDX_REF: + case VMA_DICT_GET__IDX_REF: + case VMA_DICT_GET_NO_ADD__IDX_REF: + if (res.SetArg(ParseIdx(state)) && res.SetArg(ParseRef(state))) { + return res; + } else { + return Nothing(); + } + + case VMA_ARRAY_INSERT__IDX_SRC: + case VMA_ARRAY_GET_OR_ADD__IDX_SRC: + case VMA_DICT_GET_OR_ADD__IDX_SRC: + if (res.SetArg(ParseIdx(state)) && res.SetArg(ParseSrc(state))) { + return res; + } else { + return Nothing(); + } + + case VMA_ARRAY_INSERT__IDX_DST: + case VMA_ARRAY_GET_OR_ADD__IDX_DST: + case VMA_DICT_GET_OR_ADD__IDX_DST: + if (res.SetArg(ParseIdx(state)) && res.SetArg(ParseDst(state))) { + return res; + } else { + return Nothing(); + } + + case VMA_EQUAL__POS_POS: + if (res.SetArg(ParsePos(state)) && res.SetArg(ParsePos(state))) { + return res; + } else { + return Nothing(); + } + + case VMA_SELECT_NO_ADD__PATH_REF: + case VMA_SELECT_OR_ADD__PATH_REF: + case VMA_SELECT_AND_DELETE__PATH_REF: + if (res.SetArg(ParsePath(state)) && res.SetArg(ParseRef(state))) { + return res; + } else { + return Nothing(); + } + + default: + return Nothing(); + } + } +} diff --git a/library/cpp/scheme/tests/fuzz_ops/lib/vm_parse.h b/library/cpp/scheme/tests/fuzz_ops/lib/vm_parse.h index 7c5165cd28..b4aba4e4d4 100644 --- a/library/cpp/scheme/tests/fuzz_ops/lib/vm_parse.h +++ b/library/cpp/scheme/tests/fuzz_ops/lib/vm_parse.h @@ -1,16 +1,16 @@ -#pragma once - -#include "vm_defs.h" - -namespace NSc::NUt { - - TMaybe<TIdx> ParseIdx(TVMState& st); - TMaybe<TPos> ParsePos(TVMState& state); - TMaybe<TRef> ParseRef(TVMState& state); - TMaybe<TSrc> ParseSrc(TVMState& state); - TMaybe<TDst> ParseDst(TVMState& state); - TMaybe<TPath> ParsePath(TVMState& state); - - TMaybe<TVMAction> ParseNextAction(TVMState& state); - -} +#pragma once + +#include "vm_defs.h" + +namespace NSc::NUt { + + TMaybe<TIdx> ParseIdx(TVMState& st); + TMaybe<TPos> ParsePos(TVMState& state); + TMaybe<TRef> ParseRef(TVMState& state); + TMaybe<TSrc> ParseSrc(TVMState& state); + TMaybe<TDst> ParseDst(TVMState& state); + TMaybe<TPath> ParsePath(TVMState& state); + + TMaybe<TVMAction> ParseNextAction(TVMState& state); + +} diff --git a/library/cpp/scheme/tests/fuzz_ops/lib/ya.make b/library/cpp/scheme/tests/fuzz_ops/lib/ya.make index fb38b8f426..279a2ca2d4 100644 --- a/library/cpp/scheme/tests/fuzz_ops/lib/ya.make +++ b/library/cpp/scheme/tests/fuzz_ops/lib/ya.make @@ -1,23 +1,23 @@ -LIBRARY() - -OWNER( - g:blender - g:middle - g:upper - velavokr -) - -GENERATE_ENUM_SERIALIZATION(vm_defs.h) - -SRCS( - fuzz_ops.cpp - vm_apply.cpp - vm_defs.cpp - vm_parse.cpp -) - -PEERDIR( +LIBRARY() + +OWNER( + g:blender + g:middle + g:upper + velavokr +) + +GENERATE_ENUM_SERIALIZATION(vm_defs.h) + +SRCS( + fuzz_ops.cpp + vm_apply.cpp + vm_defs.cpp + vm_parse.cpp +) + +PEERDIR( library/cpp/scheme -) - -END() +) + +END() diff --git a/library/cpp/scheme/tests/fuzz_ops/ut/vm_parse_ut.cpp b/library/cpp/scheme/tests/fuzz_ops/ut/vm_parse_ut.cpp index 006b59ebe3..ce3786a671 100644 --- a/library/cpp/scheme/tests/fuzz_ops/ut/vm_parse_ut.cpp +++ b/library/cpp/scheme/tests/fuzz_ops/ut/vm_parse_ut.cpp @@ -1,174 +1,174 @@ #include <library/cpp/scheme/tests/fuzz_ops/lib/vm_parse.h> #include <library/cpp/testing/unittest/registar.h> - -Y_UNIT_TEST_SUITE(TestParseNextAction) { - using namespace NSc::NUt; - - Y_UNIT_TEST(TestWidth) { - UNIT_ASSERT_VALUES_EQUAL(GetCountWidth(TIdx::ValueCount), 2); - UNIT_ASSERT_VALUES_EQUAL(GetCountWidth(TPos::ValueCount), 4); - UNIT_ASSERT_VALUES_EQUAL(GetCountWidth(TRef::TypeCount), 2); - UNIT_ASSERT_VALUES_EQUAL(GetCountWidth(TSrc::TypeCount), 2); - UNIT_ASSERT_VALUES_EQUAL(GetCountWidth(TDst::TypeCount), 4); - UNIT_ASSERT_VALUES_EQUAL(GetCountWidth(TPath::MaxLength), 5); - UNIT_ASSERT_VALUES_EQUAL(GetCountWidth(TVMAction::TypeCount), 6); - } - - Y_UNIT_TEST(TestParseIdx) { - { - TVMState st{"", 1, 0}; - UNIT_ASSERT(!ParseIdx(st)); - } - { - TVMState st{"\x03", 1, 0}; - auto idx = ParseIdx(st); - UNIT_ASSERT(idx); - UNIT_ASSERT_VALUES_EQUAL(idx->Idx, 3); - } - } - - void DoTestParsePosFailure(const TStringBuf inp, const ui32 memSz, const ui32 curPos) { - TVMState st{inp, memSz, curPos}; - UNIT_ASSERT(!ParsePos(st)); - } - + +Y_UNIT_TEST_SUITE(TestParseNextAction) { + using namespace NSc::NUt; + + Y_UNIT_TEST(TestWidth) { + UNIT_ASSERT_VALUES_EQUAL(GetCountWidth(TIdx::ValueCount), 2); + UNIT_ASSERT_VALUES_EQUAL(GetCountWidth(TPos::ValueCount), 4); + UNIT_ASSERT_VALUES_EQUAL(GetCountWidth(TRef::TypeCount), 2); + UNIT_ASSERT_VALUES_EQUAL(GetCountWidth(TSrc::TypeCount), 2); + UNIT_ASSERT_VALUES_EQUAL(GetCountWidth(TDst::TypeCount), 4); + UNIT_ASSERT_VALUES_EQUAL(GetCountWidth(TPath::MaxLength), 5); + UNIT_ASSERT_VALUES_EQUAL(GetCountWidth(TVMAction::TypeCount), 6); + } + + Y_UNIT_TEST(TestParseIdx) { + { + TVMState st{"", 1, 0}; + UNIT_ASSERT(!ParseIdx(st)); + } + { + TVMState st{"\x03", 1, 0}; + auto idx = ParseIdx(st); + UNIT_ASSERT(idx); + UNIT_ASSERT_VALUES_EQUAL(idx->Idx, 3); + } + } + + void DoTestParsePosFailure(const TStringBuf inp, const ui32 memSz, const ui32 curPos) { + TVMState st{inp, memSz, curPos}; + UNIT_ASSERT(!ParsePos(st)); + } + [[nodiscard]] - ui32 DoTestParsePosSuccess(TVMState& st) { - const auto pos = ParsePos(st); - UNIT_ASSERT(pos); - return pos->Pos; - } - + ui32 DoTestParsePosSuccess(TVMState& st) { + const auto pos = ParsePos(st); + UNIT_ASSERT(pos); + return pos->Pos; + } + [[nodiscard]] - ui32 DoTestParsePosSuccess(const TStringBuf inp, const ui32 memSz, const ui32 curPos) { - TVMState st{inp, memSz, curPos}; - return DoTestParsePosSuccess(st); - } - - Y_UNIT_TEST(TestParsePos) { - DoTestParsePosFailure("", 1, 0); - + ui32 DoTestParsePosSuccess(const TStringBuf inp, const ui32 memSz, const ui32 curPos) { + TVMState st{inp, memSz, curPos}; + return DoTestParsePosSuccess(st); + } + + Y_UNIT_TEST(TestParsePos) { + DoTestParsePosFailure("", 1, 0); + UNIT_ASSERT_VALUES_EQUAL(DoTestParsePosSuccess(TStringBuf("\x00"sv), 1, 0), 0); UNIT_ASSERT_VALUES_EQUAL(DoTestParsePosSuccess(TStringBuf("\x01"sv), 1, 0), 0); - + DoTestParsePosFailure(TStringBuf("\x02"sv), 1, 0); DoTestParsePosFailure(TStringBuf("\x03"sv), 2, 0); - + UNIT_ASSERT_VALUES_EQUAL(DoTestParsePosSuccess(TStringBuf("\x02"sv), 2, 0), 1); UNIT_ASSERT_VALUES_EQUAL(DoTestParsePosSuccess(TStringBuf("\x03"sv), 2, 1), 0); - + UNIT_ASSERT_VALUES_EQUAL(DoTestParsePosSuccess(TStringBuf("\x0E"sv), 8, 0), 7); UNIT_ASSERT_VALUES_EQUAL(DoTestParsePosSuccess(TStringBuf("\x0F"sv), 8, 7), 0); - - { + + { TVMState st{TStringBuf("\xDE\x7B"), 16, 0}; - UNIT_ASSERT_VALUES_EQUAL(DoTestParsePosSuccess(st), 15); - UNIT_ASSERT_VALUES_EQUAL(DoTestParsePosSuccess(st), 15); - UNIT_ASSERT_VALUES_EQUAL(DoTestParsePosSuccess(st), 15); - UNIT_ASSERT(!ParsePos(st)); - } - { + UNIT_ASSERT_VALUES_EQUAL(DoTestParsePosSuccess(st), 15); + UNIT_ASSERT_VALUES_EQUAL(DoTestParsePosSuccess(st), 15); + UNIT_ASSERT_VALUES_EQUAL(DoTestParsePosSuccess(st), 15); + UNIT_ASSERT(!ParsePos(st)); + } + { TVMState st{TStringBuf("\xFF\x7F"), 16, 15}; - UNIT_ASSERT_VALUES_EQUAL(DoTestParsePosSuccess(st), 0); - UNIT_ASSERT_VALUES_EQUAL(DoTestParsePosSuccess(st), 0); - UNIT_ASSERT_VALUES_EQUAL(DoTestParsePosSuccess(st), 0); - UNIT_ASSERT(!ParsePos(st)); - } - } - - void DoTestParseRefFailure(const TStringBuf inp, const ui32 memSz, const ui32 curPos) { - TVMState st{inp, memSz, curPos}; - UNIT_ASSERT(!ParseRef(st)); - } - + UNIT_ASSERT_VALUES_EQUAL(DoTestParsePosSuccess(st), 0); + UNIT_ASSERT_VALUES_EQUAL(DoTestParsePosSuccess(st), 0); + UNIT_ASSERT_VALUES_EQUAL(DoTestParsePosSuccess(st), 0); + UNIT_ASSERT(!ParsePos(st)); + } + } + + void DoTestParseRefFailure(const TStringBuf inp, const ui32 memSz, const ui32 curPos) { + TVMState st{inp, memSz, curPos}; + UNIT_ASSERT(!ParseRef(st)); + } + [[nodiscard]] - auto DoTestParseRefSuccess(TVMState& st) { - const auto ref = ParseRef(st); - UNIT_ASSERT(ref); - return std::make_pair(ref->Pos, ref->Type); - } - + auto DoTestParseRefSuccess(TVMState& st) { + const auto ref = ParseRef(st); + UNIT_ASSERT(ref); + return std::make_pair(ref->Pos, ref->Type); + } + [[nodiscard]] - auto DoTestParseRefSuccess(const TStringBuf inp, const ui32 memSz, const ui32 curPos) { - TVMState st{inp, memSz, curPos}; - return DoTestParseRefSuccess(st); - } - - Y_UNIT_TEST(TestParseRef) { - DoTestParseRefFailure("", 1, 0); - + auto DoTestParseRefSuccess(const TStringBuf inp, const ui32 memSz, const ui32 curPos) { + TVMState st{inp, memSz, curPos}; + return DoTestParseRefSuccess(st); + } + + Y_UNIT_TEST(TestParseRef) { + DoTestParseRefFailure("", 1, 0); + UNIT_ASSERT_VALUES_EQUAL(DoTestParseRefSuccess(TStringBuf("\x00"sv), 1, 0), std::make_pair((ui32)-1, TRef::T_CREATE_FRONT)); UNIT_ASSERT_VALUES_EQUAL(DoTestParseRefSuccess(TStringBuf("\x01"sv), 1, 0), std::make_pair((ui32)-1, TRef::T_CREATE_BACK)); UNIT_ASSERT_VALUES_EQUAL(DoTestParseRefSuccess(TStringBuf("\x0A"sv), 2, 0), std::make_pair(1u, TRef::T_REF__POS)); - + DoTestParseRefFailure(TStringBuf("\x12"), 1, 0); DoTestParseRefFailure(TStringBuf("\x03"sv), 1, 0); - - { + + { TVMState st{TStringBuf("\x7A\x7D"), 16, 0}; - UNIT_ASSERT_VALUES_EQUAL(DoTestParseRefSuccess(st), std::make_pair(15u, TRef::T_REF__POS)); - UNIT_ASSERT_VALUES_EQUAL(DoTestParseRefSuccess(st), std::make_pair(15u, TRef::T_REF__POS)); - UNIT_ASSERT_VALUES_EQUAL(DoTestParseRefSuccess(st), std::make_pair((ui32)-1, TRef::T_CREATE_BACK)); - UNIT_ASSERT(!ParseRef(st)); - } - } - - void DoTestParseSrcFailure(const TStringBuf inp, const ui32 memSz, const ui32 curPos) { - TVMState st{inp, memSz, curPos}; - UNIT_ASSERT(!ParseSrc(st)); - } - + UNIT_ASSERT_VALUES_EQUAL(DoTestParseRefSuccess(st), std::make_pair(15u, TRef::T_REF__POS)); + UNIT_ASSERT_VALUES_EQUAL(DoTestParseRefSuccess(st), std::make_pair(15u, TRef::T_REF__POS)); + UNIT_ASSERT_VALUES_EQUAL(DoTestParseRefSuccess(st), std::make_pair((ui32)-1, TRef::T_CREATE_BACK)); + UNIT_ASSERT(!ParseRef(st)); + } + } + + void DoTestParseSrcFailure(const TStringBuf inp, const ui32 memSz, const ui32 curPos) { + TVMState st{inp, memSz, curPos}; + UNIT_ASSERT(!ParseSrc(st)); + } + [[nodiscard]] - auto DoTestParseSrcSuccess(TVMState& st) { - const auto src = ParseSrc(st); - UNIT_ASSERT(src); - return std::make_pair(src->Pos, src->Type); - } - + auto DoTestParseSrcSuccess(TVMState& st) { + const auto src = ParseSrc(st); + UNIT_ASSERT(src); + return std::make_pair(src->Pos, src->Type); + } + [[nodiscard]] - auto DoTestParseSrcSuccess(const TStringBuf inp, const ui32 memSz, const ui32 curPos) { - TVMState st{inp, memSz, curPos}; - return DoTestParseSrcSuccess(st); - } - - Y_UNIT_TEST(TestParseSrc) { - DoTestParseSrcFailure("", 1, 0); - + auto DoTestParseSrcSuccess(const TStringBuf inp, const ui32 memSz, const ui32 curPos) { + TVMState st{inp, memSz, curPos}; + return DoTestParseSrcSuccess(st); + } + + Y_UNIT_TEST(TestParseSrc) { + DoTestParseSrcFailure("", 1, 0); + UNIT_ASSERT_VALUES_EQUAL(DoTestParseSrcSuccess(TStringBuf("\x08"sv), 2, 0), std::make_pair(1u, TSrc::T_LREF__POS)); UNIT_ASSERT_VALUES_EQUAL(DoTestParseSrcSuccess(TStringBuf("\x09"sv), 2, 0), std::make_pair(1u, TSrc::T_CREF__POS)); UNIT_ASSERT_VALUES_EQUAL(DoTestParseSrcSuccess(TStringBuf("\x0A"sv), 2, 0), std::make_pair(1u, TSrc::T_RREF__POS)); - + DoTestParseSrcFailure(TStringBuf("\x03"sv), 1, 0); - - { + + { TVMState st{TStringBuf("\x7A\x7D"), 16, 0}; - UNIT_ASSERT_VALUES_EQUAL(DoTestParseSrcSuccess(st), std::make_pair(15u, TSrc::T_RREF__POS)); - UNIT_ASSERT_VALUES_EQUAL(DoTestParseSrcSuccess(st), std::make_pair(15u, TSrc::T_RREF__POS)); - UNIT_ASSERT(!ParseSrc(st)); - } - } - - void DoTestParseDstFailure(const TStringBuf inp, const ui32 memSz, const ui32 curPos) { - TVMState st{inp, memSz, curPos}; - UNIT_ASSERT(!ParseDst(st)); - } - + UNIT_ASSERT_VALUES_EQUAL(DoTestParseSrcSuccess(st), std::make_pair(15u, TSrc::T_RREF__POS)); + UNIT_ASSERT_VALUES_EQUAL(DoTestParseSrcSuccess(st), std::make_pair(15u, TSrc::T_RREF__POS)); + UNIT_ASSERT(!ParseSrc(st)); + } + } + + void DoTestParseDstFailure(const TStringBuf inp, const ui32 memSz, const ui32 curPos) { + TVMState st{inp, memSz, curPos}; + UNIT_ASSERT(!ParseDst(st)); + } + [[nodiscard]] - auto DoTestParseDstSuccess(TVMState& st) { - const auto dst = ParseDst(st); - UNIT_ASSERT(dst); - return std::make_pair(dst->Pos, dst->Type); - } - + auto DoTestParseDstSuccess(TVMState& st) { + const auto dst = ParseDst(st); + UNIT_ASSERT(dst); + return std::make_pair(dst->Pos, dst->Type); + } + [[nodiscard]] - auto DoTestParseDstSuccess(const TStringBuf inp, const ui32 memSz, const ui32 curPos) { - TVMState st{inp, memSz, curPos}; - return DoTestParseDstSuccess(st); - } - - Y_UNIT_TEST(TestParseDst) { - DoTestParseDstFailure("", 1, 0); - + auto DoTestParseDstSuccess(const TStringBuf inp, const ui32 memSz, const ui32 curPos) { + TVMState st{inp, memSz, curPos}; + return DoTestParseDstSuccess(st); + } + + Y_UNIT_TEST(TestParseDst) { + DoTestParseDstFailure("", 1, 0); + UNIT_ASSERT_VALUES_EQUAL(DoTestParseDstSuccess(TStringBuf("\x00"sv), 1, 0), std::make_pair((ui32)-1, TDst::T_CREATE_FRONT_LREF)); UNIT_ASSERT_VALUES_EQUAL(DoTestParseDstSuccess(TStringBuf("\x01"sv), 1, 0), std::make_pair((ui32)-1, TDst::T_CREATE_FRONT_CREF)); UNIT_ASSERT_VALUES_EQUAL(DoTestParseDstSuccess(TStringBuf("\x02"sv), 1, 0), std::make_pair((ui32)-1, TDst::T_CREATE_FRONT_RREF)); @@ -178,48 +178,48 @@ Y_UNIT_TEST_SUITE(TestParseNextAction) { UNIT_ASSERT_VALUES_EQUAL(DoTestParseDstSuccess(TStringBuf("\x26\x00"sv), 2, 0), std::make_pair(1u, TDst::T_LREF__POS)); UNIT_ASSERT_VALUES_EQUAL(DoTestParseDstSuccess(TStringBuf("\x27\x00"sv), 2, 0), std::make_pair(1u, TDst::T_CREF__POS)); UNIT_ASSERT_VALUES_EQUAL(DoTestParseDstSuccess(TStringBuf("\x28\x00"sv), 2, 0), std::make_pair(1u, TDst::T_RREF__POS)); - + DoTestParseDstFailure(TStringBuf("\x06"sv), 1, 0); DoTestParseDstFailure(TStringBuf("\x09\x00"sv), 1, 0); - - { + + { TVMState st{TStringBuf("\x14\xE7\x09"sv), 16, 0}; - // 4=4 - UNIT_ASSERT_VALUES_EQUAL(DoTestParseDstSuccess(st), std::make_pair((ui32)-1, TDst::T_CREATE_BACK_CREF)); - // 4=8 - UNIT_ASSERT_VALUES_EQUAL(DoTestParseDstSuccess(st), std::make_pair((ui32)-1, TDst::T_CREATE_FRONT_CREF)); - // 4+1+4=17 - UNIT_ASSERT_VALUES_EQUAL(DoTestParseDstSuccess(st), std::make_pair(15u, TDst::T_CREF__POS)); - // 4=21 - UNIT_ASSERT_VALUES_EQUAL(DoTestParseDstSuccess(st), std::make_pair((ui32)-1, TDst::T_CREATE_BACK_CREF)); - UNIT_ASSERT(!ParseDst(st)); - } - } - - void DoTestParsePathFailure(const TStringBuf inp, const ui32 memSz, const ui32 curPos) { - TVMState st{inp, memSz, curPos}; - UNIT_ASSERT(!ParsePath(st)); - } - + // 4=4 + UNIT_ASSERT_VALUES_EQUAL(DoTestParseDstSuccess(st), std::make_pair((ui32)-1, TDst::T_CREATE_BACK_CREF)); + // 4=8 + UNIT_ASSERT_VALUES_EQUAL(DoTestParseDstSuccess(st), std::make_pair((ui32)-1, TDst::T_CREATE_FRONT_CREF)); + // 4+1+4=17 + UNIT_ASSERT_VALUES_EQUAL(DoTestParseDstSuccess(st), std::make_pair(15u, TDst::T_CREF__POS)); + // 4=21 + UNIT_ASSERT_VALUES_EQUAL(DoTestParseDstSuccess(st), std::make_pair((ui32)-1, TDst::T_CREATE_BACK_CREF)); + UNIT_ASSERT(!ParseDst(st)); + } + } + + void DoTestParsePathFailure(const TStringBuf inp, const ui32 memSz, const ui32 curPos) { + TVMState st{inp, memSz, curPos}; + UNIT_ASSERT(!ParsePath(st)); + } + [[nodiscard]] - auto DoTestParsePathSuccess(TVMState& st) { - const auto path = ParsePath(st); - UNIT_ASSERT(path); - return path->Path; - } - + auto DoTestParsePathSuccess(TVMState& st) { + const auto path = ParsePath(st); + UNIT_ASSERT(path); + return path->Path; + } + [[nodiscard]] - auto DoTestParsePathSuccess(const TStringBuf inp, const ui32 memSz, const ui32 curPos) { - TVMState st{inp, memSz, curPos}; - return DoTestParsePathSuccess(st); - } - - Y_UNIT_TEST(TestParsePath) { - DoTestParsePathFailure("", 1, 0); - + auto DoTestParsePathSuccess(const TStringBuf inp, const ui32 memSz, const ui32 curPos) { + TVMState st{inp, memSz, curPos}; + return DoTestParsePathSuccess(st); + } + + Y_UNIT_TEST(TestParsePath) { + DoTestParsePathFailure("", 1, 0); + UNIT_ASSERT_VALUES_EQUAL(DoTestParsePathSuccess(TStringBuf("\x00"sv), 1, 0), TStringBuf("")); UNIT_ASSERT_VALUES_EQUAL(DoTestParsePathSuccess(TStringBuf("\x21\x0C"sv), 1, 0), TStringBuf("a")); - - DoTestParsePathFailure("\x22\x0C", 1, 0); - } -}; + + DoTestParsePathFailure("\x22\x0C", 1, 0); + } +}; diff --git a/library/cpp/scheme/tests/fuzz_ops/ut/ya.make b/library/cpp/scheme/tests/fuzz_ops/ut/ya.make index 7244ab9f95..5c933518ea 100644 --- a/library/cpp/scheme/tests/fuzz_ops/ut/ya.make +++ b/library/cpp/scheme/tests/fuzz_ops/ut/ya.make @@ -1,15 +1,15 @@ -UNITTEST() - -OWNER(velavokr) - -PEERDIR( +UNITTEST() + +OWNER(velavokr) + +PEERDIR( library/cpp/testing/unittest library/cpp/scheme library/cpp/scheme/tests/fuzz_ops/lib -) - -SRCS( - vm_parse_ut.cpp -) - -END() +) + +SRCS( + vm_parse_ut.cpp +) + +END() diff --git a/library/cpp/scheme/tests/fuzz_ops/ya.make b/library/cpp/scheme/tests/fuzz_ops/ya.make index 128094f546..025241ef20 100644 --- a/library/cpp/scheme/tests/fuzz_ops/ya.make +++ b/library/cpp/scheme/tests/fuzz_ops/ya.make @@ -1,18 +1,18 @@ -FUZZ() - -OWNER( - g:blender - g:middle - g:upper - velavokr -) - -SRCS( - fuzz_ops.cpp -) - -PEERDIR( +FUZZ() + +OWNER( + g:blender + g:middle + g:upper + velavokr +) + +SRCS( + fuzz_ops.cpp +) + +PEERDIR( library/cpp/scheme/tests/fuzz_ops/lib -) - -END() +) + +END() diff --git a/library/cpp/scheme/tests/ut/fuzz_ops_found_bugs_ut.cpp b/library/cpp/scheme/tests/ut/fuzz_ops_found_bugs_ut.cpp index 3136d9b1cc..a445b0f87c 100644 --- a/library/cpp/scheme/tests/ut/fuzz_ops_found_bugs_ut.cpp +++ b/library/cpp/scheme/tests/ut/fuzz_ops_found_bugs_ut.cpp @@ -1,12 +1,12 @@ #include <library/cpp/scheme/scheme.h> #include <library/cpp/scheme/tests/fuzz_ops/lib/fuzz_ops.h> #include <library/cpp/testing/unittest/registar.h> -#include <util/string/hex.h> - -Y_UNIT_TEST_SUITE(TTestSchemeFuzzOpsFoundBugs) { - using namespace NSc::NUt; - - Y_UNIT_TEST(TestBug1) { - FuzzOps(HexDecode("98040129000525"), true); - } -}; +#include <util/string/hex.h> + +Y_UNIT_TEST_SUITE(TTestSchemeFuzzOpsFoundBugs) { + using namespace NSc::NUt; + + Y_UNIT_TEST(TestBug1) { + FuzzOps(HexDecode("98040129000525"), true); + } +}; diff --git a/library/cpp/scheme/tests/ut/scheme_json_ut.cpp b/library/cpp/scheme/tests/ut/scheme_json_ut.cpp index 84fdb26808..daeb2654f9 100644 --- a/library/cpp/scheme/tests/ut/scheme_json_ut.cpp +++ b/library/cpp/scheme/tests/ut/scheme_json_ut.cpp @@ -1,89 +1,89 @@ #include <library/cpp/scheme/scimpl_private.h> #include <library/cpp/scheme/ut_utils/scheme_ut_utils.h> #include <library/cpp/testing/unittest/registar.h> - -#include <util/stream/null.h> -#include <util/string/subst.h> -#include <util/string/util.h> - -#include <type_traits> + +#include <util/stream/null.h> +#include <util/string/subst.h> +#include <util/string/util.h> + +#include <type_traits> #include <library/cpp/string_utils/quote/quote.h> - + using namespace std::string_view_literals; -Y_UNIT_TEST_SUITE(TSchemeJsonTest) { - Y_UNIT_TEST(TestJson) { - const char* json = "[\n" +Y_UNIT_TEST_SUITE(TSchemeJsonTest) { + Y_UNIT_TEST(TestJson) { + const char* json = "[\n" " {\n" " \"url\":\"foo\",\n" " \"title\":\"bar\",\n" " \"passages\":[\"foo\", \"bar\"],\n" " }\n" "]"; - - { - const NSc::TValue& v = NSc::TValue::FromJson(json); - UNIT_ASSERT_VALUES_EQUAL("bar", (TStringBuf)v.TrySelect("0/passages/1")); - UNIT_ASSERT(v.PathExists("0/passages/0")); - UNIT_ASSERT(v.PathExists("[0]/[passages]/[0]")); - UNIT_ASSERT(v.PathExists("[0][passages][0]")); - UNIT_ASSERT(v.PathExists("")); - UNIT_ASSERT(!v.PathExists("`")); - UNIT_ASSERT(v.TrySelect("").Has(0)); - UNIT_ASSERT(!v.PathExists("1")); - UNIT_ASSERT(!v.PathExists("0/passages1")); - UNIT_ASSERT(!v.PathExists("0/passages/2")); - UNIT_ASSERT(!v.PathExists("0/passages/2")); - UNIT_ASSERT_VALUES_EQUAL(0, (double)v.TrySelect("0/passages/2")); - UNIT_ASSERT(!v.PathExists("0/passages/2")); - } - { - const NSc::TValue& vv = NSc::TValue::FromJson("[ test ]]"); - UNIT_ASSERT(vv.IsNull()); - } - { - const char* json = "[a,b],[a,b]"; - const NSc::TValue& v = NSc::TValue::FromJson(json); - UNIT_ASSERT(v.IsNull()); - } - { - const char* json = "[null,null]"; - const NSc::TValue& v = NSc::TValue::FromJson(json); - UNIT_ASSERT(v.PathExists("1")); - UNIT_ASSERT(!v.PathExists("2")); - } - { - const char* json = "{ a : b : c }"; - NSc::TValue v; - UNIT_ASSERT(!NSc::TValue::FromJson(v, json)); - UNIT_ASSERT(v.IsNull()); - } - { - const char* json = "[a:b]"; + + { + const NSc::TValue& v = NSc::TValue::FromJson(json); + UNIT_ASSERT_VALUES_EQUAL("bar", (TStringBuf)v.TrySelect("0/passages/1")); + UNIT_ASSERT(v.PathExists("0/passages/0")); + UNIT_ASSERT(v.PathExists("[0]/[passages]/[0]")); + UNIT_ASSERT(v.PathExists("[0][passages][0]")); + UNIT_ASSERT(v.PathExists("")); + UNIT_ASSERT(!v.PathExists("`")); + UNIT_ASSERT(v.TrySelect("").Has(0)); + UNIT_ASSERT(!v.PathExists("1")); + UNIT_ASSERT(!v.PathExists("0/passages1")); + UNIT_ASSERT(!v.PathExists("0/passages/2")); + UNIT_ASSERT(!v.PathExists("0/passages/2")); + UNIT_ASSERT_VALUES_EQUAL(0, (double)v.TrySelect("0/passages/2")); + UNIT_ASSERT(!v.PathExists("0/passages/2")); + } + { + const NSc::TValue& vv = NSc::TValue::FromJson("[ test ]]"); + UNIT_ASSERT(vv.IsNull()); + } + { + const char* json = "[a,b],[a,b]"; + const NSc::TValue& v = NSc::TValue::FromJson(json); + UNIT_ASSERT(v.IsNull()); + } + { + const char* json = "[null,null]"; + const NSc::TValue& v = NSc::TValue::FromJson(json); + UNIT_ASSERT(v.PathExists("1")); + UNIT_ASSERT(!v.PathExists("2")); + } + { + const char* json = "{ a : b : c }"; + NSc::TValue v; + UNIT_ASSERT(!NSc::TValue::FromJson(v, json)); + UNIT_ASSERT(v.IsNull()); + } + { + const char* json = "[a:b]"; UNIT_ASSERT(NSc::TValue::FromJson(json).IsNull()); - } - { - UNIT_ASSERT_VALUES_EQUAL("{\n \"a\" : \"b\",\n \"c\" : \"d\"\n}", - NSc::TValue::FromJson("{a:b,c:d}").ToJson(NSc::TValue::JO_PRETTY)); - } - } - - Y_UNIT_TEST(TestSafeJson) { + } + { + UNIT_ASSERT_VALUES_EQUAL("{\n \"a\" : \"b\",\n \"c\" : \"d\"\n}", + NSc::TValue::FromJson("{a:b,c:d}").ToJson(NSc::TValue::JO_PRETTY)); + } + } + + Y_UNIT_TEST(TestSafeJson) { TString ss; - ss.reserve(256); - - for (int i = 0; i < 256; ++i) { - ss.append((char)i); - } - - NSc::TValue v; - v[ss] = "xxx"; - v["xxx"] = ss; - - UNIT_ASSERT_VALUES_EQUAL("{\"xxx\":null}", v.ToJson(NSc::TValue::JO_SKIP_UNSAFE)); - UNIT_ASSERT_VALUES_EQUAL("{\"xxx\":null}", v.ToJson(NSc::TValue::JO_SAFE)); - - UNIT_ASSERT_VALUES_EQUAL("{" + ss.reserve(256); + + for (int i = 0; i < 256; ++i) { + ss.append((char)i); + } + + NSc::TValue v; + v[ss] = "xxx"; + v["xxx"] = ss; + + UNIT_ASSERT_VALUES_EQUAL("{\"xxx\":null}", v.ToJson(NSc::TValue::JO_SKIP_UNSAFE)); + UNIT_ASSERT_VALUES_EQUAL("{\"xxx\":null}", v.ToJson(NSc::TValue::JO_SAFE)); + + UNIT_ASSERT_VALUES_EQUAL("{" "\"\\u0000\\u0001\\u0002\\u0003\\u0004\\u0005\\u0006\\u0007\\b\\t\\n\\u000B\\f\\r" "\\u000E\\u000F\\u0010\\u0011\\u0012\\u0013\\u0014\\u0015\\u0016\\u0017\\u0018" "\\u0019\\u001A\\u001B\\u001C\\u001D\\u001E\\u001F !\\\"#$%&'()*+,-./0123456789" @@ -110,33 +110,33 @@ Y_UNIT_TEST_SUITE(TSchemeJsonTest) { "\\xF5\\xF6\\xF7\\xF8\\xF9\\xFA\\xFB\\xFC\\xFD\\xFE\\xFF\"" "}", v.ToJson(NSc::TValue::JO_SORT_KEYS)); - UNIT_ASSERT(NSc::TValue::Equal(v, NSc::TValue::FromJson(v.ToJson()))); - - { - NSc::TValue value; + UNIT_ASSERT(NSc::TValue::Equal(v, NSc::TValue::FromJson(v.ToJson()))); + + { + NSc::TValue value; TString articleName{"\xC2\xC2\xCF"}; - value["text"] = articleName; - UNIT_ASSERT_VALUES_EQUAL(value.ToJson(), "{\"text\":\"\xC2\xC2\xCF\"}"); - UNIT_ASSERT_VALUES_EQUAL(value.ToJsonSafe(), "{\"text\":null}"); - } - } - - Y_UNIT_TEST(TestJsonEscape) { + value["text"] = articleName; + UNIT_ASSERT_VALUES_EQUAL(value.ToJson(), "{\"text\":\"\xC2\xC2\xCF\"}"); + UNIT_ASSERT_VALUES_EQUAL(value.ToJsonSafe(), "{\"text\":null}"); + } + } + + Y_UNIT_TEST(TestJsonEscape) { NSc::TValue v("\10\7\6\5\4\3\2\1\0"sv); - UNIT_ASSERT_VALUES_EQUAL(v.ToJson(), "\"\\b\\u0007\\u0006\\u0005\\u0004\\u0003\\u0002\\u0001\\u0000\""); - } - - Y_UNIT_TEST(TestStrictJson) { - UNIT_ASSERT_NO_EXCEPTION(NSc::TValue::FromJsonThrow("{a:b}")); - UNIT_ASSERT_EXCEPTION(NSc::TValue::FromJsonThrow("{a:b}", NSc::TValue::JO_PARSER_STRICT), yexception); - UNIT_ASSERT_NO_EXCEPTION(NSc::TValue::FromJsonThrow("{\"a\":\"b\"}", NSc::TValue::JO_PARSER_STRICT)); - } - - Y_UNIT_TEST(TestJsonValue) { - NSc::TValue a = NSc::NUt::AssertFromJson("{a:[null,-1,2,3.4,str,{b:{c:d}}],e:f}"); - NSc::TValue b = NSc::TValue::FromJsonValue(a.ToJsonValue()); - UNIT_ASSERT_JSON_EQ_JSON(a, b); - } + UNIT_ASSERT_VALUES_EQUAL(v.ToJson(), "\"\\b\\u0007\\u0006\\u0005\\u0004\\u0003\\u0002\\u0001\\u0000\""); + } + + Y_UNIT_TEST(TestStrictJson) { + UNIT_ASSERT_NO_EXCEPTION(NSc::TValue::FromJsonThrow("{a:b}")); + UNIT_ASSERT_EXCEPTION(NSc::TValue::FromJsonThrow("{a:b}", NSc::TValue::JO_PARSER_STRICT), yexception); + UNIT_ASSERT_NO_EXCEPTION(NSc::TValue::FromJsonThrow("{\"a\":\"b\"}", NSc::TValue::JO_PARSER_STRICT)); + } + + Y_UNIT_TEST(TestJsonValue) { + NSc::TValue a = NSc::NUt::AssertFromJson("{a:[null,-1,2,3.4,str,{b:{c:d}}],e:f}"); + NSc::TValue b = NSc::TValue::FromJsonValue(a.ToJsonValue()); + UNIT_ASSERT_JSON_EQ_JSON(a, b); + } Y_UNIT_TEST(TestJsonEmptyContainers) { { @@ -151,11 +151,11 @@ Y_UNIT_TEST_SUITE(TSchemeJsonTest) { } } - Y_UNIT_TEST(TestDuplicateKeys) { + Y_UNIT_TEST(TestDuplicateKeys) { const TStringBuf duplicatedKeys = "{\"a\":[{\"b\":1, \"b\":42}]}"; UNIT_ASSERT_NO_EXCEPTION(NSc::TValue::FromJsonThrow(duplicatedKeys)); UNIT_ASSERT_EXCEPTION(NSc::TValue::FromJsonThrow(duplicatedKeys, NSc::TValue::JO_PARSER_DISALLOW_DUPLICATE_KEYS), yexception); UNIT_ASSERT(NSc::TValue::FromJson(duplicatedKeys).IsDict()); UNIT_ASSERT(NSc::TValue::FromJson(duplicatedKeys, NSc::TValue::JO_PARSER_DISALLOW_DUPLICATE_KEYS).IsNull()); } -}; +}; diff --git a/library/cpp/scheme/tests/ut/scheme_merge_ut.cpp b/library/cpp/scheme/tests/ut/scheme_merge_ut.cpp index 00575e4268..2a06cf110d 100644 --- a/library/cpp/scheme/tests/ut/scheme_merge_ut.cpp +++ b/library/cpp/scheme/tests/ut/scheme_merge_ut.cpp @@ -1,172 +1,172 @@ #include <library/cpp/scheme/scimpl_private.h> #include <library/cpp/scheme/ut_utils/scheme_ut_utils.h> #include <library/cpp/testing/unittest/registar.h> - -#include <util/stream/null.h> + +#include <util/stream/null.h> #include <library/cpp/string_utils/quote/quote.h> -#include <util/string/subst.h> -#include <util/string/util.h> - -#include <type_traits> - - -Y_UNIT_TEST_SUITE(TSchemeMergeTest) { - - void DoTestReverseMerge(TStringBuf lhs, TStringBuf rhs, TStringBuf res) { - NSc::TValue v = NSc::TValue::FromJson(lhs); - v.ReverseMerge(NSc::TValue::FromJson(rhs)); - UNIT_ASSERT(NSc::TValue::Equal(v, NSc::TValue::FromJson(res))); - } - - Y_UNIT_TEST(TestReverseMerge) { - DoTestReverseMerge("{a:{x:y, b:c}}", "{a:{u:w, b:d}}", "{a:{u:w, x:y, b:c}}"); - DoTestReverseMerge("null", "{x:y}", "{x:y}"); - DoTestReverseMerge("null", "[b]", "[b]"); - DoTestReverseMerge("[a]", "[b]", "[a]"); - DoTestReverseMerge("{x:null}", "{x:b}", "{x:b}"); +#include <util/string/subst.h> +#include <util/string/util.h> + +#include <type_traits> + + +Y_UNIT_TEST_SUITE(TSchemeMergeTest) { + + void DoTestReverseMerge(TStringBuf lhs, TStringBuf rhs, TStringBuf res) { + NSc::TValue v = NSc::TValue::FromJson(lhs); + v.ReverseMerge(NSc::TValue::FromJson(rhs)); + UNIT_ASSERT(NSc::TValue::Equal(v, NSc::TValue::FromJson(res))); + } + + Y_UNIT_TEST(TestReverseMerge) { + DoTestReverseMerge("{a:{x:y, b:c}}", "{a:{u:w, b:d}}", "{a:{u:w, x:y, b:c}}"); + DoTestReverseMerge("null", "{x:y}", "{x:y}"); + DoTestReverseMerge("null", "[b]", "[b]"); + DoTestReverseMerge("[a]", "[b]", "[a]"); + DoTestReverseMerge("{x:null}", "{x:b}", "{x:b}"); } - Y_UNIT_TEST(TestMerge) { - TStringBuf data = "{ a : [ { b : 1, d : { e : -1.e5 } }, { f : 0, g : [ h, i ] } ] }"; - NSc::TValue v = NSc::TValue::FromJson(data); - UNIT_ASSERT_VALUES_EQUAL(v.ToJson(true), v.Clone().ToJson(true)); + Y_UNIT_TEST(TestMerge) { + TStringBuf data = "{ a : [ { b : 1, d : { e : -1.e5 } }, { f : 0, g : [ h, i ] } ] }"; + NSc::TValue v = NSc::TValue::FromJson(data); + UNIT_ASSERT_VALUES_EQUAL(v.ToJson(true), v.Clone().ToJson(true)); UNIT_ASSERT(v.Has("a")); - UNIT_ASSERT(v["a"].Has(1)); - UNIT_ASSERT(v["a"][0].Has("b")); - UNIT_ASSERT(v["a"][0].Has("d")); - UNIT_ASSERT(1 == v["a"][0]["b"]); - UNIT_ASSERT(v["a"][0]["d"].Has("e")); - UNIT_ASSERT(-1.e5 == v["a"][0]["d"]["e"]); - UNIT_ASSERT(v["a"][1].Has("f")); - UNIT_ASSERT(v["a"][1].Has("g")); - UNIT_ASSERT(0. == v["a"][1]["f"]); - UNIT_ASSERT(v["a"][1]["g"].IsArray()); - UNIT_ASSERT(v["a"][1]["g"].Has(1)); + UNIT_ASSERT(v["a"].Has(1)); + UNIT_ASSERT(v["a"][0].Has("b")); + UNIT_ASSERT(v["a"][0].Has("d")); + UNIT_ASSERT(1 == v["a"][0]["b"]); + UNIT_ASSERT(v["a"][0]["d"].Has("e")); + UNIT_ASSERT(-1.e5 == v["a"][0]["d"]["e"]); + UNIT_ASSERT(v["a"][1].Has("f")); + UNIT_ASSERT(v["a"][1].Has("g")); + UNIT_ASSERT(0. == v["a"][1]["f"]); + UNIT_ASSERT(v["a"][1]["g"].IsArray()); + UNIT_ASSERT(v["a"][1]["g"].Has(1)); UNIT_ASSERT(TStringBuf("h") == v["a"][1]["g"][0]); UNIT_ASSERT(TStringBuf("i") == v["a"][1]["g"][1]); - - { - TStringBuf data = "{ a : [ { d : 42 }, { g : [ 3 ] } ], q : r }"; - - NSc::TValue v1 = NSc::TValue::FromJson(data); - UNIT_ASSERT_VALUES_EQUAL(v1.ToJson(true), v1.Clone().ToJson(true)); - UNIT_ASSERT(NSc::TValue::Equal(v1, v1.FromJson(v1.ToJson()))); - - NSc::TValue v2; - v2.MergeUpdate(v["a"]); + + { + TStringBuf data = "{ a : [ { d : 42 }, { g : [ 3 ] } ], q : r }"; + + NSc::TValue v1 = NSc::TValue::FromJson(data); + UNIT_ASSERT_VALUES_EQUAL(v1.ToJson(true), v1.Clone().ToJson(true)); + UNIT_ASSERT(NSc::TValue::Equal(v1, v1.FromJson(v1.ToJson()))); + + NSc::TValue v2; + v2.MergeUpdate(v["a"]); UNIT_ASSERT_C(NSc::TValue::Equal(v["a"], v2), Sprintf("\n%s\n!=\n%s\n", v["a"].ToJson().data(), v2.ToJson().data())); - - v.MergeUpdate(v1); + + v.MergeUpdate(v1); UNIT_ASSERT_C(!NSc::TValue::Equal(v["a"], v2), Sprintf("\n%s\n!=\n%s\n", v["a"].ToJson().data(), v2.ToJson().data())); - v2.MergeUpdate(v1["a"]); + v2.MergeUpdate(v1["a"]); UNIT_ASSERT_C(NSc::TValue::Equal(v["a"], v2), Sprintf("\n%s\n!=\n%s\n", v["a"].ToJson().data(), v2.ToJson().data())); - } - + } + UNIT_ASSERT(v.Has("a")); UNIT_ASSERT(v.Has("q")); UNIT_ASSERT(TStringBuf("r") == v["q"]); - UNIT_ASSERT(v["a"].Has(1)); - UNIT_ASSERT(!v["a"][0].Has("b")); - UNIT_ASSERT(v["a"][0].Has("d")); - UNIT_ASSERT(!v["a"][0]["d"].IsArray()); - UNIT_ASSERT(!v["a"][0]["d"].IsDict()); - UNIT_ASSERT(42 == v["a"][0]["d"]); - UNIT_ASSERT(!v["a"][1].Has("f")); - UNIT_ASSERT(v["a"][1].Has("g")); - UNIT_ASSERT(v["a"][1]["g"].IsArray()); - UNIT_ASSERT(!v["a"][1]["g"].Has(1)); - UNIT_ASSERT(3 == v["a"][1]["g"][0]); - } - - Y_UNIT_TEST(TestMerge1) { - TStringBuf data = "[ { a : { b : d } } ]"; - - NSc::TValue wcopy = NSc::TValue::FromJson(data); - - TStringBuf data1 = "[ { a : { b : c } } ]"; - - wcopy.MergeUpdateJson(data1); - - { - TString json = wcopy.ToJson(true); - SubstGlobal(json, "\"", ""); - UNIT_ASSERT_VALUES_EQUAL(json, "[{a:{b:c}}]"); - } - } - - Y_UNIT_TEST(TestMerge2) { - TStringBuf data = "{ a : { b : c }, q : { x : y } }"; - - NSc::TValue wcopy = NSc::TValue::FromJson(data); - - TStringBuf data1 = "{ a : { e : f } }"; - - wcopy.MergeUpdateJson(data1); - - { - TString json = wcopy.ToJson(true); - SubstGlobal(json, "\"", ""); - UNIT_ASSERT_VALUES_EQUAL(json, "{a:{b:c,e:f},q:{x:y}}"); - } - } - - Y_UNIT_TEST(TestMerge3) { - TStringBuf data = "{ g : { x : { a : { b : c }, q : { x : y } }, y : fff } }"; - - NSc::TValue wcopy = NSc::TValue::FromJson(data); - - TStringBuf data1 = "{ g : { x : { a : { e : f } } } }"; - - wcopy.MergeUpdateJson(data1); - - { - TString json = wcopy.ToJson(true); - SubstGlobal(json, "\"", ""); - UNIT_ASSERT_VALUES_EQUAL(json, "{g:{x:{a:{b:c,e:f},q:{x:y}},y:fff}}"); - } - } - - Y_UNIT_TEST(TestMerge4) { - TStringBuf data = "{ a : 1, b : { c : 2, d : { q : f } } }"; - NSc::TValue val = NSc::TValue::FromJson(data); - - TStringBuf data1 = "{ a : 2, b : { c : 3, d : { q : e }, g : h } }"; - - val.MergeUpdateJson(data1); - - { - TString json = val.ToJson(true); - SubstGlobal(json, "\"", ""); - - UNIT_ASSERT_VALUES_EQUAL(json, "{a:2,b:{c:3,d:{q:e},g:h}}"); - } - } - - Y_UNIT_TEST(TestMerge5) { - NSc::TValue v0; - v0.GetOrAdd("x").MergeUpdate(NSc::TValue(1)); - UNIT_ASSERT_VALUES_EQUAL(v0.ToJson(), "{\"x\":1}"); - } - - Y_UNIT_TEST(TestMerge6) { - NSc::TValue va = NSc::TValue::FromJson("{\"x\":\"abc\",\"y\":\"def\"}"); - NSc::TValue vb = va.Get("y"); - NSc::TValue diff; - diff["y"] = vb; - va.MergeUpdate(diff); - UNIT_ASSERT_VALUES_EQUAL(va.ToJson(), "{\"x\":\"abc\",\"y\":\"def\"}"); - UNIT_ASSERT_VALUES_EQUAL(vb.ToJson(), "\"def\""); - UNIT_ASSERT_VALUES_EQUAL(diff.ToJson(), "{\"y\":\"def\"}"); - } - - Y_UNIT_TEST(TestMerge7) { - NSc::TValue v; - v["a"] = NSc::TValue::FromJson("[0.125,0.12,0.1,0.08,0.06]"); - UNIT_ASSERT_JSON_EQ_JSON(v, "{a:[0.125,0.12,0.1,0.08,0.06]}"); - + UNIT_ASSERT(v["a"].Has(1)); + UNIT_ASSERT(!v["a"][0].Has("b")); + UNIT_ASSERT(v["a"][0].Has("d")); + UNIT_ASSERT(!v["a"][0]["d"].IsArray()); + UNIT_ASSERT(!v["a"][0]["d"].IsDict()); + UNIT_ASSERT(42 == v["a"][0]["d"]); + UNIT_ASSERT(!v["a"][1].Has("f")); + UNIT_ASSERT(v["a"][1].Has("g")); + UNIT_ASSERT(v["a"][1]["g"].IsArray()); + UNIT_ASSERT(!v["a"][1]["g"].Has(1)); + UNIT_ASSERT(3 == v["a"][1]["g"][0]); + } + + Y_UNIT_TEST(TestMerge1) { + TStringBuf data = "[ { a : { b : d } } ]"; + + NSc::TValue wcopy = NSc::TValue::FromJson(data); + + TStringBuf data1 = "[ { a : { b : c } } ]"; + + wcopy.MergeUpdateJson(data1); + + { + TString json = wcopy.ToJson(true); + SubstGlobal(json, "\"", ""); + UNIT_ASSERT_VALUES_EQUAL(json, "[{a:{b:c}}]"); + } + } + + Y_UNIT_TEST(TestMerge2) { + TStringBuf data = "{ a : { b : c }, q : { x : y } }"; + + NSc::TValue wcopy = NSc::TValue::FromJson(data); + + TStringBuf data1 = "{ a : { e : f } }"; + + wcopy.MergeUpdateJson(data1); + + { + TString json = wcopy.ToJson(true); + SubstGlobal(json, "\"", ""); + UNIT_ASSERT_VALUES_EQUAL(json, "{a:{b:c,e:f},q:{x:y}}"); + } + } + + Y_UNIT_TEST(TestMerge3) { + TStringBuf data = "{ g : { x : { a : { b : c }, q : { x : y } }, y : fff } }"; + + NSc::TValue wcopy = NSc::TValue::FromJson(data); + + TStringBuf data1 = "{ g : { x : { a : { e : f } } } }"; + + wcopy.MergeUpdateJson(data1); + + { + TString json = wcopy.ToJson(true); + SubstGlobal(json, "\"", ""); + UNIT_ASSERT_VALUES_EQUAL(json, "{g:{x:{a:{b:c,e:f},q:{x:y}},y:fff}}"); + } + } + + Y_UNIT_TEST(TestMerge4) { + TStringBuf data = "{ a : 1, b : { c : 2, d : { q : f } } }"; + NSc::TValue val = NSc::TValue::FromJson(data); + + TStringBuf data1 = "{ a : 2, b : { c : 3, d : { q : e }, g : h } }"; + + val.MergeUpdateJson(data1); + + { + TString json = val.ToJson(true); + SubstGlobal(json, "\"", ""); + + UNIT_ASSERT_VALUES_EQUAL(json, "{a:2,b:{c:3,d:{q:e},g:h}}"); + } + } + + Y_UNIT_TEST(TestMerge5) { + NSc::TValue v0; + v0.GetOrAdd("x").MergeUpdate(NSc::TValue(1)); + UNIT_ASSERT_VALUES_EQUAL(v0.ToJson(), "{\"x\":1}"); + } + + Y_UNIT_TEST(TestMerge6) { + NSc::TValue va = NSc::TValue::FromJson("{\"x\":\"abc\",\"y\":\"def\"}"); + NSc::TValue vb = va.Get("y"); + NSc::TValue diff; + diff["y"] = vb; + va.MergeUpdate(diff); + UNIT_ASSERT_VALUES_EQUAL(va.ToJson(), "{\"x\":\"abc\",\"y\":\"def\"}"); + UNIT_ASSERT_VALUES_EQUAL(vb.ToJson(), "\"def\""); + UNIT_ASSERT_VALUES_EQUAL(diff.ToJson(), "{\"y\":\"def\"}"); + } + + Y_UNIT_TEST(TestMerge7) { + NSc::TValue v; + v["a"] = NSc::TValue::FromJson("[0.125,0.12,0.1,0.08,0.06]"); + UNIT_ASSERT_JSON_EQ_JSON(v, "{a:[0.125,0.12,0.1,0.08,0.06]}"); + NSc::TValue a = v.TrySelectOrAdd("a")->MergeUpdateJson("[1,2,3]"); - - UNIT_ASSERT_JSON_EQ_JSON(a, "[1,2,3]"); - UNIT_ASSERT_JSON_EQ_JSON(v, "{a:[1,2,3]}"); - } -}; + + UNIT_ASSERT_JSON_EQ_JSON(a, "[1,2,3]"); + UNIT_ASSERT_JSON_EQ_JSON(v, "{a:[1,2,3]}"); + } +}; diff --git a/library/cpp/scheme/tests/ut/scheme_path_ut.cpp b/library/cpp/scheme/tests/ut/scheme_path_ut.cpp index 2980979e3b..0d4d79d483 100644 --- a/library/cpp/scheme/tests/ut/scheme_path_ut.cpp +++ b/library/cpp/scheme/tests/ut/scheme_path_ut.cpp @@ -1,81 +1,81 @@ #include <library/cpp/scheme/scimpl_private.h> #include <library/cpp/scheme/ut_utils/scheme_ut_utils.h> #include <library/cpp/testing/unittest/registar.h> - -#include <util/stream/null.h> -#include <util/string/subst.h> -#include <util/string/util.h> - -#include <type_traits> + +#include <util/stream/null.h> +#include <util/string/subst.h> +#include <util/string/util.h> + +#include <type_traits> #include <library/cpp/string_utils/quote/quote.h> - -Y_UNIT_TEST_SUITE(TSchemePathTest) { - void DoTestSelect(TStringBuf path, TStringBuf expected, TStringBuf delexpected) { - NSc::TValue v; - UNIT_ASSERT(!v.PathExists(path)); - UNIT_ASSERT(NSc::TValue::PathValid(path)); + +Y_UNIT_TEST_SUITE(TSchemePathTest) { + void DoTestSelect(TStringBuf path, TStringBuf expected, TStringBuf delexpected) { + NSc::TValue v; + UNIT_ASSERT(!v.PathExists(path)); + UNIT_ASSERT(NSc::TValue::PathValid(path)); UNIT_ASSERT(NSc::TValue::Same(v.TrySelect(path), NSc::Null())); *v.TrySelectOrAdd(path) = 1; - NSc::NUt::AssertSchemeJson(expected, v); - UNIT_ASSERT(v.PathExists(path)); + NSc::NUt::AssertSchemeJson(expected, v); + UNIT_ASSERT(v.PathExists(path)); UNIT_ASSERT(1 == v.TrySelectOrAdd(path)->GetNumber()); UNIT_ASSERT(1 == v.TrySelect(path).GetNumber()); UNIT_ASSERT(1 == v.TrySelectAndDelete(path).GetNumber()); UNIT_ASSERT(NSc::TValue::Same(v.TrySelectAndDelete(path), NSc::Null())); - NSc::NUt::AssertSchemeJson(delexpected, v); - UNIT_ASSERT(!v.PathExists(path)); + NSc::NUt::AssertSchemeJson(delexpected, v); + UNIT_ASSERT(!v.PathExists(path)); UNIT_ASSERT(NSc::TValue::Same(v.TrySelect(path), NSc::Null())); - } - - Y_UNIT_TEST(TestSelect) { - NSc::TValue v; + } + + Y_UNIT_TEST(TestSelect) { + NSc::TValue v; UNIT_ASSERT(!v.PathValid(" ")); - UNIT_ASSERT(v.PathExists("")); - UNIT_ASSERT(v.PathExists("//")); - + UNIT_ASSERT(v.PathExists("")); + UNIT_ASSERT(v.PathExists("//")); + UNIT_ASSERT(NSc::TValue::Same(v, *v.TrySelectOrAdd("//"))); NSc::NUt::AssertSchemeJson("null", v); UNIT_ASSERT(NSc::TValue::Same(v.TrySelectAndDelete("//"), NSc::Null())); NSc::NUt::AssertSchemeJson("null", v); - - v.SetDict(); + + v.SetDict(); UNIT_ASSERT(NSc::TValue::Same(v, *v.TrySelectOrAdd("//"))); NSc::NUt::AssertSchemeJson("{}", v); UNIT_ASSERT(NSc::TValue::Same(v.TrySelectAndDelete("//"), NSc::Null())); NSc::NUt::AssertSchemeJson("{}", v); - - v.SetArray(); + + v.SetArray(); UNIT_ASSERT(NSc::TValue::Same(v, *v.TrySelectOrAdd("//"))); NSc::NUt::AssertSchemeJson("[]", v); UNIT_ASSERT(NSc::TValue::Same(v.TrySelectAndDelete("//"), NSc::Null())); NSc::NUt::AssertSchemeJson("[]", v); - - DoTestSelect("[]", "{'':1}", "{}"); - DoTestSelect("[ ]", "{' ':1}", "{}"); - DoTestSelect("[0]", "[1]", "[]"); - DoTestSelect("[1]", "[null,1]", "[null]"); - DoTestSelect("foo/[0]/bar", "{foo:[{bar:1}]}", "{foo:[{}]}"); - DoTestSelect("foo/1/bar", "{foo:[null,{bar:1}]}", "{foo:[null,{}]}"); - DoTestSelect("foo[-1]bar", "{foo:{'-1':{bar:1}}}", "{foo:{'-1':{}}}"); - DoTestSelect("'foo'/\"0\"/'bar'", "{foo:{'0':{bar:1}}}", "{foo:{'0':{}}}"); - DoTestSelect("'\\''", "{'\\'':1}", "{}"); - } - - Y_UNIT_TEST(TestSelectAndMerge) { - NSc::TValue v; + + DoTestSelect("[]", "{'':1}", "{}"); + DoTestSelect("[ ]", "{' ':1}", "{}"); + DoTestSelect("[0]", "[1]", "[]"); + DoTestSelect("[1]", "[null,1]", "[null]"); + DoTestSelect("foo/[0]/bar", "{foo:[{bar:1}]}", "{foo:[{}]}"); + DoTestSelect("foo/1/bar", "{foo:[null,{bar:1}]}", "{foo:[null,{}]}"); + DoTestSelect("foo[-1]bar", "{foo:{'-1':{bar:1}}}", "{foo:{'-1':{}}}"); + DoTestSelect("'foo'/\"0\"/'bar'", "{foo:{'0':{bar:1}}}", "{foo:{'0':{}}}"); + DoTestSelect("'\\''", "{'\\'':1}", "{}"); + } + + Y_UNIT_TEST(TestSelectAndMerge) { + NSc::TValue v; v.TrySelectOrAdd("blender/enabled")->MergeUpdateJson("1"); - UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::FromJson("1").ToJson(), "1"); - UNIT_ASSERT_VALUES_EQUAL(v.ToJson(), "{\"blender\":{\"enabled\":1}}"); - } - - Y_UNIT_TEST(TestPathEscapes) { + UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::FromJson("1").ToJson(), "1"); + UNIT_ASSERT_VALUES_EQUAL(v.ToJson(), "{\"blender\":{\"enabled\":1}}"); + } + + Y_UNIT_TEST(TestPathEscapes) { UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("a"), "a"); UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath(""), R"=("")="); UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("[]"), R"=("[]")="); - UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("[]ab"), R"=("[]ab")="); - UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("a[]b"), R"=("a[]b")="); - UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("ab[]"), R"=("ab[]")="); - UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("[ab]"), R"=("[ab]")="); + UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("[]ab"), R"=("[]ab")="); + UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("a[]b"), R"=("a[]b")="); + UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("ab[]"), R"=("ab[]")="); + UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("[ab]"), R"=("[ab]")="); UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("[ab"), R"=("[ab")="); UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("a[b"), R"=("a[b")="); UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("ab["), R"=("ab[")="); @@ -83,77 +83,77 @@ Y_UNIT_TEST_SUITE(TSchemePathTest) { UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("a]b"), R"=("a]b")="); UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("ab]"), R"=("ab]")="); UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath(R"=(\)="), R"=("\\")="); - UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath(R"=(\\)="), R"=("\\\\")="); + UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath(R"=(\\)="), R"=("\\\\")="); UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("/"), R"=("/")="); UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("//"), R"=("//")="); UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("///"), R"=("///")="); UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("/ab"), R"=("/ab")="); UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("a/b"), R"=("a/b")="); UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("ab/"), R"=("ab/")="); - UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("//ab"), R"=("//ab")="); - UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("a//b"), R"=("a//b")="); - UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("ab//"), R"=("ab//")="); - UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("6400"), R"=("6400")="); - - { - NSc::TValue val; + UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("//ab"), R"=("//ab")="); + UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("a//b"), R"=("a//b")="); + UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("ab//"), R"=("ab//")="); + UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("6400"), R"=("6400")="); + + { + NSc::TValue val; *val.TrySelectOrAdd("") = 100; const TString res = R"=(100)="; - UNIT_ASSERT_VALUES_EQUAL(val.ToJson(), res); - } - { - NSc::TValue val; + UNIT_ASSERT_VALUES_EQUAL(val.ToJson(), res); + } + { + NSc::TValue val; *val.TrySelectOrAdd("a") = 100; const TString res = R"=({"a":100})="; - UNIT_ASSERT_VALUES_EQUAL(val.ToJson(), res); - } - { - NSc::TValue val; + UNIT_ASSERT_VALUES_EQUAL(val.ToJson(), res); + } + { + NSc::TValue val; *val.TrySelectOrAdd(R"=(////)=") = 100; const TString res = R"=(100)="; - UNIT_ASSERT_VALUES_EQUAL(val.ToJson(), res); - } - { - NSc::TValue val; + UNIT_ASSERT_VALUES_EQUAL(val.ToJson(), res); + } + { + NSc::TValue val; *val.TrySelectOrAdd(R"=()=") = 100; const TString res = R"=(100)="; - UNIT_ASSERT_VALUES_EQUAL(val.ToJson(), res); - } - { - NSc::TValue val; + UNIT_ASSERT_VALUES_EQUAL(val.ToJson(), res); + } + { + NSc::TValue val; *val.TrySelectOrAdd(R"=("")=") = 100; const TString res = R"=({"":100})="; - UNIT_ASSERT_VALUES_EQUAL(val.ToJson(), res); - } - { - NSc::TValue val; + UNIT_ASSERT_VALUES_EQUAL(val.ToJson(), res); + } + { + NSc::TValue val; *val.TrySelectOrAdd(R"=("[1]")=") = 100; const TString res = R"=({"[1]":100})="; - UNIT_ASSERT_VALUES_EQUAL(val.ToJson(), res); - } - { - NSc::TValue val; + UNIT_ASSERT_VALUES_EQUAL(val.ToJson(), res); + } + { + NSc::TValue val; *val.TrySelectOrAdd(R"=("\"\"")=") = 100; const TString res = R"=({"\"\"":100})="; - UNIT_ASSERT_VALUES_EQUAL(val.ToJson(), res); - } - { - NSc::TValue val; + UNIT_ASSERT_VALUES_EQUAL(val.ToJson(), res); + } + { + NSc::TValue val; *val.TrySelectOrAdd(R"=("/10/")=") = 100; const TString res = R"=({"/10/":100})="; - UNIT_ASSERT_VALUES_EQUAL(val.ToJson(), res); - } - { - NSc::TValue val; + UNIT_ASSERT_VALUES_EQUAL(val.ToJson(), res); + } + { + NSc::TValue val; *val.TrySelectOrAdd(R"=(/"[10]"//""/"\"/10/\""///)=") = 100; const TString res = R"=({"[10]":{"":{"\"/10/\"":100}}})="; - UNIT_ASSERT_VALUES_EQUAL(val.ToJson(), res); - } - { - NSc::TValue val; + UNIT_ASSERT_VALUES_EQUAL(val.ToJson(), res); + } + { + NSc::TValue val; *val.TrySelectOrAdd(R"=(/"[10]"//""/"\"/10/\""///)=") = 100; const TString res = R"=({"[10]":{"":{"\"/10/\"":100}}})="; - UNIT_ASSERT_VALUES_EQUAL(val.ToJson(), res); - } - } -}; + UNIT_ASSERT_VALUES_EQUAL(val.ToJson(), res); + } + } +}; diff --git a/library/cpp/scheme/tests/ut/scheme_proto_ut.cpp b/library/cpp/scheme/tests/ut/scheme_proto_ut.cpp index 6436ebb432..e711a0d092 100644 --- a/library/cpp/scheme/tests/ut/scheme_proto_ut.cpp +++ b/library/cpp/scheme/tests/ut/scheme_proto_ut.cpp @@ -2,24 +2,24 @@ #include <library/cpp/scheme/scheme.h> #include <library/cpp/scheme/tests/ut/scheme_ut.pb.h> #include <library/cpp/testing/unittest/registar.h> - -Y_UNIT_TEST_SUITE(TSchemeProtoTest) { - void DoTestProtobuf(bool fromProto, bool mapAsDict); - - Y_UNIT_TEST(TestFromProtobuf) { - DoTestProtobuf(true, false); + +Y_UNIT_TEST_SUITE(TSchemeProtoTest) { + void DoTestProtobuf(bool fromProto, bool mapAsDict); + + Y_UNIT_TEST(TestFromProtobuf) { + DoTestProtobuf(true, false); } - Y_UNIT_TEST(TestToProtobuf) { - DoTestProtobuf(false, false); + Y_UNIT_TEST(TestToProtobuf) { + DoTestProtobuf(false, false); } - Y_UNIT_TEST(TestFromProtobufWithDict) { - DoTestProtobuf(true, true); + Y_UNIT_TEST(TestFromProtobufWithDict) { + DoTestProtobuf(true, true); } - Y_UNIT_TEST(TestToProtobufWithDict) { - DoTestProtobuf(false, true); + Y_UNIT_TEST(TestToProtobufWithDict) { + DoTestProtobuf(false, true); } template <class T> @@ -33,9 +33,9 @@ Y_UNIT_TEST_SUITE(TSchemeProtoTest) { } } - void DoTestProtobuf(bool fromProto, bool mapAsDict) { - NSc::TMessage m; - NSc::TValue v; + void DoTestProtobuf(bool fromProto, bool mapAsDict) { + NSc::TMessage m; + NSc::TValue v; m.SetDouble((double)1 / 3), v["Double"] = (double)1 / 3; m.SetFloat((float)1 / 3), v["Float"] = (float)1 / 3; m.SetInt32(1000000), v["Int32"] = 1000000; @@ -52,7 +52,7 @@ Y_UNIT_TEST_SUITE(TSchemeProtoTest) { m.SetString("String"), v["String"] = "String"; m.SetBytes("Bytes"), v["Bytes"] = "Bytes"; m.SetEnum(NSc::VALUE1), v["Enum"] = "VALUE1"; - + auto& mapDoublesP = *m.mutable_mapdoubles(); auto& mapDoublesV = v["MapDoubles"]; mapDoublesP["pi"] = 3.14; @@ -85,56 +85,56 @@ Y_UNIT_TEST_SUITE(TSchemeProtoTest) { m.AddDoubles((double)1 / 3), v["Doubles"][0] = (double)1 / 3; m.AddDoubles((double)1 / 4), v["Doubles"][1] = (double)1 / 4; - + m.AddFloats((float)1 / 3), v["Floats"][0] = (float)1 / 3; m.AddFloats((float)1 / 4), v["Floats"][1] = (float)1 / 4; - + m.AddInt32s(1000000), v["Int32s"][0] = 1000000; m.AddInt32s(2000000), v["Int32s"][1] = 2000000; - + m.AddInt64s(1000000000000LL), v["Int64s"][0] = 1000000000000LL; m.AddInt64s(2000000000000LL), v["Int64s"][1] = 2000000000000LL; - + m.AddUInt32s(555555), v["UInt32s"][0] = 555555; m.AddUInt32s(655555), v["UInt32s"][1] = 655555; - + m.AddUInt64s(555555555555LL); v["UInt64s"][0] = 555555555555LL; m.AddUInt64s(655555555555LL); v["UInt64s"][1] = 655555555555LL; - + m.AddSInt32s(-555555), v["SInt32s"][0] = -555555; m.AddSInt32s(-655555), v["SInt32s"][1] = -655555; - + m.AddSInt64s(-555555555555LL), v["SInt64s"][0] = -555555555555LL; m.AddSInt64s(-655555555555LL), v["SInt64s"][1] = -655555555555LL; - + m.AddFixed32s(123456), v["Fixed32s"][0] = 123456; m.AddFixed32s(223456), v["Fixed32s"][1] = 223456; - + m.AddFixed64s(123456123456LL), v["Fixed64s"][0] = 123456123456LL; m.AddFixed64s(223456123456LL), v["Fixed64s"][1] = 223456123456LL; - + m.AddSFixed32s(-123456), v["SFixed32s"][0] = -123456; m.AddSFixed32s(-223456), v["SFixed32s"][1] = -223456; - + m.AddSFixed64s(-123456123456LL), v["SFixed64s"][0] = -123456123456LL; m.AddSFixed64s(-223456123456LL), v["SFixed64s"][1] = -223456123456LL; - + m.AddBools(false), v["Bools"][0] = false; m.AddBools(true), v["Bools"][1] = true; - + m.AddStrings("String1"), v["Strings"][0] = "String1"; m.AddStrings("String2"), v["Strings"][1] = "String2"; - + m.AddBytess("Bytes1"), v["Bytess"][0] = "Bytes1"; m.AddBytess("Bytes2"), v["Bytess"][1] = "Bytes2"; - + m.AddEnums(NSc::VALUE1), v["Enums"][0] = "VALUE1"; m.AddEnums(NSc::VALUE2), v["Enums"][1] = "VALUE2"; - - NSc::TMessage2 m2; - NSc::TValue v2; + + NSc::TMessage2 m2; + NSc::TValue v2; m2.SetDouble((double)1 / 3), v2["Double"] = (double)1 / 3; m2.SetFloat((float)1 / 3), v2["Float"] = (float)1 / 3; m2.SetInt32(1000000), v2["Int32"] = 1000000; @@ -151,62 +151,62 @@ Y_UNIT_TEST_SUITE(TSchemeProtoTest) { m2.SetString("String"), v2["String"] = "String"; m2.SetBytes("Bytes"), v2["Bytes"] = "Bytes"; m2.SetEnum(NSc::VALUE1), v2["Enum"] = "VALUE1"; - + m2.AddDoubles((double)1 / 3), v2["Doubles"][0] = (double)1 / 3; m2.AddDoubles((double)1 / 4), v2["Doubles"][1] = (double)1 / 4; - + m2.AddFloats((float)1 / 3), v2["Floats"][0] = (float)1 / 3; m2.AddFloats((float)1 / 4), v2["Floats"][1] = (float)1 / 4; - + m2.AddInt32s(1000000), v2["Int32s"][0] = 1000000; m2.AddInt32s(2000000), v2["Int32s"][1] = 2000000; - + m2.AddInt64s(1000000000000LL), v2["Int64s"][0] = 1000000000000LL; m2.AddInt64s(2000000000000LL), v2["Int64s"][1] = 2000000000000LL; - + m2.AddUInt32s(555555), v2["UInt32s"][0] = 555555; m2.AddUInt32s(655555), v2["UInt32s"][1] = 655555; - + m2.AddUInt64s(555555555555LL); v2["UInt64s"][0] = 555555555555LL; m2.AddUInt64s(655555555555LL); v2["UInt64s"][1] = 655555555555LL; - + m2.AddSInt32s(-555555), v2["SInt32s"][0] = -555555; m2.AddSInt32s(-655555), v2["SInt32s"][1] = -655555; - + m2.AddSInt64s(-555555555555LL), v2["SInt64s"][0] = -555555555555LL; m2.AddSInt64s(-655555555555LL), v2["SInt64s"][1] = -655555555555LL; - + m2.AddFixed32s(123456), v2["Fixed32s"][0] = 123456; m2.AddFixed32s(223456), v2["Fixed32s"][1] = 223456; - + m2.AddFixed64s(123456123456LL), v2["Fixed64s"][0] = 123456123456LL; m2.AddFixed64s(223456123456LL), v2["Fixed64s"][1] = 223456123456LL; - + m2.AddSFixed32s(-123456), v2["SFixed32s"][0] = -123456; m2.AddSFixed32s(-223456), v2["SFixed32s"][1] = -223456; - + m2.AddSFixed64s(-123456123456LL), v2["SFixed64s"][0] = -123456123456LL; m2.AddSFixed64s(-223456123456LL), v2["SFixed64s"][1] = -223456123456LL; - + m2.AddBools(false), v2["Bools"][0] = false; m2.AddBools(true), v2["Bools"][1] = true; - + m2.AddStrings("String1"), v2["Strings"][0] = "String1"; m2.AddStrings("String2"), v2["Strings"][1] = "String2"; - + m2.AddBytess("Bytes1"), v2["Bytess"][0] = "Bytes1"; m2.AddBytess("Bytes2"), v2["Bytess"][1] = "Bytes2"; - + m2.AddEnums(NSc::VALUE1), v2["Enums"][0] = "VALUE1"; m2.AddEnums(NSc::VALUE2), v2["Enums"][1] = "VALUE2"; - + *(m.MutableMessage()) = m2, v["Message"] = v2; - + *(m.AddMessages()) = m2, v["Messages"][0] = v2; *(m.AddMessages()) = m2, v["Messages"][1] = v2; - + if (fromProto) { UNIT_ASSERT(NSc::TValue::Equal(v, NSc::TValue::From(m, mapAsDict))); } else { @@ -216,5 +216,5 @@ Y_UNIT_TEST_SUITE(TSchemeProtoTest) { TString differentPath; UNIT_ASSERT_C(NProtoBuf::IsEqual(m, proto, &differentPath), differentPath); } - } -}; + } +}; diff --git a/library/cpp/scheme/tests/ut/scheme_ut.cpp b/library/cpp/scheme/tests/ut/scheme_ut.cpp index d19f7d0268..1a5d07c31b 100644 --- a/library/cpp/scheme/tests/ut/scheme_ut.cpp +++ b/library/cpp/scheme/tests/ut/scheme_ut.cpp @@ -1,775 +1,775 @@ #include <library/cpp/scheme/scimpl_private.h> #include <library/cpp/scheme/ut_utils/scheme_ut_utils.h> #include <library/cpp/testing/unittest/registar.h> - -#include <util/stream/null.h> + +#include <util/stream/null.h> #include <library/cpp/string_utils/quote/quote.h> -#include <util/string/subst.h> -#include <util/string/util.h> - -#include <type_traits> - -Y_UNIT_TEST_SUITE(TSchemeTest) { - - Y_UNIT_TEST(TestNaN) { - UNIT_ASSERT_VALUES_EQUAL("null", NSc::TValue(std::numeric_limits<double>::quiet_NaN()).ToJson()); - UNIT_ASSERT_VALUES_EQUAL("null", NSc::TValue(-std::numeric_limits<double>::infinity()).ToJson()); - UNIT_ASSERT_VALUES_EQUAL("null", NSc::TValue(std::numeric_limits<double>::infinity()).ToJson()); - UNIT_ASSERT_VALUES_EQUAL("1", NSc::TValue(1.0).ToJson()); - } - - Y_UNIT_TEST(TestNumbers) { - { - NSc::TValue vd; - UNIT_ASSERT_VALUES_EQUAL(2.5, vd.GetNumberMutable(2.5)); - UNIT_ASSERT_VALUES_EQUAL(2, vd.GetIntNumberMutable(-1)); - } - { - NSc::TValue vi; - UNIT_ASSERT_VALUES_EQUAL(2, vi.GetIntNumberMutable(2)); - UNIT_ASSERT_VALUES_EQUAL(2., vi.GetNumberMutable(-1)); - } - { - NSc::TValue vb = NSc::TValue::FromJson("true"); - - UNIT_ASSERT_VALUES_EQUAL("true", vb.ToJson()); - - UNIT_ASSERT(vb.IsBool()); - UNIT_ASSERT(vb.IsIntNumber()); - UNIT_ASSERT(vb.IsNumber()); - UNIT_ASSERT_VALUES_EQUAL(true, vb.GetBool()); - UNIT_ASSERT_VALUES_EQUAL(1, vb.GetIntNumber()); - UNIT_ASSERT_VALUES_EQUAL(1.0, vb.GetNumber()); - - NSc::TValue vb1 = vb.Clone(); - - UNIT_ASSERT(NSc::TValue::Equal(vb, vb1)); - - UNIT_ASSERT_VALUES_EQUAL(true, vb.GetBool()); - UNIT_ASSERT_VALUES_EQUAL(1, vb.GetIntNumber()); - UNIT_ASSERT_VALUES_EQUAL(1.0, vb.GetNumber()); - UNIT_ASSERT(vb.IsBool()); - UNIT_ASSERT_VALUES_EQUAL(1, vb.GetIntNumberMutable()); - UNIT_ASSERT(!vb.IsBool()); - - UNIT_ASSERT(NSc::TValue::Equal(vb, vb1)); - - UNIT_ASSERT(vb1.IsBool()); - UNIT_ASSERT_VALUES_EQUAL(1.0, vb1.GetNumberMutable()); - UNIT_ASSERT(!vb1.IsBool()); - - vb.SetBool(true); - - UNIT_ASSERT(vb.IsBool()); - UNIT_ASSERT(NSc::TValue::Equal(vb, vb1)); - - vb = NSc::TValue::FromJson("false"); - - UNIT_ASSERT_VALUES_EQUAL("false", vb.ToJson()); - UNIT_ASSERT(!NSc::TValue::Equal(vb, vb1)); - - UNIT_ASSERT(vb.IsBool()); - UNIT_ASSERT(vb.IsIntNumber()); - UNIT_ASSERT(vb.IsNumber()); - UNIT_ASSERT_VALUES_EQUAL(false, vb.GetBool()); - UNIT_ASSERT_VALUES_EQUAL(0.0, vb.GetNumber()); - UNIT_ASSERT_VALUES_EQUAL(0, vb.GetIntNumber()); - - NSc::TValue vd = NSc::TValue::FromJson("1.0"); - - UNIT_ASSERT(vd.IsNumber()); - UNIT_ASSERT(!vd.IsIntNumber()); - UNIT_ASSERT_VALUES_EQUAL(1.0, vd.GetNumber()); - UNIT_ASSERT_VALUES_EQUAL(1, vd.GetIntNumber()); - UNIT_ASSERT_VALUES_EQUAL(1.0, vd.GetNumberMutable()); - - NSc::TValue vi = NSc::TValue::FromJson("1"); - - UNIT_ASSERT(vi.IsNumber()); - UNIT_ASSERT(vi.IsIntNumber()); - UNIT_ASSERT_VALUES_EQUAL(1.0, vi.GetNumber()); - UNIT_ASSERT_VALUES_EQUAL(1, vi.GetIntNumber()); - UNIT_ASSERT_VALUES_EQUAL(1, vi.GetIntNumberMutable()); - - UNIT_ASSERT(NSc::TValue::Equal(vd, vi)); - - vd.SetNumber(1.5); - UNIT_ASSERT(vd.IsNumber()); - UNIT_ASSERT(!vd.IsIntNumber()); - UNIT_ASSERT_VALUES_EQUAL(1.5, vd.GetNumber()); - UNIT_ASSERT_VALUES_EQUAL(1, vd.GetIntNumber()); - UNIT_ASSERT_VALUES_EQUAL(1.5, vd.GetNumberMutable()); - - UNIT_ASSERT(!NSc::TValue::Equal(vd, vi)); - - UNIT_ASSERT_VALUES_EQUAL("1", vi.ToJson()); - UNIT_ASSERT_VALUES_EQUAL("1.5", vd.ToJson()); - - UNIT_ASSERT_VALUES_EQUAL(1, vd.GetIntNumberMutable()); - UNIT_ASSERT(NSc::TValue::Equal(vd, vi)); - vd.SetIntNumber(2); - UNIT_ASSERT(!NSc::TValue::Equal(vd, vi)); - vi.SetNumber(2.); - UNIT_ASSERT(NSc::TValue::Equal(vd, vi)); - vd.SetNumber(2.); - UNIT_ASSERT(NSc::TValue::Equal(vd, vi)); - vi.SetIntNumber(5); - vd.MergeUpdate(vi); - UNIT_ASSERT(vd.IsNumber()); - UNIT_ASSERT(vd.IsIntNumber()); - UNIT_ASSERT_VALUES_EQUAL(5, vd.GetIntNumber()); - vd.SetNumber(3.3); - vi.MergeUpdate(vd); - UNIT_ASSERT(vi.IsNumber()); - UNIT_ASSERT(!vi.IsIntNumber()); - UNIT_ASSERT_VALUES_EQUAL(3.3, vi.GetNumber()); - - vi.SetIntNumber(Max<i64>()); - UNIT_ASSERT_VALUES_EQUAL("9223372036854775807", vi.ToJson()); - } - } - - template <typename T> - void DoTestForce(T t) { - UNIT_ASSERT_VALUES_EQUAL_C(i64(t), NSc::TValue(i64(t)).ForceIntNumber(), ToString(t)); - UNIT_ASSERT_VALUES_EQUAL_C(double(t), NSc::TValue(double(t)).ForceNumber(), ToString(t)); - +#include <util/string/subst.h> +#include <util/string/util.h> + +#include <type_traits> + +Y_UNIT_TEST_SUITE(TSchemeTest) { + + Y_UNIT_TEST(TestNaN) { + UNIT_ASSERT_VALUES_EQUAL("null", NSc::TValue(std::numeric_limits<double>::quiet_NaN()).ToJson()); + UNIT_ASSERT_VALUES_EQUAL("null", NSc::TValue(-std::numeric_limits<double>::infinity()).ToJson()); + UNIT_ASSERT_VALUES_EQUAL("null", NSc::TValue(std::numeric_limits<double>::infinity()).ToJson()); + UNIT_ASSERT_VALUES_EQUAL("1", NSc::TValue(1.0).ToJson()); + } + + Y_UNIT_TEST(TestNumbers) { + { + NSc::TValue vd; + UNIT_ASSERT_VALUES_EQUAL(2.5, vd.GetNumberMutable(2.5)); + UNIT_ASSERT_VALUES_EQUAL(2, vd.GetIntNumberMutable(-1)); + } + { + NSc::TValue vi; + UNIT_ASSERT_VALUES_EQUAL(2, vi.GetIntNumberMutable(2)); + UNIT_ASSERT_VALUES_EQUAL(2., vi.GetNumberMutable(-1)); + } + { + NSc::TValue vb = NSc::TValue::FromJson("true"); + + UNIT_ASSERT_VALUES_EQUAL("true", vb.ToJson()); + + UNIT_ASSERT(vb.IsBool()); + UNIT_ASSERT(vb.IsIntNumber()); + UNIT_ASSERT(vb.IsNumber()); + UNIT_ASSERT_VALUES_EQUAL(true, vb.GetBool()); + UNIT_ASSERT_VALUES_EQUAL(1, vb.GetIntNumber()); + UNIT_ASSERT_VALUES_EQUAL(1.0, vb.GetNumber()); + + NSc::TValue vb1 = vb.Clone(); + + UNIT_ASSERT(NSc::TValue::Equal(vb, vb1)); + + UNIT_ASSERT_VALUES_EQUAL(true, vb.GetBool()); + UNIT_ASSERT_VALUES_EQUAL(1, vb.GetIntNumber()); + UNIT_ASSERT_VALUES_EQUAL(1.0, vb.GetNumber()); + UNIT_ASSERT(vb.IsBool()); + UNIT_ASSERT_VALUES_EQUAL(1, vb.GetIntNumberMutable()); + UNIT_ASSERT(!vb.IsBool()); + + UNIT_ASSERT(NSc::TValue::Equal(vb, vb1)); + + UNIT_ASSERT(vb1.IsBool()); + UNIT_ASSERT_VALUES_EQUAL(1.0, vb1.GetNumberMutable()); + UNIT_ASSERT(!vb1.IsBool()); + + vb.SetBool(true); + + UNIT_ASSERT(vb.IsBool()); + UNIT_ASSERT(NSc::TValue::Equal(vb, vb1)); + + vb = NSc::TValue::FromJson("false"); + + UNIT_ASSERT_VALUES_EQUAL("false", vb.ToJson()); + UNIT_ASSERT(!NSc::TValue::Equal(vb, vb1)); + + UNIT_ASSERT(vb.IsBool()); + UNIT_ASSERT(vb.IsIntNumber()); + UNIT_ASSERT(vb.IsNumber()); + UNIT_ASSERT_VALUES_EQUAL(false, vb.GetBool()); + UNIT_ASSERT_VALUES_EQUAL(0.0, vb.GetNumber()); + UNIT_ASSERT_VALUES_EQUAL(0, vb.GetIntNumber()); + + NSc::TValue vd = NSc::TValue::FromJson("1.0"); + + UNIT_ASSERT(vd.IsNumber()); + UNIT_ASSERT(!vd.IsIntNumber()); + UNIT_ASSERT_VALUES_EQUAL(1.0, vd.GetNumber()); + UNIT_ASSERT_VALUES_EQUAL(1, vd.GetIntNumber()); + UNIT_ASSERT_VALUES_EQUAL(1.0, vd.GetNumberMutable()); + + NSc::TValue vi = NSc::TValue::FromJson("1"); + + UNIT_ASSERT(vi.IsNumber()); + UNIT_ASSERT(vi.IsIntNumber()); + UNIT_ASSERT_VALUES_EQUAL(1.0, vi.GetNumber()); + UNIT_ASSERT_VALUES_EQUAL(1, vi.GetIntNumber()); + UNIT_ASSERT_VALUES_EQUAL(1, vi.GetIntNumberMutable()); + + UNIT_ASSERT(NSc::TValue::Equal(vd, vi)); + + vd.SetNumber(1.5); + UNIT_ASSERT(vd.IsNumber()); + UNIT_ASSERT(!vd.IsIntNumber()); + UNIT_ASSERT_VALUES_EQUAL(1.5, vd.GetNumber()); + UNIT_ASSERT_VALUES_EQUAL(1, vd.GetIntNumber()); + UNIT_ASSERT_VALUES_EQUAL(1.5, vd.GetNumberMutable()); + + UNIT_ASSERT(!NSc::TValue::Equal(vd, vi)); + + UNIT_ASSERT_VALUES_EQUAL("1", vi.ToJson()); + UNIT_ASSERT_VALUES_EQUAL("1.5", vd.ToJson()); + + UNIT_ASSERT_VALUES_EQUAL(1, vd.GetIntNumberMutable()); + UNIT_ASSERT(NSc::TValue::Equal(vd, vi)); + vd.SetIntNumber(2); + UNIT_ASSERT(!NSc::TValue::Equal(vd, vi)); + vi.SetNumber(2.); + UNIT_ASSERT(NSc::TValue::Equal(vd, vi)); + vd.SetNumber(2.); + UNIT_ASSERT(NSc::TValue::Equal(vd, vi)); + vi.SetIntNumber(5); + vd.MergeUpdate(vi); + UNIT_ASSERT(vd.IsNumber()); + UNIT_ASSERT(vd.IsIntNumber()); + UNIT_ASSERT_VALUES_EQUAL(5, vd.GetIntNumber()); + vd.SetNumber(3.3); + vi.MergeUpdate(vd); + UNIT_ASSERT(vi.IsNumber()); + UNIT_ASSERT(!vi.IsIntNumber()); + UNIT_ASSERT_VALUES_EQUAL(3.3, vi.GetNumber()); + + vi.SetIntNumber(Max<i64>()); + UNIT_ASSERT_VALUES_EQUAL("9223372036854775807", vi.ToJson()); + } + } + + template <typename T> + void DoTestForce(T t) { + UNIT_ASSERT_VALUES_EQUAL_C(i64(t), NSc::TValue(i64(t)).ForceIntNumber(), ToString(t)); + UNIT_ASSERT_VALUES_EQUAL_C(double(t), NSc::TValue(double(t)).ForceNumber(), ToString(t)); + UNIT_ASSERT_VALUES_EQUAL_C(i64(t), NSc::TValue(TStringBuf(ToString(i64(t)))).ForceIntNumber(), ToString(t)); UNIT_ASSERT_VALUES_EQUAL_C(ToString(double(t)), ToString(NSc::TValue(TStringBuf(ToString(double(t)))).ForceNumber()), ToString(t)); - + UNIT_ASSERT_VALUES_EQUAL_C(ToString(i64(t)), NSc::TValue(TStringBuf(ToString(i64(t)))).ForceString(), ToString(t)); UNIT_ASSERT_VALUES_EQUAL_C(ToString(double(t)), NSc::TValue(TStringBuf(ToString(double(t)))).ForceString(), ToString(t)); - } - - Y_UNIT_TEST(TestForce) { - DoTestForce(Max<i64>()); - DoTestForce(Min<i64>()); - DoTestForce(1.5); - DoTestForce(-1.5); - - UNIT_ASSERT_VALUES_EQUAL(1, NSc::TValue("32a").ForceIntNumber(1)); - UNIT_ASSERT_VALUES_EQUAL(1.5, NSc::TValue("32a").ForceNumber(1.5)); - } - - template <typename T> - void DoCheckRelations(T t, T tless, T tmore, const NSc::TValue& v, TStringBuf ss) { - UNIT_ASSERT_C((t == v), ss); - UNIT_ASSERT_C(!(t != v), ss); - UNIT_ASSERT_C((t <= v), ss); - UNIT_ASSERT_C((t >= v), ss); - UNIT_ASSERT_C(!(t < v), ss); - UNIT_ASSERT_C(!(t > v), ss); - - UNIT_ASSERT_C(!(tless == v), ss); - UNIT_ASSERT_C((tless != v), ss); - UNIT_ASSERT_C((tless <= v), ss); - UNIT_ASSERT_C(!(tless >= v), ss); - UNIT_ASSERT_C((tless < v), ss); - UNIT_ASSERT_C(!(tless > v), ss); - UNIT_ASSERT_C(!(tmore == v), ss); - UNIT_ASSERT_C((tmore != v), ss); - UNIT_ASSERT_C(!(tmore <= v), ss); - UNIT_ASSERT_C((tmore >= v), ss); - UNIT_ASSERT_C(!(tmore < v), ss); - UNIT_ASSERT_C((tmore > v), ss); - } - - void DoCheckRelations(const NSc::TValue& t, const NSc::TValue&, const NSc::TValue&, const NSc::TValue& v, TStringBuf ss) { - UNIT_ASSERT_C((t == v), ss); - UNIT_ASSERT_C(!(t != v), ss); - } - + } + + Y_UNIT_TEST(TestForce) { + DoTestForce(Max<i64>()); + DoTestForce(Min<i64>()); + DoTestForce(1.5); + DoTestForce(-1.5); + + UNIT_ASSERT_VALUES_EQUAL(1, NSc::TValue("32a").ForceIntNumber(1)); + UNIT_ASSERT_VALUES_EQUAL(1.5, NSc::TValue("32a").ForceNumber(1.5)); + } + + template <typename T> + void DoCheckRelations(T t, T tless, T tmore, const NSc::TValue& v, TStringBuf ss) { + UNIT_ASSERT_C((t == v), ss); + UNIT_ASSERT_C(!(t != v), ss); + UNIT_ASSERT_C((t <= v), ss); + UNIT_ASSERT_C((t >= v), ss); + UNIT_ASSERT_C(!(t < v), ss); + UNIT_ASSERT_C(!(t > v), ss); + + UNIT_ASSERT_C(!(tless == v), ss); + UNIT_ASSERT_C((tless != v), ss); + UNIT_ASSERT_C((tless <= v), ss); + UNIT_ASSERT_C(!(tless >= v), ss); + UNIT_ASSERT_C((tless < v), ss); + UNIT_ASSERT_C(!(tless > v), ss); + UNIT_ASSERT_C(!(tmore == v), ss); + UNIT_ASSERT_C((tmore != v), ss); + UNIT_ASSERT_C(!(tmore <= v), ss); + UNIT_ASSERT_C((tmore >= v), ss); + UNIT_ASSERT_C(!(tmore < v), ss); + UNIT_ASSERT_C((tmore > v), ss); + } + + void DoCheckRelations(const NSc::TValue& t, const NSc::TValue&, const NSc::TValue&, const NSc::TValue& v, TStringBuf ss) { + UNIT_ASSERT_C((t == v), ss); + UNIT_ASSERT_C(!(t != v), ss); + } + // void DoCheckRelations(bool t, bool, bool, const NSc::TValue& v, TStringBuf ss) { // UNIT_ASSERT_C((t == v), ss); // UNIT_ASSERT_C(!(t != v), ss); // } - template <typename T> - void DoCheckAssignment(T t, T tless, T tmore, TStringBuf s, TStringBuf ss) { + template <typename T> + void DoCheckAssignment(T t, T tless, T tmore, TStringBuf s, TStringBuf ss) { bool expectint = std::is_integral<T>::value; bool expectnum = std::is_arithmetic<T>::value; - bool expectbool = std::is_same<bool, T>::value; - - { - NSc::TValue v(t); - UNIT_ASSERT_VALUES_EQUAL_C(expectnum, v.IsNumber(), ss); - UNIT_ASSERT_VALUES_EQUAL_C(expectint, v.IsIntNumber(), ss); - UNIT_ASSERT_VALUES_EQUAL_C(expectbool, v.IsBool(), ss); - UNIT_ASSERT_VALUES_EQUAL_C(s, v.ToJson(), ss); - DoCheckRelations(t, tless, tmore, v, ss); - } - { - NSc::TValue v; + bool expectbool = std::is_same<bool, T>::value; + + { + NSc::TValue v(t); + UNIT_ASSERT_VALUES_EQUAL_C(expectnum, v.IsNumber(), ss); + UNIT_ASSERT_VALUES_EQUAL_C(expectint, v.IsIntNumber(), ss); + UNIT_ASSERT_VALUES_EQUAL_C(expectbool, v.IsBool(), ss); + UNIT_ASSERT_VALUES_EQUAL_C(s, v.ToJson(), ss); + DoCheckRelations(t, tless, tmore, v, ss); + } + { + NSc::TValue v; UNIT_ASSERT(v.IsNull()); - v = t; + v = t; UNIT_ASSERT(!v.IsNull()); - UNIT_ASSERT_VALUES_EQUAL_C(expectnum, v.IsNumber(), ss); - UNIT_ASSERT_VALUES_EQUAL_C(expectint, v.IsIntNumber(), ss); - UNIT_ASSERT_VALUES_EQUAL_C(expectbool, v.IsBool(), ss); - UNIT_ASSERT_VALUES_EQUAL_C(s, v.ToJson(), ss); - DoCheckRelations(t, tless, tmore, v, ss); - } - } - - template <size_t N> - void DoCheckAssignmentArr(const char (&t)[N], const char (&tless)[N], const char (&tmore)[N], TStringBuf s, TStringBuf ss) { - { - NSc::TValue v(t); - - UNIT_ASSERT_C(v.IsString(), ss); - UNIT_ASSERT_VALUES_EQUAL_C(s, v.ToJson(), ss); - DoCheckRelations(t, tless, tmore, v, ss); - } - { - NSc::TValue v; - v = t; - UNIT_ASSERT_C(v.IsString(), ss); - UNIT_ASSERT_VALUES_EQUAL_C(s, v.ToJson(), ss); - DoCheckRelations(t, tless, tmore, v, ss); - } - } - - template <typename T> - void DoCheckAssignmentNum(T t, T tless, T tmore, TStringBuf s, TStringBuf ss) { - DoCheckAssignment(t, tless, tmore, s, ss); - { - NSc::TValue v; - T tt = (v = t); - UNIT_ASSERT_VALUES_EQUAL_C(t, tt, ss); - } - } - - Y_UNIT_TEST(TestAssignments) { - for (int i = -2; i < 3; ++i) { + UNIT_ASSERT_VALUES_EQUAL_C(expectnum, v.IsNumber(), ss); + UNIT_ASSERT_VALUES_EQUAL_C(expectint, v.IsIntNumber(), ss); + UNIT_ASSERT_VALUES_EQUAL_C(expectbool, v.IsBool(), ss); + UNIT_ASSERT_VALUES_EQUAL_C(s, v.ToJson(), ss); + DoCheckRelations(t, tless, tmore, v, ss); + } + } + + template <size_t N> + void DoCheckAssignmentArr(const char (&t)[N], const char (&tless)[N], const char (&tmore)[N], TStringBuf s, TStringBuf ss) { + { + NSc::TValue v(t); + + UNIT_ASSERT_C(v.IsString(), ss); + UNIT_ASSERT_VALUES_EQUAL_C(s, v.ToJson(), ss); + DoCheckRelations(t, tless, tmore, v, ss); + } + { + NSc::TValue v; + v = t; + UNIT_ASSERT_C(v.IsString(), ss); + UNIT_ASSERT_VALUES_EQUAL_C(s, v.ToJson(), ss); + DoCheckRelations(t, tless, tmore, v, ss); + } + } + + template <typename T> + void DoCheckAssignmentNum(T t, T tless, T tmore, TStringBuf s, TStringBuf ss) { + DoCheckAssignment(t, tless, tmore, s, ss); + { + NSc::TValue v; + T tt = (v = t); + UNIT_ASSERT_VALUES_EQUAL_C(t, tt, ss); + } + } + + Y_UNIT_TEST(TestAssignments) { + for (int i = -2; i < 3; ++i) { TString ii = ToString(i); - int iless = i - 1; - int imore = i + 1; - DoCheckAssignmentNum<signed char>(i, iless, imore, ii, "schar"); - DoCheckAssignmentNum<short>(i, iless, imore, ii, "short"); - DoCheckAssignmentNum<int>(i, iless, imore, ii, "int"); - DoCheckAssignmentNum<long>(i, iless, imore, ii, "long"); - DoCheckAssignmentNum<long long>(i, iless, imore, ii, "longlong"); - DoCheckAssignmentNum<i8>(i, iless, imore, ii, "i8"); - DoCheckAssignmentNum<i16>(i, iless, imore, ii, "i16"); - DoCheckAssignmentNum<i32>(i, iless, imore, ii, "i32"); - DoCheckAssignmentNum<i64>(i, iless, imore, ii, "i64"); - - DoCheckAssignmentNum<float>(i, iless, imore, ii, "float"); - DoCheckAssignmentNum<double>(i, iless, imore, ii, "double"); - } - + int iless = i - 1; + int imore = i + 1; + DoCheckAssignmentNum<signed char>(i, iless, imore, ii, "schar"); + DoCheckAssignmentNum<short>(i, iless, imore, ii, "short"); + DoCheckAssignmentNum<int>(i, iless, imore, ii, "int"); + DoCheckAssignmentNum<long>(i, iless, imore, ii, "long"); + DoCheckAssignmentNum<long long>(i, iless, imore, ii, "longlong"); + DoCheckAssignmentNum<i8>(i, iless, imore, ii, "i8"); + DoCheckAssignmentNum<i16>(i, iless, imore, ii, "i16"); + DoCheckAssignmentNum<i32>(i, iless, imore, ii, "i32"); + DoCheckAssignmentNum<i64>(i, iless, imore, ii, "i64"); + + DoCheckAssignmentNum<float>(i, iless, imore, ii, "float"); + DoCheckAssignmentNum<double>(i, iless, imore, ii, "double"); + } + // DoCheckAssignment<bool>(true, true, true, "true", "bool"); // DoCheckAssignment<bool>(false, false, false, "false", "bool"); - - for (int i = 1; i < 3; ++i) { + + for (int i = 1; i < 3; ++i) { TString ii = ToString(i); - int iless = i - 1; - int imore = i + 1; - - DoCheckAssignmentNum<char>(i, iless, imore, ii, "char"); - - DoCheckAssignmentNum<signed char>(i, iless, imore, ii, "schar"); - DoCheckAssignmentNum<short>(i, iless, imore, ii, "short"); - DoCheckAssignmentNum<int>(i, iless, imore, ii, "int"); - DoCheckAssignmentNum<long>(i, iless, imore, ii, "long"); - DoCheckAssignmentNum<long long>(i, iless, imore, ii, "longlong"); - DoCheckAssignmentNum<i8>(i, iless, imore, ii, "i8"); - DoCheckAssignmentNum<i16>(i, iless, imore, ii, "i16"); - DoCheckAssignmentNum<i32>(i, iless, imore, ii, "i32"); - DoCheckAssignmentNum<i64>(i, iless, imore, ii, "i64"); - - DoCheckAssignmentNum<unsigned char>(i, iless, imore, ii, "uchar"); - DoCheckAssignmentNum<unsigned short>(i, iless, imore, ii, "ushort"); - DoCheckAssignmentNum<unsigned int>(i, iless, imore, ii, "uint"); - DoCheckAssignmentNum<unsigned long>(i, iless, imore, ii, "ulong"); - DoCheckAssignmentNum<unsigned long long>(i, iless, imore, ii, "ulonglong"); - DoCheckAssignmentNum<ui8>(i, iless, imore, ii, "ui8"); - DoCheckAssignmentNum<ui16>(i, iless, imore, ii, "ui16"); - DoCheckAssignmentNum<ui32>(i, iless, imore, ii, "ui32"); - DoCheckAssignmentNum<ui64>(i, iless, imore, ii, "ui64"); - - DoCheckAssignmentNum<float>(i, iless, imore, ii, "float"); - DoCheckAssignmentNum<double>(i, iless, imore, ii, "double"); - } - + int iless = i - 1; + int imore = i + 1; + + DoCheckAssignmentNum<char>(i, iless, imore, ii, "char"); + + DoCheckAssignmentNum<signed char>(i, iless, imore, ii, "schar"); + DoCheckAssignmentNum<short>(i, iless, imore, ii, "short"); + DoCheckAssignmentNum<int>(i, iless, imore, ii, "int"); + DoCheckAssignmentNum<long>(i, iless, imore, ii, "long"); + DoCheckAssignmentNum<long long>(i, iless, imore, ii, "longlong"); + DoCheckAssignmentNum<i8>(i, iless, imore, ii, "i8"); + DoCheckAssignmentNum<i16>(i, iless, imore, ii, "i16"); + DoCheckAssignmentNum<i32>(i, iless, imore, ii, "i32"); + DoCheckAssignmentNum<i64>(i, iless, imore, ii, "i64"); + + DoCheckAssignmentNum<unsigned char>(i, iless, imore, ii, "uchar"); + DoCheckAssignmentNum<unsigned short>(i, iless, imore, ii, "ushort"); + DoCheckAssignmentNum<unsigned int>(i, iless, imore, ii, "uint"); + DoCheckAssignmentNum<unsigned long>(i, iless, imore, ii, "ulong"); + DoCheckAssignmentNum<unsigned long long>(i, iless, imore, ii, "ulonglong"); + DoCheckAssignmentNum<ui8>(i, iless, imore, ii, "ui8"); + DoCheckAssignmentNum<ui16>(i, iless, imore, ii, "ui16"); + DoCheckAssignmentNum<ui32>(i, iless, imore, ii, "ui32"); + DoCheckAssignmentNum<ui64>(i, iless, imore, ii, "ui64"); + + DoCheckAssignmentNum<float>(i, iless, imore, ii, "float"); + DoCheckAssignmentNum<double>(i, iless, imore, ii, "double"); + } + TString uuu = "uuu"; TString uua = "uua"; TString uuz = "uuz"; - DoCheckAssignment<char*>(uuu.begin(), uua.begin(), uuz.begin(), "\"uuu\"", "char*"); - DoCheckAssignment<const char*>("www", "wwa", "wwz", "\"www\"", "const char*"); - DoCheckAssignmentArr("xxx", "xxa", "xxz", "\"xxx\"", "const char[]"); - DoCheckAssignment<TStringBuf>("yyy", "yya", "yyz", "\"yyy\"", "TStringBuf"); + DoCheckAssignment<char*>(uuu.begin(), uua.begin(), uuz.begin(), "\"uuu\"", "char*"); + DoCheckAssignment<const char*>("www", "wwa", "wwz", "\"www\"", "const char*"); + DoCheckAssignmentArr("xxx", "xxa", "xxz", "\"xxx\"", "const char[]"); + DoCheckAssignment<TStringBuf>("yyy", "yya", "yyz", "\"yyy\"", "TStringBuf"); #if defined(_MSC_VER) //TODO #else DoCheckAssignment<TString>("ttt", "tta", "ttz", "\"ttt\"", "TString"); #endif - - NSc::TValue v; - v.SetDict(); - DoCheckAssignment<NSc::TValue>(v, v, v, "{}", "TValue"); - DoCheckAssignment<NSc::TValue&>(v, v, v, "{}", "TValue&"); - DoCheckAssignment<const NSc::TValue&>(v, v, v, "{}", "const TValue&"); - - NSc::TValue v1{1}; - UNIT_ASSERT_VALUES_EQUAL(v1.ToJson(), "1"); - } - - Y_UNIT_TEST(TestAssignmentDictChild) { + + NSc::TValue v; + v.SetDict(); + DoCheckAssignment<NSc::TValue>(v, v, v, "{}", "TValue"); + DoCheckAssignment<NSc::TValue&>(v, v, v, "{}", "TValue&"); + DoCheckAssignment<const NSc::TValue&>(v, v, v, "{}", "const TValue&"); + + NSc::TValue v1{1}; + UNIT_ASSERT_VALUES_EQUAL(v1.ToJson(), "1"); + } + + Y_UNIT_TEST(TestAssignmentDictChild) { + { + NSc::TValue v; + { + NSc::TValue b; + v["a"] = b; + } + v = v["a"]; + } { NSc::TValue v; { NSc::TValue b; v["a"] = b; } - v = v["a"]; - } - { - NSc::TValue v; - { - NSc::TValue b; - v["a"] = b; - } v = v.Get("a"); - } - { - NSc::TValue v; - { - NSc::TValue b; - v["a"] = b; - } - v = std::move(v["a"]); - } - } - - Y_UNIT_TEST(TestInsert) { - NSc::TValue v; - v.Insert(0, "b"); - v.Insert(0, "a"); - v.Insert(2, "d"); - v.Insert(2, "c"); - UNIT_ASSERT_VALUES_EQUAL(v.ToJson(), R"(["a","b","c","d"])"); - + } + { + NSc::TValue v; + { + NSc::TValue b; + v["a"] = b; + } + v = std::move(v["a"]); + } + } + + Y_UNIT_TEST(TestInsert) { + NSc::TValue v; + v.Insert(0, "b"); + v.Insert(0, "a"); + v.Insert(2, "d"); + v.Insert(2, "c"); + UNIT_ASSERT_VALUES_EQUAL(v.ToJson(), R"(["a","b","c","d"])"); + v.AppendAll({1, 2, 3}); - UNIT_ASSERT_VALUES_EQUAL(v.ToJson(), R"(["a","b","c","d",1,2,3])"); - + UNIT_ASSERT_VALUES_EQUAL(v.ToJson(), R"(["a","b","c","d",1,2,3])"); + TVector<int> d{4, 5, 6}; - v.AppendAll(d); - UNIT_ASSERT_VALUES_EQUAL(v.ToJson(), R"(["a","b","c","d",1,2,3,4,5,6])"); - UNIT_ASSERT_VALUES_EQUAL(d.size(), 3u); + v.AppendAll(d); + UNIT_ASSERT_VALUES_EQUAL(v.ToJson(), R"(["a","b","c","d",1,2,3,4,5,6])"); + UNIT_ASSERT_VALUES_EQUAL(d.size(), 3u); TVector<TStringBuf> s{"x", "y", "z"}; - v.AppendAll(s.begin(), s.end()); - UNIT_ASSERT_VALUES_EQUAL(v.ToJson(), R"(["a","b","c","d",1,2,3,4,5,6,"x","y","z"])"); - - UNIT_ASSERT_VALUES_EQUAL(v.Clone().Clear().AppendAll(s).ToJson(), R"(["x","y","z"])"); + v.AppendAll(s.begin(), s.end()); + UNIT_ASSERT_VALUES_EQUAL(v.ToJson(), R"(["a","b","c","d",1,2,3,4,5,6,"x","y","z"])"); + + UNIT_ASSERT_VALUES_EQUAL(v.Clone().Clear().AppendAll(s).ToJson(), R"(["x","y","z"])"); UNIT_ASSERT_VALUES_EQUAL(v.Clone().Clear().AppendAll(TVector<TStringBuf>()).ToJson(), R"([])"); - + v.AddAll({{"a", "b"}, {"c", "d"}}); - UNIT_ASSERT_VALUES_EQUAL(v.ToJson(), R"({"a":"b","c":"d"})"); - } - - Y_UNIT_TEST(TestFrontBack) { - NSc::TValue v; - const NSc::TValue& vv = v; + UNIT_ASSERT_VALUES_EQUAL(v.ToJson(), R"({"a":"b","c":"d"})"); + } + + Y_UNIT_TEST(TestFrontBack) { + NSc::TValue v; + const NSc::TValue& vv = v; UNIT_ASSERT(NSc::TValue::Same(vv.Front(), NSc::Null())); UNIT_ASSERT(NSc::TValue::Same(vv.Back(), NSc::Null())); - UNIT_ASSERT(!vv.IsArray()); - v.Back() = "a"; - UNIT_ASSERT_VALUES_EQUAL("a", vv.Front().GetString()); - UNIT_ASSERT_VALUES_EQUAL("a", vv.Back().GetString()); - UNIT_ASSERT(vv.IsArray()); - v.Push("b"); - UNIT_ASSERT_VALUES_EQUAL("a", vv.Front().GetString()); - UNIT_ASSERT_VALUES_EQUAL("b", vv.Back().GetString()); - - UNIT_ASSERT_VALUES_EQUAL("b", v.Pop().GetString()); - - UNIT_ASSERT_VALUES_EQUAL("a", vv.Front().GetString()); - UNIT_ASSERT_VALUES_EQUAL("a", vv.Back().GetString()); - - UNIT_ASSERT_VALUES_EQUAL("a", v.Pop().GetString()); - + UNIT_ASSERT(!vv.IsArray()); + v.Back() = "a"; + UNIT_ASSERT_VALUES_EQUAL("a", vv.Front().GetString()); + UNIT_ASSERT_VALUES_EQUAL("a", vv.Back().GetString()); + UNIT_ASSERT(vv.IsArray()); + v.Push("b"); + UNIT_ASSERT_VALUES_EQUAL("a", vv.Front().GetString()); + UNIT_ASSERT_VALUES_EQUAL("b", vv.Back().GetString()); + + UNIT_ASSERT_VALUES_EQUAL("b", v.Pop().GetString()); + + UNIT_ASSERT_VALUES_EQUAL("a", vv.Front().GetString()); + UNIT_ASSERT_VALUES_EQUAL("a", vv.Back().GetString()); + + UNIT_ASSERT_VALUES_EQUAL("a", v.Pop().GetString()); + UNIT_ASSERT(NSc::TValue::Same(vv.Front(), NSc::Null())); UNIT_ASSERT(NSc::TValue::Same(vv.Back(), NSc::Null())); - - v.Front() = "a"; - UNIT_ASSERT_VALUES_EQUAL("a", vv.Front().GetString()); - UNIT_ASSERT_VALUES_EQUAL("a", vv.Back().GetString()); - } - - Y_UNIT_TEST(TestAssign) { - NSc::TValue v; - v.SetArray(); - v.Push() = "test"; - UNIT_ASSERT_VALUES_EQUAL(v.ToJson(), "[\"test\"]"); - UNIT_ASSERT(NSc::TValue::SamePool(v[0], v)); - } - - NSc::TValue MutableRef(const NSc::TValue& v) { - return v; - } - - NSc::TValue Clone(const NSc::TValue& v) { - NSc::TValue v1 = v.Clone(); - UNIT_ASSERT_VALUES_EQUAL(v1.ToJson(true), v.ToJson(true)); - return v1; - } - - Y_UNIT_TEST(TestCOW) { - NSc::TValue vd = NSc::TValue::FromJson("{ a : 1, b : c}"); - NSc::TValue va = NSc::TValue::FromJson("[ x, y]"); - NSc::TValue vs = NSc::TValue::FromJson("foo"); - NSc::TValue vn = NSc::TValue::FromJson("1"); + + v.Front() = "a"; + UNIT_ASSERT_VALUES_EQUAL("a", vv.Front().GetString()); + UNIT_ASSERT_VALUES_EQUAL("a", vv.Back().GetString()); + } + + Y_UNIT_TEST(TestAssign) { + NSc::TValue v; + v.SetArray(); + v.Push() = "test"; + UNIT_ASSERT_VALUES_EQUAL(v.ToJson(), "[\"test\"]"); + UNIT_ASSERT(NSc::TValue::SamePool(v[0], v)); + } + + NSc::TValue MutableRef(const NSc::TValue& v) { + return v; + } + + NSc::TValue Clone(const NSc::TValue& v) { + NSc::TValue v1 = v.Clone(); + UNIT_ASSERT_VALUES_EQUAL(v1.ToJson(true), v.ToJson(true)); + return v1; + } + + Y_UNIT_TEST(TestCOW) { + NSc::TValue vd = NSc::TValue::FromJson("{ a : 1, b : c}"); + NSc::TValue va = NSc::TValue::FromJson("[ x, y]"); + NSc::TValue vs = NSc::TValue::FromJson("foo"); + NSc::TValue vn = NSc::TValue::FromJson("1"); TString sd = "{\"a\":1,\"b\":\"c\"}"; TString sa = "[\"x\",\"y\"]"; TString ss = "\"foo\""; TString sn = "1"; - UNIT_ASSERT_VALUES_EQUAL(sd, vd.ToJson(true)); - UNIT_ASSERT_VALUES_EQUAL(sa, va.ToJson(true)); - UNIT_ASSERT_VALUES_EQUAL(ss, vs.ToJson(true)); - UNIT_ASSERT_VALUES_EQUAL(sn, vn.ToJson(true)); - - { - NSc::TValue v2 = MutableRef(vn); - v2 = -1; - - UNIT_ASSERT_VALUES_EQUAL(sn, vn.ToJson(true)); - - NSc::TValue v3 = Clone(vn); - v3 = -1; - - UNIT_ASSERT_VALUES_EQUAL(v3.ToJson(true), v2.ToJson(true)); - } - - { - NSc::TValue v2 = MutableRef(vs); - v2 = "xxx"; - - UNIT_ASSERT_VALUES_EQUAL(ss, vs.ToJson(true)); - - NSc::TValue v3 = Clone(vs); - v3 = "xxx"; - - UNIT_ASSERT_VALUES_EQUAL(v3.ToJson(true), v2.ToJson(true)); - } - - { - NSc::TValue v2 = MutableRef(vd); - v2["a"] = "zzz"; - - UNIT_ASSERT_VALUES_EQUAL(sd, vd.ToJson(true)); - - NSc::TValue v3 = Clone(vd); - v3["a"] = "zzz"; - - UNIT_ASSERT_VALUES_EQUAL(v3.ToJson(true), v2.ToJson(true)); - } - - { - NSc::TValue v2 = MutableRef(va); - v2[0] = "zzz"; - - UNIT_ASSERT_VALUES_EQUAL(sa, va.ToJson(true)); - - NSc::TValue v3 = Clone(va); - v3[0] = "zzz"; - - UNIT_ASSERT_VALUES_EQUAL(v3.ToJson(true), v2.ToJson(true)); - } - - { - NSc::TValue v2 = MutableRef(va); - - UNIT_ASSERT_VALUES_EQUAL(sa, va.ToJson(true)); - - NSc::TValue v3 = Clone(va); - - UNIT_ASSERT_VALUES_EQUAL(v3.ToJson(true), v2.ToJson(true)); - } - - { - NSc::TValue v2 = MutableRef(va); - v2.ClearArray(); - - UNIT_ASSERT_VALUES_EQUAL(sa, va.ToJson(true)); - - NSc::TValue v3 = Clone(va); - v3.ClearArray(); - - UNIT_ASSERT_VALUES_EQUAL(v3.ToJson(true), v2.ToJson(true)); - } - } - - Y_UNIT_TEST(TestOperators) { - UNIT_ASSERT("test" == NSc::TValue("test")); - UNIT_ASSERT(NSc::TValue("test") == "test"); - - UNIT_ASSERT("test1" != NSc::TValue("test")); - UNIT_ASSERT(NSc::TValue("test") != "test1"); - - UNIT_ASSERT("test1" > NSc::TValue("test")); - UNIT_ASSERT(NSc::TValue("test") < "test1"); - - UNIT_ASSERT("test" < NSc::TValue("test1")); - UNIT_ASSERT(NSc::TValue("test1") > "test"); - - UNIT_ASSERT(1 == NSc::TValue(1)); - UNIT_ASSERT(NSc::TValue(1) == 1); - - UNIT_ASSERT(2 != NSc::TValue(1)); - UNIT_ASSERT(NSc::TValue(1) != 2); - - UNIT_ASSERT(1 < NSc::TValue(2)); - UNIT_ASSERT(NSc::TValue(2) > 1); - - UNIT_ASSERT(2 > NSc::TValue(1)); - UNIT_ASSERT(NSc::TValue(1) < 2); - + UNIT_ASSERT_VALUES_EQUAL(sd, vd.ToJson(true)); + UNIT_ASSERT_VALUES_EQUAL(sa, va.ToJson(true)); + UNIT_ASSERT_VALUES_EQUAL(ss, vs.ToJson(true)); + UNIT_ASSERT_VALUES_EQUAL(sn, vn.ToJson(true)); + + { + NSc::TValue v2 = MutableRef(vn); + v2 = -1; + + UNIT_ASSERT_VALUES_EQUAL(sn, vn.ToJson(true)); + + NSc::TValue v3 = Clone(vn); + v3 = -1; + + UNIT_ASSERT_VALUES_EQUAL(v3.ToJson(true), v2.ToJson(true)); + } + + { + NSc::TValue v2 = MutableRef(vs); + v2 = "xxx"; + + UNIT_ASSERT_VALUES_EQUAL(ss, vs.ToJson(true)); + + NSc::TValue v3 = Clone(vs); + v3 = "xxx"; + + UNIT_ASSERT_VALUES_EQUAL(v3.ToJson(true), v2.ToJson(true)); + } + + { + NSc::TValue v2 = MutableRef(vd); + v2["a"] = "zzz"; + + UNIT_ASSERT_VALUES_EQUAL(sd, vd.ToJson(true)); + + NSc::TValue v3 = Clone(vd); + v3["a"] = "zzz"; + + UNIT_ASSERT_VALUES_EQUAL(v3.ToJson(true), v2.ToJson(true)); + } + + { + NSc::TValue v2 = MutableRef(va); + v2[0] = "zzz"; + + UNIT_ASSERT_VALUES_EQUAL(sa, va.ToJson(true)); + + NSc::TValue v3 = Clone(va); + v3[0] = "zzz"; + + UNIT_ASSERT_VALUES_EQUAL(v3.ToJson(true), v2.ToJson(true)); + } + + { + NSc::TValue v2 = MutableRef(va); + + UNIT_ASSERT_VALUES_EQUAL(sa, va.ToJson(true)); + + NSc::TValue v3 = Clone(va); + + UNIT_ASSERT_VALUES_EQUAL(v3.ToJson(true), v2.ToJson(true)); + } + + { + NSc::TValue v2 = MutableRef(va); + v2.ClearArray(); + + UNIT_ASSERT_VALUES_EQUAL(sa, va.ToJson(true)); + + NSc::TValue v3 = Clone(va); + v3.ClearArray(); + + UNIT_ASSERT_VALUES_EQUAL(v3.ToJson(true), v2.ToJson(true)); + } + } + + Y_UNIT_TEST(TestOperators) { + UNIT_ASSERT("test" == NSc::TValue("test")); + UNIT_ASSERT(NSc::TValue("test") == "test"); + + UNIT_ASSERT("test1" != NSc::TValue("test")); + UNIT_ASSERT(NSc::TValue("test") != "test1"); + + UNIT_ASSERT("test1" > NSc::TValue("test")); + UNIT_ASSERT(NSc::TValue("test") < "test1"); + + UNIT_ASSERT("test" < NSc::TValue("test1")); + UNIT_ASSERT(NSc::TValue("test1") > "test"); + + UNIT_ASSERT(1 == NSc::TValue(1)); + UNIT_ASSERT(NSc::TValue(1) == 1); + + UNIT_ASSERT(2 != NSc::TValue(1)); + UNIT_ASSERT(NSc::TValue(1) != 2); + + UNIT_ASSERT(1 < NSc::TValue(2)); + UNIT_ASSERT(NSc::TValue(2) > 1); + + UNIT_ASSERT(2 > NSc::TValue(1)); + UNIT_ASSERT(NSc::TValue(1) < 2); + UNIT_ASSERT(TString("test") == NSc::TValue("test")); - } - - Y_UNIT_TEST(TestDestructor) { - NSc::TValue v; - const NSc::TValue& v1 = v; - v1.GetString(); - v1.GetArray(); - v1.GetNumber(); - v1.GetDict(); - v.GetString(); - v.GetArray(); - v.GetNumber(); - v.GetDict(); - } - - void DoTestSamePool(TStringBuf json, TStringBuf jpath) { - NSc::TValue v = NSc::TValue::FromJson(json); - UNIT_ASSERT_C(NSc::TValue::SamePool(v, v.TrySelect(jpath)), json); - } - - Y_UNIT_TEST(TestSamePool) { - DoTestSamePool("", ""); - DoTestSamePool("a", ""); - DoTestSamePool("[a]", "0"); - DoTestSamePool("{a:b}", "a"); - DoTestSamePool("{a:{b:c}}", "a/b"); - DoTestSamePool("{a:{b:[c, {}]}}", "a/b/1"); - DoTestSamePool("{a:{b:[c, {d:{e:[]}}]}}", "a/b/1/d/e"); + } + + Y_UNIT_TEST(TestDestructor) { + NSc::TValue v; + const NSc::TValue& v1 = v; + v1.GetString(); + v1.GetArray(); + v1.GetNumber(); + v1.GetDict(); + v.GetString(); + v.GetArray(); + v.GetNumber(); + v.GetDict(); + } + + void DoTestSamePool(TStringBuf json, TStringBuf jpath) { + NSc::TValue v = NSc::TValue::FromJson(json); + UNIT_ASSERT_C(NSc::TValue::SamePool(v, v.TrySelect(jpath)), json); + } + + Y_UNIT_TEST(TestSamePool) { + DoTestSamePool("", ""); + DoTestSamePool("a", ""); + DoTestSamePool("[a]", "0"); + DoTestSamePool("{a:b}", "a"); + DoTestSamePool("{a:{b:c}}", "a/b"); + DoTestSamePool("{a:{b:[c, {}]}}", "a/b/1"); + DoTestSamePool("{a:{b:[c, {d:{e:[]}}]}}", "a/b/1/d/e"); UNIT_ASSERT(!NSc::TValue::SamePool(NSc::TValue(), NSc::TValue())); UNIT_ASSERT(!NSc::TValue::SamePool(NSc::Null().Clone(), NSc::Null())); - UNIT_ASSERT(!NSc::TValue::SamePool(NSc::TValue() = 0, NSc::TValue())); - UNIT_ASSERT(!NSc::TValue::SamePool(NSc::TValue::FromJson("a"), NSc::TValue::FromJson("a"))); - NSc::TValue v, vv; - v["x"] = vv; - UNIT_ASSERT(!NSc::TValue::SamePool(v, vv)); - UNIT_ASSERT(!NSc::TValue::SamePool(v, v["x"])); + UNIT_ASSERT(!NSc::TValue::SamePool(NSc::TValue() = 0, NSc::TValue())); + UNIT_ASSERT(!NSc::TValue::SamePool(NSc::TValue::FromJson("a"), NSc::TValue::FromJson("a"))); + NSc::TValue v, vv; + v["x"] = vv; + UNIT_ASSERT(!NSc::TValue::SamePool(v, vv)); + UNIT_ASSERT(!NSc::TValue::SamePool(v, v["x"])); v = vv; UNIT_ASSERT(NSc::TValue::SamePool(v, vv)); - } - - Y_UNIT_TEST(TestLoopDetection) { - NSc::NImpl::GetTlsInstance<NSc::NImpl::TSelfLoopContext>().ReportingMode - = NSc::NImpl::TSelfLoopContext::EMode::Stderr; - - NSc::TValue x; - - x["a"]["x"] = x; - x["b"][0] = x; - - UNIT_ASSERT(x.IsSameOrAncestorOf(x)); - UNIT_ASSERT(x.IsSameOrAncestorOf(x.Get("a"))); - UNIT_ASSERT(x.Get("a").IsSameOrAncestorOf(x)); - - UNIT_ASSERT_VALUES_EQUAL(x.ToJson(), "{\"a\":{\"x\":null},\"b\":[null]}"); - - NSc::TValue y = x.Clone(); - - UNIT_ASSERT(y.Has("a")); - UNIT_ASSERT(y.Get("a").Has("x")); - UNIT_ASSERT(y.Get("a").Get("x").IsNull()); - UNIT_ASSERT(y.Get("a").Get("x").IsNull()); - - UNIT_ASSERT(y.Has("b")); - UNIT_ASSERT(y.Get("b").Has(0)); - UNIT_ASSERT(y.Get("b").Get(0).IsNull()); - - UNIT_ASSERT_VALUES_EQUAL(y.ToJson(), "{\"a\":{\"x\":null},\"b\":[null]}"); - - NSc::TValue z; - z.MergeUpdate(x); - - UNIT_ASSERT(z.Has("a")); - UNIT_ASSERT(z.Get("a").Has("x")); - UNIT_ASSERT(z.Get("a").Get("x").IsNull()); - - UNIT_ASSERT(z.Has("b")); - UNIT_ASSERT(z.Get("b").Has(0)); - UNIT_ASSERT(z.Get("b").Get(0).IsNull()); - - UNIT_ASSERT_VALUES_EQUAL(z.ToJson(), "{\"a\":{\"x\":null},\"b\":[null]}"); - - x["a"].Delete("x"); - x["b"].Delete(0); - - UNIT_ASSERT(x.IsSameOrAncestorOf(x)); - UNIT_ASSERT(x.IsSameOrAncestorOf(x.Get("a"))); - UNIT_ASSERT(!x.Get("a").IsSameOrAncestorOf(x)); - } - - Y_UNIT_TEST(TestLoopDetectionThrow) { - NSc::NImpl::GetTlsInstance<NSc::NImpl::TSelfLoopContext>().ReportingMode - = NSc::NImpl::TSelfLoopContext::EMode::Throw; - - { - NSc::TValue x; - x["a"]["x"] = x; - - UNIT_ASSERT(x.IsSameOrAncestorOf(x)); - UNIT_ASSERT(x.IsSameOrAncestorOf(x.Get("a"))); - UNIT_ASSERT(x.Get("a").IsSameOrAncestorOf(x)); - - UNIT_ASSERT_EXCEPTION(x.ToJson(), NSc::TSchemeException); - UNIT_ASSERT_EXCEPTION(x.Clone(), NSc::TSchemeException); - - NSc::TValue z; - UNIT_ASSERT_EXCEPTION(z.MergeUpdate(x), NSc::TSchemeException); - - UNIT_ASSERT_VALUES_EQUAL(z.ToJson(), "{\"a\":{\"x\":null}}"); - - x["a"].Delete("x"); - - UNIT_ASSERT(x.IsSameOrAncestorOf(x)); - UNIT_ASSERT(x.IsSameOrAncestorOf(x.Get("a"))); - UNIT_ASSERT(!x.Get("a").IsSameOrAncestorOf(x)); - - UNIT_ASSERT_VALUES_EQUAL(x.ToJson(), "{\"a\":{}}"); - } - - { - NSc::TValue x; - x["a"][0] = x; - - UNIT_ASSERT(x.IsSameOrAncestorOf(x)); - UNIT_ASSERT(x.IsSameOrAncestorOf(x.Get("a"))); - UNIT_ASSERT(x.Get("a").IsSameOrAncestorOf(x)); - - UNIT_ASSERT_EXCEPTION(x.ToJson(), NSc::TSchemeException); - UNIT_ASSERT_EXCEPTION(x.Clone(), NSc::TSchemeException); - - NSc::TValue z; - UNIT_ASSERT_EXCEPTION(z.MergeUpdate(x), NSc::TSchemeException); - - UNIT_ASSERT_VALUES_EQUAL(z.ToJson(), "{\"a\":[null]}"); - - x["a"].Delete(0); - - UNIT_ASSERT(x.IsSameOrAncestorOf(x)); - UNIT_ASSERT(x.IsSameOrAncestorOf(x.Get("a"))); - UNIT_ASSERT(!x.Get("a").IsSameOrAncestorOf(x)); - - UNIT_ASSERT_VALUES_EQUAL(x.ToJson(), "{\"a\":[]}"); - } - } - - Y_UNIT_TEST(TestIsSameOrAncestorOf) { - NSc::TValue x; - UNIT_ASSERT(x.IsSameOrAncestorOf(x)); - - x["a"] = NSc::Null(); - UNIT_ASSERT(x.IsSameOrAncestorOf(x.Get("a"))); - - NSc::TValue a = 1; - NSc::TValue b = 2; - NSc::TValue c = 3; - NSc::TValue d = 4; - - x["a"] = a; - x["b"] = b; - x["c"] = a; - UNIT_ASSERT(x.IsSameOrAncestorOf(a)); - UNIT_ASSERT(x.IsSameOrAncestorOf(b)); - UNIT_ASSERT(x.Get("a").IsSameOrAncestorOf(a)); - UNIT_ASSERT(x.Get("b").IsSameOrAncestorOf(b)); - UNIT_ASSERT(x.Get("c").IsSameOrAncestorOf(a)); - - UNIT_ASSERT(!x.Get("a").IsSameOrAncestorOf(b)); - UNIT_ASSERT(!x.Get("b").IsSameOrAncestorOf(a)); - - UNIT_ASSERT(!x.Get("a").Get(0).IsSameOrAncestorOf(a)); - - b.Push() = c; - b.Push() = d; - b.Push() = c; - - UNIT_ASSERT(x.Get("b").IsSameOrAncestorOf(b)); - UNIT_ASSERT(x.IsSameOrAncestorOf(c)); - UNIT_ASSERT(x.IsSameOrAncestorOf(d)); - UNIT_ASSERT(x.Get("b").Get(0).IsSameOrAncestorOf(c)); - UNIT_ASSERT(x.Get("b").Get(1).IsSameOrAncestorOf(d)); - UNIT_ASSERT(x.Get("b").Get(2).IsSameOrAncestorOf(c)); - - UNIT_ASSERT(b.Get(0).IsSameOrAncestorOf(b.Get(2))); - UNIT_ASSERT(b.Get(2).IsSameOrAncestorOf(b.Get(0))); - UNIT_ASSERT(b.Get(0).IsSameOrAncestorOf(c)); - UNIT_ASSERT(b.Get(1).IsSameOrAncestorOf(d)); - - UNIT_ASSERT(!b.Get(0).IsSameOrAncestorOf(d)); - UNIT_ASSERT(!b.Get(1).IsSameOrAncestorOf(c)); - } - - static void ByVal(NSc::TValue v) { - v["VAL"] = 1; - } - - static void ByRef(NSc::TValue& v) { - ByVal(v); - } - - static void ByRefAndModify(NSc::TValue& v) { - v["REF"] = 1; - ByVal(v); - } - - Y_UNIT_TEST(TestMove) { - using namespace NSc; - { - TValue v = TValue::FromJson("{}"); - ByRef(v); - UNIT_ASSERT_VALUES_EQUAL(v.ToJson(), "{\"VAL\":1}"); - } - { - TValue v = TValue::FromJson("{}"); - ByVal(v); - UNIT_ASSERT_VALUES_EQUAL(v.ToJson(), "{\"VAL\":1}"); - } - { - TValue v; - ByVal(v); - UNIT_ASSERT_VALUES_EQUAL(v.ToJson(), "{\"VAL\":1}"); - } - { - TValue v = TValue::FromJson("{}"); - ByRefAndModify(v); - UNIT_ASSERT_VALUES_EQUAL(v.ToJson(), "{\"REF\":1,\"VAL\":1}"); - } - { - TValue v = TValue::FromJson("{foo:bar}"); - TValue w(std::move(v)); - UNIT_ASSERT(v.IsNull()); + } + + Y_UNIT_TEST(TestLoopDetection) { + NSc::NImpl::GetTlsInstance<NSc::NImpl::TSelfLoopContext>().ReportingMode + = NSc::NImpl::TSelfLoopContext::EMode::Stderr; + + NSc::TValue x; + + x["a"]["x"] = x; + x["b"][0] = x; + + UNIT_ASSERT(x.IsSameOrAncestorOf(x)); + UNIT_ASSERT(x.IsSameOrAncestorOf(x.Get("a"))); + UNIT_ASSERT(x.Get("a").IsSameOrAncestorOf(x)); + + UNIT_ASSERT_VALUES_EQUAL(x.ToJson(), "{\"a\":{\"x\":null},\"b\":[null]}"); + + NSc::TValue y = x.Clone(); + + UNIT_ASSERT(y.Has("a")); + UNIT_ASSERT(y.Get("a").Has("x")); + UNIT_ASSERT(y.Get("a").Get("x").IsNull()); + UNIT_ASSERT(y.Get("a").Get("x").IsNull()); + + UNIT_ASSERT(y.Has("b")); + UNIT_ASSERT(y.Get("b").Has(0)); + UNIT_ASSERT(y.Get("b").Get(0).IsNull()); + + UNIT_ASSERT_VALUES_EQUAL(y.ToJson(), "{\"a\":{\"x\":null},\"b\":[null]}"); + + NSc::TValue z; + z.MergeUpdate(x); + + UNIT_ASSERT(z.Has("a")); + UNIT_ASSERT(z.Get("a").Has("x")); + UNIT_ASSERT(z.Get("a").Get("x").IsNull()); + + UNIT_ASSERT(z.Has("b")); + UNIT_ASSERT(z.Get("b").Has(0)); + UNIT_ASSERT(z.Get("b").Get(0).IsNull()); + + UNIT_ASSERT_VALUES_EQUAL(z.ToJson(), "{\"a\":{\"x\":null},\"b\":[null]}"); + + x["a"].Delete("x"); + x["b"].Delete(0); + + UNIT_ASSERT(x.IsSameOrAncestorOf(x)); + UNIT_ASSERT(x.IsSameOrAncestorOf(x.Get("a"))); + UNIT_ASSERT(!x.Get("a").IsSameOrAncestorOf(x)); + } + + Y_UNIT_TEST(TestLoopDetectionThrow) { + NSc::NImpl::GetTlsInstance<NSc::NImpl::TSelfLoopContext>().ReportingMode + = NSc::NImpl::TSelfLoopContext::EMode::Throw; + + { + NSc::TValue x; + x["a"]["x"] = x; + + UNIT_ASSERT(x.IsSameOrAncestorOf(x)); + UNIT_ASSERT(x.IsSameOrAncestorOf(x.Get("a"))); + UNIT_ASSERT(x.Get("a").IsSameOrAncestorOf(x)); + + UNIT_ASSERT_EXCEPTION(x.ToJson(), NSc::TSchemeException); + UNIT_ASSERT_EXCEPTION(x.Clone(), NSc::TSchemeException); + + NSc::TValue z; + UNIT_ASSERT_EXCEPTION(z.MergeUpdate(x), NSc::TSchemeException); + + UNIT_ASSERT_VALUES_EQUAL(z.ToJson(), "{\"a\":{\"x\":null}}"); + + x["a"].Delete("x"); + + UNIT_ASSERT(x.IsSameOrAncestorOf(x)); + UNIT_ASSERT(x.IsSameOrAncestorOf(x.Get("a"))); + UNIT_ASSERT(!x.Get("a").IsSameOrAncestorOf(x)); + + UNIT_ASSERT_VALUES_EQUAL(x.ToJson(), "{\"a\":{}}"); + } + + { + NSc::TValue x; + x["a"][0] = x; + + UNIT_ASSERT(x.IsSameOrAncestorOf(x)); + UNIT_ASSERT(x.IsSameOrAncestorOf(x.Get("a"))); + UNIT_ASSERT(x.Get("a").IsSameOrAncestorOf(x)); + + UNIT_ASSERT_EXCEPTION(x.ToJson(), NSc::TSchemeException); + UNIT_ASSERT_EXCEPTION(x.Clone(), NSc::TSchemeException); + + NSc::TValue z; + UNIT_ASSERT_EXCEPTION(z.MergeUpdate(x), NSc::TSchemeException); + + UNIT_ASSERT_VALUES_EQUAL(z.ToJson(), "{\"a\":[null]}"); + + x["a"].Delete(0); + + UNIT_ASSERT(x.IsSameOrAncestorOf(x)); + UNIT_ASSERT(x.IsSameOrAncestorOf(x.Get("a"))); + UNIT_ASSERT(!x.Get("a").IsSameOrAncestorOf(x)); + + UNIT_ASSERT_VALUES_EQUAL(x.ToJson(), "{\"a\":[]}"); + } + } + + Y_UNIT_TEST(TestIsSameOrAncestorOf) { + NSc::TValue x; + UNIT_ASSERT(x.IsSameOrAncestorOf(x)); + + x["a"] = NSc::Null(); + UNIT_ASSERT(x.IsSameOrAncestorOf(x.Get("a"))); + + NSc::TValue a = 1; + NSc::TValue b = 2; + NSc::TValue c = 3; + NSc::TValue d = 4; + + x["a"] = a; + x["b"] = b; + x["c"] = a; + UNIT_ASSERT(x.IsSameOrAncestorOf(a)); + UNIT_ASSERT(x.IsSameOrAncestorOf(b)); + UNIT_ASSERT(x.Get("a").IsSameOrAncestorOf(a)); + UNIT_ASSERT(x.Get("b").IsSameOrAncestorOf(b)); + UNIT_ASSERT(x.Get("c").IsSameOrAncestorOf(a)); + + UNIT_ASSERT(!x.Get("a").IsSameOrAncestorOf(b)); + UNIT_ASSERT(!x.Get("b").IsSameOrAncestorOf(a)); + + UNIT_ASSERT(!x.Get("a").Get(0).IsSameOrAncestorOf(a)); + + b.Push() = c; + b.Push() = d; + b.Push() = c; + + UNIT_ASSERT(x.Get("b").IsSameOrAncestorOf(b)); + UNIT_ASSERT(x.IsSameOrAncestorOf(c)); + UNIT_ASSERT(x.IsSameOrAncestorOf(d)); + UNIT_ASSERT(x.Get("b").Get(0).IsSameOrAncestorOf(c)); + UNIT_ASSERT(x.Get("b").Get(1).IsSameOrAncestorOf(d)); + UNIT_ASSERT(x.Get("b").Get(2).IsSameOrAncestorOf(c)); + + UNIT_ASSERT(b.Get(0).IsSameOrAncestorOf(b.Get(2))); + UNIT_ASSERT(b.Get(2).IsSameOrAncestorOf(b.Get(0))); + UNIT_ASSERT(b.Get(0).IsSameOrAncestorOf(c)); + UNIT_ASSERT(b.Get(1).IsSameOrAncestorOf(d)); + + UNIT_ASSERT(!b.Get(0).IsSameOrAncestorOf(d)); + UNIT_ASSERT(!b.Get(1).IsSameOrAncestorOf(c)); + } + + static void ByVal(NSc::TValue v) { + v["VAL"] = 1; + } + + static void ByRef(NSc::TValue& v) { + ByVal(v); + } + + static void ByRefAndModify(NSc::TValue& v) { + v["REF"] = 1; + ByVal(v); + } + + Y_UNIT_TEST(TestMove) { + using namespace NSc; + { + TValue v = TValue::FromJson("{}"); + ByRef(v); + UNIT_ASSERT_VALUES_EQUAL(v.ToJson(), "{\"VAL\":1}"); + } + { + TValue v = TValue::FromJson("{}"); + ByVal(v); + UNIT_ASSERT_VALUES_EQUAL(v.ToJson(), "{\"VAL\":1}"); + } + { + TValue v; + ByVal(v); + UNIT_ASSERT_VALUES_EQUAL(v.ToJson(), "{\"VAL\":1}"); + } + { + TValue v = TValue::FromJson("{}"); + ByRefAndModify(v); + UNIT_ASSERT_VALUES_EQUAL(v.ToJson(), "{\"REF\":1,\"VAL\":1}"); + } + { + TValue v = TValue::FromJson("{foo:bar}"); + TValue w(std::move(v)); + UNIT_ASSERT(v.IsNull()); v = static_cast<TValue&>(v); - UNIT_ASSERT(v.IsNull()); - UNIT_ASSERT_VALUES_EQUAL(w.Get("foo").GetString(), "bar"); - v = std::move(w); - UNIT_ASSERT_VALUES_EQUAL(v.Get("foo").GetString(), "bar"); - UNIT_ASSERT(w.IsNull()); - UNIT_ASSERT(w.Get("foo").IsNull()); // no crash here - w["foo"] = "baz"; // no crash here - UNIT_ASSERT(w.IsDict()); - UNIT_ASSERT_VALUES_EQUAL(w.Get("foo").GetString(), "baz"); - } + UNIT_ASSERT(v.IsNull()); + UNIT_ASSERT_VALUES_EQUAL(w.Get("foo").GetString(), "bar"); + v = std::move(w); + UNIT_ASSERT_VALUES_EQUAL(v.Get("foo").GetString(), "bar"); + UNIT_ASSERT(w.IsNull()); + UNIT_ASSERT(w.Get("foo").IsNull()); // no crash here + w["foo"] = "baz"; // no crash here + UNIT_ASSERT(w.IsDict()); + UNIT_ASSERT_VALUES_EQUAL(w.Get("foo").GetString(), "baz"); + } UNIT_ASSERT(NSc::TValue::DefaultValue().IsNull()); - } - + } + //SPI-25156 Y_UNIT_TEST(TestMoveNotCorruptingDefault) { using namespace NSc; @@ -779,72 +779,72 @@ Y_UNIT_TEST_SUITE(TSchemeTest) { UNIT_ASSERT(NSc::TValue::DefaultValue().IsNull()); } - Y_UNIT_TEST(TestCopyFrom) { - { + Y_UNIT_TEST(TestCopyFrom) { + { TString sa = "[1,2]"; - const NSc::TValue& va = NSc::TValue::FromJson(sa); - NSc::TValue vb = va; - vb.CopyFrom(va); - UNIT_ASSERT_VALUES_EQUAL(va.ToJson(), sa); - UNIT_ASSERT_VALUES_EQUAL(vb.ToJson(), sa); - } - { + const NSc::TValue& va = NSc::TValue::FromJson(sa); + NSc::TValue vb = va; + vb.CopyFrom(va); + UNIT_ASSERT_VALUES_EQUAL(va.ToJson(), sa); + UNIT_ASSERT_VALUES_EQUAL(vb.ToJson(), sa); + } + { TString sa = "[1,2]"; - NSc::TValue va = NSc::TValue::FromJson(sa); - NSc::TValue vb = va; - vb.CopyFrom(va); - UNIT_ASSERT_VALUES_EQUAL(va.ToJson(), sa); - UNIT_ASSERT_VALUES_EQUAL(vb.ToJson(), sa); - } - { + NSc::TValue va = NSc::TValue::FromJson(sa); + NSc::TValue vb = va; + vb.CopyFrom(va); + UNIT_ASSERT_VALUES_EQUAL(va.ToJson(), sa); + UNIT_ASSERT_VALUES_EQUAL(vb.ToJson(), sa); + } + { TString sa = "[1,2]"; - NSc::TValue va = NSc::TValue::FromJson(sa); - const NSc::TValue& vb = va; - va.CopyFrom(vb); - UNIT_ASSERT_VALUES_EQUAL(va.ToJson(), sa); - UNIT_ASSERT_VALUES_EQUAL(vb.ToJson(), sa); - } - { + NSc::TValue va = NSc::TValue::FromJson(sa); + const NSc::TValue& vb = va; + va.CopyFrom(vb); + UNIT_ASSERT_VALUES_EQUAL(va.ToJson(), sa); + UNIT_ASSERT_VALUES_EQUAL(vb.ToJson(), sa); + } + { TString sa = "[1,2]"; - NSc::TValue va = NSc::TValue::FromJson(sa); - NSc::TValue vb = va; - va.CopyFrom(vb); - UNIT_ASSERT_VALUES_EQUAL(va.ToJson(), sa); - UNIT_ASSERT_VALUES_EQUAL(vb.ToJson(), sa); - } - { - NSc::TValue va = NSc::TValue::FromJson("{\"x\":\"ab\",\"y\":{\"p\":\"cd\",\"q\":\"ef\"}}"); - NSc::TValue vb = va.Get("y"); - va.CopyFrom(vb); + NSc::TValue va = NSc::TValue::FromJson(sa); + NSc::TValue vb = va; + va.CopyFrom(vb); + UNIT_ASSERT_VALUES_EQUAL(va.ToJson(), sa); + UNIT_ASSERT_VALUES_EQUAL(vb.ToJson(), sa); + } + { + NSc::TValue va = NSc::TValue::FromJson("{\"x\":\"ab\",\"y\":{\"p\":\"cd\",\"q\":\"ef\"}}"); + NSc::TValue vb = va.Get("y"); + va.CopyFrom(vb); TString sa = "{\"p\":\"cd\",\"q\":\"ef\"}"; - UNIT_ASSERT_VALUES_EQUAL(va.ToJson(), sa); - UNIT_ASSERT_VALUES_EQUAL(vb.ToJson(), sa); - } - { - NSc::TValue va = NSc::TValue::FromJson("{\"x\":\"ab\",\"y\":{\"p\":\"cd\",\"q\":\"ef\"}}"); - const NSc::TValue& vb = va.Get("y"); - va.CopyFrom(vb); + UNIT_ASSERT_VALUES_EQUAL(va.ToJson(), sa); + UNIT_ASSERT_VALUES_EQUAL(vb.ToJson(), sa); + } + { + NSc::TValue va = NSc::TValue::FromJson("{\"x\":\"ab\",\"y\":{\"p\":\"cd\",\"q\":\"ef\"}}"); + const NSc::TValue& vb = va.Get("y"); + va.CopyFrom(vb); TString sa = "{\"p\":\"cd\",\"q\":\"ef\"}"; - UNIT_ASSERT_VALUES_EQUAL(va.ToJson(), sa); - } - { - NSc::TValue va = NSc::TValue::FromJson("{\"x\":\"ab\",\"y\":{\"p\":\"cd\",\"q\":\"ef\"}}"); - NSc::TValue vb = va.Get("y"); - va = vb; + UNIT_ASSERT_VALUES_EQUAL(va.ToJson(), sa); + } + { + NSc::TValue va = NSc::TValue::FromJson("{\"x\":\"ab\",\"y\":{\"p\":\"cd\",\"q\":\"ef\"}}"); + NSc::TValue vb = va.Get("y"); + va = vb; TString sa = "{\"p\":\"cd\",\"q\":\"ef\"}"; - UNIT_ASSERT_VALUES_EQUAL(va.ToJson(), sa); - UNIT_ASSERT_VALUES_EQUAL(vb.ToJson(), sa); - } - { - NSc::TValue va = NSc::TValue::FromJson("{\"x\":\"ab\",\"y\":{\"p\":\"cd\",\"q\":\"ef\"}}"); - const NSc::TValue& vb = va.Get("y"); - va = vb; + UNIT_ASSERT_VALUES_EQUAL(va.ToJson(), sa); + UNIT_ASSERT_VALUES_EQUAL(vb.ToJson(), sa); + } + { + NSc::TValue va = NSc::TValue::FromJson("{\"x\":\"ab\",\"y\":{\"p\":\"cd\",\"q\":\"ef\"}}"); + const NSc::TValue& vb = va.Get("y"); + va = vb; TString sa = "{\"p\":\"cd\",\"q\":\"ef\"}"; - UNIT_ASSERT_VALUES_EQUAL(va.ToJson(), sa); - UNIT_ASSERT_VALUES_EQUAL(vb.ToJson(), sa); - } - } - + UNIT_ASSERT_VALUES_EQUAL(va.ToJson(), sa); + UNIT_ASSERT_VALUES_EQUAL(vb.ToJson(), sa); + } + } + Y_UNIT_TEST(TestCopyingDictIntoSelf) { //Found by fuzzing NSc::TValue a; NSc::TValue b = a.GetOrAdd("aa"); @@ -862,7 +862,7 @@ Y_UNIT_TEST_SUITE(TSchemeTest) { UNIT_ASSERT_VALUES_EQUAL(a, NSc::TValue::FromJsonThrow("{\"aa\": {\"aa\": null}}")); } - Y_UNIT_TEST(TestGetNoAdd) { + Y_UNIT_TEST(TestGetNoAdd) { NSc::TValue v = NSc::NUt::AssertFromJson("{a:[null,-1,2,3.4],b:3,c:{d:5}}"); UNIT_ASSERT(v.GetNoAdd("a") != nullptr); UNIT_ASSERT(v.GetNoAdd("b") != nullptr); @@ -876,4 +876,4 @@ Y_UNIT_TEST_SUITE(TSchemeTest) { const NSc::TValue expectedResult = NSc::NUt::AssertFromJson("{a:[null,-1,2,3.4],b:3,c:{d:5,e:{f:42}}}"); UNIT_ASSERT_VALUES_EQUAL(v, expectedResult); } -}; +}; diff --git a/library/cpp/scheme/tests/ut/ya.make b/library/cpp/scheme/tests/ut/ya.make index 738cb0e7ad..9f54791414 100644 --- a/library/cpp/scheme/tests/ut/ya.make +++ b/library/cpp/scheme/tests/ut/ya.make @@ -1,7 +1,7 @@ -UNITTEST() +UNITTEST() + +OWNER(velavokr) -OWNER(velavokr) - PEERDIR( library/cpp/protobuf/util library/cpp/scheme/tests/fuzz_ops/lib @@ -11,12 +11,12 @@ PEERDIR( ) SRCS( - fuzz_ops_found_bugs_ut.cpp - scheme_cast_ut.cpp - scheme_json_ut.cpp - scheme_merge_ut.cpp - scheme_path_ut.cpp - scheme_proto_ut.cpp + fuzz_ops_found_bugs_ut.cpp + scheme_cast_ut.cpp + scheme_json_ut.cpp + scheme_merge_ut.cpp + scheme_path_ut.cpp + scheme_proto_ut.cpp scheme_ut.cpp scheme_ut.proto ) diff --git a/library/cpp/scheme/tests/ya.make b/library/cpp/scheme/tests/ya.make index f47eaf4574..741cc9a2da 100644 --- a/library/cpp/scheme/tests/ya.make +++ b/library/cpp/scheme/tests/ya.make @@ -1,13 +1,13 @@ -OWNER( - g:blender - g:middle - g:upper - velavokr -) - -RECURSE( - fuzz_json - fuzz_ops - fuzz_ops/ut - ut -) +OWNER( + g:blender + g:middle + g:upper + velavokr +) + +RECURSE( + fuzz_json + fuzz_ops + fuzz_ops/ut + ut +) diff --git a/library/cpp/scheme/ut_utils/scheme_ut_utils.cpp b/library/cpp/scheme/ut_utils/scheme_ut_utils.cpp index b520eceab5..0bbdab10e8 100644 --- a/library/cpp/scheme/ut_utils/scheme_ut_utils.cpp +++ b/library/cpp/scheme/ut_utils/scheme_ut_utils.cpp @@ -1,44 +1,44 @@ -#include "scheme_ut_utils.h" - +#include "scheme_ut_utils.h" + #include <library/cpp/colorizer/colors.h> - -#include <util/stream/str.h> - -namespace NSc { - namespace NUt { - NSc::TValue AssertFromJson(TStringBuf val) { - try { - return TValue::FromJsonThrow(val); - } catch (const TSchemeParseException& e) { - TStringStream s; - NColorizer::TColors colors; + +#include <util/stream/str.h> + +namespace NSc { + namespace NUt { + NSc::TValue AssertFromJson(TStringBuf val) { + try { + return TValue::FromJsonThrow(val); + } catch (const TSchemeParseException& e) { + TStringStream s; + NColorizer::TColors colors; s << "\n" << colors.YellowColor() << "Reason:" << colors.OldColor() << "\n" << e.Reason; s << "\n" << colors.YellowColor() << "Where:" << colors.OldColor() << "\n" << val.SubStr(0, e.Offset) << colors.RedColor() << val.SubStr(e.Offset) << colors.OldColor() << "\n"; - UNIT_FAIL_IMPL("could not parse json", s.Str()); - return NSc::Null(); - } catch (const yexception& e) { - TStringStream s; + UNIT_FAIL_IMPL("could not parse json", s.Str()); + return NSc::Null(); + } catch (const yexception& e) { + TStringStream s; s << '\n' << val; - UNIT_FAIL_IMPL("could not parse json", s.Str()); - return NSc::Null(); - } - } - - void AssertScheme(const TValue& expected, const TValue& actual) { - UNIT_ASSERT_JSON_EQ_JSON(actual, expected); - } - - void AssertSchemeJson(TStringBuf expected, const NSc::TValue& actual) { - UNIT_ASSERT_JSON_EQ_JSON(actual, expected); - } - - void AssertJsonJson(TStringBuf expected, TStringBuf actual) { - UNIT_ASSERT_JSON_EQ_JSON(actual, expected); - } - } -} + UNIT_FAIL_IMPL("could not parse json", s.Str()); + return NSc::Null(); + } + } + + void AssertScheme(const TValue& expected, const TValue& actual) { + UNIT_ASSERT_JSON_EQ_JSON(actual, expected); + } + + void AssertSchemeJson(TStringBuf expected, const NSc::TValue& actual) { + UNIT_ASSERT_JSON_EQ_JSON(actual, expected); + } + + void AssertJsonJson(TStringBuf expected, TStringBuf actual) { + UNIT_ASSERT_JSON_EQ_JSON(actual, expected); + } + } +} diff --git a/library/cpp/scheme/ut_utils/scheme_ut_utils.h b/library/cpp/scheme/ut_utils/scheme_ut_utils.h index fe04641370..eb3ea15b2a 100644 --- a/library/cpp/scheme/ut_utils/scheme_ut_utils.h +++ b/library/cpp/scheme/ut_utils/scheme_ut_utils.h @@ -1,29 +1,29 @@ -#pragma once - +#pragma once + #include <library/cpp/json/json_prettifier.h> #include <library/cpp/scheme/scheme.h> #include <library/cpp/json/json_value.h> #include <library/cpp/json/json_writer.h> #include <library/cpp/testing/unittest/registar.h> #include <util/string/cast.h> - -namespace NSc { - namespace NUt { - TValue AssertFromJson(TStringBuf json); - + +namespace NSc { + namespace NUt { + TValue AssertFromJson(TStringBuf json); + inline TString NormalizeJson(const NSc::TValue& sc) { - return sc.ToJson(true); - } - + return sc.ToJson(true); + } + inline TString NormalizeJson(const NJson::TJsonValue& sc) { - return NJson::WriteJson(sc, false, true, false); - } - - template <class TStr> + return NJson::WriteJson(sc, false, true, false); + } + + template <class TStr> inline TString NormalizeJson(const TStr& val) { - return AssertFromJson(val).ToJson(true); - } - + return AssertFromJson(val).ToJson(true); + } + #define UNIT_ASSERT_JSON_EQ_JSON_C(A, B, c) \ do { \ const TString _a = NSc::NUt::NormalizeJson(A); \ @@ -34,22 +34,22 @@ namespace NSc { Sprintf("%s\n!=\n%s\n%s\n%s", _a.data(), _b.data(), \ ::NUnitTest::ColoredDiff(NJson::PrettifyJson(_a), NJson::PrettifyJson(_b), " \t\n,:\"{}[]").data(), ToString(c).data())); \ } \ - } while (false) - -#define UNIT_ASSERT_JSON_EQ_JSON(A, B) UNIT_ASSERT_JSON_EQ_JSON_C(A, B, "") - + } while (false) + +#define UNIT_ASSERT_JSON_EQ_JSON(A, B) UNIT_ASSERT_JSON_EQ_JSON_C(A, B, "") + inline TString DumpJson(const TValue& json) { - return NJson::CompactifyJson(json.ToJson(true), true, true); - } - - // deprecated + return NJson::CompactifyJson(json.ToJson(true), true, true); + } + + // deprecated inline TString DumpJsonVS(const TValue& expected, const TValue& fact) { - return DumpJson(expected) + "(expected) != (fact)" + DumpJson(fact); - } - - void AssertScheme(const TValue& expected, const TValue& real); - void AssertSchemeJson(TStringBuf expected, const TValue& real); - void AssertJsonJson(TStringBuf expected, TStringBuf real); - - } -} + return DumpJson(expected) + "(expected) != (fact)" + DumpJson(fact); + } + + void AssertScheme(const TValue& expected, const TValue& real); + void AssertSchemeJson(TStringBuf expected, const TValue& real); + void AssertJsonJson(TStringBuf expected, TStringBuf real); + + } +} diff --git a/library/cpp/scheme/ut_utils/ya.make b/library/cpp/scheme/ut_utils/ya.make index cfa7387a94..7661262e1b 100644 --- a/library/cpp/scheme/ut_utils/ya.make +++ b/library/cpp/scheme/ut_utils/ya.make @@ -1,16 +1,16 @@ -LIBRARY() - -OWNER(velavokr) - -SRCS( - scheme_ut_utils.cpp -) - -PEERDIR( +LIBRARY() + +OWNER(velavokr) + +SRCS( + scheme_ut_utils.cpp +) + +PEERDIR( library/cpp/colorizer library/cpp/json library/cpp/scheme library/cpp/testing/unittest -) - -END() +) + +END() diff --git a/library/cpp/scheme/ya.make b/library/cpp/scheme/ya.make index ff4c491c21..bac08ba5a4 100644 --- a/library/cpp/scheme/ya.make +++ b/library/cpp/scheme/ya.make @@ -1,25 +1,25 @@ -LIBRARY() - -OWNER(velavokr) - -SRCS( - scheme.cpp +LIBRARY() + +OWNER(velavokr) + +SRCS( + scheme.cpp scheme_cast.h scimpl.h scimpl_defs.h - scimpl_private.cpp + scimpl_private.cpp scimpl_protobuf.cpp - scimpl_select.rl6 - scimpl_json_read.cpp - scimpl_json_write.cpp -) - -PEERDIR( + scimpl_select.rl6 + scimpl_json_read.cpp + scimpl_json_write.cpp +) + +PEERDIR( contrib/libs/protobuf library/cpp/json library/cpp/string_utils/relaxed_escaper -) - +) + GENERATE_ENUM_SERIALIZATION(scheme.h) - + END() diff --git a/library/cpp/streams/bzip2/bzip2.h b/library/cpp/streams/bzip2/bzip2.h index 7ac43ffa66..2322277ef6 100644 --- a/library/cpp/streams/bzip2/bzip2.h +++ b/library/cpp/streams/bzip2/bzip2.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <util/stream/input.h> #include <util/stream/output.h> diff --git a/library/cpp/streams/lz/lz.cpp b/library/cpp/streams/lz/lz.cpp index 484fd3cf7c..b65bb3ed96 100644 --- a/library/cpp/streams/lz/lz.cpp +++ b/library/cpp/streams/lz/lz.cpp @@ -460,78 +460,78 @@ DEF_COMPRESSOR(TLzfCompress, TFastLZ) DEF_DECOMPRESSOR(TLzfDecompress, TFastLZ) /* - * LZ4 - */ -class TLZ4 { + * LZ4 + */ +class TLZ4 { public: static const char signature[]; - + static inline size_t Hint(size_t len) noexcept { return Max<size_t>((size_t)(len * 1.06), 100); } - + inline size_t Compress(const char* data, size_t len, char* ptr, size_t dstMaxSize) { return LZ4_compress_default(data, ptr, len, dstMaxSize); } - + inline size_t Decompress(const char* data, size_t len, char* ptr, size_t max) { int res = LZ4_decompress_safe(data, ptr, len, max); if (res < 0) ythrow TDecompressorError(); return res; } - + inline void InitFromStream(IInputStream*) const noexcept { } - + static inline bool SaveIncompressibleChunks() noexcept { return false; } -}; - -const char TLZ4::signature[] = "LZ.4"; - -DEF_COMPRESSOR(TLz4Compress, TLZ4) -DEF_DECOMPRESSOR(TLz4Decompress, TLZ4) - -/* - * Snappy - */ -class TSnappy { +}; + +const char TLZ4::signature[] = "LZ.4"; + +DEF_COMPRESSOR(TLz4Compress, TLZ4) +DEF_DECOMPRESSOR(TLz4Decompress, TLZ4) + +/* + * Snappy + */ +class TSnappy { public: static const char signature[]; - + static inline size_t Hint(size_t len) noexcept { return Max<size_t>(snappy::MaxCompressedLength(len), 100); } - + inline size_t Compress(const char* data, size_t len, char* ptr, size_t /*dstMaxSize*/) { size_t reslen = 0; snappy::RawCompress(data, len, ptr, &reslen); return reslen; } - + inline size_t Decompress(const char* data, size_t len, char* ptr, size_t) { size_t srclen = 0; if (!snappy::GetUncompressedLength(data, len, &srclen) || !snappy::RawUncompress(data, len, ptr)) ythrow TDecompressorError(); return srclen; } - + inline void InitFromStream(IInputStream*) const noexcept { } - + static inline bool SaveIncompressibleChunks() noexcept { return false; } -}; - -const char TSnappy::signature[] = "Snap"; - -DEF_COMPRESSOR(TSnappyCompress, TSnappy) -DEF_DECOMPRESSOR(TSnappyDecompress, TSnappy) - -/* +}; + +const char TSnappy::signature[] = "Snap"; + +DEF_COMPRESSOR(TSnappyCompress, TSnappy) +DEF_DECOMPRESSOR(TSnappyDecompress, TSnappy) + +/* * QuickLZ */ class TQuickLZBase { diff --git a/library/cpp/streams/lz/lz.h b/library/cpp/streams/lz/lz.h index 9e97c3389b..3a2eaad88b 100644 --- a/library/cpp/streams/lz/lz.h +++ b/library/cpp/streams/lz/lz.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <util/stream/output.h> #include <util/stream/input.h> @@ -23,7 +23,7 @@ struct TDecompressorError: public yexception { /** * @addtogroup Streams_Archs * @{ - */ + */ /** * Lz4 compressing stream. @@ -34,17 +34,17 @@ class TLz4Compress: public IOutputStream { public: TLz4Compress(IOutputStream* slave, ui16 maxBlockSize = 1 << 15); ~TLz4Compress() override; - + private: void DoWrite(const void* buf, size_t len) override; void DoFlush() override; void DoFinish() override; - + private: class TImpl; THolder<TImpl> Impl_; -}; - +}; + /** * Lz4 decompressing stream. * @@ -54,35 +54,35 @@ class TLz4Decompress: public IInputStream { public: TLz4Decompress(IInputStream* slave); ~TLz4Decompress() override; - + private: size_t DoRead(void* buf, size_t len) override; - + private: class TImpl; THolder<TImpl> Impl_; -}; - +}; + /** * Snappy compressing stream. * * @see http://code.google.com/p/snappy/ - */ + */ class TSnappyCompress: public IOutputStream { public: TSnappyCompress(IOutputStream* slave, ui16 maxBlockSize = 1 << 15); ~TSnappyCompress() override; - + private: void DoWrite(const void* buf, size_t len) override; void DoFlush() override; void DoFinish() override; - + private: class TImpl; THolder<TImpl> Impl_; -}; - +}; + /** * Snappy decompressing stream. * @@ -92,15 +92,15 @@ class TSnappyDecompress: public IInputStream { public: TSnappyDecompress(IInputStream* slave); ~TSnappyDecompress() override; - + private: size_t DoRead(void* buf, size_t len) override; - + private: class TImpl; THolder<TImpl> Impl_; -}; - +}; + /** * MiniLZO compressing stream. */ diff --git a/library/cpp/streams/lz/lz_ut.cpp b/library/cpp/streams/lz/lz_ut.cpp index 1b84978710..6876f070fc 100644 --- a/library/cpp/streams/lz/lz_ut.cpp +++ b/library/cpp/streams/lz/lz_ut.cpp @@ -203,7 +203,7 @@ Y_UNIT_TEST_SUITE(TLzTest) { TestCompress<TLzqCompress>(); TestDecompress<TLzqDecompress>(); } - + Y_UNIT_TEST(TestLzq151_1) { TestCompress<TLzqCompressX<TLzqCompress::V_1_51, 1, TLzqCompress::M_0>>(); TestDecompress<TLzqDecompress>(); @@ -235,14 +235,14 @@ Y_UNIT_TEST_SUITE(TLzTest) { } Y_UNIT_TEST(TestLz4) { - TestCompress<TLz4Compress>(); - TestDecompress<TLz4Decompress>(); - } - + TestCompress<TLz4Compress>(); + TestDecompress<TLz4Decompress>(); + } + Y_UNIT_TEST(TestSnappy) { - TestCompress<TSnappyCompress>(); - TestDecompress<TSnappyDecompress>(); - } + TestCompress<TSnappyCompress>(); + TestDecompress<TSnappyDecompress>(); + } Y_UNIT_TEST(TestGeneric) { TestMixedDecompress<TLzoCompress>(); diff --git a/library/cpp/streams/lzma/lzma.h b/library/cpp/streams/lzma/lzma.h index 79f891b4ab..ca1e06e9ef 100644 --- a/library/cpp/streams/lzma/lzma.h +++ b/library/cpp/streams/lzma/lzma.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <util/stream/input.h> #include <util/stream/output.h> diff --git a/library/cpp/streams/zc_memory_input/zc_memory_input.h b/library/cpp/streams/zc_memory_input/zc_memory_input.h index b0beed9e91..c939d8e426 100644 --- a/library/cpp/streams/zc_memory_input/zc_memory_input.h +++ b/library/cpp/streams/zc_memory_input/zc_memory_input.h @@ -31,14 +31,14 @@ public: } template <class T> - Y_FORCE_INLINE T LoadPOD() { + Y_FORCE_INLINE T LoadPOD() { const char* buf = nullptr; if (!ReadFixed(buf, sizeof(T))) ythrow yexception() << "TZCMemoryInput::LoadPOD failed: not enough data (" << Avail() << " of " << sizeof(T) << " bytes)"; - T res; - memcpy(&res, buf, sizeof(T)); - return res; + T res; + memcpy(&res, buf, sizeof(T)); + return res; } template <class T> diff --git a/library/cpp/string_utils/base64/base64.h b/library/cpp/string_utils/base64/base64.h index ba7e90b144..f778a6425a 100644 --- a/library/cpp/string_utils/base64/base64.h +++ b/library/cpp/string_utils/base64/base64.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <util/system/defaults.h> #include <util/generic/strbuf.h> diff --git a/library/cpp/string_utils/base64/fuzz/uneven/main.cpp b/library/cpp/string_utils/base64/fuzz/uneven/main.cpp index 9e1e934e09..915e81a7e5 100644 --- a/library/cpp/string_utils/base64/fuzz/uneven/main.cpp +++ b/library/cpp/string_utils/base64/fuzz/uneven/main.cpp @@ -1,10 +1,10 @@ -#include <library/cpp/string_utils/base64/base64.h> - -#include <util/system/types.h> -#include <util/system/yassert.h> - -extern "C" int LLVMFuzzerTestOneInput(const ui8* data, size_t size) { - const TStringBuf example{reinterpret_cast<const char*>(data), size}; - Y_UNUSED(Base64DecodeUneven(example)); - return 0; -} +#include <library/cpp/string_utils/base64/base64.h> + +#include <util/system/types.h> +#include <util/system/yassert.h> + +extern "C" int LLVMFuzzerTestOneInput(const ui8* data, size_t size) { + const TStringBuf example{reinterpret_cast<const char*>(data), size}; + Y_UNUSED(Base64DecodeUneven(example)); + return 0; +} diff --git a/library/cpp/string_utils/base64/fuzz/uneven/ya.make b/library/cpp/string_utils/base64/fuzz/uneven/ya.make index 5ebcb21ecf..18cb18ef52 100644 --- a/library/cpp/string_utils/base64/fuzz/uneven/ya.make +++ b/library/cpp/string_utils/base64/fuzz/uneven/ya.make @@ -1,15 +1,15 @@ -FUZZ() - -OWNER( - g:util -) - -SRCS( - main.cpp -) - -PEERDIR( - library/cpp/string_utils/base64 -) - -END() +FUZZ() + +OWNER( + g:util +) + +SRCS( + main.cpp +) + +PEERDIR( + library/cpp/string_utils/base64 +) + +END() diff --git a/library/cpp/string_utils/base64/fuzz/ya.make b/library/cpp/string_utils/base64/fuzz/ya.make index b4e427326d..bef82061c4 100644 --- a/library/cpp/string_utils/base64/fuzz/ya.make +++ b/library/cpp/string_utils/base64/fuzz/ya.make @@ -6,5 +6,5 @@ OWNER( RECURSE( generic lib - uneven + uneven ) diff --git a/library/cpp/string_utils/levenshtein_diff/levenshtein_diff.h b/library/cpp/string_utils/levenshtein_diff/levenshtein_diff.h index 0dd8e49251..8a240bfed8 100644 --- a/library/cpp/string_utils/levenshtein_diff/levenshtein_diff.h +++ b/library/cpp/string_utils/levenshtein_diff/levenshtein_diff.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <util/draft/matrix.h> #include <util/generic/algorithm.h> diff --git a/library/cpp/string_utils/relaxed_escaper/relaxed_escaper.h b/library/cpp/string_utils/relaxed_escaper/relaxed_escaper.h index 97d2a48fd0..d7ea7c1259 100644 --- a/library/cpp/string_utils/relaxed_escaper/relaxed_escaper.h +++ b/library/cpp/string_utils/relaxed_escaper/relaxed_escaper.h @@ -1,14 +1,14 @@ -#pragma once - -#include <util/stream/output.h> -#include <util/string/escape.h> +#pragma once + +#include <util/stream/output.h> +#include <util/string/escape.h> #include <util/memory/tempbuf.h> #include <util/generic/strbuf.h> - -namespace NEscJ { + +namespace NEscJ { // almost copypaste from util/string/escape.h // todo: move there (note difference in IsPrintable and handling of string) - + inline char HexDigit(char value) { if (value < 10) return '0' + value; @@ -17,32 +17,32 @@ namespace NEscJ { } inline char OctDigit(char value) { - return '0' + value; + return '0' + value; } - + inline bool IsUTF8(ui8 c) { return c < 0xf5 && c != 0xC0 && c != 0xC1; } - + inline bool IsControl(ui8 c) { return c < 0x20 || c == 0x7f; } - + inline bool IsPrintable(ui8 c) { return IsUTF8(c) && !IsControl(c); } - + inline bool IsHexDigit(ui8 c) { return (c >= '0' && c <= '9') || (c >= 'A' && c <= 'F') || (c >= 'a' && c <= 'f'); } - + inline bool IsOctDigit(ui8 c) { return c >= '0' && c <= '7'; } - + struct TEscapeUtil { static const size_t ESCAPE_C_BUFFER_SIZE = 6; - + template <bool asunicode> static inline size_t EscapeJ(ui8 c, ui8 next, char r[ESCAPE_C_BUFFER_SIZE], TStringBuf safe, TStringBuf unsafe) { // (1) Printable characters go as-is, except backslash and double quote. @@ -109,88 +109,88 @@ namespace NEscJ { return 4; } } - + static inline size_t EscapeJ(ui8 c, ui8 next, char r[ESCAPE_C_BUFFER_SIZE], TStringBuf safe, TStringBuf unsafe) { return EscapeJ<false>(c, next, r, safe, unsafe); - } + } }; inline size_t SuggestBuffer(size_t len) { return len * TEscapeUtil::ESCAPE_C_BUFFER_SIZE; } - + template <bool tounicode> inline size_t EscapeJ(const char* str, size_t len, char* out, TStringBuf safe = TStringBuf(), TStringBuf unsafe = TStringBuf()) { char* out0 = out; char buffer[TEscapeUtil::ESCAPE_C_BUFFER_SIZE]; - + size_t i, j; for (i = 0, j = 0; i < len; ++i) { size_t rlen = TEscapeUtil::EscapeJ<tounicode>(str[i], (i + 1 < len ? str[i + 1] : 0), buffer, safe, unsafe); - + if (rlen > 1) { strncpy(out, str + j, i - j); out += i - j; j = i + 1; - + strncpy(out, buffer, rlen); out += rlen; } } - + if (j > 0) { strncpy(out, str + j, len - j); out += len - j; } else { strncpy(out, str, len); out += len; - } - + } + return out - out0; - } - + } + template <bool quote, bool tounicode> inline void EscapeJ(TStringBuf in, IOutputStream& out, TStringBuf safe = TStringBuf(), TStringBuf unsafe = TStringBuf()) { TTempBuf b(SuggestBuffer(in.size()) + 2); - + if (quote) b.Append("\"", 1); - + b.Proceed(EscapeJ<tounicode>(in.data(), in.size(), b.Current(), safe, unsafe)); - + if (quote) b.Append("\"", 1); - + out.Write(b.Data(), b.Filled()); } - + template <bool quote, bool tounicode> inline void EscapeJ(TStringBuf in, TString& out, TStringBuf safe = TStringBuf(), TStringBuf unsafe = TStringBuf()) { TTempBuf b(SuggestBuffer(in.size()) + 2); - + if (quote) b.Append("\"", 1); - + b.Proceed(EscapeJ<tounicode>(in.data(), in.size(), b.Current(), safe, unsafe)); - + if (quote) b.Append("\"", 1); - + out.append(b.Data(), b.Filled()); } - + template <bool quote, bool tounicode> inline TString EscapeJ(TStringBuf in, TStringBuf safe = TStringBuf(), TStringBuf unsafe = TStringBuf()) { TString s; EscapeJ<quote, tounicode>(in, s, safe, unsafe); return s; } - + // If the template parameter "tounicode" is ommited, then use the default value false inline size_t EscapeJ(const char* str, size_t len, char* out, TStringBuf safe = TStringBuf(), TStringBuf unsafe = TStringBuf()) { return EscapeJ<false>(str, len, out, safe, unsafe); } - + template <bool quote> inline void EscapeJ(TStringBuf in, IOutputStream& out, TStringBuf safe = TStringBuf(), TStringBuf unsafe = TStringBuf()) { EscapeJ<quote, false>(in, out, safe, unsafe); diff --git a/library/cpp/string_utils/relaxed_escaper/relaxed_escaper_ut.cpp b/library/cpp/string_utils/relaxed_escaper/relaxed_escaper_ut.cpp index 9132fbad91..768555ea3a 100644 --- a/library/cpp/string_utils/relaxed_escaper/relaxed_escaper_ut.cpp +++ b/library/cpp/string_utils/relaxed_escaper/relaxed_escaper_ut.cpp @@ -1,57 +1,57 @@ -#include "relaxed_escaper.h" - +#include "relaxed_escaper.h" + #include <library/cpp/testing/unittest/registar.h> - + #define RESC_FIXED_STR(s) TStringBuf(s, sizeof(s) - 1) static const TStringBuf CommonTestData[] = { - // Should be valid UTF-8. - RESC_FIXED_STR("http://ya.ru/"), RESC_FIXED_STR("http://ya.ru/"), - RESC_FIXED_STR("http://ya.ru/\\x17\\n"), RESC_FIXED_STR("http://ya.ru/\x17\n"), - - RESC_FIXED_STR("http://ya.ru/\\0"), RESC_FIXED_STR("http://ya.ru/\0"), - RESC_FIXED_STR("http://ya.ru/\\0\\0"), RESC_FIXED_STR("http://ya.ru/\0\0"), + // Should be valid UTF-8. + RESC_FIXED_STR("http://ya.ru/"), RESC_FIXED_STR("http://ya.ru/"), + RESC_FIXED_STR("http://ya.ru/\\x17\\n"), RESC_FIXED_STR("http://ya.ru/\x17\n"), + + RESC_FIXED_STR("http://ya.ru/\\0"), RESC_FIXED_STR("http://ya.ru/\0"), + RESC_FIXED_STR("http://ya.ru/\\0\\0"), RESC_FIXED_STR("http://ya.ru/\0\0"), RESC_FIXED_STR("http://ya.ru/\\0\\0000"), RESC_FIXED_STR("http://ya.ru/\0\0" "0"), RESC_FIXED_STR("http://ya.ru/\\0\\0001"), RESC_FIXED_STR("http://ya.ru/\0\x00" "1"), - + RESC_FIXED_STR("\\2\\4\\00678"), RESC_FIXED_STR("\2\4\6" "78"), - RESC_FIXED_STR("\\2\\4\\689"), RESC_FIXED_STR("\2\4\689"), - - RESC_FIXED_STR("\\\"Hello\\\", Alice said."), RESC_FIXED_STR("\"Hello\", Alice said."), - RESC_FIXED_STR("Slash\\\\dash!"), RESC_FIXED_STR("Slash\\dash!"), - RESC_FIXED_STR("There\\nare\\r\\nnewlines."), RESC_FIXED_STR("There\nare\r\nnewlines."), + RESC_FIXED_STR("\\2\\4\\689"), RESC_FIXED_STR("\2\4\689"), + + RESC_FIXED_STR("\\\"Hello\\\", Alice said."), RESC_FIXED_STR("\"Hello\", Alice said."), + RESC_FIXED_STR("Slash\\\\dash!"), RESC_FIXED_STR("Slash\\dash!"), + RESC_FIXED_STR("There\\nare\\r\\nnewlines."), RESC_FIXED_STR("There\nare\r\nnewlines."), RESC_FIXED_STR("There\\tare\\ttabs."), RESC_FIXED_STR("There\tare\ttabs.")}; -#undef RESC_FIXED_STR - +#undef RESC_FIXED_STR + Y_UNIT_TEST_SUITE(TRelaxedEscaperTest) { Y_UNIT_TEST(TestEscaper) { - using namespace NEscJ; + using namespace NEscJ; for (size_t i = 0; i < Y_ARRAY_SIZE(CommonTestData); i += 2) { TString expected(CommonTestData[i].data(), CommonTestData[i].size()); TString source(CommonTestData[i + 1].data(), CommonTestData[i + 1].size()); TString actual(EscapeJ<false>(source)); TString actual2(UnescapeC(expected)); - - UNIT_ASSERT_VALUES_EQUAL(expected, actual); - UNIT_ASSERT_VALUES_EQUAL(source, actual2); - } - - UNIT_ASSERT_VALUES_EQUAL("http://ya.ru/\\x17\\n\xAB", EscapeJ<false>("http://ya.ru/\x17\n\xab")); + + UNIT_ASSERT_VALUES_EQUAL(expected, actual); + UNIT_ASSERT_VALUES_EQUAL(source, actual2); + } + + UNIT_ASSERT_VALUES_EQUAL("http://ya.ru/\\x17\\n\xAB", EscapeJ<false>("http://ya.ru/\x17\n\xab")); TString s = EscapeJ<false, true>("http://ya.ru/\x17\n\xab\xff"); - UNIT_ASSERT_VALUES_EQUAL("http://ya.ru/\\u0017\\n\xAB\\xFF", s); - UNIT_ASSERT_VALUES_EQUAL("http://ya.ru/\\x17\n\xAB", EscapeJ<false>("http://ya.ru/\x17\n\xab", "\n")); - UNIT_ASSERT_VALUES_EQUAL("http:\\x2F\\x2Fya.ru\\x2F\\x17\n\xAB'", EscapeJ<false>("http://ya.ru/\x17\n\xab'", "\n'", "/")); - UNIT_ASSERT_VALUES_EQUAL("http://ya.ru/\x17\n\xab", UnescapeC("http:\\x2F\\x2Fya.ru\\x2F\\x17\n\xAB")); - UNIT_ASSERT_VALUES_EQUAL("http://ya.ru/\x17\n\xab", UnescapeC("http://ya.ru/\\x17\\n\xAB")); - UNIT_ASSERT_VALUES_EQUAL("h", EscapeJ<false>("h")); - UNIT_ASSERT_VALUES_EQUAL("\"h\"", EscapeJ<true>("h")); - UNIT_ASSERT_VALUES_EQUAL("h", UnescapeC("h")); - UNIT_ASSERT_VALUES_EQUAL("\\xFF", EscapeJ<false>("\xFF")); - UNIT_ASSERT_VALUES_EQUAL("\"\\xFF\"", EscapeJ<true>("\xFF")); - UNIT_ASSERT_VALUES_EQUAL("\xFF", UnescapeC("\\xFF")); - + UNIT_ASSERT_VALUES_EQUAL("http://ya.ru/\\u0017\\n\xAB\\xFF", s); + UNIT_ASSERT_VALUES_EQUAL("http://ya.ru/\\x17\n\xAB", EscapeJ<false>("http://ya.ru/\x17\n\xab", "\n")); + UNIT_ASSERT_VALUES_EQUAL("http:\\x2F\\x2Fya.ru\\x2F\\x17\n\xAB'", EscapeJ<false>("http://ya.ru/\x17\n\xab'", "\n'", "/")); + UNIT_ASSERT_VALUES_EQUAL("http://ya.ru/\x17\n\xab", UnescapeC("http:\\x2F\\x2Fya.ru\\x2F\\x17\n\xAB")); + UNIT_ASSERT_VALUES_EQUAL("http://ya.ru/\x17\n\xab", UnescapeC("http://ya.ru/\\x17\\n\xAB")); + UNIT_ASSERT_VALUES_EQUAL("h", EscapeJ<false>("h")); + UNIT_ASSERT_VALUES_EQUAL("\"h\"", EscapeJ<true>("h")); + UNIT_ASSERT_VALUES_EQUAL("h", UnescapeC("h")); + UNIT_ASSERT_VALUES_EQUAL("\\xFF", EscapeJ<false>("\xFF")); + UNIT_ASSERT_VALUES_EQUAL("\"\\xFF\"", EscapeJ<true>("\xFF")); + UNIT_ASSERT_VALUES_EQUAL("\xFF", UnescapeC("\\xFF")); + UNIT_ASSERT_VALUES_EQUAL("\\377f", EscapeJ<false>("\xff" "f")); UNIT_ASSERT_VALUES_EQUAL("\xff" @@ -62,5 +62,5 @@ Y_UNIT_TEST_SUITE(TRelaxedEscaperTest) { UNIT_ASSERT_VALUES_EQUAL("\xff" "g", UnescapeC("\\xFFg")); - } -} + } +} diff --git a/library/cpp/string_utils/url/url.cpp b/library/cpp/string_utils/url/url.cpp index 5050e2bf3a..85f4ac5d69 100644 --- a/library/cpp/string_utils/url/url.cpp +++ b/library/cpp/string_utils/url/url.cpp @@ -48,13 +48,13 @@ namespace { const TChar httpsPrefix[] = {'h', 't', 't', 'p', 's', ':', '/', '/', 0}; if (urlSize.Has(7) && Compare1Case2(url, httpPrefix, 7) == 0) return 7; - if (!ignorehttps && urlSize.Has(8) && Compare1Case2(url, httpsPrefix, 8) == 0) + if (!ignorehttps && urlSize.Has(8) && Compare1Case2(url, httpsPrefix, 8) == 0) return 8; return 0; } template <typename T> - inline T CutHttpPrefixImpl(const T& url, bool ignorehttps) { + inline T CutHttpPrefixImpl(const T& url, bool ignorehttps) { size_t prefixSize = GetHttpPrefixSizeImpl<typename T::char_type>(url.data(), TKnownSize(url.size()), ignorehttps); if (prefixSize) return url.substr(prefixSize); @@ -90,11 +90,11 @@ size_t GetHttpPrefixSize(const TWtringBuf url, bool ignorehttps) noexcept { } TStringBuf CutHttpPrefix(const TStringBuf url, bool ignorehttps) noexcept { - return CutHttpPrefixImpl(url, ignorehttps); + return CutHttpPrefixImpl(url, ignorehttps); } TWtringBuf CutHttpPrefix(const TWtringBuf url, bool ignorehttps) noexcept { - return CutHttpPrefixImpl(url, ignorehttps); + return CutHttpPrefixImpl(url, ignorehttps); } size_t GetSchemePrefixSize(const TStringBuf url) noexcept { diff --git a/library/cpp/string_utils/url/url.h b/library/cpp/string_utils/url/url.h index 3c12e514fa..84137ccc57 100644 --- a/library/cpp/string_utils/url/url.h +++ b/library/cpp/string_utils/url/url.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <util/generic/fwd.h> #include <util/generic/strbuf.h> @@ -49,7 +49,7 @@ TStringBuf GetSchemePrefix(const TStringBuf url) noexcept; //! @return a new URL without protocol prefix Y_PURE_FUNCTION TStringBuf CutHttpPrefix(const TStringBuf url, bool ignorehttps = false) noexcept; - + Y_PURE_FUNCTION TWtringBuf CutHttpPrefix(const TWtringBuf url, bool ignorehttps = false) noexcept; diff --git a/library/cpp/string_utils/url/url_ut.cpp b/library/cpp/string_utils/url/url_ut.cpp index 36fdaa0f6d..1588013893 100644 --- a/library/cpp/string_utils/url/url_ut.cpp +++ b/library/cpp/string_utils/url/url_ut.cpp @@ -19,8 +19,8 @@ Y_UNIT_TEST_SUITE(TUtilUrlTest) { UNIT_ASSERT_VALUES_EQUAL("www.ya.ru", GetHost("www.ya.ru:8080/bebe")); UNIT_ASSERT_VALUES_EQUAL("www.ya.ru", GetHost("https://www.ya.ru:8080/bebe")); UNIT_ASSERT_VALUES_EQUAL("ya.ru:8080", GetHostAndPort("ya.ru:8080/bebe")); - // irl RFC3986 sometimes gets ignored - UNIT_ASSERT_VALUES_EQUAL("pravda-kmv.ru", GetHost("pravda-kmv.ru?page=news&id=6973")); + // irl RFC3986 sometimes gets ignored + UNIT_ASSERT_VALUES_EQUAL("pravda-kmv.ru", GetHost("pravda-kmv.ru?page=news&id=6973")); UNIT_ASSERT_VALUES_EQUAL("pravda-kmv.ru", GetHostAndPort("pravda-kmv.ru?page=news&id=6973")); // check simple string UNIT_ASSERT_VALUES_EQUAL("some_blender_url", GetHost("some_blender_url")); @@ -99,9 +99,9 @@ Y_UNIT_TEST_SUITE(TUtilUrlTest) { UNIT_ASSERT_VALUES_EQUAL("ya.ru/zzz", CutHttpPrefix("http://ya.ru/zzz")); UNIT_ASSERT_VALUES_EQUAL("ya.ru/zzz", CutHttpPrefix("http://ya.ru/zzz", true)); UNIT_ASSERT_VALUES_EQUAL("ya.ru/zzz", CutHttpPrefix("https://ya.ru/zzz")); - UNIT_ASSERT_VALUES_EQUAL("https://ya.ru/zzz", CutHttpPrefix("https://ya.ru/zzz", true)); + UNIT_ASSERT_VALUES_EQUAL("https://ya.ru/zzz", CutHttpPrefix("https://ya.ru/zzz", true)); UNIT_ASSERT_VALUES_EQUAL("", CutHttpPrefix("https://")); // is that right? - UNIT_ASSERT_VALUES_EQUAL("https://", CutHttpPrefix("https://", true)); // is that right? + UNIT_ASSERT_VALUES_EQUAL("https://", CutHttpPrefix("https://", true)); // is that right? } Y_UNIT_TEST(TestMisc) { diff --git a/library/cpp/svnversion/svnversion.cpp b/library/cpp/svnversion/svnversion.cpp index 14744014b7..4c9761fa75 100644 --- a/library/cpp/svnversion/svnversion.cpp +++ b/library/cpp/svnversion/svnversion.cpp @@ -11,11 +11,11 @@ extern "C" void PrintProgramSvnVersion() { puts(GetProgramSvnVersion()); } -extern "C" void PrintSvnVersionAndExit0() { - PrintProgramSvnVersion(); - exit(0); -} - +extern "C" void PrintSvnVersionAndExit0() { + PrintProgramSvnVersion(); + exit(0); +} + extern "C" void PrintSvnVersionAndExitEx(int argc, char* argv[], const char* opts) { if (2 == argc) { for (TStringBuf all = opts, versionOpt; all.NextTok(';', versionOpt);) { diff --git a/library/cpp/svnversion/svnversion.h b/library/cpp/svnversion/svnversion.h index f4e62a6df0..b99615daa9 100644 --- a/library/cpp/svnversion/svnversion.h +++ b/library/cpp/svnversion/svnversion.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #if !defined(FROM_IMPL) #define PROGRAM_VERSION GetProgramSvnVersion() diff --git a/library/cpp/testing/benchmark/bench.cpp b/library/cpp/testing/benchmark/bench.cpp index b03792edc9..08d8708005 100644 --- a/library/cpp/testing/benchmark/bench.cpp +++ b/library/cpp/testing/benchmark/bench.cpp @@ -276,7 +276,7 @@ namespace { NCpu::TParams params{n}; F(params); - }, opts.TimeBudget, *this); + }, opts.TimeBudget, *this); } enum EOutFormat { diff --git a/library/cpp/testing/common/env.h b/library/cpp/testing/common/env.h index e8f445db3b..7b89aa1bed 100644 --- a/library/cpp/testing/common/env.h +++ b/library/cpp/testing/common/env.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <unordered_map> diff --git a/library/cpp/testing/unittest/registar.cpp b/library/cpp/testing/unittest/registar.cpp index 7679b6647c..3679b768ed 100644 --- a/library/cpp/testing/unittest/registar.cpp +++ b/library/cpp/testing/unittest/registar.cpp @@ -74,13 +74,13 @@ NUnitTest::TTestBase* ::NUnitTest::NPrivate::GetCurrentTest() { struct TDiffColorizer { NColorizer::TColors Colors; - bool Reverse = false; + bool Reverse = false; - explicit TDiffColorizer(bool reverse = false) - : Reverse(reverse) + explicit TDiffColorizer(bool reverse = false) + : Reverse(reverse) { } - + TString Special(TStringBuf str) const { return ToString(Colors.YellowColor()) + str; } @@ -96,14 +96,14 @@ struct TDiffColorizer { TString Right(TArrayRef<const char> str) const { return ToString(GetRightColor()) + TString(str.begin(), str.end()); } - - TStringBuf GetLeftColor() const { - return Reverse ? Colors.RedColor() : Colors.GreenColor(); - } - - TStringBuf GetRightColor() const { - return Reverse ? Colors.GreenColor() : Colors.RedColor(); - } + + TStringBuf GetLeftColor() const { + return Reverse ? Colors.RedColor() : Colors.GreenColor(); + } + + TStringBuf GetRightColor() const { + return Reverse ? Colors.GreenColor() : Colors.RedColor(); + } }; struct TTraceDiffFormatter { diff --git a/library/cpp/testing/unittest/registar.h b/library/cpp/testing/unittest/registar.h index 45da84eaca..44517a0092 100644 --- a/library/cpp/testing/unittest/registar.h +++ b/library/cpp/testing/unittest/registar.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <library/cpp/dbg_output/dump.h> @@ -15,7 +15,7 @@ #include <util/string/builder.h> #include <util/string/cast.h> #include <util/string/printf.h> - + #include <util/system/defaults.h> #include <util/system/type_name.h> #include <util/system/spinlock.h> @@ -382,7 +382,7 @@ public: \ } \ } while (false) -//doubles +//doubles // UNIT_ASSERT_DOUBLES_EQUAL_DEPRECATED* macros do not handle NaNs correctly (see IGNIETFERRO-1419) and are for backward compatibility // only. Consider switching to regular UNIT_ASSERT_DOUBLES_EQUAL* macros if you're still using the deprecated version. #define UNIT_ASSERT_DOUBLES_EQUAL_DEPRECATED_C(E, A, D, C) \ @@ -422,9 +422,9 @@ public: \ } \ } while (false) -#define UNIT_ASSERT_DOUBLES_EQUAL(E, A, D) UNIT_ASSERT_DOUBLES_EQUAL_C(E, A, D, "") +#define UNIT_ASSERT_DOUBLES_EQUAL(E, A, D) UNIT_ASSERT_DOUBLES_EQUAL_C(E, A, D, "") -//strings +//strings #define UNIT_ASSERT_STRINGS_EQUAL_C(A, B, C) \ do { \ const TString _a(A); \ @@ -435,7 +435,7 @@ public: \ } \ } while (false) -#define UNIT_ASSERT_STRINGS_EQUAL(A, B) UNIT_ASSERT_STRINGS_EQUAL_C(A, B, "") +#define UNIT_ASSERT_STRINGS_EQUAL(A, B) UNIT_ASSERT_STRINGS_EQUAL_C(A, B, "") #define UNIT_ASSERT_STRING_CONTAINS_C(A, B, C) \ do { \ @@ -471,7 +471,7 @@ public: \ #define UNIT_ASSERT_STRINGS_UNEQUAL(A, B) UNIT_ASSERT_STRINGS_UNEQUAL_C(A, B, "") -//bool +//bool #define UNIT_ASSERT_C(A, C) \ do { \ if (!(A)) { \ @@ -479,9 +479,9 @@ public: \ } \ } while (false) -#define UNIT_ASSERT(A) UNIT_ASSERT_C(A, "") +#define UNIT_ASSERT(A) UNIT_ASSERT_C(A, "") -//general +//general #define UNIT_ASSERT_EQUAL_C(A, B, C) \ do { \ if (!((A) == (B))) { \ @@ -489,7 +489,7 @@ public: \ } \ } while (false) -#define UNIT_ASSERT_EQUAL(A, B) UNIT_ASSERT_EQUAL_C(A, B, "") +#define UNIT_ASSERT_EQUAL(A, B) UNIT_ASSERT_EQUAL_C(A, B, "") #define UNIT_ASSERT_UNEQUAL_C(A, B, C) \ do { \ @@ -498,8 +498,8 @@ public: \ } \ } while (false) -#define UNIT_ASSERT_UNEQUAL(A, B) UNIT_ASSERT_UNEQUAL_C(A, B, "") - +#define UNIT_ASSERT_UNEQUAL(A, B) UNIT_ASSERT_UNEQUAL_C(A, B, "") + #define UNIT_ASSERT_LT_C(A, B, C) \ do { \ if (!((A) < (B))) { \ @@ -703,7 +703,7 @@ public: \ } } -//values +//values #define UNIT_ASSERT_VALUES_EQUAL_IMPL(A, B, C, EQflag, EQstr, NEQstr) \ do { \ TString _as; \ @@ -727,7 +727,7 @@ public: \ #define UNIT_ASSERT_VALUES_UNEQUAL_C(A, B, C) \ UNIT_ASSERT_VALUES_EQUAL_IMPL(A, B, C, false, "!=", "==") -#define UNIT_ASSERT_VALUES_EQUAL(A, B) UNIT_ASSERT_VALUES_EQUAL_C(A, B, "") +#define UNIT_ASSERT_VALUES_EQUAL(A, B) UNIT_ASSERT_VALUES_EQUAL_C(A, B, "") #define UNIT_ASSERT_VALUES_UNEQUAL(A, B) UNIT_ASSERT_VALUES_UNEQUAL_C(A, B, "") // Checks that test will fail while executing given expression diff --git a/library/cpp/testing/unittest/tests_data.h b/library/cpp/testing/unittest/tests_data.h index bc93f5f59c..6536bc1ae6 100644 --- a/library/cpp/testing/unittest/tests_data.h +++ b/library/cpp/testing/unittest/tests_data.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <library/cpp/testing/common/env.h> diff --git a/library/cpp/uri/common.h b/library/cpp/uri/common.h index a08c0ccf9d..8025357763 100644 --- a/library/cpp/uri/common.h +++ b/library/cpp/uri/common.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <util/stream/output.h> #include <util/system/compat.h> diff --git a/library/cpp/uri/http_url.h b/library/cpp/uri/http_url.h index efbc5375de..7c8e8d844d 100644 --- a/library/cpp/uri/http_url.h +++ b/library/cpp/uri/http_url.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include "uri.h" #include "other.h" diff --git a/library/cpp/uri/other.h b/library/cpp/uri/other.h index ee38c6b942..7aec22e77b 100644 --- a/library/cpp/uri/other.h +++ b/library/cpp/uri/other.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <util/generic/string.h> diff --git a/library/cpp/uri/uri.h b/library/cpp/uri/uri.h index 7f9e63ed17..3b6c19fe4a 100644 --- a/library/cpp/uri/uri.h +++ b/library/cpp/uri/uri.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include "common.h" #include "encode.h" diff --git a/library/cpp/ya.make b/library/cpp/ya.make index bc0bb82f82..8c1193b007 100644 --- a/library/cpp/ya.make +++ b/library/cpp/ya.make @@ -172,7 +172,7 @@ RECURSE( ipv6_address/ut iterator json - json/fast_sax + json/fast_sax json/flex_buffers json/flex_buffers/ut json/fuzzy_test |