diff options
author | pkalinnikov <pkalinnikov@yandex-team.ru> | 2022-02-10 16:50:15 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:50:15 +0300 |
commit | d507a9366b2ab84411afe63fea9fba5498891e1b (patch) | |
tree | 5d5cb817648f650d76cf1076100726fd9b8448e8 | |
parent | 9e33e026829d561d6fd46d72b88c367952e08075 (diff) | |
download | ydb-d507a9366b2ab84411afe63fea9fba5498891e1b.tar.gz |
Restoring authorship annotation for <pkalinnikov@yandex-team.ru>. Commit 2 of 2.
51 files changed, 2428 insertions, 2428 deletions
diff --git a/library/cpp/actors/core/actorid.cpp b/library/cpp/actors/core/actorid.cpp index 8de83ebc0d..ccda035eac 100644 --- a/library/cpp/actors/core/actorid.cpp +++ b/library/cpp/actors/core/actorid.cpp @@ -1,6 +1,6 @@ #include "actorid.h" #include <util/string/builder.h> -#include <util/string/cast.h> +#include <util/string/cast.h> namespace NActors { void TActorId::Out(IOutputStream& o) const { @@ -26,9 +26,9 @@ namespace NActors { semicolons[1] = str.find(':', semicolons[0] + 1); if (semicolons[1] == TStringBuf::npos) return false; - + bool success = TryFromString(buf + 1, semicolons[0] - 1, Raw.N.NodeId) && TryFromString(buf + semicolons[0] + 1, semicolons[1] - semicolons[0] - 1, Raw.N.LocalId) && TryFromString(buf + semicolons[1] + 1, sz - semicolons[1] - 2, Raw.N.Hint); - + return success; } } diff --git a/library/cpp/containers/bitseq/bititerator.h b/library/cpp/containers/bitseq/bititerator.h index 616c9bbac4..52dadd3798 100644 --- a/library/cpp/containers/bitseq/bititerator.h +++ b/library/cpp/containers/bitseq/bititerator.h @@ -1,138 +1,138 @@ -#pragma once - -#include "traits.h" - +#pragma once + +#include "traits.h" + #include <library/cpp/pop_count/popcount.h> -template <typename T> -class TBitIterator { -public: - using TWord = T; - using TTraits = TBitSeqTraits<TWord>; - -public: - TBitIterator(const T* data = nullptr) - : Current(0) - , Mask(0) - , Data(data) +template <typename T> +class TBitIterator { +public: + using TWord = T; + using TTraits = TBitSeqTraits<TWord>; + +public: + TBitIterator(const T* data = nullptr) + : Current(0) + , Mask(0) + , Data(data) { } - - /// Get the word next to the one we are currenlty iterating over. - const TWord* NextWord() const { - return Data; - } - - /// Get the next bit without moving the iterator. - bool Peek() const { - return Mask ? (Current & Mask) : (*Data & 1); - } - - /// Get the next bit and move forward. - /// TODO: Implement inversed iteration as well. - bool Next() { - if (!Mask) { - Current = *Data++; - Mask = 1; - } - const bool bit = Current & Mask; - Mask <<= 1; - return bit; - } - - /// Get the next count bits without moving the iterator. - TWord Peek(ui8 count) const { - if (!count) - return 0; + + /// Get the word next to the one we are currenlty iterating over. + const TWord* NextWord() const { + return Data; + } + + /// Get the next bit without moving the iterator. + bool Peek() const { + return Mask ? (Current & Mask) : (*Data & 1); + } + + /// Get the next bit and move forward. + /// TODO: Implement inversed iteration as well. + bool Next() { + if (!Mask) { + Current = *Data++; + Mask = 1; + } + const bool bit = Current & Mask; + Mask <<= 1; + return bit; + } + + /// Get the next count bits without moving the iterator. + TWord Peek(ui8 count) const { + if (!count) + return 0; Y_VERIFY_DEBUG(count <= TTraits::NumBits); - - if (!Mask) - return *Data & TTraits::ElemMask(count); - + + if (!Mask) + return *Data & TTraits::ElemMask(count); + auto usedBits = (size_t)PopCount(Mask - 1); - TWord result = Current >> usedBits; - auto leftInCurrent = TTraits::NumBits - usedBits; - if (count <= leftInCurrent) - return result & TTraits::ElemMask(count); - - count -= leftInCurrent; - result |= (*Data & TTraits::ElemMask(count)) << leftInCurrent; - return result; - } - - /// Get the next count bits and move forward by count bits. - TWord Read(ui8 count) { - if (!count) - return 0; + TWord result = Current >> usedBits; + auto leftInCurrent = TTraits::NumBits - usedBits; + if (count <= leftInCurrent) + return result & TTraits::ElemMask(count); + + count -= leftInCurrent; + result |= (*Data & TTraits::ElemMask(count)) << leftInCurrent; + return result; + } + + /// Get the next count bits and move forward by count bits. + TWord Read(ui8 count) { + if (!count) + return 0; Y_VERIFY_DEBUG(count <= TTraits::NumBits); - - if (!Mask) { - Current = *Data++; - Mask = 1 << count; - return Current & TTraits::ElemMask(count); - } - + + if (!Mask) { + Current = *Data++; + Mask = 1 << count; + return Current & TTraits::ElemMask(count); + } + auto usedBits = (size_t)PopCount(Mask - 1); - TWord result = Current >> usedBits; - auto leftInCurrent = TTraits::NumBits - usedBits; - if (count < leftInCurrent) { - Mask <<= count; - return result & TTraits::ElemMask(count); - } - - count -= leftInCurrent; - if (count) { - Current = *Data++; - Mask = 1 << count; - result |= (Current & TTraits::ElemMask(count)) << leftInCurrent; - } else { - Mask = 0; - } - - return result; - } - - /// Move the iterator forward by count bits. - void Forward(int count) { - if (!count) - return; - + TWord result = Current >> usedBits; + auto leftInCurrent = TTraits::NumBits - usedBits; + if (count < leftInCurrent) { + Mask <<= count; + return result & TTraits::ElemMask(count); + } + + count -= leftInCurrent; + if (count) { + Current = *Data++; + Mask = 1 << count; + result |= (Current & TTraits::ElemMask(count)) << leftInCurrent; + } else { + Mask = 0; + } + + return result; + } + + /// Move the iterator forward by count bits. + void Forward(int count) { + if (!count) + return; + int leftInCurrent = (size_t)PopCount(~(Mask - 1)); - if (count < leftInCurrent) { - Mask <<= count; - return; - } - - count -= leftInCurrent; - Data += count >> TTraits::DivShift; - auto remainder = count & TTraits::ModMask; - - if (remainder) { - Current = *Data++; - Mask = 1 << remainder; - } else { - Current = 0; - Mask = 0; - } - } - - /// Skip trailing bits of the current word and move by count words forward. - void Align(int count = 0) { - Current = 0; - if (Mask) - Mask = 0; - Data += count; - } - - /// Initialize the iterator. - void Reset(const TWord* data) { - Current = 0; - Mask = 0; - Data = data; - } - -private: - TWord Current; - TWord Mask; - const TWord* Data; -}; + if (count < leftInCurrent) { + Mask <<= count; + return; + } + + count -= leftInCurrent; + Data += count >> TTraits::DivShift; + auto remainder = count & TTraits::ModMask; + + if (remainder) { + Current = *Data++; + Mask = 1 << remainder; + } else { + Current = 0; + Mask = 0; + } + } + + /// Skip trailing bits of the current word and move by count words forward. + void Align(int count = 0) { + Current = 0; + if (Mask) + Mask = 0; + Data += count; + } + + /// Initialize the iterator. + void Reset(const TWord* data) { + Current = 0; + Mask = 0; + Data = data; + } + +private: + TWord Current; + TWord Mask; + const TWord* Data; +}; diff --git a/library/cpp/containers/bitseq/bititerator_ut.cpp b/library/cpp/containers/bitseq/bititerator_ut.cpp index 5f34e0de84..ed0925866f 100644 --- a/library/cpp/containers/bitseq/bititerator_ut.cpp +++ b/library/cpp/containers/bitseq/bititerator_ut.cpp @@ -1,109 +1,109 @@ -#include "bititerator.h" - +#include "bititerator.h" + #include <library/cpp/testing/unittest/registar.h> -#include <util/generic/vector.h> - +#include <util/generic/vector.h> + Y_UNIT_TEST_SUITE(TBitIteratorTest) { TVector<ui16> GenWords() { TVector<ui16> words(1, 0); - for (ui16 word = 1; word; ++word) - words.push_back(word); - return words; - } - - template <typename TWord> + for (ui16 word = 1; word; ++word) + words.push_back(word); + return words; + } + + template <typename TWord> void AssertPeekRead(TBitIterator<TWord> & iter, ui8 count, TWord expected) { - auto peek = iter.Peek(count); - auto read = iter.Read(count); - UNIT_ASSERT_EQUAL(peek, read); - UNIT_ASSERT_EQUAL(peek, expected); - } - + auto peek = iter.Peek(count); + auto read = iter.Read(count); + UNIT_ASSERT_EQUAL(peek, read); + UNIT_ASSERT_EQUAL(peek, expected); + } + Y_UNIT_TEST(TestNextAndPeek) { - const auto& words = GenWords(); - - TBitIterator<ui16> iter(words.data()); - ui16 word = 0; - for (int i = 0; i != (1 << 16); ++i, ++word) { - for (int bit = 0; bit != 16; ++bit) { - auto peek = iter.Peek(); - auto next = iter.Next(); - UNIT_ASSERT_EQUAL(peek, next); - UNIT_ASSERT_EQUAL(peek, (word >> bit) & 1); - } - UNIT_ASSERT_EQUAL(iter.NextWord(), words.data() + i + 1); - } - - UNIT_ASSERT_EQUAL(iter.NextWord(), words.data() + words.size()); - } - + const auto& words = GenWords(); + + TBitIterator<ui16> iter(words.data()); + ui16 word = 0; + for (int i = 0; i != (1 << 16); ++i, ++word) { + for (int bit = 0; bit != 16; ++bit) { + auto peek = iter.Peek(); + auto next = iter.Next(); + UNIT_ASSERT_EQUAL(peek, next); + UNIT_ASSERT_EQUAL(peek, (word >> bit) & 1); + } + UNIT_ASSERT_EQUAL(iter.NextWord(), words.data() + i + 1); + } + + UNIT_ASSERT_EQUAL(iter.NextWord(), words.data() + words.size()); + } + Y_UNIT_TEST(TestAlignedReadAndPeek) { - const auto& words = GenWords(); - - TBitIterator<ui16> iter(words.data()); - ui16 word = 0; - for (int i = 0; i != (1 << 16); ++i, ++word) { - AssertPeekRead(iter, 16, word); - UNIT_ASSERT_EQUAL(iter.NextWord(), words.data() + i + 1); - } - - UNIT_ASSERT_EQUAL(iter.NextWord(), words.data() + words.size()); - } - + const auto& words = GenWords(); + + TBitIterator<ui16> iter(words.data()); + ui16 word = 0; + for (int i = 0; i != (1 << 16); ++i, ++word) { + AssertPeekRead(iter, 16, word); + UNIT_ASSERT_EQUAL(iter.NextWord(), words.data() + i + 1); + } + + UNIT_ASSERT_EQUAL(iter.NextWord(), words.data() + words.size()); + } + Y_UNIT_TEST(TestForward) { TVector<ui32> words; - words.push_back((1 << 10) | (1 << 20) | (1 << 25)); - words.push_back(1 | (1 << 5) | (1 << 6) | (1 << 30)); - for (int i = 0; i < 3; ++i) - words.push_back(0); - words.push_back(1 << 10); - - TBitIterator<ui32> iter(words.data()); - UNIT_ASSERT(!iter.Next()); - UNIT_ASSERT(!iter.Next()); - UNIT_ASSERT(!iter.Next()); - iter.Forward(6); - UNIT_ASSERT(!iter.Next()); - UNIT_ASSERT(iter.Next()); - UNIT_ASSERT(!iter.Next()); - iter.Forward(8); - UNIT_ASSERT(iter.Next()); - iter.Forward(4); - UNIT_ASSERT(iter.Next()); - iter.Forward(5); - UNIT_ASSERT(!iter.Next()); - UNIT_ASSERT(iter.Next()); - iter.Forward(4); - UNIT_ASSERT(iter.Next()); - - iter.Reset(words.data()); - iter.Forward(38); - UNIT_ASSERT(iter.Next()); - UNIT_ASSERT(!iter.Next()); - UNIT_ASSERT_EQUAL(iter.NextWord(), words.data() + 2); - - iter.Forward(24 + 32 * 3 + 9); - UNIT_ASSERT(!iter.Next()); - UNIT_ASSERT(iter.Next()); - UNIT_ASSERT(!iter.Next()); - UNIT_ASSERT_EQUAL(iter.NextWord(), words.data() + 6); - } - + words.push_back((1 << 10) | (1 << 20) | (1 << 25)); + words.push_back(1 | (1 << 5) | (1 << 6) | (1 << 30)); + for (int i = 0; i < 3; ++i) + words.push_back(0); + words.push_back(1 << 10); + + TBitIterator<ui32> iter(words.data()); + UNIT_ASSERT(!iter.Next()); + UNIT_ASSERT(!iter.Next()); + UNIT_ASSERT(!iter.Next()); + iter.Forward(6); + UNIT_ASSERT(!iter.Next()); + UNIT_ASSERT(iter.Next()); + UNIT_ASSERT(!iter.Next()); + iter.Forward(8); + UNIT_ASSERT(iter.Next()); + iter.Forward(4); + UNIT_ASSERT(iter.Next()); + iter.Forward(5); + UNIT_ASSERT(!iter.Next()); + UNIT_ASSERT(iter.Next()); + iter.Forward(4); + UNIT_ASSERT(iter.Next()); + + iter.Reset(words.data()); + iter.Forward(38); + UNIT_ASSERT(iter.Next()); + UNIT_ASSERT(!iter.Next()); + UNIT_ASSERT_EQUAL(iter.NextWord(), words.data() + 2); + + iter.Forward(24 + 32 * 3 + 9); + UNIT_ASSERT(!iter.Next()); + UNIT_ASSERT(iter.Next()); + UNIT_ASSERT(!iter.Next()); + UNIT_ASSERT_EQUAL(iter.NextWord(), words.data() + 6); + } + Y_UNIT_TEST(TestUnalignedReadAndPeek) { TVector<ui32> words; - words.push_back((1 << 10) | (1 << 20) | (1 << 25)); - words.push_back(1 | (1 << 5) | (1 << 6) | (1 << 30)); - for (int i = 0; i < 5; ++i) - words.push_back(1 | (1 << 10)); - - TBitIterator<ui32> iter(words.data()); - AssertPeekRead(iter, 5, ui32(0)); - AssertPeekRead(iter, 7, ui32(1 << 5)); - AssertPeekRead(iter, 21, ui32((1 << 8) | (1 << 13) | (1 << 20))); - AssertPeekRead(iter, 32, (words[1] >> 1) | (1 << 31)); - iter.Forward(8); - UNIT_ASSERT(!iter.Next()); - UNIT_ASSERT(iter.Next()); - UNIT_ASSERT(!iter.Next()); - } -} + words.push_back((1 << 10) | (1 << 20) | (1 << 25)); + words.push_back(1 | (1 << 5) | (1 << 6) | (1 << 30)); + for (int i = 0; i < 5; ++i) + words.push_back(1 | (1 << 10)); + + TBitIterator<ui32> iter(words.data()); + AssertPeekRead(iter, 5, ui32(0)); + AssertPeekRead(iter, 7, ui32(1 << 5)); + AssertPeekRead(iter, 21, ui32((1 << 8) | (1 << 13) | (1 << 20))); + AssertPeekRead(iter, 32, (words[1] >> 1) | (1 << 31)); + iter.Forward(8); + UNIT_ASSERT(!iter.Next()); + UNIT_ASSERT(iter.Next()); + UNIT_ASSERT(!iter.Next()); + } +} diff --git a/library/cpp/containers/bitseq/bitvector.h b/library/cpp/containers/bitseq/bitvector.h index 87ed1560b8..3f8fd81ee5 100644 --- a/library/cpp/containers/bitseq/bitvector.h +++ b/library/cpp/containers/bitseq/bitvector.h @@ -1,158 +1,158 @@ -#pragma once - -#include "traits.h" - +#pragma once + +#include "traits.h" + #include <library/cpp/pop_count/popcount.h> -#include <util/generic/vector.h> -#include <util/ysaveload.h> - -template <typename T> +#include <util/generic/vector.h> +#include <util/ysaveload.h> + +template <typename T> class TReadonlyBitVector; template <typename T> -class TBitVector { -public: - using TWord = T; - using TTraits = TBitSeqTraits<TWord>; - -private: +class TBitVector { +public: + using TWord = T; + using TTraits = TBitSeqTraits<TWord>; + +private: friend class TReadonlyBitVector<T>; - ui64 Size_; + ui64 Size_; TVector<TWord> Data_; - -public: - TBitVector() - : Size_(0) - , Data_(0) - { - } - - TBitVector(ui64 size) - : Size_(size) - , Data_(static_cast<size_t>((Size_ + TTraits::ModMask) >> TTraits::DivShift), 0) - { - } - + +public: + TBitVector() + : Size_(0) + , Data_(0) + { + } + + TBitVector(ui64 size) + : Size_(size) + , Data_(static_cast<size_t>((Size_ + TTraits::ModMask) >> TTraits::DivShift), 0) + { + } + virtual ~TBitVector() = default; - - void Clear() { - Size_ = 0; - Data_.clear(); - } - - void Resize(ui64 size) { - Size_ = size; - Data_.resize((Size_ + TTraits::ModMask) >> TTraits::DivShift); - } - - void Swap(TBitVector& other) { - DoSwap(Size_, other.Size_); - DoSwap(Data_, other.Data_); - } - - bool Set(ui64 pos) { + + void Clear() { + Size_ = 0; + Data_.clear(); + } + + void Resize(ui64 size) { + Size_ = size; + Data_.resize((Size_ + TTraits::ModMask) >> TTraits::DivShift); + } + + void Swap(TBitVector& other) { + DoSwap(Size_, other.Size_); + DoSwap(Data_, other.Data_); + } + + bool Set(ui64 pos) { Y_ASSERT(pos < Size_); - TWord& val = Data_[pos >> TTraits::DivShift]; - if (val & TTraits::BitMask(pos & TTraits::ModMask)) - return false; - val |= TTraits::BitMask(pos & TTraits::ModMask); - return true; - } - - bool Test(ui64 pos) const { + TWord& val = Data_[pos >> TTraits::DivShift]; + if (val & TTraits::BitMask(pos & TTraits::ModMask)) + return false; + val |= TTraits::BitMask(pos & TTraits::ModMask); + return true; + } + + bool Test(ui64 pos) const { return TTraits::Test(Data(), pos, Size_); - } - - void Reset(ui64 pos) { + } + + void Reset(ui64 pos) { Y_ASSERT(pos < Size_); - Data_[pos >> TTraits::DivShift] &= ~TTraits::BitMask(pos & TTraits::ModMask); - } - - TWord Get(ui64 pos, ui8 width, TWord mask) const { + Data_[pos >> TTraits::DivShift] &= ~TTraits::BitMask(pos & TTraits::ModMask); + } + + TWord Get(ui64 pos, ui8 width, TWord mask) const { return TTraits::Get(Data(), pos, width, mask, Size_); - } - - TWord Get(ui64 pos, ui8 width) const { - return Get(pos, width, TTraits::ElemMask(width)); - } - - void Set(ui64 pos, TWord value, ui8 width, TWord mask) { + } + + TWord Get(ui64 pos, ui8 width) const { + return Get(pos, width, TTraits::ElemMask(width)); + } + + void Set(ui64 pos, TWord value, ui8 width, TWord mask) { if (!width) return; Y_ASSERT((pos + width) <= Size_); - size_t word = pos >> TTraits::DivShift; - TWord shift1 = pos & TTraits::ModMask; - TWord shift2 = TTraits::NumBits - shift1; - Data_[word] &= ~(mask << shift1); - Data_[word] |= (value & mask) << shift1; - if (shift2 < width) { + size_t word = pos >> TTraits::DivShift; + TWord shift1 = pos & TTraits::ModMask; + TWord shift2 = TTraits::NumBits - shift1; + Data_[word] &= ~(mask << shift1); + Data_[word] |= (value & mask) << shift1; + if (shift2 < width) { Data_[word + 1] &= ~(mask >> shift2); Data_[word + 1] |= (value & mask) >> shift2; - } - } - - void Set(ui64 pos, TWord value, ui8 width) { - Set(pos, value, width, TTraits::ElemMask(width)); - } - - void Append(TWord value, ui8 width, TWord mask) { + } + } + + void Set(ui64 pos, TWord value, ui8 width) { + Set(pos, value, width, TTraits::ElemMask(width)); + } + + void Append(TWord value, ui8 width, TWord mask) { if (!width) return; - if (Data_.size() * TTraits::NumBits < Size_ + width) { - Data_.push_back(0); - } - Size_ += width; - Set(Size_ - width, value, width, mask); - } - - void Append(TWord value, ui8 width) { - Append(value, width, TTraits::ElemMask(width)); - } - - size_t Count() const { - size_t count = 0; - for (size_t i = 0; i < Data_.size(); ++i) { + if (Data_.size() * TTraits::NumBits < Size_ + width) { + Data_.push_back(0); + } + Size_ += width; + Set(Size_ - width, value, width, mask); + } + + void Append(TWord value, ui8 width) { + Append(value, width, TTraits::ElemMask(width)); + } + + size_t Count() const { + size_t count = 0; + for (size_t i = 0; i < Data_.size(); ++i) { count += (size_t)PopCount(Data_[i]); - } - return count; - } - - ui64 Size() const { - return Size_; - } - - size_t Words() const { - return Data_.size(); - } - - const TWord* Data() const { + } + return count; + } + + ui64 Size() const { + return Size_; + } + + size_t Words() const { + return Data_.size(); + } + + const TWord* Data() const { return Data_.data(); - } - + } + void Save(IOutputStream* out) const { - ::Save(out, Size_); - ::Save(out, Data_); - } - + ::Save(out, Size_); + ::Save(out, Data_); + } + void Load(IInputStream* inp) { - ::Load(inp, Size_); - ::Load(inp, Data_); - } - - ui64 Space() const { - return CHAR_BIT * (sizeof(Size_) + + ::Load(inp, Size_); + ::Load(inp, Data_); + } + + ui64 Space() const { + return CHAR_BIT * (sizeof(Size_) + Data_.size() * sizeof(TWord)); - } - + } + void Print(IOutputStream& out, size_t truncate = 128) { - for (size_t i = 0; i < Data_.size() && i < truncate; ++i) { - for (int j = TTraits::NumBits - 1; j >= 0; --j) { - size_t pos = TTraits::NumBits * i + j; - out << (pos < Size_ && Test(pos) ? '1' : '0'); - } - out << " "; - } - out << Endl; - } -}; + for (size_t i = 0; i < Data_.size() && i < truncate; ++i) { + for (int j = TTraits::NumBits - 1; j >= 0; --j) { + size_t pos = TTraits::NumBits * i + j; + out << (pos < Size_ && Test(pos) ? '1' : '0'); + } + out << " "; + } + out << Endl; + } +}; diff --git a/library/cpp/containers/bitseq/bitvector_ut.cpp b/library/cpp/containers/bitseq/bitvector_ut.cpp index e0598dbceb..6137adab1e 100644 --- a/library/cpp/containers/bitseq/bitvector_ut.cpp +++ b/library/cpp/containers/bitseq/bitvector_ut.cpp @@ -1,71 +1,71 @@ -#include "bitvector.h" +#include "bitvector.h" #include "readonly_bitvector.h" - + #include <library/cpp/testing/unittest/registar.h> - + #include <util/memory/blob.h> #include <util/stream/buffer.h> Y_UNIT_TEST_SUITE(TBitVectorTest) { Y_UNIT_TEST(TestEmpty) { - TBitVector<ui64> v64; - UNIT_ASSERT_EQUAL(v64.Size(), 0); - UNIT_ASSERT_EQUAL(v64.Words(), 0); - - TBitVector<ui32> v32(0); - UNIT_ASSERT_EQUAL(v32.Size(), 0); - UNIT_ASSERT_EQUAL(v32.Words(), 0); - } - + TBitVector<ui64> v64; + UNIT_ASSERT_EQUAL(v64.Size(), 0); + UNIT_ASSERT_EQUAL(v64.Words(), 0); + + TBitVector<ui32> v32(0); + UNIT_ASSERT_EQUAL(v32.Size(), 0); + UNIT_ASSERT_EQUAL(v32.Words(), 0); + } + Y_UNIT_TEST(TestOneWord) { - TBitVector<ui32> v; - v.Append(1, 1); - v.Append(0, 1); - v.Append(1, 3); - v.Append(10, 4); - v.Append(100500, 20); - - UNIT_ASSERT_EQUAL(v.Get(0, 1), 1); - UNIT_ASSERT(v.Test(0)); - UNIT_ASSERT_EQUAL(v.Get(1, 1), 0); - UNIT_ASSERT_EQUAL(v.Get(2, 3), 1); - UNIT_ASSERT_EQUAL(v.Get(5, 4), 10); - UNIT_ASSERT_EQUAL(v.Get(9, 20), 100500); - - v.Reset(0); - v.Set(9, 1234, 15); - UNIT_ASSERT_EQUAL(v.Get(0, 1), 0); - UNIT_ASSERT(!v.Test(0)); - UNIT_ASSERT_EQUAL(v.Get(9, 15), 1234); - - UNIT_ASSERT_EQUAL(v.Size(), 29); - UNIT_ASSERT_EQUAL(v.Words(), 1); - } - + TBitVector<ui32> v; + v.Append(1, 1); + v.Append(0, 1); + v.Append(1, 3); + v.Append(10, 4); + v.Append(100500, 20); + + UNIT_ASSERT_EQUAL(v.Get(0, 1), 1); + UNIT_ASSERT(v.Test(0)); + UNIT_ASSERT_EQUAL(v.Get(1, 1), 0); + UNIT_ASSERT_EQUAL(v.Get(2, 3), 1); + UNIT_ASSERT_EQUAL(v.Get(5, 4), 10); + UNIT_ASSERT_EQUAL(v.Get(9, 20), 100500); + + v.Reset(0); + v.Set(9, 1234, 15); + UNIT_ASSERT_EQUAL(v.Get(0, 1), 0); + UNIT_ASSERT(!v.Test(0)); + UNIT_ASSERT_EQUAL(v.Get(9, 15), 1234); + + UNIT_ASSERT_EQUAL(v.Size(), 29); + UNIT_ASSERT_EQUAL(v.Words(), 1); + } + Y_UNIT_TEST(TestManyWords) { - static const int BITS = 10; - TBitVector<ui64> v; - - for (int i = 0, end = (1 << BITS); i < end; ++i) - v.Append(i, BITS); - - UNIT_ASSERT_EQUAL(v.Size(), BITS * (1 << BITS)); - UNIT_ASSERT_EQUAL(v.Words(), (v.Size() + 63) / 64); - for (int i = 0, end = (1 << BITS); i < end; ++i) - UNIT_ASSERT_EQUAL(v.Get(i * BITS, BITS), (ui64)i); - } - + static const int BITS = 10; + TBitVector<ui64> v; + + for (int i = 0, end = (1 << BITS); i < end; ++i) + v.Append(i, BITS); + + UNIT_ASSERT_EQUAL(v.Size(), BITS * (1 << BITS)); + UNIT_ASSERT_EQUAL(v.Words(), (v.Size() + 63) / 64); + for (int i = 0, end = (1 << BITS); i < end; ++i) + UNIT_ASSERT_EQUAL(v.Get(i * BITS, BITS), (ui64)i); + } + Y_UNIT_TEST(TestMaxWordSize) { - TBitVector<ui32> v; - for (int i = 0; i < 100; ++i) - v.Append(i, 32); - - for (int i = 0; i < 100; ++i) - UNIT_ASSERT_EQUAL(v.Get(i * 32, 32), (ui32)i); - - v.Set(10 * 32, 100500, 32); - UNIT_ASSERT_EQUAL(v.Get(10 * 32, 32), 100500); - } + TBitVector<ui32> v; + for (int i = 0; i < 100; ++i) + v.Append(i, 32); + + for (int i = 0; i < 100; ++i) + UNIT_ASSERT_EQUAL(v.Get(i * 32, 32), (ui32)i); + + v.Set(10 * 32, 100500, 32); + UNIT_ASSERT_EQUAL(v.Get(10 * 32, 32), 100500); + } Y_UNIT_TEST(TestReadonlyVector) { TBitVector<ui64> v(100); @@ -83,4 +83,4 @@ Y_UNIT_TEST_SUITE(TBitVectorTest) { UNIT_ASSERT_VALUES_EQUAL(rv.Test(i), i % 3 == 0); } } -} +} diff --git a/library/cpp/containers/bitseq/traits.h b/library/cpp/containers/bitseq/traits.h index 9136d6e972..2330b1b4f2 100644 --- a/library/cpp/containers/bitseq/traits.h +++ b/library/cpp/containers/bitseq/traits.h @@ -1,30 +1,30 @@ -#pragma once - +#pragma once + #include <util/generic/bitops.h> -#include <util/generic/typetraits.h> +#include <util/generic/typetraits.h> #include <util/system/yassert.h> - -template <typename TWord> -struct TBitSeqTraits { + +template <typename TWord> +struct TBitSeqTraits { static constexpr ui8 NumBits = CHAR_BIT * sizeof(TWord); static constexpr TWord ModMask = static_cast<TWord>(NumBits - 1); static constexpr TWord DivShift = MostSignificantBitCT(NumBits); - - static inline TWord ElemMask(ui8 count) { - // NOTE: Shifting by the type's length is UB, so we need this workaround. + + static inline TWord ElemMask(ui8 count) { + // NOTE: Shifting by the type's length is UB, so we need this workaround. if (Y_LIKELY(count)) - return TWord(-1) >> (NumBits - count); - return 0; - } - - static inline TWord BitMask(ui8 pos) { - return TWord(1) << pos; - } - - static size_t NumOfWords(size_t bits) { - return (bits + NumBits - 1) >> DivShift; - } - + return TWord(-1) >> (NumBits - count); + return 0; + } + + static inline TWord BitMask(ui8 pos) { + return TWord(1) << pos; + } + + static size_t NumOfWords(size_t bits) { + return (bits + NumBits - 1) >> DivShift; + } + static bool Test(const TWord* data, ui64 pos, ui64 size) { Y_ASSERT(pos < size); return data[pos >> DivShift] & BitMask(pos & ModMask); @@ -45,5 +45,5 @@ struct TBitSeqTraits { } static_assert(std::is_unsigned<TWord>::value, "Expected std::is_unsigned<T>::value."); - static_assert((NumBits & (NumBits - 1)) == 0, "NumBits should be a power of 2."); -}; + static_assert((NumBits & (NumBits - 1)) == 0, "NumBits should be a power of 2."); +}; diff --git a/library/cpp/containers/bitseq/ut/ya.make b/library/cpp/containers/bitseq/ut/ya.make index 9485ead99e..7155e82c06 100644 --- a/library/cpp/containers/bitseq/ut/ya.make +++ b/library/cpp/containers/bitseq/ut/ya.make @@ -1,10 +1,10 @@ UNITTEST_FOR(library/cpp/containers/bitseq) - + OWNER(g:util) - -SRCS( - bititerator_ut.cpp - bitvector_ut.cpp -) - -END() + +SRCS( + bititerator_ut.cpp + bitvector_ut.cpp +) + +END() diff --git a/library/cpp/containers/bitseq/ya.make b/library/cpp/containers/bitseq/ya.make index 74594abfc0..7090956c55 100644 --- a/library/cpp/containers/bitseq/ya.make +++ b/library/cpp/containers/bitseq/ya.make @@ -1,7 +1,7 @@ -LIBRARY() - +LIBRARY() + OWNER(g:util) - + PEERDIR( util/draft library/cpp/pop_count @@ -12,4 +12,4 @@ SRCS( readonly_bitvector.cpp ) -END() +END() diff --git a/library/cpp/packedtypes/longs.h b/library/cpp/packedtypes/longs.h index a91d5df178..084098d705 100644 --- a/library/cpp/packedtypes/longs.h +++ b/library/cpp/packedtypes/longs.h @@ -91,7 +91,7 @@ struct mem_traits { #define MY_14(x) ((ui16)(x) < PACK1LIM ? 1 : 2) #define MY_28(x) ((ui32)(x) < PACK2LIM ? MY_14(x) : ((ui32)(x) < PACK3LIM ? 3 : 4)) - + #define MY_32(x) ((ui32)(x) < PACK4LIM ? MY_28(x) : 5) #define MY_64(x) ((ui64)(x) < PACK4LIM ? MY_28(x) : ((ui64)(x) < PACK6LIM ? ((ui64)(x) < PACK5LIM ? 5 : 6) : ((ui64)(x) < PACK7LIM ? 7 : ((ui64)(x) < PACK8LIM ? 8 : 9)))) @@ -141,7 +141,7 @@ struct mem_traits { (ret) = 5; \ } \ MACRO_END - + #define PACK_64(x, buf, how, ret) \ MACRO_BEGIN \ if ((ui64)(x) < PACK4LIM) { \ @@ -227,13 +227,13 @@ struct mem_traits { (ret) = 5; \ } \ MACRO_END - + #define UNPACK_32(x, buf, how, ret) \ MACRO_BEGIN \ ui8 firstByte = how::get_8(buf); \ DO_UNPACK_32(firstByte, x, buf, how, ret); \ MACRO_END - + #define DO_UNPACK_64(firstByte, x, buf, how, ret) \ MACRO_BEGIN \ if (firstByte < 0xF0) { \ @@ -292,13 +292,13 @@ inline int len_long(const i64& longVal) { inline int in_long(i32& longVal, const char* ptrBuf) { int ret; - UNPACK_32(longVal, ptrBuf, mem_traits, ret); + UNPACK_32(longVal, ptrBuf, mem_traits, ret); return ret; } inline int out_long(const i32& longVal, char* ptrBuf) { int ret; - PACK_32(longVal, ptrBuf, mem_traits, ret); + PACK_32(longVal, ptrBuf, mem_traits, ret); return ret; } @@ -311,7 +311,7 @@ inline const C* Unpack32(T& x, const C* src) { int pkLen = 0; const char* c = reinterpret_cast<const char*>(src); Y_UNUSED(pkLen); - UNPACK_32(x, c, mem_traits, pkLen); + UNPACK_32(x, c, mem_traits, pkLen); Y_ASSERT(pkLen); return reinterpret_cast<const C*>(c); } @@ -331,7 +331,7 @@ inline C* Pack32(const T& x, C* dest) { int pkLen = 0; Y_UNUSED(pkLen); char* c = reinterpret_cast<char*>(dest); - PACK_32(x, c, mem_traits, pkLen); + PACK_32(x, c, mem_traits, pkLen); Y_ASSERT(pkLen); return reinterpret_cast<C*>(c); } diff --git a/library/cpp/packedtypes/longs_ut.cpp b/library/cpp/packedtypes/longs_ut.cpp index ae5fc54c25..8b06c934d2 100644 --- a/library/cpp/packedtypes/longs_ut.cpp +++ b/library/cpp/packedtypes/longs_ut.cpp @@ -9,60 +9,60 @@ Y_UNIT_TEST_SUITE(TLongsTest) { Y_UNIT_TEST(TestLongs) { - i16 x16 = 40; - i64 x64 = 40; - i64 y64; + i16 x16 = 40; + i64 x64 = 40; + i64 y64; TString s; - s += Sprintf("x16=0x%x\n", (int)x16); + s += Sprintf("x16=0x%x\n", (int)x16); s += Sprintf("LO_8(x16)=0x%x HI_8(x16)=0x%x\n\n", (int)Lo8(x16), (int)Hi8(x16)); - - char buf[100]; - memset(buf, 0, 100); + + char buf[100]; + memset(buf, 0, 100); char* p = buf; - int l = out_long(x64, buf); - s += Sprintf("x64=0x%" PRIi64 "\n", x64); + int l = out_long(x64, buf); + s += Sprintf("x64=0x%" PRIi64 "\n", x64); s += Sprintf("LO_32(x64)=0x%" PRIu32 " HI_32(x64)=0x%" PRIu32 "\n", (ui32)Lo32(x64), (ui32)Hi32(x64)); - s += Sprintf("buf=%s, l=%d: ", buf, l); - for (int i = 0; i < l; i++) { - s += Sprintf("0x%02x ", buf[i]); - } - s += Sprintf("\n"); + s += Sprintf("buf=%s, l=%d: ", buf, l); + for (int i = 0; i < l; i++) { + s += Sprintf("0x%02x ", buf[i]); + } + s += Sprintf("\n"); - p = buf; - in_long(y64, p); - s += Sprintf("x=0x%" PRIi64 " y=0x%" PRIi64 "\n", x64, y64); - if (x64 != y64) { - s += Sprintf("Error: y64 != x64\n"); - } else { - s += Sprintf("OK\n"); - } + p = buf; + in_long(y64, p); + s += Sprintf("x=0x%" PRIi64 " y=0x%" PRIi64 "\n", x64, y64); + if (x64 != y64) { + s += Sprintf("Error: y64 != x64\n"); + } else { + s += Sprintf("OK\n"); + } UNIT_ASSERT_EQUAL(Crc<ui64>(s.data(), s.size()), 7251624297500315779ULL); // WTF? - } - - template <typename TSignedInt> - void TestOneValue(TSignedInt value) { - char buffer[sizeof(TSignedInt) + 1]; - auto bytes = out_long(value, buffer); - TSignedInt readValue = 0; - auto readBytes = in_long(readValue, buffer); - UNIT_ASSERT_EQUAL(bytes, readBytes); - UNIT_ASSERT_EQUAL(bytes, len_long(value)); - UNIT_ASSERT_EQUAL(value, readValue); - } - - template <typename TSignedInt> - void TestCornerCasesImpl(int maxPow) { - for (int i = 0; i <= maxPow; ++i) { + } + + template <typename TSignedInt> + void TestOneValue(TSignedInt value) { + char buffer[sizeof(TSignedInt) + 1]; + auto bytes = out_long(value, buffer); + TSignedInt readValue = 0; + auto readBytes = in_long(readValue, buffer); + UNIT_ASSERT_EQUAL(bytes, readBytes); + UNIT_ASSERT_EQUAL(bytes, len_long(value)); + UNIT_ASSERT_EQUAL(value, readValue); + } + + template <typename TSignedInt> + void TestCornerCasesImpl(int maxPow) { + for (int i = 0; i <= maxPow; ++i) { TestOneValue<TSignedInt>((TSignedInt)(1ull << i)); TestOneValue<TSignedInt>((TSignedInt)((1ull << i) - 1)); TestOneValue<TSignedInt>((TSignedInt)((1ull << i) + 1)); } - } + } Y_UNIT_TEST(TestCornerCases) { - TestCornerCasesImpl<i32>(31); - TestCornerCasesImpl<i64>(63); - } -} + TestCornerCasesImpl<i32>(31); + TestCornerCasesImpl<i64>(63); + } +} diff --git a/library/cpp/packedtypes/ut/ya.make b/library/cpp/packedtypes/ut/ya.make index 357b03bdd4..a203115e71 100644 --- a/library/cpp/packedtypes/ut/ya.make +++ b/library/cpp/packedtypes/ut/ya.make @@ -13,7 +13,7 @@ SRCS( longs_ut.cpp packed_ut.cpp packedfloat_ut.cpp - zigzag_ut.cpp + zigzag_ut.cpp ) END() diff --git a/library/cpp/packedtypes/ya.make b/library/cpp/packedtypes/ya.make index 5f72331c7d..4c2c950619 100644 --- a/library/cpp/packedtypes/ya.make +++ b/library/cpp/packedtypes/ya.make @@ -15,7 +15,7 @@ SRCS( packed.h packedfloat.cpp packedfloat.h - zigzag.h + zigzag.h ) END() diff --git a/library/cpp/packedtypes/zigzag.h b/library/cpp/packedtypes/zigzag.h index 2adc715902..548403f838 100644 --- a/library/cpp/packedtypes/zigzag.h +++ b/library/cpp/packedtypes/zigzag.h @@ -1,21 +1,21 @@ -#pragma once - -#include <util/generic/typetraits.h> - +#pragma once + +#include <util/generic/typetraits.h> + #include <limits.h> -//! Convert signed values to unsigned. Convenient for further varint encoding. -//! See https://developers.google.com/protocol-buffers/docs/encoding#types for details. - -template <typename TSignedInt> +//! Convert signed values to unsigned. Convenient for further varint encoding. +//! See https://developers.google.com/protocol-buffers/docs/encoding#types for details. + +template <typename TSignedInt> inline auto ZigZagEncode(TSignedInt n) -> std::make_unsigned_t<TSignedInt> { static_assert(std::is_signed<TSignedInt>::value && std::is_integral<TSignedInt>::value, "Expected signed integral type."); auto un = static_cast<std::make_unsigned_t<TSignedInt>>(n); return (un << 1) ^ (n >> (CHAR_BIT * sizeof(TSignedInt) - 1)); -} - -template <typename TUnsignedInt> +} + +template <typename TUnsignedInt> inline auto ZigZagDecode(TUnsignedInt n) -> std::make_signed_t<TUnsignedInt> { static_assert(std::is_unsigned<TUnsignedInt>::value, "Expected unsigned integral type."); return (n >> 1) ^ -static_cast<std::make_signed_t<TUnsignedInt>>(n & 1); -} +} diff --git a/library/cpp/packedtypes/zigzag_ut.cpp b/library/cpp/packedtypes/zigzag_ut.cpp index cd3284d55e..13b78b7481 100644 --- a/library/cpp/packedtypes/zigzag_ut.cpp +++ b/library/cpp/packedtypes/zigzag_ut.cpp @@ -1,38 +1,38 @@ -#include "zigzag.h" - +#include "zigzag.h" + #include <library/cpp/testing/unittest/registar.h> - + Y_UNIT_TEST_SUITE(TZigZagTest) { - template <typename T> - void TestEncodeDecode(T value) { - auto encoded = ZigZagEncode(value); - UNIT_ASSERT(encoded >= 0); - auto decoded = ZigZagDecode(encoded); - UNIT_ASSERT(decoded == value); - static_assert(sizeof(value) == sizeof(decoded), "sizeof mismatch"); - static_assert(sizeof(value) == sizeof(encoded), "sizeof mismatch"); - } - - template <typename TSignedInt> - void TestImpl() { - static const int bits = CHAR_BIT * sizeof(TSignedInt); - for (int p = 0; p + 1 < bits; ++p) { - TSignedInt value = 1; - value <<= p; - - TestEncodeDecode(value); - TestEncodeDecode(-value); - TestEncodeDecode(value - 1); - TestEncodeDecode(-value + 1); - } - - TestEncodeDecode((TSignedInt)123); - TestEncodeDecode((TSignedInt)-123); - } - + template <typename T> + void TestEncodeDecode(T value) { + auto encoded = ZigZagEncode(value); + UNIT_ASSERT(encoded >= 0); + auto decoded = ZigZagDecode(encoded); + UNIT_ASSERT(decoded == value); + static_assert(sizeof(value) == sizeof(decoded), "sizeof mismatch"); + static_assert(sizeof(value) == sizeof(encoded), "sizeof mismatch"); + } + + template <typename TSignedInt> + void TestImpl() { + static const int bits = CHAR_BIT * sizeof(TSignedInt); + for (int p = 0; p + 1 < bits; ++p) { + TSignedInt value = 1; + value <<= p; + + TestEncodeDecode(value); + TestEncodeDecode(-value); + TestEncodeDecode(value - 1); + TestEncodeDecode(-value + 1); + } + + TestEncodeDecode((TSignedInt)123); + TestEncodeDecode((TSignedInt)-123); + } + Y_UNIT_TEST(TestSigned) { - TestImpl<i16>(); - TestImpl<i32>(); - TestImpl<i64>(); - } -} + TestImpl<i16>(); + TestImpl<i32>(); + TestImpl<i64>(); + } +} diff --git a/util/datetime/parser.rl6 b/util/datetime/parser.rl6 index 1f5e430c5c..931f09eae1 100644 --- a/util/datetime/parser.rl6 +++ b/util/datetime/parser.rl6 @@ -773,8 +773,8 @@ bool TDurationParser::ParsePart(const char* input, size_t len) { return cs != %%{ write error; }%%; } -static inline ui64 DecPower(ui64 part, i32 power) { - if (power >= 0) +static inline ui64 DecPower(ui64 part, i32 power) { + if (power >= 0) return part * Power(10, power); return part / Power(10, -power); } diff --git a/util/datetime/parser_ut.cpp b/util/datetime/parser_ut.cpp index 7c031e46a7..61364af997 100644 --- a/util/datetime/parser_ut.cpp +++ b/util/datetime/parser_ut.cpp @@ -587,11 +587,11 @@ Y_UNIT_TEST_SUITE(TDurationParseTest) { UNIT_ASSERT_VALUES_EQUAL(TDuration::MilliSeconds(7196400), TDuration::Parse("1.999h")); UNIT_ASSERT_VALUES_EQUAL(TDuration::MilliSeconds(7199640), TDuration::Parse("1.9999h")); - UNIT_ASSERT_EQUAL(TDuration::Minutes(15), TDuration::Parse("15m")); - UNIT_ASSERT_EQUAL(TDuration::Hours(10), TDuration::Parse("10h")); - UNIT_ASSERT_EQUAL(TDuration::Days(365), TDuration::Parse("365d")); - UNIT_ASSERT_EQUAL(TDuration::Hours(36), TDuration::Parse("1.5d")); - + UNIT_ASSERT_EQUAL(TDuration::Minutes(15), TDuration::Parse("15m")); + UNIT_ASSERT_EQUAL(TDuration::Hours(10), TDuration::Parse("10h")); + UNIT_ASSERT_EQUAL(TDuration::Days(365), TDuration::Parse("365d")); + UNIT_ASSERT_EQUAL(TDuration::Hours(36), TDuration::Parse("1.5d")); + UNIT_ASSERT_VALUES_EQUAL(TDuration::Hours(24), TDuration::Parse("1d")); UNIT_ASSERT_VALUES_EQUAL(TDuration::Hours(36), TDuration::Parse("1.5d")); UNIT_ASSERT_VALUES_EQUAL(TDuration::Minutes(2448), TDuration::Parse("1.7d")); @@ -618,10 +618,10 @@ Y_UNIT_TEST_SUITE(TDurationParseTest) { UNIT_ASSERT_VALUES_EQUAL(TDuration::Seconds(112), TDuration::Parse("112")); UNIT_ASSERT_VALUES_EQUAL(TDuration::MicroSeconds(14456), TDuration::Parse("14456us")); - - UNIT_ASSERT_VALUES_EQUAL(TDuration::MicroSeconds(1), TDuration::Parse("1000ns")); - UNIT_ASSERT_VALUES_EQUAL(TDuration::MicroSeconds(1), TDuration::Parse("0.000001s")); - - UNIT_ASSERT_EQUAL(TDuration(), TDuration::Parse("10ns")); // TDuration has 1us precision. + + UNIT_ASSERT_VALUES_EQUAL(TDuration::MicroSeconds(1), TDuration::Parse("1000ns")); + UNIT_ASSERT_VALUES_EQUAL(TDuration::MicroSeconds(1), TDuration::Parse("0.000001s")); + + UNIT_ASSERT_EQUAL(TDuration(), TDuration::Parse("10ns")); // TDuration has 1us precision. } } diff --git a/util/generic/vector.h b/util/generic/vector.h index 1725aa22b4..a5b258955a 100644 --- a/util/generic/vector.h +++ b/util/generic/vector.h @@ -124,7 +124,7 @@ public: } #endif - inline void crop(size_type size) { + inline void crop(size_type size) { if (this->size() > size) { this->erase(this->begin() + size, this->end()); } diff --git a/util/stream/hex.cpp b/util/stream/hex.cpp index 6d91cbe1cf..1c05330504 100644 --- a/util/stream/hex.cpp +++ b/util/stream/hex.cpp @@ -1,30 +1,30 @@ -#include "hex.h" - +#include "hex.h" + #include "output.h" -#include <util/string/hex.h> - +#include <util/string/hex.h> + void HexEncode(const void* in, size_t len, IOutputStream& out) { - static const size_t NUM_OF_BYTES = 32; - char buffer[NUM_OF_BYTES * 2]; - - auto current = static_cast<const char*>(in); - for (size_t take = 0; len; current += take, len -= take) { - take = Min(NUM_OF_BYTES, len); - HexEncode(current, take, buffer); - out.Write(buffer, take * 2); - } -} - + static const size_t NUM_OF_BYTES = 32; + char buffer[NUM_OF_BYTES * 2]; + + auto current = static_cast<const char*>(in); + for (size_t take = 0; len; current += take, len -= take) { + take = Min(NUM_OF_BYTES, len); + HexEncode(current, take, buffer); + out.Write(buffer, take * 2); + } +} + void HexDecode(const void* in, size_t len, IOutputStream& out) { Y_ENSURE(!(len & 1), TStringBuf("Odd buffer length passed to HexDecode")); - - static const size_t NUM_OF_BYTES = 32; - char buffer[NUM_OF_BYTES]; - - auto current = static_cast<const char*>(in); - for (size_t take = 0; len; current += take, len -= take) { - take = Min(NUM_OF_BYTES * 2, len); - HexDecode(current, take, buffer); - out.Write(buffer, take / 2); - } -} + + static const size_t NUM_OF_BYTES = 32; + char buffer[NUM_OF_BYTES]; + + auto current = static_cast<const char*>(in); + for (size_t take = 0; len; current += take, len -= take) { + take = Min(NUM_OF_BYTES * 2, len); + HexDecode(current, take, buffer); + out.Write(buffer, take / 2); + } +} diff --git a/util/stream/hex.h b/util/stream/hex.h index 68451d781f..a018933b1b 100644 --- a/util/stream/hex.h +++ b/util/stream/hex.h @@ -1,8 +1,8 @@ -#pragma once - -#include <util/system/types.h> - +#pragma once + +#include <util/system/types.h> + class IOutputStream; - + void HexEncode(const void* in, size_t len, IOutputStream& out); void HexDecode(const void* in, size_t len, IOutputStream& out); diff --git a/util/stream/hex_ut.cpp b/util/stream/hex_ut.cpp index 1a6bf5a6d3..5074a0b616 100644 --- a/util/stream/hex_ut.cpp +++ b/util/stream/hex_ut.cpp @@ -1,29 +1,29 @@ -#include "hex.h" - +#include "hex.h" + #include <library/cpp/testing/unittest/registar.h> #include "str.h" - + Y_UNIT_TEST_SUITE(THexCodingTest) { void TestImpl(const TString& data) { TString encoded; - TStringOutput encodedOut(encoded); + TStringOutput encodedOut(encoded); HexEncode(data.data(), data.size(), encodedOut); - + UNIT_ASSERT_EQUAL(encoded.size(), data.size() * 2); - + TString decoded; - TStringOutput decodedOut(decoded); + TStringOutput decodedOut(decoded); HexDecode(encoded.data(), encoded.size(), decodedOut); - - UNIT_ASSERT_EQUAL(decoded, data); - } - + + UNIT_ASSERT_EQUAL(decoded, data); + } + Y_UNIT_TEST(TestEncodeDecodeToStream) { TString data = "100ABAcaba500,$%0987123456 \n\t\x01\x02\x03."; - TestImpl(data); - } - + TestImpl(data); + } + Y_UNIT_TEST(TestEmpty) { - TestImpl(""); - } -} + TestImpl(""); + } +} diff --git a/util/string/escape.cpp b/util/string/escape.cpp index bbe8bbd0d2..cd09a7dbd0 100644 --- a/util/string/escape.cpp +++ b/util/string/escape.cpp @@ -242,9 +242,9 @@ static TStr& DoUnescapeC(const TChar* p, size_t sz, TStr& res) { default: res.append(*p); break; - case 'a': - res.append('\a'); - break; + case 'a': + res.append('\a'); + break; case 'b': res.append('\b'); break; @@ -260,9 +260,9 @@ static TStr& DoUnescapeC(const TChar* p, size_t sz, TStr& res) { case 't': res.append('\t'); break; - case 'v': - res.append('\v'); - break; + case 'v': + res.append('\v'); + break; case 'u': { ui16 cp[2]; @@ -318,13 +318,13 @@ static TStr& DoUnescapeC(const TChar* p, size_t sz, TStr& res) { } ++p; - } else { + } else { const auto r = std::basic_string_view<TChar>(p, pe - p).find('\\'); const auto n = r != std::string::npos ? p + r : pe; res.append(p, n); p = n; - } + } } return res; @@ -363,43 +363,43 @@ template TUtf16String& UnescapeCImpl<TUtf16String::TChar>(const TUtf16String::TC template char* UnescapeC<char>(const char* str, size_t len, char* buf); -template <class TChar> -size_t UnescapeCCharLen(const TChar* begin, const TChar* end) { +template <class TChar> +size_t UnescapeCCharLen(const TChar* begin, const TChar* end) { if (begin >= end) { - return 0; + return 0; } if (*begin != '\\') { - return 1; + return 1; } if (++begin == end) { - return 1; + return 1; } - - switch (*begin) { - default: - return 2; - case 'u': - return CountHex<4>(begin + 1, end) == 4 ? 6 : 2; - case 'U': - return CountHex<8>(begin + 1, end) == 8 ? 10 : 2; - case 'x': - return 2 + CountHex<2>(begin + 1, end); - case '0': - case '1': - case '2': - case '3': - return 1 + CountOct<3>(begin, end); // >= 2 - case '4': - case '5': - case '6': - case '7': - return 1 + CountOct<2>(begin, end); // >= 2 - } -} - -template size_t UnescapeCCharLen<char>(const char* begin, const char* end); + + switch (*begin) { + default: + return 2; + case 'u': + return CountHex<4>(begin + 1, end) == 4 ? 6 : 2; + case 'U': + return CountHex<8>(begin + 1, end) == 8 ? 10 : 2; + case 'x': + return 2 + CountHex<2>(begin + 1, end); + case '0': + case '1': + case '2': + case '3': + return 1 + CountOct<3>(begin, end); // >= 2 + case '4': + case '5': + case '6': + case '7': + return 1 + CountOct<2>(begin, end); // >= 2 + } +} + +template size_t UnescapeCCharLen<char>(const char* begin, const char* end); template size_t UnescapeCCharLen<TUtf16String::TChar>(const TUtf16String::TChar* begin, const TUtf16String::TChar* end); - + TString& EscapeC(const TStringBuf str, TString& s) { return EscapeC(str.data(), str.size(), s); } diff --git a/util/string/escape.h b/util/string/escape.h index 9ab9cf574d..b01be65b0e 100644 --- a/util/string/escape.h +++ b/util/string/escape.h @@ -61,10 +61,10 @@ TUtf16String& UnescapeC(const TWtringBuf str, TUtf16String& res); TString UnescapeC(const TStringBuf str); TUtf16String UnescapeC(const TWtringBuf wtr); - -/// Returns number of chars in escape sequence. -/// - 0, if begin >= end -/// - 1, if [begin, end) starts with an unescaped char -/// - at least 2 (including '\'), if [begin, end) starts with an escaped symbol -template <class TChar> -size_t UnescapeCCharLen(const TChar* begin, const TChar* end); + +/// Returns number of chars in escape sequence. +/// - 0, if begin >= end +/// - 1, if [begin, end) starts with an unescaped char +/// - at least 2 (including '\'), if [begin, end) starts with an escaped symbol +template <class TChar> +size_t UnescapeCCharLen(const TChar* begin, const TChar* end); diff --git a/util/string/escape_ut.cpp b/util/string/escape_ut.cpp index c3e6e3a05b..cd38ecffd3 100644 --- a/util/string/escape_ut.cpp +++ b/util/string/escape_ut.cpp @@ -103,33 +103,33 @@ Y_UNIT_TEST_SUITE(TEscapeCTest) { // UNIT_ASSERT_VALUES_EQUAL("[x]?z", EscapeC(TString("??(x??)?z"))); UNIT_ASSERT_VALUES_EQUAL("\\x3F?x\\x3F\\x3F?z", EscapeC(TString("??x???z"))); } - + Y_UNIT_TEST(TestUnescapeCCharLen) { - auto test = [](const char* str, size_t len) { - UNIT_ASSERT_EQUAL(UnescapeCCharLen(str, str + strlen(str)), len); - }; - - test("", 0); - test("abc", 1); - test("\\", 1); - test("\\\\", 2); - test("\\#", 2); - test("\\n10", 2); - test("\\r\\n", 2); - test("\\x05abc", 4); - test("\\u11117777", 6); - test("\\u123yyy", 2); - test("\\U11117777cccc", 10); - test("\\U111yyy", 2); - test("\\0\\1", 2); - test("\\01\\1", 3); - test("\\012\\1", 4); - test("\\0123\\1", 4); - test("\\4\\1", 2); - test("\\40\\1", 3); - test("\\400\\1", 3); - test("\\4xxx", 2); - } + auto test = [](const char* str, size_t len) { + UNIT_ASSERT_EQUAL(UnescapeCCharLen(str, str + strlen(str)), len); + }; + + test("", 0); + test("abc", 1); + test("\\", 1); + test("\\\\", 2); + test("\\#", 2); + test("\\n10", 2); + test("\\r\\n", 2); + test("\\x05abc", 4); + test("\\u11117777", 6); + test("\\u123yyy", 2); + test("\\U11117777cccc", 10); + test("\\U111yyy", 2); + test("\\0\\1", 2); + test("\\01\\1", 3); + test("\\012\\1", 4); + test("\\0123\\1", 4); + test("\\4\\1", 2); + test("\\40\\1", 3); + test("\\400\\1", 3); + test("\\4xxx", 2); + } Y_UNIT_TEST(TestUnbounded) { char buf[100000]; diff --git a/util/string/hex.cpp b/util/string/hex.cpp index 75aa031639..667397987f 100644 --- a/util/string/hex.cpp +++ b/util/string/hex.cpp @@ -56,8 +56,8 @@ TString HexEncode(const void* in, size_t len) { TString HexDecode(const void* in, size_t len) { TString ret; - ret.ReserveAndResize(len >> 1); - HexDecode(in, len, ret.begin()); + ret.ReserveAndResize(len >> 1); + HexDecode(in, len, ret.begin()); return ret; } diff --git a/util/string/hex.h b/util/string/hex.h index 81f9dd42ec..af3d2d528f 100644 --- a/util/string/hex.h +++ b/util/string/hex.h @@ -32,8 +32,8 @@ TString HexEncode(const void* in, size_t len); inline TString HexEncode(const TStringBuf h) { return HexEncode(h.data(), h.size()); -} - +} + //! Convert a hex string @c in of @c len chars (case-insensitive) to array of ints stored at @c ptr and return this array. /*! @note len must be even (len % 2 == 0), otherwise an exception will be thrown. * @return @c ptr, which is an array of chars, where each char holds the numeric value @@ -52,8 +52,8 @@ void* HexDecode(const void* in, size_t len, void* ptr); * @example HexDecode("beef", 4) => {190, 239} */ TString HexDecode(const void* in, size_t len); - + //! Convert an ASCII hex-string (case-insensitive) to the binary form. Note that h.Size() must be even (+h % 2 == 0). inline TString HexDecode(const TStringBuf h) { return HexDecode(h.data(), h.size()); -} +} diff --git a/ydb/core/client/server/msgbus_server_proxy.cpp b/ydb/core/client/server/msgbus_server_proxy.cpp index 5b9546e4c4..7f3146d21a 100644 --- a/ydb/core/client/server/msgbus_server_proxy.cpp +++ b/ydb/core/client/server/msgbus_server_proxy.cpp @@ -143,8 +143,8 @@ TBusResponse* ProposeTransactionStatusToResponse(EResponseStatus status, if (result.HasSchemeShardReportedId()) response->Record.SetSchemeTagId(result.GetSchemeShardReportedId()); - if (result.HasTimings()) - response->Record.MutableProxyTimings()->CopyFrom(result.GetTimings()); + if (result.HasTimings()) + response->Record.MutableProxyTimings()->CopyFrom(result.GetTimings()); return response.Release(); } diff --git a/ydb/core/driver_lib/base_utils/format_info.cpp b/ydb/core/driver_lib/base_utils/format_info.cpp index 5ea2d39c37..e5d53a7b80 100644 --- a/ydb/core/driver_lib/base_utils/format_info.cpp +++ b/ydb/core/driver_lib/base_utils/format_info.cpp @@ -66,7 +66,7 @@ void TCmdFormatInfoConfig::Parse(int argc, char **argv) { using namespace NLastGetopt; TOpts opts = TOpts::Default(); - opts.AddLongOption('p', "pdisk-path", "path to pdisk to read format info").RequiredArgument("PATH").Required() + opts.AddLongOption('p', "pdisk-path", "path to pdisk to read format info").RequiredArgument("PATH").Required() .StoreResult(&Path); opts.AddLongOption('k', "main-key", "encryption main-key to use while reading").RequiredArgument("NUM") .Optional().StoreResult(&MainKey); // TODO: make required diff --git a/ydb/core/driver_lib/base_utils/format_util.cpp b/ydb/core/driver_lib/base_utils/format_util.cpp index 5cd3aea6a1..e6f99b1634 100644 --- a/ydb/core/driver_lib/base_utils/format_util.cpp +++ b/ydb/core/driver_lib/base_utils/format_util.cpp @@ -46,7 +46,7 @@ void TCmdFormatUtilConfig::Parse(int argc, char **argv) { using namespace NLastGetopt; TOpts opts = TOpts::Default(); - opts.AddLongOption("format-file", "Blob storage fromat config file").RequiredArgument("PATH").Required() + opts.AddLongOption("format-file", "Blob storage fromat config file").RequiredArgument("PATH").Required() .StoreResult(&FormatFile); opts.AddLongOption('n', "node-id", "Node ID to get drive info for").RequiredArgument("NUM").Required() .StoreResult(&NodeId); diff --git a/ydb/core/driver_lib/base_utils/node_by_host.cpp b/ydb/core/driver_lib/base_utils/node_by_host.cpp index 1ef7896a6e..81344f4dae 100644 --- a/ydb/core/driver_lib/base_utils/node_by_host.cpp +++ b/ydb/core/driver_lib/base_utils/node_by_host.cpp @@ -48,7 +48,7 @@ void TCmdNodeByHostConfig::Parse(int argc, char **argv) { using namespace NLastGetopt; TOpts opts = TOpts::Default(); - opts.AddLongOption("naming-file", "static nameservice config file").RequiredArgument("PATH").Required() + opts.AddLongOption("naming-file", "static nameservice config file").RequiredArgument("PATH").Required() .StoreResult(&NamingFile); opts.AddLongOption('n', "hostname", "Hostname to detect node id for").RequiredArgument("STR").Required() .StoreResult(&Hostname); diff --git a/ydb/core/driver_lib/cli_utils/cli_minikql_compile_and_exec.cpp b/ydb/core/driver_lib/cli_utils/cli_minikql_compile_and_exec.cpp index afc315228f..b4429d5721 100644 --- a/ydb/core/driver_lib/cli_utils/cli_minikql_compile_and_exec.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_minikql_compile_and_exec.cpp @@ -143,10 +143,10 @@ void TCmdCompileAndExecMiniKQLConfig::Parse(int argc, char **argv) { using namespace NLastGetopt; TOpts opts = TOpts::Default(); - opts.AddLongOption('f', "pgm-txt", "path to file with text program").RequiredArgument("PATH").StoreResult(&PathToTextPgm); - opts.AddLongOption('b', "pgm-bin", "path to file with serialized program").RequiredArgument("PATH").StoreResult(&PathToBinPgm); - opts.AddLongOption("params-txt", "path to file with text program params").RequiredArgument("PATH").StoreResult(&PathToTextParams); - opts.AddLongOption("params-bin", "path to file with serialized program params").RequiredArgument("PATH").StoreResult(&PathToBinParams); + opts.AddLongOption('f', "pgm-txt", "path to file with text program").RequiredArgument("PATH").StoreResult(&PathToTextPgm); + opts.AddLongOption('b', "pgm-bin", "path to file with serialized program").RequiredArgument("PATH").StoreResult(&PathToBinPgm); + opts.AddLongOption("params-txt", "path to file with text program params").RequiredArgument("PATH").StoreResult(&PathToTextParams); + opts.AddLongOption("params-bin", "path to file with serialized program params").RequiredArgument("PATH").StoreResult(&PathToBinParams); ConfigureBaseLastGetopt(opts); diff --git a/ydb/core/driver_lib/cli_utils/cli_scheme_initroot.cpp b/ydb/core/driver_lib/cli_utils/cli_scheme_initroot.cpp index e6dbcdf267..ebca272cc4 100644 --- a/ydb/core/driver_lib/cli_utils/cli_scheme_initroot.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_scheme_initroot.cpp @@ -71,7 +71,7 @@ void TCmdSchemeInitShardConfig::Parse(int argc, char **argv) { TOpts opts = TOpts::Default(); opts.AddLongOption('n', "name", "domain name").Required().RequiredArgument("STR").StoreResult(&TagName); opts.AddLongOption("config", "apply global config").RequiredArgument("STR").StoreResult(&configPb); - opts.AddLongOption("config-file", "load global config from file").RequiredArgument("PATH").StoreResult(&configPbFile); + opts.AddLongOption("config-file", "load global config from file").RequiredArgument("PATH").StoreResult(&configPbFile); ConfigureBaseLastGetopt(opts); TOptsParseResult res(&opts, argc, argv); diff --git a/ydb/core/persqueue/type_codecs.h b/ydb/core/persqueue/type_codecs.h index 2499c45653..788e0fce0e 100644 --- a/ydb/core/persqueue/type_codecs.h +++ b/ydb/core/persqueue/type_codecs.h @@ -1,69 +1,69 @@ -#pragma once - -#include "type_codecs_defs.h" -#include "type_coders.h" -#include "type_decoders.h" - +#pragma once + +#include "type_codecs_defs.h" +#include "type_coders.h" +#include "type_decoders.h" + namespace NKikimr { namespace NScheme { - -template <typename TCoder, typename TDecoder> -class TCodecImpl : public ICodec { -public: + +template <typename TCoder, typename TDecoder> +class TCodecImpl : public ICodec { +public: static inline TCodecSig Sig() { return TCoder::Sig(); } - -public: - TCodecImpl() { + +public: + TCodecImpl() { Y_VERIFY(TCoder::Sig() == TDecoder::Sig(), "Codecs signatures mismatch (cd: %u, dc: %u).", ui16(TCoder::Sig()), ui16(TDecoder::Sig())); - } - - TCodecSig Signature() const override { + } + + TCodecSig Signature() const override { Y_VERIFY_DEBUG(TCoder::Sig() == TDecoder::Sig()); return TCoder::Sig(); - } - - TAutoPtr<IChunkCoder> MakeChunk(TFlatBlobDataOutputStream* output) const override { - return new TCoder(output); - } - - IChunkDecoder::TPtr ReadChunk(const TDataRef& data) const override { - return new TDecoder(data); - } - - IChunkDecoder::TPtr ReadChunk(const TDataRef& data, const TTypeCodecs* codecs) const override { - return TDecoder::ReadNextImpl(data, codecs); - } -}; - -//////////////////////////////////////////////////////////////////////////////// - -template <size_t Size, bool IsNullable> -class TFixedLenCodec : public TCodecImpl<TFixedLenCoder<Size, IsNullable>, TFixedLenDecoder<Size, IsNullable>> { }; - -template <bool IsNullable> -class TVarLenCodec : public TCodecImpl<TVarLenCoder<IsNullable>, TVarLenDecoder<IsNullable>> { }; - -template <typename TIntType, bool IsNullable> -class TVarIntCodec : public TCodecImpl<TVarIntCoder<TIntType, IsNullable>, TVarIntDecoder<TIntType, IsNullable>> { }; - -template <typename TIntType, bool IsNullable> -class TZigZagCodec : public TCodecImpl<TZigZagCoder<TIntType, IsNullable>, TZigZagDecoder<TIntType, IsNullable>> { }; - -template <typename TIntType, bool IsNullable> -class TDeltaVarIntCodec : public TCodecImpl<TDeltaVarIntCoder<TIntType, IsNullable>, TDeltaVarIntDecoder<TIntType, IsNullable>> { }; - -template <typename TIntType, bool IsNullable> -class TDeltaRevVarIntCodec : public TCodecImpl<TDeltaRevVarIntCoder<TIntType, IsNullable>, TDeltaRevVarIntDecoder<TIntType, IsNullable>> { }; - -template <typename TIntType, bool IsNullable> -class TDeltaZigZagCodec : public TCodecImpl<TDeltaZigZagCoder<TIntType, IsNullable>, TDeltaZigZagDecoder<TIntType, IsNullable>> { }; - -template <bool IsNullable> -class TBoolCodec : public TCodecImpl<TBoolCoder<IsNullable>, TBoolDecoder<IsNullable>> { }; - + } + + TAutoPtr<IChunkCoder> MakeChunk(TFlatBlobDataOutputStream* output) const override { + return new TCoder(output); + } + + IChunkDecoder::TPtr ReadChunk(const TDataRef& data) const override { + return new TDecoder(data); + } + + IChunkDecoder::TPtr ReadChunk(const TDataRef& data, const TTypeCodecs* codecs) const override { + return TDecoder::ReadNextImpl(data, codecs); + } +}; + +//////////////////////////////////////////////////////////////////////////////// + +template <size_t Size, bool IsNullable> +class TFixedLenCodec : public TCodecImpl<TFixedLenCoder<Size, IsNullable>, TFixedLenDecoder<Size, IsNullable>> { }; + +template <bool IsNullable> +class TVarLenCodec : public TCodecImpl<TVarLenCoder<IsNullable>, TVarLenDecoder<IsNullable>> { }; + +template <typename TIntType, bool IsNullable> +class TVarIntCodec : public TCodecImpl<TVarIntCoder<TIntType, IsNullable>, TVarIntDecoder<TIntType, IsNullable>> { }; + +template <typename TIntType, bool IsNullable> +class TZigZagCodec : public TCodecImpl<TZigZagCoder<TIntType, IsNullable>, TZigZagDecoder<TIntType, IsNullable>> { }; + +template <typename TIntType, bool IsNullable> +class TDeltaVarIntCodec : public TCodecImpl<TDeltaVarIntCoder<TIntType, IsNullable>, TDeltaVarIntDecoder<TIntType, IsNullable>> { }; + +template <typename TIntType, bool IsNullable> +class TDeltaRevVarIntCodec : public TCodecImpl<TDeltaRevVarIntCoder<TIntType, IsNullable>, TDeltaRevVarIntDecoder<TIntType, IsNullable>> { }; + +template <typename TIntType, bool IsNullable> +class TDeltaZigZagCodec : public TCodecImpl<TDeltaZigZagCoder<TIntType, IsNullable>, TDeltaZigZagDecoder<TIntType, IsNullable>> { }; + +template <bool IsNullable> +class TBoolCodec : public TCodecImpl<TBoolCoder<IsNullable>, TBoolDecoder<IsNullable>> { }; + } // namespace NScheme } // namespace NKikimr - + diff --git a/ydb/core/persqueue/type_codecs_defs.cpp b/ydb/core/persqueue/type_codecs_defs.cpp index ab35da6846..f333be5cfc 100644 --- a/ydb/core/persqueue/type_codecs_defs.cpp +++ b/ydb/core/persqueue/type_codecs_defs.cpp @@ -1,112 +1,112 @@ -#include "type_codecs.h" - +#include "type_codecs.h" + #include <ydb/core/scheme/scheme_types_defs.h> -#include <util/generic/typetraits.h> - +#include <util/generic/typetraits.h> + namespace NKikimr { namespace NScheme { - -void InitDefaults(TTypeCodecs* codecs, TCodecType type) { - auto nullable = TCodecSig(type, true); - auto nonNullable = TCodecSig(type, false); - - codecs->AddAlias(TCodecSig(TCodecType::Default, true), nullable); - codecs->AddAlias(TCodecSig(TCodecType::Default, false), nonNullable); - - codecs->AddAlias(TCodecSig(TCodecType::ZeroCopy, true), nullable); - codecs->AddAlias(TCodecSig(TCodecType::ZeroCopy, false), nonNullable); - codecs->AddAlias(TCodecSig(TCodecType::Compact, true), nullable); - codecs->AddAlias(TCodecSig(TCodecType::Compact, false), nonNullable); - codecs->AddAlias(TCodecSig(TCodecType::RandomAccess, true), nullable); - codecs->AddAlias(TCodecSig(TCodecType::RandomAccess, false), nonNullable); - - // codecs->AddAlias(TCodecSig(TCodecType::Adaptive, true), nullable); - // codecs->AddAlias(TCodecSig(TCodecType::Adaptive, false), nonNullable); -} - -template <size_t Size> -inline void AddFixedLen(TTypeCodecs* codecs, bool initDefaults = true) { - codecs->AddCodec<TFixedLenCodec<Size, true>>(); - codecs->AddCodec<TFixedLenCodec<Size, false>>(); - if (initDefaults) - InitDefaults(codecs, TCodecType::FixedLen); -}; - -template <typename TType> -inline void AddFixedLen(TTypeCodecs* codecs, bool initDefaults = true) { - AddFixedLen<sizeof(typename TType::TValueType)>(codecs, initDefaults); -} - -template <typename TType> -void AddIntCodecs(TTypeCodecs* codecs) { + +void InitDefaults(TTypeCodecs* codecs, TCodecType type) { + auto nullable = TCodecSig(type, true); + auto nonNullable = TCodecSig(type, false); + + codecs->AddAlias(TCodecSig(TCodecType::Default, true), nullable); + codecs->AddAlias(TCodecSig(TCodecType::Default, false), nonNullable); + + codecs->AddAlias(TCodecSig(TCodecType::ZeroCopy, true), nullable); + codecs->AddAlias(TCodecSig(TCodecType::ZeroCopy, false), nonNullable); + codecs->AddAlias(TCodecSig(TCodecType::Compact, true), nullable); + codecs->AddAlias(TCodecSig(TCodecType::Compact, false), nonNullable); + codecs->AddAlias(TCodecSig(TCodecType::RandomAccess, true), nullable); + codecs->AddAlias(TCodecSig(TCodecType::RandomAccess, false), nonNullable); + + // codecs->AddAlias(TCodecSig(TCodecType::Adaptive, true), nullable); + // codecs->AddAlias(TCodecSig(TCodecType::Adaptive, false), nonNullable); +} + +template <size_t Size> +inline void AddFixedLen(TTypeCodecs* codecs, bool initDefaults = true) { + codecs->AddCodec<TFixedLenCodec<Size, true>>(); + codecs->AddCodec<TFixedLenCodec<Size, false>>(); + if (initDefaults) + InitDefaults(codecs, TCodecType::FixedLen); +}; + +template <typename TType> +inline void AddFixedLen(TTypeCodecs* codecs, bool initDefaults = true) { + AddFixedLen<sizeof(typename TType::TValueType)>(codecs, initDefaults); +} + +template <typename TType> +void AddIntCodecs(TTypeCodecs* codecs) { static_assert(std::is_integral<typename TType::TValueType>::value, "Not an integral type."); using TSigned = std::make_signed_t<typename TType::TValueType>; using TUnsigned = std::make_unsigned_t<typename TType::TValueType>; - - codecs->AddCodec<TVarIntCodec<TUnsigned, true>>(); - codecs->AddCodec<TVarIntCodec<TUnsigned, false>>(); - codecs->AddCodec<TZigZagCodec<TSigned, true>>(); - codecs->AddCodec<TZigZagCodec<TSigned, false>>(); - codecs->AddCodec<TDeltaVarIntCodec<TUnsigned, true>>(); - codecs->AddCodec<TDeltaVarIntCodec<TUnsigned, false>>(); - codecs->AddCodec<TDeltaRevVarIntCodec<TUnsigned, true>>(); - codecs->AddCodec<TDeltaRevVarIntCodec<TUnsigned, false>>(); - codecs->AddCodec<TDeltaZigZagCodec<TSigned, true>>(); - codecs->AddCodec<TDeltaZigZagCodec<TSigned, false>>(); - + + codecs->AddCodec<TVarIntCodec<TUnsigned, true>>(); + codecs->AddCodec<TVarIntCodec<TUnsigned, false>>(); + codecs->AddCodec<TZigZagCodec<TSigned, true>>(); + codecs->AddCodec<TZigZagCodec<TSigned, false>>(); + codecs->AddCodec<TDeltaVarIntCodec<TUnsigned, true>>(); + codecs->AddCodec<TDeltaVarIntCodec<TUnsigned, false>>(); + codecs->AddCodec<TDeltaRevVarIntCodec<TUnsigned, true>>(); + codecs->AddCodec<TDeltaRevVarIntCodec<TUnsigned, false>>(); + codecs->AddCodec<TDeltaZigZagCodec<TSigned, true>>(); + codecs->AddCodec<TDeltaZigZagCodec<TSigned, false>>(); + if (std::is_signed<typename TType::TValueType>::value) { - codecs->AddAlias(TCodecSig(TCodecType::Compact, true), TCodecSig(TCodecType::ZigZag, true), true); - codecs->AddAlias(TCodecSig(TCodecType::Compact, false), TCodecSig(TCodecType::ZigZag, false), true); + codecs->AddAlias(TCodecSig(TCodecType::Compact, true), TCodecSig(TCodecType::ZigZag, true), true); + codecs->AddAlias(TCodecSig(TCodecType::Compact, false), TCodecSig(TCodecType::ZigZag, false), true); } else { // std::is_unsigned<typename TType::TValueType>::value) - codecs->AddAlias(TCodecSig(TCodecType::Compact, true), TCodecSig(TCodecType::VarInt, true), true); - codecs->AddAlias(TCodecSig(TCodecType::Compact, false), TCodecSig(TCodecType::VarInt, false), true); - } -} - + codecs->AddAlias(TCodecSig(TCodecType::Compact, true), TCodecSig(TCodecType::VarInt, true), true); + codecs->AddAlias(TCodecSig(TCodecType::Compact, false), TCodecSig(TCodecType::VarInt, false), true); + } +} + TTypeCodecs::TTypeCodecs(TTypeId typeId) { - using namespace NScheme; - AddCodec<TVarLenCodec<true>>(); - AddCodec<TVarLenCodec<false>>(); - - switch (typeId) { + using namespace NScheme; + AddCodec<TVarLenCodec<true>>(); + AddCodec<TVarLenCodec<false>>(); + + switch (typeId) { case NTypeIds::Int32: - AddFixedLen<TInt32>(this); - AddIntCodecs<TInt32>(this); - break; + AddFixedLen<TInt32>(this); + AddIntCodecs<TInt32>(this); + break; case NTypeIds::Uint32: - AddFixedLen<TUint32>(this); - AddIntCodecs<TUint32>(this); - break; + AddFixedLen<TUint32>(this); + AddIntCodecs<TUint32>(this); + break; case NTypeIds::Int64: - AddFixedLen<TInt64>(this); - AddIntCodecs<TInt64>(this); - break; + AddFixedLen<TInt64>(this); + AddIntCodecs<TInt64>(this); + break; case NTypeIds::Uint64: - AddFixedLen<TUint64>(this); - AddIntCodecs<TUint64>(this); - break; + AddFixedLen<TUint64>(this); + AddIntCodecs<TUint64>(this); + break; case NTypeIds::Byte: AddFixedLen<TUint8>(this); - break; + break; case NTypeIds::Bool: - AddFixedLen<TBool>(this, false); - AddCodec<TBoolCodec<true>>(); - AddCodec<TBoolCodec<false>>(); - InitDefaults(this, TCodecType::Bool); - break; - + AddFixedLen<TBool>(this, false); + AddCodec<TBoolCodec<true>>(); + AddCodec<TBoolCodec<false>>(); + InitDefaults(this, TCodecType::Bool); + break; + case NTypeIds::Double: - AddFixedLen<TDouble>(this); - break; + AddFixedLen<TDouble>(this); + break; case NTypeIds::Float: - AddFixedLen<TFloat>(this); - break; - + AddFixedLen<TFloat>(this); + break; + case NTypeIds::PairUi64Ui64: - AddFixedLen<TPairUi64Ui64>(this); - break; - + AddFixedLen<TPairUi64Ui64>(this); + break; + case NTypeIds::String: case NTypeIds::String4k: case NTypeIds::String2m: @@ -114,15 +114,15 @@ TTypeCodecs::TTypeCodecs(TTypeId typeId) { case NTypeIds::Json: case NTypeIds::JsonDocument: case NTypeIds::DyNumber: - InitDefaults(this, TCodecType::VarLen); - break; - + InitDefaults(this, TCodecType::VarLen); + break; + case NTypeIds::ActorId: AddFixedLen<TActorId>(this); - break; + break; case NTypeIds::StepOrderId: - AddFixedLen<TStepOrderId>(this); - break; + AddFixedLen<TStepOrderId>(this); + break; case NTypeIds::Decimal: AddFixedLen<TDecimal>(this); break; @@ -139,8 +139,8 @@ TTypeCodecs::TTypeCodecs(TTypeId typeId) { case NTypeIds::Interval: AddFixedLen<TInterval>(this); break; - } -} - + } +} + } // namespace NScheme } // namespace NKikimr diff --git a/ydb/core/persqueue/type_codecs_defs.h b/ydb/core/persqueue/type_codecs_defs.h index 1581c2949a..90c55d4b89 100644 --- a/ydb/core/persqueue/type_codecs_defs.h +++ b/ydb/core/persqueue/type_codecs_defs.h @@ -1,391 +1,391 @@ -#pragma once - +#pragma once + #include <ydb/core/scheme/scheme_type_id.h> #include <ydb/core/util/blob_data_stream.h> - -#include <util/generic/hash.h> -#include <util/generic/ptr.h> -#include <util/generic/singleton.h> -#include <util/system/types.h> + +#include <util/generic/hash.h> +#include <util/generic/ptr.h> +#include <util/generic/singleton.h> +#include <util/system/types.h> #include <util/system/unaligned_mem.h> - -#include <string.h> - + +#include <string.h> + namespace NKikimr { namespace NScheme { - -class TDataRef; -class TTypeCodecs; - -/***************************************************************************//** - * TCodecType identifies an algorithm of data serialization/deserialization. - ******************************************************************************/ -enum class TCodecType : ui16 { - Unset = 0, // Use default writer from a different source (e.g. doc-wise). - - // Universal IType-dependent aliases. - - Default = 1, // Flat as-is writer, no deserialization required on read. - ZeroCopy, // Decoders return refs on chunk buffer data. - Compact, // Good compression, might require data unpacking on read. - RandomAccess, // Decoder supports random access to values out of the box. - Adaptive, // Best compression algo is chosen on finishing the chunk. - - // Type-specific codecs. - - FixedLen = 10, // For types with a fixed byte size. - - VarLen = 20, // E.g. for strings, blobs. - VarLenVarInt = 23, // Uses VarInt to save sizes. TODO - - VarInt = 30, // Unsigned ints. - DeltaVarInt = 31, // Increasing unsigned ints. - DeltaRevVarInt = 32, // Decreasing unsigned ints. - ZigZag = 40, // Signed ints. - DeltaZigZag = 41, // Monotone or densely distributed signed ints. - // TODO: Add some bit-wise int compression algorithms. - - Bool = 50, // 1 bit per bool (2 bits per nullable bool). - - FixedLenLz = 60, // TODO - VarLenLz = 61, // TODO - // TODO: Add gzip/snappy or some other general-case compression algos. - - MaxId = 0x3FF, // 10 bits -}; - -/***************************************************************************//** - * TCodecSig identifies a concrete instance of a codec within a concrete IType - * context. - * - * Internal structure: - * --------------------------------------------- - * | Reserved : 5 | IsNullable : 1 | Type : 10 | - * --------------------------------------------- - * - * NOTE: Could not use union of ui16 and bitfield with constexpr constructor. - ******************************************************************************/ -#pragma pack(push, 2) -class TCodecSig { -public: + +class TDataRef; +class TTypeCodecs; + +/***************************************************************************//** + * TCodecType identifies an algorithm of data serialization/deserialization. + ******************************************************************************/ +enum class TCodecType : ui16 { + Unset = 0, // Use default writer from a different source (e.g. doc-wise). + + // Universal IType-dependent aliases. + + Default = 1, // Flat as-is writer, no deserialization required on read. + ZeroCopy, // Decoders return refs on chunk buffer data. + Compact, // Good compression, might require data unpacking on read. + RandomAccess, // Decoder supports random access to values out of the box. + Adaptive, // Best compression algo is chosen on finishing the chunk. + + // Type-specific codecs. + + FixedLen = 10, // For types with a fixed byte size. + + VarLen = 20, // E.g. for strings, blobs. + VarLenVarInt = 23, // Uses VarInt to save sizes. TODO + + VarInt = 30, // Unsigned ints. + DeltaVarInt = 31, // Increasing unsigned ints. + DeltaRevVarInt = 32, // Decreasing unsigned ints. + ZigZag = 40, // Signed ints. + DeltaZigZag = 41, // Monotone or densely distributed signed ints. + // TODO: Add some bit-wise int compression algorithms. + + Bool = 50, // 1 bit per bool (2 bits per nullable bool). + + FixedLenLz = 60, // TODO + VarLenLz = 61, // TODO + // TODO: Add gzip/snappy or some other general-case compression algos. + + MaxId = 0x3FF, // 10 bits +}; + +/***************************************************************************//** + * TCodecSig identifies a concrete instance of a codec within a concrete IType + * context. + * + * Internal structure: + * --------------------------------------------- + * | Reserved : 5 | IsNullable : 1 | Type : 10 | + * --------------------------------------------- + * + * NOTE: Could not use union of ui16 and bitfield with constexpr constructor. + ******************************************************************************/ +#pragma pack(push, 2) +class TCodecSig { +public: TCodecSig(ui16 raw = 0) - : Raw(raw) - { } - - TCodecSig(TCodecType type) - : Raw(ui16(type)) - { } - + : Raw(raw) + { } + + TCodecSig(TCodecType type) + : Raw(ui16(type)) + { } + TCodecSig(TCodecType type, bool isNullable) - : Raw((ui16(isNullable) << 10) | ui16(type)) - { } - + : Raw((ui16(isNullable) << 10) | ui16(type)) + { } + operator ui16() const { - return Raw; - } - + return Raw; + } + TCodecType Type() const { - return static_cast<TCodecType>(Raw & ((1 << 10) - 1)); - } - + return static_cast<TCodecType>(Raw & ((1 << 10) - 1)); + } + bool IsNullable() const { - return Raw & (1 << 10); - } - -private: - ui16 Raw; -}; -#pragma pack(pop) -static_assert(sizeof(TCodecSig) == 2, "Expected sizeof(TCodecSig) == 2."); - -/***************************************************************************//** - * IChunkCoder is used to serialize a single data chunk. - ******************************************************************************/ -class IChunkCoder { -public: - virtual ~IChunkCoder() { } - - virtual TCodecSig Signature() const = 0; - - inline void AddData(const char* data, size_t size) { - ++ValuesConsumed; - ConsumedSize += size; - DoAddData(data, size); - } - - void AddNull() { - ++ValuesConsumed; - DoAddNull(); - } - - inline ui32 GetValuesConsumed() const { - return ValuesConsumed; - } - - inline size_t GetConsumedSize() const { - return ConsumedSize; - } - - virtual size_t GetEstimatedSize() const = 0; - virtual void Seal() = 0; - -protected: - virtual void DoAddData(const char* data, size_t size) = 0; - virtual void DoAddNull() = 0; - -protected: - ui32 ValuesConsumed; - size_t ConsumedSize; -}; - -/***************************************************************************//** - * IChunkIterator is a forward iterator over the chunk values. - * NOTE: No explicit end of values is provided, it should be known to clients. - ******************************************************************************/ -class IChunkIterator { -public: - virtual ~IChunkIterator() { } - - virtual TDataRef Next() = 0; - virtual TDataRef Peek() const = 0; -}; - -class IChunkBidirIterator : public IChunkIterator { -public: - virtual void Back() = 0; -}; - -/***************************************************************************//** - * IChunkDecoder provides an interface to values of the encoded chunk. - ******************************************************************************/ -class IChunkDecoder : public TThrRefBase { -public: - using TPtr = TIntrusiveConstPtr<IChunkDecoder>; - - virtual TCodecSig Signature() const = 0; - - virtual TAutoPtr<IChunkIterator> MakeIterator() const = 0; - virtual TAutoPtr<IChunkBidirIterator> MakeBidirIterator() const = 0; - virtual TDataRef GetValue(size_t index) const = 0; - - static IChunkDecoder::TPtr ReadChunk(const TDataRef&, const TTypeCodecs* codecs); - virtual IChunkDecoder::TPtr ReadNext(const TDataRef&, const TTypeCodecs* codecs) const = 0; - //< ReadNext looks up in the given TTypeCodecs only if the Sig doesn't match 'this' codec. -}; - -/***************************************************************************//** - * ICodec encapsulates type serialization/deserialization. - ******************************************************************************/ -class ICodec { -public: - virtual ~ICodec() { } - - virtual TCodecSig Signature() const = 0; - - virtual TAutoPtr<IChunkCoder> MakeChunk(TFlatBlobDataOutputStream*) const = 0; - virtual IChunkDecoder::TPtr ReadChunk(const TDataRef&) const = 0; - - /// Read the chunk using 'this' codec (if the codec signature matches), - /// or look up a different codec in the table. - virtual IChunkDecoder::TPtr ReadChunk(const TDataRef&, const TTypeCodecs* codecs) const = 0; -}; - -/***************************************************************************//** + return Raw & (1 << 10); + } + +private: + ui16 Raw; +}; +#pragma pack(pop) +static_assert(sizeof(TCodecSig) == 2, "Expected sizeof(TCodecSig) == 2."); + +/***************************************************************************//** + * IChunkCoder is used to serialize a single data chunk. + ******************************************************************************/ +class IChunkCoder { +public: + virtual ~IChunkCoder() { } + + virtual TCodecSig Signature() const = 0; + + inline void AddData(const char* data, size_t size) { + ++ValuesConsumed; + ConsumedSize += size; + DoAddData(data, size); + } + + void AddNull() { + ++ValuesConsumed; + DoAddNull(); + } + + inline ui32 GetValuesConsumed() const { + return ValuesConsumed; + } + + inline size_t GetConsumedSize() const { + return ConsumedSize; + } + + virtual size_t GetEstimatedSize() const = 0; + virtual void Seal() = 0; + +protected: + virtual void DoAddData(const char* data, size_t size) = 0; + virtual void DoAddNull() = 0; + +protected: + ui32 ValuesConsumed; + size_t ConsumedSize; +}; + +/***************************************************************************//** + * IChunkIterator is a forward iterator over the chunk values. + * NOTE: No explicit end of values is provided, it should be known to clients. + ******************************************************************************/ +class IChunkIterator { +public: + virtual ~IChunkIterator() { } + + virtual TDataRef Next() = 0; + virtual TDataRef Peek() const = 0; +}; + +class IChunkBidirIterator : public IChunkIterator { +public: + virtual void Back() = 0; +}; + +/***************************************************************************//** + * IChunkDecoder provides an interface to values of the encoded chunk. + ******************************************************************************/ +class IChunkDecoder : public TThrRefBase { +public: + using TPtr = TIntrusiveConstPtr<IChunkDecoder>; + + virtual TCodecSig Signature() const = 0; + + virtual TAutoPtr<IChunkIterator> MakeIterator() const = 0; + virtual TAutoPtr<IChunkBidirIterator> MakeBidirIterator() const = 0; + virtual TDataRef GetValue(size_t index) const = 0; + + static IChunkDecoder::TPtr ReadChunk(const TDataRef&, const TTypeCodecs* codecs); + virtual IChunkDecoder::TPtr ReadNext(const TDataRef&, const TTypeCodecs* codecs) const = 0; + //< ReadNext looks up in the given TTypeCodecs only if the Sig doesn't match 'this' codec. +}; + +/***************************************************************************//** + * ICodec encapsulates type serialization/deserialization. + ******************************************************************************/ +class ICodec { +public: + virtual ~ICodec() { } + + virtual TCodecSig Signature() const = 0; + + virtual TAutoPtr<IChunkCoder> MakeChunk(TFlatBlobDataOutputStream*) const = 0; + virtual IChunkDecoder::TPtr ReadChunk(const TDataRef&) const = 0; + + /// Read the chunk using 'this' codec (if the codec signature matches), + /// or look up a different codec in the table. + virtual IChunkDecoder::TPtr ReadChunk(const TDataRef&, const TTypeCodecs* codecs) const = 0; +}; + +/***************************************************************************//** * TDataRef can either share the data (TString) or keep a reference (TStringBuf). - * It uses short string optimization (SSO) to store small data (<= 16b). + * It uses short string optimization (SSO) to store small data (<= 16b). * TODO: Move to ydb/core/util - ******************************************************************************/ -class TDataRef { -public: - /// Create null by default. - TDataRef() - : Data_(nullptr) - , Size_(0) - , ShortSize_(0) - , IsNull_(1) - { } - - /// No ownership of data is taken. - TDataRef(const char* data, size_t size) - : Data_(data) - , Size_(size) - , ShortSize_(LONG_SIZE) - , IsNull_(0) - { } - - TDataRef(const TStringBuf& data) + ******************************************************************************/ +class TDataRef { +public: + /// Create null by default. + TDataRef() + : Data_(nullptr) + , Size_(0) + , ShortSize_(0) + , IsNull_(1) + { } + + /// No ownership of data is taken. + TDataRef(const char* data, size_t size) + : Data_(data) + , Size_(size) + , ShortSize_(LONG_SIZE) + , IsNull_(0) + { } + + TDataRef(const TStringBuf& data) : TDataRef(data.data(), data.size()) - { } - - /// Copy and take ownership of a small piece of data (<= 16b). - TDataRef(const char* data, size_t size, bool) - : ShortSize_(size) - , IsNull_(0) - { + { } + + /// Copy and take ownership of a small piece of data (<= 16b). + TDataRef(const char* data, size_t size, bool) + : ShortSize_(size) + , IsNull_(0) + { Y_VERIFY_DEBUG(size <= INTRUSIVE_SIZE); if (size) { ::memcpy(IntrusiveData_, data, size); } - } - + } + /// Ownership of the TString is taken with zero-copy. TDataRef(const TString& data) : TDataRef(data, data.data(), data.size()) - { } - + { } + /// Ownership of the TString is taken with zero-copy. TDataRef(const TString& data, const char* begin, size_t size) - : SharedData_(data) - , Data_(begin) - , Size_(size) - , ShortSize_(LONG_SIZE) - , IsNull_(0) - { } - + : SharedData_(data) + , Data_(begin) + , Size_(size) + , ShortSize_(LONG_SIZE) + , IsNull_(0) + { } + /// Ownership of the TString is taken with zero-copy. TDataRef(const TString& data, size_t begin, size_t size) : TDataRef(data, data.data() + begin, size) - { } - - bool operator==(const TDataRef& other) const { - if (IsNull_ || other.IsNull_) - return IsNull_ == other.IsNull_; - return ToStringBuf() == other.ToStringBuf(); - } - - const char* Data() const { - return ShortSize_ != LONG_SIZE ? IntrusiveData_ : Data_ ; - } - - size_t Size() const { - return ShortSize_ != LONG_SIZE ? ShortSize_ : Size_; - } - - const char* End() const { - return ShortSize_ != LONG_SIZE ? IntrusiveData_ + ShortSize_ : Data_ + Size_; - } - - bool IsNull() const { - return IsNull_; - } - - TStringBuf ToStringBuf() const { - return ShortSize_ != LONG_SIZE ? TStringBuf(IntrusiveData_, ShortSize_) : TStringBuf(Data_, Size_); - } - - /// Static factory methods. - - static inline TDataRef Ref(const char* data, size_t size) { - return TDataRef(data, size); - } - - static inline TDataRef CopyShort(const char* data, size_t size) { - return TDataRef(data, size, true); - } - - static inline TDataRef CopyLong(const char* data, size_t size) { + { } + + bool operator==(const TDataRef& other) const { + if (IsNull_ || other.IsNull_) + return IsNull_ == other.IsNull_; + return ToStringBuf() == other.ToStringBuf(); + } + + const char* Data() const { + return ShortSize_ != LONG_SIZE ? IntrusiveData_ : Data_ ; + } + + size_t Size() const { + return ShortSize_ != LONG_SIZE ? ShortSize_ : Size_; + } + + const char* End() const { + return ShortSize_ != LONG_SIZE ? IntrusiveData_ + ShortSize_ : Data_ + Size_; + } + + bool IsNull() const { + return IsNull_; + } + + TStringBuf ToStringBuf() const { + return ShortSize_ != LONG_SIZE ? TStringBuf(IntrusiveData_, ShortSize_) : TStringBuf(Data_, Size_); + } + + /// Static factory methods. + + static inline TDataRef Ref(const char* data, size_t size) { + return TDataRef(data, size); + } + + static inline TDataRef CopyShort(const char* data, size_t size) { + return TDataRef(data, size, true); + } + + static inline TDataRef CopyLong(const char* data, size_t size) { return TDataRef(TString(data, size)); - } - - static inline TDataRef Copy(const char* data, size_t size) { + } + + static inline TDataRef Copy(const char* data, size_t size) { return size <= INTRUSIVE_SIZE ? TDataRef(data, size, true) : TDataRef(TString(data, size)); - } - -private: - static const size_t INTRUSIVE_SIZE = 16; - static const ui8 LONG_SIZE = 0x1F; - + } + +private: + static const size_t INTRUSIVE_SIZE = 16; + static const ui8 LONG_SIZE = 0x1F; + TString SharedData_; // FIXME: It's a bottleneck (slows down a document traverse). - - union { - char IntrusiveData_[INTRUSIVE_SIZE]; - - struct { - const char* Data_; - size_t Size_; - }; - }; - - struct { - ui8 ShortSize_ : 5; - ui8 IsNull_ : 1; - // Reserved : 2; - }; -}; - -/***************************************************************************//** - * TTypeCodecs represents a set of codecs available to use with the IType - * instance. - * TODO: Use this class instead of TTypeSerializerRegistry. - ******************************************************************************/ -class TTypeCodecs { -public: + + union { + char IntrusiveData_[INTRUSIVE_SIZE]; + + struct { + const char* Data_; + size_t Size_; + }; + }; + + struct { + ui8 ShortSize_ : 5; + ui8 IsNull_ : 1; + // Reserved : 2; + }; +}; + +/***************************************************************************//** + * TTypeCodecs represents a set of codecs available to use with the IType + * instance. + * TODO: Use this class instead of TTypeSerializerRegistry. + ******************************************************************************/ +class TTypeCodecs { +public: TTypeCodecs(TTypeId typeId = 0); - - template <bool IsNullable> - const ICodec* GetDefaultCodec() const { - const ICodec* codec = IsNullable ? DefaultNullable : DefaultNonNullable; + + template <bool IsNullable> + const ICodec* GetDefaultCodec() const { + const ICodec* codec = IsNullable ? DefaultNullable : DefaultNonNullable; Y_VERIFY(codec, "No default codec."); - return codec; - } - - const ICodec* GetCodec(TCodecSig sig) const { - auto iter = Codecs.find(sig); + return codec; + } + + const ICodec* GetCodec(TCodecSig sig) const { + auto iter = Codecs.find(sig); Y_VERIFY(iter != Codecs.end(), "Unregistered codec (%u).", ui16(sig)); - return iter->second; - } - - bool Has(TCodecSig sig) const { - return Codecs.find(sig) != Codecs.end(); - } - - template <typename TCodec> - const ICodec* AddCodec() { - auto codec = Singleton<TCodec>(); + return iter->second; + } + + bool Has(TCodecSig sig) const { + return Codecs.find(sig) != Codecs.end(); + } + + template <typename TCodec> + const ICodec* AddCodec() { + auto codec = Singleton<TCodec>(); auto inserted = Codecs.insert(std::make_pair(TCodec::Sig(), codec)); Y_VERIFY(inserted.second, "Codec signature collision (%u).", ui16(TCodec::Sig())); - return codec; - } - - const ICodec* AddAlias(TCodecSig from, TCodecSig to, bool force = false) { - auto iter = Codecs.find(to); + return codec; + } + + const ICodec* AddAlias(TCodecSig from, TCodecSig to, bool force = false) { + auto iter = Codecs.find(to); Y_VERIFY(iter != Codecs.end(), "Aliasing an unregistered codec (%u -> %u).", ui16(from), ui16(to)); - return AddAlias(from, iter->second, force); - } - - const ICodec* AddAlias(TCodecSig from, const ICodec* to, bool force = false) { + return AddAlias(from, iter->second, force); + } + + const ICodec* AddAlias(TCodecSig from, const ICodec* to, bool force = false) { Y_VERIFY(to, "Aliasing an unregistered codec (%u -> nullptr).", ui16(from)); - auto& alias = Codecs[from]; + auto& alias = Codecs[from]; Y_VERIFY(force || !alias, "Codec signature collision (%u).", ui16(from)); - alias = to; - - // Cache the default codecs. - if (from.Type() == TCodecType::Default) { - if (from.IsNullable()) - DefaultNullable = to; - else - DefaultNonNullable = to; - } - - return to; - } - -private: + alias = to; + + // Cache the default codecs. + if (from.Type() == TCodecType::Default) { + if (from.IsNullable()) + DefaultNullable = to; + else + DefaultNonNullable = to; + } + + return to; + } + +private: using TIdToCodec = THashMap<ui16, const ICodec*>; - TIdToCodec Codecs; - - const ICodec* DefaultNullable; - const ICodec* DefaultNonNullable; -}; - -//////////////////////////////////////////////////////////////////////////////// -inline IChunkDecoder::TPtr IChunkDecoder::ReadChunk(const TDataRef& data, const TTypeCodecs* codecs) { + TIdToCodec Codecs; + + const ICodec* DefaultNullable; + const ICodec* DefaultNonNullable; +}; + +//////////////////////////////////////////////////////////////////////////////// +inline IChunkDecoder::TPtr IChunkDecoder::ReadChunk(const TDataRef& data, const TTypeCodecs* codecs) { Y_VERIFY_DEBUG(data.Size() >= sizeof(TCodecSig)); const TCodecSig sig = ReadUnaligned<TCodecSig>(data.Data()); - auto codec = codecs->GetCodec(sig); + auto codec = codecs->GetCodec(sig); Y_VERIFY(codec, "Unregistered codec (%u).", ui16(sig)); - return codec->ReadChunk(data); -} - + return codec->ReadChunk(data); +} + } // namespace NScheme } // namespace NKikimr diff --git a/ydb/core/persqueue/type_codecs_ut.cpp b/ydb/core/persqueue/type_codecs_ut.cpp index e3c2a1259e..d9f0b64755 100644 --- a/ydb/core/persqueue/type_codecs_ut.cpp +++ b/ydb/core/persqueue/type_codecs_ut.cpp @@ -1,15 +1,15 @@ -#include "type_codecs.h" -#include "type_codecs_defs.h" +#include "type_codecs.h" +#include "type_codecs_defs.h" #include <ydb/core/scheme_types/scheme_types_defs.h> - + #include <library/cpp/testing/unittest/registar.h> -#include <util/generic/vector.h> +#include <util/generic/vector.h> #include <util/random/fast.h> -#include <util/datetime/base.h> - +#include <util/datetime/base.h> + namespace NKikimr { - + using ICodec = NScheme::ICodec; using TTypeCodecs = NScheme::TTypeCodecs; using TCodecSig = NScheme::TCodecSig; @@ -17,244 +17,244 @@ using TCodecType = NScheme::TCodecType; using TDataRef = NScheme::TDataRef; Y_UNIT_TEST_SUITE(TTypeCodecsTest) { - + void Metrics(const TVector<TDataRef>& values, const ICodec* codec) { - TAutoPtr<TFlatBlobDataOutputStream> output(new TFlatBlobDataOutputStream()); - auto chunk = codec->MakeChunk(output.Get()); - - auto start = TInstant::Now(); - for (const auto& value : values) { - if (value.IsNull()) - chunk->AddNull(); - else - chunk->AddData(value.Data(), value.Size()); - } - chunk->Seal(); - auto duration = TInstant::Now() - start; - - Cerr << "Size: " << output->GetCurrentOffset() << Endl; - Cerr << "Create chunk: " << duration << Endl; - - auto reading = codec->ReadChunk(output->GetBuffer()); - output.Reset(nullptr); - - start = TInstant::Now(); - for (size_t i = 0, size = values.size(); i != size; ++i) { - auto value = reading->GetValue(i); + TAutoPtr<TFlatBlobDataOutputStream> output(new TFlatBlobDataOutputStream()); + auto chunk = codec->MakeChunk(output.Get()); + + auto start = TInstant::Now(); + for (const auto& value : values) { + if (value.IsNull()) + chunk->AddNull(); + else + chunk->AddData(value.Data(), value.Size()); + } + chunk->Seal(); + auto duration = TInstant::Now() - start; + + Cerr << "Size: " << output->GetCurrentOffset() << Endl; + Cerr << "Create chunk: " << duration << Endl; + + auto reading = codec->ReadChunk(output->GetBuffer()); + output.Reset(nullptr); + + start = TInstant::Now(); + for (size_t i = 0, size = values.size(); i != size; ++i) { + auto value = reading->GetValue(i); Y_UNUSED(value); - } - Cerr << "Read by index: " << TInstant::Now() - start << Endl; - - auto iter = reading->MakeIterator(); - start = TInstant::Now(); - for (size_t i = 0; i != values.size(); ++i) { - auto value = iter->Next(); + } + Cerr << "Read by index: " << TInstant::Now() - start << Endl; + + auto iter = reading->MakeIterator(); + start = TInstant::Now(); + for (size_t i = 0; i != values.size(); ++i) { + auto value = iter->Next(); Y_UNUSED(value); - } - Cerr << "Iterate: " << TInstant::Now() - start << Endl; - } - + } + Cerr << "Iterate: " << TInstant::Now() - start << Endl; + } + void TestImpl(const TVector<TDataRef>& values, const ICodec* codec) { - TAutoPtr<TFlatBlobDataOutputStream> output(new TFlatBlobDataOutputStream()); - auto chunk = codec->MakeChunk(output.Get()); - for (const auto& value : values) { - if (value.IsNull()) - chunk->AddNull(); - else - chunk->AddData(value.Data(), value.Size()); - } - chunk->Seal(); - - auto reading = codec->ReadChunk(output->GetBuffer()); - auto iter = reading->MakeIterator(); - output.Reset(nullptr); - - for (size_t i = 0; i != values.size(); ++i) { - const auto& value = values[i]; - auto value1 = reading->GetValue(i); - auto value2 = iter->Peek(); - auto value3 = iter->Next(); - UNIT_ASSERT_EQUAL(value, value1); - UNIT_ASSERT_EQUAL(value1, value2); - UNIT_ASSERT_EQUAL(value1, value3); - UNIT_ASSERT_EQUAL(value2, value3); - } - - Metrics(values, codec); - }; - + TAutoPtr<TFlatBlobDataOutputStream> output(new TFlatBlobDataOutputStream()); + auto chunk = codec->MakeChunk(output.Get()); + for (const auto& value : values) { + if (value.IsNull()) + chunk->AddNull(); + else + chunk->AddData(value.Data(), value.Size()); + } + chunk->Seal(); + + auto reading = codec->ReadChunk(output->GetBuffer()); + auto iter = reading->MakeIterator(); + output.Reset(nullptr); + + for (size_t i = 0; i != values.size(); ++i) { + const auto& value = values[i]; + auto value1 = reading->GetValue(i); + auto value2 = iter->Peek(); + auto value3 = iter->Next(); + UNIT_ASSERT_EQUAL(value, value1); + UNIT_ASSERT_EQUAL(value1, value2); + UNIT_ASSERT_EQUAL(value1, value3); + UNIT_ASSERT_EQUAL(value2, value3); + } + + Metrics(values, codec); + }; + Y_UNIT_TEST(TestBoolCodec) { - static const bool VALUE_FALSE = false; - static const bool VALUE_TRUE = true; - + static const bool VALUE_FALSE = false; + static const bool VALUE_TRUE = true; + THolder<TTypeCodecs> codecs(new TTypeCodecs(NScheme::TBool::TypeId)); - + TVector<TDataRef> values(1000, TDataRef((const char*)&VALUE_FALSE, sizeof(VALUE_FALSE))); - for (int i = 0; i < 100; ++i) { - values[i * 2] = TDataRef((const char*)&VALUE_TRUE, sizeof(VALUE_TRUE)); - values[500 + i] = TDataRef((const char*)&VALUE_TRUE, sizeof(VALUE_TRUE)); - values[700 + 3 * i] = TDataRef((const char*)&VALUE_TRUE, sizeof(VALUE_TRUE)); - } - - auto codec = codecs->GetCodec(TCodecSig(TCodecType::Bool, false)); - TestImpl(values, codec); - - for (int i = 0; i < 200; ++i) { - values[i * 5] = TDataRef(); - } - codec = codecs->GetCodec(TCodecSig(TCodecType::Bool, true)); - TestImpl(values, codec); - } - + for (int i = 0; i < 100; ++i) { + values[i * 2] = TDataRef((const char*)&VALUE_TRUE, sizeof(VALUE_TRUE)); + values[500 + i] = TDataRef((const char*)&VALUE_TRUE, sizeof(VALUE_TRUE)); + values[700 + 3 * i] = TDataRef((const char*)&VALUE_TRUE, sizeof(VALUE_TRUE)); + } + + auto codec = codecs->GetCodec(TCodecSig(TCodecType::Bool, false)); + TestImpl(values, codec); + + for (int i = 0; i < 200; ++i) { + values[i * 5] = TDataRef(); + } + codec = codecs->GetCodec(TCodecSig(TCodecType::Bool, true)); + TestImpl(values, codec); + } + Y_UNIT_TEST(TestFixedLenCodec) { THolder<TTypeCodecs> codecs(new TTypeCodecs(NScheme::TInt32::TypeId)); - + TVector<TDataRef> values; - for (int i = 0; i < 1000; ++i) { - int data = i; - values.push_back(TDataRef((const char*)&data, sizeof(data), true)); - data = -i; - values.push_back(TDataRef((const char*)&data, sizeof(data), true)); - } - - auto codec = codecs->GetCodec(TCodecSig(TCodecType::FixedLen, false)); - TestImpl(values, codec); - - for (int i = 0; i < 200; ++i) { - values[i * 5] = TDataRef(); - values[1200 + i] = TDataRef(); - } - codec = codecs->GetCodec(TCodecSig(TCodecType::FixedLen, true)); - TestImpl(values, codec); - } - + for (int i = 0; i < 1000; ++i) { + int data = i; + values.push_back(TDataRef((const char*)&data, sizeof(data), true)); + data = -i; + values.push_back(TDataRef((const char*)&data, sizeof(data), true)); + } + + auto codec = codecs->GetCodec(TCodecSig(TCodecType::FixedLen, false)); + TestImpl(values, codec); + + for (int i = 0; i < 200; ++i) { + values[i * 5] = TDataRef(); + values[1200 + i] = TDataRef(); + } + codec = codecs->GetCodec(TCodecSig(TCodecType::FixedLen, true)); + TestImpl(values, codec); + } + Y_UNIT_TEST(TestVarLenCodec) { THolder<TTypeCodecs> codecs(new TTypeCodecs(NScheme::TString::TypeId)); - + TReallyFastRng32 rand(100500); - + TVector<TDataRef> values; - for (int i = 0; i < 1000; ++i) { + for (int i = 0; i < 1000; ++i) { TVector<char> value(rand.Uniform(10)); - for (char& c : value) + for (char& c : value) c = 'a' + rand.Uniform(26); - values.push_back(TDataRef(value.data(), value.size(), true)); - } - - auto codec = codecs->GetCodec(TCodecSig(TCodecType::VarLen, false)); - TestImpl(values, codec); - - for (int i = 0; i < 100; ++i) { - values[i * 5] = TDataRef(); - values[800 + i] = TDataRef(); - } - codec = codecs->GetCodec(TCodecSig(TCodecType::VarLen, true)); - TestImpl(values, codec); - } - + values.push_back(TDataRef(value.data(), value.size(), true)); + } + + auto codec = codecs->GetCodec(TCodecSig(TCodecType::VarLen, false)); + TestImpl(values, codec); + + for (int i = 0; i < 100; ++i) { + values[i * 5] = TDataRef(); + values[800 + i] = TDataRef(); + } + codec = codecs->GetCodec(TCodecSig(TCodecType::VarLen, true)); + TestImpl(values, codec); + } + Y_UNIT_TEST(TestVarIntCodec) { THolder<TTypeCodecs> codecs(new TTypeCodecs(NScheme::TUint32::TypeId)); - + TReallyFastRng32 rand(100500); - + TVector<TDataRef> values; - for (int i = 0; i < 1000; ++i) { + for (int i = 0; i < 1000; ++i) { ui32 value = rand.Uniform(100500); - values.push_back(TDataRef((const char*)&value, sizeof(value), true)); - } - - auto codec = codecs->GetCodec(TCodecSig(TCodecType::VarInt, false)); - TestImpl(values, codec); - - for (int i = 0; i < 100; ++i) { - values[i * 5] = TDataRef(); - values[800 + i] = TDataRef(); - } - codec = codecs->GetCodec(TCodecSig(TCodecType::VarInt, true)); - TestImpl(values, codec); - } - + values.push_back(TDataRef((const char*)&value, sizeof(value), true)); + } + + auto codec = codecs->GetCodec(TCodecSig(TCodecType::VarInt, false)); + TestImpl(values, codec); + + for (int i = 0; i < 100; ++i) { + values[i * 5] = TDataRef(); + values[800 + i] = TDataRef(); + } + codec = codecs->GetCodec(TCodecSig(TCodecType::VarInt, true)); + TestImpl(values, codec); + } + Y_UNIT_TEST(TestZigZagCodec) { THolder<TTypeCodecs> codecs(new TTypeCodecs(NScheme::TUint32::TypeId)); - + TReallyFastRng32 rand(100500); - + TVector<TDataRef> values; - for (int i = 0; i < 1000; ++i) { + for (int i = 0; i < 1000; ++i) { i32 value = rand.Uniform(100500); - if (i & 1) - value = -value; - values.push_back(TDataRef((const char*)&value, sizeof(value), true)); - } - - auto codec = codecs->GetCodec(TCodecSig(TCodecType::ZigZag, false)); - TestImpl(values, codec); - - for (int i = 0; i < 100; ++i) { - values[i * 5] = TDataRef(); - values[800 + i] = TDataRef(); - } - codec = codecs->GetCodec(TCodecSig(TCodecType::ZigZag, true)); - TestImpl(values, codec); - } - - void TestDeltaVarIntCodecImpl(TCodecType type, bool rev) { + if (i & 1) + value = -value; + values.push_back(TDataRef((const char*)&value, sizeof(value), true)); + } + + auto codec = codecs->GetCodec(TCodecSig(TCodecType::ZigZag, false)); + TestImpl(values, codec); + + for (int i = 0; i < 100; ++i) { + values[i * 5] = TDataRef(); + values[800 + i] = TDataRef(); + } + codec = codecs->GetCodec(TCodecSig(TCodecType::ZigZag, true)); + TestImpl(values, codec); + } + + void TestDeltaVarIntCodecImpl(TCodecType type, bool rev) { THolder<TTypeCodecs> codecs(new TTypeCodecs(NScheme::TUint32::TypeId)); - + TReallyFastRng32 rand(100500); - + TVector<TDataRef> values; - ui32 value = 2000000; - for (int i = 0; i < 1000; ++i) { - if (rev) + ui32 value = 2000000; + for (int i = 0; i < 1000; ++i) { + if (rev) value -= rand.Uniform(1000); - else + else value += rand.Uniform(1000); - values.push_back(TDataRef((const char*)&value, sizeof(value), true)); - } - - auto codec = codecs->GetCodec(TCodecSig(type, false)); - TestImpl(values, codec); - - for (int i = 0; i < 100; ++i) { - values[i * 5] = TDataRef(); - values[800 + i] = TDataRef(); - } - codec = codecs->GetCodec(TCodecSig(type, true)); - TestImpl(values, codec); - } - + values.push_back(TDataRef((const char*)&value, sizeof(value), true)); + } + + auto codec = codecs->GetCodec(TCodecSig(type, false)); + TestImpl(values, codec); + + for (int i = 0; i < 100; ++i) { + values[i * 5] = TDataRef(); + values[800 + i] = TDataRef(); + } + codec = codecs->GetCodec(TCodecSig(type, true)); + TestImpl(values, codec); + } + Y_UNIT_TEST(TestDeltaVarIntCodecAndRev) { - TestDeltaVarIntCodecImpl(TCodecType::DeltaVarInt, false); - TestDeltaVarIntCodecImpl(TCodecType::DeltaRevVarInt, true); - } - + TestDeltaVarIntCodecImpl(TCodecType::DeltaVarInt, false); + TestDeltaVarIntCodecImpl(TCodecType::DeltaRevVarInt, true); + } + Y_UNIT_TEST(TestDeltaZigZagCodec) { THolder<TTypeCodecs> codecs(new TTypeCodecs(NScheme::TInt32::TypeId)); - + TReallyFastRng32 rand(100500); - + TVector<TDataRef> values; - ui32 value = -17; - for (int i = 0; i < 1000; ++i) { + ui32 value = -17; + for (int i = 0; i < 1000; ++i) { value += rand.Uniform(1000); - if (rand.GenRand() & 1) - value = -value; - values.push_back(TDataRef((const char*)&value, sizeof(value), true)); - } - - auto codec = codecs->GetCodec(TCodecSig(TCodecType::DeltaZigZag, false)); - TestImpl(values, codec); - - for (int i = 0; i < 100; ++i) { - values[i * 5] = TDataRef(); - values[800 + i] = TDataRef(); - } - codec = codecs->GetCodec(TCodecSig(TCodecType::DeltaZigZag, true)); - TestImpl(values, codec); - } - -} - + if (rand.GenRand() & 1) + value = -value; + values.push_back(TDataRef((const char*)&value, sizeof(value), true)); + } + + auto codec = codecs->GetCodec(TCodecSig(TCodecType::DeltaZigZag, false)); + TestImpl(values, codec); + + for (int i = 0; i < 100; ++i) { + values[i * 5] = TDataRef(); + values[800 + i] = TDataRef(); + } + codec = codecs->GetCodec(TCodecSig(TCodecType::DeltaZigZag, true)); + TestImpl(values, codec); + } + +} + } // namespace NKikimr - + diff --git a/ydb/core/persqueue/type_coders.h b/ydb/core/persqueue/type_coders.h index 672b6c874f..26c2566d32 100644 --- a/ydb/core/persqueue/type_coders.h +++ b/ydb/core/persqueue/type_coders.h @@ -1,311 +1,311 @@ -#pragma once - -#include "type_codecs_defs.h" - +#pragma once + +#include "type_codecs_defs.h" + #include <library/cpp/containers/bitseq/bitvector.h> #include <library/cpp/packedtypes/longs.h> #include <library/cpp/packedtypes/zigzag.h> - -#include <util/generic/typetraits.h> - + +#include <util/generic/typetraits.h> + namespace NKikimr { namespace NScheme { - -//////////////////////////////////////////////////////////////////////////////// - -template <ui16 Type, bool IsNullable> -class TChunkCoderBase : public IChunkCoder { -public: + +//////////////////////////////////////////////////////////////////////////////// + +template <ui16 Type, bool IsNullable> +class TChunkCoderBase : public IChunkCoder { +public: static inline TCodecSig Sig() { return TCodecSig(TCodecType(Type), IsNullable); } - -public: + +public: TCodecSig Signature() const override { return Sig(); } -}; - -//////////////////////////////////////////////////////////////////////////////// - -template <bool IsNullable> -class TCoderMask; - -template <> -class TCoderMask<true> { -public: - inline size_t MaxSize() const { - return Mask.Words() * sizeof(TMask::TWord) + sizeof(ui32); - } - - inline void AddNull() { Mask.Append(0, 1); } - inline void AddNonNull() { Mask.Append(1, 1); } - - inline void Seal(TFlatBlobDataOutputStream* output) { - const ui32 maskSize = Mask.Words() * sizeof(TMask::TWord); - output->Write((const char*)Mask.Data(), maskSize); - output->Write((const char*)&maskSize, sizeof(maskSize)); // TODO: Reduce size of small masks (embed size into the first word). - } - -private: - using TMask = TBitVector<ui16>; - TMask Mask; -}; - -template <> -class TCoderMask<false> { -public: - inline size_t MaxSize() const { return 0; } +}; + +//////////////////////////////////////////////////////////////////////////////// + +template <bool IsNullable> +class TCoderMask; + +template <> +class TCoderMask<true> { +public: + inline size_t MaxSize() const { + return Mask.Words() * sizeof(TMask::TWord) + sizeof(ui32); + } + + inline void AddNull() { Mask.Append(0, 1); } + inline void AddNonNull() { Mask.Append(1, 1); } + + inline void Seal(TFlatBlobDataOutputStream* output) { + const ui32 maskSize = Mask.Words() * sizeof(TMask::TWord); + output->Write((const char*)Mask.Data(), maskSize); + output->Write((const char*)&maskSize, sizeof(maskSize)); // TODO: Reduce size of small masks (embed size into the first word). + } + +private: + using TMask = TBitVector<ui16>; + TMask Mask; +}; + +template <> +class TCoderMask<false> { +public: + inline size_t MaxSize() const { return 0; } inline void AddNull() { Y_FAIL("Null values are not supported."); } - inline void AddNonNull() { } - inline void Seal(TFlatBlobDataOutputStream*) { } -}; - -//////////////////////////////////////////////////////////////////////////////// - -template <size_t Size, bool IsNullable> -class TFixedLenCoder : public TChunkCoderBase<ui16(TCodecType::FixedLen), IsNullable> { -public: - TFixedLenCoder(TFlatBlobDataOutputStream* output) - : DataSize(0) - , Output(output) - { + inline void AddNonNull() { } + inline void Seal(TFlatBlobDataOutputStream*) { } +}; + +//////////////////////////////////////////////////////////////////////////////// + +template <size_t Size, bool IsNullable> +class TFixedLenCoder : public TChunkCoderBase<ui16(TCodecType::FixedLen), IsNullable> { +public: + TFixedLenCoder(TFlatBlobDataOutputStream* output) + : DataSize(0) + , Output(output) + { output->WritePOD(TFixedLenCoder::Sig()); - } - - size_t GetEstimatedSize() const override { - return sizeof(TCodecSig) + DataSize + Mask.MaxSize(); - } - - void Seal() override { - Mask.Seal(Output); - } - -protected: - void DoAddData(const char* data, size_t size) override { + } + + size_t GetEstimatedSize() const override { + return sizeof(TCodecSig) + DataSize + Mask.MaxSize(); + } + + void Seal() override { + Mask.Seal(Output); + } + +protected: + void DoAddData(const char* data, size_t size) override { Y_VERIFY(size == Size, "Size mismatch."); - Mask.AddNonNull(); - DataSize += Size; - Output->Write(data, size); - } - - void DoAddNull() override { - Mask.AddNull(); - static char zero[Size ? Size : 1]; - Output->Write(zero, Size); - // TODO: Consider using succinct Rank structure instead. - } - -private: - size_t DataSize; - TCoderMask<IsNullable> Mask; - TFlatBlobDataOutputStream* Output; -}; - -//////////////////////////////////////////////////////////////////////////////// - -template <bool IsNullable> -class TVarLenCoder : public TChunkCoderBase<ui16(TCodecType::VarLen), IsNullable> { -public: - TVarLenCoder(TFlatBlobDataOutputStream* output) - : DataSize(0) - , Output(output) - { + Mask.AddNonNull(); + DataSize += Size; + Output->Write(data, size); + } + + void DoAddNull() override { + Mask.AddNull(); + static char zero[Size ? Size : 1]; + Output->Write(zero, Size); + // TODO: Consider using succinct Rank structure instead. + } + +private: + size_t DataSize; + TCoderMask<IsNullable> Mask; + TFlatBlobDataOutputStream* Output; +}; + +//////////////////////////////////////////////////////////////////////////////// + +template <bool IsNullable> +class TVarLenCoder : public TChunkCoderBase<ui16(TCodecType::VarLen), IsNullable> { +public: + TVarLenCoder(TFlatBlobDataOutputStream* output) + : DataSize(0) + , Output(output) + { output->WritePOD(TVarLenCoder::Sig()); - } - - size_t GetEstimatedSize() const override { - return sizeof(TCodecSig) + DataSize + Offsets.size() * sizeof(ui32) + Mask.MaxSize() + sizeof(ui32); - } - - void Seal() override { - Output->Write((const char*)Offsets.data(), Offsets.size() * sizeof(ui32)); - Mask.Seal(Output); - } - -protected: - void DoAddData(const char* data, size_t size) override { - Mask.AddNonNull(); - DataSize += size; - Offsets.push_back(DataSize); - Output->Write(data, size); - } - - void DoAddNull() override { - Mask.AddNull(); - Offsets.push_back(DataSize); - } - -private: - ui32 DataSize; + } + + size_t GetEstimatedSize() const override { + return sizeof(TCodecSig) + DataSize + Offsets.size() * sizeof(ui32) + Mask.MaxSize() + sizeof(ui32); + } + + void Seal() override { + Output->Write((const char*)Offsets.data(), Offsets.size() * sizeof(ui32)); + Mask.Seal(Output); + } + +protected: + void DoAddData(const char* data, size_t size) override { + Mask.AddNonNull(); + DataSize += size; + Offsets.push_back(DataSize); + Output->Write(data, size); + } + + void DoAddNull() override { + Mask.AddNull(); + Offsets.push_back(DataSize); + } + +private: + ui32 DataSize; TVector<ui32> Offsets; - TCoderMask<IsNullable> Mask; - TFlatBlobDataOutputStream* Output; -}; - -//////////////////////////////////////////////////////////////////////////////// - -template <typename TIntType> -class TVarIntValueCoder { -public: - using TType = TIntType; + TCoderMask<IsNullable> Mask; + TFlatBlobDataOutputStream* Output; +}; + +//////////////////////////////////////////////////////////////////////////////// + +template <typename TIntType> +class TVarIntValueCoder { +public: + using TType = TIntType; using TSigned = std::make_signed_t<TType>; - - inline size_t Save(TFlatBlobDataOutputStream* output, TType value) { - const auto outValue = static_cast<i64>(value); // TODO: fix out_long(i32) - char varIntOut[sizeof(outValue) + 1]; - auto bytes = out_long(outValue, varIntOut); - output->Write(varIntOut, bytes); - return bytes; - } -}; - -template <typename TIntType> -class TZigZagValueCoder { -public: - using TType = TIntType; + + inline size_t Save(TFlatBlobDataOutputStream* output, TType value) { + const auto outValue = static_cast<i64>(value); // TODO: fix out_long(i32) + char varIntOut[sizeof(outValue) + 1]; + auto bytes = out_long(outValue, varIntOut); + output->Write(varIntOut, bytes); + return bytes; + } +}; + +template <typename TIntType> +class TZigZagValueCoder { +public: + using TType = TIntType; using TSigned = std::make_signed_t<TType>; - - inline size_t Save(TFlatBlobDataOutputStream* output, TType value) { - const auto zigZagged = static_cast<i64>(ZigZagEncode(value)); + + inline size_t Save(TFlatBlobDataOutputStream* output, TType value) { + const auto zigZagged = static_cast<i64>(ZigZagEncode(value)); char varIntOut[sizeof(zigZagged) + 1]; - auto bytes = out_long(zigZagged, varIntOut); - output->Write(varIntOut, bytes); - return bytes; - } -}; - -template <typename TValueCoder, bool Rev = false> -class TDeltaValueCoder : public TValueCoder { -public: - using TType = typename TValueCoder::TType; - - inline size_t Save(TFlatBlobDataOutputStream* output, TType value) { - auto delta = Rev ? Last - value : value - Last; - Last = value; - return TValueCoder::Save(output, delta); - } - -private: - TType Last = 0; -}; - -template <typename TValueCoder, ui16 CodecType, bool IsNullable> -class TByteAlignedIntCoder : public TChunkCoderBase<CodecType, IsNullable> { -public: - using TType = typename TValueCoder::TType; - - TByteAlignedIntCoder(TFlatBlobDataOutputStream* output) - : DataSize(0) - , Output(output) - { + auto bytes = out_long(zigZagged, varIntOut); + output->Write(varIntOut, bytes); + return bytes; + } +}; + +template <typename TValueCoder, bool Rev = false> +class TDeltaValueCoder : public TValueCoder { +public: + using TType = typename TValueCoder::TType; + + inline size_t Save(TFlatBlobDataOutputStream* output, TType value) { + auto delta = Rev ? Last - value : value - Last; + Last = value; + return TValueCoder::Save(output, delta); + } + +private: + TType Last = 0; +}; + +template <typename TValueCoder, ui16 CodecType, bool IsNullable> +class TByteAlignedIntCoder : public TChunkCoderBase<CodecType, IsNullable> { +public: + using TType = typename TValueCoder::TType; + + TByteAlignedIntCoder(TFlatBlobDataOutputStream* output) + : DataSize(0) + , Output(output) + { output->WritePOD(TByteAlignedIntCoder::Sig()); - } - - size_t GetEstimatedSize() const override { - return sizeof(TCodecSig) + DataSize + Mask.MaxSize(); - } - - void Seal() override { - Mask.Seal(Output); - } - -protected: - void DoAddData(const char* data, size_t size) override { + } + + size_t GetEstimatedSize() const override { + return sizeof(TCodecSig) + DataSize + Mask.MaxSize(); + } + + void Seal() override { + Mask.Seal(Output); + } + +protected: + void DoAddData(const char* data, size_t size) override { Y_VERIFY(size == sizeof(TType)); - Mask.AddNonNull(); + Mask.AddNonNull(); DataSize += ValueCoder.Save(Output, ReadUnaligned<TType>(data)); - } - - void DoAddNull() override { - Mask.AddNull(); - } - -private: - size_t DataSize; - TCoderMask<IsNullable> Mask; - TFlatBlobDataOutputStream* Output; - TValueCoder ValueCoder; -}; - -template <typename TIntType, bool IsNullable> -class TVarIntCoder : public TByteAlignedIntCoder<TVarIntValueCoder<TIntType>, ui16(TCodecType::VarInt), IsNullable> { -public: - TVarIntCoder(TFlatBlobDataOutputStream* output) - : TByteAlignedIntCoder<TVarIntValueCoder<TIntType>, ui16(TCodecType::VarInt), IsNullable>(output) - { } -}; - -template <typename TIntType, bool IsNullable> -class TZigZagCoder : public TByteAlignedIntCoder<TZigZagValueCoder<TIntType>, ui16(TCodecType::ZigZag), IsNullable> { -public: - TZigZagCoder(TFlatBlobDataOutputStream* output) - : TByteAlignedIntCoder<TZigZagValueCoder<TIntType>, ui16(TCodecType::ZigZag), IsNullable>(output) - { } -}; - -template <typename TIntType, bool IsNullable> -class TDeltaVarIntCoder : public TByteAlignedIntCoder<TDeltaValueCoder<TVarIntValueCoder<TIntType>>, ui16(TCodecType::DeltaVarInt), IsNullable> { -public: - TDeltaVarIntCoder(TFlatBlobDataOutputStream* output) - : TByteAlignedIntCoder<TDeltaValueCoder<TVarIntValueCoder<TIntType>>, ui16(TCodecType::DeltaVarInt), IsNullable>(output) - { } -}; - -template <typename TIntType, bool IsNullable> -class TDeltaRevVarIntCoder : public TByteAlignedIntCoder<TDeltaValueCoder<TVarIntValueCoder<TIntType>, true>, ui16(TCodecType::DeltaRevVarInt), IsNullable> { -public: - TDeltaRevVarIntCoder(TFlatBlobDataOutputStream* output) - : TByteAlignedIntCoder<TDeltaValueCoder<TVarIntValueCoder<TIntType>, true>, ui16(TCodecType::DeltaRevVarInt), IsNullable>(output) - { } -}; - -template <typename TIntType, bool IsNullable> -class TDeltaZigZagCoder : public TByteAlignedIntCoder<TDeltaValueCoder<TZigZagValueCoder<TIntType>>, ui16(TCodecType::DeltaZigZag), IsNullable> { -public: - TDeltaZigZagCoder(TFlatBlobDataOutputStream* output) - : TByteAlignedIntCoder<TDeltaValueCoder<TZigZagValueCoder<TIntType>>, ui16(TCodecType::DeltaZigZag), IsNullable>(output) - { } -}; - -//////////////////////////////////////////////////////////////////////////////// - -template <bool IsNullable> -class TBoolCoder : public TChunkCoderBase<ui16(TCodecType::Bool), IsNullable> { -public: - TBoolCoder(TFlatBlobDataOutputStream* output) - : Output(output) - { + } + + void DoAddNull() override { + Mask.AddNull(); + } + +private: + size_t DataSize; + TCoderMask<IsNullable> Mask; + TFlatBlobDataOutputStream* Output; + TValueCoder ValueCoder; +}; + +template <typename TIntType, bool IsNullable> +class TVarIntCoder : public TByteAlignedIntCoder<TVarIntValueCoder<TIntType>, ui16(TCodecType::VarInt), IsNullable> { +public: + TVarIntCoder(TFlatBlobDataOutputStream* output) + : TByteAlignedIntCoder<TVarIntValueCoder<TIntType>, ui16(TCodecType::VarInt), IsNullable>(output) + { } +}; + +template <typename TIntType, bool IsNullable> +class TZigZagCoder : public TByteAlignedIntCoder<TZigZagValueCoder<TIntType>, ui16(TCodecType::ZigZag), IsNullable> { +public: + TZigZagCoder(TFlatBlobDataOutputStream* output) + : TByteAlignedIntCoder<TZigZagValueCoder<TIntType>, ui16(TCodecType::ZigZag), IsNullable>(output) + { } +}; + +template <typename TIntType, bool IsNullable> +class TDeltaVarIntCoder : public TByteAlignedIntCoder<TDeltaValueCoder<TVarIntValueCoder<TIntType>>, ui16(TCodecType::DeltaVarInt), IsNullable> { +public: + TDeltaVarIntCoder(TFlatBlobDataOutputStream* output) + : TByteAlignedIntCoder<TDeltaValueCoder<TVarIntValueCoder<TIntType>>, ui16(TCodecType::DeltaVarInt), IsNullable>(output) + { } +}; + +template <typename TIntType, bool IsNullable> +class TDeltaRevVarIntCoder : public TByteAlignedIntCoder<TDeltaValueCoder<TVarIntValueCoder<TIntType>, true>, ui16(TCodecType::DeltaRevVarInt), IsNullable> { +public: + TDeltaRevVarIntCoder(TFlatBlobDataOutputStream* output) + : TByteAlignedIntCoder<TDeltaValueCoder<TVarIntValueCoder<TIntType>, true>, ui16(TCodecType::DeltaRevVarInt), IsNullable>(output) + { } +}; + +template <typename TIntType, bool IsNullable> +class TDeltaZigZagCoder : public TByteAlignedIntCoder<TDeltaValueCoder<TZigZagValueCoder<TIntType>>, ui16(TCodecType::DeltaZigZag), IsNullable> { +public: + TDeltaZigZagCoder(TFlatBlobDataOutputStream* output) + : TByteAlignedIntCoder<TDeltaValueCoder<TZigZagValueCoder<TIntType>>, ui16(TCodecType::DeltaZigZag), IsNullable>(output) + { } +}; + +//////////////////////////////////////////////////////////////////////////////// + +template <bool IsNullable> +class TBoolCoder : public TChunkCoderBase<ui16(TCodecType::Bool), IsNullable> { +public: + TBoolCoder(TFlatBlobDataOutputStream* output) + : Output(output) + { output->WritePOD(TBoolCoder::Sig()); - } - - size_t GetEstimatedSize() const override { - return sizeof(TCodecSig) + Mask.Words() * sizeof(TMask::TWord); - } - - void Seal() override { - Output->Write((const char*)Mask.Data(), Mask.Words() * sizeof(TMask::TWord)); - } - -protected: - void DoAddData(const char* data, size_t size) override { + } + + size_t GetEstimatedSize() const override { + return sizeof(TCodecSig) + Mask.Words() * sizeof(TMask::TWord); + } + + void Seal() override { + Output->Write((const char*)Mask.Data(), Mask.Words() * sizeof(TMask::TWord)); + } + +protected: + void DoAddData(const char* data, size_t size) override { Y_VERIFY_DEBUG(size == sizeof(bool)); - if (IsNullable) - Mask.Append(1, 1); - Mask.Append(*(const bool*)data, 1); - } - - void DoAddNull() override { + if (IsNullable) + Mask.Append(1, 1); + Mask.Append(*(const bool*)data, 1); + } + + void DoAddNull() override { Y_VERIFY(IsNullable, "Null values are not supported."); - Mask.Append(0, 2); - } - -protected: - using TMask = TBitVector<ui16>; - TMask Mask; - TFlatBlobDataOutputStream* Output; -}; - -//////////////////////////////////////////////////////////////////////////////// - + Mask.Append(0, 2); + } + +protected: + using TMask = TBitVector<ui16>; + TMask Mask; + TFlatBlobDataOutputStream* Output; +}; + +//////////////////////////////////////////////////////////////////////////////// + } // namespace NScheme } // namespace NKikimr - + diff --git a/ydb/core/persqueue/type_decoders.h b/ydb/core/persqueue/type_decoders.h index 9dfe62584e..30a8846923 100644 --- a/ydb/core/persqueue/type_decoders.h +++ b/ydb/core/persqueue/type_decoders.h @@ -1,564 +1,564 @@ -#pragma once - -#include "type_codecs_defs.h" - +#pragma once + +#include "type_codecs_defs.h" + #include <library/cpp/containers/bitseq/bititerator.h> #include <library/cpp/packedtypes/longs.h> #include <library/cpp/packedtypes/zigzag.h> - -#include <util/generic/deque.h> -#include <util/generic/map.h> -#include <util/generic/ptr.h> -#include <util/generic/strbuf.h> + +#include <util/generic/deque.h> +#include <util/generic/map.h> +#include <util/generic/ptr.h> +#include <util/generic/strbuf.h> #include <util/system/unaligned_mem.h> - + namespace NKikimr { namespace NScheme { - + using TMaskIterator = TBitIterator<ui8>; - -//////////////////////////////////////////////////////////////////////////////// - -class TDefaultChunkDecoder : public IChunkDecoder { -public: - TAutoPtr<IChunkIterator> MakeIterator() const override { - return new TDefaultChunkIterator(this); - } - - TAutoPtr<IChunkBidirIterator> MakeBidirIterator() const override { - return new TDefaultChunkBidirIterator(this); - } - -private: - template <typename TIterator> - class TDefaultChunkIteratorOps : public TIterator { - public: - TDefaultChunkIteratorOps(const IChunkDecoder* decoder) - : Decoder(decoder) - , Index(0) - { } - - TDataRef Next() override { - return Decoder->GetValue(Index++); - } - - TDataRef Peek() const override { - return Decoder->GetValue(Index); - } - - protected: - const IChunkDecoder* Decoder; - size_t Index; - }; - - using TDefaultChunkIterator = TDefaultChunkIteratorOps<IChunkIterator>; - - class TDefaultChunkBidirIterator : public TDefaultChunkIteratorOps<IChunkBidirIterator> { - public: - TDefaultChunkBidirIterator(const IChunkDecoder* decoder) - : TDefaultChunkIteratorOps<IChunkBidirIterator>(decoder) - { } - - void Back() override { + +//////////////////////////////////////////////////////////////////////////////// + +class TDefaultChunkDecoder : public IChunkDecoder { +public: + TAutoPtr<IChunkIterator> MakeIterator() const override { + return new TDefaultChunkIterator(this); + } + + TAutoPtr<IChunkBidirIterator> MakeBidirIterator() const override { + return new TDefaultChunkBidirIterator(this); + } + +private: + template <typename TIterator> + class TDefaultChunkIteratorOps : public TIterator { + public: + TDefaultChunkIteratorOps(const IChunkDecoder* decoder) + : Decoder(decoder) + , Index(0) + { } + + TDataRef Next() override { + return Decoder->GetValue(Index++); + } + + TDataRef Peek() const override { + return Decoder->GetValue(Index); + } + + protected: + const IChunkDecoder* Decoder; + size_t Index; + }; + + using TDefaultChunkIterator = TDefaultChunkIteratorOps<IChunkIterator>; + + class TDefaultChunkBidirIterator : public TDefaultChunkIteratorOps<IChunkBidirIterator> { + public: + TDefaultChunkBidirIterator(const IChunkDecoder* decoder) + : TDefaultChunkIteratorOps<IChunkBidirIterator>(decoder) + { } + + void Back() override { Y_VERIFY_DEBUG(Index > 0); - --Index; - } - }; -}; - -template <typename TDerived, ui16 Type, bool IsNullable> -class TChunkDecoderBase : public TDefaultChunkDecoder { -public: + --Index; + } + }; +}; + +template <typename TDerived, ui16 Type, bool IsNullable> +class TChunkDecoderBase : public TDefaultChunkDecoder { +public: inline static TCodecSig Sig() { return TCodecSig(TCodecType(Type), IsNullable); } - + TCodecSig Signature() const override { return Sig(); } - IChunkDecoder::TPtr ReadNext(const TDataRef& data, const TTypeCodecs* codecs) const override { - return ReadNextImpl(data, codecs); - } - - static inline IChunkDecoder::TPtr ReadNextImpl(const TDataRef& data, const TTypeCodecs* codecs) { + IChunkDecoder::TPtr ReadNext(const TDataRef& data, const TTypeCodecs* codecs) const override { + return ReadNextImpl(data, codecs); + } + + static inline IChunkDecoder::TPtr ReadNextImpl(const TDataRef& data, const TTypeCodecs* codecs) { Y_VERIFY_DEBUG(data.Size() >= sizeof(TCodecSig)); const TCodecSig sig = ReadUnaligned<TCodecSig>(data.Data()); if (Y_LIKELY(sig == Sig())) { - return new TDerived(data); - } else { - auto codec = codecs->GetCodec(sig); + return new TDerived(data); + } else { + auto codec = codecs->GetCodec(sig); Y_VERIFY(codec, "Unknown codec signature."); - return codec->ReadChunk(data); - } - } - -protected: - static inline void VerifySignature(const TDataRef& data) { + return codec->ReadChunk(data); + } + } + +protected: + static inline void VerifySignature(const TDataRef& data) { Y_VERIFY(data.Size() >= sizeof(TCodecSig)); Y_VERIFY(ReadUnaligned<TCodecSig>(data.Data()) == Sig()); - } -}; - -//////////////////////////////////////////////////////////////////////////////// - -template <bool IsNullable> -class TDecoderMask; - -template <bool IsNullable> -class TDecoderMaskIterator; - -template <> -class TDecoderMask<true> { -public: - TDecoderMask(const TDataRef& data) + } +}; + +//////////////////////////////////////////////////////////////////////////////// + +template <bool IsNullable> +class TDecoderMask; + +template <bool IsNullable> +class TDecoderMaskIterator; + +template <> +class TDecoderMask<true> { +public: + TDecoderMask(const TDataRef& data) : MaskSize(ReadUnaligned<ui32>(data.End() - sizeof(ui32))) , Bits((const ui8*)(data.End() - Size())) - { } - - inline size_t Size() const { return MaskSize + sizeof(ui32); } - - inline bool IsNotNull(size_t index) const { - TMaskIterator iter(Bits); - iter.Forward(index); - return iter.Peek(); - } - -private: - size_t MaskSize; + { } + + inline size_t Size() const { return MaskSize + sizeof(ui32); } + + inline bool IsNotNull(size_t index) const { + TMaskIterator iter(Bits); + iter.Forward(index); + return iter.Peek(); + } + +private: + size_t MaskSize; const ui8* Bits; -}; - -template <> -class TDecoderMaskIterator<true> { -public: - TDecoderMaskIterator(const TDataRef& data) { - const char* sizePtr = data.Data() + data.Size() - sizeof(ui32); +}; + +template <> +class TDecoderMaskIterator<true> { +public: + TDecoderMaskIterator(const TDataRef& data) { + const char* sizePtr = data.Data() + data.Size() - sizeof(ui32); BitIter.Reset((const ui8*)(sizePtr - ReadUnaligned<ui32>(sizePtr))); - } - inline bool IsNotNull() const { return BitIter.Peek(); } - inline bool Next() { return BitIter.Next(); } - -private: - TMaskIterator BitIter; -}; - -template <> -class TDecoderMask<false> { -public: - TDecoderMask(const TDataRef&) { } - - inline size_t Size() const { return 0; } - inline bool IsNotNull(size_t) const { return true; } -}; - -template <> -class TDecoderMaskIterator<false> { -public: - TDecoderMaskIterator(const TDataRef&) { } - - inline bool IsNotNull() const { return true; } - inline bool Next() { return true; } -}; - -//////////////////////////////////////////////////////////////////////////////// - -template <size_t Size, bool IsNullable> -class TFixedLenDecoder : public TChunkDecoderBase<TFixedLenDecoder<Size, IsNullable>, ui16(TCodecType::FixedLen), IsNullable> { -public: - TFixedLenDecoder(const TDataRef& data) - : Data(data) - , Mask(Data) - { - TFixedLenDecoder::VerifySignature(data); - } - - TAutoPtr<IChunkIterator> MakeIterator() const override { + } + inline bool IsNotNull() const { return BitIter.Peek(); } + inline bool Next() { return BitIter.Next(); } + +private: + TMaskIterator BitIter; +}; + +template <> +class TDecoderMask<false> { +public: + TDecoderMask(const TDataRef&) { } + + inline size_t Size() const { return 0; } + inline bool IsNotNull(size_t) const { return true; } +}; + +template <> +class TDecoderMaskIterator<false> { +public: + TDecoderMaskIterator(const TDataRef&) { } + + inline bool IsNotNull() const { return true; } + inline bool Next() { return true; } +}; + +//////////////////////////////////////////////////////////////////////////////// + +template <size_t Size, bool IsNullable> +class TFixedLenDecoder : public TChunkDecoderBase<TFixedLenDecoder<Size, IsNullable>, ui16(TCodecType::FixedLen), IsNullable> { +public: + TFixedLenDecoder(const TDataRef& data) + : Data(data) + , Mask(Data) + { + TFixedLenDecoder::VerifySignature(data); + } + + TAutoPtr<IChunkIterator> MakeIterator() const override { return new TChunkIterator(Data, Mask.Size()); - } - - TDataRef GetValue(size_t index) const override { - if (Mask.IsNotNull(index)) { - const char* data = Data.Data() + sizeof(TCodecSig) + index * Size; - return TDataRef(data, Size); - } - return TDataRef(); - } - -private: - class TChunkIterator : public IChunkIterator { - public: + } + + TDataRef GetValue(size_t index) const override { + if (Mask.IsNotNull(index)) { + const char* data = Data.Data() + sizeof(TCodecSig) + index * Size; + return TDataRef(data, Size); + } + return TDataRef(); + } + +private: + class TChunkIterator : public IChunkIterator { + public: TChunkIterator(const TDataRef& data, const size_t maskSize) - : Current(data.Data() + sizeof(TCodecSig)) + : Current(data.Data() + sizeof(TCodecSig)) , End(data.End() - maskSize) - , MaskIter(data) - { } - - TDataRef Next() override { - const char* data = Current; - Current += Size; + , MaskIter(data) + { } + + TDataRef Next() override { + const char* data = Current; + Current += Size; Y_VERIFY(Current <= End); - return MaskIter.Next() ? TDataRef(data, Size) : TDataRef(); - } - - TDataRef Peek() const override { + return MaskIter.Next() ? TDataRef(data, Size) : TDataRef(); + } + + TDataRef Peek() const override { Y_VERIFY(Current + Size <= End); - return MaskIter.IsNotNull() ? TDataRef(Current, Size) : TDataRef(); - } - - private: - const char* Current; + return MaskIter.IsNotNull() ? TDataRef(Current, Size) : TDataRef(); + } + + private: + const char* Current; const char* End; - TDecoderMaskIterator<IsNullable> MaskIter; - }; - -private: - TDataRef Data; - TDecoderMask<IsNullable> Mask; -}; - -//////////////////////////////////////////////////////////////////////////////// - -template <bool IsNullable> -class TVarLenDecoder : public TChunkDecoderBase<TVarLenDecoder<IsNullable>, ui16(TCodecType::VarLen), IsNullable> { -public: - TVarLenDecoder(const TDataRef& data) - : Data(data) - , Sizes((const ui32*)(data.Data() + sizeof(TCodecSig))) - , Mask(Data) - { - TVarLenDecoder::VerifySignature(data); - const char* sizesEnd = data.End() - Mask.Size(); - if ((const char*)(Sizes + 1) <= sizesEnd) { + TDecoderMaskIterator<IsNullable> MaskIter; + }; + +private: + TDataRef Data; + TDecoderMask<IsNullable> Mask; +}; + +//////////////////////////////////////////////////////////////////////////////// + +template <bool IsNullable> +class TVarLenDecoder : public TChunkDecoderBase<TVarLenDecoder<IsNullable>, ui16(TCodecType::VarLen), IsNullable> { +public: + TVarLenDecoder(const TDataRef& data) + : Data(data) + , Sizes((const ui32*)(data.Data() + sizeof(TCodecSig))) + , Mask(Data) + { + TVarLenDecoder::VerifySignature(data); + const char* sizesEnd = data.End() - Mask.Size(); + if ((const char*)(Sizes + 1) <= sizesEnd) { auto dataSize = ReadUnaligned<ui32>((const ui32*)sizesEnd - 1); - Sizes = (const ui32*)((const char*)Sizes + dataSize); - } - } - - TAutoPtr<IChunkIterator> MakeIterator() const override { + Sizes = (const ui32*)((const char*)Sizes + dataSize); + } + } + + TAutoPtr<IChunkIterator> MakeIterator() const override { return new TChunkIterator(Data, Sizes, Data.End() - Mask.Size()); - } - - TDataRef GetValue(size_t index) const override { - if (Mask.IsNotNull(index)) { + } + + TDataRef GetValue(size_t index) const override { + if (Mask.IsNotNull(index)) { Y_VERIFY(ReadUnaligned<ui32>(Sizes + index) + sizeof(TCodecSig) <= Data.Size() - Mask.Size()); ui32 begin = index ? ReadUnaligned<ui32>(Sizes + index - 1) : 0; return TDataRef(Data.Data() + sizeof(TCodecSig) + begin, ReadUnaligned<ui32>(Sizes + index) - begin); - } - return TDataRef(); - } - -private: - class TChunkIterator : public IChunkIterator { - public: + } + return TDataRef(); + } + +private: + class TChunkIterator : public IChunkIterator { + public: TChunkIterator(const TDataRef& data, const ui32* sizes, const char* end) - : Current(data.Data() + sizeof(TCodecSig)) + : Current(data.Data() + sizeof(TCodecSig)) , End(end) - , LastOffset(0) - , CurrentOffset(sizes) - , MaskIter(data) - { } - - TDataRef Next() override { - if (MaskIter.Next()) { + , LastOffset(0) + , CurrentOffset(sizes) + , MaskIter(data) + { } + + TDataRef Next() override { + if (MaskIter.Next()) { ui32 endOffset = ReadUnaligned<ui32>(CurrentOffset); CurrentOffset++; - ui32 size = endOffset - LastOffset; - LastOffset = endOffset; - - const char* data = Current; - Current += size; + ui32 size = endOffset - LastOffset; + LastOffset = endOffset; + + const char* data = Current; + Current += size; Y_VERIFY(data + size <= End); - return TDataRef(data, size); - } - ++CurrentOffset; - return TDataRef(); - } - - TDataRef Peek() const override { + return TDataRef(data, size); + } + ++CurrentOffset; + return TDataRef(); + } + + TDataRef Peek() const override { if (MaskIter.IsNotNull()) { Y_VERIFY(Current + ReadUnaligned<ui32>(CurrentOffset) - LastOffset <= End); return TDataRef(Current, ReadUnaligned<ui32>(CurrentOffset) - LastOffset); } - return TDataRef(); - } - - private: - const char* Current; + return TDataRef(); + } + + private: + const char* Current; const char* End; - ui32 LastOffset; - const ui32* CurrentOffset; - TDecoderMaskIterator<IsNullable> MaskIter; - }; - -private: - TDataRef Data; - const ui32* Sizes; - TDecoderMask<IsNullable> Mask; -}; - -//////////////////////////////////////////////////////////////////////////////// - -template <typename TIntType> -class TVarIntValueDecoder { -public: - using TType = TIntType; - + ui32 LastOffset; + const ui32* CurrentOffset; + TDecoderMaskIterator<IsNullable> MaskIter; + }; + +private: + TDataRef Data; + const ui32* Sizes; + TDecoderMask<IsNullable> Mask; +}; + +//////////////////////////////////////////////////////////////////////////////// + +template <typename TIntType> +class TVarIntValueDecoder { +public: + using TType = TIntType; + inline TType Peek(const char* data, const char* end) const { - i64 value; + i64 value; auto bytes = in_long(value, data); Y_VERIFY(data + bytes <= end); - return value; - } - + return value; + } + inline size_t Load(const char* data, const char* end, TType& value) const { - i64 loaded = 0; - auto bytes = in_long(loaded, data); + i64 loaded = 0; + auto bytes = in_long(loaded, data); Y_VERIFY(data + bytes <= end); - value = loaded; - return bytes; - } -}; - -template <typename TIntType> -class TZigZagValueDecoder { -public: - using TType = TIntType; + value = loaded; + return bytes; + } +}; + +template <typename TIntType> +class TZigZagValueDecoder { +public: + using TType = TIntType; using TUnsigned = std::make_unsigned_t<TType>; - + inline TType Peek(const char* data, const char* end) const { - i64 value; + i64 value; auto bytes = in_long(value, data); Y_VERIFY(data + bytes <= end); - return ZigZagDecode(static_cast<TUnsigned>(value)); - } - + return ZigZagDecode(static_cast<TUnsigned>(value)); + } + inline size_t Load(const char* data, const char* end, TType& value) const { - i64 loaded = 0; - auto bytes = in_long(loaded, data); + i64 loaded = 0; + auto bytes = in_long(loaded, data); Y_VERIFY(data + bytes <= end); - value = ZigZagDecode(static_cast<TUnsigned>(loaded)); - return bytes; - } -}; - -template <typename TValueDecoder, bool Rev = false> -class TDeltaValueDecoder : public TValueDecoder { -public: - using TType = typename TValueDecoder::TType; - + value = ZigZagDecode(static_cast<TUnsigned>(loaded)); + return bytes; + } +}; + +template <typename TValueDecoder, bool Rev = false> +class TDeltaValueDecoder : public TValueDecoder { +public: + using TType = typename TValueDecoder::TType; + inline TType Peek(const char* data, const char* end) const { - TType value; + TType value; TValueDecoder::Load(data, end, value); - return Rev ? Last - value : Last + value; - } - + return Rev ? Last - value : Last + value; + } + inline size_t Load(const char* data, const char* end, TType& value) { auto bytes = TValueDecoder::Load(data, end, value); - if (Rev) - Last -= value; - else - Last += value; - value = Last; - return bytes; - } - -private: - TType Last = 0; -}; - -template <typename TDerived, typename TValueDecoder, ui16 CodecType, bool IsNullable> -class TIntDecoderImpl : public TChunkDecoderBase<TDerived, CodecType, IsNullable> { -public: - using TType = typename TValueDecoder::TType; - - TIntDecoderImpl(const TDataRef& data) - : Data(data) - , Mask(Data) - , Fetched(Data.Data() + sizeof(TCodecSig)) - , MaskIter(Data) - { - TIntDecoderImpl::VerifySignature(data); - } - - TAutoPtr<IChunkIterator> MakeIterator() const override { + if (Rev) + Last -= value; + else + Last += value; + value = Last; + return bytes; + } + +private: + TType Last = 0; +}; + +template <typename TDerived, typename TValueDecoder, ui16 CodecType, bool IsNullable> +class TIntDecoderImpl : public TChunkDecoderBase<TDerived, CodecType, IsNullable> { +public: + using TType = typename TValueDecoder::TType; + + TIntDecoderImpl(const TDataRef& data) + : Data(data) + , Mask(Data) + , Fetched(Data.Data() + sizeof(TCodecSig)) + , MaskIter(Data) + { + TIntDecoderImpl::VerifySignature(data); + } + + TAutoPtr<IChunkIterator> MakeIterator() const override { return new TChunkIterator(Data, Mask.Size()); - } - - TDataRef GetValue(size_t index) const override { - if (Mask.IsNotNull(index)) { - FetchTo(index); - return GetFromCache(index); - } - return TDataRef(); - } - -private: - inline void FetchTo(size_t index) const { - if (index < Cache.size()) - return; - TType value; - for (size_t count = index + 1 - Cache.size(); count; --count) - if (MaskIter.Next()) { + } + + TDataRef GetValue(size_t index) const override { + if (Mask.IsNotNull(index)) { + FetchTo(index); + return GetFromCache(index); + } + return TDataRef(); + } + +private: + inline void FetchTo(size_t index) const { + if (index < Cache.size()) + return; + TType value; + for (size_t count = index + 1 - Cache.size(); count; --count) + if (MaskIter.Next()) { Fetched += ValueDecoder.Load(Fetched, Data.End() - Mask.Size(), value); - Cache.push_back(value); - } else { - Cache.push_back(0); - } - } - - inline TDataRef GetFromCache(size_t index) const { + Cache.push_back(value); + } else { + Cache.push_back(0); + } + } + + inline TDataRef GetFromCache(size_t index) const { Y_VERIFY_DEBUG(index < Cache.size()); - return TDataRef((const char*)&Cache[index], sizeof(TType)); - } - -private: - class TChunkIterator : public IChunkIterator { - public: + return TDataRef((const char*)&Cache[index], sizeof(TType)); + } + +private: + class TChunkIterator : public IChunkIterator { + public: TChunkIterator(const TDataRef& data, const size_t maskSize) - : Current(data.Data() + sizeof(TCodecSig)) + : Current(data.Data() + sizeof(TCodecSig)) , End(data.End() - maskSize) - , MaskIter(data) - { } - - TDataRef Next() override { - if (MaskIter.Next()) { - TType value; + , MaskIter(data) + { } + + TDataRef Next() override { + if (MaskIter.Next()) { + TType value; Current += ValueDecoder.Load(Current, End, value); - return TDataRef((const char*)&value, sizeof(value), true); - } - return TDataRef(); - } - - TDataRef Peek() const override { - if (MaskIter.IsNotNull()) { + return TDataRef((const char*)&value, sizeof(value), true); + } + return TDataRef(); + } + + TDataRef Peek() const override { + if (MaskIter.IsNotNull()) { const TType value = ValueDecoder.Peek(Current, End); - return TDataRef((const char*)&value, sizeof(value), true); - } - return TDataRef(); - } - - private: - const char* Current; + return TDataRef((const char*)&value, sizeof(value), true); + } + return TDataRef(); + } + + private: + const char* Current; const char* End; - TDecoderMaskIterator<IsNullable> MaskIter; - TValueDecoder ValueDecoder; - }; - + TDecoderMaskIterator<IsNullable> MaskIter; + TValueDecoder ValueDecoder; + }; + using TCache = TDeque<TType>; - -private: - TDataRef Data; - TDecoderMask<IsNullable> Mask; - - mutable const char* Fetched; - mutable TDecoderMaskIterator<IsNullable> MaskIter; - mutable TCache Cache; - mutable TValueDecoder ValueDecoder; -}; - -template <typename TIntType, bool IsNullable> -class TVarIntDecoder : public TIntDecoderImpl<TVarIntDecoder<TIntType, IsNullable>, TVarIntValueDecoder<TIntType>, ui16(TCodecType::VarInt), IsNullable> { -public: - TVarIntDecoder(const TDataRef& data) - : TIntDecoderImpl<TVarIntDecoder<TIntType, IsNullable>, TVarIntValueDecoder<TIntType>, ui16(TCodecType::VarInt), IsNullable>(data) - { } -}; - -template <typename TIntType, bool IsNullable> -class TZigZagDecoder : public TIntDecoderImpl<TZigZagDecoder<TIntType, IsNullable>, TZigZagValueDecoder<TIntType>, ui16(TCodecType::ZigZag), IsNullable> { -public: - TZigZagDecoder(const TDataRef& data) - : TIntDecoderImpl<TZigZagDecoder<TIntType, IsNullable>, TZigZagValueDecoder<TIntType>, ui16(TCodecType::ZigZag), IsNullable>(data) - { } -}; - -template <typename TIntType, bool IsNullable> -class TDeltaVarIntDecoder : public TIntDecoderImpl<TDeltaVarIntDecoder<TIntType, IsNullable>, TDeltaValueDecoder<TVarIntValueDecoder<TIntType>>, ui16(TCodecType::DeltaVarInt), IsNullable> { -public: - TDeltaVarIntDecoder(const TDataRef& data) - : TIntDecoderImpl<TDeltaVarIntDecoder<TIntType, IsNullable>, TDeltaValueDecoder<TVarIntValueDecoder<TIntType>>, ui16(TCodecType::DeltaVarInt), IsNullable>(data) - { } -}; - -template <typename TIntType, bool IsNullable> -class TDeltaRevVarIntDecoder : public TIntDecoderImpl<TDeltaRevVarIntDecoder<TIntType, IsNullable>, TDeltaValueDecoder<TVarIntValueDecoder<TIntType>, true>, ui16(TCodecType::DeltaRevVarInt), IsNullable> { -public: - TDeltaRevVarIntDecoder(const TDataRef& data) - : TIntDecoderImpl<TDeltaRevVarIntDecoder<TIntType, IsNullable>, TDeltaValueDecoder<TVarIntValueDecoder<TIntType>, true>, ui16(TCodecType::DeltaRevVarInt), IsNullable>(data) - { } -}; - -template <typename TIntType, bool IsNullable> -class TDeltaZigZagDecoder : public TIntDecoderImpl<TDeltaZigZagDecoder<TIntType, IsNullable>, TDeltaValueDecoder<TZigZagValueDecoder<TIntType>>, ui16(TCodecType::DeltaZigZag), IsNullable> { -public: - TDeltaZigZagDecoder(const TDataRef& data) - : TIntDecoderImpl<TDeltaZigZagDecoder<TIntType, IsNullable>, TDeltaValueDecoder<TZigZagValueDecoder<TIntType>>, ui16(TCodecType::DeltaZigZag), IsNullable>(data) - { } -}; - -/// TODO: FetchAll / FetchOnDemand / DoNotCache options. - -//////////////////////////////////////////////////////////////////////////////// - -template <bool IsNullable> -class TBoolDecoder : public TChunkDecoderBase<TBoolDecoder<IsNullable>, ui16(TCodecType::Bool), IsNullable> { -public: - TBoolDecoder(const TDataRef& data) - : Data(data) - { - TBoolDecoder::VerifySignature(data); - } - - TAutoPtr<IChunkIterator> MakeIterator() const override { - return new TChunkIterator(Data.Data() + sizeof(TCodecSig)); - } - - TDataRef GetValue(size_t index) const override { + +private: + TDataRef Data; + TDecoderMask<IsNullable> Mask; + + mutable const char* Fetched; + mutable TDecoderMaskIterator<IsNullable> MaskIter; + mutable TCache Cache; + mutable TValueDecoder ValueDecoder; +}; + +template <typename TIntType, bool IsNullable> +class TVarIntDecoder : public TIntDecoderImpl<TVarIntDecoder<TIntType, IsNullable>, TVarIntValueDecoder<TIntType>, ui16(TCodecType::VarInt), IsNullable> { +public: + TVarIntDecoder(const TDataRef& data) + : TIntDecoderImpl<TVarIntDecoder<TIntType, IsNullable>, TVarIntValueDecoder<TIntType>, ui16(TCodecType::VarInt), IsNullable>(data) + { } +}; + +template <typename TIntType, bool IsNullable> +class TZigZagDecoder : public TIntDecoderImpl<TZigZagDecoder<TIntType, IsNullable>, TZigZagValueDecoder<TIntType>, ui16(TCodecType::ZigZag), IsNullable> { +public: + TZigZagDecoder(const TDataRef& data) + : TIntDecoderImpl<TZigZagDecoder<TIntType, IsNullable>, TZigZagValueDecoder<TIntType>, ui16(TCodecType::ZigZag), IsNullable>(data) + { } +}; + +template <typename TIntType, bool IsNullable> +class TDeltaVarIntDecoder : public TIntDecoderImpl<TDeltaVarIntDecoder<TIntType, IsNullable>, TDeltaValueDecoder<TVarIntValueDecoder<TIntType>>, ui16(TCodecType::DeltaVarInt), IsNullable> { +public: + TDeltaVarIntDecoder(const TDataRef& data) + : TIntDecoderImpl<TDeltaVarIntDecoder<TIntType, IsNullable>, TDeltaValueDecoder<TVarIntValueDecoder<TIntType>>, ui16(TCodecType::DeltaVarInt), IsNullable>(data) + { } +}; + +template <typename TIntType, bool IsNullable> +class TDeltaRevVarIntDecoder : public TIntDecoderImpl<TDeltaRevVarIntDecoder<TIntType, IsNullable>, TDeltaValueDecoder<TVarIntValueDecoder<TIntType>, true>, ui16(TCodecType::DeltaRevVarInt), IsNullable> { +public: + TDeltaRevVarIntDecoder(const TDataRef& data) + : TIntDecoderImpl<TDeltaRevVarIntDecoder<TIntType, IsNullable>, TDeltaValueDecoder<TVarIntValueDecoder<TIntType>, true>, ui16(TCodecType::DeltaRevVarInt), IsNullable>(data) + { } +}; + +template <typename TIntType, bool IsNullable> +class TDeltaZigZagDecoder : public TIntDecoderImpl<TDeltaZigZagDecoder<TIntType, IsNullable>, TDeltaValueDecoder<TZigZagValueDecoder<TIntType>>, ui16(TCodecType::DeltaZigZag), IsNullable> { +public: + TDeltaZigZagDecoder(const TDataRef& data) + : TIntDecoderImpl<TDeltaZigZagDecoder<TIntType, IsNullable>, TDeltaValueDecoder<TZigZagValueDecoder<TIntType>>, ui16(TCodecType::DeltaZigZag), IsNullable>(data) + { } +}; + +/// TODO: FetchAll / FetchOnDemand / DoNotCache options. + +//////////////////////////////////////////////////////////////////////////////// + +template <bool IsNullable> +class TBoolDecoder : public TChunkDecoderBase<TBoolDecoder<IsNullable>, ui16(TCodecType::Bool), IsNullable> { +public: + TBoolDecoder(const TDataRef& data) + : Data(data) + { + TBoolDecoder::VerifySignature(data); + } + + TAutoPtr<IChunkIterator> MakeIterator() const override { + return new TChunkIterator(Data.Data() + sizeof(TCodecSig)); + } + + TDataRef GetValue(size_t index) const override { TMaskIterator iter((const ui8*)(Data.Data() + sizeof(TCodecSig))); - iter.Forward(IsNullable ? index * 2 : index); - if (IsNullable && !iter.Next()) - return TDataRef(); - return GetValue(iter.Peek()); - } - -private: - class TChunkIterator : public IChunkIterator { - public: - TChunkIterator(const char* data) + iter.Forward(IsNullable ? index * 2 : index); + if (IsNullable && !iter.Next()) + return TDataRef(); + return GetValue(iter.Peek()); + } + +private: + class TChunkIterator : public IChunkIterator { + public: + TChunkIterator(const char* data) : BitIter((const ui8*)data) - { } - - TDataRef Next() override { - return IsNullable ? ParseNullable(BitIter.Read(2)) : GetValue(BitIter.Next()); - } - - TDataRef Peek() const override { - return IsNullable ? ParseNullable(BitIter.Peek(2)) : GetValue(BitIter.Peek()); - } - - private: - static inline TDataRef ParseNullable(ui16 word) { - return (word & 1) ? GetValue(word >> 1) : TDataRef(); - } - - private: - TMaskIterator BitIter; - }; - -private: - static const bool Values[2]; - - static inline TDataRef GetValue(int index) { - return TDataRef((const char*)&Values[index], sizeof(bool)); - } - - TDataRef Data; -}; - -template <bool IsNullable> -const bool TBoolDecoder<IsNullable>::Values[2] = {false, true}; - -//////////////////////////////////////////////////////////////////////////////// - + { } + + TDataRef Next() override { + return IsNullable ? ParseNullable(BitIter.Read(2)) : GetValue(BitIter.Next()); + } + + TDataRef Peek() const override { + return IsNullable ? ParseNullable(BitIter.Peek(2)) : GetValue(BitIter.Peek()); + } + + private: + static inline TDataRef ParseNullable(ui16 word) { + return (word & 1) ? GetValue(word >> 1) : TDataRef(); + } + + private: + TMaskIterator BitIter; + }; + +private: + static const bool Values[2]; + + static inline TDataRef GetValue(int index) { + return TDataRef((const char*)&Values[index], sizeof(bool)); + } + + TDataRef Data; +}; + +template <bool IsNullable> +const bool TBoolDecoder<IsNullable>::Values[2] = {false, true}; + +//////////////////////////////////////////////////////////////////////////////// + } // namespace NScheme } // namespace NKikimr - + diff --git a/ydb/core/protos/msgbus.proto b/ydb/core/protos/msgbus.proto index da4b93fbf1..df7cda5980 100644 --- a/ydb/core/protos/msgbus.proto +++ b/ydb/core/protos/msgbus.proto @@ -158,7 +158,7 @@ message TResponse { ]; // Program "exit code": see NMiniKQL::IEngine::EStatus. optional NKikimrMiniKQL.TResult ExecutionEngineEvaluatedResponse = 103; - + optional bytes MiniKQLErrors = 104; optional bytes DataShardErrors = 105; repeated fixed64 ComplainingDataShards = 106; @@ -171,7 +171,7 @@ message TResponse { optional uint32 BSAssignedID = 40; - optional NKikimrTxUserProxy.TTxProxyTimings ProxyTimings = 50; + optional NKikimrTxUserProxy.TTxProxyTimings ProxyTimings = 50; optional int32 SchemeStatus = 300; // for enum definition see flat_tx_scheme.proto - enum EStatus optional uint32 SchemeTagId = 301; diff --git a/ydb/core/protos/tx_proxy.proto b/ydb/core/protos/tx_proxy.proto index 4594205b27..dedd2f9cbb 100644 --- a/ydb/core/protos/tx_proxy.proto +++ b/ydb/core/protos/tx_proxy.proto @@ -196,18 +196,18 @@ message TEvProposeTransaction { optional string RequestType = 8; } -message TTxProxyTimings { - optional uint64 WallClockAccepted = 10; - optional uint64 WallClockResolved = 11; - optional uint64 WallClockPrepared = 12; - optional uint64 WallClockPlanned = 13; - optional uint64 WallClockNow = 14; - optional uint64 ElapsedPrepareExec = 15; - optional uint64 ElapsedExecExec = 16; - optional uint64 ElapsedPrepareComplete = 17; - optional uint64 ElapsedExecComplete = 18; -} - +message TTxProxyTimings { + optional uint64 WallClockAccepted = 10; + optional uint64 WallClockResolved = 11; + optional uint64 WallClockPrepared = 12; + optional uint64 WallClockPlanned = 13; + optional uint64 WallClockNow = 14; + optional uint64 ElapsedPrepareExec = 15; + optional uint64 ElapsedExecExec = 16; + optional uint64 ElapsedPrepareComplete = 17; + optional uint64 ElapsedExecComplete = 18; +} + message TEvProposeTransactionStatus { optional uint32 Status = 1; optional fixed64 TxId = 2; @@ -218,7 +218,7 @@ message TEvProposeTransactionStatus { optional uint32 ExecutionEngineStatus = 10; optional uint32 ExecutionEngineResponseStatus = 11; - optional bytes ExecutionEngineResponse = 12; // Native document. + optional bytes ExecutionEngineResponse = 12; // Native document. optional bytes MiniKQLErrors = 13; optional bytes DataShardErrors = 14; repeated fixed64 ComplainingDataShards = 15; @@ -228,7 +228,7 @@ message TEvProposeTransactionStatus { optional NKikimrMiniKQL.TResult ExecutionEngineEvaluatedResponse = 18; optional NKikimrQueryStats.TTxStats TxStats = 19; - optional TTxProxyTimings Timings = 20; + optional TTxProxyTimings Timings = 20; optional bool HadFollowerReads = 21; optional uint32 SchemeShardStatus = 30; diff --git a/ydb/core/scheme/scheme_types_auto.h b/ydb/core/scheme/scheme_types_auto.h index 426d4e62aa..06264b4c73 100644 --- a/ydb/core/scheme/scheme_types_auto.h +++ b/ydb/core/scheme/scheme_types_auto.h @@ -1,53 +1,53 @@ -#pragma once - -#include "scheme_type_id.h" -#include "scheme_types_defs.h" - - -namespace NKikimr { -namespace NScheme { - -template <typename TRawType> -struct TTypeAuto; - -template <typename TRawType> -struct TTypeIsAuto { - enum { - Value = false - }; -}; - -#define DEFINE_TYPE_AUTO(TRawType, TImplType) \ -template<> \ -struct TTypeAuto<TRawType> { \ - static const TTypeId TypeId = TImplType::TypeId; \ - using TInstance = typename TImplType::TInstance; \ - using TType = TImplType; \ -}; \ -template <> \ -struct TTypeIsAuto<TRawType> { \ - enum { \ - Value = true \ - }; \ -}; - -DEFINE_TYPE_AUTO(i32, TInt32); -DEFINE_TYPE_AUTO(ui32, TUint32); -DEFINE_TYPE_AUTO(i64, TInt64); -DEFINE_TYPE_AUTO(ui64, TUint64); +#pragma once + +#include "scheme_type_id.h" +#include "scheme_types_defs.h" + + +namespace NKikimr { +namespace NScheme { + +template <typename TRawType> +struct TTypeAuto; + +template <typename TRawType> +struct TTypeIsAuto { + enum { + Value = false + }; +}; + +#define DEFINE_TYPE_AUTO(TRawType, TImplType) \ +template<> \ +struct TTypeAuto<TRawType> { \ + static const TTypeId TypeId = TImplType::TypeId; \ + using TInstance = typename TImplType::TInstance; \ + using TType = TImplType; \ +}; \ +template <> \ +struct TTypeIsAuto<TRawType> { \ + enum { \ + Value = true \ + }; \ +}; + +DEFINE_TYPE_AUTO(i32, TInt32); +DEFINE_TYPE_AUTO(ui32, TUint32); +DEFINE_TYPE_AUTO(i64, TInt64); +DEFINE_TYPE_AUTO(ui64, TUint64); DEFINE_TYPE_AUTO(ui8, TUint8); -DEFINE_TYPE_AUTO(bool, TBool); -DEFINE_TYPE_AUTO(float, TFloat); -DEFINE_TYPE_AUTO(double, TDouble); - +DEFINE_TYPE_AUTO(bool, TBool); +DEFINE_TYPE_AUTO(float, TFloat); +DEFINE_TYPE_AUTO(double, TDouble); + DEFINE_TYPE_AUTO(TString, TString); DEFINE_TYPE_AUTO(const char*, TString); - + using TPairUi64Ui64Type = std::pair<ui64, ui64>; -DEFINE_TYPE_AUTO(TPairUi64Ui64Type, TPairUi64Ui64); - -#undef DEFINE_TYPE_AUTO - -} // namespace NScheme -} // namespace NKikimr - +DEFINE_TYPE_AUTO(TPairUi64Ui64Type, TPairUi64Ui64); + +#undef DEFINE_TYPE_AUTO + +} // namespace NScheme +} // namespace NKikimr + diff --git a/ydb/core/scheme/tablet_scheme_defs.h b/ydb/core/scheme/tablet_scheme_defs.h index 48b6be6f03..697638079d 100644 --- a/ydb/core/scheme/tablet_scheme_defs.h +++ b/ydb/core/scheme/tablet_scheme_defs.h @@ -32,15 +32,15 @@ public: bool IsSingular() const { return !ValueType; } - - inline bool operator==(const TTagDetails& other) const { - return Idx == other.Idx && ValueType == other.ValueType && PayloadType == other.PayloadType - && (Name.empty() || other.Name.empty() || Name == other.Name); - } - - inline bool operator!=(const TTagDetails& other) const { - return !operator==(other); - } + + inline bool operator==(const TTagDetails& other) const { + return Idx == other.Idx && ValueType == other.ValueType && PayloadType == other.PayloadType + && (Name.empty() || other.Name.empty() || Name == other.Name); + } + + inline bool operator!=(const TTagDetails& other) const { + return !operator==(other); + } }; //////////////////////////////////////////// diff --git a/ydb/core/scheme_types/scheme_raw_type_value.h b/ydb/core/scheme_types/scheme_raw_type_value.h index 011be30395..1cb0946b6d 100644 --- a/ydb/core/scheme_types/scheme_raw_type_value.h +++ b/ydb/core/scheme_types/scheme_raw_type_value.h @@ -42,7 +42,7 @@ public: TString ToString() const { TStringBuilder builder; - builder << "(type:" << ValueType; + builder << "(type:" << ValueType; if (!IsEmpty()) { builder << ", value:" << TString((const char*)Buffer, BufferSize).Quote(); } @@ -65,7 +65,7 @@ private: }; } // namspace NKikimr - + inline IOutputStream& operator << (IOutputStream& out, const NKikimr::TRawTypeValue& v) { out << v.ToString(); return out; diff --git a/ydb/core/scheme_types/scheme_type_registry.cpp b/ydb/core/scheme_types/scheme_type_registry.cpp index 6f17812d43..d737254361 100644 --- a/ydb/core/scheme_types/scheme_type_registry.cpp +++ b/ydb/core/scheme_types/scheme_type_registry.cpp @@ -5,7 +5,7 @@ #include <util/digest/murmur.h> #include <util/generic/algorithm.h> - + #define REGISTER_TYPE(name, size, ...) RegisterType<T##name>(); namespace NKikimr { diff --git a/ydb/core/scheme_types/scheme_type_registry.h b/ydb/core/scheme_types/scheme_type_registry.h index 9c264e2aca..a82af19267 100644 --- a/ydb/core/scheme_types/scheme_type_registry.h +++ b/ydb/core/scheme_types/scheme_type_registry.h @@ -9,7 +9,7 @@ #include <util/generic/vector.h> #include <util/string/builder.h> - + namespace NKikimr { namespace NScheme { @@ -24,7 +24,7 @@ public: // template <typename T> - void RegisterType() { + void RegisterType() { RegisterType(Singleton<T>()); } @@ -40,14 +40,14 @@ public: // ITypeSP GetType(TTypeId typeId) const { - if (typeId) { + if (typeId) { auto iter = TypeByIdMap.find(typeId); - if (iter != TypeByIdMap.end()) { + if (iter != TypeByIdMap.end()) { Y_VERIFY_DEBUG(iter->second); - return iter->second; - } + return iter->second; + } } - return typeId; + return typeId; } ::TString GetTypeName(TTypeId typeId) const { @@ -62,32 +62,32 @@ public: if (!typeId) ythrow yexception() << "Type id must be non zero"; - auto type = GetType(typeId); + auto type = GetType(typeId); if (Y_LIKELY(type)) - return type; + return type; ythrow yexception() << "Unknown type: " << typeId; } const IType* GetType(const TStringBuf& name) const { auto iter = TypeByNameMap.find(name); - return iter != TypeByNameMap.end() ? iter->second : nullptr; - } + return iter != TypeByNameMap.end() ? iter->second : nullptr; + } const IType* GetKnownType(const TStringBuf& name) const { - auto type = GetType(name); + auto type = GetType(name); if (Y_LIKELY(type)) - return type; - ythrow yexception() << "Unknown type: " << name; + return type; + ythrow yexception() << "Unknown type: " << name; } TVector<const IType*> GetTypes() const { TVector<const IType*> types; - types.reserve(TypeByIdMap.size()); - for (const auto& entry : TypeByIdMap) - types.push_back(entry.second); - return types; - } - + types.reserve(TypeByIdMap.size()); + for (const auto& entry : TypeByIdMap) + types.push_back(entry.second); + return types; + } + TTypeMetadataRegistry& GetTypeMetadataRegistry() { return TypeMetadataRegistry; } diff --git a/ydb/core/scheme_types/scheme_types.h b/ydb/core/scheme_types/scheme_types.h index c887f6fdb2..3034843a24 100644 --- a/ydb/core/scheme_types/scheme_types.h +++ b/ydb/core/scheme_types/scheme_types.h @@ -7,7 +7,7 @@ #include <typeinfo> - + namespace NKikimr { namespace NScheme { @@ -38,7 +38,7 @@ friend class TTypeRegistry; }; //////////////////////////////////////////////////////// -class ITypeSP { +class ITypeSP { public: // ITypeSP(const IType* t = nullptr) @@ -52,13 +52,13 @@ public: {} // - const IType* operator->() const noexcept { return Type; } - explicit operator bool() const noexcept { return TypeId != 0; } + const IType* operator->() const noexcept { return Type; } + explicit operator bool() const noexcept { return TypeId != 0; } - bool IsKnownType() const noexcept { return Type != nullptr; } + bool IsKnownType() const noexcept { return Type != nullptr; } - TTypeId GetTypeId() const noexcept { return TypeId; } - const IType* GetType() const noexcept { return Type; } + TTypeId GetTypeId() const noexcept { return TypeId; } + const IType* GetType() const noexcept { return Type; } private: const IType* Type; diff --git a/ydb/core/scheme_types/scheme_types_defs.cpp b/ydb/core/scheme_types/scheme_types_defs.cpp index 9ce564e437..eeaba2e9af 100644 --- a/ydb/core/scheme_types/scheme_types_defs.cpp +++ b/ydb/core/scheme_types/scheme_types_defs.cpp @@ -34,7 +34,7 @@ namespace NNames { DECLARE_TYPED_TYPE_NAME(DyNumber); } - + void WriteEscapedValue(IOutputStream &out, const char *data, size_t size) { static const size_t BUFFER_SIZE = 32; char buffer[BUFFER_SIZE]; diff --git a/ydb/core/scheme_types/scheme_types_defs.h b/ydb/core/scheme_types/scheme_types_defs.h index 92ddd58f92..bb06b09702 100644 --- a/ydb/core/scheme_types/scheme_types_defs.h +++ b/ydb/core/scheme_types/scheme_types_defs.h @@ -4,7 +4,7 @@ #include "scheme_raw_type_value.h" #include <util/charset/utf8.h> -#include <util/generic/hash.h> +#include <util/generic/hash.h> #include <util/stream/output.h> // for IOutputStream #include <util/string/ascii.h> @@ -38,12 +38,12 @@ public: }; TTypedType() = default; - + // IType interface const char* GetName() const override { return Name_; } - + static constexpr ui32 GetFixedSize() { return sizeof(T); } @@ -169,9 +169,9 @@ void WriteEscapedValue(IOutputStream &out, const char *data, size_t size); class TString : public TStringBase<TString, NTypeIds::String, NNames::String> {}; class TUtf8 : public TStringBase<TUtf8, NTypeIds::Utf8, NNames::Utf8> { -public: -}; - +public: +}; + class TYson : public TStringBase<TYson, NTypeIds::Yson, NNames::Yson> { public: }; diff --git a/ydb/core/tx/tx_proxy/datareq.cpp b/ydb/core/tx/tx_proxy/datareq.cpp index 207a2d30f8..af48b42d60 100644 --- a/ydb/core/tx/tx_proxy/datareq.cpp +++ b/ydb/core/tx/tx_proxy/datareq.cpp @@ -756,23 +756,23 @@ void TDataReq::ReportStatus(TEvTxUserProxy::TEvProposeTransactionStatus::EStatus } if (ProxyFlags & TEvTxUserProxy::TEvProposeTransaction::ProxyTrackWallClock) { - auto *timings = x->Record.MutableTimings(); + auto *timings = x->Record.MutableTimings(); if (WallClockAccepted.GetValue()) - timings->SetWallClockAccepted(WallClockAccepted.MicroSeconds()); + timings->SetWallClockAccepted(WallClockAccepted.MicroSeconds()); if (WallClockResolved.GetValue()) - timings->SetWallClockResolved(WallClockResolved.MicroSeconds()); + timings->SetWallClockResolved(WallClockResolved.MicroSeconds()); if (WallClockPrepared.GetValue()) - timings->SetWallClockPrepared(WallClockPrepared.MicroSeconds()); + timings->SetWallClockPrepared(WallClockPrepared.MicroSeconds()); if (WallClockPlanned.GetValue()) - timings->SetWallClockPlanned(WallClockPlanned.MicroSeconds()); + timings->SetWallClockPlanned(WallClockPlanned.MicroSeconds()); if (ElapsedExecExec.GetValue()) - timings->SetElapsedExecExec(ElapsedExecExec.MicroSeconds()); + timings->SetElapsedExecExec(ElapsedExecExec.MicroSeconds()); if (ElapsedExecComplete.GetValue()) - timings->SetElapsedExecComplete(ElapsedExecComplete.MicroSeconds()); + timings->SetElapsedExecComplete(ElapsedExecComplete.MicroSeconds()); if (ElapsedPrepareExec.GetValue()) - timings->SetElapsedPrepareExec(ElapsedExecExec.MicroSeconds()); + timings->SetElapsedPrepareExec(ElapsedExecExec.MicroSeconds()); if (ElapsedPrepareComplete.GetValue()) - timings->SetElapsedPrepareComplete(ElapsedExecComplete.MicroSeconds()); + timings->SetElapsedPrepareComplete(ElapsedExecComplete.MicroSeconds()); timings->SetWallClockNow(Now().MicroSeconds()); } diff --git a/ydb/core/util/any64.h b/ydb/core/util/any64.h index b4a81ac232..8f810dd2bf 100644 --- a/ydb/core/util/any64.h +++ b/ydb/core/util/any64.h @@ -1,77 +1,77 @@ -#pragma once - -#include <util/system/types.h> -#include <util/system/yassert.h> - -namespace NKikimr { - -class TAny64 { -public: - TAny64(ui64 raw = 0) - : Raw(raw) - { } - - template <typename T> - TAny64(T* pointer) - : Pointer(static_cast<void*>(pointer)) - { /* no-op */ } - - template <typename T> - TAny64(const T& value) - : Raw(value) - { /* no-op */ } - - void Reset() { - Raw = 0; - } - - template <typename T> - T* Release() { - T* result = static_cast<T*>(Pointer); - Raw = 0; - return result; - } - - template <typename T> - void SetPtr(T* pointer) { - Pointer = static_cast<void*>(pointer); - } - - template <typename T> - void SetValue(const T& value) { - Raw = value; - } - - template <typename T> - T* Ptr() { - return static_cast<T*>(Pointer); - } - - template <typename T> - const T* Ptr() const { - return static_cast<const T*>(Pointer); - } - - template <typename T> - T* MutablePtr() const { - return const_cast<T*>(Ptr<T>()); - } - - template <typename T> - T Value() const { - return T(Raw); - } - - operator bool() const { return Pointer; } - -private: - union { - void* Pointer; - ui64 Raw; - }; -}; - -static_assert(sizeof(TAny64) == sizeof(ui64), "Expected TAny64 size violated."); - -} // namespace NKikimr - +#pragma once + +#include <util/system/types.h> +#include <util/system/yassert.h> + +namespace NKikimr { + +class TAny64 { +public: + TAny64(ui64 raw = 0) + : Raw(raw) + { } + + template <typename T> + TAny64(T* pointer) + : Pointer(static_cast<void*>(pointer)) + { /* no-op */ } + + template <typename T> + TAny64(const T& value) + : Raw(value) + { /* no-op */ } + + void Reset() { + Raw = 0; + } + + template <typename T> + T* Release() { + T* result = static_cast<T*>(Pointer); + Raw = 0; + return result; + } + + template <typename T> + void SetPtr(T* pointer) { + Pointer = static_cast<void*>(pointer); + } + + template <typename T> + void SetValue(const T& value) { + Raw = value; + } + + template <typename T> + T* Ptr() { + return static_cast<T*>(Pointer); + } + + template <typename T> + const T* Ptr() const { + return static_cast<const T*>(Pointer); + } + + template <typename T> + T* MutablePtr() const { + return const_cast<T*>(Ptr<T>()); + } + + template <typename T> + T Value() const { + return T(Raw); + } + + operator bool() const { return Pointer; } + +private: + union { + void* Pointer; + ui64 Raw; + }; +}; + +static_assert(sizeof(TAny64) == sizeof(ui64), "Expected TAny64 size violated."); + +} // namespace NKikimr + diff --git a/ydb/library/yql/minikql/mkql_node.h b/ydb/library/yql/minikql/mkql_node.h index 37ec7dbf14..ee74455ea6 100644 --- a/ydb/library/yql/minikql/mkql_node.h +++ b/ydb/library/yql/minikql/mkql_node.h @@ -52,10 +52,10 @@ struct TRuntimeNode { : Data(node, isImmediate) {} - explicit operator bool() const { - return Data.GetPtr(); - } - + explicit operator bool() const { + return Data.GetPtr(); + } + ~TRuntimeNode() {} TType* GetRuntimeType() const; diff --git a/ydb/public/lib/base/defs.h b/ydb/public/lib/base/defs.h index 9e5968fe66..97e933c93a 100644 --- a/ydb/public/lib/base/defs.h +++ b/ydb/public/lib/base/defs.h @@ -8,35 +8,35 @@ namespace NKikimr { -enum class EDataReqStatusExcerpt { - Unknown, // must not happen - Complete, // request success - InProgress, // request success, but result is not ready yet - RejectedForNow, // request rejected for some reason, guarantied to have no meaningful side effects - LostInSpaceAndTime, // smth happened with request, we don't know what (i.e. timeout), side effects (successful execution inclusive) possible - Error, // error with request - InternalError, // smth weird, report to -}; - -inline const char* EDataReqStatusExcerptStr(EDataReqStatusExcerpt status) { - switch (status) { - case EDataReqStatusExcerpt::Complete: - return "Complete"; - case EDataReqStatusExcerpt::InProgress: - return "In progress"; - case EDataReqStatusExcerpt::RejectedForNow: - return "Rejected"; - case EDataReqStatusExcerpt::LostInSpaceAndTime: - return "Lost in space and time"; - case EDataReqStatusExcerpt::Error: - return "Request error"; - case EDataReqStatusExcerpt::InternalError: - return "Internal error"; - default: - return "Unknown error"; - } +enum class EDataReqStatusExcerpt { + Unknown, // must not happen + Complete, // request success + InProgress, // request success, but result is not ready yet + RejectedForNow, // request rejected for some reason, guarantied to have no meaningful side effects + LostInSpaceAndTime, // smth happened with request, we don't know what (i.e. timeout), side effects (successful execution inclusive) possible + Error, // error with request + InternalError, // smth weird, report to +}; + +inline const char* EDataReqStatusExcerptStr(EDataReqStatusExcerpt status) { + switch (status) { + case EDataReqStatusExcerpt::Complete: + return "Complete"; + case EDataReqStatusExcerpt::InProgress: + return "In progress"; + case EDataReqStatusExcerpt::RejectedForNow: + return "Rejected"; + case EDataReqStatusExcerpt::LostInSpaceAndTime: + return "Lost in space and time"; + case EDataReqStatusExcerpt::Error: + return "Request error"; + case EDataReqStatusExcerpt::InternalError: + return "Internal error"; + default: + return "Unknown error"; + } } - + namespace NTxProxy { #define TXUSERPROXY_RESULT_STATUS_MAP(XX) \ XX(Unknown, 0) \ @@ -124,7 +124,7 @@ namespace NTxProxy { } } }; -} +} } |