diff options
author | yazevnul <yazevnul@yandex-team.ru> | 2022-02-10 16:46:48 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:46:48 +0300 |
commit | 9abfb1a53b7f7b791444d1378e645d8fad9b06ed (patch) | |
tree | 49e222ea1c5804306084bb3ae065bb702625360f /library/cpp/containers/comptrie | |
parent | 8cbc307de0221f84c80c42dcbe07d40727537e2c (diff) | |
download | ydb-9abfb1a53b7f7b791444d1378e645d8fad9b06ed.tar.gz |
Restoring authorship annotation for <yazevnul@yandex-team.ru>. Commit 2 of 2.
Diffstat (limited to 'library/cpp/containers/comptrie')
-rw-r--r-- | library/cpp/containers/comptrie/chunked_helpers_trie.h | 6 | ||||
-rw-r--r-- | library/cpp/containers/comptrie/comptrie_builder.h | 16 | ||||
-rw-r--r-- | library/cpp/containers/comptrie/comptrie_builder.inl | 82 | ||||
-rw-r--r-- | library/cpp/containers/comptrie/comptrie_impl.h | 12 | ||||
-rw-r--r-- | library/cpp/containers/comptrie/comptrie_trie.h | 34 | ||||
-rw-r--r-- | library/cpp/containers/comptrie/comptrie_ut.cpp | 122 | ||||
-rw-r--r-- | library/cpp/containers/comptrie/loader/loader.h | 2 | ||||
-rw-r--r-- | library/cpp/containers/comptrie/loader/loader_ut.cpp | 4 | ||||
-rw-r--r-- | library/cpp/containers/comptrie/make_fast_layout.h | 2 | ||||
-rw-r--r-- | library/cpp/containers/comptrie/minimize.h | 2 | ||||
-rw-r--r-- | library/cpp/containers/comptrie/write_trie_backwards.h | 4 |
11 files changed, 143 insertions, 143 deletions
diff --git a/library/cpp/containers/comptrie/chunked_helpers_trie.h b/library/cpp/containers/comptrie/chunked_helpers_trie.h index 993a9f800f..cfa35f5ba2 100644 --- a/library/cpp/containers/comptrie/chunked_helpers_trie.h +++ b/library/cpp/containers/comptrie/chunked_helpers_trie.h @@ -48,7 +48,7 @@ public: return Builder.Find(key, strlen(key), &dummy); } - void Save(IOutputStream& out) const { + void Save(IOutputStream& out) const { Builder.Save(out); } @@ -164,7 +164,7 @@ public: } } - void Save(IOutputStream& out, bool minimize = false) const { + void Save(IOutputStream& out, bool minimize = false) const { if (minimize) { CompactTrieMinimize<TBuilder>(out, Builder, false); } else { @@ -191,7 +191,7 @@ public: Values.push_back(TValue(key, value)); } - void Save(IOutputStream& out) { + void Save(IOutputStream& out) { Sort(Values.begin(), Values.end()); TTrieMapWriter<T, true> writer; for (typename TValues::const_iterator toValue = Values.begin(); toValue != Values.end(); ++toValue) diff --git a/library/cpp/containers/comptrie/comptrie_builder.h b/library/cpp/containers/comptrie/comptrie_builder.h index 8b5adf060c..cf7d2e39a3 100644 --- a/library/cpp/containers/comptrie/comptrie_builder.h +++ b/library/cpp/containers/comptrie/comptrie_builder.h @@ -81,8 +81,8 @@ public: return FindLongestPrefix(key.data(), key.size(), prefixLen, value); } - size_t Save(IOutputStream& os) const; - size_t SaveAndDestroy(IOutputStream& os); + size_t Save(IOutputStream& os) const; + size_t SaveAndDestroy(IOutputStream& os); size_t SaveToFile(const TString& fileName) const { TFixedBufferFileOutput out(fileName); return Save(out); @@ -118,10 +118,10 @@ protected: // If you want both minimization and fast layout, do the minimization first. template <class TPacker> -size_t CompactTrieMinimize(IOutputStream& os, const char* data, size_t datalength, bool verbose = false, const TPacker& packer = TPacker(), NCompactTrie::EMinimizeMode mode = NCompactTrie::MM_DEFAULT); +size_t CompactTrieMinimize(IOutputStream& os, const char* data, size_t datalength, bool verbose = false, const TPacker& packer = TPacker(), NCompactTrie::EMinimizeMode mode = NCompactTrie::MM_DEFAULT); template <class TTrieBuilder> -size_t CompactTrieMinimize(IOutputStream& os, const TTrieBuilder& builder, bool verbose = false); +size_t CompactTrieMinimize(IOutputStream& os, const TTrieBuilder& builder, bool verbose = false); //---------------------------------------------------------------------------------------------------------------- // Lay the trie in memory in such a way that there are less cache misses when jumping from root to leaf. @@ -143,17 +143,17 @@ size_t CompactTrieMinimize(IOutputStream& os, const TTrieBuilder& builder, bool // (there is not much difference between these papers, actually). // template <class TPacker> -size_t CompactTrieMakeFastLayout(IOutputStream& os, const char* data, size_t datalength, bool verbose = false, const TPacker& packer = TPacker()); +size_t CompactTrieMakeFastLayout(IOutputStream& os, const char* data, size_t datalength, bool verbose = false, const TPacker& packer = TPacker()); template <class TTrieBuilder> -size_t CompactTrieMakeFastLayout(IOutputStream& os, const TTrieBuilder& builder, bool verbose = false); +size_t CompactTrieMakeFastLayout(IOutputStream& os, const TTrieBuilder& builder, bool verbose = false); // Composition of minimization and fast layout template <class TPacker> -size_t CompactTrieMinimizeAndMakeFastLayout(IOutputStream& os, const char* data, size_t datalength, bool verbose = false, const TPacker& packer = TPacker()); +size_t CompactTrieMinimizeAndMakeFastLayout(IOutputStream& os, const char* data, size_t datalength, bool verbose = false, const TPacker& packer = TPacker()); template <class TTrieBuilder> -size_t CompactTrieMinimizeAndMakeFastLayout(IOutputStream& os, const TTrieBuilder& builder, bool verbose = false); +size_t CompactTrieMinimizeAndMakeFastLayout(IOutputStream& os, const TTrieBuilder& builder, bool verbose = false); // Implementation details moved here. #include "comptrie_builder.inl" diff --git a/library/cpp/containers/comptrie/comptrie_builder.inl b/library/cpp/containers/comptrie/comptrie_builder.inl index 612a1bbe95..f273fa6571 100644 --- a/library/cpp/containers/comptrie/comptrie_builder.inl +++ b/library/cpp/containers/comptrie/comptrie_builder.inl @@ -53,18 +53,18 @@ protected: bool FindLongestPrefixImpl(const char* keyptr, size_t keylen, size_t* prefixLen, TData* value) const; size_t NodeMeasureSubtree(TNode* thiz) const; - ui64 NodeSaveSubtree(TNode* thiz, IOutputStream& os) const; - ui64 NodeSaveSubtreeAndDestroy(TNode* thiz, IOutputStream& osy); + ui64 NodeSaveSubtree(TNode* thiz, IOutputStream& os) const; + ui64 NodeSaveSubtreeAndDestroy(TNode* thiz, IOutputStream& osy); void NodeBufferSubtree(TNode* thiz); size_t NodeMeasureLeafValue(TNode* thiz) const; - ui64 NodeSaveLeafValue(TNode* thiz, IOutputStream& os) const; + ui64 NodeSaveLeafValue(TNode* thiz, IOutputStream& os) const; virtual ui64 ArcMeasure(const TArc* thiz, size_t leftsize, size_t rightsize) const; virtual ui64 ArcSaveSelf(const TArc* thiz, IOutputStream& os) const; - ui64 ArcSave(const TArc* thiz, IOutputStream& os) const; - ui64 ArcSaveAndDestroy(const TArc* thiz, IOutputStream& os); + ui64 ArcSave(const TArc* thiz, IOutputStream& os) const; + ui64 ArcSaveAndDestroy(const TArc* thiz, IOutputStream& os); public: TCompactTrieBuilderImpl(TCompactTrieBuilderFlags flags, TPacker packer, IAllocator* alloc); @@ -83,8 +83,8 @@ public: bool FindEntry(const TSymbol* key, size_t keylen, TData* value) const; bool FindLongestPrefix(const TSymbol* key, size_t keylen, size_t* prefixlen, TData* value) const; - size_t Save(IOutputStream& os) const; - size_t SaveAndDestroy(IOutputStream& os); + size_t Save(IOutputStream& os) const; + size_t SaveAndDestroy(IOutputStream& os); void Clear(); @@ -118,8 +118,8 @@ public: virtual ~ISubtree() = default; virtual bool IsLast() const = 0; virtual ui64 Measure(const TBuilderImpl* builder) const = 0; - virtual ui64 Save(const TBuilderImpl* builder, IOutputStream& os) const = 0; - virtual ui64 SaveAndDestroy(TBuilderImpl* builder, IOutputStream& os) = 0; + virtual ui64 Save(const TBuilderImpl* builder, IOutputStream& os) const = 0; + virtual ui64 SaveAndDestroy(TBuilderImpl* builder, IOutputStream& os) = 0; virtual void Destroy(TBuilderImpl*) { } // Tries to find key in subtree. @@ -135,7 +135,7 @@ public: typedef typename TCompactVector<TArc>::const_iterator const_iterator; TArcSet() { - Y_ASSERT(reinterpret_cast<ISubtree*>(this) == static_cast<void*>(this)); // This assumption is used in TNode::Subtree() + Y_ASSERT(reinterpret_cast<ISubtree*>(this) == static_cast<void*>(this)); // This assumption is used in TNode::Subtree() } iterator Find(char ch); @@ -166,17 +166,17 @@ public: return builder->ArcMeasure(&(*this)[median], leftsize, rightsize); } - ui64 Save(const TBuilderImpl* builder, IOutputStream& os) const override { + ui64 Save(const TBuilderImpl* builder, IOutputStream& os) const override { return SaveRange(builder, 0, this->size(), os); } - ui64 SaveAndDestroy(TBuilderImpl* builder, IOutputStream& os) override { + ui64 SaveAndDestroy(TBuilderImpl* builder, IOutputStream& os) override { ui64 result = SaveRangeAndDestroy(builder, 0, this->size(), os); Destroy(builder); return result; } - ui64 SaveRange(const TBuilderImpl* builder, size_t from, size_t to, IOutputStream& os) const { + ui64 SaveRange(const TBuilderImpl* builder, size_t from, size_t to, IOutputStream& os) const { if (from >= to) return 0; @@ -188,7 +188,7 @@ public: return written; } - ui64 SaveRangeAndDestroy(TBuilderImpl* builder, size_t from, size_t to, IOutputStream& os) { + ui64 SaveRangeAndDestroy(TBuilderImpl* builder, size_t from, size_t to, IOutputStream& os) { if (from >= to) return 0; @@ -209,7 +209,7 @@ public: } ~TArcSet() override { - Y_ASSERT(this->empty()); + Y_ASSERT(this->empty()); } }; @@ -218,7 +218,7 @@ public: TArrayWithSizeHolder<char> Buffer; TBufferedSubtree() { - Y_ASSERT(reinterpret_cast<ISubtree*>(this) == static_cast<void*>(this)); // This assumption is used in TNode::Subtree() + Y_ASSERT(reinterpret_cast<ISubtree*>(this) == static_cast<void*>(this)); // This assumption is used in TNode::Subtree() } bool IsLast() const override { @@ -255,12 +255,12 @@ public: return Buffer.Size(); } - ui64 Save(const TBuilderImpl*, IOutputStream& os) const override { + ui64 Save(const TBuilderImpl*, IOutputStream& os) const override { os.Write(Buffer.Get(), Buffer.Size()); return Buffer.Size(); } - ui64 SaveAndDestroy(TBuilderImpl* builder, IOutputStream& os) override { + ui64 SaveAndDestroy(TBuilderImpl* builder, IOutputStream& os) override { ui64 result = Save(builder, os); TArrayWithSizeHolder<char>().Swap(Buffer); return result; @@ -284,7 +284,7 @@ public: Data->FileName = fileName; Data->Size = size; - Y_ASSERT(reinterpret_cast<ISubtree*>(this) == static_cast<void*>(this)); // This assumption is used in TNode::Subtree() + Y_ASSERT(reinterpret_cast<ISubtree*>(this) == static_cast<void*>(this)); // This assumption is used in TNode::Subtree() } bool IsLast() const override { @@ -320,7 +320,7 @@ public: return Data->Size; } - ui64 Save(const TBuilderImpl*, IOutputStream& os) const override { + ui64 Save(const TBuilderImpl*, IOutputStream& os) const override { TUnbufferedFileInput is(Data->FileName); ui64 written = TransferData(&is, &os); if (written != Data->Size) @@ -328,7 +328,7 @@ public: return written; } - ui64 SaveAndDestroy(TBuilderImpl* builder, IOutputStream& os) override { + ui64 SaveAndDestroy(TBuilderImpl* builder, IOutputStream& os) override { return Save(builder, os); } }; @@ -412,7 +412,7 @@ public: ~TNode() { Subtree()->~ISubtree(); - Y_ASSERT(PayloadType == DATA_ABSENT); + Y_ASSERT(PayloadType == DATA_ABSENT); } }; @@ -457,12 +457,12 @@ bool TCompactTrieBuilder<T, D, S>::FindLongestPrefix( } template <class T, class D, class S> -size_t TCompactTrieBuilder<T, D, S>::Save(IOutputStream& os) const { +size_t TCompactTrieBuilder<T, D, S>::Save(IOutputStream& os) const { return Impl->Save(os); } template <class T, class D, class S> -size_t TCompactTrieBuilder<T, D, S>::SaveAndDestroy(IOutputStream& os) { +size_t TCompactTrieBuilder<T, D, S>::SaveAndDestroy(IOutputStream& os) { return Impl->SaveAndDestroy(os); } @@ -509,13 +509,13 @@ void TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::ConvertSymbolArrayTo for (size_t i = 0; i < keylen; ++i) { TSymbol label = key[i]; for (int j = (int)NCompactTrie::ExtraBits<TSymbol>(); j >= 0; j -= 8) { - Y_ASSERT(ckeyptr < buf.Data() + buflen); + Y_ASSERT(ckeyptr < buf.Data() + buflen); *(ckeyptr++) = (char)(label >> j); } } buf.Proceed(buflen); - Y_ASSERT(ckeyptr == buf.Data() + buf.Filled()); + Y_ASSERT(ckeyptr == buf.Data() + buf.Filled()); } template <class T, class D, class S> @@ -750,7 +750,7 @@ void TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::Clear() { } template <class T, class D, class S> -size_t TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::Save(IOutputStream& os) const { +size_t TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::Save(IOutputStream& os) const { const size_t len = NodeMeasureSubtree(Root); if (len != NodeSaveSubtree(Root, os)) ythrow yexception() << "something wrong"; @@ -759,7 +759,7 @@ size_t TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::Save(IOutputStream } template <class T, class D, class S> -size_t TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::SaveAndDestroy(IOutputStream& os) { +size_t TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::SaveAndDestroy(IOutputStream& os) { const size_t len = NodeMeasureSubtree(Root); if (len != NodeSaveSubtreeAndDestroy(Root, os)) ythrow yexception() << "something wrong"; @@ -829,12 +829,12 @@ size_t TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::NodeMeasureSubtree } template <class T, class D, class S> -ui64 TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::NodeSaveSubtree(TNode* thiz, IOutputStream& os) const { +ui64 TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::NodeSaveSubtree(TNode* thiz, IOutputStream& os) const { return thiz->Subtree()->Save(this, os); } template <class T, class D, class S> -ui64 TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::NodeSaveSubtreeAndDestroy(TNode* thiz, IOutputStream& os) { +ui64 TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::NodeSaveSubtreeAndDestroy(TNode* thiz, IOutputStream& os) { return thiz->Subtree()->SaveAndDestroy(this, os); } @@ -853,7 +853,7 @@ void TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::NodeBufferSubtree(TN TMemoryOutput bufout(buffer.Get(), buffer.Size()); ui64 written = arcSet->Save(this, bufout); - Y_ASSERT(written == bufferLength); + Y_ASSERT(written == bufferLength); arcSet->Destroy(this); arcSet->~TArcSet(); @@ -872,7 +872,7 @@ size_t TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::NodeMeasureLeafVal } template <class T, class D, class S> -ui64 TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::NodeSaveLeafValue(TNode* thiz, IOutputStream& os) const { +ui64 TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::NodeSaveLeafValue(TNode* thiz, IOutputStream& os) const { if (!thiz->IsFinal()) return 0; @@ -919,7 +919,7 @@ ui64 TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::ArcMeasure( } template <class T, class D, class S> -ui64 TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::ArcSaveSelf(const TArc* thiz, IOutputStream& os) const { +ui64 TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::ArcSaveSelf(const TArc* thiz, IOutputStream& os) const { using namespace NCompactTrie; ui64 written = 0; @@ -962,14 +962,14 @@ ui64 TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::ArcSaveSelf(const TA } template <class T, class D, class S> -ui64 TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::ArcSave(const TArc* thiz, IOutputStream& os) const { +ui64 TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::ArcSave(const TArc* thiz, IOutputStream& os) const { ui64 written = ArcSaveSelf(thiz, os); written += NodeSaveSubtree(thiz->Node, os); return written; } template <class T, class D, class S> -ui64 TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::ArcSaveAndDestroy(const TArc* thiz, IOutputStream& os) { +ui64 TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::ArcSaveAndDestroy(const TArc* thiz, IOutputStream& os) { ui64 written = ArcSaveSelf(thiz, os); written += NodeSaveSubtreeAndDestroy(thiz->Node, os); return written; @@ -1061,13 +1061,13 @@ const typename TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::TNode* // as you expect it to, and can destroy the trie in the making. template <class TPacker> -size_t CompactTrieMinimize(IOutputStream& os, const char* data, size_t datalength, bool verbose /*= false*/, const TPacker& packer /*= TPacker()*/, NCompactTrie::EMinimizeMode mode) { +size_t CompactTrieMinimize(IOutputStream& os, const char* data, size_t datalength, bool verbose /*= false*/, const TPacker& packer /*= TPacker()*/, NCompactTrie::EMinimizeMode mode) { using namespace NCompactTrie; return CompactTrieMinimizeImpl(os, data, datalength, verbose, &packer, mode); } template <class TTrieBuilder> -size_t CompactTrieMinimize(IOutputStream& os, const TTrieBuilder& builder, bool verbose /*=false*/) { +size_t CompactTrieMinimize(IOutputStream& os, const TTrieBuilder& builder, bool verbose /*=false*/) { TBufferStream buftmp; size_t len = builder.Save(buftmp); return CompactTrieMinimize<typename TTrieBuilder::TPacker>(os, buftmp.Buffer().Data(), len, verbose); @@ -1093,27 +1093,27 @@ size_t CompactTrieMinimize(IOutputStream& os, const TTrieBuilder& builder, bool // (there is not much difference between these papers, actually). // template <class TPacker> -size_t CompactTrieMakeFastLayout(IOutputStream& os, const char* data, size_t datalength, bool verbose /*= false*/, const TPacker& packer /*= TPacker()*/) { +size_t CompactTrieMakeFastLayout(IOutputStream& os, const char* data, size_t datalength, bool verbose /*= false*/, const TPacker& packer /*= TPacker()*/) { using namespace NCompactTrie; return CompactTrieMakeFastLayoutImpl(os, data, datalength, verbose, &packer); } template <class TTrieBuilder> -size_t CompactTrieMakeFastLayout(IOutputStream& os, const TTrieBuilder& builder, bool verbose /*=false*/) { +size_t CompactTrieMakeFastLayout(IOutputStream& os, const TTrieBuilder& builder, bool verbose /*=false*/) { TBufferStream buftmp; size_t len = builder.Save(buftmp); return CompactTrieMakeFastLayout<typename TTrieBuilder::TPacker>(os, buftmp.Buffer().Data(), len, verbose); } template <class TPacker> -size_t CompactTrieMinimizeAndMakeFastLayout(IOutputStream& os, const char* data, size_t datalength, bool verbose/*=false*/, const TPacker& packer/*= TPacker()*/) { +size_t CompactTrieMinimizeAndMakeFastLayout(IOutputStream& os, const char* data, size_t datalength, bool verbose/*=false*/, const TPacker& packer/*= TPacker()*/) { TBufferStream buftmp; size_t len = CompactTrieMinimize(buftmp, data, datalength, verbose, packer); return CompactTrieMakeFastLayout(os, buftmp.Buffer().Data(), len, verbose, packer); } template <class TTrieBuilder> -size_t CompactTrieMinimizeAndMakeFastLayout(IOutputStream& os, const TTrieBuilder& builder, bool verbose /*=false*/) { +size_t CompactTrieMinimizeAndMakeFastLayout(IOutputStream& os, const TTrieBuilder& builder, bool verbose /*=false*/) { TBufferStream buftmp; size_t len = CompactTrieMinimize(buftmp, builder, verbose); return CompactTrieMakeFastLayout<typename TTrieBuilder::TPacker>(os, buftmp.Buffer().Data(), len, verbose); diff --git a/library/cpp/containers/comptrie/comptrie_impl.h b/library/cpp/containers/comptrie/comptrie_impl.h index 607b8e5d32..f41c38311a 100644 --- a/library/cpp/containers/comptrie/comptrie_impl.h +++ b/library/cpp/containers/comptrie/comptrie_impl.h @@ -18,7 +18,7 @@ namespace NCompactTrie { Y_FORCE_INLINE size_t UnpackOffset(const char* p, size_t len); size_t MeasureOffset(size_t offset); size_t PackOffset(char* buffer, size_t offset); - static inline ui64 ArcSaveOffset(size_t offset, IOutputStream& os); + static inline ui64 ArcSaveOffset(size_t offset, IOutputStream& os); Y_FORCE_INLINE char LeapByte(const char*& datapos, const char* dataend, char label); template <class T> @@ -37,7 +37,7 @@ namespace NCompactTrie { } const size_t offsetlength = flags & MT_SIZEMASK; const size_t offset = UnpackOffset(datapos + 1, offsetlength); - Y_ASSERT(offset); + Y_ASSERT(offset); datapos += offset; } @@ -89,7 +89,7 @@ namespace NCompTriePrivate { } namespace NCompactTrie { - static inline ui64 ArcSaveOffset(size_t offset, IOutputStream& os) { + static inline ui64 ArcSaveOffset(size_t offset, IOutputStream& os) { using namespace NCompactTrie; if (!offset) @@ -127,7 +127,7 @@ namespace NCompactTrie { // These links are created during minimization: original uncompressed // tree does not need them. (If we find a way to package 3 offset lengths // into 1 byte, we could get rid of them; but it looks like they do no harm. - Y_ASSERT(datapos < dataend); + Y_ASSERT(datapos < dataend); offsetlength = flags & MT_SIZEMASK; offset = UnpackOffset(datapos, offsetlength); if (!offset) @@ -185,7 +185,7 @@ namespace NCompactTrie { template <typename TSymbol, class TPacker> Y_FORCE_INLINE bool Advance(const char*& datapos, const char* const dataend, const char*& value, TSymbol label, TPacker packer) { - Y_ASSERT(datapos < dataend); + Y_ASSERT(datapos < dataend); char flags = MT_NEXT; for (int i = (int)ExtraBits<TSymbol>(); i >= 0; i -= 8) { flags = LeapByte(datapos, dataend, (char)(label >> i)); @@ -195,7 +195,7 @@ namespace NCompactTrie { value = nullptr; - Y_ASSERT(datapos <= dataend); + Y_ASSERT(datapos <= dataend); if ((flags & MT_FINAL)) { value = datapos; datapos += packer.SkipLeaf(datapos); diff --git a/library/cpp/containers/comptrie/comptrie_trie.h b/library/cpp/containers/comptrie/comptrie_trie.h index 25a288f23d..40ec1e52b3 100644 --- a/library/cpp/containers/comptrie/comptrie_trie.h +++ b/library/cpp/containers/comptrie/comptrie_trie.h @@ -11,7 +11,7 @@ #include <util/generic/vector.h> #include <util/generic/yexception.h> #include <util/memory/blob.h> -#include <util/stream/input.h> +#include <util/stream/input.h> #include <utility> template <class T, class D, class S> @@ -54,16 +54,16 @@ protected: public: TCompactTrie() = default; - TCompactTrie(const char* d, size_t len, TPacker packer); - TCompactTrie(const char* d, size_t len) + TCompactTrie(const char* d, size_t len, TPacker packer); + TCompactTrie(const char* d, size_t len) : TCompactTrie{d, len, TPacker{}} { - } - - TCompactTrie(const TBlob& data, TPacker packer); - explicit TCompactTrie(const TBlob& data) + } + + TCompactTrie(const TBlob& data, TPacker packer); + explicit TCompactTrie(const TBlob& data) : TCompactTrie{data, TPacker{}} { - } - + } + // Skipper should be initialized with &Packer, not with &other.Packer, so you have to redefine these. TCompactTrie(const TCompactTrie& other); TCompactTrie(TCompactTrie&& other) noexcept; @@ -185,7 +185,7 @@ public: // LowerBound of X cannot be greater than X. TConstIterator UpperBound(const TKeyBuf& key) const; - void Print(IOutputStream& os); + void Print(IOutputStream& os); size_t Size() const; @@ -212,7 +212,7 @@ private: TArrayHolder<char> Storage; public: - TCompactTrieHolder(IInputStream& is, size_t len); + TCompactTrieHolder(IInputStream& is, size_t len); }; //------------------------// @@ -308,7 +308,7 @@ void TCompactTrie<T, D, S>::Init(const TBlob& data, TPacker packer) { const char* emptypos = datapos; char flags = LeapByte(emptypos, dataend, 0); if (emptypos && (flags & MT_FINAL)) { - Y_ASSERT(emptypos <= dataend); + Y_ASSERT(emptypos <= dataend); EmptyValue = emptypos; } } @@ -375,7 +375,7 @@ bool TCompactTrie<T, D, S>::FindTails(const TSymbol* key, size_t keylen, TCompac if (key == keyend) { if (datapos) { - Y_ASSERT(datapos >= datastart); + Y_ASSERT(datapos >= datastart); res = TCompactTrie<T, D, S>(TBlob::NoCopy(datapos, dataend - datapos), value); } else { res = TCompactTrie<T, D, S>(value); @@ -406,7 +406,7 @@ inline bool TCompactTrie<T, D, S>::FindTails(TSymbol label, TCompactTrie<T, D, S return false; if (datapos) { - Y_ASSERT(datapos >= datastart); + Y_ASSERT(datapos >= datastart); res = TCompactTrie<T, D, S>(TBlob::NoCopy(datapos, dataend - datapos), value); } else { res = TCompactTrie<T, D, S>(value); @@ -452,7 +452,7 @@ typename TCompactTrie<T, D, S>::TConstIterator TCompactTrie<T, D, S>::UpperBound } template <class T, class D, class S> -void TCompactTrie<T, D, S>::Print(IOutputStream& os) { +void TCompactTrie<T, D, S>::Print(IOutputStream& os) { typedef typename ::TCompactTrieKeySelector<T>::TKeyBuf TSBuffer; for (TConstIterator it = Begin(); it != End(); ++it) { os << TSBuffer((*it).first.data(), (*it).first.size()) << "\t" << (*it).second << Endl; @@ -504,7 +504,7 @@ bool TCompactTrie<T, D, S>::LookupLongestPrefix(const TSymbol* key, size_t keyle return found; // no such arc } - Y_ASSERT(datapos <= dataend); + Y_ASSERT(datapos <= dataend); if ((flags & MT_FINAL)) { prefixLen = keylen - (keyend - key) - (i ? 1 : 0); valuepos = datapos; @@ -558,7 +558,7 @@ void TCompactTrie<T, D, S>::LookupPhrases( // TCompactTrieHolder template <class T, class D, class S> -TCompactTrieHolder<T, D, S>::TCompactTrieHolder(IInputStream& is, size_t len) +TCompactTrieHolder<T, D, S>::TCompactTrieHolder(IInputStream& is, size_t len) : Storage(new char[len]) { if (is.Load(Storage.Get(), len) != len) { diff --git a/library/cpp/containers/comptrie/comptrie_ut.cpp b/library/cpp/containers/comptrie/comptrie_ut.cpp index 6095f78cf8..74bee09b5d 100644 --- a/library/cpp/containers/comptrie/comptrie_ut.cpp +++ b/library/cpp/containers/comptrie/comptrie_ut.cpp @@ -1,7 +1,7 @@ #include <util/random/shuffle.h> #include <library/cpp/testing/unittest/registar.h> -#include <util/stream/output.h> +#include <util/stream/output.h> #include <utility> #include <util/charset/wide.h> @@ -105,7 +105,7 @@ private: UNIT_TEST(TestFirstSymbolIteratorChar32); UNIT_TEST(TestArrayPacker); - + UNIT_TEST(TestBuilderFindLongestPrefix); UNIT_TEST(TestBuilderFindLongestPrefixWithEmptyValue); @@ -118,7 +118,7 @@ private: static const char* SampleData[]; template <class T> - void CreateTrie(IOutputStream& out, bool minimize, bool useFastLayout); + void CreateTrie(IOutputStream& out, bool minimize, bool useFastLayout); template <class T> void CheckData(const char* src, size_t len); @@ -241,7 +241,7 @@ public: void TestFirstSymbolIterator32(); void TestFirstSymbolIteratorChar32(); - void TestArrayPacker(); + void TestArrayPacker(); void TestBuilderFindLongestPrefix(); void TestBuilderFindLongestPrefix(size_t keysCount, double branchProbability, bool isPrefixGrouped, bool hasEmptyKey); @@ -297,17 +297,17 @@ typename TCompactTrie<T>::TKey MakeWideKey(const TStringBuf& buf) { } template <class T> -void TCompactTrieTest::CreateTrie(IOutputStream& out, bool minimize, bool useFastLayout) { +void TCompactTrieTest::CreateTrie(IOutputStream& out, bool minimize, bool useFastLayout) { TCompactTrieBuilder<T> builder; - for (auto& i : SampleData) { - size_t len = strlen(i); + for (auto& i : SampleData) { + size_t len = strlen(i); - builder.Add(MakeWideKey<T>(i, len), len * 2); + builder.Add(MakeWideKey<T>(i, len), len * 2); } TBufferOutput tmp2; - IOutputStream& currentOutput = useFastLayout ? tmp2 : out; + IOutputStream& currentOutput = useFastLayout ? tmp2 : out; if (minimize) { TBufferOutput buftmp; builder.Save(buftmp); @@ -361,14 +361,14 @@ template <class T> void TCompactTrieTest::CheckData(const char* data, size_t datalen) { TCompactTrie<T> trie(data, datalen); - UNIT_ASSERT_VALUES_EQUAL(Y_ARRAY_SIZE(SampleData), trie.Size()); + UNIT_ASSERT_VALUES_EQUAL(Y_ARRAY_SIZE(SampleData), trie.Size()); - for (auto& i : SampleData) { - size_t len = strlen(i); + for (auto& i : SampleData) { + size_t len = strlen(i); ui64 value = 0; size_t prefixLen = 0; - typename TCompactTrie<T>::TKey key = MakeWideKey<T>(i, len); + typename TCompactTrie<T>::TKey key = MakeWideKey<T>(i, len); UNIT_ASSERT(trie.Find(key, &value)); UNIT_ASSERT_EQUAL(len * 2, value); UNIT_ASSERT(trie.FindLongestPrefix(key, &prefixLen, &value)); @@ -376,7 +376,7 @@ void TCompactTrieTest::CheckData(const char* data, size_t datalen) { UNIT_ASSERT_EQUAL(len * 2, value); TString badkey("bb"); - badkey += i; + badkey += i; key = MakeWideKey<T>(badkey); UNIT_ASSERT(!trie.Find(key)); value = 123; @@ -386,7 +386,7 @@ void TCompactTrieTest::CheckData(const char* data, size_t datalen) { UNIT_ASSERT_EQUAL(1, prefixLen); UNIT_ASSERT_EQUAL(2, value); - badkey = i; + badkey = i; badkey += "x"; key = MakeWideKey<T>(badkey); UNIT_ASSERT(!trie.Find(key)); @@ -425,10 +425,10 @@ void TCompactTrieTest::CheckIterator(const char* data, size_t datalen) { typedef typename TCompactTrie<T>::TValueType TValue; TMap<TKey, ui64> stored; - for (auto& i : SampleData) { - size_t len = strlen(i); + for (auto& i : SampleData) { + size_t len = strlen(i); - stored[MakeWideKey<T>(i, len)] = len * 2; + stored[MakeWideKey<T>(i, len)] = len * 2; } TCompactTrie<T> trie(data, datalen); @@ -567,7 +567,7 @@ void TCompactTrieTest::TestPhraseSearch() { TBufferOutput bufout; TCompactTrieBuilder<char> builder; - for (size_t i = 0; i < Y_ARRAY_SIZE(phrases); i++) { + for (size_t i = 0; i < Y_ARRAY_SIZE(phrases); i++) { builder.Add(phrases[i], strlen(phrases[i]), i); } builder.Save(bufout); @@ -576,8 +576,8 @@ void TCompactTrieTest::TestPhraseSearch() { TVector<TCompactTrie<char>::TPhraseMatch> matches; trie.FindPhrases(goodphrase, strlen(goodphrase), matches); - UNIT_ASSERT(matches.size() == Y_ARRAY_SIZE(phrases)); - for (size_t i = 0; i < Y_ARRAY_SIZE(phrases); i++) { + UNIT_ASSERT(matches.size() == Y_ARRAY_SIZE(phrases)); + for (size_t i = 0; i < Y_ARRAY_SIZE(phrases); i++) { UNIT_ASSERT(matches[i].first == strlen(phrases[i])); UNIT_ASSERT(matches[i].second == i); } @@ -734,11 +734,11 @@ void TCompactTrieTest::TestFindTailsImpl(const TString& prefix) { TMap<TString, ui64> input; - for (auto& i : SampleData) { + for (auto& i : SampleData) { TString temp = i; ui64 val = temp.size() * 2; builder.Add(temp.data(), temp.size(), val); - if (temp.StartsWith(prefix)) { + if (temp.StartsWith(prefix)) { input[temp.substr(prefix.size())] = val; } } @@ -789,10 +789,10 @@ void TCompactTrieTest::TestPrefixGrouped() { "Tumen", }; - for (size_t i = 0; i < Y_ARRAY_SIZE(data); ++i) { + for (size_t i = 0; i < Y_ARRAY_SIZE(data); ++i) { ui32 val = strlen(data[i]) + 1; b1.Add(data[i], strlen(data[i]), val); - for (size_t j = 0; j < Y_ARRAY_SIZE(data); ++j) { + for (size_t j = 0; j < Y_ARRAY_SIZE(data); ++j) { ui32 mustHave = strlen(data[j]) + 1; ui32 found = 0; if (j <= i) { @@ -813,10 +813,10 @@ void TCompactTrieTest::TestPrefixGrouped() { //t1.Print(Cerr); - for (auto& i : data) { + for (auto& i : data) { ui32 v; - UNIT_ASSERT(t1.Find(i, strlen(i), &v)); - UNIT_ASSERT_VALUES_EQUAL(strlen(i) + 1, v); + UNIT_ASSERT(t1.Find(i, strlen(i), &v)); + UNIT_ASSERT_VALUES_EQUAL(strlen(i) + 1, v); } } @@ -831,7 +831,7 @@ void TCompactTrieTest::CrashTestPrefixGrouped() { }; bool wasException = false; try { - for (size_t i = 0; i < Y_ARRAY_SIZE(data); ++i) { + for (size_t i = 0; i < Y_ARRAY_SIZE(data); ++i) { builder.Add(data[i], strlen(data[i]), i + 1); } } catch (const yexception& e) { @@ -947,7 +947,7 @@ void TCompactTrieTest::TestUniqueImpl(bool isPrefixGrouped) { "Fry", "Tumen", }; - for (size_t i = 0; i < Y_ARRAY_SIZE(data); ++i) { + for (size_t i = 0; i < Y_ARRAY_SIZE(data); ++i) { UNIT_ASSERT_C(builder.Add(data[i], strlen(data[i]), i + 1), i); } bool wasException = false; @@ -973,7 +973,7 @@ void TCompactTrieTest::TestAddRetValue() { "Fry", "Tumen", }; - for (size_t i = 0; i < Y_ARRAY_SIZE(data); ++i) { + for (size_t i = 0; i < Y_ARRAY_SIZE(data); ++i) { UNIT_ASSERT(builder.Add(data[i], strlen(data[i]), i + 1)); UNIT_ASSERT(!builder.Add(data[i], strlen(data[i]), i + 2)); ui32 value; @@ -995,10 +995,10 @@ void TCompactTrieTest::TestClear() { "Fry", "Tumen", }; - for (size_t i = 0; i < Y_ARRAY_SIZE(data); ++i) { + for (size_t i = 0; i < Y_ARRAY_SIZE(data); ++i) { builder.Add(data[i], strlen(data[i]), i + 1); } - UNIT_ASSERT(builder.GetEntryCount() == Y_ARRAY_SIZE(data)); + UNIT_ASSERT(builder.GetEntryCount() == Y_ARRAY_SIZE(data)); builder.Clear(); UNIT_ASSERT(builder.GetEntryCount() == 0); UNIT_ASSERT(builder.GetNodeCount() == 1); @@ -1105,7 +1105,7 @@ void TCompactTrieTest::TestTrieSet() { // Tests for trie with vector (list, set) values TVector<TUtf16String> TCompactTrieTest::GetSampleKeys(size_t nKeys) const { - Y_ASSERT(nKeys <= 10); + Y_ASSERT(nKeys <= 10); TString sampleKeys[] = {"a", "b", "ac", "bd", "abe", "bcf", "deg", "ah", "xy", "abc"}; TVector<TUtf16String> result; for (size_t i = 0; i < nKeys; i++) @@ -1332,7 +1332,7 @@ void TCompactTrieTest::TestSearchIterImpl() { TStringBuf("abbbc"), TStringBuf("bdfaa"), }; - for (size_t i = 0; i < Y_ARRAY_SIZE(data); ++i) { + for (size_t i = 0; i < Y_ARRAY_SIZE(data); ++i) { builder.Add(TConvertKey<TChar>::Convert(data[i]), i + 1); } builder.Save(buffer); @@ -1434,37 +1434,37 @@ void TCompactTrieTest::TestFirstSymbolIteratorChar32() { } -void TCompactTrieTest::TestArrayPacker() { +void TCompactTrieTest::TestArrayPacker() { using TDataInt = std::array<int, 2>; const std::pair<TString, TDataInt> dataXxx{"xxx", {{15, 16}}}; const std::pair<TString, TDataInt> dataYyy{"yyy", {{20, 30}}}; - - TCompactTrieBuilder<char, TDataInt> trieBuilderOne; - trieBuilderOne.Add(dataXxx.first, dataXxx.second); - trieBuilderOne.Add(dataYyy.first, dataYyy.second); - - TBufferOutput bufferOne; - trieBuilderOne.Save(bufferOne); - - const TCompactTrie<char, TDataInt> trieOne(bufferOne.Buffer().Data(), bufferOne.Buffer().Size()); - UNIT_ASSERT_VALUES_EQUAL(dataXxx.second, trieOne.Get(dataXxx.first)); - UNIT_ASSERT_VALUES_EQUAL(dataYyy.second, trieOne.Get(dataYyy.first)); - + + TCompactTrieBuilder<char, TDataInt> trieBuilderOne; + trieBuilderOne.Add(dataXxx.first, dataXxx.second); + trieBuilderOne.Add(dataYyy.first, dataYyy.second); + + TBufferOutput bufferOne; + trieBuilderOne.Save(bufferOne); + + const TCompactTrie<char, TDataInt> trieOne(bufferOne.Buffer().Data(), bufferOne.Buffer().Size()); + UNIT_ASSERT_VALUES_EQUAL(dataXxx.second, trieOne.Get(dataXxx.first)); + UNIT_ASSERT_VALUES_EQUAL(dataYyy.second, trieOne.Get(dataYyy.first)); + using TDataStroka = std::array<TString, 2>; const std::pair<TString, TDataStroka> dataZzz{"zzz", {{"hello", "there"}}}; const std::pair<TString, TDataStroka> dataWww{"www", {{"half", "life"}}}; - - TCompactTrieBuilder<char, TDataStroka> trieBuilderTwo; - trieBuilderTwo.Add(dataZzz.first, dataZzz.second); - trieBuilderTwo.Add(dataWww.first, dataWww.second); - - TBufferOutput bufferTwo; - trieBuilderTwo.Save(bufferTwo); - - const TCompactTrie<char, TDataStroka> trieTwo(bufferTwo.Buffer().Data(), bufferTwo.Buffer().Size()); - UNIT_ASSERT_VALUES_EQUAL(dataZzz.second, trieTwo.Get(dataZzz.first)); - UNIT_ASSERT_VALUES_EQUAL(dataWww.second, trieTwo.Get(dataWww.first)); -} + + TCompactTrieBuilder<char, TDataStroka> trieBuilderTwo; + trieBuilderTwo.Add(dataZzz.first, dataZzz.second); + trieBuilderTwo.Add(dataWww.first, dataWww.second); + + TBufferOutput bufferTwo; + trieBuilderTwo.Save(bufferTwo); + + const TCompactTrie<char, TDataStroka> trieTwo(bufferTwo.Buffer().Data(), bufferTwo.Buffer().Size()); + UNIT_ASSERT_VALUES_EQUAL(dataZzz.second, trieTwo.Get(dataZzz.first)); + UNIT_ASSERT_VALUES_EQUAL(dataWww.second, trieTwo.Get(dataWww.first)); +} void TCompactTrieTest::TestBuilderFindLongestPrefix() { const size_t sizes[] = {10, 100}; @@ -1517,7 +1517,7 @@ void TCompactTrieTest::TestBuilderFindLongestPrefix(size_t keysCount, double bra } else { size_t max = 0; for (size_t k = 0; k < i; ++k) - if (keys[k].Size() < otherKey.Size() && keys[k].Size() > max && otherKey.StartsWith(keys[k])) + if (keys[k].Size() < otherKey.Size() && keys[k].Size() > max && otherKey.StartsWith(keys[k])) max = keys[k].Size(); expectedSize = max; } diff --git a/library/cpp/containers/comptrie/loader/loader.h b/library/cpp/containers/comptrie/loader/loader.h index 478c820abc..ee10e9b451 100644 --- a/library/cpp/containers/comptrie/loader/loader.h +++ b/library/cpp/containers/comptrie/loader/loader.h @@ -12,7 +12,7 @@ TrieType LoadTrieFromArchive(const TString& key, bool ignoreErrors = false) { TArchiveReader archive(TBlob::NoCopy(data, sizeof(data))); if (archive.Has(key)) { - TAutoPtr<IInputStream> trie = archive.ObjectByKey(key); + TAutoPtr<IInputStream> trie = archive.ObjectByKey(key); return TrieType(TBlob::FromStream(*trie)); } if (!ignoreErrors) { diff --git a/library/cpp/containers/comptrie/loader/loader_ut.cpp b/library/cpp/containers/comptrie/loader/loader_ut.cpp index f146bfaff5..345063a31e 100644 --- a/library/cpp/containers/comptrie/loader/loader_ut.cpp +++ b/library/cpp/containers/comptrie/loader/loader_ut.cpp @@ -10,8 +10,8 @@ namespace { }; } -Y_UNIT_TEST_SUITE(ArchiveLoaderTests) { - Y_UNIT_TEST(BaseTest) { +Y_UNIT_TEST_SUITE(ArchiveLoaderTests) { + Y_UNIT_TEST(BaseTest) { TDummyTrie trie = LoadTrieFromArchive<TDummyTrie>("/dummy.trie", DATA, true); UNIT_ASSERT_EQUAL(trie.Size(), 3); diff --git a/library/cpp/containers/comptrie/make_fast_layout.h b/library/cpp/containers/comptrie/make_fast_layout.h index 6bb4ff0d55..b8fab5d65b 100644 --- a/library/cpp/containers/comptrie/make_fast_layout.h +++ b/library/cpp/containers/comptrie/make_fast_layout.h @@ -3,7 +3,7 @@ #include "leaf_skipper.h" #include <cstddef> -class IOutputStream; +class IOutputStream; namespace NCompactTrie { // Return value: size of the resulting trie. diff --git a/library/cpp/containers/comptrie/minimize.h b/library/cpp/containers/comptrie/minimize.h index 5973564844..baaa431d04 100644 --- a/library/cpp/containers/comptrie/minimize.h +++ b/library/cpp/containers/comptrie/minimize.h @@ -3,7 +3,7 @@ #include "leaf_skipper.h" #include <cstddef> -class IOutputStream; +class IOutputStream; namespace NCompactTrie { size_t MeasureOffset(size_t offset); diff --git a/library/cpp/containers/comptrie/write_trie_backwards.h b/library/cpp/containers/comptrie/write_trie_backwards.h index 78e1e6ade4..634e6b811a 100644 --- a/library/cpp/containers/comptrie/write_trie_backwards.h +++ b/library/cpp/containers/comptrie/write_trie_backwards.h @@ -17,7 +17,7 @@ namespace NCompactTrie { struct TOpaqueTrie; - size_t WriteTrieBackwards(IOutputStream& os, TReverseNodeEnumerator& enumerator, bool verbose); - size_t WriteTrieBackwardsNoAlloc(IOutputStream& os, TReverseNodeEnumerator& enumerator, TOpaqueTrie& trie, EMinimizeMode mode); + size_t WriteTrieBackwards(IOutputStream& os, TReverseNodeEnumerator& enumerator, bool verbose); + size_t WriteTrieBackwardsNoAlloc(IOutputStream& os, TReverseNodeEnumerator& enumerator, TOpaqueTrie& trie, EMinimizeMode mode); } |