diff options
author | Anton Samokhvalov <pg83@yandex.ru> | 2022-02-10 16:45:15 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:45:15 +0300 |
commit | 72cb13b4aff9bc9cf22e49251bc8fd143f82538f (patch) | |
tree | da2c34829458c7d4e74bdfbdf85dff449e9e7fb8 /library/cpp/codecs/codecs.cpp | |
parent | 778e51ba091dc39e7b7fcab2b9cf4dbedfb6f2b5 (diff) | |
download | ydb-72cb13b4aff9bc9cf22e49251bc8fd143f82538f.tar.gz |
Restoring authorship annotation for Anton Samokhvalov <pg83@yandex.ru>. Commit 1 of 2.
Diffstat (limited to 'library/cpp/codecs/codecs.cpp')
-rw-r--r-- | library/cpp/codecs/codecs.cpp | 264 |
1 files changed, 132 insertions, 132 deletions
diff --git a/library/cpp/codecs/codecs.cpp b/library/cpp/codecs/codecs.cpp index b17a3156d2..bc60d10cf3 100644 --- a/library/cpp/codecs/codecs.cpp +++ b/library/cpp/codecs/codecs.cpp @@ -4,187 +4,187 @@ #include <util/stream/mem.h> namespace NCodecs { - void ICodec::Store(IOutputStream* out, TCodecPtr p) { - if (!p.Get()) { - ::Save(out, (ui16)0); - return; - } - - Y_ENSURE_EX(p->AlreadyTrained(), TCodecException() << "untrained codec " << p->GetName()); - const TString& n = p->GetName(); + void ICodec::Store(IOutputStream* out, TCodecPtr p) { + if (!p.Get()) { + ::Save(out, (ui16)0); + return; + } + + Y_ENSURE_EX(p->AlreadyTrained(), TCodecException() << "untrained codec " << p->GetName()); + const TString& n = p->GetName(); Y_VERIFY(n.size() <= Max<ui16>()); ::Save(out, (ui16)n.size()); out->Write(n.data(), n.size()); - p->Save(out); + p->Save(out); } - TCodecPtr ICodec::Restore(IInputStream* in) { - ui16 l = 0; - ::Load(in, l); + TCodecPtr ICodec::Restore(IInputStream* in) { + ui16 l = 0; + ::Load(in, l); - if (!l) { - return nullptr; - } + if (!l) { + return nullptr; + } - TString n; - n.resize(l); + TString n; + n.resize(l); - Y_ENSURE_EX(in->Load(n.begin(), l) == l, TCodecException()); + Y_ENSURE_EX(in->Load(n.begin(), l) == l, TCodecException()); - TCodecPtr p = ICodec::GetInstance(n); - p->Load(in); - p->Trained = true; - return p; - } + TCodecPtr p = ICodec::GetInstance(n); + p->Load(in); + p->Trained = true; + return p; + } - TCodecPtr ICodec::RestoreFromString(TStringBuf s) { + TCodecPtr ICodec::RestoreFromString(TStringBuf s) { TMemoryInput minp{s.data(), s.size()}; - return Restore(&minp); - } + return Restore(&minp); + } - TString ICodec::GetNameSafe(TCodecPtr p) { - return !p ? TString("none") : p->GetName(); - } + TString ICodec::GetNameSafe(TCodecPtr p) { + return !p ? TString("none") : p->GetName(); + } - ui8 TPipelineCodec::Encode(TStringBuf in, TBuffer& out) const { + ui8 TPipelineCodec::Encode(TStringBuf in, TBuffer& out) const { size_t res = Traits().ApproximateSizeOnEncode(in.size()); - out.Reserve(res); - out.Clear(); + out.Reserve(res); + out.Clear(); - if (Pipeline.empty()) { + if (Pipeline.empty()) { out.Append(in.data(), in.size()); - return 0; - } else if (Pipeline.size() == 1) { - return Pipeline.front()->Encode(in, out); - } + return 0; + } else if (Pipeline.size() == 1) { + return Pipeline.front()->Encode(in, out); + } - ui8 freelastbits = 0; + ui8 freelastbits = 0; - auto buffer = TBufferTlsCache::TlsInstance().Item(); - TBuffer& tmp = buffer.Get(); - tmp.Reserve(res); + auto buffer = TBufferTlsCache::TlsInstance().Item(); + TBuffer& tmp = buffer.Get(); + tmp.Reserve(res); - for (auto it = Pipeline.begin(); it != Pipeline.end(); ++it) { - if (it != Pipeline.begin()) { - tmp.Clear(); - tmp.Swap(out); + for (auto it = Pipeline.begin(); it != Pipeline.end(); ++it) { + if (it != Pipeline.begin()) { + tmp.Clear(); + tmp.Swap(out); in = TStringBuf{tmp.data(), tmp.size()}; - } - freelastbits = (*it)->Encode(in, out); - } + } + freelastbits = (*it)->Encode(in, out); + } - return freelastbits; + return freelastbits; } - void TPipelineCodec::Decode(TStringBuf in, TBuffer& out) const { + void TPipelineCodec::Decode(TStringBuf in, TBuffer& out) const { size_t res = Traits().ApproximateSizeOnDecode(in.size()); - out.Reserve(res); - out.Clear(); + out.Reserve(res); + out.Clear(); - if (Pipeline.empty()) { + if (Pipeline.empty()) { out.Append(in.data(), in.size()); - return; - } else if (Pipeline.size() == 1) { - Pipeline.front()->Decode(in, out); - return; - } + return; + } else if (Pipeline.size() == 1) { + Pipeline.front()->Decode(in, out); + return; + } - auto buffer = TBufferTlsCache::TlsInstance().Item(); + auto buffer = TBufferTlsCache::TlsInstance().Item(); - TBuffer& tmp = buffer.Get(); - tmp.Reserve(res); + TBuffer& tmp = buffer.Get(); + tmp.Reserve(res); - for (TPipeline::const_reverse_iterator it = Pipeline.rbegin(); it != Pipeline.rend(); ++it) { - if (it != Pipeline.rbegin()) { - tmp.Clear(); - tmp.Swap(out); + for (TPipeline::const_reverse_iterator it = Pipeline.rbegin(); it != Pipeline.rend(); ++it) { + if (it != Pipeline.rbegin()) { + tmp.Clear(); + tmp.Swap(out); in = TStringBuf{tmp.data(), tmp.size()}; - } - (*it)->Decode(in, out); + } + (*it)->Decode(in, out); } } - void TPipelineCodec::Save(IOutputStream* out) const { - for (const auto& it : Pipeline) - it->Save(out); + void TPipelineCodec::Save(IOutputStream* out) const { + for (const auto& it : Pipeline) + it->Save(out); } - void TPipelineCodec::Load(IInputStream* in) { - for (const auto& it : Pipeline) { - it->Load(in); - it->SetTrained(true); - } + void TPipelineCodec::Load(IInputStream* in) { + for (const auto& it : Pipeline) { + it->Load(in); + it->SetTrained(true); + } } - void TPipelineCodec::SetTrained(bool t) { - for (const auto& it : Pipeline) { - it->SetTrained(t); - } + void TPipelineCodec::SetTrained(bool t) { + for (const auto& it : Pipeline) { + it->SetTrained(t); + } } - TPipelineCodec& TPipelineCodec::AddCodec(TCodecPtr codec) { - if (!codec) - return *this; - - TCodecTraits tr = codec->Traits(); - - if (!MyName) { - MyTraits.AssumesStructuredInput = tr.AssumesStructuredInput; - MyTraits.SizeOfInputElement = tr.SizeOfInputElement; - } else { - MyName.append(':'); - } - - MyName.append(codec->GetName()); - MyTraits.PreservesPrefixGrouping &= tr.PreservesPrefixGrouping; - MyTraits.PaddingBit = tr.PaddingBit; - MyTraits.NeedsTraining |= tr.NeedsTraining; - MyTraits.Irreversible |= tr.Irreversible; - MyTraits.SizeOnEncodeAddition = MyTraits.SizeOnEncodeAddition * tr.SizeOnEncodeMultiplier + tr.SizeOnEncodeAddition; - MyTraits.SizeOnEncodeMultiplier *= tr.SizeOnEncodeMultiplier; - MyTraits.SizeOnDecodeMultiplier *= tr.SizeOnDecodeMultiplier; - MyTraits.RecommendedSampleSize = Max(MyTraits.RecommendedSampleSize, tr.RecommendedSampleSize); - - Pipeline.push_back(codec); - return *this; + TPipelineCodec& TPipelineCodec::AddCodec(TCodecPtr codec) { + if (!codec) + return *this; + + TCodecTraits tr = codec->Traits(); + + if (!MyName) { + MyTraits.AssumesStructuredInput = tr.AssumesStructuredInput; + MyTraits.SizeOfInputElement = tr.SizeOfInputElement; + } else { + MyName.append(':'); + } + + MyName.append(codec->GetName()); + MyTraits.PreservesPrefixGrouping &= tr.PreservesPrefixGrouping; + MyTraits.PaddingBit = tr.PaddingBit; + MyTraits.NeedsTraining |= tr.NeedsTraining; + MyTraits.Irreversible |= tr.Irreversible; + MyTraits.SizeOnEncodeAddition = MyTraits.SizeOnEncodeAddition * tr.SizeOnEncodeMultiplier + tr.SizeOnEncodeAddition; + MyTraits.SizeOnEncodeMultiplier *= tr.SizeOnEncodeMultiplier; + MyTraits.SizeOnDecodeMultiplier *= tr.SizeOnDecodeMultiplier; + MyTraits.RecommendedSampleSize = Max(MyTraits.RecommendedSampleSize, tr.RecommendedSampleSize); + + Pipeline.push_back(codec); + return *this; } - void TPipelineCodec::DoLearnX(ISequenceReader& in, double sampleSizeMult) { - if (!Traits().NeedsTraining) { - return; - } + void TPipelineCodec::DoLearnX(ISequenceReader& in, double sampleSizeMult) { + if (!Traits().NeedsTraining) { + return; + } - if (Pipeline.size() == 1) { - Pipeline.back()->Learn(in); - return; - } + if (Pipeline.size() == 1) { + Pipeline.back()->Learn(in); + return; + } - TVector<TBuffer> trainingInput; + TVector<TBuffer> trainingInput; - TStringBuf r; - while (in.NextRegion(r)) { + TStringBuf r; + while (in.NextRegion(r)) { trainingInput.emplace_back(r.data(), r.size()); } - - TBuffer buff; - for (const auto& it : Pipeline) { - it->LearnX(trainingInput.begin(), trainingInput.end(), sampleSizeMult); - - for (auto& bit : trainingInput) { - buff.Clear(); + + TBuffer buff; + for (const auto& it : Pipeline) { + it->LearnX(trainingInput.begin(), trainingInput.end(), sampleSizeMult); + + for (auto& bit : trainingInput) { + buff.Clear(); it->Encode(TStringBuf{bit.data(), bit.size()}, buff); - buff.Swap(bit); - } - } + buff.Swap(bit); + } + } } - bool TPipelineCodec::AlreadyTrained() const { - for (const auto& it : Pipeline) { - if (!it->AlreadyTrained()) - return false; - } - - return true; + bool TPipelineCodec::AlreadyTrained() const { + for (const auto& it : Pipeline) { + if (!it->AlreadyTrained()) + return false; + } + + return true; } } |