aboutsummaryrefslogtreecommitdiffstats
path: root/library/cpp/codecs/codecs.cpp
diff options
context:
space:
mode:
authorAnton Samokhvalov <pg83@yandex.ru>2022-02-10 16:45:15 +0300
committerDaniil Cherednik <dcherednik@yandex-team.ru>2022-02-10 16:45:15 +0300
commit72cb13b4aff9bc9cf22e49251bc8fd143f82538f (patch)
treeda2c34829458c7d4e74bdfbdf85dff449e9e7fb8 /library/cpp/codecs/codecs.cpp
parent778e51ba091dc39e7b7fcab2b9cf4dbedfb6f2b5 (diff)
downloadydb-72cb13b4aff9bc9cf22e49251bc8fd143f82538f.tar.gz
Restoring authorship annotation for Anton Samokhvalov <pg83@yandex.ru>. Commit 1 of 2.
Diffstat (limited to 'library/cpp/codecs/codecs.cpp')
-rw-r--r--library/cpp/codecs/codecs.cpp264
1 files changed, 132 insertions, 132 deletions
diff --git a/library/cpp/codecs/codecs.cpp b/library/cpp/codecs/codecs.cpp
index b17a3156d2..bc60d10cf3 100644
--- a/library/cpp/codecs/codecs.cpp
+++ b/library/cpp/codecs/codecs.cpp
@@ -4,187 +4,187 @@
#include <util/stream/mem.h>
namespace NCodecs {
- void ICodec::Store(IOutputStream* out, TCodecPtr p) {
- if (!p.Get()) {
- ::Save(out, (ui16)0);
- return;
- }
-
- Y_ENSURE_EX(p->AlreadyTrained(), TCodecException() << "untrained codec " << p->GetName());
- const TString& n = p->GetName();
+ void ICodec::Store(IOutputStream* out, TCodecPtr p) {
+ if (!p.Get()) {
+ ::Save(out, (ui16)0);
+ return;
+ }
+
+ Y_ENSURE_EX(p->AlreadyTrained(), TCodecException() << "untrained codec " << p->GetName());
+ const TString& n = p->GetName();
Y_VERIFY(n.size() <= Max<ui16>());
::Save(out, (ui16)n.size());
out->Write(n.data(), n.size());
- p->Save(out);
+ p->Save(out);
}
- TCodecPtr ICodec::Restore(IInputStream* in) {
- ui16 l = 0;
- ::Load(in, l);
+ TCodecPtr ICodec::Restore(IInputStream* in) {
+ ui16 l = 0;
+ ::Load(in, l);
- if (!l) {
- return nullptr;
- }
+ if (!l) {
+ return nullptr;
+ }
- TString n;
- n.resize(l);
+ TString n;
+ n.resize(l);
- Y_ENSURE_EX(in->Load(n.begin(), l) == l, TCodecException());
+ Y_ENSURE_EX(in->Load(n.begin(), l) == l, TCodecException());
- TCodecPtr p = ICodec::GetInstance(n);
- p->Load(in);
- p->Trained = true;
- return p;
- }
+ TCodecPtr p = ICodec::GetInstance(n);
+ p->Load(in);
+ p->Trained = true;
+ return p;
+ }
- TCodecPtr ICodec::RestoreFromString(TStringBuf s) {
+ TCodecPtr ICodec::RestoreFromString(TStringBuf s) {
TMemoryInput minp{s.data(), s.size()};
- return Restore(&minp);
- }
+ return Restore(&minp);
+ }
- TString ICodec::GetNameSafe(TCodecPtr p) {
- return !p ? TString("none") : p->GetName();
- }
+ TString ICodec::GetNameSafe(TCodecPtr p) {
+ return !p ? TString("none") : p->GetName();
+ }
- ui8 TPipelineCodec::Encode(TStringBuf in, TBuffer& out) const {
+ ui8 TPipelineCodec::Encode(TStringBuf in, TBuffer& out) const {
size_t res = Traits().ApproximateSizeOnEncode(in.size());
- out.Reserve(res);
- out.Clear();
+ out.Reserve(res);
+ out.Clear();
- if (Pipeline.empty()) {
+ if (Pipeline.empty()) {
out.Append(in.data(), in.size());
- return 0;
- } else if (Pipeline.size() == 1) {
- return Pipeline.front()->Encode(in, out);
- }
+ return 0;
+ } else if (Pipeline.size() == 1) {
+ return Pipeline.front()->Encode(in, out);
+ }
- ui8 freelastbits = 0;
+ ui8 freelastbits = 0;
- auto buffer = TBufferTlsCache::TlsInstance().Item();
- TBuffer& tmp = buffer.Get();
- tmp.Reserve(res);
+ auto buffer = TBufferTlsCache::TlsInstance().Item();
+ TBuffer& tmp = buffer.Get();
+ tmp.Reserve(res);
- for (auto it = Pipeline.begin(); it != Pipeline.end(); ++it) {
- if (it != Pipeline.begin()) {
- tmp.Clear();
- tmp.Swap(out);
+ for (auto it = Pipeline.begin(); it != Pipeline.end(); ++it) {
+ if (it != Pipeline.begin()) {
+ tmp.Clear();
+ tmp.Swap(out);
in = TStringBuf{tmp.data(), tmp.size()};
- }
- freelastbits = (*it)->Encode(in, out);
- }
+ }
+ freelastbits = (*it)->Encode(in, out);
+ }
- return freelastbits;
+ return freelastbits;
}
- void TPipelineCodec::Decode(TStringBuf in, TBuffer& out) const {
+ void TPipelineCodec::Decode(TStringBuf in, TBuffer& out) const {
size_t res = Traits().ApproximateSizeOnDecode(in.size());
- out.Reserve(res);
- out.Clear();
+ out.Reserve(res);
+ out.Clear();
- if (Pipeline.empty()) {
+ if (Pipeline.empty()) {
out.Append(in.data(), in.size());
- return;
- } else if (Pipeline.size() == 1) {
- Pipeline.front()->Decode(in, out);
- return;
- }
+ return;
+ } else if (Pipeline.size() == 1) {
+ Pipeline.front()->Decode(in, out);
+ return;
+ }
- auto buffer = TBufferTlsCache::TlsInstance().Item();
+ auto buffer = TBufferTlsCache::TlsInstance().Item();
- TBuffer& tmp = buffer.Get();
- tmp.Reserve(res);
+ TBuffer& tmp = buffer.Get();
+ tmp.Reserve(res);
- for (TPipeline::const_reverse_iterator it = Pipeline.rbegin(); it != Pipeline.rend(); ++it) {
- if (it != Pipeline.rbegin()) {
- tmp.Clear();
- tmp.Swap(out);
+ for (TPipeline::const_reverse_iterator it = Pipeline.rbegin(); it != Pipeline.rend(); ++it) {
+ if (it != Pipeline.rbegin()) {
+ tmp.Clear();
+ tmp.Swap(out);
in = TStringBuf{tmp.data(), tmp.size()};
- }
- (*it)->Decode(in, out);
+ }
+ (*it)->Decode(in, out);
}
}
- void TPipelineCodec::Save(IOutputStream* out) const {
- for (const auto& it : Pipeline)
- it->Save(out);
+ void TPipelineCodec::Save(IOutputStream* out) const {
+ for (const auto& it : Pipeline)
+ it->Save(out);
}
- void TPipelineCodec::Load(IInputStream* in) {
- for (const auto& it : Pipeline) {
- it->Load(in);
- it->SetTrained(true);
- }
+ void TPipelineCodec::Load(IInputStream* in) {
+ for (const auto& it : Pipeline) {
+ it->Load(in);
+ it->SetTrained(true);
+ }
}
- void TPipelineCodec::SetTrained(bool t) {
- for (const auto& it : Pipeline) {
- it->SetTrained(t);
- }
+ void TPipelineCodec::SetTrained(bool t) {
+ for (const auto& it : Pipeline) {
+ it->SetTrained(t);
+ }
}
- TPipelineCodec& TPipelineCodec::AddCodec(TCodecPtr codec) {
- if (!codec)
- return *this;
-
- TCodecTraits tr = codec->Traits();
-
- if (!MyName) {
- MyTraits.AssumesStructuredInput = tr.AssumesStructuredInput;
- MyTraits.SizeOfInputElement = tr.SizeOfInputElement;
- } else {
- MyName.append(':');
- }
-
- MyName.append(codec->GetName());
- MyTraits.PreservesPrefixGrouping &= tr.PreservesPrefixGrouping;
- MyTraits.PaddingBit = tr.PaddingBit;
- MyTraits.NeedsTraining |= tr.NeedsTraining;
- MyTraits.Irreversible |= tr.Irreversible;
- MyTraits.SizeOnEncodeAddition = MyTraits.SizeOnEncodeAddition * tr.SizeOnEncodeMultiplier + tr.SizeOnEncodeAddition;
- MyTraits.SizeOnEncodeMultiplier *= tr.SizeOnEncodeMultiplier;
- MyTraits.SizeOnDecodeMultiplier *= tr.SizeOnDecodeMultiplier;
- MyTraits.RecommendedSampleSize = Max(MyTraits.RecommendedSampleSize, tr.RecommendedSampleSize);
-
- Pipeline.push_back(codec);
- return *this;
+ TPipelineCodec& TPipelineCodec::AddCodec(TCodecPtr codec) {
+ if (!codec)
+ return *this;
+
+ TCodecTraits tr = codec->Traits();
+
+ if (!MyName) {
+ MyTraits.AssumesStructuredInput = tr.AssumesStructuredInput;
+ MyTraits.SizeOfInputElement = tr.SizeOfInputElement;
+ } else {
+ MyName.append(':');
+ }
+
+ MyName.append(codec->GetName());
+ MyTraits.PreservesPrefixGrouping &= tr.PreservesPrefixGrouping;
+ MyTraits.PaddingBit = tr.PaddingBit;
+ MyTraits.NeedsTraining |= tr.NeedsTraining;
+ MyTraits.Irreversible |= tr.Irreversible;
+ MyTraits.SizeOnEncodeAddition = MyTraits.SizeOnEncodeAddition * tr.SizeOnEncodeMultiplier + tr.SizeOnEncodeAddition;
+ MyTraits.SizeOnEncodeMultiplier *= tr.SizeOnEncodeMultiplier;
+ MyTraits.SizeOnDecodeMultiplier *= tr.SizeOnDecodeMultiplier;
+ MyTraits.RecommendedSampleSize = Max(MyTraits.RecommendedSampleSize, tr.RecommendedSampleSize);
+
+ Pipeline.push_back(codec);
+ return *this;
}
- void TPipelineCodec::DoLearnX(ISequenceReader& in, double sampleSizeMult) {
- if (!Traits().NeedsTraining) {
- return;
- }
+ void TPipelineCodec::DoLearnX(ISequenceReader& in, double sampleSizeMult) {
+ if (!Traits().NeedsTraining) {
+ return;
+ }
- if (Pipeline.size() == 1) {
- Pipeline.back()->Learn(in);
- return;
- }
+ if (Pipeline.size() == 1) {
+ Pipeline.back()->Learn(in);
+ return;
+ }
- TVector<TBuffer> trainingInput;
+ TVector<TBuffer> trainingInput;
- TStringBuf r;
- while (in.NextRegion(r)) {
+ TStringBuf r;
+ while (in.NextRegion(r)) {
trainingInput.emplace_back(r.data(), r.size());
}
-
- TBuffer buff;
- for (const auto& it : Pipeline) {
- it->LearnX(trainingInput.begin(), trainingInput.end(), sampleSizeMult);
-
- for (auto& bit : trainingInput) {
- buff.Clear();
+
+ TBuffer buff;
+ for (const auto& it : Pipeline) {
+ it->LearnX(trainingInput.begin(), trainingInput.end(), sampleSizeMult);
+
+ for (auto& bit : trainingInput) {
+ buff.Clear();
it->Encode(TStringBuf{bit.data(), bit.size()}, buff);
- buff.Swap(bit);
- }
- }
+ buff.Swap(bit);
+ }
+ }
}
- bool TPipelineCodec::AlreadyTrained() const {
- for (const auto& it : Pipeline) {
- if (!it->AlreadyTrained())
- return false;
- }
-
- return true;
+ bool TPipelineCodec::AlreadyTrained() const {
+ for (const auto& it : Pipeline) {
+ if (!it->AlreadyTrained())
+ return false;
+ }
+
+ return true;
}
}