aboutsummaryrefslogtreecommitdiffstats
path: root/library/cpp/codecs/codecs.cpp
diff options
context:
space:
mode:
authorRuslan Kovalev <ruslan.a.kovalev@gmail.com>2022-02-10 16:46:45 +0300
committerDaniil Cherednik <dcherednik@yandex-team.ru>2022-02-10 16:46:45 +0300
commit9123176b341b6f2658cff5132482b8237c1416c8 (patch)
tree49e222ea1c5804306084bb3ae065bb702625360f /library/cpp/codecs/codecs.cpp
parent59e19371de37995fcb36beb16cd6ec030af960bc (diff)
downloadydb-9123176b341b6f2658cff5132482b8237c1416c8.tar.gz
Restoring authorship annotation for Ruslan Kovalev <ruslan.a.kovalev@gmail.com>. Commit 2 of 2.
Diffstat (limited to 'library/cpp/codecs/codecs.cpp')
-rw-r--r--library/cpp/codecs/codecs.cpp102
1 files changed, 51 insertions, 51 deletions
diff --git a/library/cpp/codecs/codecs.cpp b/library/cpp/codecs/codecs.cpp
index d2265dd9f9..b17a3156d2 100644
--- a/library/cpp/codecs/codecs.cpp
+++ b/library/cpp/codecs/codecs.cpp
@@ -1,69 +1,69 @@
-#include "codecs.h"
-#include "tls_cache.h"
-
-#include <util/stream/mem.h>
-
-namespace NCodecs {
+#include "codecs.h"
+#include "tls_cache.h"
+
+#include <util/stream/mem.h>
+
+namespace NCodecs {
void ICodec::Store(IOutputStream* out, TCodecPtr p) {
if (!p.Get()) {
::Save(out, (ui16)0);
return;
}
-
+
Y_ENSURE_EX(p->AlreadyTrained(), TCodecException() << "untrained codec " << p->GetName());
const TString& n = p->GetName();
Y_VERIFY(n.size() <= Max<ui16>());
::Save(out, (ui16)n.size());
out->Write(n.data(), n.size());
p->Save(out);
- }
-
+ }
+
TCodecPtr ICodec::Restore(IInputStream* in) {
ui16 l = 0;
::Load(in, l);
-
+
if (!l) {
return nullptr;
}
-
+
TString n;
n.resize(l);
-
+
Y_ENSURE_EX(in->Load(n.begin(), l) == l, TCodecException());
-
+
TCodecPtr p = ICodec::GetInstance(n);
p->Load(in);
p->Trained = true;
return p;
}
-
+
TCodecPtr ICodec::RestoreFromString(TStringBuf s) {
TMemoryInput minp{s.data(), s.size()};
return Restore(&minp);
}
-
+
TString ICodec::GetNameSafe(TCodecPtr p) {
return !p ? TString("none") : p->GetName();
}
-
+
ui8 TPipelineCodec::Encode(TStringBuf in, TBuffer& out) const {
size_t res = Traits().ApproximateSizeOnEncode(in.size());
out.Reserve(res);
out.Clear();
-
+
if (Pipeline.empty()) {
out.Append(in.data(), in.size());
return 0;
} else if (Pipeline.size() == 1) {
return Pipeline.front()->Encode(in, out);
}
-
+
ui8 freelastbits = 0;
-
+
auto buffer = TBufferTlsCache::TlsInstance().Item();
TBuffer& tmp = buffer.Get();
tmp.Reserve(res);
-
+
for (auto it = Pipeline.begin(); it != Pipeline.end(); ++it) {
if (it != Pipeline.begin()) {
tmp.Clear();
@@ -72,15 +72,15 @@ namespace NCodecs {
}
freelastbits = (*it)->Encode(in, out);
}
-
+
return freelastbits;
- }
-
+ }
+
void TPipelineCodec::Decode(TStringBuf in, TBuffer& out) const {
size_t res = Traits().ApproximateSizeOnDecode(in.size());
out.Reserve(res);
out.Clear();
-
+
if (Pipeline.empty()) {
out.Append(in.data(), in.size());
return;
@@ -88,12 +88,12 @@ namespace NCodecs {
Pipeline.front()->Decode(in, out);
return;
}
-
+
auto buffer = TBufferTlsCache::TlsInstance().Item();
-
+
TBuffer& tmp = buffer.Get();
tmp.Reserve(res);
-
+
for (TPipeline::const_reverse_iterator it = Pipeline.rbegin(); it != Pipeline.rend(); ++it) {
if (it != Pipeline.rbegin()) {
tmp.Clear();
@@ -101,40 +101,40 @@ namespace NCodecs {
in = TStringBuf{tmp.data(), tmp.size()};
}
(*it)->Decode(in, out);
- }
- }
-
+ }
+ }
+
void TPipelineCodec::Save(IOutputStream* out) const {
for (const auto& it : Pipeline)
it->Save(out);
- }
-
+ }
+
void TPipelineCodec::Load(IInputStream* in) {
for (const auto& it : Pipeline) {
it->Load(in);
it->SetTrained(true);
}
- }
-
+ }
+
void TPipelineCodec::SetTrained(bool t) {
for (const auto& it : Pipeline) {
it->SetTrained(t);
}
- }
-
+ }
+
TPipelineCodec& TPipelineCodec::AddCodec(TCodecPtr codec) {
if (!codec)
return *this;
-
+
TCodecTraits tr = codec->Traits();
-
+
if (!MyName) {
MyTraits.AssumesStructuredInput = tr.AssumesStructuredInput;
MyTraits.SizeOfInputElement = tr.SizeOfInputElement;
} else {
MyName.append(':');
}
-
+
MyName.append(codec->GetName());
MyTraits.PreservesPrefixGrouping &= tr.PreservesPrefixGrouping;
MyTraits.PaddingBit = tr.PaddingBit;
@@ -144,27 +144,27 @@ namespace NCodecs {
MyTraits.SizeOnEncodeMultiplier *= tr.SizeOnEncodeMultiplier;
MyTraits.SizeOnDecodeMultiplier *= tr.SizeOnDecodeMultiplier;
MyTraits.RecommendedSampleSize = Max(MyTraits.RecommendedSampleSize, tr.RecommendedSampleSize);
-
+
Pipeline.push_back(codec);
return *this;
- }
-
+ }
+
void TPipelineCodec::DoLearnX(ISequenceReader& in, double sampleSizeMult) {
if (!Traits().NeedsTraining) {
return;
}
-
+
if (Pipeline.size() == 1) {
Pipeline.back()->Learn(in);
return;
}
-
+
TVector<TBuffer> trainingInput;
-
+
TStringBuf r;
while (in.NextRegion(r)) {
trainingInput.emplace_back(r.data(), r.size());
- }
+ }
TBuffer buff;
for (const auto& it : Pipeline) {
@@ -176,8 +176,8 @@ namespace NCodecs {
buff.Swap(bit);
}
}
- }
-
+ }
+
bool TPipelineCodec::AlreadyTrained() const {
for (const auto& it : Pipeline) {
if (!it->AlreadyTrained())
@@ -185,6 +185,6 @@ namespace NCodecs {
}
return true;
- }
-
-}
+ }
+
+}