aboutsummaryrefslogtreecommitdiffstats
path: root/library/cpp/archive/yarchive.cpp
diff options
context:
space:
mode:
authorAnton Samokhvalov <pg83@yandex.ru>2022-02-10 16:45:15 +0300
committerDaniil Cherednik <dcherednik@yandex-team.ru>2022-02-10 16:45:15 +0300
commit72cb13b4aff9bc9cf22e49251bc8fd143f82538f (patch)
treeda2c34829458c7d4e74bdfbdf85dff449e9e7fb8 /library/cpp/archive/yarchive.cpp
parent778e51ba091dc39e7b7fcab2b9cf4dbedfb6f2b5 (diff)
downloadydb-72cb13b4aff9bc9cf22e49251bc8fd143f82538f.tar.gz
Restoring authorship annotation for Anton Samokhvalov <pg83@yandex.ru>. Commit 1 of 2.
Diffstat (limited to 'library/cpp/archive/yarchive.cpp')
-rw-r--r--library/cpp/archive/yarchive.cpp552
1 files changed, 276 insertions, 276 deletions
diff --git a/library/cpp/archive/yarchive.cpp b/library/cpp/archive/yarchive.cpp
index 1becc3e5da..7adefc714d 100644
--- a/library/cpp/archive/yarchive.cpp
+++ b/library/cpp/archive/yarchive.cpp
@@ -1,5 +1,5 @@
-#include "yarchive.h"
-
+#include "yarchive.h"
+
#include <util/generic/algorithm.h>
#include <util/generic/hash.h>
#include <util/generic/utility.h>
@@ -14,145 +14,145 @@
#include <util/stream/zlib.h>
#include <util/system/byteorder.h>
#include <util/ysaveload.h>
-
-template <class T>
+
+template <class T>
static inline void ESSave(IOutputStream* out, const T& t_in) {
- T t = HostToLittle(t_in);
-
- out->Write((const void*)&t, sizeof(t));
-}
-
+ T t = HostToLittle(t_in);
+
+ out->Write((const void*)&t, sizeof(t));
+}
+
static inline void ESSave(IOutputStream* out, const TString& s) {
ESSave(out, (ui32) s.size());
out->Write(s.data(), s.size());
-}
-
-template <class T>
+}
+
+template <class T>
static inline T ESLoad(IInputStream* in) {
- T t = T();
-
- if (in->Load(&t, sizeof(t)) != sizeof(t)) {
+ T t = T();
+
+ if (in->Load(&t, sizeof(t)) != sizeof(t)) {
ythrow TSerializeException() << "malformed archive";
- }
-
- return LittleToHost(t);
-}
-
-template <>
+ }
+
+ return LittleToHost(t);
+}
+
+template <>
inline TString ESLoad<TString>(IInputStream* in) {
- size_t len = ESLoad<ui32>(in);
+ size_t len = ESLoad<ui32>(in);
TString ret;
- TTempBuf tmp;
-
- while (len) {
- const size_t toread = Min(len, tmp.Size());
- const size_t readed = in->Read(tmp.Data(), toread);
-
- if (!readed) {
+ TTempBuf tmp;
+
+ while (len) {
+ const size_t toread = Min(len, tmp.Size());
+ const size_t readed = in->Read(tmp.Data(), toread);
+
+ if (!readed) {
ythrow TSerializeException() << "malformed archive";
- }
-
- ret.append(tmp.Data(), readed);
- len -= readed;
- }
-
- return ret;
-}
-
-namespace {
- class TArchiveRecordDescriptor: public TSimpleRefCount<TArchiveRecordDescriptor> {
- public:
+ }
+
+ ret.append(tmp.Data(), readed);
+ len -= readed;
+ }
+
+ return ret;
+}
+
+namespace {
+ class TArchiveRecordDescriptor: public TSimpleRefCount<TArchiveRecordDescriptor> {
+ public:
inline TArchiveRecordDescriptor(ui64 off, ui64 len, const TString& name)
- : Off_(off)
- , Len_(len)
- , Name_(name)
- {
- }
-
+ : Off_(off)
+ , Len_(len)
+ , Name_(name)
+ {
+ }
+
inline TArchiveRecordDescriptor(IInputStream* in)
- : Off_(ESLoad<ui64>(in))
- , Len_(ESLoad<ui64>(in))
+ : Off_(ESLoad<ui64>(in))
+ , Len_(ESLoad<ui64>(in))
, Name_(ESLoad<TString>(in))
- {
- }
-
+ {
+ }
+
inline ~TArchiveRecordDescriptor() = default;
-
+
inline void SaveTo(IOutputStream* out) const {
- ESSave(out, Off_);
- ESSave(out, Len_);
- ESSave(out, Name_);
- }
-
+ ESSave(out, Off_);
+ ESSave(out, Len_);
+ ESSave(out, Name_);
+ }
+
inline const TString& Name() const noexcept {
- return Name_;
- }
-
+ return Name_;
+ }
+
inline ui64 Length() const noexcept {
- return Len_;
- }
-
+ return Len_;
+ }
+
inline ui64 Offset() const noexcept {
- return Off_;
- }
-
- private:
- ui64 Off_;
- ui64 Len_;
+ return Off_;
+ }
+
+ private:
+ ui64 Off_;
+ ui64 Len_;
TString Name_;
- };
-
- typedef TIntrusivePtr<TArchiveRecordDescriptor> TArchiveRecordDescriptorRef;
-}
-
-class TArchiveWriter::TImpl {
+ };
+
+ typedef TIntrusivePtr<TArchiveRecordDescriptor> TArchiveRecordDescriptorRef;
+}
+
+class TArchiveWriter::TImpl {
using TDict = THashMap<TString, TArchiveRecordDescriptorRef>;
-
-public:
+
+public:
inline TImpl(IOutputStream& out, bool compress)
- : Off_(0)
- , Out_(&out)
- , UseCompression(compress)
- {
- }
-
+ : Off_(0)
+ , Out_(&out)
+ , UseCompression(compress)
+ {
+ }
+
inline ~TImpl() = default;
-
- inline void Flush() {
- Out_->Flush();
- }
-
- inline void Finish() {
+
+ inline void Flush() {
+ Out_->Flush();
+ }
+
+ inline void Finish() {
TCountingOutput out(Out_);
-
- {
- TZLibCompress compress(&out);
-
- ESSave(&compress, (ui32)Dict_.size());
-
+
+ {
+ TZLibCompress compress(&out);
+
+ ESSave(&compress, (ui32)Dict_.size());
+
for (const auto& kv : Dict_) {
kv.second->SaveTo(&compress);
- }
-
- ESSave(&compress, static_cast<ui8>(UseCompression));
-
- compress.Finish();
- }
-
+ }
+
+ ESSave(&compress, static_cast<ui8>(UseCompression));
+
+ compress.Finish();
+ }
+
ESSave(Out_, out.Counter());
-
- Out_->Flush();
- }
-
+
+ Out_->Flush();
+ }
+
inline void Add(const TString& key, IInputStream* src) {
Y_ENSURE(!Dict_.contains(key), "key " << key.data() << " already stored");
-
+
TCountingOutput out(Out_);
- if (UseCompression) {
- TZLibCompress compress(&out);
- TransferData(src, &compress);
- compress.Finish();
- } else {
+ if (UseCompression) {
+ TZLibCompress compress(&out);
+ TransferData(src, &compress);
+ compress.Finish();
+ } else {
size_t skip_size = ArchiveWriterDefaultDataAlignment - Off_ % ArchiveWriterDefaultDataAlignment;
if (skip_size == ArchiveWriterDefaultDataAlignment) {
skip_size = 0;
@@ -162,16 +162,16 @@ public:
Off_ += 1;
skip_size -= 1;
}
- TransferData(src, &out);
- out.Finish();
- }
-
+ TransferData(src, &out);
+ out.Finish();
+ }
+
TArchiveRecordDescriptorRef descr(new TArchiveRecordDescriptor(Off_, out.Counter(), key));
-
- Dict_[key] = descr;
+
+ Dict_[key] = descr;
Off_ += out.Counter();
- }
-
+ }
+
inline void AddSynonym(const TString& existingKey, const TString& newKey) {
Y_ENSURE(Dict_.contains(existingKey), "key " << existingKey.data() << " not stored yet");
Y_ENSURE(!Dict_.contains(newKey), "key " << newKey.data() << " already stored");
@@ -182,208 +182,208 @@ public:
Dict_[newKey] = descr;
}
-private:
- ui64 Off_;
+private:
+ ui64 Off_;
IOutputStream* Out_;
- TDict Dict_;
- const bool UseCompression;
-};
-
+ TDict Dict_;
+ const bool UseCompression;
+};
+
TArchiveWriter::TArchiveWriter(IOutputStream* out, bool compress)
: Impl_(new TImpl(*out, compress))
-{
-}
-
+{
+}
+
TArchiveWriter::~TArchiveWriter() {
- try {
- Finish();
- } catch (...) {
- }
-}
-
-void TArchiveWriter::Flush() {
- if (Impl_.Get()) {
- Impl_->Flush();
- }
-}
-
-void TArchiveWriter::Finish() {
- if (Impl_.Get()) {
- Impl_->Finish();
- Impl_.Destroy();
- }
-}
-
+ try {
+ Finish();
+ } catch (...) {
+ }
+}
+
+void TArchiveWriter::Flush() {
+ if (Impl_.Get()) {
+ Impl_->Flush();
+ }
+}
+
+void TArchiveWriter::Finish() {
+ if (Impl_.Get()) {
+ Impl_->Finish();
+ Impl_.Destroy();
+ }
+}
+
void TArchiveWriter::Add(const TString& key, IInputStream* src) {
Y_ENSURE(Impl_.Get(), "archive already closed");
-
- Impl_->Add(key, src);
-}
-
+
+ Impl_->Add(key, src);
+}
+
void TArchiveWriter::AddSynonym(const TString& existingKey, const TString& newKey) {
Y_ENSURE(Impl_.Get(), "archive already closed");
Impl_->AddSynonym(existingKey, newKey);
}
-namespace {
- class TArchiveInputStreamBase {
- public:
- inline TArchiveInputStreamBase(const TBlob& b)
- : Blob_(b)
- , Input_(b.Data(), b.Size())
- {
- }
-
- protected:
- TBlob Blob_;
- TMemoryInput Input_;
- };
-
- class TArchiveInputStream: public TArchiveInputStreamBase, public TZLibDecompress {
- public:
- inline TArchiveInputStream(const TBlob& b)
- : TArchiveInputStreamBase(b)
+namespace {
+ class TArchiveInputStreamBase {
+ public:
+ inline TArchiveInputStreamBase(const TBlob& b)
+ : Blob_(b)
+ , Input_(b.Data(), b.Size())
+ {
+ }
+
+ protected:
+ TBlob Blob_;
+ TMemoryInput Input_;
+ };
+
+ class TArchiveInputStream: public TArchiveInputStreamBase, public TZLibDecompress {
+ public:
+ inline TArchiveInputStream(const TBlob& b)
+ : TArchiveInputStreamBase(b)
, TZLibDecompress(&Input_)
- {
- }
-
+ {
+ }
+
~TArchiveInputStream() override = default;
- };
-}
-
-class TArchiveReader::TImpl {
+ };
+}
+
+class TArchiveReader::TImpl {
typedef THashMap<TString, TArchiveRecordDescriptorRef> TDict;
-
-public:
- inline TImpl(const TBlob& blob)
- : Blob_(blob)
- , UseDecompression(true)
- {
- ReadDict();
- }
-
+
+public:
+ inline TImpl(const TBlob& blob)
+ : Blob_(blob)
+ , UseDecompression(true)
+ {
+ ReadDict();
+ }
+
inline ~TImpl() = default;
-
- inline void ReadDict() {
+
+ inline void ReadDict() {
Y_ENSURE(Blob_.Size() >= sizeof(ui64), "too small blob");
-
- const char* end = (const char*)Blob_.End();
- const char* ptr = end - sizeof(ui64);
- ui64 dictlen = 0;
- memcpy(&dictlen, ptr, sizeof(ui64));
- dictlen = LittleToHost(dictlen);
-
+
+ const char* end = (const char*)Blob_.End();
+ const char* ptr = end - sizeof(ui64);
+ ui64 dictlen = 0;
+ memcpy(&dictlen, ptr, sizeof(ui64));
+ dictlen = LittleToHost(dictlen);
+
Y_ENSURE(dictlen <= Blob_.Size() - sizeof(ui64), "bad blob");
-
- const char* beg = ptr - dictlen;
- TMemoryInput mi(beg, dictlen);
+
+ const char* beg = ptr - dictlen;
+ TMemoryInput mi(beg, dictlen);
TZLibDecompress d(&mi);
- const ui32 count = ESLoad<ui32>(&d);
-
- for (size_t i = 0; i < count; ++i) {
- TArchiveRecordDescriptorRef descr(new TArchiveRecordDescriptor(&d));
-
- Recs_.push_back(descr);
- Dict_[descr->Name()] = descr;
- }
+ const ui32 count = ESLoad<ui32>(&d);
+
+ for (size_t i = 0; i < count; ++i) {
+ TArchiveRecordDescriptorRef descr(new TArchiveRecordDescriptor(&d));
+
+ Recs_.push_back(descr);
+ Dict_[descr->Name()] = descr;
+ }
Sort(Recs_.begin(), Recs_.end(), [](const auto& lhs, const auto& rhs) -> bool {
return lhs->Offset() < rhs->Offset();
});
-
- try {
- UseDecompression = static_cast<bool>(ESLoad<ui8>(&d));
- } catch (const TSerializeException&) {
- // that's ok - just old format
- UseDecompression = true;
- }
- }
-
+
+ try {
+ UseDecompression = static_cast<bool>(ESLoad<ui8>(&d));
+ } catch (const TSerializeException&) {
+ // that's ok - just old format
+ UseDecompression = true;
+ }
+ }
+
inline size_t Count() const noexcept {
- return Recs_.size();
- }
-
+ return Recs_.size();
+ }
+
inline TString KeyByIndex(size_t n) const {
- if (n < Count()) {
- return Recs_[n]->Name();
- }
-
- ythrow yexception() << "incorrect index";
- }
+ if (n < Count()) {
+ return Recs_[n]->Name();
+ }
+
+ ythrow yexception() << "incorrect index";
+ }
inline bool Has(const TStringBuf key) const {
return Dict_.contains(key);
- }
+ }
inline TAutoPtr<IInputStream> ObjectByKey(const TStringBuf key) const {
- TBlob subBlob = BlobByKey(key);
-
- if (UseDecompression) {
- return new TArchiveInputStream(subBlob);
- } else {
- return new TMemoryInput(subBlob.Data(), subBlob.Length());
+ TBlob subBlob = BlobByKey(key);
+
+ if (UseDecompression) {
+ return new TArchiveInputStream(subBlob);
+ } else {
+ return new TMemoryInput(subBlob.Data(), subBlob.Length());
}
- }
+ }
inline TBlob ObjectBlobByKey(const TStringBuf key) const {
- TBlob subBlob = BlobByKey(key);
+ TBlob subBlob = BlobByKey(key);
- if (UseDecompression) {
- TArchiveInputStream st(subBlob);
- return TBlob::FromStream(st);
- } else {
- return subBlob;
+ if (UseDecompression) {
+ TArchiveInputStream st(subBlob);
+ return TBlob::FromStream(st);
+ } else {
+ return subBlob;
}
- }
+ }
inline TBlob BlobByKey(const TStringBuf key) const {
const auto it = Dict_.find(key);
-
+
Y_ENSURE(it != Dict_.end(), "key " << key.data() << " not found");
-
- const size_t off = it->second->Offset();
- const size_t len = it->second->Length();
-
- /*
- * TODO - overflow check
- */
-
- return Blob_.SubBlob(off, off + len);
- }
-
- inline bool Compressed() const {
- return UseDecompression;
- }
-
-private:
- TBlob Blob_;
+
+ const size_t off = it->second->Offset();
+ const size_t len = it->second->Length();
+
+ /*
+ * TODO - overflow check
+ */
+
+ return Blob_.SubBlob(off, off + len);
+ }
+
+ inline bool Compressed() const {
+ return UseDecompression;
+ }
+
+private:
+ TBlob Blob_;
TVector<TArchiveRecordDescriptorRef> Recs_;
- TDict Dict_;
- bool UseDecompression;
-};
-
-TArchiveReader::TArchiveReader(const TBlob& data)
- : Impl_(new TImpl(data))
-{
-}
-
+ TDict Dict_;
+ bool UseDecompression;
+};
+
+TArchiveReader::TArchiveReader(const TBlob& data)
+ : Impl_(new TImpl(data))
+{
+}
+
TArchiveReader::~TArchiveReader() {}
-
+
size_t TArchiveReader::Count() const noexcept {
- return Impl_->Count();
-}
-
+ return Impl_->Count();
+}
+
TString TArchiveReader::KeyByIndex(size_t n) const {
- return Impl_->KeyByIndex(n);
-}
-
+ return Impl_->KeyByIndex(n);
+}
+
bool TArchiveReader::Has(const TStringBuf key) const {
return Impl_->Has(key);
}
TAutoPtr<IInputStream> TArchiveReader::ObjectByKey(const TStringBuf key) const {
- return Impl_->ObjectByKey(key);
-}
+ return Impl_->ObjectByKey(key);
+}
TBlob TArchiveReader::ObjectBlobByKey(const TStringBuf key) const {
return Impl_->ObjectBlobByKey(key);