diff options
author | Anton Samokhvalov <pg83@yandex.ru> | 2022-02-10 16:45:15 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:45:15 +0300 |
commit | 72cb13b4aff9bc9cf22e49251bc8fd143f82538f (patch) | |
tree | da2c34829458c7d4e74bdfbdf85dff449e9e7fb8 /library/cpp/streams | |
parent | 778e51ba091dc39e7b7fcab2b9cf4dbedfb6f2b5 (diff) | |
download | ydb-72cb13b4aff9bc9cf22e49251bc8fd143f82538f.tar.gz |
Restoring authorship annotation for Anton Samokhvalov <pg83@yandex.ru>. Commit 1 of 2.
Diffstat (limited to 'library/cpp/streams')
21 files changed, 1664 insertions, 1664 deletions
diff --git a/library/cpp/streams/brotli/brotli.cpp b/library/cpp/streams/brotli/brotli.cpp index 38052cb688..ce2567724a 100644 --- a/library/cpp/streams/brotli/brotli.cpp +++ b/library/cpp/streams/brotli/brotli.cpp @@ -122,7 +122,7 @@ void TBrotliCompress::DoFinish() { //////////////////////////////////////////////////////////////////////////////// -class TBrotliDecompress::TImpl: public TAdditionalStorage<TImpl> { +class TBrotliDecompress::TImpl: public TAdditionalStorage<TImpl> { public: TImpl(IInputStream* slave) : Slave_(slave) @@ -190,7 +190,7 @@ private: bool SubstreamFinished_ = false; bool InputExhausted_ = false; - const ui8* InputBuffer_ = nullptr; + const ui8* InputBuffer_ = nullptr; size_t InputAvailable_ = 0; unsigned char* TmpBuf() noexcept { @@ -220,7 +220,7 @@ private: }; TBrotliDecompress::TBrotliDecompress(IInputStream* slave, size_t bufferSize) - : Impl_(new (bufferSize) TImpl(slave)) + : Impl_(new (bufferSize) TImpl(slave)) { } diff --git a/library/cpp/streams/brotli/brotli.h b/library/cpp/streams/brotli/brotli.h index b3af869e29..1d60b41851 100644 --- a/library/cpp/streams/brotli/brotli.h +++ b/library/cpp/streams/brotli/brotli.h @@ -9,7 +9,7 @@ * @{ */ -class TBrotliCompress: public IOutputStream { +class TBrotliCompress: public IOutputStream { public: static constexpr int BEST_QUALITY = 11; @@ -32,7 +32,7 @@ public: //////////////////////////////////////////////////////////////////////////////// -class TBrotliDecompress: public IInputStream { +class TBrotliDecompress: public IInputStream { public: /** @param slave stream to read compressed data from diff --git a/library/cpp/streams/bzip2/bzip2.cpp b/library/cpp/streams/bzip2/bzip2.cpp index bccc5c6807..37c484fed7 100644 --- a/library/cpp/streams/bzip2/bzip2.cpp +++ b/library/cpp/streams/bzip2/bzip2.cpp @@ -1,204 +1,204 @@ -#include "bzip2.h" - -#include <util/memory/addstorage.h> -#include <util/generic/scope.h> - -#include <contrib/libs/libbz2/bzlib.h> - -class TBZipDecompress::TImpl: public TAdditionalStorage<TImpl> { -public: +#include "bzip2.h" + +#include <util/memory/addstorage.h> +#include <util/generic/scope.h> + +#include <contrib/libs/libbz2/bzlib.h> + +class TBZipDecompress::TImpl: public TAdditionalStorage<TImpl> { +public: inline TImpl(IInputStream* input) - : Stream_(input) - { - Zero(BzStream_); - Init(); - } - + : Stream_(input) + { + Zero(BzStream_); + Init(); + } + inline ~TImpl() { - Clear(); - } - - inline void Init() { + Clear(); + } + + inline void Init() { if (BZ2_bzDecompressInit(&BzStream_, 0, 0) != BZ_OK) { - ythrow TBZipDecompressError() << "can not init bzip engine"; - } - } - + ythrow TBZipDecompressError() << "can not init bzip engine"; + } + } + inline void Clear() noexcept { BZ2_bzDecompressEnd(&BzStream_); - } - - inline size_t Read(void* buf, size_t size) { - BzStream_.next_out = (char*)buf; - BzStream_.avail_out = size; - - while (true) { - if (BzStream_.avail_in == 0) { - if (FillInputBuffer() == 0) { - return 0; - } - } - + } + + inline size_t Read(void* buf, size_t size) { + BzStream_.next_out = (char*)buf; + BzStream_.avail_out = size; + + while (true) { + if (BzStream_.avail_in == 0) { + if (FillInputBuffer() == 0) { + return 0; + } + } + switch (BZ2_bzDecompress(&BzStream_)) { - case BZ_STREAM_END: { - Clear(); - Init(); + case BZ_STREAM_END: { + Clear(); + Init(); [[fallthrough]]; - } - - case BZ_OK: { - const size_t processed = size - BzStream_.avail_out; - - if (processed) { - return processed; - } - - break; - } - - default: - ythrow TBZipDecompressError() << "bzip error"; - } - } - } - - inline size_t FillInputBuffer() { - BzStream_.next_in = (char*)AdditionalData(); - BzStream_.avail_in = Stream_->Read(BzStream_.next_in, AdditionalDataLength()); - - return BzStream_.avail_in; - } - -private: + } + + case BZ_OK: { + const size_t processed = size - BzStream_.avail_out; + + if (processed) { + return processed; + } + + break; + } + + default: + ythrow TBZipDecompressError() << "bzip error"; + } + } + } + + inline size_t FillInputBuffer() { + BzStream_.next_in = (char*)AdditionalData(); + BzStream_.avail_in = Stream_->Read(BzStream_.next_in, AdditionalDataLength()); + + return BzStream_.avail_in; + } + +private: IInputStream* Stream_; - bz_stream BzStream_; -}; - + bz_stream BzStream_; +}; + TBZipDecompress::TBZipDecompress(IInputStream* input, size_t bufLen) - : Impl_(new (bufLen) TImpl(input)) -{ -} - + : Impl_(new (bufLen) TImpl(input)) +{ +} + TBZipDecompress::~TBZipDecompress() { -} - -size_t TBZipDecompress::DoRead(void* buf, size_t size) { - return Impl_->Read(buf, size); -} - -class TBZipCompress::TImpl: public TAdditionalStorage<TImpl> { -public: +} + +size_t TBZipDecompress::DoRead(void* buf, size_t size) { + return Impl_->Read(buf, size); +} + +class TBZipCompress::TImpl: public TAdditionalStorage<TImpl> { +public: inline TImpl(IOutputStream* stream, size_t level) - : Stream_(stream) - { - Zero(BzStream_); - + : Stream_(stream) + { + Zero(BzStream_); + if (BZ2_bzCompressInit(&BzStream_, level, 0, 0) != BZ_OK) { - ythrow TBZipCompressError() << "can not init bzip engine"; - } - - BzStream_.next_out = TmpBuf(); - BzStream_.avail_out = TmpBufLen(); - } - + ythrow TBZipCompressError() << "can not init bzip engine"; + } + + BzStream_.next_out = TmpBuf(); + BzStream_.avail_out = TmpBufLen(); + } + inline ~TImpl() { BZ2_bzCompressEnd(&BzStream_); - } - - inline void Write(const void* buf, size_t size) { - BzStream_.next_in = (char*)buf; - BzStream_.avail_in = size; - - Y_DEFER { - BzStream_.next_in = 0; - BzStream_.avail_in = 0; - }; - - while (BzStream_.avail_in) { + } + + inline void Write(const void* buf, size_t size) { + BzStream_.next_in = (char*)buf; + BzStream_.avail_in = size; + + Y_DEFER { + BzStream_.next_in = 0; + BzStream_.avail_in = 0; + }; + + while (BzStream_.avail_in) { const int ret = BZ2_bzCompress(&BzStream_, BZ_RUN); - - switch (ret) { - case BZ_RUN_OK: - continue; - - case BZ_PARAM_ERROR: - case BZ_OUTBUFF_FULL: - Stream_->Write(TmpBuf(), TmpBufLen() - BzStream_.avail_out); - BzStream_.next_out = TmpBuf(); - BzStream_.avail_out = TmpBufLen(); - - break; - - default: - ythrow TBZipCompressError() << "bzip error(" << ret << ", " << BzStream_.avail_out << ")"; - } - } - } - - inline void Flush() { - /* - * TODO ? - */ - } - - inline void Finish() { + + switch (ret) { + case BZ_RUN_OK: + continue; + + case BZ_PARAM_ERROR: + case BZ_OUTBUFF_FULL: + Stream_->Write(TmpBuf(), TmpBufLen() - BzStream_.avail_out); + BzStream_.next_out = TmpBuf(); + BzStream_.avail_out = TmpBufLen(); + + break; + + default: + ythrow TBZipCompressError() << "bzip error(" << ret << ", " << BzStream_.avail_out << ")"; + } + } + } + + inline void Flush() { + /* + * TODO ? + */ + } + + inline void Finish() { int ret = BZ2_bzCompress(&BzStream_, BZ_FINISH); - - while (ret != BZ_STREAM_END) { - Stream_->Write(TmpBuf(), TmpBufLen() - BzStream_.avail_out); - BzStream_.next_out = TmpBuf(); - BzStream_.avail_out = TmpBufLen(); - + + while (ret != BZ_STREAM_END) { + Stream_->Write(TmpBuf(), TmpBufLen() - BzStream_.avail_out); + BzStream_.next_out = TmpBuf(); + BzStream_.avail_out = TmpBufLen(); + ret = BZ2_bzCompress(&BzStream_, BZ_FINISH); - } - - Stream_->Write(TmpBuf(), TmpBufLen() - BzStream_.avail_out); - } - -private: + } + + Stream_->Write(TmpBuf(), TmpBufLen() - BzStream_.avail_out); + } + +private: inline char* TmpBuf() noexcept { - return (char*)AdditionalData(); - } - + return (char*)AdditionalData(); + } + inline size_t TmpBufLen() const noexcept { - return AdditionalDataLength(); - } - -private: + return AdditionalDataLength(); + } + +private: IOutputStream* Stream_; - bz_stream BzStream_; -}; - + bz_stream BzStream_; +}; + TBZipCompress::TBZipCompress(IOutputStream* out, size_t compressionLevel, size_t bufLen) - : Impl_(new (bufLen) TImpl(out, compressionLevel)) -{ -} - + : Impl_(new (bufLen) TImpl(out, compressionLevel)) +{ +} + TBZipCompress::~TBZipCompress() { - try { - Finish(); - } catch (...) { - } -} - -void TBZipCompress::DoWrite(const void* buf, size_t size) { + try { + Finish(); + } catch (...) { + } +} + +void TBZipCompress::DoWrite(const void* buf, size_t size) { if (!Impl_) { - ythrow TBZipCompressError() << "can not write to finished bzip stream"; + ythrow TBZipCompressError() << "can not write to finished bzip stream"; } - Impl_->Write(buf, size); -} - -void TBZipCompress::DoFlush() { + Impl_->Write(buf, size); +} + +void TBZipCompress::DoFlush() { if (Impl_) { Impl_->Flush(); } -} - -void TBZipCompress::DoFinish() { +} + +void TBZipCompress::DoFinish() { THolder<TImpl> impl(Impl_.Release()); if (impl) { impl->Finish(); } -} +} diff --git a/library/cpp/streams/bzip2/bzip2.h b/library/cpp/streams/bzip2/bzip2.h index 2322277ef6..e007ab9b6c 100644 --- a/library/cpp/streams/bzip2/bzip2.h +++ b/library/cpp/streams/bzip2/bzip2.h @@ -1,53 +1,53 @@ #pragma once - -#include <util/stream/input.h> -#include <util/stream/output.h> + +#include <util/stream/input.h> +#include <util/stream/output.h> #include <util/generic/ptr.h> #include <util/generic/yexception.h> - -#define BZIP_BUF_LEN (8 * 1024) -#define BZIP_COMPRESSION_LEVEL 6 - + +#define BZIP_BUF_LEN (8 * 1024) +#define BZIP_COMPRESSION_LEVEL 6 + /** * @addtogroup Streams_Archs * @{ */ -class TBZipException: public yexception { -}; - -class TBZipDecompressError: public TBZipException { -}; - -class TBZipCompressError: public TBZipException { -}; - +class TBZipException: public yexception { +}; + +class TBZipDecompressError: public TBZipException { +}; + +class TBZipCompressError: public TBZipException { +}; + class TBZipDecompress: public IInputStream { -public: +public: TBZipDecompress(IInputStream* input, size_t bufLen = BZIP_BUF_LEN); ~TBZipDecompress() override; - -private: + +private: size_t DoRead(void* buf, size_t size) override; - -private: - class TImpl; - THolder<TImpl> Impl_; -}; - + +private: + class TImpl; + THolder<TImpl> Impl_; +}; + class TBZipCompress: public IOutputStream { -public: +public: TBZipCompress(IOutputStream* out, size_t compressionLevel = BZIP_COMPRESSION_LEVEL, size_t bufLen = BZIP_BUF_LEN); ~TBZipCompress() override; - -private: + +private: void DoWrite(const void* buf, size_t size) override; void DoFlush() override; void DoFinish() override; - -public: - class TImpl; - THolder<TImpl> Impl_; -}; - + +public: + class TImpl; + THolder<TImpl> Impl_; +}; + /** @} */ diff --git a/library/cpp/streams/bzip2/bzip2_ut.cpp b/library/cpp/streams/bzip2/bzip2_ut.cpp index 69a98f296c..0038735599 100644 --- a/library/cpp/streams/bzip2/bzip2_ut.cpp +++ b/library/cpp/streams/bzip2/bzip2_ut.cpp @@ -1,41 +1,41 @@ -#include "bzip2.h" - +#include "bzip2.h" + #include <library/cpp/testing/unittest/registar.h> - + #include <util/stream/file.h> -#include <util/system/tempfile.h> - -#define ZDATA "./zdata" - +#include <util/system/tempfile.h> + +#define ZDATA "./zdata" + Y_UNIT_TEST_SUITE(TBZipTest) { static const TString data = "8s7d5vc6s5vc67sa4c65ascx6asd4xcv76adsfxv76s"; - + Y_UNIT_TEST(TestCompress) { TUnbufferedFileOutput o(ZDATA); - TBZipCompress c(&o); - + TBZipCompress c(&o); + c.Write(data.data(), data.size()); - c.Finish(); - o.Finish(); - } - + c.Finish(); + o.Finish(); + } + Y_UNIT_TEST(TestDecompress) { - TTempFile tmp(ZDATA); - - { + TTempFile tmp(ZDATA); + + { TUnbufferedFileInput i(ZDATA); - TBZipDecompress d(&i); - - UNIT_ASSERT_EQUAL(d.ReadLine(), data); - } - } - + TBZipDecompress d(&i); + + UNIT_ASSERT_EQUAL(d.ReadLine(), data); + } + } + Y_UNIT_TEST(TestCorrupted) { - TMemoryInput i("blablabla", 10); - TBZipDecompress d(&i); - - UNIT_ASSERT_EXCEPTION(d.ReadLine(), TBZipDecompressError); - } -} - -#undef ZDATA + TMemoryInput i("blablabla", 10); + TBZipDecompress d(&i); + + UNIT_ASSERT_EXCEPTION(d.ReadLine(), TBZipDecompressError); + } +} + +#undef ZDATA diff --git a/library/cpp/streams/bzip2/ut/ya.make b/library/cpp/streams/bzip2/ut/ya.make index 5ef91498ca..4b2b7d1f60 100644 --- a/library/cpp/streams/bzip2/ut/ya.make +++ b/library/cpp/streams/bzip2/ut/ya.make @@ -1,12 +1,12 @@ UNITTEST_FOR(library/cpp/streams/bzip2) - + OWNER( pg g:util ) - -SRCS( - bzip2_ut.cpp -) - -END() + +SRCS( + bzip2_ut.cpp +) + +END() diff --git a/library/cpp/streams/bzip2/ya.make b/library/cpp/streams/bzip2/ya.make index 122a35837c..9c860f269e 100644 --- a/library/cpp/streams/bzip2/ya.make +++ b/library/cpp/streams/bzip2/ya.make @@ -1,16 +1,16 @@ -LIBRARY() - +LIBRARY() + OWNER( pg g:util ) - -PEERDIR( - contrib/libs/libbz2 -) - -SRCS( - bzip2.cpp -) - -END() + +PEERDIR( + contrib/libs/libbz2 +) + +SRCS( + bzip2.cpp +) + +END() diff --git a/library/cpp/streams/lz/lz.cpp b/library/cpp/streams/lz/lz.cpp index b65bb3ed96..0b0849866a 100644 --- a/library/cpp/streams/lz/lz.cpp +++ b/library/cpp/streams/lz/lz.cpp @@ -1,169 +1,169 @@ -#include "lz.h" - +#include "lz.h" + #include <util/system/yassert.h> -#include <util/system/byteorder.h> -#include <util/memory/addstorage.h> +#include <util/system/byteorder.h> +#include <util/memory/addstorage.h> #include <util/generic/buffer.h> #include <util/generic/utility.h> -#include <util/generic/singleton.h> +#include <util/generic/singleton.h> #include <util/generic/yexception.h> -#include <util/stream/mem.h> - -#include <contrib/libs/lz4/lz4.h> -#include <contrib/libs/fastlz/fastlz.h> -#include <contrib/libs/snappy/snappy.h> -#include <contrib/libs/quicklz/quicklz.h> -#include <contrib/libs/minilzo/minilzo.h> - +#include <util/stream/mem.h> + +#include <contrib/libs/lz4/lz4.h> +#include <contrib/libs/fastlz/fastlz.h> +#include <contrib/libs/snappy/snappy.h> +#include <contrib/libs/quicklz/quicklz.h> +#include <contrib/libs/minilzo/minilzo.h> + static inline ui8 HostToLittle(ui8 t) noexcept { - return t; -} - + return t; +} + static inline ui8 LittleToHost(ui8 t) noexcept { - return t; -} - -struct TCommonData { - static const size_t overhead = sizeof(ui16) + sizeof(ui8); -}; - + return t; +} + +struct TCommonData { + static const size_t overhead = sizeof(ui16) + sizeof(ui8); +}; + const size_t SIGNATURE_SIZE = 4; -template <class TCompressor, class TBase> -class TCompressorBase: public TAdditionalStorage<TCompressorBase<TCompressor, TBase>>, public TCompressor, public TCommonData { -public: +template <class TCompressor, class TBase> +class TCompressorBase: public TAdditionalStorage<TCompressorBase<TCompressor, TBase>>, public TCompressor, public TCommonData { +public: inline TCompressorBase(IOutputStream* slave, ui16 blockSize) - : Slave_(slave) - , BlockSize_(blockSize) - { - /* + : Slave_(slave) + , BlockSize_(blockSize) + { + /* * save signature */ - static_assert(sizeof(TCompressor::signature) - 1 == SIGNATURE_SIZE, "expect sizeof(TCompressor::signature) - 1 == SIGNATURE_SIZE"); - Slave_->Write(TCompressor::signature, sizeof(TCompressor::signature) - 1); + static_assert(sizeof(TCompressor::signature) - 1 == SIGNATURE_SIZE, "expect sizeof(TCompressor::signature) - 1 == SIGNATURE_SIZE"); + Slave_->Write(TCompressor::signature, sizeof(TCompressor::signature) - 1); - /* + /* * save version */ - this->Save((ui32)1); + this->Save((ui32)1); - /* + /* * save block size */ - this->Save(BlockSize()); - } - + this->Save(BlockSize()); + } + inline ~TCompressorBase() { - } - - inline void Write(const char* buf, size_t len) { - while (len) { - const ui16 toWrite = (ui16)Min<size_t>(len, this->BlockSize()); - - this->WriteBlock(buf, toWrite); - - buf += toWrite; - len -= toWrite; - } - } - - inline void Flush() { - } - - inline void Finish() { - this->Flush(); + } + + inline void Write(const char* buf, size_t len) { + while (len) { + const ui16 toWrite = (ui16)Min<size_t>(len, this->BlockSize()); + + this->WriteBlock(buf, toWrite); + + buf += toWrite; + len -= toWrite; + } + } + + inline void Flush() { + } + + inline void Finish() { + this->Flush(); this->WriteBlock(nullptr, 0); - } - - template <class T> + } + + template <class T> static inline void Save(T t, IOutputStream* out) { - t = HostToLittle(t); - - out->Write(&t, sizeof(t)); - } - - template <class T> - inline void Save(T t) { - Save(t, Slave_); - } - -private: + t = HostToLittle(t); + + out->Write(&t, sizeof(t)); + } + + template <class T> + inline void Save(T t) { + Save(t, Slave_); + } + +private: inline void* Block() const noexcept { - return this->AdditionalData(); - } - + return this->AdditionalData(); + } + inline ui16 BlockSize() const noexcept { - return BlockSize_; - } - - inline void WriteBlock(const void* ptr, ui16 len) { + return BlockSize_; + } + + inline void WriteBlock(const void* ptr, ui16 len) { Y_ASSERT(len <= this->BlockSize()); - - ui8 compressed = false; - - if (len) { + + ui8 compressed = false; + + if (len) { const size_t out = this->Compress((const char*)ptr, len, (char*)Block(), this->AdditionalDataLength()); // catch compressor buffer overrun (e.g. SEARCH-2043) //Y_VERIFY(out <= this->Hint(this->BlockSize())); - - if (out < len || TCompressor::SaveIncompressibleChunks()) { - compressed = true; - ptr = Block(); - len = (ui16)out; - } - } - - char tmp[overhead]; - TMemoryOutput header(tmp, sizeof(tmp)); - - this->Save(len, &header); - this->Save(compressed, &header); - + + if (out < len || TCompressor::SaveIncompressibleChunks()) { + compressed = true; + ptr = Block(); + len = (ui16)out; + } + } + + char tmp[overhead]; + TMemoryOutput header(tmp, sizeof(tmp)); + + this->Save(len, &header); + this->Save(compressed, &header); + using TPart = IOutputStream::TPart; if (ptr) { const TPart parts[] = { TPart(tmp, sizeof(tmp)), TPart(ptr, len), }; - + Slave_->Write(parts, sizeof(parts) / sizeof(*parts)); } else { Slave_->Write(tmp, sizeof(tmp)); } - } - -private: + } + +private: IOutputStream* Slave_; - const ui16 BlockSize_; -}; - -template <class T> + const ui16 BlockSize_; +}; + +template <class T> static inline T GLoad(IInputStream* input) { - T t; - - if (input->Load(&t, sizeof(t)) != sizeof(t)) { - ythrow TDecompressorError() << "stream error"; - } - - return LittleToHost(t); -} - + T t; + + if (input->Load(&t, sizeof(t)) != sizeof(t)) { + ythrow TDecompressorError() << "stream error"; + } + + return LittleToHost(t); +} + class TDecompressSignature { -public: +public: inline TDecompressSignature(IInputStream* input) { - if (input->Load(Buffer_, SIGNATURE_SIZE) != SIGNATURE_SIZE) { - ythrow TDecompressorError() << "can not load stream signature"; + if (input->Load(Buffer_, SIGNATURE_SIZE) != SIGNATURE_SIZE) { + ythrow TDecompressorError() << "can not load stream signature"; } - } + } - template <class TDecompressor> - inline bool Check() const { - static_assert(sizeof(TDecompressor::signature) - 1 == SIGNATURE_SIZE, "expect sizeof(TDecompressor::signature) - 1 == SIGNATURE_SIZE"); - return memcmp(TDecompressor::signature, Buffer_, SIGNATURE_SIZE) == 0; - } + template <class TDecompressor> + inline bool Check() const { + static_assert(sizeof(TDecompressor::signature) - 1 == SIGNATURE_SIZE, "expect sizeof(TDecompressor::signature) - 1 == SIGNATURE_SIZE"); + return memcmp(TDecompressor::signature, Buffer_, SIGNATURE_SIZE) == 0; + } -private: - char Buffer_[SIGNATURE_SIZE]; +private: + char Buffer_[SIGNATURE_SIZE]; }; template <class TDecompressor> @@ -177,316 +177,316 @@ static inline IInputStream* ConsumeSignature(IInputStream* input) { template <class TDecompressor> class TDecompressorBaseImpl: public TDecompressor, public TCommonData { -public: - static inline ui32 CheckVer(ui32 v) { - if (v != 1) { +public: + static inline ui32 CheckVer(ui32 v) { + if (v != 1) { ythrow yexception() << TStringBuf("incorrect stream version: ") << v; - } - - return v; - } - + } + + return v; + } + inline TDecompressorBaseImpl(IInputStream* slave) - : Slave_(slave) + : Slave_(slave) , Input_(nullptr, 0) - , Eof_(false) - , Version_(CheckVer(Load<ui32>())) - , BlockSize_(Load<ui16>()) - , OutBufSize_(TDecompressor::Hint(BlockSize_)) + , Eof_(false) + , Version_(CheckVer(Load<ui32>())) + , BlockSize_(Load<ui16>()) + , OutBufSize_(TDecompressor::Hint(BlockSize_)) , Tmp_(2 * OutBufSize_) , In_(Tmp_.Data()) - , Out_(In_ + OutBufSize_) - { - this->InitFromStream(Slave_); - } - + , Out_(In_ + OutBufSize_) + { + this->InitFromStream(Slave_); + } + inline ~TDecompressorBaseImpl() { - } - - inline size_t Read(void* buf, size_t len) { - size_t ret = Input_.Read(buf, len); - - if (ret) { - return ret; - } - - if (Eof_) { - return 0; - } - - this->FillNextBlock(); - - ret = Input_.Read(buf, len); - - if (ret) { - return ret; - } - - Eof_ = true; - - return 0; - } - - inline void FillNextBlock() { - char tmp[overhead]; - - if (Slave_->Load(tmp, sizeof(tmp)) != sizeof(tmp)) { - ythrow TDecompressorError() << "can not read block header"; - } - - TMemoryInput header(tmp, sizeof(tmp)); - - const ui16 len = GLoad<ui16>(&header); + } + + inline size_t Read(void* buf, size_t len) { + size_t ret = Input_.Read(buf, len); + + if (ret) { + return ret; + } + + if (Eof_) { + return 0; + } + + this->FillNextBlock(); + + ret = Input_.Read(buf, len); + + if (ret) { + return ret; + } + + Eof_ = true; + + return 0; + } + + inline void FillNextBlock() { + char tmp[overhead]; + + if (Slave_->Load(tmp, sizeof(tmp)) != sizeof(tmp)) { + ythrow TDecompressorError() << "can not read block header"; + } + + TMemoryInput header(tmp, sizeof(tmp)); + + const ui16 len = GLoad<ui16>(&header); if (len > Tmp_.Capacity()) { ythrow TDecompressorError() << "invalid len inside block header"; } - const ui8 compressed = GLoad<ui8>(&header); - - if (compressed > 1) { - ythrow TDecompressorError() << "broken header"; - } - - if (Slave_->Load(In_, len) != len) { - ythrow TDecompressorError() << "can not read data"; - } - - if (compressed) { - const size_t ret = this->Decompress(In_, len, Out_, OutBufSize_); - - Input_.Reset(Out_, ret); - } else { - Input_.Reset(In_, len); - } - } - - template <class T> - inline T Load() { - return GLoad<T>(Slave_); - } - -protected: + const ui8 compressed = GLoad<ui8>(&header); + + if (compressed > 1) { + ythrow TDecompressorError() << "broken header"; + } + + if (Slave_->Load(In_, len) != len) { + ythrow TDecompressorError() << "can not read data"; + } + + if (compressed) { + const size_t ret = this->Decompress(In_, len, Out_, OutBufSize_); + + Input_.Reset(Out_, ret); + } else { + Input_.Reset(In_, len); + } + } + + template <class T> + inline T Load() { + return GLoad<T>(Slave_); + } + +protected: IInputStream* Slave_; - TMemoryInput Input_; - bool Eof_; - const ui32 Version_; - const ui16 BlockSize_; - const size_t OutBufSize_; + TMemoryInput Input_; + bool Eof_; + const ui32 Version_; + const ui16 BlockSize_; + const size_t OutBufSize_; TBuffer Tmp_; - char* In_; - char* Out_; -}; - + char* In_; + char* Out_; +}; + template <class TDecompressor, class TBase> class TDecompressorBase: public TDecompressorBaseImpl<TDecompressor> { -public: +public: inline TDecompressorBase(IInputStream* slave) - : TDecompressorBaseImpl<TDecompressor>(ConsumeSignature<TDecompressor>(slave)) - { - } + : TDecompressorBaseImpl<TDecompressor>(ConsumeSignature<TDecompressor>(slave)) + { + } inline ~TDecompressorBase() { - } + } }; -#define DEF_COMPRESSOR_COMMON(rname, name) \ - rname::~rname() { \ - try { \ - Finish(); \ - } catch (...) { \ - } \ - } \ - \ - void rname::DoWrite(const void* buf, size_t len) { \ - if (!Impl_) { \ - ythrow yexception() << "can not write to finalized stream"; \ - } \ - \ - Impl_->Write((const char*)buf, len); \ - } \ - \ - void rname::DoFlush() { \ - if (!Impl_) { \ - ythrow yexception() << "can not flush finalized stream"; \ - } \ - \ - Impl_->Flush(); \ - } \ - \ - void rname::DoFinish() { \ - THolder<TImpl> impl(Impl_.Release()); \ - \ - if (impl) { \ - impl->Finish(); \ - } \ - } - -#define DEF_COMPRESSOR(rname, name) \ - class rname::TImpl: public TCompressorBase<name, TImpl> { \ - public: \ +#define DEF_COMPRESSOR_COMMON(rname, name) \ + rname::~rname() { \ + try { \ + Finish(); \ + } catch (...) { \ + } \ + } \ + \ + void rname::DoWrite(const void* buf, size_t len) { \ + if (!Impl_) { \ + ythrow yexception() << "can not write to finalized stream"; \ + } \ + \ + Impl_->Write((const char*)buf, len); \ + } \ + \ + void rname::DoFlush() { \ + if (!Impl_) { \ + ythrow yexception() << "can not flush finalized stream"; \ + } \ + \ + Impl_->Flush(); \ + } \ + \ + void rname::DoFinish() { \ + THolder<TImpl> impl(Impl_.Release()); \ + \ + if (impl) { \ + impl->Finish(); \ + } \ + } + +#define DEF_COMPRESSOR(rname, name) \ + class rname::TImpl: public TCompressorBase<name, TImpl> { \ + public: \ inline TImpl(IOutputStream* out, ui16 blockSize) \ - : TCompressorBase<name, TImpl>(out, blockSize) { \ - } \ - }; \ - \ + : TCompressorBase<name, TImpl>(out, blockSize) { \ + } \ + }; \ + \ rname::rname(IOutputStream* slave, ui16 blockSize) \ - : Impl_(new (TImpl::Hint(blockSize)) TImpl(slave, blockSize)) { \ - } \ - \ - DEF_COMPRESSOR_COMMON(rname, name) - -#define DEF_DECOMPRESSOR(rname, name) \ - class rname::TImpl: public TDecompressorBase<name, TImpl> { \ - public: \ + : Impl_(new (TImpl::Hint(blockSize)) TImpl(slave, blockSize)) { \ + } \ + \ + DEF_COMPRESSOR_COMMON(rname, name) + +#define DEF_DECOMPRESSOR(rname, name) \ + class rname::TImpl: public TDecompressorBase<name, TImpl> { \ + public: \ inline TImpl(IInputStream* in) \ - : TDecompressorBase<name, TImpl>(in) { \ - } \ - }; \ - \ + : TDecompressorBase<name, TImpl>(in) { \ + } \ + }; \ + \ rname::rname(IInputStream* slave) \ - : Impl_(new TImpl(slave)) { \ - } \ - \ - rname::~rname() { \ - } \ - \ - size_t rname::DoRead(void* buf, size_t len) { \ - return Impl_->Read(buf, len); \ - } - -/* - * MiniLzo - */ -class TMiniLzo { - class TInit { - public: - inline TInit() { - if (lzo_init() != LZO_E_OK) { - ythrow yexception() << "can not init lzo engine"; - } - } - }; - -public: - static const char signature[]; - - inline TMiniLzo() { - Singleton<TInit>(); - } - + : Impl_(new TImpl(slave)) { \ + } \ + \ + rname::~rname() { \ + } \ + \ + size_t rname::DoRead(void* buf, size_t len) { \ + return Impl_->Read(buf, len); \ + } + +/* + * MiniLzo + */ +class TMiniLzo { + class TInit { + public: + inline TInit() { + if (lzo_init() != LZO_E_OK) { + ythrow yexception() << "can not init lzo engine"; + } + } + }; + +public: + static const char signature[]; + + inline TMiniLzo() { + Singleton<TInit>(); + } + inline ~TMiniLzo() { - } - + } + static inline size_t Hint(size_t len) noexcept { // see SEARCH-2043 and, e.g. examples at // http://stackoverflow.com/questions/4235019/how-to-get-lzo-to-work-with-a-file-stream return len + (len / 16) + 64 + 3; - } - + } + static inline bool SaveIncompressibleChunks() noexcept { - return false; - } -}; - -const char TMiniLzo::signature[] = "YLZO"; - -template <size_t N> -class TFixedArray { -public: + return false; + } +}; + +const char TMiniLzo::signature[] = "YLZO"; + +template <size_t N> +class TFixedArray { +public: inline TFixedArray() noexcept { - memset(WorkMem_, 0, sizeof(WorkMem_)); - } - -protected: - char WorkMem_[N]; -}; - -class TMiniLzoCompressor: public TMiniLzo, public TFixedArray<LZO1X_MEM_COMPRESS + 1> { -public: + memset(WorkMem_, 0, sizeof(WorkMem_)); + } + +protected: + char WorkMem_[N]; +}; + +class TMiniLzoCompressor: public TMiniLzo, public TFixedArray<LZO1X_MEM_COMPRESS + 1> { +public: inline size_t Compress(const char* data, size_t len, char* ptr, size_t /*dstMaxSize*/) { - lzo_uint out = 0; - lzo1x_1_compress((const lzo_bytep)data, len, (lzo_bytep)ptr, &out, WorkMem_); - - return out; - } -}; - -class TMiniLzoDecompressor: public TMiniLzo, public TFixedArray<LZO1X_MEM_DECOMPRESS + 1> { -public: - inline size_t Decompress(const char* data, size_t len, char* ptr, size_t /*max*/) { - lzo_uint ret = 0; - - lzo1x_decompress((const lzo_bytep)data, len, (lzo_bytep)ptr, &ret, WorkMem_); - - return ret; - } - + lzo_uint out = 0; + lzo1x_1_compress((const lzo_bytep)data, len, (lzo_bytep)ptr, &out, WorkMem_); + + return out; + } +}; + +class TMiniLzoDecompressor: public TMiniLzo, public TFixedArray<LZO1X_MEM_DECOMPRESS + 1> { +public: + inline size_t Decompress(const char* data, size_t len, char* ptr, size_t /*max*/) { + lzo_uint ret = 0; + + lzo1x_decompress((const lzo_bytep)data, len, (lzo_bytep)ptr, &ret, WorkMem_); + + return ret; + } + inline void InitFromStream(IInputStream*) const noexcept { - } -}; - -DEF_COMPRESSOR(TLzoCompress, TMiniLzoCompressor) -DEF_DECOMPRESSOR(TLzoDecompress, TMiniLzoDecompressor) - -/* - * FastLZ - */ -class TFastLZ { -public: - static const char signature[]; - + } +}; + +DEF_COMPRESSOR(TLzoCompress, TMiniLzoCompressor) +DEF_DECOMPRESSOR(TLzoDecompress, TMiniLzoDecompressor) + +/* + * FastLZ + */ +class TFastLZ { +public: + static const char signature[]; + static inline size_t Hint(size_t len) noexcept { - return Max<size_t>((size_t)(len * 1.06), 100); - } - + return Max<size_t>((size_t)(len * 1.06), 100); + } + inline size_t Compress(const char* data, size_t len, char* ptr, size_t /*dstMaxSize*/) { - return fastlz_compress(data, len, ptr); - } - - inline size_t Decompress(const char* data, size_t len, char* ptr, size_t max) { - return fastlz_decompress(data, len, ptr, max); - } - + return fastlz_compress(data, len, ptr); + } + + inline size_t Decompress(const char* data, size_t len, char* ptr, size_t max) { + return fastlz_decompress(data, len, ptr, max); + } + inline void InitFromStream(IInputStream*) const noexcept { - } - + } + static inline bool SaveIncompressibleChunks() noexcept { - return false; - } -}; - -const char TFastLZ::signature[] = "YLZF"; - -DEF_COMPRESSOR(TLzfCompress, TFastLZ) -DEF_DECOMPRESSOR(TLzfDecompress, TFastLZ) - -/* + return false; + } +}; + +const char TFastLZ::signature[] = "YLZF"; + +DEF_COMPRESSOR(TLzfCompress, TFastLZ) +DEF_DECOMPRESSOR(TLzfDecompress, TFastLZ) + +/* * LZ4 */ class TLZ4 { -public: - static const char signature[]; +public: + static const char signature[]; static inline size_t Hint(size_t len) noexcept { - return Max<size_t>((size_t)(len * 1.06), 100); - } + return Max<size_t>((size_t)(len * 1.06), 100); + } inline size_t Compress(const char* data, size_t len, char* ptr, size_t dstMaxSize) { return LZ4_compress_default(data, ptr, len, dstMaxSize); - } + } - inline size_t Decompress(const char* data, size_t len, char* ptr, size_t max) { + inline size_t Decompress(const char* data, size_t len, char* ptr, size_t max) { int res = LZ4_decompress_safe(data, ptr, len, max); - if (res < 0) - ythrow TDecompressorError(); - return res; - } + if (res < 0) + ythrow TDecompressorError(); + return res; + } inline void InitFromStream(IInputStream*) const noexcept { - } + } static inline bool SaveIncompressibleChunks() noexcept { - return false; - } + return false; + } }; const char TLZ4::signature[] = "LZ.4"; @@ -498,32 +498,32 @@ DEF_DECOMPRESSOR(TLz4Decompress, TLZ4) * Snappy */ class TSnappy { -public: - static const char signature[]; +public: + static const char signature[]; static inline size_t Hint(size_t len) noexcept { return Max<size_t>(snappy::MaxCompressedLength(len), 100); - } + } inline size_t Compress(const char* data, size_t len, char* ptr, size_t /*dstMaxSize*/) { - size_t reslen = 0; - snappy::RawCompress(data, len, ptr, &reslen); - return reslen; - } - - inline size_t Decompress(const char* data, size_t len, char* ptr, size_t) { - size_t srclen = 0; - if (!snappy::GetUncompressedLength(data, len, &srclen) || !snappy::RawUncompress(data, len, ptr)) - ythrow TDecompressorError(); - return srclen; - } + size_t reslen = 0; + snappy::RawCompress(data, len, ptr, &reslen); + return reslen; + } + + inline size_t Decompress(const char* data, size_t len, char* ptr, size_t) { + size_t srclen = 0; + if (!snappy::GetUncompressedLength(data, len, &srclen) || !snappy::RawUncompress(data, len, ptr)) + ythrow TDecompressorError(); + return srclen; + } inline void InitFromStream(IInputStream*) const noexcept { - } + } static inline bool SaveIncompressibleChunks() noexcept { - return false; - } + return false; + } }; const char TSnappy::signature[] = "Snap"; @@ -532,117 +532,117 @@ DEF_COMPRESSOR(TSnappyCompress, TSnappy) DEF_DECOMPRESSOR(TSnappyDecompress, TSnappy) /* - * QuickLZ - */ -class TQuickLZBase { -public: - static const char signature[]; - + * QuickLZ + */ +class TQuickLZBase { +public: + static const char signature[]; + static inline size_t Hint(size_t len) noexcept { - return len + 500; - } - - inline TQuickLZBase() + return len + 500; + } + + inline TQuickLZBase() : Table_(nullptr) - { - } - - inline void Init(unsigned ver, unsigned lev, unsigned mod, unsigned type) { - Table_ = LzqTable(ver, lev, mod); - - if (!Table_) { - ythrow yexception() << "unsupported lzq stream(" << ver << ", " << lev << ", " << mod << ")"; - } - - const size_t size = Table_->Setting(3) + Table_->Setting(type); - - Mem_.Reset(::operator new(size)); - memset(Mem_.Get(), 0, size); - } - + { + } + + inline void Init(unsigned ver, unsigned lev, unsigned mod, unsigned type) { + Table_ = LzqTable(ver, lev, mod); + + if (!Table_) { + ythrow yexception() << "unsupported lzq stream(" << ver << ", " << lev << ", " << mod << ")"; + } + + const size_t size = Table_->Setting(3) + Table_->Setting(type); + + Mem_.Reset(::operator new(size)); + memset(Mem_.Get(), 0, size); + } + inline bool SaveIncompressibleChunks() const noexcept { // we must save incompressible chunks "as is" // after compressor run in streaming mode - return Table_->Setting(3); - } - -protected: - const TQuickLZMethods* Table_; - THolder<void> Mem_; -}; - -const char TQuickLZBase::signature[] = "YLZQ"; - -class TQuickLZCompress: public TQuickLZBase { -public: + return Table_->Setting(3); + } + +protected: + const TQuickLZMethods* Table_; + THolder<void> Mem_; +}; + +const char TQuickLZBase::signature[] = "YLZQ"; + +class TQuickLZCompress: public TQuickLZBase { +public: inline size_t Compress(const char* data, size_t len, char* ptr, size_t /*dstMaxSize*/) { - return Table_->Compress(data, ptr, len, (char*)Mem_.Get()); - } -}; - -class TQuickLZDecompress: public TQuickLZBase { -public: - inline size_t Decompress(const char* data, size_t /*len*/, char* ptr, size_t /*max*/) { - return Table_->Decompress(data, ptr, (char*)Mem_.Get()); - } - + return Table_->Compress(data, ptr, len, (char*)Mem_.Get()); + } +}; + +class TQuickLZDecompress: public TQuickLZBase { +public: + inline size_t Decompress(const char* data, size_t /*len*/, char* ptr, size_t /*max*/) { + return Table_->Decompress(data, ptr, (char*)Mem_.Get()); + } + inline void InitFromStream(IInputStream* in) { - const ui8 ver = ::GLoad<ui8>(in); - const ui8 lev = ::GLoad<ui8>(in); - const ui8 mod = ::GLoad<ui8>(in); - - Init(ver, lev, mod, 2); - } -}; - -class TLzqCompress::TImpl: public TCompressorBase<TQuickLZCompress, TImpl> { -public: + const ui8 ver = ::GLoad<ui8>(in); + const ui8 lev = ::GLoad<ui8>(in); + const ui8 mod = ::GLoad<ui8>(in); + + Init(ver, lev, mod, 2); + } +}; + +class TLzqCompress::TImpl: public TCompressorBase<TQuickLZCompress, TImpl> { +public: inline TImpl(IOutputStream* out, ui16 blockSize, EVersion ver, unsigned level, EMode mode) - : TCompressorBase<TQuickLZCompress, TImpl>(out, blockSize) - { - memset(AdditionalData(), 0, AdditionalDataLength()); - - Init(ver, level, mode, 1); - - Save((ui8)ver); - Save((ui8)level); - Save((ui8)mode); - } -}; - + : TCompressorBase<TQuickLZCompress, TImpl>(out, blockSize) + { + memset(AdditionalData(), 0, AdditionalDataLength()); + + Init(ver, level, mode, 1); + + Save((ui8)ver); + Save((ui8)level); + Save((ui8)mode); + } +}; + TLzqCompress::TLzqCompress(IOutputStream* slave, ui16 blockSize, EVersion ver, unsigned level, EMode mode) - : Impl_(new (TImpl::Hint(blockSize)) TImpl(slave, blockSize, ver, level, mode)) -{ -} - -DEF_COMPRESSOR_COMMON(TLzqCompress, TQuickLZCompress) -DEF_DECOMPRESSOR(TLzqDecompress, TQuickLZDecompress) - -namespace { - template <class T> - struct TInputHolder { + : Impl_(new (TImpl::Hint(blockSize)) TImpl(slave, blockSize, ver, level, mode)) +{ +} + +DEF_COMPRESSOR_COMMON(TLzqCompress, TQuickLZCompress) +DEF_DECOMPRESSOR(TLzqDecompress, TQuickLZDecompress) + +namespace { + template <class T> + struct TInputHolder { static inline T Set(T t) noexcept { - return t; - } - }; - - template <class T> - struct TInputHolder<TAutoPtr<T>> { + return t; + } + }; + + template <class T> + struct TInputHolder<TAutoPtr<T>> { inline T* Set(TAutoPtr<T> v) noexcept { - V_ = v; - - return V_.Get(); - } - - TAutoPtr<T> V_; - }; - - // Decompressing input streams without signature verification - template <class TInput, class TDecompressor> + V_ = v; + + return V_.Get(); + } + + TAutoPtr<T> V_; + }; + + // Decompressing input streams without signature verification + template <class TInput, class TDecompressor> class TLzDecompressInput: public TInputHolder<TInput>, public IInputStream { public: - inline TLzDecompressInput(TInput in) - : Impl_(this->Set(in)) + inline TLzDecompressInput(TInput in) + : Impl_(this->Set(in)) { } @@ -653,79 +653,79 @@ namespace { private: TDecompressorBaseImpl<TDecompressor> Impl_; - }; -} + }; +} -template <class T> +template <class T> static TAutoPtr<IInputStream> TryOpenLzDecompressorX(const TDecompressSignature& s, T input) { if (s.Check<TLZ4>()) - return new TLzDecompressInput<T, TLZ4>(input); + return new TLzDecompressInput<T, TLZ4>(input); if (s.Check<TSnappy>()) - return new TLzDecompressInput<T, TSnappy>(input); + return new TLzDecompressInput<T, TSnappy>(input); if (s.Check<TMiniLzo>()) - return new TLzDecompressInput<T, TMiniLzoDecompressor>(input); + return new TLzDecompressInput<T, TMiniLzoDecompressor>(input); if (s.Check<TFastLZ>()) - return new TLzDecompressInput<T, TFastLZ>(input); + return new TLzDecompressInput<T, TFastLZ>(input); if (s.Check<TQuickLZDecompress>()) - return new TLzDecompressInput<T, TQuickLZDecompress>(input); + return new TLzDecompressInput<T, TQuickLZDecompress>(input); return nullptr; } -template <class T> +template <class T> static inline TAutoPtr<IInputStream> TryOpenLzDecompressorImpl(const TStringBuf& signature, T input) { if (signature.size() == SIGNATURE_SIZE) { TMemoryInput mem(signature.data(), signature.size()); TDecompressSignature s(&mem); - - return TryOpenLzDecompressorX(s, input); + + return TryOpenLzDecompressorX(s, input); } return nullptr; } -template <class T> +template <class T> static inline TAutoPtr<IInputStream> TryOpenLzDecompressorImpl(T input) { - TDecompressSignature s(&*input); + TDecompressSignature s(&*input); - return TryOpenLzDecompressorX(s, input); -} - -template <class T> + return TryOpenLzDecompressorX(s, input); +} + +template <class T> static inline TAutoPtr<IInputStream> OpenLzDecompressorImpl(T input) { TAutoPtr<IInputStream> ret = TryOpenLzDecompressorImpl(input); - - if (!ret) { + + if (!ret) { ythrow TDecompressorError() << "Unknown compression format"; - } + } return ret; } - + TAutoPtr<IInputStream> OpenLzDecompressor(IInputStream* input) { - return OpenLzDecompressorImpl(input); -} - + return OpenLzDecompressorImpl(input); +} + TAutoPtr<IInputStream> TryOpenLzDecompressor(IInputStream* input) { - return TryOpenLzDecompressorImpl(input); -} - + return TryOpenLzDecompressorImpl(input); +} + TAutoPtr<IInputStream> TryOpenLzDecompressor(const TStringBuf& signature, IInputStream* input) { - return TryOpenLzDecompressorImpl(signature, input); -} - + return TryOpenLzDecompressorImpl(signature, input); +} + TAutoPtr<IInputStream> OpenOwnedLzDecompressor(TAutoPtr<IInputStream> input) { - return OpenLzDecompressorImpl(input); -} - + return OpenLzDecompressorImpl(input); +} + TAutoPtr<IInputStream> TryOpenOwnedLzDecompressor(TAutoPtr<IInputStream> input) { - return TryOpenLzDecompressorImpl(input); -} - + return TryOpenLzDecompressorImpl(input); +} + TAutoPtr<IInputStream> TryOpenOwnedLzDecompressor(const TStringBuf& signature, TAutoPtr<IInputStream> input) { - return TryOpenLzDecompressorImpl(signature, input); -} + return TryOpenLzDecompressorImpl(signature, input); +} diff --git a/library/cpp/streams/lz/lz.h b/library/cpp/streams/lz/lz.h index 3a2eaad88b..95120fffed 100644 --- a/library/cpp/streams/lz/lz.h +++ b/library/cpp/streams/lz/lz.h @@ -1,7 +1,7 @@ #pragma once - -#include <util/stream/output.h> -#include <util/stream/input.h> + +#include <util/stream/output.h> +#include <util/stream/input.h> #include <util/generic/ptr.h> #include <util/generic/yexception.h> @@ -17,7 +17,7 @@ * for some comparisons. */ -struct TDecompressorError: public yexception { +struct TDecompressorError: public yexception { }; /** @@ -31,18 +31,18 @@ struct TDecompressorError: public yexception { * @see http://code.google.com/p/lz4/ */ class TLz4Compress: public IOutputStream { -public: +public: TLz4Compress(IOutputStream* slave, ui16 maxBlockSize = 1 << 15); ~TLz4Compress() override; -private: +private: void DoWrite(const void* buf, size_t len) override; void DoFlush() override; void DoFinish() override; -private: - class TImpl; - THolder<TImpl> Impl_; +private: + class TImpl; + THolder<TImpl> Impl_; }; /** @@ -51,16 +51,16 @@ private: * @see http://code.google.com/p/lz4/ */ class TLz4Decompress: public IInputStream { -public: +public: TLz4Decompress(IInputStream* slave); ~TLz4Decompress() override; -private: +private: size_t DoRead(void* buf, size_t len) override; -private: - class TImpl; - THolder<TImpl> Impl_; +private: + class TImpl; + THolder<TImpl> Impl_; }; /** @@ -69,18 +69,18 @@ private: * @see http://code.google.com/p/snappy/ */ class TSnappyCompress: public IOutputStream { -public: +public: TSnappyCompress(IOutputStream* slave, ui16 maxBlockSize = 1 << 15); ~TSnappyCompress() override; -private: +private: void DoWrite(const void* buf, size_t len) override; void DoFlush() override; void DoFinish() override; -private: - class TImpl; - THolder<TImpl> Impl_; +private: + class TImpl; + THolder<TImpl> Impl_; }; /** @@ -89,138 +89,138 @@ private: * @see http://code.google.com/p/snappy/ */ class TSnappyDecompress: public IInputStream { -public: +public: TSnappyDecompress(IInputStream* slave); ~TSnappyDecompress() override; -private: +private: size_t DoRead(void* buf, size_t len) override; -private: - class TImpl; - THolder<TImpl> Impl_; +private: + class TImpl; + THolder<TImpl> Impl_; }; /** * MiniLZO compressing stream. - */ + */ class TLzoCompress: public IOutputStream { -public: +public: TLzoCompress(IOutputStream* slave, ui16 maxBlockSize = 1 << 15); ~TLzoCompress() override; - -private: + +private: void DoWrite(const void* buf, size_t len) override; void DoFlush() override; void DoFinish() override; - -private: - class TImpl; - THolder<TImpl> Impl_; -}; - + +private: + class TImpl; + THolder<TImpl> Impl_; +}; + /** * MiniLZO decompressing stream. */ class TLzoDecompress: public IInputStream { -public: +public: TLzoDecompress(IInputStream* slave); ~TLzoDecompress() override; - -private: + +private: size_t DoRead(void* buf, size_t len) override; - -private: - class TImpl; - THolder<TImpl> Impl_; -}; - + +private: + class TImpl; + THolder<TImpl> Impl_; +}; + /** * FastLZ compressing stream. - */ + */ class TLzfCompress: public IOutputStream { -public: +public: TLzfCompress(IOutputStream* slave, ui16 maxBlockSize = 1 << 15); ~TLzfCompress() override; - -private: + +private: void DoWrite(const void* buf, size_t len) override; void DoFlush() override; void DoFinish() override; - -private: - class TImpl; - THolder<TImpl> Impl_; -}; - + +private: + class TImpl; + THolder<TImpl> Impl_; +}; + /** * FastLZ decompressing stream. */ class TLzfDecompress: public IInputStream { -public: +public: TLzfDecompress(IInputStream* slave); ~TLzfDecompress() override; - -private: + +private: size_t DoRead(void* buf, size_t len) override; - -private: - class TImpl; - THolder<TImpl> Impl_; -}; - + +private: + class TImpl; + THolder<TImpl> Impl_; +}; + /** * QuickLZ compressing stream. - */ + */ class TLzqCompress: public IOutputStream { -public: - enum EVersion { - V_1_31 = 0, - V_1_40 = 1, - V_1_51 = 2 - }; - - /* +public: + enum EVersion { + V_1_31 = 0, + V_1_40 = 1, + V_1_51 = 2 + }; + + /* * streaming mode - actually, backlog size */ - enum EMode { - M_0 = 0, - M_100000 = 1, - M_1000000 = 2 - }; - + enum EMode { + M_0 = 0, + M_100000 = 1, + M_1000000 = 2 + }; + TLzqCompress(IOutputStream* slave, ui16 maxBlockSize = 1 << 15, - EVersion ver = V_1_31, - unsigned level = 0, - EMode mode = M_0); + EVersion ver = V_1_31, + unsigned level = 0, + EMode mode = M_0); ~TLzqCompress() override; - -private: + +private: void DoWrite(const void* buf, size_t len) override; void DoFlush() override; void DoFinish() override; - -private: - class TImpl; - THolder<TImpl> Impl_; -}; - + +private: + class TImpl; + THolder<TImpl> Impl_; +}; + /** * QuickLZ decompressing stream. */ class TLzqDecompress: public IInputStream { -public: +public: TLzqDecompress(IInputStream* slave); ~TLzqDecompress() override; - -private: + +private: size_t DoRead(void* buf, size_t len) override; - -private: - class TImpl; - THolder<TImpl> Impl_; -}; - + +private: + class TImpl; + THolder<TImpl> Impl_; +}; + /** @} */ /** @@ -236,7 +236,7 @@ private: TAutoPtr<IInputStream> OpenLzDecompressor(IInputStream* input); TAutoPtr<IInputStream> TryOpenLzDecompressor(IInputStream* input); TAutoPtr<IInputStream> TryOpenLzDecompressor(const TStringBuf& signature, IInputStream* input); - + TAutoPtr<IInputStream> OpenOwnedLzDecompressor(TAutoPtr<IInputStream> input); TAutoPtr<IInputStream> TryOpenOwnedLzDecompressor(TAutoPtr<IInputStream> input); TAutoPtr<IInputStream> TryOpenOwnedLzDecompressor(const TStringBuf& signature, TAutoPtr<IInputStream> input); diff --git a/library/cpp/streams/lz/lz_ut.cpp b/library/cpp/streams/lz/lz_ut.cpp index 6876f070fc..54ce318dd4 100644 --- a/library/cpp/streams/lz/lz_ut.cpp +++ b/library/cpp/streams/lz/lz_ut.cpp @@ -1,19 +1,19 @@ -#include "lz.h" - +#include "lz.h" + #include <library/cpp/testing/unittest/registar.h> #include <library/cpp/resource/resource.h> - + #include <util/stream/file.h> #include <util/generic/vector.h> -#include <util/system/tempfile.h> -#include <util/generic/singleton.h> - -#define LDATA "./ldata" +#include <util/system/tempfile.h> +#include <util/generic/singleton.h> + +#define LDATA "./ldata" #define LDATA_RANDOM "./ldata.random" - + static const TString data = "aa aaa aa aaa aa aaa bb bbb bb bbb bb bbb"; - -namespace { + +namespace { /** * Produces well-formed random crap **/ @@ -51,7 +51,7 @@ namespace { for (auto size : sizes) { result.push_back(RandomString(size)); } - result.push_back(NResource::Find("/request.data")); + result.push_back(NResource::Find("/request.data")); return result; } @@ -60,11 +60,11 @@ namespace { } struct TRandomData: public TVector<TString> { - inline TRandomData() { - InitRandomData().swap(*this); - } - }; -} + inline TRandomData() { + InitRandomData().swap(*this); + } + }; +} static const TVector<size_t> bufferSizes = { 127, @@ -73,32 +73,32 @@ static const TVector<size_t> bufferSizes = { }; namespace { - template <TLzqCompress::EVersion Ver, int Level, TLzqCompress::EMode Mode> - struct TLzqCompressX: public TLzqCompress { + template <TLzqCompress::EVersion Ver, int Level, TLzqCompress::EMode Mode> + struct TLzqCompressX: public TLzqCompress { inline TLzqCompressX(IOutputStream* out, size_t bufLen) - : TLzqCompress(out, bufLen, Ver, Level, Mode) - { - } - }; -} - -template <class C> + : TLzqCompress(out, bufLen, Ver, Level, Mode) + { + } + }; +} + +template <class C> static inline void TestGoodDataCompress() { TFixedBufferFileOutput o(LDATA); - C c(&o, 1024); - + C c(&o, 1024); + TString d = data; - - for (size_t i = 0; i < 10; ++i) { + + for (size_t i = 0; i < 10; ++i) { c.Write(d.data(), d.size()); - c << Endl; - d = d + d; - } - - c.Finish(); - o.Finish(); -} - + c << Endl; + d = d + d; + } + + c.Finish(); + o.Finish(); +} + template <class C> static inline void TestIncompressibleDataCompress(const TString& d, size_t bufferSize) { TString testFileName = TestFileName(d, bufferSize); @@ -113,30 +113,30 @@ template <class C> static inline void TestCompress() { TestGoodDataCompress<C>(); for (auto bufferSize : bufferSizes) { - for (auto rd : *Singleton<TRandomData>()) { + for (auto rd : *Singleton<TRandomData>()) { TestIncompressibleDataCompress<C>(rd, bufferSize); } } } -template <class D> +template <class D> static inline void TestGoodDataDecompress() { - TTempFile tmpFile(LDATA); - - { + TTempFile tmpFile(LDATA); + + { TFileInput i1(LDATA); D ld(&i1); - + TString d = data; - + for (size_t i2 = 0; i2 < 10; ++i2) { - UNIT_ASSERT_EQUAL(ld.ReadLine(), d); - - d = d + d; - } - } -} - + UNIT_ASSERT_EQUAL(ld.ReadLine(), d); + + d = d + d; + } + } +} + template <class D> static inline void TestIncompressibleDataDecompress(const TString& d, size_t bufferSize) { TString testFileName = TestFileName(d, bufferSize); @@ -154,25 +154,25 @@ template <class D> static inline void TestDecompress() { TestGoodDataDecompress<D>(); for (auto bufferSize : bufferSizes) { - for (auto rd : *Singleton<TRandomData>()) { + for (auto rd : *Singleton<TRandomData>()) { TestIncompressibleDataDecompress<D>(rd, bufferSize); } } } class TMixedDecompress: public IInputStream { -public: +public: TMixedDecompress(IInputStream* input) - : Slave_(OpenLzDecompressor(input).Release()) - { - } + : Slave_(OpenLzDecompressor(input).Release()) + { + } -private: +private: size_t DoRead(void* buf, size_t len) override { - return Slave_->Read(buf, len); - } + return Slave_->Read(buf, len); + } -private: +private: THolder<IInputStream> Slave_; }; @@ -190,50 +190,50 @@ static inline void TestDecompressError() { Y_UNIT_TEST_SUITE(TLzTest) { Y_UNIT_TEST(TestLzo) { - TestCompress<TLzoCompress>(); - TestDecompress<TLzoDecompress>(); - } - + TestCompress<TLzoCompress>(); + TestDecompress<TLzoDecompress>(); + } + Y_UNIT_TEST(TestLzf) { - TestCompress<TLzfCompress>(); - TestDecompress<TLzfDecompress>(); - } - + TestCompress<TLzfCompress>(); + TestDecompress<TLzfDecompress>(); + } + Y_UNIT_TEST(TestLzq) { - TestCompress<TLzqCompress>(); - TestDecompress<TLzqDecompress>(); - } + TestCompress<TLzqCompress>(); + TestDecompress<TLzqDecompress>(); + } Y_UNIT_TEST(TestLzq151_1) { - TestCompress<TLzqCompressX<TLzqCompress::V_1_51, 1, TLzqCompress::M_0>>(); - TestDecompress<TLzqDecompress>(); - } - + TestCompress<TLzqCompressX<TLzqCompress::V_1_51, 1, TLzqCompress::M_0>>(); + TestDecompress<TLzqDecompress>(); + } + Y_UNIT_TEST(TestLzq151_2) { - TestCompress<TLzqCompressX<TLzqCompress::V_1_51, 2, TLzqCompress::M_100000>>(); - TestDecompress<TLzqDecompress>(); - } - + TestCompress<TLzqCompressX<TLzqCompress::V_1_51, 2, TLzqCompress::M_100000>>(); + TestDecompress<TLzqDecompress>(); + } + Y_UNIT_TEST(TestLzq151_3) { - TestCompress<TLzqCompressX<TLzqCompress::V_1_51, 3, TLzqCompress::M_1000000>>(); - TestDecompress<TLzqDecompress>(); - } - + TestCompress<TLzqCompressX<TLzqCompress::V_1_51, 3, TLzqCompress::M_1000000>>(); + TestDecompress<TLzqDecompress>(); + } + Y_UNIT_TEST(TestLzq140_1) { - TestCompress<TLzqCompressX<TLzqCompress::V_1_40, 1, TLzqCompress::M_0>>(); - TestDecompress<TLzqDecompress>(); - } - + TestCompress<TLzqCompressX<TLzqCompress::V_1_40, 1, TLzqCompress::M_0>>(); + TestDecompress<TLzqDecompress>(); + } + Y_UNIT_TEST(TestLzq140_2) { - TestCompress<TLzqCompressX<TLzqCompress::V_1_40, 2, TLzqCompress::M_100000>>(); - TestDecompress<TLzqDecompress>(); - } - + TestCompress<TLzqCompressX<TLzqCompress::V_1_40, 2, TLzqCompress::M_100000>>(); + TestDecompress<TLzqDecompress>(); + } + Y_UNIT_TEST(TestLzq140_3) { - TestCompress<TLzqCompressX<TLzqCompress::V_1_40, 3, TLzqCompress::M_1000000>>(); - TestDecompress<TLzqDecompress>(); - } - + TestCompress<TLzqCompressX<TLzqCompress::V_1_40, 3, TLzqCompress::M_1000000>>(); + TestDecompress<TLzqDecompress>(); + } + Y_UNIT_TEST(TestLz4) { TestCompress<TLz4Compress>(); TestDecompress<TLz4Decompress>(); @@ -260,21 +260,21 @@ Y_UNIT_TEST_SUITE(TLzTest) { TestDecompressError<TSnappyDecompress, TBufferedOutput>(); TestDecompressError<TMixedDecompress, TBufferedOutput>(); } - + Y_UNIT_TEST(TestFactory) { - TStringStream ss; - - { - TLz4Compress c(&ss); - - c.Write("123456789", 9); - c.Finish(); - } - + TStringStream ss; + + { + TLz4Compress c(&ss); + + c.Write("123456789", 9); + c.Finish(); + } + TAutoPtr<IInputStream> is(OpenOwnedLzDecompressor(new TStringInput(ss.Str()))); - - UNIT_ASSERT_EQUAL(is->ReadAll(), "123456789"); - } + + UNIT_ASSERT_EQUAL(is->ReadAll(), "123456789"); + } Y_UNIT_TEST(TestYQ609) { auto data = NResource::Find("/yq_609.data"); @@ -284,4 +284,4 @@ Y_UNIT_TEST_SUITE(TLzTest) { TLz4Decompress d(&input); UNIT_ASSERT_EXCEPTION(d.ReadAll(), TDecompressorError); } -} +} diff --git a/library/cpp/streams/lz/ut/ya.make b/library/cpp/streams/lz/ut/ya.make index 18288c8ac9..74e5a169c8 100644 --- a/library/cpp/streams/lz/ut/ya.make +++ b/library/cpp/streams/lz/ut/ya.make @@ -1,18 +1,18 @@ UNITTEST_FOR(library/cpp/streams/lz) - + OWNER( pg g:util ) - -RESOURCE( - random.data /random.data - request.data /request.data + +RESOURCE( + random.data /random.data + request.data /request.data yq_609.data /yq_609.data -) - -SRCS( - lz_ut.cpp -) - -END() +) + +SRCS( + lz_ut.cpp +) + +END() diff --git a/library/cpp/streams/lz/ya.make b/library/cpp/streams/lz/ya.make index e5eea0b096..7f83071128 100644 --- a/library/cpp/streams/lz/ya.make +++ b/library/cpp/streams/lz/ya.make @@ -1,20 +1,20 @@ -LIBRARY() - +LIBRARY() + OWNER( pg g:util ) - -PEERDIR( - contrib/libs/fastlz - contrib/libs/lz4 - contrib/libs/minilzo - contrib/libs/quicklz - contrib/libs/snappy -) - -SRCS( - lz.cpp -) - -END() + +PEERDIR( + contrib/libs/fastlz + contrib/libs/lz4 + contrib/libs/minilzo + contrib/libs/quicklz + contrib/libs/snappy +) + +SRCS( + lz.cpp +) + +END() diff --git a/library/cpp/streams/lzma/lzma.cpp b/library/cpp/streams/lzma/lzma.cpp index f1942fa546..f265e0a284 100644 --- a/library/cpp/streams/lzma/lzma.cpp +++ b/library/cpp/streams/lzma/lzma.cpp @@ -1,520 +1,520 @@ -#include "lzma.h" - -#include <util/stream/mem.h> -#include <util/system/context.h> -#include <util/generic/cast.h> -#include <util/memory/addstorage.h> -#include <util/generic/ptr.h> -#include <util/generic/intrlist.h> -#include <util/generic/scope.h> - -extern "C" { -#include <contrib/libs/lzmasdk/LzmaEnc.h> -#include <contrib/libs/lzmasdk/LzmaDec.h> -} - -namespace { - class TMemoryGc { - class TAllocation: public TIntrusiveListItem<TAllocation>, public TAdditionalStorage<TAllocation> { - }; - - public: - inline void* Allocate(size_t len) { - if (len > 1024 * 1024 * 1024) { - return nullptr; - } - - TAllocation* ret = new (len) TAllocation; - - Allocs_.PushBack(ret); - - return ret->AdditionalData(); - } - +#include "lzma.h" + +#include <util/stream/mem.h> +#include <util/system/context.h> +#include <util/generic/cast.h> +#include <util/memory/addstorage.h> +#include <util/generic/ptr.h> +#include <util/generic/intrlist.h> +#include <util/generic/scope.h> + +extern "C" { +#include <contrib/libs/lzmasdk/LzmaEnc.h> +#include <contrib/libs/lzmasdk/LzmaDec.h> +} + +namespace { + class TMemoryGc { + class TAllocation: public TIntrusiveListItem<TAllocation>, public TAdditionalStorage<TAllocation> { + }; + + public: + inline void* Allocate(size_t len) { + if (len > 1024 * 1024 * 1024) { + return nullptr; + } + + TAllocation* ret = new (len) TAllocation; + + Allocs_.PushBack(ret); + + return ret->AdditionalData(); + } + inline void Deallocate(void* ptr) noexcept { - if (ptr) { - delete TAllocation::ObjectFromData(ptr); - } - } - - private: - TIntrusiveListWithAutoDelete<TAllocation, TDelete> Allocs_; - }; - - template <class T> - class TInverseFilter { - class TTrampoLine: public ITrampoLine { - public: - inline TTrampoLine(TInverseFilter* parent) - : Parent_(parent) - { - } - + if (ptr) { + delete TAllocation::ObjectFromData(ptr); + } + } + + private: + TIntrusiveListWithAutoDelete<TAllocation, TDelete> Allocs_; + }; + + template <class T> + class TInverseFilter { + class TTrampoLine: public ITrampoLine { + public: + inline TTrampoLine(TInverseFilter* parent) + : Parent_(parent) + { + } + void DoRun() override { - Parent_->RunFilter(); - } - - private: - TInverseFilter* Parent_; - }; - + Parent_->RunFilter(); + } + + private: + TInverseFilter* Parent_; + }; + class TInput: public IInputStream { - public: - inline TInput(TInverseFilter* parent) - : Parent_(parent) - { - } - + public: + inline TInput(TInverseFilter* parent) + : Parent_(parent) + { + } + ~TInput() override { - } - + } + size_t DoRead(void* ptr, size_t len) override { - return Parent_->ReadImpl(ptr, len); - } - - private: - TInverseFilter* Parent_; - }; - + return Parent_->ReadImpl(ptr, len); + } + + private: + TInverseFilter* Parent_; + }; + class TOutput: public IOutputStream { - public: - inline TOutput(TInverseFilter* parent) - : Parent_(parent) - { - } - + public: + inline TOutput(TInverseFilter* parent) + : Parent_(parent) + { + } + ~TOutput() override { - } - + } + void DoWrite(const void* ptr, size_t len) override { - Parent_->WriteImpl(ptr, len); - } - - private: - TInverseFilter* Parent_; - }; - - public: + Parent_->WriteImpl(ptr, len); + } + + private: + TInverseFilter* Parent_; + }; + + public: inline TInverseFilter(IOutputStream* slave, T* filter) - : Slave_(slave) - , Filter_(filter) - , TrampoLine_(this) - , FilterCtx_(FilterClosure()) - , Finished_(false) + : Slave_(slave) + , Filter_(filter) + , TrampoLine_(this) + , FilterCtx_(FilterClosure()) + , Finished_(false) , In_(nullptr, 0) - { - } - + { + } + virtual ~TInverseFilter() { - if (!UncaughtException()) { - try { - Finish(); - } catch (...) { - } - } else { - //rely on gc - } - } - - inline void Write(const void* ptr, size_t len) { - In_.Reset(ptr, len); - - Y_DEFER { - In_.Reset(0, 0); - }; - - while (In_.Avail()) { - SwitchTo(); - } - } - - inline void Finish() { - if (!Finished_) { - Finished_ = true; - SwitchTo(); - } - } - - private: - inline void RunFilter() { - try { - TInput in(this); - TOutput out(this); - - (*Filter_)(&in, &out); - } catch (...) { - Err_ = std::current_exception(); - } - - SwitchFrom(); - } - - inline TContClosure FilterClosure() { + if (!UncaughtException()) { + try { + Finish(); + } catch (...) { + } + } else { + //rely on gc + } + } + + inline void Write(const void* ptr, size_t len) { + In_.Reset(ptr, len); + + Y_DEFER { + In_.Reset(0, 0); + }; + + while (In_.Avail()) { + SwitchTo(); + } + } + + inline void Finish() { + if (!Finished_) { + Finished_ = true; + SwitchTo(); + } + } + + private: + inline void RunFilter() { + try { + TInput in(this); + TOutput out(this); + + (*Filter_)(&in, &out); + } catch (...) { + Err_ = std::current_exception(); + } + + SwitchFrom(); + } + + inline TContClosure FilterClosure() { return {&TrampoLine_, TArrayRef(Stack_, sizeof(Stack_))}; - } - - inline size_t ReadImpl(void* ptr, size_t len) { - while (!Finished_) { - const size_t ret = In_.Read(ptr, len); - - if (ret) { - return ret; - } - - SwitchFrom(); - } - - return 0; - } - - inline void WriteImpl(const void* ptr, size_t len) { + } + + inline size_t ReadImpl(void* ptr, size_t len) { + while (!Finished_) { + const size_t ret = In_.Read(ptr, len); + + if (ret) { + return ret; + } + + SwitchFrom(); + } + + return 0; + } + + inline void WriteImpl(const void* ptr, size_t len) { Y_ASSERT(!Out_.Avail()); - - Out_.Reset(ptr, len); - - while (Out_.Avail()) { - SwitchFrom(); - } - } - - inline bool FlushImpl() { - if (Out_.Avail()) { + + Out_.Reset(ptr, len); + + while (Out_.Avail()) { + SwitchFrom(); + } + } + + inline bool FlushImpl() { + if (Out_.Avail()) { TransferData(&Out_, Slave_); Out_.Reset(nullptr, 0); - - return true; - } - - return false; - } - - inline void SwitchTo() { - do { - CurrentCtx_.SwitchTo(&FilterCtx_); - - if (Err_) { - Finished_ = true; - - std::rethrow_exception(Err_); - } - } while (FlushImpl()); - } - - inline void SwitchFrom() { - FilterCtx_.SwitchTo(&CurrentCtx_); - } - - private: + + return true; + } + + return false; + } + + inline void SwitchTo() { + do { + CurrentCtx_.SwitchTo(&FilterCtx_); + + if (Err_) { + Finished_ = true; + + std::rethrow_exception(Err_); + } + } while (FlushImpl()); + } + + inline void SwitchFrom() { + FilterCtx_.SwitchTo(&CurrentCtx_); + } + + private: IOutputStream* Slave_; - T* Filter_; - TTrampoLine TrampoLine_; - char Stack_[16 * 1024]; - TContMachineContext FilterCtx_; - TContMachineContext CurrentCtx_; - bool Finished_; - TMemoryInput In_; - TMemoryInput Out_; - std::exception_ptr Err_; - }; - - class TLzma { - public: - class TLzmaInput: public ISeqInStream { - public: + T* Filter_; + TTrampoLine TrampoLine_; + char Stack_[16 * 1024]; + TContMachineContext FilterCtx_; + TContMachineContext CurrentCtx_; + bool Finished_; + TMemoryInput In_; + TMemoryInput Out_; + std::exception_ptr Err_; + }; + + class TLzma { + public: + class TLzmaInput: public ISeqInStream { + public: inline TLzmaInput(IInputStream* slave) - : Slave_(slave) - { - Read = ReadFunc; - } - - private: - static inline SRes ReadFunc(const ISeqInStream* p, void* ptr, size_t* len) { - *len = const_cast<TLzmaInput*>(static_cast<const TLzmaInput*>(p))->Slave_->Read(ptr, *len); - - return SZ_OK; - } - - private: + : Slave_(slave) + { + Read = ReadFunc; + } + + private: + static inline SRes ReadFunc(const ISeqInStream* p, void* ptr, size_t* len) { + *len = const_cast<TLzmaInput*>(static_cast<const TLzmaInput*>(p))->Slave_->Read(ptr, *len); + + return SZ_OK; + } + + private: IInputStream* Slave_; - }; - - class TLzmaOutput: public ISeqOutStream { - public: + }; + + class TLzmaOutput: public ISeqOutStream { + public: inline TLzmaOutput(IOutputStream* slave) - : Slave_(slave) - { - Write = WriteFunc; - } - - private: - static inline size_t WriteFunc(const ISeqOutStream* p, const void* ptr, size_t len) { - const_cast<TLzmaOutput*>(static_cast<const TLzmaOutput*>(p))->Slave_->Write(ptr, len); - - return len; - } - - private: + : Slave_(slave) + { + Write = WriteFunc; + } + + private: + static inline size_t WriteFunc(const ISeqOutStream* p, const void* ptr, size_t len) { + const_cast<TLzmaOutput*>(static_cast<const TLzmaOutput*>(p))->Slave_->Write(ptr, len); + + return len; + } + + private: IOutputStream* Slave_; - }; - - class TAlloc: public ISzAlloc { - public: - inline TAlloc() { - Alloc = AllocFunc; - Free = FreeFunc; - } - - private: - static void* AllocFunc(const ISzAlloc* t, size_t len) { - return static_cast<TAlloc*>(((ISzAlloc*)t))->Gc_.Allocate(len); - } - - static void FreeFunc(const ISzAlloc* t, void* p) { - static_cast<TAlloc*>(((ISzAlloc*)t))->Gc_.Deallocate(p); - } - - private: - TMemoryGc Gc_; - }; - + }; + + class TAlloc: public ISzAlloc { + public: + inline TAlloc() { + Alloc = AllocFunc; + Free = FreeFunc; + } + + private: + static void* AllocFunc(const ISzAlloc* t, size_t len) { + return static_cast<TAlloc*>(((ISzAlloc*)t))->Gc_.Allocate(len); + } + + static void FreeFunc(const ISzAlloc* t, void* p) { + static_cast<TAlloc*>(((ISzAlloc*)t))->Gc_.Deallocate(p); + } + + private: + TMemoryGc Gc_; + }; + inline ISzAlloc* Alloc() noexcept { - return &Alloc_; - } - - static inline void Check(SRes r) { - if (r != SZ_OK) { - ythrow yexception() << "lzma error(" << r << ")"; - } - } - - private: - TAlloc Alloc_; - }; - - class TLzmaCompressBase: public TLzma { - public: - inline TLzmaCompressBase(size_t level) - : H_(LzmaEnc_Create(Alloc())) - { - if (!H_) { - ythrow yexception() << "can not init lzma engine"; - } - - LzmaEncProps_Init(&Props_); - - Props_.level = level; - Props_.dictSize = 0; - Props_.lc = -1; - Props_.lp = -1; - Props_.pb = -1; - Props_.fb = -1; - Props_.numThreads = -1; - Props_.writeEndMark = 1; - - Check(LzmaEnc_SetProps(H_, &Props_)); - size_t bufLen = sizeof(PropsBuf_); - Zero(PropsBuf_); - Check(LzmaEnc_WriteProperties(H_, PropsBuf_, &bufLen)); - } - + return &Alloc_; + } + + static inline void Check(SRes r) { + if (r != SZ_OK) { + ythrow yexception() << "lzma error(" << r << ")"; + } + } + + private: + TAlloc Alloc_; + }; + + class TLzmaCompressBase: public TLzma { + public: + inline TLzmaCompressBase(size_t level) + : H_(LzmaEnc_Create(Alloc())) + { + if (!H_) { + ythrow yexception() << "can not init lzma engine"; + } + + LzmaEncProps_Init(&Props_); + + Props_.level = level; + Props_.dictSize = 0; + Props_.lc = -1; + Props_.lp = -1; + Props_.pb = -1; + Props_.fb = -1; + Props_.numThreads = -1; + Props_.writeEndMark = 1; + + Check(LzmaEnc_SetProps(H_, &Props_)); + size_t bufLen = sizeof(PropsBuf_); + Zero(PropsBuf_); + Check(LzmaEnc_WriteProperties(H_, PropsBuf_, &bufLen)); + } + inline ~TLzmaCompressBase() { - LzmaEnc_Destroy(H_, Alloc(), Alloc()); - } - + LzmaEnc_Destroy(H_, Alloc(), Alloc()); + } + inline void operator()(IInputStream* in, IOutputStream* out) { - TLzmaInput input(in); - TLzmaOutput output(out); - - out->Write(PropsBuf_, sizeof(PropsBuf_)); - + TLzmaInput input(in); + TLzmaOutput output(out); + + out->Write(PropsBuf_, sizeof(PropsBuf_)); + Check(LzmaEnc_Encode(H_, &output, &input, nullptr, Alloc(), Alloc())); - } - - private: - CLzmaEncHandle H_; - CLzmaEncProps Props_; - Byte PropsBuf_[LZMA_PROPS_SIZE]; - }; -} - -class TLzmaCompress::TImpl: public TLzmaCompressBase, public TInverseFilter<TLzmaCompressBase> { -public: + } + + private: + CLzmaEncHandle H_; + CLzmaEncProps Props_; + Byte PropsBuf_[LZMA_PROPS_SIZE]; + }; +} + +class TLzmaCompress::TImpl: public TLzmaCompressBase, public TInverseFilter<TLzmaCompressBase> { +public: inline TImpl(IOutputStream* slave, size_t level) - : TLzmaCompressBase(level) - , TInverseFilter<TLzmaCompressBase>(slave, this) - { - } -}; - -class TLzmaDecompress::TImpl: public TLzma { -public: - inline TImpl() - : InBegin_(nullptr) - , InEnd_(nullptr) - { - LzmaDec_Construct(&H_); - } + : TLzmaCompressBase(level) + , TInverseFilter<TLzmaCompressBase>(slave, this) + { + } +}; + +class TLzmaDecompress::TImpl: public TLzma { +public: + inline TImpl() + : InBegin_(nullptr) + , InEnd_(nullptr) + { + LzmaDec_Construct(&H_); + } inline virtual ~TImpl() { - LzmaDec_Free(&H_, Alloc()); - } - - inline size_t Read(void* ptr, size_t len) { - Byte* pos = (Byte*)ptr; - Byte* end = pos + len; - - retry: - size_t availLen = InEnd_ - InBegin_; - size_t bufLen = end - pos; - ELzmaStatus status; - - Check(LzmaDec_DecodeToBuf(&H_, pos, &bufLen, (Byte*)InBegin_, &availLen, LZMA_FINISH_ANY, &status)); - - InBegin_ += availLen; - pos += bufLen; - - if (status == LZMA_STATUS_NEEDS_MORE_INPUT) { + LzmaDec_Free(&H_, Alloc()); + } + + inline size_t Read(void* ptr, size_t len) { + Byte* pos = (Byte*)ptr; + Byte* end = pos + len; + + retry: + size_t availLen = InEnd_ - InBegin_; + size_t bufLen = end - pos; + ELzmaStatus status; + + Check(LzmaDec_DecodeToBuf(&H_, pos, &bufLen, (Byte*)InBegin_, &availLen, LZMA_FINISH_ANY, &status)); + + InBegin_ += availLen; + pos += bufLen; + + if (status == LZMA_STATUS_NEEDS_MORE_INPUT) { Y_ASSERT(InEnd_ == InBegin_); - if (!Fill()) { - ythrow yexception() << "incomplete lzma stream"; - } - - goto retry; - } - - return pos - (Byte*)ptr; - } - -private: - virtual bool Fill() = 0; - -protected: - CLzmaDec H_; - char* InBegin_; - char* InEnd_; + if (!Fill()) { + ythrow yexception() << "incomplete lzma stream"; + } + + goto retry; + } + + return pos - (Byte*)ptr; + } + +private: + virtual bool Fill() = 0; + +protected: + CLzmaDec H_; + char* InBegin_; + char* InEnd_; }; class TLzmaDecompress::TImplStream: public TImpl { -public: +public: inline TImplStream(IInputStream* slave) - : Slave_(slave) - { - Byte buf[LZMA_PROPS_SIZE]; - - if (Slave_->Load(buf, sizeof(buf)) != sizeof(buf)) - ythrow yexception() << "can't read lzma header"; - - Check(LzmaDec_Allocate(&H_, buf, sizeof(buf), Alloc())); - LzmaDec_Init(&H_); - } - -private: - bool Fill() override { - size_t size = Slave_->Read(In_, sizeof(In_)); - InBegin_ = In_; - InEnd_ = In_ + size; - - return size; - } - -private: + : Slave_(slave) + { + Byte buf[LZMA_PROPS_SIZE]; + + if (Slave_->Load(buf, sizeof(buf)) != sizeof(buf)) + ythrow yexception() << "can't read lzma header"; + + Check(LzmaDec_Allocate(&H_, buf, sizeof(buf), Alloc())); + LzmaDec_Init(&H_); + } + +private: + bool Fill() override { + size_t size = Slave_->Read(In_, sizeof(In_)); + InBegin_ = In_; + InEnd_ = In_ + size; + + return size; + } + +private: IInputStream* Slave_; - char In_[4096]; -}; - + char In_[4096]; +}; + class TLzmaDecompress::TImplZeroCopy: public TLzmaDecompress::TImpl { public: inline TImplZeroCopy(IZeroCopyInput* in) : Input_(in) - { - if (!Fill()) - ythrow yexception() << "can't read lzma header"; - - char buf[LZMA_PROPS_SIZE]; - char* header; - if (InEnd_ - InBegin_ >= LZMA_PROPS_SIZE) { - header = InBegin_; - InBegin_ += LZMA_PROPS_SIZE; - } else { - //bad luck, first part is less than header - //try to copy header part by part to the local buffer - const char* end = buf + sizeof(buf); - char* pos = buf; - while (1) { - size_t left = end - pos; - size_t avail = InEnd_ - InBegin_; - if (left < avail) { - memcpy(pos, InBegin_, left); - InBegin_ += left; - break; - } else { - memcpy(pos, InBegin_, avail); - pos += avail; - if (!Fill()) { - ythrow yexception() << "can't read lzma header"; + { + if (!Fill()) + ythrow yexception() << "can't read lzma header"; + + char buf[LZMA_PROPS_SIZE]; + char* header; + if (InEnd_ - InBegin_ >= LZMA_PROPS_SIZE) { + header = InBegin_; + InBegin_ += LZMA_PROPS_SIZE; + } else { + //bad luck, first part is less than header + //try to copy header part by part to the local buffer + const char* end = buf + sizeof(buf); + char* pos = buf; + while (1) { + size_t left = end - pos; + size_t avail = InEnd_ - InBegin_; + if (left < avail) { + memcpy(pos, InBegin_, left); + InBegin_ += left; + break; + } else { + memcpy(pos, InBegin_, avail); + pos += avail; + if (!Fill()) { + ythrow yexception() << "can't read lzma header"; } } } - header = buf; - } + header = buf; + } - Check(LzmaDec_Allocate(&H_, (Byte*)header, LZMA_PROPS_SIZE, Alloc())); + Check(LzmaDec_Allocate(&H_, (Byte*)header, LZMA_PROPS_SIZE, Alloc())); - LzmaDec_Init(&H_); - } + LzmaDec_Init(&H_); + } private: - bool Fill() override { + bool Fill() override { size_t size = Input_->Next(&InBegin_); - + if (size) { - InEnd_ = InBegin_ + size; - + InEnd_ = InBegin_ + size; + return true; - } - - return false; - } + } + + return false; + } IZeroCopyInput* Input_; }; TLzmaCompress::TLzmaCompress(IOutputStream* slave, size_t level) - : Impl_(new TImpl(slave, level)) -{ -} - + : Impl_(new TImpl(slave, level)) +{ +} + TLzmaCompress::~TLzmaCompress() { -} - -void TLzmaCompress::DoWrite(const void* buf, size_t len) { - if (!Impl_) { - ythrow yexception() << "can not write to finished lzma stream"; - } - - Impl_->Write(buf, len); -} - -void TLzmaCompress::DoFinish() { +} + +void TLzmaCompress::DoWrite(const void* buf, size_t len) { + if (!Impl_) { + ythrow yexception() << "can not write to finished lzma stream"; + } + + Impl_->Write(buf, len); +} + +void TLzmaCompress::DoFinish() { THolder<TImpl> impl(Impl_.Release()); if (impl) { impl->Finish(); - } -} - + } +} + TLzmaDecompress::TLzmaDecompress(IInputStream* slave) : Impl_(new TImplStream(slave)) -{ -} - +{ +} + TLzmaDecompress::TLzmaDecompress(IZeroCopyInput* input) : Impl_(new TImplZeroCopy(input)) { } TLzmaDecompress::~TLzmaDecompress() { -} - -size_t TLzmaDecompress::DoRead(void* buf, size_t len) { - return Impl_->Read(buf, len); -} +} + +size_t TLzmaDecompress::DoRead(void* buf, size_t len) { + return Impl_->Read(buf, len); +} diff --git a/library/cpp/streams/lzma/lzma.h b/library/cpp/streams/lzma/lzma.h index ca1e06e9ef..af5399c73c 100644 --- a/library/cpp/streams/lzma/lzma.h +++ b/library/cpp/streams/lzma/lzma.h @@ -1,37 +1,37 @@ #pragma once - -#include <util/stream/input.h> -#include <util/stream/output.h> -#include <util/stream/zerocopy.h> - -#include <util/generic/ptr.h> - + +#include <util/stream/input.h> +#include <util/stream/output.h> +#include <util/stream/zerocopy.h> + +#include <util/generic/ptr.h> + class TLzmaCompress: public IOutputStream { -public: +public: TLzmaCompress(IOutputStream* slave, size_t level = 7); ~TLzmaCompress() override; - -private: + +private: void DoWrite(const void* buf, size_t len) override; void DoFinish() override; - -private: - class TImpl; - THolder<TImpl> Impl_; -}; - + +private: + class TImpl; + THolder<TImpl> Impl_; +}; + class TLzmaDecompress: public IInputStream { -public: +public: TLzmaDecompress(IInputStream* slave); TLzmaDecompress(IZeroCopyInput* input); ~TLzmaDecompress() override; - -private: + +private: size_t DoRead(void* buf, size_t len) override; - -private: - class TImpl; - class TImplStream; - class TImplZeroCopy; - THolder<TImpl> Impl_; -}; + +private: + class TImpl; + class TImplStream; + class TImplZeroCopy; + THolder<TImpl> Impl_; +}; diff --git a/library/cpp/streams/lzma/lzma_ut.cpp b/library/cpp/streams/lzma/lzma_ut.cpp index 847e98d1ca..47a3aa4bf3 100644 --- a/library/cpp/streams/lzma/lzma_ut.cpp +++ b/library/cpp/streams/lzma/lzma_ut.cpp @@ -1,127 +1,127 @@ -#include "lzma.h" - +#include "lzma.h" + #include <library/cpp/testing/unittest/registar.h> - -#include <util/stream/mem.h> -#include <util/random/fast.h> -#include <util/random/random.h> - + +#include <util/stream/mem.h> +#include <util/random/fast.h> +#include <util/random/random.h> + class TStrokaByOneByte: public IZeroCopyInput { -public: +public: TStrokaByOneByte(const TString& s) - : Data(s) - , Pos(s.data()) - { - } + : Data(s) + , Pos(s.data()) + { + } -private: +private: size_t DoNext(const void** ptr, size_t len) override { - if (Pos < Data.end()) { + if (Pos < Data.end()) { len = Min(len, static_cast<size_t>(1)); *ptr = Pos; Pos += len; return len; - } else { + } else { return 0; } - } + } TString Data; - const char* Pos; + const char* Pos; }; -class TLzmaTest: public TTestBase { - UNIT_TEST_SUITE(TLzmaTest); - UNIT_TEST(Test1) - UNIT_TEST(Test2) - UNIT_TEST_SUITE_END(); - -private: +class TLzmaTest: public TTestBase { + UNIT_TEST_SUITE(TLzmaTest); + UNIT_TEST(Test1) + UNIT_TEST(Test2) + UNIT_TEST_SUITE_END(); + +private: inline TString GenData() { TString data; - TReallyFastRng32 rnd(RandomNumber<ui64>()); - - for (size_t i = 0; i < 50000; ++i) { - const char ch = rnd.Uniform(256); - const size_t len = 1 + rnd.Uniform(10); - + TReallyFastRng32 rnd(RandomNumber<ui64>()); + + for (size_t i = 0; i < 50000; ++i) { + const char ch = rnd.Uniform(256); + const size_t len = 1 + rnd.Uniform(10); + data += TString(len, ch); - } - - return data; - } - - inline void Test2() { + } + + return data; + } + + inline void Test2() { class TExcOutput: public IOutputStream { - public: + public: ~TExcOutput() override { - } - + } + void DoWrite(const void*, size_t) override { - throw 12345; - } - }; - + throw 12345; + } + }; + TString data(GenData()); TMemoryInput mi(data.data(), data.size()); - TExcOutput out; - - try { - TLzmaCompress c(&out); - + TExcOutput out; + + try { + TLzmaCompress c(&out); + TransferData(&mi, &c); - } catch (int i) { - UNIT_ASSERT_EQUAL(i, 12345); - } - } - - inline void Test1() { + } catch (int i) { + UNIT_ASSERT_EQUAL(i, 12345); + } + } + + inline void Test1() { TString data(GenData()); TString data1; TString res; - - { + + { TMemoryInput mi(data.data(), data.size()); - TStringOutput so(res); - TLzmaCompress c(&so); - + TStringOutput so(res); + TLzmaCompress c(&so); + TransferData(&mi, &c); - - c.Finish(); - } - - { + + c.Finish(); + } + + { TMemoryInput mi(res.data(), res.size()); - TStringOutput so(data1); + TStringOutput so(data1); TLzmaDecompress d((IInputStream*)&mi); - - TransferData(&d, &so); - } - - UNIT_ASSERT_EQUAL(data, data1); - - data1.clear(); - { + + TransferData(&d, &so); + } + + UNIT_ASSERT_EQUAL(data, data1); + + data1.clear(); + { TMemoryInput mi(res.data(), res.size()); - TStringOutput so(data1); + TStringOutput so(data1); TLzmaDecompress d(&mi); - TransferData(&d, &so); - } + TransferData(&d, &so); + } - UNIT_ASSERT_EQUAL(data, data1); + UNIT_ASSERT_EQUAL(data, data1); - data1.clear(); - { - TStrokaByOneByte mi(res); - TStringOutput so(data1); + data1.clear(); + { + TStrokaByOneByte mi(res); + TStringOutput so(data1); TLzmaDecompress d(&mi); - TransferData(&d, &so); - } - - UNIT_ASSERT_EQUAL(data, data1); - } -}; + TransferData(&d, &so); + } -UNIT_TEST_SUITE_REGISTRATION(TLzmaTest); + UNIT_ASSERT_EQUAL(data, data1); + } +}; + +UNIT_TEST_SUITE_REGISTRATION(TLzmaTest); diff --git a/library/cpp/streams/lzma/ut/ya.make b/library/cpp/streams/lzma/ut/ya.make index 01624f0259..81981221a7 100644 --- a/library/cpp/streams/lzma/ut/ya.make +++ b/library/cpp/streams/lzma/ut/ya.make @@ -1,12 +1,12 @@ UNITTEST_FOR(library/cpp/streams/lzma) - -OWNER( - pg + +OWNER( + pg g:util -) - -SRCS( - lzma_ut.cpp -) - -END() +) + +SRCS( + lzma_ut.cpp +) + +END() diff --git a/library/cpp/streams/lzma/ya.make b/library/cpp/streams/lzma/ya.make index 38c05145c4..dc96e8b7fe 100644 --- a/library/cpp/streams/lzma/ya.make +++ b/library/cpp/streams/lzma/ya.make @@ -1,16 +1,16 @@ -LIBRARY() - -OWNER( - pg +LIBRARY() + +OWNER( + pg g:util -) - -PEERDIR( - contrib/libs/lzmasdk -) - -SRCS( - lzma.cpp -) - -END() +) + +PEERDIR( + contrib/libs/lzmasdk +) + +SRCS( + lzma.cpp +) + +END() diff --git a/library/cpp/streams/ya.make b/library/cpp/streams/ya.make index 7426a874ee..67ffb63236 100644 --- a/library/cpp/streams/ya.make +++ b/library/cpp/streams/ya.make @@ -1,26 +1,26 @@ -RECURSE( +RECURSE( + brotli + brotli/ut + base64 brotli - brotli/ut - base64 - brotli - bzip2 - bzip2/ut - factory - factory/ut - fields_io - fields_io/ut + bzip2 + bzip2/ut + factory + factory/ut + fields_io + fields_io/ut growing_file_input growing_file_input/ut - lz - lz/ut - lzma - lzma/ut + lz + lz/ut + lzma + lzma/ut lzop lzop/ut - special - special/ut + special + special/ut xz - zc_memory_input + zc_memory_input zstd zstd/ut -) +) diff --git a/library/cpp/streams/zc_memory_input/ya.make b/library/cpp/streams/zc_memory_input/ya.make index bc94d6f1ed..c86449b1b5 100644 --- a/library/cpp/streams/zc_memory_input/ya.make +++ b/library/cpp/streams/zc_memory_input/ya.make @@ -1,12 +1,12 @@ -LIBRARY() - +LIBRARY() + OWNER( pg g:util ) - -SRCS( - zc_memory_input.cpp -) - -END() + +SRCS( + zc_memory_input.cpp +) + +END() diff --git a/library/cpp/streams/zc_memory_input/zc_memory_input.cpp b/library/cpp/streams/zc_memory_input/zc_memory_input.cpp index 682099a239..398344153e 100644 --- a/library/cpp/streams/zc_memory_input/zc_memory_input.cpp +++ b/library/cpp/streams/zc_memory_input/zc_memory_input.cpp @@ -1 +1 @@ -#include "zc_memory_input.h" +#include "zc_memory_input.h" diff --git a/library/cpp/streams/zc_memory_input/zc_memory_input.h b/library/cpp/streams/zc_memory_input/zc_memory_input.h index c939d8e426..2ffda1504e 100644 --- a/library/cpp/streams/zc_memory_input/zc_memory_input.h +++ b/library/cpp/streams/zc_memory_input/zc_memory_input.h @@ -2,23 +2,23 @@ #include <util/stream/mem.h> #include <util/system/defaults.h> -#include <util/generic/yexception.h> +#include <util/generic/yexception.h> /// Zero-copy memory input with fixed read -class TZCMemoryInput: public TMemoryInput { +class TZCMemoryInput: public TMemoryInput { public: - TZCMemoryInput() { - } + TZCMemoryInput() { + } TZCMemoryInput(const char* dataPtr, size_t size) : TMemoryInput(dataPtr, size) - { - } + { + } TZCMemoryInput(TMemoryInput& rhs) : TMemoryInput(rhs.Buf(), rhs.Avail()) - { - } + { + } /// if there's 'size' data read it, otherwise just return false Y_FORCE_INLINE bool ReadFixed(const char*& buf, size_t size) { @@ -30,7 +30,7 @@ public: return false; } - template <class T> + template <class T> Y_FORCE_INLINE T LoadPOD() { const char* buf = nullptr; if (!ReadFixed(buf, sizeof(T))) @@ -41,7 +41,7 @@ public: return res; } - template <class T> + template <class T> Y_FORCE_INLINE void ReadPOD(T& x) { x = LoadPOD<T>(); } |