diff options
author | Devtools Arcadia <arcadia-devtools@yandex-team.ru> | 2022-02-07 18:08:42 +0300 |
---|---|---|
committer | Devtools Arcadia <arcadia-devtools@mous.vla.yp-c.yandex.net> | 2022-02-07 18:08:42 +0300 |
commit | 1110808a9d39d4b808aef724c861a2e1a38d2a69 (patch) | |
tree | e26c9fed0de5d9873cce7e00bc214573dc2195b7 /library/cpp/streams | |
download | ydb-1110808a9d39d4b808aef724c861a2e1a38d2a69.tar.gz |
intermediate changes
ref:cde9a383711a11544ce7e107a78147fb96cc4029
Diffstat (limited to 'library/cpp/streams')
32 files changed, 3173 insertions, 0 deletions
diff --git a/library/cpp/streams/brotli/brotli.cpp b/library/cpp/streams/brotli/brotli.cpp new file mode 100644 index 0000000000..38052cb688 --- /dev/null +++ b/library/cpp/streams/brotli/brotli.cpp @@ -0,0 +1,231 @@ +#include "brotli.h" + +#include <contrib/libs/brotli/include/brotli/decode.h> +#include <contrib/libs/brotli/include/brotli/encode.h> + +#include <util/generic/yexception.h> +#include <util/memory/addstorage.h> + +namespace { + struct TAllocator { + static void* Allocate(void* /* opaque */, size_t size) { + return ::operator new(size); + } + + static void Deallocate(void* /* opaque */, void* ptr) noexcept { + ::operator delete(ptr); + } + }; + +} + +class TBrotliCompress::TImpl { +public: + TImpl(IOutputStream* slave, int quality) + : Slave_(slave) + , EncoderState_(BrotliEncoderCreateInstance(&TAllocator::Allocate, &TAllocator::Deallocate, nullptr)) + { + if (!EncoderState_) { + ythrow yexception() << "Brotli encoder initialization failed"; + } + + auto res = BrotliEncoderSetParameter( + EncoderState_, + BROTLI_PARAM_QUALITY, + quality); + + if (!res) { + BrotliEncoderDestroyInstance(EncoderState_); + ythrow yexception() << "Failed to set brotli encoder quality to " << quality; + } + } + + ~TImpl() { + BrotliEncoderDestroyInstance(EncoderState_); + } + + void Write(const void* buffer, size_t size) { + DoWrite(buffer, size, BROTLI_OPERATION_PROCESS); + } + + void Flush() { + DoWrite(nullptr, 0, BROTLI_OPERATION_FLUSH); + } + + void Finish() { + Flush(); + DoWrite(nullptr, 0, BROTLI_OPERATION_FINISH); + Y_VERIFY(BrotliEncoderIsFinished(EncoderState_)); + } + +private: + IOutputStream* Slave_; + BrotliEncoderState* EncoderState_; + + void DoWrite(const void* buffer, size_t size, BrotliEncoderOperation operation) { + size_t availableOut = 0; + ui8* outputBuffer = nullptr; + + const ui8* uBuffer = static_cast<const ui8*>(buffer); + + do { + auto result = BrotliEncoderCompressStream( + EncoderState_, + operation, + &size, + &uBuffer, + &availableOut, + &outputBuffer, + nullptr); + + if (result == BROTLI_FALSE) { + ythrow yexception() << "Brotli encoder failed to process buffer"; + } + + size_t outputLength = 0; + const ui8* output = BrotliEncoderTakeOutput(EncoderState_, &outputLength); + if (outputLength > 0) { + Slave_->Write(output, outputLength); + } + } while (size > 0 || BrotliEncoderHasMoreOutput(EncoderState_)); + } +}; + +TBrotliCompress::TBrotliCompress(IOutputStream* slave, int quality) { + Impl_.Reset(new TImpl(slave, quality)); +} + +TBrotliCompress::~TBrotliCompress() { + try { + Finish(); + } catch (...) { + } +} + +void TBrotliCompress::DoWrite(const void* buffer, size_t size) { + Impl_->Write(buffer, size); +} + +void TBrotliCompress::DoFlush() { + if (Impl_) { + Impl_->Flush(); + } +} + +void TBrotliCompress::DoFinish() { + THolder<TImpl> impl(Impl_.Release()); + + if (impl) { + impl->Finish(); + } +} + +//////////////////////////////////////////////////////////////////////////////// + +class TBrotliDecompress::TImpl: public TAdditionalStorage<TImpl> { +public: + TImpl(IInputStream* slave) + : Slave_(slave) + { + InitDecoder(); + } + + ~TImpl() { + FreeDecoder(); + } + + size_t Read(void* buffer, size_t size) { + Y_ASSERT(size > 0); + + ui8* outBuffer = static_cast<ui8*>(buffer); + size_t availableOut = size; + size_t decompressedSize = 0; + + BrotliDecoderResult result; + do { + if (InputAvailable_ == 0 && !InputExhausted_) { + InputBuffer_ = TmpBuf(); + InputAvailable_ = Slave_->Read((void*)InputBuffer_, TmpBufLen()); + if (InputAvailable_ == 0) { + InputExhausted_ = true; + } + } + + if (SubstreamFinished_ && !InputExhausted_) { + ResetState(); + } + + result = BrotliDecoderDecompressStream( + DecoderState_, + &InputAvailable_, + &InputBuffer_, + &availableOut, + &outBuffer, + nullptr); + + decompressedSize = size - availableOut; + SubstreamFinished_ = (result == BROTLI_DECODER_RESULT_SUCCESS); + + if (result == BROTLI_DECODER_RESULT_ERROR) { + BrotliDecoderErrorCode code = BrotliDecoderGetErrorCode(DecoderState_); + ythrow yexception() << "Brotli decoder failed to decompress buffer: " + << BrotliDecoderErrorString(code); + } else if (result == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) { + Y_VERIFY(availableOut != size, + "Buffer passed to read in Brotli decoder is too small"); + break; + } + } while (decompressedSize == 0 && result == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT && !InputExhausted_); + + if (!SubstreamFinished_ && decompressedSize == 0) { + ythrow yexception() << "Input stream is incomplete"; + } + + return decompressedSize; + } + +private: + IInputStream* Slave_; + BrotliDecoderState* DecoderState_; + + bool SubstreamFinished_ = false; + bool InputExhausted_ = false; + const ui8* InputBuffer_ = nullptr; + size_t InputAvailable_ = 0; + + unsigned char* TmpBuf() noexcept { + return static_cast<unsigned char*>(AdditionalData()); + } + + size_t TmpBufLen() const noexcept { + return AdditionalDataLength(); + } + + void InitDecoder() { + DecoderState_ = BrotliDecoderCreateInstance(&TAllocator::Allocate, &TAllocator::Deallocate, nullptr); + if (!DecoderState_) { + ythrow yexception() << "Brotli decoder initialization failed"; + } + } + + void FreeDecoder() { + BrotliDecoderDestroyInstance(DecoderState_); + } + + void ResetState() { + Y_VERIFY(BrotliDecoderIsFinished(DecoderState_)); + FreeDecoder(); + InitDecoder(); + } +}; + +TBrotliDecompress::TBrotliDecompress(IInputStream* slave, size_t bufferSize) + : Impl_(new (bufferSize) TImpl(slave)) +{ +} + +TBrotliDecompress::~TBrotliDecompress() = default; + +size_t TBrotliDecompress::DoRead(void* buffer, size_t size) { + return Impl_->Read(buffer, size); +} diff --git a/library/cpp/streams/brotli/brotli.h b/library/cpp/streams/brotli/brotli.h new file mode 100644 index 0000000000..b3af869e29 --- /dev/null +++ b/library/cpp/streams/brotli/brotli.h @@ -0,0 +1,52 @@ +#pragma once + +#include <util/generic/ptr.h> +#include <util/stream/input.h> +#include <util/stream/output.h> + +/** + * @addtogroup Streams_Archs + * @{ + */ + +class TBrotliCompress: public IOutputStream { +public: + static constexpr int BEST_QUALITY = 11; + + /** + @param slave stream to write compressed data to + @param quality the higher the quality, the slower and better the compression. Range is 0 to 11. + */ + explicit TBrotliCompress(IOutputStream* slave, int quality = BEST_QUALITY); + ~TBrotliCompress() override; + +private: + void DoWrite(const void* buffer, size_t size) override; + void DoFlush() override; + void DoFinish() override; + +public: + class TImpl; + THolder<TImpl> Impl_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +class TBrotliDecompress: public IInputStream { +public: + /** + @param slave stream to read compressed data from + @param bufferSize approximate size of buffer compressed data is read in + */ + explicit TBrotliDecompress(IInputStream* slave, size_t bufferSize = 8 * 1024); + ~TBrotliDecompress() override; + +private: + size_t DoRead(void* buffer, size_t size) override; + +private: + class TImpl; + THolder<TImpl> Impl_; +}; + +/** @} */ diff --git a/library/cpp/streams/brotli/brotli_ut.cpp b/library/cpp/streams/brotli/brotli_ut.cpp new file mode 100644 index 0000000000..aeb2e284dc --- /dev/null +++ b/library/cpp/streams/brotli/brotli_ut.cpp @@ -0,0 +1,89 @@ +#include "brotli.h" + +#include <library/cpp/testing/unittest/registar.h> + +#include <util/random/fast.h> + +Y_UNIT_TEST_SUITE(TBrotliTestSuite) { + TString Compress(TString data) { + TString compressed; + TStringOutput output(compressed); + TBrotliCompress compressStream(&output, 11); + compressStream.Write(data.data(), data.size()); + compressStream.Finish(); + output.Finish(); + return compressed; + } + + TString Decompress(TString data) { + TStringInput input(data); + TBrotliDecompress decompressStream(&input); + return decompressStream.ReadAll(); + } + + void TestCase(const TString& s) { + UNIT_ASSERT_VALUES_EQUAL(s, Decompress(Compress(s))); + } + + TString GenerateRandomString(size_t size) { + TReallyFastRng32 rng(42); + TString result; + result.reserve(size + sizeof(ui64)); + while (result.size() < size) { + ui64 value = rng.GenRand64(); + result += TStringBuf(reinterpret_cast<const char*>(&value), sizeof(value)); + } + result.resize(size); + return result; + } + + Y_UNIT_TEST(TestHelloWorld) { + TestCase("hello world"); + } + + Y_UNIT_TEST(TestFlush) { + TStringStream ss; + TBrotliCompress compressStream(&ss); + TBrotliDecompress decompressStream(&ss); + + for (size_t i = 0; i < 3; ++i) { + TString s = GenerateRandomString(1 << 15); + compressStream.Write(s.data(), s.size()); + compressStream.Flush(); + + TString r(s.size(), '*'); + decompressStream.Load((char*)r.data(), r.size()); + + UNIT_ASSERT_VALUES_EQUAL(s, r); + } + } + + Y_UNIT_TEST(TestSeveralStreams) { + auto s1 = GenerateRandomString(1 << 15); + auto s2 = GenerateRandomString(1 << 15); + auto c1 = Compress(s1); + auto c2 = Compress(s2); + UNIT_ASSERT_VALUES_EQUAL(s1 + s2, Decompress(c1 + c2)); + } + + Y_UNIT_TEST(TestIncompleteStream) { + TString manyAs(64 * 1024, 'a'); + auto compressed = Compress(manyAs); + TString truncated(compressed.data(), compressed.size() - 1); + UNIT_CHECK_GENERATED_EXCEPTION(Decompress(truncated), std::exception); + } + + Y_UNIT_TEST(Test64KB) { + auto manyAs = TString(64 * 1024, 'a'); + TString str("Hello from the Matrix!@#% How are you?}{\n\t\a"); + TestCase(manyAs + str + manyAs); + } + + Y_UNIT_TEST(Test1MB) { + TestCase(GenerateRandomString(1 * 1024 * 1024)); + } + + Y_UNIT_TEST(TestEmpty) { + TestCase(""); + } +} diff --git a/library/cpp/streams/brotli/ut/ya.make b/library/cpp/streams/brotli/ut/ya.make new file mode 100644 index 0000000000..243462f1b2 --- /dev/null +++ b/library/cpp/streams/brotli/ut/ya.make @@ -0,0 +1,12 @@ +UNITTEST_FOR(library/cpp/streams/brotli) + +OWNER( + levysotsky + g:util +) + +SRCS( + brotli_ut.cpp +) + +END() diff --git a/library/cpp/streams/brotli/ya.make b/library/cpp/streams/brotli/ya.make new file mode 100644 index 0000000000..fa2bfec9cc --- /dev/null +++ b/library/cpp/streams/brotli/ya.make @@ -0,0 +1,17 @@ +LIBRARY() + +OWNER( + levysotsky + g:util +) + +PEERDIR( + contrib/libs/brotli/enc + contrib/libs/brotli/dec +) + +SRCS( + brotli.cpp +) + +END() diff --git a/library/cpp/streams/bzip2/bzip2.cpp b/library/cpp/streams/bzip2/bzip2.cpp new file mode 100644 index 0000000000..bccc5c6807 --- /dev/null +++ b/library/cpp/streams/bzip2/bzip2.cpp @@ -0,0 +1,204 @@ +#include "bzip2.h" + +#include <util/memory/addstorage.h> +#include <util/generic/scope.h> + +#include <contrib/libs/libbz2/bzlib.h> + +class TBZipDecompress::TImpl: public TAdditionalStorage<TImpl> { +public: + inline TImpl(IInputStream* input) + : Stream_(input) + { + Zero(BzStream_); + Init(); + } + + inline ~TImpl() { + Clear(); + } + + inline void Init() { + if (BZ2_bzDecompressInit(&BzStream_, 0, 0) != BZ_OK) { + ythrow TBZipDecompressError() << "can not init bzip engine"; + } + } + + inline void Clear() noexcept { + BZ2_bzDecompressEnd(&BzStream_); + } + + inline size_t Read(void* buf, size_t size) { + BzStream_.next_out = (char*)buf; + BzStream_.avail_out = size; + + while (true) { + if (BzStream_.avail_in == 0) { + if (FillInputBuffer() == 0) { + return 0; + } + } + + switch (BZ2_bzDecompress(&BzStream_)) { + case BZ_STREAM_END: { + Clear(); + Init(); + [[fallthrough]]; + } + + case BZ_OK: { + const size_t processed = size - BzStream_.avail_out; + + if (processed) { + return processed; + } + + break; + } + + default: + ythrow TBZipDecompressError() << "bzip error"; + } + } + } + + inline size_t FillInputBuffer() { + BzStream_.next_in = (char*)AdditionalData(); + BzStream_.avail_in = Stream_->Read(BzStream_.next_in, AdditionalDataLength()); + + return BzStream_.avail_in; + } + +private: + IInputStream* Stream_; + bz_stream BzStream_; +}; + +TBZipDecompress::TBZipDecompress(IInputStream* input, size_t bufLen) + : Impl_(new (bufLen) TImpl(input)) +{ +} + +TBZipDecompress::~TBZipDecompress() { +} + +size_t TBZipDecompress::DoRead(void* buf, size_t size) { + return Impl_->Read(buf, size); +} + +class TBZipCompress::TImpl: public TAdditionalStorage<TImpl> { +public: + inline TImpl(IOutputStream* stream, size_t level) + : Stream_(stream) + { + Zero(BzStream_); + + if (BZ2_bzCompressInit(&BzStream_, level, 0, 0) != BZ_OK) { + ythrow TBZipCompressError() << "can not init bzip engine"; + } + + BzStream_.next_out = TmpBuf(); + BzStream_.avail_out = TmpBufLen(); + } + + inline ~TImpl() { + BZ2_bzCompressEnd(&BzStream_); + } + + inline void Write(const void* buf, size_t size) { + BzStream_.next_in = (char*)buf; + BzStream_.avail_in = size; + + Y_DEFER { + BzStream_.next_in = 0; + BzStream_.avail_in = 0; + }; + + while (BzStream_.avail_in) { + const int ret = BZ2_bzCompress(&BzStream_, BZ_RUN); + + switch (ret) { + case BZ_RUN_OK: + continue; + + case BZ_PARAM_ERROR: + case BZ_OUTBUFF_FULL: + Stream_->Write(TmpBuf(), TmpBufLen() - BzStream_.avail_out); + BzStream_.next_out = TmpBuf(); + BzStream_.avail_out = TmpBufLen(); + + break; + + default: + ythrow TBZipCompressError() << "bzip error(" << ret << ", " << BzStream_.avail_out << ")"; + } + } + } + + inline void Flush() { + /* + * TODO ? + */ + } + + inline void Finish() { + int ret = BZ2_bzCompress(&BzStream_, BZ_FINISH); + + while (ret != BZ_STREAM_END) { + Stream_->Write(TmpBuf(), TmpBufLen() - BzStream_.avail_out); + BzStream_.next_out = TmpBuf(); + BzStream_.avail_out = TmpBufLen(); + + ret = BZ2_bzCompress(&BzStream_, BZ_FINISH); + } + + Stream_->Write(TmpBuf(), TmpBufLen() - BzStream_.avail_out); + } + +private: + inline char* TmpBuf() noexcept { + return (char*)AdditionalData(); + } + + inline size_t TmpBufLen() const noexcept { + return AdditionalDataLength(); + } + +private: + IOutputStream* Stream_; + bz_stream BzStream_; +}; + +TBZipCompress::TBZipCompress(IOutputStream* out, size_t compressionLevel, size_t bufLen) + : Impl_(new (bufLen) TImpl(out, compressionLevel)) +{ +} + +TBZipCompress::~TBZipCompress() { + try { + Finish(); + } catch (...) { + } +} + +void TBZipCompress::DoWrite(const void* buf, size_t size) { + if (!Impl_) { + ythrow TBZipCompressError() << "can not write to finished bzip stream"; + } + + Impl_->Write(buf, size); +} + +void TBZipCompress::DoFlush() { + if (Impl_) { + Impl_->Flush(); + } +} + +void TBZipCompress::DoFinish() { + THolder<TImpl> impl(Impl_.Release()); + + if (impl) { + impl->Finish(); + } +} diff --git a/library/cpp/streams/bzip2/bzip2.h b/library/cpp/streams/bzip2/bzip2.h new file mode 100644 index 0000000000..2322277ef6 --- /dev/null +++ b/library/cpp/streams/bzip2/bzip2.h @@ -0,0 +1,53 @@ +#pragma once + +#include <util/stream/input.h> +#include <util/stream/output.h> +#include <util/generic/ptr.h> +#include <util/generic/yexception.h> + +#define BZIP_BUF_LEN (8 * 1024) +#define BZIP_COMPRESSION_LEVEL 6 + +/** + * @addtogroup Streams_Archs + * @{ + */ + +class TBZipException: public yexception { +}; + +class TBZipDecompressError: public TBZipException { +}; + +class TBZipCompressError: public TBZipException { +}; + +class TBZipDecompress: public IInputStream { +public: + TBZipDecompress(IInputStream* input, size_t bufLen = BZIP_BUF_LEN); + ~TBZipDecompress() override; + +private: + size_t DoRead(void* buf, size_t size) override; + +private: + class TImpl; + THolder<TImpl> Impl_; +}; + +class TBZipCompress: public IOutputStream { +public: + TBZipCompress(IOutputStream* out, size_t compressionLevel = BZIP_COMPRESSION_LEVEL, size_t bufLen = BZIP_BUF_LEN); + ~TBZipCompress() override; + +private: + void DoWrite(const void* buf, size_t size) override; + void DoFlush() override; + void DoFinish() override; + +public: + class TImpl; + THolder<TImpl> Impl_; +}; + +/** @} */ diff --git a/library/cpp/streams/bzip2/bzip2_ut.cpp b/library/cpp/streams/bzip2/bzip2_ut.cpp new file mode 100644 index 0000000000..69a98f296c --- /dev/null +++ b/library/cpp/streams/bzip2/bzip2_ut.cpp @@ -0,0 +1,41 @@ +#include "bzip2.h" + +#include <library/cpp/testing/unittest/registar.h> + +#include <util/stream/file.h> +#include <util/system/tempfile.h> + +#define ZDATA "./zdata" + +Y_UNIT_TEST_SUITE(TBZipTest) { + static const TString data = "8s7d5vc6s5vc67sa4c65ascx6asd4xcv76adsfxv76s"; + + Y_UNIT_TEST(TestCompress) { + TUnbufferedFileOutput o(ZDATA); + TBZipCompress c(&o); + + c.Write(data.data(), data.size()); + c.Finish(); + o.Finish(); + } + + Y_UNIT_TEST(TestDecompress) { + TTempFile tmp(ZDATA); + + { + TUnbufferedFileInput i(ZDATA); + TBZipDecompress d(&i); + + UNIT_ASSERT_EQUAL(d.ReadLine(), data); + } + } + + Y_UNIT_TEST(TestCorrupted) { + TMemoryInput i("blablabla", 10); + TBZipDecompress d(&i); + + UNIT_ASSERT_EXCEPTION(d.ReadLine(), TBZipDecompressError); + } +} + +#undef ZDATA diff --git a/library/cpp/streams/bzip2/ut/ya.make b/library/cpp/streams/bzip2/ut/ya.make new file mode 100644 index 0000000000..5ef91498ca --- /dev/null +++ b/library/cpp/streams/bzip2/ut/ya.make @@ -0,0 +1,12 @@ +UNITTEST_FOR(library/cpp/streams/bzip2) + +OWNER( + pg + g:util +) + +SRCS( + bzip2_ut.cpp +) + +END() diff --git a/library/cpp/streams/bzip2/ya.make b/library/cpp/streams/bzip2/ya.make new file mode 100644 index 0000000000..122a35837c --- /dev/null +++ b/library/cpp/streams/bzip2/ya.make @@ -0,0 +1,16 @@ +LIBRARY() + +OWNER( + pg + g:util +) + +PEERDIR( + contrib/libs/libbz2 +) + +SRCS( + bzip2.cpp +) + +END() diff --git a/library/cpp/streams/lz/lz.cpp b/library/cpp/streams/lz/lz.cpp new file mode 100644 index 0000000000..b65bb3ed96 --- /dev/null +++ b/library/cpp/streams/lz/lz.cpp @@ -0,0 +1,731 @@ +#include "lz.h" + +#include <util/system/yassert.h> +#include <util/system/byteorder.h> +#include <util/memory/addstorage.h> +#include <util/generic/buffer.h> +#include <util/generic/utility.h> +#include <util/generic/singleton.h> +#include <util/generic/yexception.h> +#include <util/stream/mem.h> + +#include <contrib/libs/lz4/lz4.h> +#include <contrib/libs/fastlz/fastlz.h> +#include <contrib/libs/snappy/snappy.h> +#include <contrib/libs/quicklz/quicklz.h> +#include <contrib/libs/minilzo/minilzo.h> + +static inline ui8 HostToLittle(ui8 t) noexcept { + return t; +} + +static inline ui8 LittleToHost(ui8 t) noexcept { + return t; +} + +struct TCommonData { + static const size_t overhead = sizeof(ui16) + sizeof(ui8); +}; + +const size_t SIGNATURE_SIZE = 4; + +template <class TCompressor, class TBase> +class TCompressorBase: public TAdditionalStorage<TCompressorBase<TCompressor, TBase>>, public TCompressor, public TCommonData { +public: + inline TCompressorBase(IOutputStream* slave, ui16 blockSize) + : Slave_(slave) + , BlockSize_(blockSize) + { + /* + * save signature + */ + static_assert(sizeof(TCompressor::signature) - 1 == SIGNATURE_SIZE, "expect sizeof(TCompressor::signature) - 1 == SIGNATURE_SIZE"); + Slave_->Write(TCompressor::signature, sizeof(TCompressor::signature) - 1); + + /* + * save version + */ + this->Save((ui32)1); + + /* + * save block size + */ + this->Save(BlockSize()); + } + + inline ~TCompressorBase() { + } + + inline void Write(const char* buf, size_t len) { + while (len) { + const ui16 toWrite = (ui16)Min<size_t>(len, this->BlockSize()); + + this->WriteBlock(buf, toWrite); + + buf += toWrite; + len -= toWrite; + } + } + + inline void Flush() { + } + + inline void Finish() { + this->Flush(); + this->WriteBlock(nullptr, 0); + } + + template <class T> + static inline void Save(T t, IOutputStream* out) { + t = HostToLittle(t); + + out->Write(&t, sizeof(t)); + } + + template <class T> + inline void Save(T t) { + Save(t, Slave_); + } + +private: + inline void* Block() const noexcept { + return this->AdditionalData(); + } + + inline ui16 BlockSize() const noexcept { + return BlockSize_; + } + + inline void WriteBlock(const void* ptr, ui16 len) { + Y_ASSERT(len <= this->BlockSize()); + + ui8 compressed = false; + + if (len) { + const size_t out = this->Compress((const char*)ptr, len, (char*)Block(), this->AdditionalDataLength()); + // catch compressor buffer overrun (e.g. SEARCH-2043) + //Y_VERIFY(out <= this->Hint(this->BlockSize())); + + if (out < len || TCompressor::SaveIncompressibleChunks()) { + compressed = true; + ptr = Block(); + len = (ui16)out; + } + } + + char tmp[overhead]; + TMemoryOutput header(tmp, sizeof(tmp)); + + this->Save(len, &header); + this->Save(compressed, &header); + + using TPart = IOutputStream::TPart; + if (ptr) { + const TPart parts[] = { + TPart(tmp, sizeof(tmp)), + TPart(ptr, len), + }; + + Slave_->Write(parts, sizeof(parts) / sizeof(*parts)); + } else { + Slave_->Write(tmp, sizeof(tmp)); + } + } + +private: + IOutputStream* Slave_; + const ui16 BlockSize_; +}; + +template <class T> +static inline T GLoad(IInputStream* input) { + T t; + + if (input->Load(&t, sizeof(t)) != sizeof(t)) { + ythrow TDecompressorError() << "stream error"; + } + + return LittleToHost(t); +} + +class TDecompressSignature { +public: + inline TDecompressSignature(IInputStream* input) { + if (input->Load(Buffer_, SIGNATURE_SIZE) != SIGNATURE_SIZE) { + ythrow TDecompressorError() << "can not load stream signature"; + } + } + + template <class TDecompressor> + inline bool Check() const { + static_assert(sizeof(TDecompressor::signature) - 1 == SIGNATURE_SIZE, "expect sizeof(TDecompressor::signature) - 1 == SIGNATURE_SIZE"); + return memcmp(TDecompressor::signature, Buffer_, SIGNATURE_SIZE) == 0; + } + +private: + char Buffer_[SIGNATURE_SIZE]; +}; + +template <class TDecompressor> +static inline IInputStream* ConsumeSignature(IInputStream* input) { + TDecompressSignature sign(input); + if (!sign.Check<TDecompressor>()) { + ythrow TDecompressorError() << "incorrect signature"; + } + return input; +} + +template <class TDecompressor> +class TDecompressorBaseImpl: public TDecompressor, public TCommonData { +public: + static inline ui32 CheckVer(ui32 v) { + if (v != 1) { + ythrow yexception() << TStringBuf("incorrect stream version: ") << v; + } + + return v; + } + + inline TDecompressorBaseImpl(IInputStream* slave) + : Slave_(slave) + , Input_(nullptr, 0) + , Eof_(false) + , Version_(CheckVer(Load<ui32>())) + , BlockSize_(Load<ui16>()) + , OutBufSize_(TDecompressor::Hint(BlockSize_)) + , Tmp_(2 * OutBufSize_) + , In_(Tmp_.Data()) + , Out_(In_ + OutBufSize_) + { + this->InitFromStream(Slave_); + } + + inline ~TDecompressorBaseImpl() { + } + + inline size_t Read(void* buf, size_t len) { + size_t ret = Input_.Read(buf, len); + + if (ret) { + return ret; + } + + if (Eof_) { + return 0; + } + + this->FillNextBlock(); + + ret = Input_.Read(buf, len); + + if (ret) { + return ret; + } + + Eof_ = true; + + return 0; + } + + inline void FillNextBlock() { + char tmp[overhead]; + + if (Slave_->Load(tmp, sizeof(tmp)) != sizeof(tmp)) { + ythrow TDecompressorError() << "can not read block header"; + } + + TMemoryInput header(tmp, sizeof(tmp)); + + const ui16 len = GLoad<ui16>(&header); + if (len > Tmp_.Capacity()) { + ythrow TDecompressorError() << "invalid len inside block header"; + } + const ui8 compressed = GLoad<ui8>(&header); + + if (compressed > 1) { + ythrow TDecompressorError() << "broken header"; + } + + if (Slave_->Load(In_, len) != len) { + ythrow TDecompressorError() << "can not read data"; + } + + if (compressed) { + const size_t ret = this->Decompress(In_, len, Out_, OutBufSize_); + + Input_.Reset(Out_, ret); + } else { + Input_.Reset(In_, len); + } + } + + template <class T> + inline T Load() { + return GLoad<T>(Slave_); + } + +protected: + IInputStream* Slave_; + TMemoryInput Input_; + bool Eof_; + const ui32 Version_; + const ui16 BlockSize_; + const size_t OutBufSize_; + TBuffer Tmp_; + char* In_; + char* Out_; +}; + +template <class TDecompressor, class TBase> +class TDecompressorBase: public TDecompressorBaseImpl<TDecompressor> { +public: + inline TDecompressorBase(IInputStream* slave) + : TDecompressorBaseImpl<TDecompressor>(ConsumeSignature<TDecompressor>(slave)) + { + } + + inline ~TDecompressorBase() { + } +}; + +#define DEF_COMPRESSOR_COMMON(rname, name) \ + rname::~rname() { \ + try { \ + Finish(); \ + } catch (...) { \ + } \ + } \ + \ + void rname::DoWrite(const void* buf, size_t len) { \ + if (!Impl_) { \ + ythrow yexception() << "can not write to finalized stream"; \ + } \ + \ + Impl_->Write((const char*)buf, len); \ + } \ + \ + void rname::DoFlush() { \ + if (!Impl_) { \ + ythrow yexception() << "can not flush finalized stream"; \ + } \ + \ + Impl_->Flush(); \ + } \ + \ + void rname::DoFinish() { \ + THolder<TImpl> impl(Impl_.Release()); \ + \ + if (impl) { \ + impl->Finish(); \ + } \ + } + +#define DEF_COMPRESSOR(rname, name) \ + class rname::TImpl: public TCompressorBase<name, TImpl> { \ + public: \ + inline TImpl(IOutputStream* out, ui16 blockSize) \ + : TCompressorBase<name, TImpl>(out, blockSize) { \ + } \ + }; \ + \ + rname::rname(IOutputStream* slave, ui16 blockSize) \ + : Impl_(new (TImpl::Hint(blockSize)) TImpl(slave, blockSize)) { \ + } \ + \ + DEF_COMPRESSOR_COMMON(rname, name) + +#define DEF_DECOMPRESSOR(rname, name) \ + class rname::TImpl: public TDecompressorBase<name, TImpl> { \ + public: \ + inline TImpl(IInputStream* in) \ + : TDecompressorBase<name, TImpl>(in) { \ + } \ + }; \ + \ + rname::rname(IInputStream* slave) \ + : Impl_(new TImpl(slave)) { \ + } \ + \ + rname::~rname() { \ + } \ + \ + size_t rname::DoRead(void* buf, size_t len) { \ + return Impl_->Read(buf, len); \ + } + +/* + * MiniLzo + */ +class TMiniLzo { + class TInit { + public: + inline TInit() { + if (lzo_init() != LZO_E_OK) { + ythrow yexception() << "can not init lzo engine"; + } + } + }; + +public: + static const char signature[]; + + inline TMiniLzo() { + Singleton<TInit>(); + } + + inline ~TMiniLzo() { + } + + static inline size_t Hint(size_t len) noexcept { + // see SEARCH-2043 and, e.g. examples at + // http://stackoverflow.com/questions/4235019/how-to-get-lzo-to-work-with-a-file-stream + return len + (len / 16) + 64 + 3; + } + + static inline bool SaveIncompressibleChunks() noexcept { + return false; + } +}; + +const char TMiniLzo::signature[] = "YLZO"; + +template <size_t N> +class TFixedArray { +public: + inline TFixedArray() noexcept { + memset(WorkMem_, 0, sizeof(WorkMem_)); + } + +protected: + char WorkMem_[N]; +}; + +class TMiniLzoCompressor: public TMiniLzo, public TFixedArray<LZO1X_MEM_COMPRESS + 1> { +public: + inline size_t Compress(const char* data, size_t len, char* ptr, size_t /*dstMaxSize*/) { + lzo_uint out = 0; + lzo1x_1_compress((const lzo_bytep)data, len, (lzo_bytep)ptr, &out, WorkMem_); + + return out; + } +}; + +class TMiniLzoDecompressor: public TMiniLzo, public TFixedArray<LZO1X_MEM_DECOMPRESS + 1> { +public: + inline size_t Decompress(const char* data, size_t len, char* ptr, size_t /*max*/) { + lzo_uint ret = 0; + + lzo1x_decompress((const lzo_bytep)data, len, (lzo_bytep)ptr, &ret, WorkMem_); + + return ret; + } + + inline void InitFromStream(IInputStream*) const noexcept { + } +}; + +DEF_COMPRESSOR(TLzoCompress, TMiniLzoCompressor) +DEF_DECOMPRESSOR(TLzoDecompress, TMiniLzoDecompressor) + +/* + * FastLZ + */ +class TFastLZ { +public: + static const char signature[]; + + static inline size_t Hint(size_t len) noexcept { + return Max<size_t>((size_t)(len * 1.06), 100); + } + + inline size_t Compress(const char* data, size_t len, char* ptr, size_t /*dstMaxSize*/) { + return fastlz_compress(data, len, ptr); + } + + inline size_t Decompress(const char* data, size_t len, char* ptr, size_t max) { + return fastlz_decompress(data, len, ptr, max); + } + + inline void InitFromStream(IInputStream*) const noexcept { + } + + static inline bool SaveIncompressibleChunks() noexcept { + return false; + } +}; + +const char TFastLZ::signature[] = "YLZF"; + +DEF_COMPRESSOR(TLzfCompress, TFastLZ) +DEF_DECOMPRESSOR(TLzfDecompress, TFastLZ) + +/* + * LZ4 + */ +class TLZ4 { +public: + static const char signature[]; + + static inline size_t Hint(size_t len) noexcept { + return Max<size_t>((size_t)(len * 1.06), 100); + } + + inline size_t Compress(const char* data, size_t len, char* ptr, size_t dstMaxSize) { + return LZ4_compress_default(data, ptr, len, dstMaxSize); + } + + inline size_t Decompress(const char* data, size_t len, char* ptr, size_t max) { + int res = LZ4_decompress_safe(data, ptr, len, max); + if (res < 0) + ythrow TDecompressorError(); + return res; + } + + inline void InitFromStream(IInputStream*) const noexcept { + } + + static inline bool SaveIncompressibleChunks() noexcept { + return false; + } +}; + +const char TLZ4::signature[] = "LZ.4"; + +DEF_COMPRESSOR(TLz4Compress, TLZ4) +DEF_DECOMPRESSOR(TLz4Decompress, TLZ4) + +/* + * Snappy + */ +class TSnappy { +public: + static const char signature[]; + + static inline size_t Hint(size_t len) noexcept { + return Max<size_t>(snappy::MaxCompressedLength(len), 100); + } + + inline size_t Compress(const char* data, size_t len, char* ptr, size_t /*dstMaxSize*/) { + size_t reslen = 0; + snappy::RawCompress(data, len, ptr, &reslen); + return reslen; + } + + inline size_t Decompress(const char* data, size_t len, char* ptr, size_t) { + size_t srclen = 0; + if (!snappy::GetUncompressedLength(data, len, &srclen) || !snappy::RawUncompress(data, len, ptr)) + ythrow TDecompressorError(); + return srclen; + } + + inline void InitFromStream(IInputStream*) const noexcept { + } + + static inline bool SaveIncompressibleChunks() noexcept { + return false; + } +}; + +const char TSnappy::signature[] = "Snap"; + +DEF_COMPRESSOR(TSnappyCompress, TSnappy) +DEF_DECOMPRESSOR(TSnappyDecompress, TSnappy) + +/* + * QuickLZ + */ +class TQuickLZBase { +public: + static const char signature[]; + + static inline size_t Hint(size_t len) noexcept { + return len + 500; + } + + inline TQuickLZBase() + : Table_(nullptr) + { + } + + inline void Init(unsigned ver, unsigned lev, unsigned mod, unsigned type) { + Table_ = LzqTable(ver, lev, mod); + + if (!Table_) { + ythrow yexception() << "unsupported lzq stream(" << ver << ", " << lev << ", " << mod << ")"; + } + + const size_t size = Table_->Setting(3) + Table_->Setting(type); + + Mem_.Reset(::operator new(size)); + memset(Mem_.Get(), 0, size); + } + + inline bool SaveIncompressibleChunks() const noexcept { + // we must save incompressible chunks "as is" + // after compressor run in streaming mode + return Table_->Setting(3); + } + +protected: + const TQuickLZMethods* Table_; + THolder<void> Mem_; +}; + +const char TQuickLZBase::signature[] = "YLZQ"; + +class TQuickLZCompress: public TQuickLZBase { +public: + inline size_t Compress(const char* data, size_t len, char* ptr, size_t /*dstMaxSize*/) { + return Table_->Compress(data, ptr, len, (char*)Mem_.Get()); + } +}; + +class TQuickLZDecompress: public TQuickLZBase { +public: + inline size_t Decompress(const char* data, size_t /*len*/, char* ptr, size_t /*max*/) { + return Table_->Decompress(data, ptr, (char*)Mem_.Get()); + } + + inline void InitFromStream(IInputStream* in) { + const ui8 ver = ::GLoad<ui8>(in); + const ui8 lev = ::GLoad<ui8>(in); + const ui8 mod = ::GLoad<ui8>(in); + + Init(ver, lev, mod, 2); + } +}; + +class TLzqCompress::TImpl: public TCompressorBase<TQuickLZCompress, TImpl> { +public: + inline TImpl(IOutputStream* out, ui16 blockSize, EVersion ver, unsigned level, EMode mode) + : TCompressorBase<TQuickLZCompress, TImpl>(out, blockSize) + { + memset(AdditionalData(), 0, AdditionalDataLength()); + + Init(ver, level, mode, 1); + + Save((ui8)ver); + Save((ui8)level); + Save((ui8)mode); + } +}; + +TLzqCompress::TLzqCompress(IOutputStream* slave, ui16 blockSize, EVersion ver, unsigned level, EMode mode) + : Impl_(new (TImpl::Hint(blockSize)) TImpl(slave, blockSize, ver, level, mode)) +{ +} + +DEF_COMPRESSOR_COMMON(TLzqCompress, TQuickLZCompress) +DEF_DECOMPRESSOR(TLzqDecompress, TQuickLZDecompress) + +namespace { + template <class T> + struct TInputHolder { + static inline T Set(T t) noexcept { + return t; + } + }; + + template <class T> + struct TInputHolder<TAutoPtr<T>> { + inline T* Set(TAutoPtr<T> v) noexcept { + V_ = v; + + return V_.Get(); + } + + TAutoPtr<T> V_; + }; + + // Decompressing input streams without signature verification + template <class TInput, class TDecompressor> + class TLzDecompressInput: public TInputHolder<TInput>, public IInputStream { + public: + inline TLzDecompressInput(TInput in) + : Impl_(this->Set(in)) + { + } + + private: + size_t DoRead(void* buf, size_t len) override { + return Impl_.Read(buf, len); + } + + private: + TDecompressorBaseImpl<TDecompressor> Impl_; + }; +} + +template <class T> +static TAutoPtr<IInputStream> TryOpenLzDecompressorX(const TDecompressSignature& s, T input) { + if (s.Check<TLZ4>()) + return new TLzDecompressInput<T, TLZ4>(input); + + if (s.Check<TSnappy>()) + return new TLzDecompressInput<T, TSnappy>(input); + + if (s.Check<TMiniLzo>()) + return new TLzDecompressInput<T, TMiniLzoDecompressor>(input); + + if (s.Check<TFastLZ>()) + return new TLzDecompressInput<T, TFastLZ>(input); + + if (s.Check<TQuickLZDecompress>()) + return new TLzDecompressInput<T, TQuickLZDecompress>(input); + + return nullptr; +} + +template <class T> +static inline TAutoPtr<IInputStream> TryOpenLzDecompressorImpl(const TStringBuf& signature, T input) { + if (signature.size() == SIGNATURE_SIZE) { + TMemoryInput mem(signature.data(), signature.size()); + TDecompressSignature s(&mem); + + return TryOpenLzDecompressorX(s, input); + } + + return nullptr; +} + +template <class T> +static inline TAutoPtr<IInputStream> TryOpenLzDecompressorImpl(T input) { + TDecompressSignature s(&*input); + + return TryOpenLzDecompressorX(s, input); +} + +template <class T> +static inline TAutoPtr<IInputStream> OpenLzDecompressorImpl(T input) { + TAutoPtr<IInputStream> ret = TryOpenLzDecompressorImpl(input); + + if (!ret) { + ythrow TDecompressorError() << "Unknown compression format"; + } + + return ret; +} + +TAutoPtr<IInputStream> OpenLzDecompressor(IInputStream* input) { + return OpenLzDecompressorImpl(input); +} + +TAutoPtr<IInputStream> TryOpenLzDecompressor(IInputStream* input) { + return TryOpenLzDecompressorImpl(input); +} + +TAutoPtr<IInputStream> TryOpenLzDecompressor(const TStringBuf& signature, IInputStream* input) { + return TryOpenLzDecompressorImpl(signature, input); +} + +TAutoPtr<IInputStream> OpenOwnedLzDecompressor(TAutoPtr<IInputStream> input) { + return OpenLzDecompressorImpl(input); +} + +TAutoPtr<IInputStream> TryOpenOwnedLzDecompressor(TAutoPtr<IInputStream> input) { + return TryOpenLzDecompressorImpl(input); +} + +TAutoPtr<IInputStream> TryOpenOwnedLzDecompressor(const TStringBuf& signature, TAutoPtr<IInputStream> input) { + return TryOpenLzDecompressorImpl(signature, input); +} diff --git a/library/cpp/streams/lz/lz.h b/library/cpp/streams/lz/lz.h new file mode 100644 index 0000000000..3a2eaad88b --- /dev/null +++ b/library/cpp/streams/lz/lz.h @@ -0,0 +1,242 @@ +#pragma once + +#include <util/stream/output.h> +#include <util/stream/input.h> +#include <util/generic/ptr.h> +#include <util/generic/yexception.h> + +/** + * @file + * + * All lz compressors compress blocks. `Write` method splits input data into + * blocks, compresses each block and then writes each compressed block to the + * underlying output stream. Thus compression classes are not buffered. + * MaxBlockSize parameter specified max allowed block size. + * + * See http://altdevblogaday.com/2011/04/22/survey-of-fast-compression-algorithms-part-1/ + * for some comparisons. + */ + +struct TDecompressorError: public yexception { +}; + +/** + * @addtogroup Streams_Archs + * @{ + */ + +/** + * Lz4 compressing stream. + * + * @see http://code.google.com/p/lz4/ + */ +class TLz4Compress: public IOutputStream { +public: + TLz4Compress(IOutputStream* slave, ui16 maxBlockSize = 1 << 15); + ~TLz4Compress() override; + +private: + void DoWrite(const void* buf, size_t len) override; + void DoFlush() override; + void DoFinish() override; + +private: + class TImpl; + THolder<TImpl> Impl_; +}; + +/** + * Lz4 decompressing stream. + * + * @see http://code.google.com/p/lz4/ + */ +class TLz4Decompress: public IInputStream { +public: + TLz4Decompress(IInputStream* slave); + ~TLz4Decompress() override; + +private: + size_t DoRead(void* buf, size_t len) override; + +private: + class TImpl; + THolder<TImpl> Impl_; +}; + +/** + * Snappy compressing stream. + * + * @see http://code.google.com/p/snappy/ + */ +class TSnappyCompress: public IOutputStream { +public: + TSnappyCompress(IOutputStream* slave, ui16 maxBlockSize = 1 << 15); + ~TSnappyCompress() override; + +private: + void DoWrite(const void* buf, size_t len) override; + void DoFlush() override; + void DoFinish() override; + +private: + class TImpl; + THolder<TImpl> Impl_; +}; + +/** + * Snappy decompressing stream. + * + * @see http://code.google.com/p/snappy/ + */ +class TSnappyDecompress: public IInputStream { +public: + TSnappyDecompress(IInputStream* slave); + ~TSnappyDecompress() override; + +private: + size_t DoRead(void* buf, size_t len) override; + +private: + class TImpl; + THolder<TImpl> Impl_; +}; + +/** + * MiniLZO compressing stream. + */ +class TLzoCompress: public IOutputStream { +public: + TLzoCompress(IOutputStream* slave, ui16 maxBlockSize = 1 << 15); + ~TLzoCompress() override; + +private: + void DoWrite(const void* buf, size_t len) override; + void DoFlush() override; + void DoFinish() override; + +private: + class TImpl; + THolder<TImpl> Impl_; +}; + +/** + * MiniLZO decompressing stream. + */ +class TLzoDecompress: public IInputStream { +public: + TLzoDecompress(IInputStream* slave); + ~TLzoDecompress() override; + +private: + size_t DoRead(void* buf, size_t len) override; + +private: + class TImpl; + THolder<TImpl> Impl_; +}; + +/** + * FastLZ compressing stream. + */ +class TLzfCompress: public IOutputStream { +public: + TLzfCompress(IOutputStream* slave, ui16 maxBlockSize = 1 << 15); + ~TLzfCompress() override; + +private: + void DoWrite(const void* buf, size_t len) override; + void DoFlush() override; + void DoFinish() override; + +private: + class TImpl; + THolder<TImpl> Impl_; +}; + +/** + * FastLZ decompressing stream. + */ +class TLzfDecompress: public IInputStream { +public: + TLzfDecompress(IInputStream* slave); + ~TLzfDecompress() override; + +private: + size_t DoRead(void* buf, size_t len) override; + +private: + class TImpl; + THolder<TImpl> Impl_; +}; + +/** + * QuickLZ compressing stream. + */ +class TLzqCompress: public IOutputStream { +public: + enum EVersion { + V_1_31 = 0, + V_1_40 = 1, + V_1_51 = 2 + }; + + /* + * streaming mode - actually, backlog size + */ + enum EMode { + M_0 = 0, + M_100000 = 1, + M_1000000 = 2 + }; + + TLzqCompress(IOutputStream* slave, ui16 maxBlockSize = 1 << 15, + EVersion ver = V_1_31, + unsigned level = 0, + EMode mode = M_0); + ~TLzqCompress() override; + +private: + void DoWrite(const void* buf, size_t len) override; + void DoFlush() override; + void DoFinish() override; + +private: + class TImpl; + THolder<TImpl> Impl_; +}; + +/** + * QuickLZ decompressing stream. + */ +class TLzqDecompress: public IInputStream { +public: + TLzqDecompress(IInputStream* slave); + ~TLzqDecompress() override; + +private: + size_t DoRead(void* buf, size_t len) override; + +private: + class TImpl; + THolder<TImpl> Impl_; +}; + +/** @} */ + +/** + * Reads a compression signature from the provided input stream and returns a + * corresponding decompressing stream. + * + * Note that returned stream doesn't own the provided input stream, thus it's + * up to the user to free them both. + * + * @param input Stream to decompress. + * @return Decompressing proxy input stream. + */ +TAutoPtr<IInputStream> OpenLzDecompressor(IInputStream* input); +TAutoPtr<IInputStream> TryOpenLzDecompressor(IInputStream* input); +TAutoPtr<IInputStream> TryOpenLzDecompressor(const TStringBuf& signature, IInputStream* input); + +TAutoPtr<IInputStream> OpenOwnedLzDecompressor(TAutoPtr<IInputStream> input); +TAutoPtr<IInputStream> TryOpenOwnedLzDecompressor(TAutoPtr<IInputStream> input); +TAutoPtr<IInputStream> TryOpenOwnedLzDecompressor(const TStringBuf& signature, TAutoPtr<IInputStream> input); diff --git a/library/cpp/streams/lz/lz_ut.cpp b/library/cpp/streams/lz/lz_ut.cpp new file mode 100644 index 0000000000..6876f070fc --- /dev/null +++ b/library/cpp/streams/lz/lz_ut.cpp @@ -0,0 +1,287 @@ +#include "lz.h" + +#include <library/cpp/testing/unittest/registar.h> +#include <library/cpp/resource/resource.h> + +#include <util/stream/file.h> +#include <util/generic/vector.h> +#include <util/system/tempfile.h> +#include <util/generic/singleton.h> + +#define LDATA "./ldata" +#define LDATA_RANDOM "./ldata.random" + +static const TString data = "aa aaa aa aaa aa aaa bb bbb bb bbb bb bbb"; + +namespace { + /** + * Produces well-formed random crap + **/ + TString RandomString(size_t size) { + TString entropy(NResource::Find("/random.data")); + TString result; + size_t seed = 1; + size_t j = 0; + for (size_t i = 0; i < size; ++i) { + seed *= 3; + char sym; + do { + sym = char((seed ^ i) % 256); + if (!sym) { + seed += 1; + } + } while (!sym); + Y_ASSERT(sym); + j = (j + 1) % entropy.size(); + result += char(sym + entropy[j]); + } + return result; + } + + TVector<TString> InitRandomData() { + static const TVector<size_t> sizes = { + 0, + 1, + 127, + 2017, + 32767, + }; + + TVector<TString> result; + for (auto size : sizes) { + result.push_back(RandomString(size)); + } + result.push_back(NResource::Find("/request.data")); + return result; + } + + TString TestFileName(const TString& d, size_t bufferSize) { + return LDATA_RANDOM + TString(".") + ToString(d.size()) + TString(".") + ToString(bufferSize); + } + + struct TRandomData: public TVector<TString> { + inline TRandomData() { + InitRandomData().swap(*this); + } + }; +} + +static const TVector<size_t> bufferSizes = { + 127, + 1024, + 32768, +}; + +namespace { + template <TLzqCompress::EVersion Ver, int Level, TLzqCompress::EMode Mode> + struct TLzqCompressX: public TLzqCompress { + inline TLzqCompressX(IOutputStream* out, size_t bufLen) + : TLzqCompress(out, bufLen, Ver, Level, Mode) + { + } + }; +} + +template <class C> +static inline void TestGoodDataCompress() { + TFixedBufferFileOutput o(LDATA); + C c(&o, 1024); + + TString d = data; + + for (size_t i = 0; i < 10; ++i) { + c.Write(d.data(), d.size()); + c << Endl; + d = d + d; + } + + c.Finish(); + o.Finish(); +} + +template <class C> +static inline void TestIncompressibleDataCompress(const TString& d, size_t bufferSize) { + TString testFileName = TestFileName(d, bufferSize); + TFixedBufferFileOutput o(testFileName); + C c(&o, bufferSize); + c.Write(d.data(), d.size()); + c.Finish(); + o.Finish(); +} + +template <class C> +static inline void TestCompress() { + TestGoodDataCompress<C>(); + for (auto bufferSize : bufferSizes) { + for (auto rd : *Singleton<TRandomData>()) { + TestIncompressibleDataCompress<C>(rd, bufferSize); + } + } +} + +template <class D> +static inline void TestGoodDataDecompress() { + TTempFile tmpFile(LDATA); + + { + TFileInput i1(LDATA); + D ld(&i1); + + TString d = data; + + for (size_t i2 = 0; i2 < 10; ++i2) { + UNIT_ASSERT_EQUAL(ld.ReadLine(), d); + + d = d + d; + } + } +} + +template <class D> +static inline void TestIncompressibleDataDecompress(const TString& d, size_t bufferSize) { + TString testFileName = TestFileName(d, bufferSize); + TTempFile tmpFile(testFileName); + + { + TFileInput i(testFileName); + D ld(&i); + + UNIT_ASSERT_EQUAL(ld.ReadAll(), d); + } +} + +template <class D> +static inline void TestDecompress() { + TestGoodDataDecompress<D>(); + for (auto bufferSize : bufferSizes) { + for (auto rd : *Singleton<TRandomData>()) { + TestIncompressibleDataDecompress<D>(rd, bufferSize); + } + } +} + +class TMixedDecompress: public IInputStream { +public: + TMixedDecompress(IInputStream* input) + : Slave_(OpenLzDecompressor(input).Release()) + { + } + +private: + size_t DoRead(void* buf, size_t len) override { + return Slave_->Read(buf, len); + } + +private: + THolder<IInputStream> Slave_; +}; + +template <class C> +static inline void TestMixedDecompress() { + TestCompress<C>(); + TestDecompress<TMixedDecompress>(); +} + +template <class D, class C> +static inline void TestDecompressError() { + TestCompress<C>(); + UNIT_ASSERT_EXCEPTION(TestDecompress<D>(), TDecompressorError); +} + +Y_UNIT_TEST_SUITE(TLzTest) { + Y_UNIT_TEST(TestLzo) { + TestCompress<TLzoCompress>(); + TestDecompress<TLzoDecompress>(); + } + + Y_UNIT_TEST(TestLzf) { + TestCompress<TLzfCompress>(); + TestDecompress<TLzfDecompress>(); + } + + Y_UNIT_TEST(TestLzq) { + TestCompress<TLzqCompress>(); + TestDecompress<TLzqDecompress>(); + } + + Y_UNIT_TEST(TestLzq151_1) { + TestCompress<TLzqCompressX<TLzqCompress::V_1_51, 1, TLzqCompress::M_0>>(); + TestDecompress<TLzqDecompress>(); + } + + Y_UNIT_TEST(TestLzq151_2) { + TestCompress<TLzqCompressX<TLzqCompress::V_1_51, 2, TLzqCompress::M_100000>>(); + TestDecompress<TLzqDecompress>(); + } + + Y_UNIT_TEST(TestLzq151_3) { + TestCompress<TLzqCompressX<TLzqCompress::V_1_51, 3, TLzqCompress::M_1000000>>(); + TestDecompress<TLzqDecompress>(); + } + + Y_UNIT_TEST(TestLzq140_1) { + TestCompress<TLzqCompressX<TLzqCompress::V_1_40, 1, TLzqCompress::M_0>>(); + TestDecompress<TLzqDecompress>(); + } + + Y_UNIT_TEST(TestLzq140_2) { + TestCompress<TLzqCompressX<TLzqCompress::V_1_40, 2, TLzqCompress::M_100000>>(); + TestDecompress<TLzqDecompress>(); + } + + Y_UNIT_TEST(TestLzq140_3) { + TestCompress<TLzqCompressX<TLzqCompress::V_1_40, 3, TLzqCompress::M_1000000>>(); + TestDecompress<TLzqDecompress>(); + } + + Y_UNIT_TEST(TestLz4) { + TestCompress<TLz4Compress>(); + TestDecompress<TLz4Decompress>(); + } + + Y_UNIT_TEST(TestSnappy) { + TestCompress<TSnappyCompress>(); + TestDecompress<TSnappyDecompress>(); + } + + Y_UNIT_TEST(TestGeneric) { + TestMixedDecompress<TLzoCompress>(); + TestMixedDecompress<TLzfCompress>(); + TestMixedDecompress<TLzqCompress>(); + TestMixedDecompress<TLz4Compress>(); + TestMixedDecompress<TSnappyCompress>(); + } + + Y_UNIT_TEST(TestDecompressorError) { + TestDecompressError<TLzoDecompress, TLzfCompress>(); + TestDecompressError<TLzfDecompress, TLzqCompress>(); + TestDecompressError<TLzqDecompress, TLz4Compress>(); + TestDecompressError<TLz4Decompress, TSnappyCompress>(); + TestDecompressError<TSnappyDecompress, TBufferedOutput>(); + TestDecompressError<TMixedDecompress, TBufferedOutput>(); + } + + Y_UNIT_TEST(TestFactory) { + TStringStream ss; + + { + TLz4Compress c(&ss); + + c.Write("123456789", 9); + c.Finish(); + } + + TAutoPtr<IInputStream> is(OpenOwnedLzDecompressor(new TStringInput(ss.Str()))); + + UNIT_ASSERT_EQUAL(is->ReadAll(), "123456789"); + } + + Y_UNIT_TEST(TestYQ609) { + auto data = NResource::Find("/yq_609.data"); + + TMemoryInput input(data.Data(), data.Size()); + + TLz4Decompress d(&input); + UNIT_ASSERT_EXCEPTION(d.ReadAll(), TDecompressorError); + } +} diff --git a/library/cpp/streams/lz/ut/random.data b/library/cpp/streams/lz/ut/random.data Binary files differnew file mode 100644 index 0000000000..6b3c159b58 --- /dev/null +++ b/library/cpp/streams/lz/ut/random.data diff --git a/library/cpp/streams/lz/ut/request.data b/library/cpp/streams/lz/ut/request.data new file mode 100644 index 0000000000..6a29bfe6af --- /dev/null +++ b/library/cpp/streams/lz/ut/request.data @@ -0,0 +1 @@ +pron=snipfactors&pron=grmult_QUICK_SAMOHOD_ON_MIDDLE%3Ad%3A0.1%3A0.1&pron=umrfull&pron=usemiddleownerdata&pron=onlyfastrank&pron=sortbyfastmn&pron=rtb2000000&pron=tbs120000&pron=multbs0.63&pron=versiontbs2&pron=wffrm_ru.fast.download&pron=whrm_ru.hast.download&pron=rcy-lzo&qtree=cHic7V0NdFXVlb7nvpuXxyFhXgNx4Z0ljURpYFkbwZlxaR2s80apf0VmlrVvqgNIgBRI4kuE6KxZTYJABIZg_VlqUSnIj2hfIAiEAAEELag479k1SrV1KuMvrXUW2pnxr875ufe-87PvvS_JKyrGtQzvvrP3vufs_e2zz88-5-ErcEmsKB4daVSgKrPaKDNsY7QxzhhvXFgSM-IG-d6oMqqNS4omFU02rjemRmejlch4ABlrkNGFjD3IIP8dQkYGzbBfQfgHmLPFCBsVVzztlqb6m6bNsFGFJ7ZYEGtMMqjY2Y8fOd0V67Io0quNC9ClDTEUN2yXYrRRRfiq0SQ0-S6T1M2YjVJXYLc0jkayd1xgTEHtSXNdenqM8NIXD5-OySfzG43kc2T6kLhpI_ejYaNv0I-zjWbzNjNmzi5uKG5BBqmbfbAUz1JaNyzbmunJbM4uIf-3ZZfTRqKKgFa-id1GKpxQW58pYY1VKEmb38SkyebktW6TVxVhhShuek03SFtN28x0sQbGicBYtj2zObM108u-YaXZDvZ5GPkczWyTyzLbnM-IfN4pfN7tfCaazC4UZK0QeLtzNPnI8eipnEU5Gqd-JeR7K7vQeUNJ3LItUtvdzpNBytq8t0QIV5v2di6hPbs8R-XUBeDP7BL4Nws17hY-i5rao9HHCU0ssydQ370F0beqG_q0rAA655roFb5fJryhjWlyuGJ5w9U98SKUei6Kyzx4dme6vrIQFYHB4Aa2kNeq14-qIPDgJcsK2vKT7XIiAFW7egDkDpi94ySgidWH4LvQGMIEQ04Jc6dqobPvJtrrpnGmCjmeNIxoMco9jDrm7FizQcIYEsLY00W4XgljZZktmW1E6N7MLspK-qteKZQNAULZymI3lAHcUDg7aLFwBlCTkLayWA5p_4IBOtJlIDegtyYNoifEvAvSuOtDw6dXxIfYI4mYXk0gwMk0fMzEI1Ri0pfuFLos1JKMpHfsJ1WIsCrsESCwJ9TFOV1Yxb9Mzl8Rx31U8njVwhqUXW32-gB5E8L_oACZKV-CrglAd7HpQpfR62C1Lv17hlVWTNC52HTRiaaaBJ2VmJU4eEAXmFPQEoJHb3hJG6hUdnEUGDy2EJu0MmfuzewOHTw-Pio3eJQ4IW9bXuQMHiVKOmAeJXvaOIxzNIr-qQZ6GQqY_lOtJhYFdqsuYW1Kd-9jf_c7XSvc7f45Au9Awubw_DyH4fZRFKCDSPu-w0lzU1prPe8nChEc4DGj1gpW13G4RLS_ZF3-rpxtx2EFKxJ1DtqU9hwcl2mJL8jUKEetOML_mYAjtGV6icAW2pFm20Id4eDUnCNInJAj_Ak5jiBREkc4OFV2hLXUsBKRbNhnkubGNDSN9MYpRh6GzEGRR69eZ9Qrw8AUwOgNuZlJqeqlSvZB9UsQ_p6ienPCeKC7RD7qJtSQisdQNpsUjh5a7f43YXzV3dVEwUwE1e9thlCTByJ4ilKTyPTaJqkqEcDyb6cjbl0oPVSZbSazNy0mRiYM8vLAZZiWxFFuJNGvpYHUVZqcO5Lm6q1JtEUQxXg4-1Bil0hTzQznwbQjtXWzPGk_wBaRViOuWBBxa7aS_7tyJuXyanLVadSkkYcIfU-dJ3o0jhLR82rmKDCJSDAR1zm6EP6-Ypxo5onMrmxHqH0e8MzjcECx9bvMQA4BsdEDETm6no2dsr7E11dNfelpQW3djPoFjaH9yfFzvKUnhwUC1gnekbgkpOLHz5Gx9V0c5YUqvDq9qkdgeA1X4XW1IEqAhLU6vbVXFQaAioiLOJKZuAnYrbZQNbO9M4nWqXD1W_1aDXQezQ22ETzUSh_xVgsJNaTWiUyrpJAolFDLGh2FSQFTARSIlCqOpguZeKRZYVUVVRujS2M_jrcg8hjJ41FmVZ8QQSJMGSSGNi_5MkFNuWFMS9iHEP6hosAYXamcUdM4x0ZjR6MqR48xQI9rHq509egxQdpMMm16JKOp4iSN_hX2CgVome1pGQoCSnUsjLZoK6qs5BbLbdzDFp6EKQRMVvWgheTopChrkv8ycgbhGb6aGlIx5ltV51WcN7biS6Qx-y2Ea9Qm1dbNr6lrqk9R4-daFdCkT372757SPG7_VXKPRG_TldgrLPAq-WgOAuoQVjXtu5GCmbVWLF5-91VTEvadhcTMS7p3WdRsIF4sQLkfPenFLsYI6bWWzwppsa7TSZgV5KtPNsiryWPb4QWEk-pYiW2oAKiBYvLLH47wxkw-uyoz-ZCJ76ggbcBU-L2UUJQsicRi5ce2zk3YfzRxrTpfIBOrLjoKpqtgmSekzhOK79sPPeYBR-GFtPGhM2OQKR3NSDNnhSRgxvQzhOMysbN8gJzpxV4yvVjfKYz1CjG1EGaY0tqPPrVA6oDKsYSRPI6Iv7I-_qgOxCjfapMsEAUs0H3fsZW5oSFnglQ_lw8NOYGORTLA4kXCkKAfcEQ-Ac1IHiA6KL-otDlhbzH1aE3GT3Vz67nrBcae54_m-mmXCWrukxxpHo3e4snYKwyZa4QPBfnI8kpYYiGc2qJOTZy3h455Lty8IGFvRPo8r2HGTEmBUJ_V9bCnQEoP6S7B-yxSqmvtLEy_F_qsSPu6tDKB8Kv872nl_7OBQOBlHe_FJGCmSNgM7XLuXW95cwqHB2pEHZ9SOBR6Q67AblmhO2C3uetGkOZ-uIrY6q4RBQzHR4rwgjw2GaDgXJDNhgNBmw1ocK9hcK-hb3sNr5biZg3PO-gCL10IJZ-6ef4HMCaD8LwvFyAAMRCgf1niAFqj1gG9fygW6bZrG-40fmw5MH1w131w131w1z10193rQr9K6T0cVwuF9u-QfPbU6zWYxfuc8tWX5DP-bp4wJfsnC-3ZDiFpqhprXb0SsgRpbIp5rs5BOn3fKaky3fOfkPNx4p5SMk68dPX8hP1YKW4ICIXb3VTIscFDOp8QuN0_H3KjHgK3ezmRSgh8Mo8QGNly8MhgDByMgQOKgb6pj4MxcDAGful6jZMZA1X_1GPg-Uof3o9sRy9-raXxa97q-oR9ZyluUuLXCPYaJUFZimAYiGAtr5a6EQwUAMWwpTyGgfR6FDs6FIOUQhwz7zhC5nIHifaKCD43n4KYHEzz_6Kk-auWF2NdhLSf4fOUR-DgWGpwLDXgLH6fKHUoQqIUvp7MsrZH9CglH6Ih3Q2pZmiUOvFf3jwLFABFqU6eQwfS61HqU4RBSil9cr9_-qSQ9CgBBNq9FIEavifKhxTLPWMGvK1_bwgw5fvFxJSfPEYGHK8Va2vHp-nnobKtNL9OMOZQwJivvzTBNaaPCMicL0aZOX04dIOuiWIfWtGkdxx8nu5AnkaAPSzT5Xb1lMpxSsuOkuGc-1xGnjF3UpmGrst7z3Q4t1N53i08E0dnecfiO1Z4z3HyTMOQv0QeqHaToWSr56YUI-rTcuF9i2RpQpvY-6iLC99EifxtQp3ZG2n36EmV38jL27l-nbcUKXU2FQlxQhFjOxm5mtJ2divPvYHvbPPeOVzQpNjyNo9jBOEocTvgL7mNuRX7YtVhxKqcYnmI199OO_CL1y5I2CdMLYmtxHOqHnpKJSRnInOkzPV1iRHy8I95IoFEp_v1WoQlCmGPkyegkM7Znef3CNGwh8VwtbsOT0MRY35eKSlOhnuukr3akRShPkGzPTNWVV59XhOZ7Zn6Jn7jnGk3zZ4Wvom_8q2R3ia-wwMp_xEnMdgh0Xfxz8duGcu4QOEJhKjZpHv_OleBsl2olozkJpqKN_OREU_a91n435Cipgr5SDl0RFDQXxzKu3p_8ddcBYZKgzT7v3wcEsqrQ30bwqFcDP4ot8V_bjxuj6H7yiGM0F76uPhwe3Q_-IQ9eJqDlboEh0hhuQHyNkBOAt0GuCS44X3d-WZgsYhLHbFiZ5a_89T1CXuXpaVGDZle09gwd1pTXb2QMQKt-i99-m9cROR4INM_F2GmzxHpbvXXOFfaF8caB_NpOUKI5eGDtP3J7ETs1Ejfxfl6darKR1qL1gw_ylaN8lofypCMsyEE08BxFLDzsQienFS7z35562CqXf_Styzmlb-hWvz6AqLFZ038j2oC9MxUTY2kQSjx-eB-T4OMAdLe61x7rFzX3IWYFYhtlCDIm5LKdX6NTYKONF6_g1SuwpGvwoGKtOZZER_97qcnHJaMuTlhd1p4qqJfnOtaJS0XAVp-cdVcV8sCG6Tr3bznE6j08PYwwkK5EMi-EOcUaSBb5VdDq_2n9Izs51y9KlwixkX_I5Qsy9uj1KJvjLzDb0vdxdByiqE5V5M5wUcRfIu29aAOE7ozO0MXdZ57_21h7wGQAB5gc1Z1IHodYvuAxETvtLMXaIo2prsJ4qx15B_FLIVa8wxf3x3ez7W5FdIMpMOv3v1KO4QS6jkcHqRw-OzYDQl7mZUXHHpC4bDi6ftmBMGhB4LDwogvHHp0OKxDIBx65L6nkI4NzR4DnZw59qN51bPQfWS_6uqDj-N04fDpi-cn7KPF-DZ14VBrG5sRhC4cZt64d5l364WPEAgkzzpLhzCHDpMdCPvQiiGg5aF0dzKykZph3T5nBrL5pG2WaV0GA05XXjVvX-VgXL3gwVlwA65lkKvHl6L0Hbs-92Gs0h_kU2m0NGltTBNlR9al9wXey3HyNyr71-wt-TTb646-EKbycfZP6GmTrxu1Cfs4AsaXT5C2tfDLYULGl7uXluXGlx4b5NLNzvDSI9K9eAw7gU-KQ-6DGYsFMeqJiYDT-l7z_4dOX0p_PCthv27iuUrz48S8XSzc9pK3qAtO0GRw1bve_YoaM6SKN_i0RqPVNbIIYY1KbfCoeIx0lHz9SCSE1n3s-FC7LA86cXXHSQuVucLWg3wU30oHIYcua0rY70XwPFXxom_lpfjtn-70zjtp3JDm3-NjUY1W1_zP6clJhUrwcrNlZdLc1Jk0N3TSA5RJ89FOeuEC7ev2sm7vcDKyPr2P_tlPQg79s47-2USKIxvon_X0z6P0zyZKt4mWbqCfNnAOUrCx8xmhC6XU7A8t7jwsLaCg1NTA6jqzH1rhTWQAsrGTD0Ro5TekeQPWp3kjHk2zhkiDFPaGavkN_VjR4xh4njrf7QfJQGO3GYwBemo1FAOPPLYWgxig3OBEF-kYoLQ6Bs7VWrxZwTwSMZ_6Z42-VzpsSzejB2ICH4Wm6ebPlbdPTdirIkBvtoigYVe2g3ivPmaDFPrmi9fkejOFGdLng45PqbS6Pt-jvZlCJeaUtXSfSiMG__DzAo2-H9zbnLD3IXy9erY5dUtjY2342eb7fuF1fQ4LZJzr-MlmTqAvoU3ATtFALuWIxWOkUfdQtz5_y60JeyGw98XfUhe69_XRf3vntV0eqFn3OntfDonesDHYLfNdHmRLd1eAdIVZKuV6eYF657VWPTF2BDdquTUtbM-6lS5xuBe7hc67X_12LrUG4IcU1uOm1gD0uqO2mxiklE-HSoswA7t3r-9LKSch0Q30YG7UDgr2TAfx4E-Bo-YEUaFHzXs_zh01J_SQze7jIKfFOsC_hen3QftPyq7N5SpDPzeTfGD-LoX5d5qnJ-xXIrhV3dS1SXe_jcTEbnpvn3yNoaSpUujEzkPHPVUFyIE0-DRHfQCXjv0VJg6gH_QAx97vUA-IbLkxYb9k4pvVg1lcaSwVpo2pcXPowaxjv3jXu4of4AcPJ3MHAajBJBSATsoVdFJR3HV41wonMQnFR9d_oLp-6T8aEvavTSCnz23V7bxN2cXkc3hOX9f7V-dy-kARkMqzyMnpAzl0tT-IsA8tpHr3tgMfReWdppmXuQIUzjaJZ6L5bJP4BnWT2GlPT-gW57H0wtwuscsFaXUv1SrNtnNodD3ej7BXKs0plLXvCQJoKwUtVApaqBS0UCmAtlIAbaW0nlwpdB2VAmgr3VnJVCgkrKQhYd5bsxL2n4CtdrdfDdXi8baxihKzrZASO3hn79EEKTHbKiWhfc47g3SFZYxwH3LQvqBI1-tP5wPst-n6y9Gz5ibsFy3cokVp8XZdMkXbkV2WXQLMGqEovXqZl3MVIAZMkeAbQgFcuiWXRnAAfUCQHhMvtUexxTKQG1wMq4j_hT0SpP9zXIryeZ7bKMwkl0PtMPlQ_tpPahP2Dgv_EzQfnFM_L3Q--ObOD6LShJAwQSA6HMlNCAmJPl6u5DPdOfWSz7Ch7TzPuc7HroC-JHZVQVxgWtcEWP66JEqrA3cpIVNn6mfq1vdAWf3MNaICx4AC9UwwmE7LAwMnFwa9HCtWvvSBU_9yLBzHpLkP0cD5_bduSdjLI3o2d2YLO5Gqd8vQKtHKp8Z72dwiI9Twe3gAlej0rvd3CEsUX60FvGHxYaRv-zhKDHT_3U0J-50onq1dK78w0-OMeNXzvBAsW1YmcvfKS6yQkf7g_sCCRKmb6RwskYRc4nF3EVYkSkduDhzhvYMpTYxOvROXX5i7WJzDIe1f3ZsPArzvAI5Z5T0VNWR2hgt4d-ABU3NlPH3aTXNmpepvqZsB3hkIHnx8dai3I55jh1z5EJ9LC1R6sLkaC8UDXqJOTfET1_90Y-Xa4BcRTilaHFpb11Qzd27tLBqkBTVqV9ZByxSL77jOVacoB9LnFKZOkUrX5zgslosBXNsbUBr2ip4pEGtKTatrnFmTCmwVtMd2d_tp3pzWFQI16ddO-rhLA16p7RYOHB4_goX1BRzshyBSMGwsf9hsNfF07bxUTWp-7U014O2G4N7RPV6qmcsL6bTXPTfFScABnlM2cI1eA8oKTjX3O9QQcp110JN0sxbtRY3kE_Sq6zcWkTH1moi-CclHeKG7GL9tieV-eYOxQBrfwQeWDgWYVu6UyasyTrrk-s7PfXXmLBxzxrwhazOcKjhfOydNuygtaAWHm40M0mPl437TyLZZtaFnpivTTUYGT9AGKktrkMf0_n5M7oJyiRUyY7t7QblEqZvzBqyQaEu9_U6CgAYFZfEyopc9FM4398xN2I9FgJvbvfrQO-pC9fL49l_FAcVQ3oB7CxRKcA9CofkiHqG4MaCSVvtP9w4wj8jHfr10uejOjtkJ-3Er2H554Prl5_eaoP1gYG-O6PaDgD0OKyQBN-8fNXGZTOxNlr1Fkkh6x75c1yCZ-su7htj32cCo-BCewCjrK-_zDRxAR-gPBuCLfvV39i6Eb1QPijo_Q9TcELoPuXTn5d5RUY8LQs2V_KSoR6MPJM7GudJ4vr8mRNszgsxyVtM9qAefXZCwVwT9xlSIJ-T3G1NrtN-Y0n8BoVA_MTXw34QiKhrBVLST5lHd-vOmhL0B6acQs4uyLZJ-oB-KOvHpmd4pRMoAaec7_BAiLdb7hArMCgTHls2rw5XX_YRFJrGrRtYl7N8W8vdoFptaOne0uSlVM68GnMBCS4YHX77MG9NxVkgpd3HIOBQgYnjRwBetz4ZEaUvMyoTiBMI_UuehDTWpmfWpedPqfCYV0Pzzg3e9E28iP6SS-Xz-KVDparkGi-WFXkD2H_NT0BnUYWLlh38y389hMntDHabtrssFh8nsDXSYzF4fh8ns7aPDGORpKToTs19MKTs9huxy4j9fSxdPPOMv0ZUXVRjfpHWodmiIFhya08svfu22iWcYRstEXxq7nBZzGvxN-k81cmkshwaXU94zxuMbLtDkWN67ai--AnyXpcq5y06qcixBzv0N_xoix7_OVjzmyel48o_fPuPYvrGqnKhA89RzV4HvytH4vwvHsUMTKz990e_-FpKTozm9_J1ru0No_N9Fpj-hNPF43HsX03Pzm7ep7yKjBQ8_2bXVE8-Y1ZpVbUEisEfz4WcTQZoRAk3VDzsuVmnKHGzgWGxKNIbKzOsm8y-pYr0vJ5Mvh5MvS0jrhpIvi8mXkWsm1_BvaT3Eb4k__D_2jF58&rdba=clon&relev=ad_cat%3DO;ad_filtr%3D10;cl%3D2;cm2%3D0.0181118;cm%3D0.0869565;country%3Dru;ct%3D520-87%3A11886-13;dmoz_th%3D174-83%7C179-12;dnorm%3D32+autocad+bit+windows+xp+%D1%80%D1%83%D1%81%D1%81%D0%BA%D0%B8%D0%B9+%D1%82%D0%BE%D1%80%D1%80%D0%B5%D0%BD%D1%82;dnorm_nav%3D32+autocad+bit+windows+xp+%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D1%82%D0%BD%D0%BE+%D1%80%D1%83%D1%81%D1%81%D0%BA%D0%B8%D0%B9+%D1%81%D0%BA%D0%B0%D1%87%D0%B0%D1%82%D1%8C+%D1%82%D0%BE%D1%80%D1%80%D0%B5%D0%BD%D1%82;dnorm_old%3D32+autocad+bit+windows+xp+%D1%80%D1%83%D1%81%D1%81%D0%BA%D0%B8%D0%B9+%D1%82%D0%BE%D1%80%D1%80%D0%B5%D0%BD%D1%82;dnorm_w%3Dautoc+%D1%81%D0%BA%D0%B0%D1%87%D0%B0+%D0%B1%D0%B5%D1%81%D0%BF%D0%BB+%D0%BD%D0%B0+%D1%80%D1%83%D1%81%D1%81%D0%BA+%D1%82%D0%BE%D1%80%D1%80%D0%B5+32+bit+%D0%B4%D0%BB%D1%8F+windo+xp;fe%3D6.48773;fir%3D0.267373;forum%3D0.59255;fqwg%3D5.218048216e-15;frqcls%3Dstrong_tail;fsm%3D177;fsml%3D11;geov%3D-0.000436704;hpq%3D0;iad%3D1;iad_vw%3D-33.57460785;il%3D0;ilp%3D0.001022;im%3D536870912;imgintent%3D0.56;is_music_ext%3D0.000;isorg%3D0.460;issite%3D0.460;issoft%3D0.750;ldatopicru%3D197;mum%3D0;mut%3D15;navmx%3D0.268605;norm%3Dautocad+%D1%81%D0%BA%D0%B0%D1%87%D0%B0%D1%82%D1%8C+%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D1%82%D0%BD%D0%BE+%D0%BD%D0%B0+%D1%80%D1%83%D1%81%D1%81%D0%BA%D0%BE%D0%BC+%D1%82%D0%BE%D1%80%D1%80%D0%B5%D0%BD%D1%82+32+bit+%D0%B4%D0%BB%D1%8F+windows+xp;ow%3D52;prel%3D3;qc%3D3;qlu%3D1;qmpl%3Dru;qr2r%3D0;qr2u%3D1;qr2v%3D0;qrr%3D0;qru%3D1;qrv%3D0;qt%3Dru_nloc;relev_locale%3Dru;synnorm%3D32bit+autcad+pyc+winxp+%D1%82%D0%BE%D1%80%D0%B5%D0%BD%D1%82;syq%3D1.136661591e-07;th3561%3D0.0179;th3973%3D0.0682;topicality%3D36.0447;tvm%3D0.00222546;uil%3Dru;utq%3Ddownload+here+upon+russian+for+autodesk+inventor+auto+cad+%D0%B0%D0%B2%D1%82%D0%BE%D0%BA%D0%B0%D0%B4+autoca+pdf+torrent+%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D1%82%D0%BD%D0%BE+%D0%B7%D0%B0%D0%B3%D1%80%D1%83%D0%B7%D0%B8%D1%82%D1%8C+feed+load+%D0%B7%D0%B0%D0%B3%D1%80%D1%83%D0%B6%D0%B0%D1%82%D1%8C+%D0%B7%D0%B0%D0%BA%D0%B0%D1%87%D0%B8%D0%B2%D0%B0%D1%82%D1%8C+%D1%81%D0%BA%D0%B0%D1%87%D0%B8%D0%B2%D0%B0%D0%BD%D0%B8%D0%B5+%D1%81%D0%BA%D0%B0%D1%87%D0%B8%D0%B2%D0%B0%D1%82%D1%8C%D1%81%D1%8F+%D1%81%D0%BA%D0%B0%D1%87%D0%BA%D0%B0+galloping+skachat+besplatno+free+%D0%B1%D0%B5%D1%81%D0%BF%D0%BB+%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D1%82%D0%BD%D0%B8%D0%B9+%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D1%82%D0%BD%D0%B8%D0%BA+%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D1%82%D0%BD%D0%BD%D1%8B%D0%B9+%D0%B4%D0%B0%D1%80%D0%BE%D0%BC+gift+%D1%81%D0%B2%D0%BE%D0%B1%D0%BE%D0%B4%D0%BD%D0%BE+freely+%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D0%BD%D0%BE+%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D1%82%D0%BE+%D1%85%D0%B0%D0%BB%D1%8F%D0%B2%D0%BD%D1%8B%D0%B9+russia+%D1%80%D0%BE%D1%81%D1%81%D0%B8%D0%B9%D1%81%D0%BA%D0%B8%D0%B9+rus+%D0%B2%D0%B5%D0%BB%D0%B8%D0%BA%D0%BE%D1%80%D1%83%D1%81%D1%81%D0%BA%D0%B8%D0%B9+%D1%80%D1%83%D1%81%D0%B8%D1%81%D1%82%D0%B8%D0%BA%D0%B0+%D1%80%D1%83%D1%81%D0%B8%D1%84%D0%B8%D0%BA%D0%B0%D1%86%D0%B8%D1%8F+russification+%D1%80%D1%83%D1%81%D0%BA+%D1%80%D1%83%D1%81%D1%81+%D1%80%D1%83%D1%81%D1%81%D0%BA%D0%BE%D1%8F%D0%B7%D1%8B%D1%87%D0%BD%D1%8B%D0%B9+russkom+%D0%B1%D0%B8%D1%82%D0%BD%D1%8B%D0%B9+%D1%83%D0%BA%D1%83%D1%81%D0%B8%D1%82%D1%8C+background+intelligent+transfer+service+%D0%B1%D0%B8%D1%82+bit+%D0%B2%D0%B8%D0%BD%D0%B4%D0%BE%D0%B2%D1%81+%D0%B2%D0%B8%D0%BD%D0%B4%D0%BE%D1%83%D0%B7+%D0%B2%D0%B8%D0%BD%D0%B4%D0%BE%D1%83%D1%81+windowsxp+windows+xtreme+performance+%D1%81%D0%BA%D0%B0%D1%87%D0%B0%D1%82%D1%8C%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D1%82%D0%BD%D0%BE;vcomm%3D0.248625;vintent%3D0.95;vm%3D268435456;vnorm%3Dautocad+32+bit+%D0%B4%D0%BB%D1%8F+windows+xp;vnorm_dopp%3D32+autocad+bit+windows+xp;vnorm_syn%3D32bit+autcad+winxp;vqi2tvec%3D1%7C0rkjACu_CrE81xPoE93gAB5X2uKgUsx_JvATChj78_A%2C;vqsemvec%3D371364212%091370421533%093362250591%092760956004;wizqbundle%3DehRMWi40AQAAAACAUzAB-F8KZipZehRMWi40AQAAAACARwAB8wYSXBJGEgdhdXRvY2FkGAAgAioMCggPAHNzEAAqDQoJDgAQJw8AJAsKLAAnEAEcAIRzJxAAIJMCKh0AoDICCAI4AUAAIAAAAAAwo7-XscucoK7PAQr0ByrmB2oA8P9S1AMB9g8SjxcShgkSDtGB0LrQsNGH0LDRgtGMGAAgASoQCgwWAIfQuRAAKhYKEhIAfbLRiNC40YUYAMe90L3QvtC1EAAqEgpYAE3QtdGCLAABRAAt0LwYAH290L3Rg9GORAA9vdC-LABRtdC80YJwADsYChSgAAAcAB2DMgACdgAP0gADH7WOAAMh0L4CAQrSAD290YtcAAD-AD2w0Y-6AD-70LiiAAQRs9QACloAP7vQsFoAAS_Ri54AAy_RiwABAACKAB--AAEFA14BCjIBAjIAApAACv4BHbuOAABEARG1dgAKKgAdveYAAMwBMYvQvAABCuYAD2wBAQBYAB-4WAAAH7KCAAIRuGoBCYIAATYCCtgAAlQAD4IAACHRjpQCCZYAAAADLxAB7AIAAtYBNxQKEGQCX7XRiNGM8gABI9CwSAEMMAACqgIKSgMCeAEABgMJigAARgAN9gABLAAB5LYA8HMB5AANigAAFgAEigANdAAh0Ys-AQ8YAAEA1gMKYAATuWQDCi4AAGAAH7UaAwARu9QCDywAAAG6AAoYAAIgAna5EAAS6A0SYQQyuNCyjQEBjQQ1GgoWdwECGgAzjtGJgQII7QEBHAAAawQBQwINNgABsQACyQENHABP0Y7RiRcDAANSlAHwZdMAAhoAAusADU4AAGoAA04ACEkBAzQADzIAAQCAAA8NBQABMgABlQIKOQECGAACwQI_HAoYHgEABPMCD5wAAgOLAQ6cAAAXBQ8yAAMC7QUNbAAA6gAFxwQPOgADAiMCDToAAfIABDoADx4ABQLBBA5YAAAaRwDw2Q0cAAFYAA_mAQQBrgACYQQNcgAA5gEAYwUBSQMNVgAAHgADBwYOcgIfuXICBgGoAAEjAw9wAAIFKQQOVAABHgAPKgIHAXsHDsABACkFDWoAAhIBA5MEDtoAABYDEbhbBg1QAAGxAw0yAAFQAAJDBw0cAAJiAQ-aAQgCtQUOOAAAKgEC9QcOHAAAogAP7gMHDoUFAR4DALIBAaMHDVAAA8AAAKUICI0EAjYAAn0HDTQAAk4AASYBDWABAfQAAcAAC4MFAlIAAL4AAP8HDzgAAw-hBQECOAACNQgPigAEAQACDhwAAWwAAXAJAOAAUgICaQYOHAAAOAACkXYA8GcBOAABCgIOIAIA6wgPMgAED4QFBAAoAQKxCA4QAQFqAAOoAg6aAQKgAA42AAC8AATuBA4eAAJUAA9IBAQBCgICWAANxgACHAAB-AAPjAAAATsIDTQAACYCDxYBBQJQAACcAUcgowIqHAsA3wWgMgIIATgBQAAgRQT4AjDk_fqeq-3Q-OMBCqgCKpoC9wPwCAgBAf0XEr0GEqABEhLQsdC10YHQv9C7-QMA1gPxARgBIAEqHAoY0L_Qvh4An7UABPAlGAoUGgAELxYKUgAAHxBQAAkftVAABn21EAAS-AQSawBv0YvQuRgApQAKD1UABR--vwAFTwME-4W_AAKdtdC9EAAqGgoW8QBfsNCz0L69AAYAJwEPDQEFDz8BAT--0LyAAAI_vdGLZAADT9GL0Y8aAAYfhRoABQEaAQ_OAAFfvtC80YM2AAYAnAAPtgABEbCEAA_oAAEfsEwAAyHQvoIAD18CHgD0AQ-IAQQfvogBCBG-1gEP8AAAb9GL0LzQuKQABAFAAk8gowIqzAIAKwHovMHs0NOEgNjdAQpIKjspAfEAKQAB8QISJxIUEgTQvdCwFAXBCAoMAGEQASCjAioLbAUAdQESAWwF-AHYpuGipb-7vI8BCu0BKt8BTADxBM0AAfcPEvEEEuwCEg7RgNGD0YFyBRC4NwEEbAUhaL5RBREoAAWRFAA-vtGOFACIZwXgFAoQTgBN0LzRgxYAe7jVAPAfEAB6AAASAAw-AB-zkAAAP7jQuXwAADq8EAG4ACHRg6QAClIAH7BSAAQAugAK9iAA8Cm8ZgAADnoAH7geAQCPuNGFEAAS5AFvAQIKkQAPIQELDCwAD70AAw6VAAIPAQo-AA_7ACsP5wAGSO0AYy4C0L7QvPAAA2UC-QDW3teX_ICvr_gBCqMBKpbwAPEMhAAB-g8SmAIS-gESDtGC0L7RgNGA0LXQvdGC8ACiFgoSFgCg0L7Qst0A8hMMAAYuAE_RixAALgAAP7DQvC4AAS_QsC4AAy_RhS4AAnq1hgJQcgBv0LD_APIESAACH752AAN70YMQACoSCugAa5cBFRWXAQOnANjLxM6v-_j5l1oKRyo6lAH0GCgAAXQSJxIZEgswAQDwDTMyGAAgACoGCgIzMhABIAAqAjMyMgA4AksH-QCA877ojIGuoecBCqQBKpfvAPFthQAB9hESvQESMhIDYml0GAEgAioICgRiaXRzEAAqCQoFYml0JwsAcHMnEAAqBworAEYQARI1NADwAQsKB2JpdHRpbmcQACoKCgYNAEdlZBAATQAFNwAgQBJgADVlGABsABdlNQAVbgwAAk0AAoQAEGWEAARCAEAgkwIqg_AAB_MH2J-xsZ7qkK_2ewpJKj3wAPECKwAB8wQSLRIYEgbQtNC70Y-KAVIKCg4AYzsBEQ1LAAfSAunHnYGH9bv932YKkQEqhPIA8QNyAAHyBhLFARJBEgZ3aW5kb3f1AGEMCggOAHPnAPEaCgocAGMQACoLCgcaACYQAScAZSdzEAASQkMAPwAgAjUABzINCglCAFb8APEKRAB0ZWQQABIoEl8AD4gAAAh8AEQgkwIqK5QAB98A2MbplNn6pa_fAwpDKjffALElAAASIRIQEgJ4cDIJcAYKAnhwEAFFADoCeHAXCdiHlbXAt_bOjnUKVypKRQCwOAAB9AcSQxIsEgh-CUFkZXNrTQBgDgoKEAB1wABiKgwKIABlWAAcD50A-QDmvbXihY6InIYBCpgBKosyAR95WwAcJPEHtADwHxJiEksSCGludmVudG9yRQA0DQoJEAAARAA0DgoKDwAEVAAEEAAgEAFyAAQOAEF9AUdkAAQRnAAAWgr5ADC5xvmm0_qipSoKoAEqkpsAsIAAAfADEkwSORIE9gAB8gBhCgoGDABx8gBwCAoYAGAQAcoBQBYAVGVhAPUAIwCBZXMQACCTAiokAPECqgAQRAID8A5jYWRNAGEJCgVjYWQ1ADAICgQLACMQABUAAWEAIP4CfCAQAUcAAAqiAK_p2fTEnaq4t64BowAC4PwREpoCEvwBEhDQsNCyVQQAswoj0LSbBWAYChQYAJwcBAFTBIEyAB-1FgABjCAEURYKEkYASgQCVgTwAAI_vtGOGAADL9C5RgABingE8AIqEgoOdAAvEAGIAAGL0LwQAPcEES2jAAASAwNNBOjw7e2q0ZnMoLYBCmUqWDoCcEYAAfIFElYQAwBEASFjYUYBARADEEYyATEOAHNNATAKCipwAwBNARAoIAMAQwERGmcAB8QD6JGI7ved-9fU6wEKfypzZwDzBWEAAfUHErABEkwSCGRvd25sb2FkegNQEABkEAEfDNEeAHRzEAAqDwoLDwCUXgNxKg4KChEAb2ID0EtOABkHPQAqcydNAIWTAACCAByMggCeq4T0m6vIyclDNgWhABIkEhISA3BkZuMAIAcKCwACEQMACgAEDAICFQfosbOQvtqmg6CnAQpjKlbKABZE6QxxdG9ycmVudE8AAekMAcAAAekMwBknDwAQJx4AJAsKO-AAAK0AHA6tAPgB_Ma8_MTw4JzsAQqrCiqdCmcALwsF7wjrJPANtgfgEpoYEvgXEhLQt9Cw0LOFCODQtkIDJdGMmwIJGgBj0N8N8Du1qwAAkQAGHgAzt9C4HgAANQIKHABzttCw0LXQvLkBOxQKEHAAAAMCAH8BBk4AMLbQtd8ADS4AU7fQuNGCQwEILgABYAADKQEAl4YM8AIFKwENmgBl0Y7RidC4AwIGOi8L8GMF6QEGGAAAbgAACAEEOwMPUgAEAesBOBIKDuwAAZ0CCuoAAWYATtGMEAEYAS_Qu-oAAQKwAAFpAwzqADG10L2dAgLTAwqYAASgARG40gAKbAAAHAAAHQMNmgAAUAUNhAAALgAAAAEC4gAMhAADigEDJAK2CvAWHgARu28ED5wAAxW16QIJbgEC8gABDwQKOAAAUAABigABHAAK1AYA8WQNQAIAFgAAoAIOmAARtVgCDBgAAFIBAAUECngABQIBDzYCAyHQu5UDCjQAAkwABM0DD2gAAQCHBQ1OAQH8AARqAgweAACeAB-92gIGAx4AAaQAD4wAAgOQAwwcAANUAQFiAgAvBQaMAQX4AA2MAAQaAAQchgDwB1QAE7VMAw5uAAIaBACWAQ8cAAIDVQa7APAbcAAfvFIABQE0ARG1wAAPMgECAOwED4YCAAMyAQNfBwqOAzG30Y8gAg2iXwHw9AJUAw0cAAMQBQStBgZIAQEQAS_QvPoEAgAQBD2I0Yx2AwSeAAI0AA1sAAGIAA98BQUAJAMCHAAMeAIBfgEPbAAAA1AAAZADD0QBAwRrBwqKAACYAQBuAAKmAA5uAA9QBAEDhgADeAEK6AEBHAADEAEMiAAE2AAG8wcKYgEfuSACAgSiAALyBArmAQFkAQ3cAgRqABO4bQcK8gABNAABNgIA6AEN8gAv0Y44AgMAAAIDnAMKJAYRt14DDGQAAuwAAlYBChwABdAAAQQFCiQBBBwABEAECjoABFYAAvwGChwABDoAAr4CClYABR4CD0YDDABSBgpgAQBWAA_UBgIAKgMDvQoUAvAeMgAAUAIEfAEPHgIAAEgBDcICAzgAA1QACtwAAPgAADYCAmEKDbIBD74FAgNQjgLwAw1UBgEcAA2CAABmAAK6AA_aBBcBQQA0Ag4XAvGyD1IAAQKWAwHCBw_UAAQPoAAAAbABD04AAQDOAAPLCg0GAQGUCA-OAQUAPgEAZwsKrAEA8gAB1gQPGAIEAtgAApgFDr4AAPQAAgAICowAAF4BAjQIDW4AAw4BAowADlIAARoAD_oAAgIaBwItDQ5-AQHEAA8wAQEBcAAChAMPNgADD8IHAgNmAQ-mAAMCigAP_gYDAxwAD9gDAAHSAQC2Cg-SBAkDDAINGAEAOgACGAECHQwGNQUAewbQgtGMMgIIAV0GEAEUEvgBi5Xsua2Avp3ZAQqeCCqRCH4SX_8DAfoTKQQBAB8OIdGMMgpyHAoYGgD3ATAEANENAt8NBDQEYx25HAD3AjMEJNGLvwwQOMkCIWqPTRKxhgBdtdC90YsuAF85BPIJYAAFb9C-0Y4QALYAAAJ8AD-80Lg6AAFvOQQghADYAvEKAbIADwgBAxOwCAEKugABNgAv0YxqAAcBIDsE8QgEAS3Rg-oAADQBH7tGAAgfvOoAAHO10KgSYZgAD6ABAy0EwIQBALwBAFAADcoAT_MOwGgAAAFSAQ9oAA4A6DEN8Q4AOgAfuzwCCQHAAQq4AAAgAgKiAR-FDgEBAXQCAiIEwQBsAB-yVgECD5gAAx8EEbAfBPENAgEKlAAAMAEACgMPNgIEL9C7agICAEwAE72kAiEEglQDDWoCAZ4CIQRDAVIADyAEs9oCCsAABRwAAQwCIgSAEgI3GAoUqAIiBC8UAiIEASOeAyIEZi_QvooABB8EGTYfBLwWASHQvqIDCyIDISAEogFIAR-1ZAEEP7UdBJBABB-IQAQCAdAYEgccBBCs9wAAHAR1iAAPWAMBAhwE8QAxvtC8tgQKigAECAIP2gAjBPQEUAQKNAAAGAAADgECeAEPOgIBAiAEbwKIAA44ACAEBPIAH7hCBgQCmAERvugBDWABHAT2BAIQBwGcAwoMBhG3XgMOIgEAxAEcBBkgHARoBdgAAYYAHAT5AASSBQTsAwo6AAYQAQBABCEEUS_Rg24CIQTzAvoABHwBDuYBAAACACgICoYAIwQQHCMEZwE2AAEyASMEIrYBIwT6BA9EAwEA1gAD8AcOggAC1gAAEAkjBIpqAg5SAAHUACUE8AgPuAIGTILRjBACCgHqAAIGAQH8Aw0YAgoCNW4CCyAEphG1qAAKkgEAdAAhBCAABFEAoBoAAVIADBwAAbL6A_YADn4BAIwEDvQHAUYBI9GJJASCARQBDy4FAwQkBPEDAqYAAqYCD_wAAwQWAw_gAAMCZwJwAU4JAvwASuMKW3ULAfcBFBHp16ff677V8PFICo8IKoEhBNHvAwH3FRLkGBLAGBIUSggCNxY3uNCyIwRwFAoQHAC3sFASAh0EEBZiFBBBRQiQHAA9FgoSHABoWBbwABgKZgAASgC9vdCw0Y8QAHgSoDIAAU4AATgACmyPAZAh0LNsAAxUAF-dCHC4VAAMHbzAJBNkE75UAA9wHBIgxABdBPMDEbWmAD8eChrEAAQCOgAP5AAAQQTwET4ABgBcABOwlAAP6AADH4U8AQQAOgARuOgAD5YABQRYBwDxDAAByAEPmAEKIdGLXAAK6AF_sNC50YLQtZIAB8oSUEIBAFG9tATgOAAPrAABA8oAL9C4tAF9CPGKE7UgAQ_IAAQh0YskATcSCg6cAQKOAg8yAAcPkgIFAnAAAJICDBQDEbugAAqkAQEUAwBGAy0QAYAAH7sIAQAAXgACdgEOXgAesBYAT9GO0YJsAQMBVgIRtVIBCpIAAVQDA6QBDNYDEbXSAQwwAAa2AQ0wAA9-AAUByAACggMP0AEFBLQBDG4AABAEEb4OAw-cAQQh0L7WAA8ekwDxIjwADNoAEbsoBAysAgGmAA2GAQG0AQNwAQ-eAAADTgIKIAQCNAATuVgBCjYAAhwAAiwJAPCyggAftVYECg9MAwQD4AEA3gMPCgEBAdoAAQoBD3IAAQT4AwwQAR-1wAAJARwCCjIAAtoAH7LmAQQA2AATi7YDChIBBKAAD9wACwBkAww4AAIeAgA4AAwaAAbyAA-kAAEAbAMLFAIBlgMOIAECigQfvBgCBQNyAwx-AAM2AA9-AAMBigYPCAMAAcQBE7giAQ_GAQUC5AELPAQPRAUHD2wABQGSAwCSAQDABg8UAgEPiAAIAqgAAUQFD6YAAAEeABG1kv8A8poAA3gAALIBCnYAAjwCAJIAHYuCAwFQAQOkBw84AQUAOgAAzAAPigEBAmQGCggCAN4CEYjyAQ9SAAAA3gIF-gcPUgAAIdC-4AALQgIADAQPfgUDAFAAA9AGCk4AAvYAAioGDxABBAHoBQvwAQ9kAwUCfAED_gYNngABSgAKgAABWgEDJAUPCAEAAVAABB4EDIQAH73sAAgCcgEP0gACA1QEDxwCAAFuAAIGSRfQsAEPOgAEA0oIDx4AAHEG8glGBw8eAAUCsAIMHAEARAQCGgAPOAAFAq7cAPAOzAAAkAUPzAABAFoAAL4ID3IAAgE4BA9UAAYB0gNDAPAMADgAA5gCClYABGQCAKILChoAA4gAAXAACh4DiwnwAuQCD2oAAQG-AA8ABAoBbgFIDgRZ3QsCVQAzCBAAMwj4AZy27ceGip-f2QEK4wEq1gEzCNHEAAH9FBKMBBLoAxIU1RQARxoBEARh0L3QuNC1NQiSGAocAK7RjNGOEARQNgBv0Y9mD1A2AAEAUncPQDYABY-UFAAEBBBSmg_wAbhwAAUv0LVwAAgANgAPwABxA_AAwAAFL9GMwAAJL9GP2gADGQDwARoAA0_QuNC5GgADIdGM1ADYAvART9GM0LXuAAMAGgAB7gAPegEAAIQAD0QBBQAcAAJEAU3qAEzPAQBF-AT4AJi7qpKGyID3PQrsAyrfA-YA2s0BAfsXErYLEpALEhbmAAAZCSPRgU8TYB4KGh4AcPEMYbXQtSIAauUEwBQgACPQuxoADToAjzcJABsXQCAACU9OG_IIIAAKM7DRjyAAKxoKuADr0LvQvtGB0YwJAfAGlgACXAACHgAPOgABBBwAPyAKHDoABw3wBrNcAA_YAAIBEgEzuNC5mAAP-AACT18F8BwgAAwPQAALH7hAAAcBgAAGQAANkgEBcgEAFgEPmgADFbhSAQ0gAAJaAAFAAQfwEg8cAQUzvNGD3AAOWgEztdGCHAANXgAAEAIHXgANIAABfloAYA8EWAENnjIA8lUfsLoBCQKCAAXMAg9CAAMxuNC8HgIAAgEPpAAED7QCBwKEAAViAQ1iAAGCAA9iABAFpAANYgEl0Y5iAQ72AjG50YI6AAD-AQ0YAgWyAg5UAAMCBB8QAAQAA7IAD64CDQQgBA1cmgnwDqYDBhe1sAANLAEBTAEA8AEEbgIPIgAFBqwDDSIAtQLwHBQCCgBmAAemBA5UATK10YggBQ_-BAEB6gEAbgEPKgIFD74EBwDaAAFmAkvvAVvbBASTBecG6enM4IPgrJuYeQqUASqH1QLUdQAB8AES7gES0gESDO8BMQYAeB4ZYRAKFAB3uHcWsQ4mAD2-0LkUAJewvQLxChQKECgAMbDQvD4AClAAK7AQYgAt0YNOAD3MF-I9sNC8TAAbtRIAmLXQumMSRMEAsLCiDwIzEdiW1aqNnOWslW4KcCpjhALxBFEAAfMHEocBEikSB3NrYWNoYXT7EWANCgkPAHTLEfASKgsKHgBKEAESRisAMwwKCCsAACoABjkAKXMnDwANSABEPxEcVj8R-AC80eakh8SnqcABCosBKn9zANRtAAHwERKkAhL0ARIgCAEC8wIGOxpAEAD_AesHAR0BMCoKJjsaMywADDwa8AwmCiIoABIPVAAVH7VUABQAKAAvJArQAA5vEAGZATsnAA6XAen5xY6Ulab6tjMKrAEqnpcB8QaMAAH1CRKhAxIvEgliZXNwbGF0bm8EAWEPCgsRAHYEAfERDQoiAEwQARIeMQALIAAcUiAACFEAEGVRADUQCgxiAF_BFPAldAAOH1BUAAkBQgA1DgoKVAAAZAAIdQAPxgAhD-YAHgjVADtpbmeDACxlZBEAAKUACjoBRj0BEWo0BAerE-iJnJWDwt20i4EBCnwqcDwB8QFeAAHwBBKkARIrEgRmcmVlqAByCQoFDABwcu0XUQsAcXN0YRWxIwBOEAESNy0AcGTGAUEHLQBUuhOwRQAZczkAD2YAGUF_ABxxvAHp5MjfwIHK0NUTCpoBKo0tAdZ7AAH2CxK5AxLOARIKrgEBpgGBEAoMEgCG0YsrFTAQEgATEgI6FXAOCjoAZxABDxWwJgBavtC8EABMAE3GDVAmAByyJs8CwCYAH7BMAAAesCYAT_YCQNEAvkfLARFoHAEHLhXpy6ShvsiwpJVTCvoBKu2dANbbAAH_FBLQBBKsBBIUnQAAWwIBNgYVuTYGIAG9_BkDkhxQOgARtR4HB2ACL9GFGgDADBC5BwCRuNC8GgAGfbkQWxqAggAfuDIABS9LHDQEnbV1BvAPTAB_tdC80YMQAGgAAQ9OAAUh0Y5OAA8CAQERtc4Apw1QP7jQvLgYAMCPagAPcAEHADwBD1RIChCP6A7yBvIABQGmAQ82AAAj0LU2AA9aAQF_tcYDWhMCAbC5xwOfxY7vtvKou7MXxwICj_8UEp8FErwC_QACEbqkAQAzBzECbxBxD0A2AAFvghtgEAAeAAh_NRtgGgoWPAACgACBKh4KGhwAAl_CAfESPAAIDxwACC_RgzgACB-wrgAIP7DRhR4ACQDqAA8_Af8tBx07YgICdQP4AcaI8fa1rMGLsQEKpgIqmALJAt0GAQH_FhKnCxK4AhIWrAERvfwd4AEgASoaCh4AAz-8EAAclAeAvtC5HAAIDziNACCOOFUBkIMcAD4cChiqAMsAELPNARAFJwBgtRwABQDkBAiyVgAGP7zRgzoABY8RDPAXErw7ASIAOwAPOwEDD1cBCS_Ri8kABwNXAQ_JAAUCHwEPHgACAHIaCBAeLQiwsFsBCQ8_ASef0YuYAvEuEsgDegIhDz8BIw-WAgkPWwEQDwYDAg-yAi4PkwEpDzwAAgAJAg-2AggPDAMPD-gAAgKvAQ-zAgMPCgP_MygBPGYFA9cB6Pnp99egktmV6QEKTSpBJwGwLwAB9wgSORIgEgqVGCDRgLEBAcoBMA4KElQDACQDApcZByQDn8HXhZeluLalHj8EAPUH-xEStAESlAESENGB0LLQvtCx0L7QtGsFYhYKEhgA3aIfMBoKFncFMBwAbS4D8gMUCkwAHxBKAAMALgAPSgAFfbmnAhFLPwQHfgDY6IT9ysKFy7o5ClcqS80AuDkAAf0OEnkSLBIQ8gEFfAAwFAoYsgExLgAbTgkcRVkA2N3H666P0qrwTApTKkdZABE1WQAcS1kAAM8ZAfMDAFkAEm0sARwXVQDpn_CL1IKJ3ethCrwBKq5QBPAInAAB_RASiAMS6AISENGF0LDQu9GP0LKyDQNjHwBZAPAKHxAWAAE9tRAALAA_g9GOFgAAPovRjxYAL7EGoQEv0L5YAAGr0L70IMAWChKyAD680LgYAG7KAUDRg3IAnyCQhS4AAj6z0L4u7CPxBw8WAAARsMwAC4oAL9CwRAADINGLRAATARxTDgXorN7M1fKtjPSPAQpUKkgUARE2PBq0OxImEgZydXNzaWEKGQY1GhAcNRoAAgYCpR0HgQbY2YSlusPR2boqCm0qYFYAsk4AAfQFEngSGhIHVgCSbhgBIAIqCwoPIwhyHAAzDQoJKz4mAmEmoykQADkAByoAhCfdGw0sCPkA-6qwpt32g5SJAQrUASrGhAGgtAAB8AkS_AMS2AoMMYDQvuUgMLjQuTkIMQgAfuABANYEEV_uD_EQGgACT7jQuRA0AAM_vtGONAAGH7UaAAMv0YM0AAh9vCoUQBK4AH4WIWAaChYYAE-FAWE0AAQfuIAhDPAF6AAHL9GFaAAGEbOEAA84AQERvrYADBABmApghgAPHAABAiFiCg88AQSu1gEAgwMgvwEwDAqtBa-q4bqf8pLDtqYBdQkAsfALEmQSHBIDcnVzQgFQCAoEcnXUG_AAKgcKFQBGEAESNB4AeQoKvQFAZSAAQfEfEBXyHDQLAIBJASBAAHEFB7cB-QDj_o6a2pz4xI8BCt0BKs9JAfEIvQAB_xoS7gQSxAQSGtCy0LXQu9C40LpGIRGDVQEQusMAAVABYR4KIgAFn6gHgSogChxCAAR_ZiPAEABCAAoftSAACj-4XwwQC5ML8AKiAAkfuaIAChO-ogA_HAoYxE0BoUAADD-80YOiAAnqEPAMIAAJL9C4AgEMH75AAA0v0YVgAAw_uRABpAEIWAEQ5DEHkQcPwgELELCiASYFMCUCBY8AClIB-QDxkqauxviIwsEBCqIBKpXgANCDAAH-ExK0AhKSAhISiRqB0YHQuNGB0YLmAAJqI1EYChQaACMOADwkUBoAAX7QcwswFgpQYxRQD04AAozkBaAqFAoQTgAPMAAERAJhYgAEH7UYRALTGAADP9CwEOAAA56-0P8EHcVsC-nkupf52uG8oTcKqQEqm6UAgIkAAf8XEsoC_goVFqUAEoSjAEPRhtC4QSFxHAoYHgADj40AAf4NIwOfhR7xIhoKXAACARwAD1oABE_QtdGOHgAHAFoAD1gAB0_RjxABdAAFAVYAD5AABR-5cgAHr9DLBQCjAmrLAALA0Y9RAfkAsJKumbn30cyuAQqTASqGrADTdAAB9AkS0AESuAESCKwAEbofAmAQCgwQAGo0I6IQABIAK9GFEgCVdiNgDgoKNgBlVAHxAgwKVgAaEFQAEb5UADUSCg4wwgzyATQAB0QAK9GDVAAbtRAAdbCaABFlyAIHQga_-oj2o4emraw2CpVFDQAq9QiWACHRgREHBK0eZBAKDB4AaZ4AEpiZAGESCg4kAITuAfAIDgoKFAAr0Ys2ADuw0YUiABuDIgARvmqaAHMq0LUyAIXQlwAcq5cA-QD_-tm7xIeol4cBCqECKpTQCCACAV8DcrIGEogGEhouAQCiBKfQvtGP0LfRi9GHNgYAXwMSBt0B0BwKGEIABa8QACoiCh5tB-AiAANv0LXQtRAAYgAKf8AUYCAKHEAAAw4ncLzQuEIACy-IBPEXEB-5RAAJDyAADhGwpgAPKAEGIdC-hAAPIAAGAGoBLxAB5gAHb9C1AzBqAQn3KDBgAAefBvU14AAKEb4gAA_IAQU_tdC9PgAKAsQBD6QBBgD-AC_Rg0IACSHRiwYCD4AABT-90LA-AAkh0L4-AA-AAAkCPgEPIAEGr9D8AkzpAgbApAPpiY66wtOj_skWCpwBKo9SAhF9bA4gzgJsDgA-BjFrb238BA9sDgCRGisACRwAGigccw7QRwANRgAaRioABnEATCUF8RWAAA9IAAgaR0gAMw4KCnIAOWluZ0kAL2VkSQALD9cACQ8cAAiYDhEdMQcHIwXp1JyM4sqUsu1pCr4BKrCfAPAAngAB9QwSxAISqAISDNCxfh4HtQEAoQ8QpLQHAk4CwiYAatC-0LPQvhQAiBIo8BUQADoATNGL0LUSACrRhTgAI9CwTAAHcgAATAANOAAbjzgAW75tBvALJgA6g9GOEgAs0L6CAC3QviQAEbBcAAeAAEpPBmNIAAAYAWWsCDwBABOEAvkAgone2__i_ODlAQr7AyrtUBLzAtsBAfQOEv4IEuAIEg7Rg9C6BQUEfS1gEgoWAJeIMicA4wJxFAADKgBmEPMJoD4AIdCyLgA3tRBrGTESANvlGgHsAGEYChQwAFlKAmAyAANKAElpEsF2AAJKAAosAAIUAFmsC2BEAAMYACtvCUADXACryAQCRwUwjgB0DgQB9y1gSdGP0YKEIhLwjz-I0YzKAAITtcoAClYAMbzRghIBBvoAACoAAxIBKdGDtgAEngARtfoADzIAAQ8sAQQh0LDQAAe6ACHQuHABNA4KCswAAdwADVQAT9GL0LwaAAMFmgEGDAIAoAARuy4AD_gBAQDgAQYYAAPmACHRi5oADEQAAMYBBpwAG4gIAQU8AAE6AQ9UAAAB3AAGGAAAmAAAfAI_uNGFmgECE7jkdgAAahLyKwQCfAIJUAELCgIBpAAp0YvAAQEUAAB2AgHAAQZuAAUYAAFuAAaEAQS4ABO1CgIJcAAt0Lt6AgJKAgrWDvAIIdC7rgMGfgAGZgAPJAMDAjwCBjAAAJB_F_MLAU4BBhgABEgAEbhUAwaGAAEwAAkIAQYqAHQlJF5NBAA_AzQV-QDDn9HZ2s_81sUBCt0CKtCCBPEHPgEB9goSxgESVRIKYmFja2dyb3VuZGEDcg8KCxIAdnOJAXARAB0nEgB3RwPwEioOCkcATRABElZXADYRCg1GAF9pbmcQAGoADC5lZFgARx0KRGkA8gxEA-ESPBIiEgtpbnRlbGxpZ18j8xDJAAcTAAI9AAglAAU-AOHTARJLEgh0cmFuc2ZlcgUBWCYg8QC2AfAqDwABAQElDAovACAQAf0ABB4AAA0BKxJvTQA0EAoMIAAScvoAD18ABwH2AApfADQPCgs_AAI-AAcRIQHAABYBBZAABdYA1KMByjDyAXNlcnZpY2XVAAMPAACnAERuCoICkwADDgAB4t0jYEoQARJFSK4S8AIAowEFOQACnQAGHQAAOQEIRzQAIQRVXwMCHwQByB6fkNmj0NC6popBvxMCIPIHDQdCogESBh8EA3gGQQgOAIIzKhAO-Sgik9BzBhQKSjEgV7BJA3E6ABmLOgA4OQ9gKdCwDgBzVQ1AEAoMZl4DoLzQuD4AEL5cAEOrBxF5lwAHEQfY4__2qtGk64MoClIqRZAL8AQzAAH7DBJFEigSDtCy0LjQvdC0fwAh0YGpBwDxAw_DKgWvrNbfha7H8saFAacIAoj6DxLCAhL6AVgAQdGD0LfzAAFqKxKbYwfxBCoSCi4AHxAsAAERviwAD0QAAIy3CPAHFAoQXAAPFgABL9GDFgACH4sWAAGM0PQAQBgKFFh4C2HQuIoAAn8tAVASKPwAAl4bAVgVHPn_APkA8omD97a-ic6qAQqCAir1tgXW4wAB_xIS5AQSxgQSEgQBAdQKAEUQEQCwACQaADcXMjIAjxgHMBAAMp0AQIUYAAIZBwAHAECL0LUYJCwgtRj7CVDQvDAAAZ4No48YAAEh0LAYAHsSD_MWxgACZAA7FAoQGAAA2gAOFgAfsGAAAhG-2AA6EgoORAAuEAE2AbIHckYAAw9oAQRVHHEyAAEATAAPUADyC744AQ6yARO-sgEN2AAv0YsgAQgfuWQAAgEYBgEc9QYB2ODPj6GB5K3DQApMKkAFArIuAAH2BxI2Eh4SCTcrE3OvKjANChHiBQDGAxEQTgAC6gL4AwAAAAAw-tq7zK_BqvY-CnIqZk4AEVSGKx-BhisaASkWQRIoEhsBAjEIOAB9BxwrdAC_0omXtNqDmvtTCkcXLwZB0YXRgL0CDxcvAgO9AOmOm_CVjYH8lQMKrwEqohACEZD4DqJwEkESBnh0cmVtIxVgCwoHDgBjAgUwCgob9w4ALgSQGQA2JxAADgBpUQ2AEhhDAAc2AEMtAUVDAPcS8wT3J3QSWhILcGVyZm9ybWFuY2UYACACKhEKDRMAAFUACxMAAXYANxAKDCYAAKIAKA8KSwACcQAIEvsAAVkEn4-3-KiRx_W-bxIwCE_RhdC_-wAO8BHs9vjQu-y-q9sBEpIPCAsQABoQCgYqBBCszRkSAggAGgQA0A8KBSoDEOYKEgIIARoEAAIRAHCQBxICCAIaBAACEQBwhgISAggDGgQAAhEAcKweEgIIBBoEAAIRAHDEYBICCAUaBAAHEQAgBhoEAAJ4AIDorgMSAggHGgQAAiMAEJoRACAIGgQAAhEAcOosEgIICRoEAAI0AICuzQESAggKGgQAgB0KEwgBEAEhlQLhACCMQCoEEKKfIxICCAvLADoeChQfAFAFEPyhzz8AEQwgAAI_AJAQIXyXoXOSSYo_AHHa-RQSAggNHwACPwCQCCFY7o9JfJaNPwAwtsirPwARDiAAMR8KFT8ABX4AYAYQuJjikMIAEQ8hAAJgAJABIV24LWmnAY9gACDO2EAAERBZAQQfAHDuyi08B_aOHwAgsp8fABkRHwBhB0ZK1hsOPgB5lqYEEgIIEh8AYc2MfSSf8z4AEcBdABkTHwBnMTCaOAjUHwAZFB8AcBUHTKcN9I18AHmA3g0SAggVHwBhZmEqRwHLPgAg8uObABkWHwBhMKIgeX_1PgB55NczEgIIFx8AYawhD5xn9D4Ad9DNEhICCBgfAIEoIWSP8_0lhh8AcZDnGBICCBkfABEgtgHzDBAhbBwBtIyUQsAqBRC284YREgIIGhoECAEQAjkBkCEhxLnuH70Yi70AcYrGNxICCBugAgIfAIEBIZNP1el_OjkBBncBBh8AYTE2JtPjLh8AIMTB3AAZHD4AYXDHgNvQJJ4AcdacbBICCB0fAAIVAvABASHyO1hVremOQCoFEMzz6DoBGR4gAHASbDrXX7yLIACBjsaYZBICCB8gAAI1ApABIbYJAsO4x441AkDQ5ZaLTQMRICEABb4AUSGu6wMgXAEgvIo9ARkhHwBhMMdbmuHv8wIgnO2tAxEiHwAEnwBhrl2R_tiInwAwtv25NgIZIyAAYcP9JXiQ2CAAMKSro0AAESQgAAR-AGGLK-daiRBfAHnm1k4SAgglHwBh7EZcgN4SPAEgmMdeABEm-AMEPgBhRGexjSY2HwAg-vAQAxEnHwAxHAoSkAPxAtC9FxyK4I1AKgMQ3jwSAggoHgAEPQBh9t5A8_kRPQAgvvuYAREpHwARHhcCkAEhlLRkGAhPjFkBMMCe6iAAGSogAGFpZIFpINZ5ATDix-3YBBErIAAEXwBhtIzr7boa2AFxsvNNEgIILB8ABF8AYYY1tLkswBkBMOKug18AES0gAAQ_AGHSNJus7fwZAXnqgikSAgguHwBh8NvF-JqEtgEgoIu0AhEvHwACXgCBCCGBc64oXDJLBIHmufQGEgIIMCAAAl4AgQghOBnAMVET_AAG7AMTBfECgQEh7slu6-q8vABxkMUyEgIIMTsFAj4AgQEhMPynFpOSHwB3gJBEEgIIMh8AgQQh8KgtOBekuwB3holWEgIIMx8AgQghAReekbwQfAB54IAIEgIINB8AYdq5_0ule10Acb7sJhICCDWVBQTaAGFGheYVTpfaAImsttkQEgIINiAAYeB_xxzyESAAMNrOv-8CETcgABEfwwWRECEQW242YSd7zAPFuUcSAgg4GgQICRAKNQJwB8SqrXP5jjUCARQGGTkCBmGCqTQor5WeAHfy_SESAgg6HwCBBCHoEquIE8OXAXfE2kYSAgg7HwCBCCEMRqorWdI-ACCCk30AEDwfAC0iAAIAIQkJ7QXxCADwPy0AAIA_MgYSBBCc-QcyBRIDEKgIBwAhzAYHACGGAgcAIaARBwAhkFIHACHEYDIAIriRHQAhmgMWAJCULDIFEgMQhiQAAAA%2C;wmaxone%3D1.000;wminone%3D1.000;ydngd_mean%3D0.261882;ydngi%3D0.5063;ynhg_mean%3D0.1904;zsset%3DeNptV0ty20YQvUrvsrJLkiuu5DggMCRh4ucBYIlekZJtKbFd3Lkqt6AkgoIpkroCULlQ3usBTMpOSWj09Pz687ob9CIj7by9bJbtJf4W8urk5emZjEqTFxKbUuIyKiQzNk8Tae6autk1m3YuzbZZNpumbhfNWk5fc8/p69fy%2Bx8nJ%2BL5RYjVXhCIF1rxkrQYG7xtWiYQ5fjP8R9iyblnAxl4/gQkikCKAgoNjFeADFMLPko5mw7E9xI%2BvxWgeNI0AimTwtgXeWHDiRHfeu%2BnEnh2AlKUsQTGC0iKMWicShAaCdIRnoQPjgrSc7DW5Dlo%2BA7TZTEVMxwavxATi7E2tWJy38uMmHcww1zI0PMNiMWTj2nqMByNC1DoCwLPDSPMCAzgwjKRkRcbGYVDPDaSURoFBsI0DWRkvYSUJo%2B9aChj6gx/pTIOCwljb2QkzCOuCotc3mQjeVPGmUxCOGwSJqMgjSXyBiaSCAqChgPr2alEKQ6K0mQksWdDDxRGxvB6nAbG8gVzQeCTdDgU6CRpEk0lzfBvrKdBTEtE3yv8sWS%2BZKFflLAxi7ypZDg4S8ME8ymCOUDcMpv69GSWv5K3iiBLT8G2COdak5WDKPTFhjmE1M6m0N3CIgsf5R4Pyz1olRvfGvBjD/GRPIwznJ5nxgSgqcVMYQ0XlFBU8mk8CAEOoGxYkE5B4BhSwKtICy8CLWFEAb%2BUmSCOUwH28NA0UBxzPg5xy3kY4klUECYXIMWYTrg4EyD%2BjnmCZ/ZsgERYUvCIfJgr62TNStn2Uppbld02tWPnmK6w%2BbbZN49OdNPskH5gq2at2fjFDXZ81Vh7x5NveUnzgCys2k%2Bi%2B2/dS7OSzJbHMVcrXs0LeBUFYHFNhfld%2B7n5DhEUqTiP83a4dEHRZ71v5nRAbYDwXk/ZuyP0CrWY4l3H4q4r6LXXGrGRZqWLKzIV2fbaLeRwy31k15DP%2BOaAx7qtvNytxlYM3KErVbPmmRWurpsnZ1eFGSx9wJ57%2Btzt5JD3bES12jrf11SU0x3j3OwGupa%2B1SrXLvq5rV40Y60Tnt1%2B4AEbdfCcCmzgkyUN2mhwNmRgIHxLh3/RIfZvGcXDUOPCCGho9xBDWOGuPUW6joLuim6Dns4I3D9j4dxKMVUzqlut0rWo/eobx3RBx4CwaD/xCKJv234GLNRrW5h9LYqFGUr7yrFzVXXlfK8Ch6v2S3/mToNcuU4iehsFTInNM4GzbN88AS0Oy/QVQXeYVEgo8J0rFADSQQhmPenKazK1Tt479hNMWpLdPw8vBUy3mbJYBMjj%2Bu9u%2BBUR19A%2BOf9r4F1Q1F6Ka0Jdbb7rsuVoascTsUMzX0U/wtEJ9kRjx0K%2B7dlL7L7TwzYqghfWDoZ0zwPCMlP926suPpoYD0w1CJx1auZcAbLs7lQ7iArnvLrLoEM%2Bq2BNJ3eGVopWB59fJvdyQCveXKE1Qq%2BaE/kadLLQeof0vVQNNk70USufzrssXnbugwDJhS2PdDnDThEi034UzZ1Nn/rtX245wzXrvabAg4p0yfrYuF5U/biHCnUvLdnOO4teVHUoPQwB7NvngsWva1yBgODzEdjoCtZE0WtYVREGceYhkTsV5pRd0RdMM7fxStdeK8r6BgBfXGuf6Kz9oIVo%2B4NRUOtgwVfdw%2BSD1vwb4W6tX7VWXIJGs64fcLe%2BFIJ0M5QgACcDKQI5OX15cionZ6RnZ6/k7E9%2B5/lGu2OArx2/wMcRvj/QufktVMhoIOGFTN5LPM3fRvgyyMoIbdaizeclzvQzKZOwQJs1gzhN0Km1j95ru2In09cLhdV317Rql0eO0aLPSqStSq28Zop0Vd01kco1F1c8rrBjd6iplWujD3rtXgvwUemheO9eTxqoG8Va3ybY1RGSS%2BaKtgKubD%2B6qucawE%2BiQ1F2vYGJdimaS7dqSMXKxuGG9ZhWPbbfiJ0OUVfOsO1RFi9cPX%2Bg14jwtVItwlxw33WXVS/QOXwm0BjMdBWbgr%2BBvbnW7q6g7bu0n7uavXb5evgOcI1QG%2B7BLf8j7Fer27vKq1l51/dAN3xgV9J62WnlqqVGk78wDoVyr33isRd0FRfRXR61IYXNSvSj5t4BgTnXta%2Bjoq5W37sq476Tbg7l4qoP3k/lHpXM2VO7zdddI9zpd077VdnnrRhlY6eAa79BlRsXiidNwJVEU4uP4UnIX1exF4XJRJr1v/8wHtKdsDoCZ/uNv2HM5D%2Bak0fD;relevgeo%3D225;geochanged%3D1;;udbts%3D1459797057;downerprotobundle%3DCAwVQbwCVw%2C%2C&reqid=1461253060957657-600715567702129748010289-myt1-1909&snip=exps%3Dmarker_ratings_info%2Cmarker_movie_rating%3D0;report%3Dwww;reps%3Dall_on;snip_width%3D550;uil%3Dru;dqsigml%3D4&snip=t%3D0&text=%28autocad%3A%3A209750+^+autodesk%3A%3A288721+^+%28%28autodesk%3A%3A288721+%26/%281+1%29+inventor%3A%3A1697918%29%29+^+%28%28auto%3A%3A28861+%26/%281+1%29+cad%3A%3A171629%29%29+^+%D0%B0%D0%B2%D1%82%D0%BE%D0%BA%D0%B0%D0%B4%3A%3A1405467+^+autoca%3A%3A420234780%29+%26%26/%28-3+5%29+%28%28%D1%81%D0%BA%D0%B0%D1%87%D0%B0%D1%82%D1%8C%3A%3A691+^+download%3A%3A13863+^+pdf%3A%3A10201+^+torrent%3A%3A35211+^+%28%28%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D1%82%D0%BD%D0%BE%3A%3A456+%26/%281+1%29+%D0%B7%D0%B0%D0%B3%D1%80%D1%83%D0%B7%D0%B8%D1%82%D1%8C%3A%3A13856%29%29+^+%D0%B7%D0%B0%D0%B3%D1%80%D1%83%D0%B6%D0%B0%D1%82%D1%8C%3A%3A13856+^+%D0%B7%D0%B0%D0%BA%D0%B0%D1%87%D0%B8%D0%B2%D0%B0%D1%82%D1%8C%3A%3A112512+^+%D1%81%D0%BA%D0%B0%D1%87%D0%B8%D0%B2%D0%B0%D0%BD%D0%B8%D0%B5%3A%3A14585+^+%D1%81%D0%BA%D0%B0%D1%87%D0%B8%D0%B2%D0%B0%D1%82%D1%8C%D1%81%D1%8F%3A%3A423410+^+%D1%81%D0%BA%D0%B0%D1%87%D0%BA%D0%B0%3A%3A152424+^+skachat%3A%3A203208%29+%26/%28-64+64%29+%28%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D1%82%D0%BD%D0%BE%3A%3A456+^+besplatno%3A%3A455045+^+download%3A%3A13863+^+free%3A%3A12386+^+%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%3A%3A886571+^+%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D1%82%D0%BD%D0%B8%D0%B9%3A%3A14490854+^+%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D1%82%D0%BD%D0%B8%D0%BA%3A%3A105058695+^+%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D1%82%D0%BD%D0%BD%D1%8B%D0%B9%3A%3A280156520+^+%D0%B4%D0%B0%D1%80%D0%BE%D0%BC%3A%3A148126+^+%D1%81%D0%B2%D0%BE%D0%B1%D0%BE%D0%B4%D0%BD%D0%BE%3A%3A88910+^+%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D0%BD%D0%BE%3A%3A4669275+^+%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D1%82%D0%BE%3A%3A10775250+^+%D1%85%D0%B0%D0%BB%D1%8F%D0%B2%D0%BD%D1%8B%D0%B9%3A%3A644531%29%29+^+%D1%81%D0%BA%D0%B0%D1%87%D0%B0%D1%82%D1%8C%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D1%82%D0%BD%D0%BE%3A%3A17882331+%26/%28-64+64%29+%D0%BD%D0%B0%3A%3A131+%26/%28-64+64%29+%28%D1%80%D1%83%D1%81%D1%81%D0%BA%D0%BE%D0%BC%3A%3A1942+^+russia%3A%3A37324+^+russian%3A%3A31805+^+%D1%80%D0%BE%D1%81%D1%81%D0%B8%D0%B9%D1%81%D0%BA%D0%B8%D0%B9%3A%3A3887+^+rus%3A%3A16095+^+%D0%B2%D0%B5%D0%BB%D0%B8%D0%BA%D0%BE%D1%80%D1%83%D1%81%D1%81%D0%BA%D0%B8%D0%B9%3A%3A1918880+^+%D1%80%D1%83%D1%81%D0%B8%D1%81%D1%82%D0%B8%D0%BA%D0%B0%3A%3A27111921+^+%D1%80%D1%83%D1%81%D0%B8%D1%84%D0%B8%D0%BA%D0%B0%D1%86%D0%B8%D1%8F%3A%3A638169+^+!!%D1%80%D1%83%D1%81%D0%BA%3A%3A1076145+^+%D1%80%D1%83%D1%81%D1%81%3A%3A336053+^+%D1%80%D1%83%D1%81%D1%81%D0%BA%D0%BE%D1%8F%D0%B7%D1%8B%D1%87%D0%BD%D1%8B%D0%B9%3A%3A139984+^+russkom%3A%3A7245427%29+%26%26/%28-3+5%29+%28%D1%82%D0%BE%D1%80%D1%80%D0%B5%D0%BD%D1%82%3A%3A6178+^+torrent%3A%3A35211%29+%26%26/%28-3+5%29+32%3A%3A6178+%26/%28-1+1%29+%28bit%3A%3A27572+^+%D0%B1%D0%B8%D1%82%D0%BD%D1%8B%D0%B9%3A%3A414024+^+%D1%83%D0%BA%D1%83%D1%81%D0%B8%D1%82%D1%8C%3A%3A558080+^+%28%28background%3A%3A96185+%26/%281+1%29+intelligent%3A%3A705091+%26/%281+1%29+transfer%3A%3A180746+%26/%281+1%29+service%3A%3A17789%29%29+^+%D0%B1%D0%B8%D1%82%3A%3A65584%29+%26%26/%28-3+5%29+%D0%B4%D0%BB%D1%8F%3A%3A205+%26/%28-64+64%29+%28%28windows%3A%3A2869+^+%D0%B2%D0%B8%D0%BD%D0%B4%D0%BE%D0%B2%D1%81%3A%3A318239+^+%D0%B2%D0%B8%D0%BD%D0%B4%D0%BE%D1%83%D0%B7%3A%3A17509782+^+%D0%B2%D0%B8%D0%BD%D0%B4%D0%BE%D1%83%D1%81%3A%3A2618285%29+%26%26/%28-3+5%29+%28xp%3A%3A13143+^+windows%3A%3A2869+^+%D1%85%D1%80%3A%3A278393+^+%28%28xtreme%3A%3A579234+%26/%281+1%29+performance%3A%3A113530%29%29+^+%D1%85%D0%BF%3A%3A582849%29%29+^+windowsxp%3A%3A585285+softness%3A6&tld=ru&uil=ru&user_request=autocad+%D1%81%D0%BA%D0%B0%D1%87%D0%B0%D1%82%D1%8C+%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D1%82%D0%BD%D0%BE+%D0%BD%D0%B0+%D1%80%D1%83%D1%81%D1%81%D0%BA%D0%BE%D0%BC+%D1%82%D0%BE%D1%80%D1%80%D0%B5%D0%BD%D1%82+32+bit+%D0%B4%D0%BB%D1%8F+windows+xp
\ No newline at end of file diff --git a/library/cpp/streams/lz/ut/ya.make b/library/cpp/streams/lz/ut/ya.make new file mode 100644 index 0000000000..18288c8ac9 --- /dev/null +++ b/library/cpp/streams/lz/ut/ya.make @@ -0,0 +1,18 @@ +UNITTEST_FOR(library/cpp/streams/lz) + +OWNER( + pg + g:util +) + +RESOURCE( + random.data /random.data + request.data /request.data + yq_609.data /yq_609.data +) + +SRCS( + lz_ut.cpp +) + +END() diff --git a/library/cpp/streams/lz/ut/yq_609.data b/library/cpp/streams/lz/ut/yq_609.data Binary files differnew file mode 100644 index 0000000000..9694b0c97f --- /dev/null +++ b/library/cpp/streams/lz/ut/yq_609.data diff --git a/library/cpp/streams/lz/ya.make b/library/cpp/streams/lz/ya.make new file mode 100644 index 0000000000..e5eea0b096 --- /dev/null +++ b/library/cpp/streams/lz/ya.make @@ -0,0 +1,20 @@ +LIBRARY() + +OWNER( + pg + g:util +) + +PEERDIR( + contrib/libs/fastlz + contrib/libs/lz4 + contrib/libs/minilzo + contrib/libs/quicklz + contrib/libs/snappy +) + +SRCS( + lz.cpp +) + +END() diff --git a/library/cpp/streams/lzma/lzma.cpp b/library/cpp/streams/lzma/lzma.cpp new file mode 100644 index 0000000000..f1942fa546 --- /dev/null +++ b/library/cpp/streams/lzma/lzma.cpp @@ -0,0 +1,520 @@ +#include "lzma.h" + +#include <util/stream/mem.h> +#include <util/system/context.h> +#include <util/generic/cast.h> +#include <util/memory/addstorage.h> +#include <util/generic/ptr.h> +#include <util/generic/intrlist.h> +#include <util/generic/scope.h> + +extern "C" { +#include <contrib/libs/lzmasdk/LzmaEnc.h> +#include <contrib/libs/lzmasdk/LzmaDec.h> +} + +namespace { + class TMemoryGc { + class TAllocation: public TIntrusiveListItem<TAllocation>, public TAdditionalStorage<TAllocation> { + }; + + public: + inline void* Allocate(size_t len) { + if (len > 1024 * 1024 * 1024) { + return nullptr; + } + + TAllocation* ret = new (len) TAllocation; + + Allocs_.PushBack(ret); + + return ret->AdditionalData(); + } + + inline void Deallocate(void* ptr) noexcept { + if (ptr) { + delete TAllocation::ObjectFromData(ptr); + } + } + + private: + TIntrusiveListWithAutoDelete<TAllocation, TDelete> Allocs_; + }; + + template <class T> + class TInverseFilter { + class TTrampoLine: public ITrampoLine { + public: + inline TTrampoLine(TInverseFilter* parent) + : Parent_(parent) + { + } + + void DoRun() override { + Parent_->RunFilter(); + } + + private: + TInverseFilter* Parent_; + }; + + class TInput: public IInputStream { + public: + inline TInput(TInverseFilter* parent) + : Parent_(parent) + { + } + + ~TInput() override { + } + + size_t DoRead(void* ptr, size_t len) override { + return Parent_->ReadImpl(ptr, len); + } + + private: + TInverseFilter* Parent_; + }; + + class TOutput: public IOutputStream { + public: + inline TOutput(TInverseFilter* parent) + : Parent_(parent) + { + } + + ~TOutput() override { + } + + void DoWrite(const void* ptr, size_t len) override { + Parent_->WriteImpl(ptr, len); + } + + private: + TInverseFilter* Parent_; + }; + + public: + inline TInverseFilter(IOutputStream* slave, T* filter) + : Slave_(slave) + , Filter_(filter) + , TrampoLine_(this) + , FilterCtx_(FilterClosure()) + , Finished_(false) + , In_(nullptr, 0) + { + } + + virtual ~TInverseFilter() { + if (!UncaughtException()) { + try { + Finish(); + } catch (...) { + } + } else { + //rely on gc + } + } + + inline void Write(const void* ptr, size_t len) { + In_.Reset(ptr, len); + + Y_DEFER { + In_.Reset(0, 0); + }; + + while (In_.Avail()) { + SwitchTo(); + } + } + + inline void Finish() { + if (!Finished_) { + Finished_ = true; + SwitchTo(); + } + } + + private: + inline void RunFilter() { + try { + TInput in(this); + TOutput out(this); + + (*Filter_)(&in, &out); + } catch (...) { + Err_ = std::current_exception(); + } + + SwitchFrom(); + } + + inline TContClosure FilterClosure() { + return {&TrampoLine_, TArrayRef(Stack_, sizeof(Stack_))}; + } + + inline size_t ReadImpl(void* ptr, size_t len) { + while (!Finished_) { + const size_t ret = In_.Read(ptr, len); + + if (ret) { + return ret; + } + + SwitchFrom(); + } + + return 0; + } + + inline void WriteImpl(const void* ptr, size_t len) { + Y_ASSERT(!Out_.Avail()); + + Out_.Reset(ptr, len); + + while (Out_.Avail()) { + SwitchFrom(); + } + } + + inline bool FlushImpl() { + if (Out_.Avail()) { + TransferData(&Out_, Slave_); + Out_.Reset(nullptr, 0); + + return true; + } + + return false; + } + + inline void SwitchTo() { + do { + CurrentCtx_.SwitchTo(&FilterCtx_); + + if (Err_) { + Finished_ = true; + + std::rethrow_exception(Err_); + } + } while (FlushImpl()); + } + + inline void SwitchFrom() { + FilterCtx_.SwitchTo(&CurrentCtx_); + } + + private: + IOutputStream* Slave_; + T* Filter_; + TTrampoLine TrampoLine_; + char Stack_[16 * 1024]; + TContMachineContext FilterCtx_; + TContMachineContext CurrentCtx_; + bool Finished_; + TMemoryInput In_; + TMemoryInput Out_; + std::exception_ptr Err_; + }; + + class TLzma { + public: + class TLzmaInput: public ISeqInStream { + public: + inline TLzmaInput(IInputStream* slave) + : Slave_(slave) + { + Read = ReadFunc; + } + + private: + static inline SRes ReadFunc(const ISeqInStream* p, void* ptr, size_t* len) { + *len = const_cast<TLzmaInput*>(static_cast<const TLzmaInput*>(p))->Slave_->Read(ptr, *len); + + return SZ_OK; + } + + private: + IInputStream* Slave_; + }; + + class TLzmaOutput: public ISeqOutStream { + public: + inline TLzmaOutput(IOutputStream* slave) + : Slave_(slave) + { + Write = WriteFunc; + } + + private: + static inline size_t WriteFunc(const ISeqOutStream* p, const void* ptr, size_t len) { + const_cast<TLzmaOutput*>(static_cast<const TLzmaOutput*>(p))->Slave_->Write(ptr, len); + + return len; + } + + private: + IOutputStream* Slave_; + }; + + class TAlloc: public ISzAlloc { + public: + inline TAlloc() { + Alloc = AllocFunc; + Free = FreeFunc; + } + + private: + static void* AllocFunc(const ISzAlloc* t, size_t len) { + return static_cast<TAlloc*>(((ISzAlloc*)t))->Gc_.Allocate(len); + } + + static void FreeFunc(const ISzAlloc* t, void* p) { + static_cast<TAlloc*>(((ISzAlloc*)t))->Gc_.Deallocate(p); + } + + private: + TMemoryGc Gc_; + }; + + inline ISzAlloc* Alloc() noexcept { + return &Alloc_; + } + + static inline void Check(SRes r) { + if (r != SZ_OK) { + ythrow yexception() << "lzma error(" << r << ")"; + } + } + + private: + TAlloc Alloc_; + }; + + class TLzmaCompressBase: public TLzma { + public: + inline TLzmaCompressBase(size_t level) + : H_(LzmaEnc_Create(Alloc())) + { + if (!H_) { + ythrow yexception() << "can not init lzma engine"; + } + + LzmaEncProps_Init(&Props_); + + Props_.level = level; + Props_.dictSize = 0; + Props_.lc = -1; + Props_.lp = -1; + Props_.pb = -1; + Props_.fb = -1; + Props_.numThreads = -1; + Props_.writeEndMark = 1; + + Check(LzmaEnc_SetProps(H_, &Props_)); + size_t bufLen = sizeof(PropsBuf_); + Zero(PropsBuf_); + Check(LzmaEnc_WriteProperties(H_, PropsBuf_, &bufLen)); + } + + inline ~TLzmaCompressBase() { + LzmaEnc_Destroy(H_, Alloc(), Alloc()); + } + + inline void operator()(IInputStream* in, IOutputStream* out) { + TLzmaInput input(in); + TLzmaOutput output(out); + + out->Write(PropsBuf_, sizeof(PropsBuf_)); + + Check(LzmaEnc_Encode(H_, &output, &input, nullptr, Alloc(), Alloc())); + } + + private: + CLzmaEncHandle H_; + CLzmaEncProps Props_; + Byte PropsBuf_[LZMA_PROPS_SIZE]; + }; +} + +class TLzmaCompress::TImpl: public TLzmaCompressBase, public TInverseFilter<TLzmaCompressBase> { +public: + inline TImpl(IOutputStream* slave, size_t level) + : TLzmaCompressBase(level) + , TInverseFilter<TLzmaCompressBase>(slave, this) + { + } +}; + +class TLzmaDecompress::TImpl: public TLzma { +public: + inline TImpl() + : InBegin_(nullptr) + , InEnd_(nullptr) + { + LzmaDec_Construct(&H_); + } + inline virtual ~TImpl() { + LzmaDec_Free(&H_, Alloc()); + } + + inline size_t Read(void* ptr, size_t len) { + Byte* pos = (Byte*)ptr; + Byte* end = pos + len; + + retry: + size_t availLen = InEnd_ - InBegin_; + size_t bufLen = end - pos; + ELzmaStatus status; + + Check(LzmaDec_DecodeToBuf(&H_, pos, &bufLen, (Byte*)InBegin_, &availLen, LZMA_FINISH_ANY, &status)); + + InBegin_ += availLen; + pos += bufLen; + + if (status == LZMA_STATUS_NEEDS_MORE_INPUT) { + Y_ASSERT(InEnd_ == InBegin_); + if (!Fill()) { + ythrow yexception() << "incomplete lzma stream"; + } + + goto retry; + } + + return pos - (Byte*)ptr; + } + +private: + virtual bool Fill() = 0; + +protected: + CLzmaDec H_; + char* InBegin_; + char* InEnd_; +}; + +class TLzmaDecompress::TImplStream: public TImpl { +public: + inline TImplStream(IInputStream* slave) + : Slave_(slave) + { + Byte buf[LZMA_PROPS_SIZE]; + + if (Slave_->Load(buf, sizeof(buf)) != sizeof(buf)) + ythrow yexception() << "can't read lzma header"; + + Check(LzmaDec_Allocate(&H_, buf, sizeof(buf), Alloc())); + LzmaDec_Init(&H_); + } + +private: + bool Fill() override { + size_t size = Slave_->Read(In_, sizeof(In_)); + InBegin_ = In_; + InEnd_ = In_ + size; + + return size; + } + +private: + IInputStream* Slave_; + char In_[4096]; +}; + +class TLzmaDecompress::TImplZeroCopy: public TLzmaDecompress::TImpl { +public: + inline TImplZeroCopy(IZeroCopyInput* in) + : Input_(in) + { + if (!Fill()) + ythrow yexception() << "can't read lzma header"; + + char buf[LZMA_PROPS_SIZE]; + char* header; + if (InEnd_ - InBegin_ >= LZMA_PROPS_SIZE) { + header = InBegin_; + InBegin_ += LZMA_PROPS_SIZE; + } else { + //bad luck, first part is less than header + //try to copy header part by part to the local buffer + const char* end = buf + sizeof(buf); + char* pos = buf; + while (1) { + size_t left = end - pos; + size_t avail = InEnd_ - InBegin_; + if (left < avail) { + memcpy(pos, InBegin_, left); + InBegin_ += left; + break; + } else { + memcpy(pos, InBegin_, avail); + pos += avail; + if (!Fill()) { + ythrow yexception() << "can't read lzma header"; + } + } + } + header = buf; + } + + Check(LzmaDec_Allocate(&H_, (Byte*)header, LZMA_PROPS_SIZE, Alloc())); + + LzmaDec_Init(&H_); + } + +private: + bool Fill() override { + size_t size = Input_->Next(&InBegin_); + + if (size) { + InEnd_ = InBegin_ + size; + + return true; + } + + return false; + } + + IZeroCopyInput* Input_; +}; + +TLzmaCompress::TLzmaCompress(IOutputStream* slave, size_t level) + : Impl_(new TImpl(slave, level)) +{ +} + +TLzmaCompress::~TLzmaCompress() { +} + +void TLzmaCompress::DoWrite(const void* buf, size_t len) { + if (!Impl_) { + ythrow yexception() << "can not write to finished lzma stream"; + } + + Impl_->Write(buf, len); +} + +void TLzmaCompress::DoFinish() { + THolder<TImpl> impl(Impl_.Release()); + + if (impl) { + impl->Finish(); + } +} + +TLzmaDecompress::TLzmaDecompress(IInputStream* slave) + : Impl_(new TImplStream(slave)) +{ +} + +TLzmaDecompress::TLzmaDecompress(IZeroCopyInput* input) + : Impl_(new TImplZeroCopy(input)) +{ +} + +TLzmaDecompress::~TLzmaDecompress() { +} + +size_t TLzmaDecompress::DoRead(void* buf, size_t len) { + return Impl_->Read(buf, len); +} diff --git a/library/cpp/streams/lzma/lzma.h b/library/cpp/streams/lzma/lzma.h new file mode 100644 index 0000000000..ca1e06e9ef --- /dev/null +++ b/library/cpp/streams/lzma/lzma.h @@ -0,0 +1,37 @@ +#pragma once + +#include <util/stream/input.h> +#include <util/stream/output.h> +#include <util/stream/zerocopy.h> + +#include <util/generic/ptr.h> + +class TLzmaCompress: public IOutputStream { +public: + TLzmaCompress(IOutputStream* slave, size_t level = 7); + ~TLzmaCompress() override; + +private: + void DoWrite(const void* buf, size_t len) override; + void DoFinish() override; + +private: + class TImpl; + THolder<TImpl> Impl_; +}; + +class TLzmaDecompress: public IInputStream { +public: + TLzmaDecompress(IInputStream* slave); + TLzmaDecompress(IZeroCopyInput* input); + ~TLzmaDecompress() override; + +private: + size_t DoRead(void* buf, size_t len) override; + +private: + class TImpl; + class TImplStream; + class TImplZeroCopy; + THolder<TImpl> Impl_; +}; diff --git a/library/cpp/streams/lzma/lzma_ut.cpp b/library/cpp/streams/lzma/lzma_ut.cpp new file mode 100644 index 0000000000..847e98d1ca --- /dev/null +++ b/library/cpp/streams/lzma/lzma_ut.cpp @@ -0,0 +1,127 @@ +#include "lzma.h" + +#include <library/cpp/testing/unittest/registar.h> + +#include <util/stream/mem.h> +#include <util/random/fast.h> +#include <util/random/random.h> + +class TStrokaByOneByte: public IZeroCopyInput { +public: + TStrokaByOneByte(const TString& s) + : Data(s) + , Pos(s.data()) + { + } + +private: + size_t DoNext(const void** ptr, size_t len) override { + if (Pos < Data.end()) { + len = Min(len, static_cast<size_t>(1)); + *ptr = Pos; + Pos += len; + return len; + } else { + return 0; + } + } + + TString Data; + const char* Pos; +}; + +class TLzmaTest: public TTestBase { + UNIT_TEST_SUITE(TLzmaTest); + UNIT_TEST(Test1) + UNIT_TEST(Test2) + UNIT_TEST_SUITE_END(); + +private: + inline TString GenData() { + TString data; + TReallyFastRng32 rnd(RandomNumber<ui64>()); + + for (size_t i = 0; i < 50000; ++i) { + const char ch = rnd.Uniform(256); + const size_t len = 1 + rnd.Uniform(10); + + data += TString(len, ch); + } + + return data; + } + + inline void Test2() { + class TExcOutput: public IOutputStream { + public: + ~TExcOutput() override { + } + + void DoWrite(const void*, size_t) override { + throw 12345; + } + }; + + TString data(GenData()); + TMemoryInput mi(data.data(), data.size()); + TExcOutput out; + + try { + TLzmaCompress c(&out); + + TransferData(&mi, &c); + } catch (int i) { + UNIT_ASSERT_EQUAL(i, 12345); + } + } + + inline void Test1() { + TString data(GenData()); + TString data1; + TString res; + + { + TMemoryInput mi(data.data(), data.size()); + TStringOutput so(res); + TLzmaCompress c(&so); + + TransferData(&mi, &c); + + c.Finish(); + } + + { + TMemoryInput mi(res.data(), res.size()); + TStringOutput so(data1); + TLzmaDecompress d((IInputStream*)&mi); + + TransferData(&d, &so); + } + + UNIT_ASSERT_EQUAL(data, data1); + + data1.clear(); + { + TMemoryInput mi(res.data(), res.size()); + TStringOutput so(data1); + TLzmaDecompress d(&mi); + + TransferData(&d, &so); + } + + UNIT_ASSERT_EQUAL(data, data1); + + data1.clear(); + { + TStrokaByOneByte mi(res); + TStringOutput so(data1); + TLzmaDecompress d(&mi); + + TransferData(&d, &so); + } + + UNIT_ASSERT_EQUAL(data, data1); + } +}; + +UNIT_TEST_SUITE_REGISTRATION(TLzmaTest); diff --git a/library/cpp/streams/lzma/ut/ya.make b/library/cpp/streams/lzma/ut/ya.make new file mode 100644 index 0000000000..01624f0259 --- /dev/null +++ b/library/cpp/streams/lzma/ut/ya.make @@ -0,0 +1,12 @@ +UNITTEST_FOR(library/cpp/streams/lzma) + +OWNER( + pg + g:util +) + +SRCS( + lzma_ut.cpp +) + +END() diff --git a/library/cpp/streams/lzma/ya.make b/library/cpp/streams/lzma/ya.make new file mode 100644 index 0000000000..38c05145c4 --- /dev/null +++ b/library/cpp/streams/lzma/ya.make @@ -0,0 +1,16 @@ +LIBRARY() + +OWNER( + pg + g:util +) + +PEERDIR( + contrib/libs/lzmasdk +) + +SRCS( + lzma.cpp +) + +END() diff --git a/library/cpp/streams/ya.make b/library/cpp/streams/ya.make new file mode 100644 index 0000000000..7426a874ee --- /dev/null +++ b/library/cpp/streams/ya.make @@ -0,0 +1,26 @@ +RECURSE( + brotli + brotli/ut + base64 + brotli + bzip2 + bzip2/ut + factory + factory/ut + fields_io + fields_io/ut + growing_file_input + growing_file_input/ut + lz + lz/ut + lzma + lzma/ut + lzop + lzop/ut + special + special/ut + xz + zc_memory_input + zstd + zstd/ut +) diff --git a/library/cpp/streams/zc_memory_input/ya.make b/library/cpp/streams/zc_memory_input/ya.make new file mode 100644 index 0000000000..bc94d6f1ed --- /dev/null +++ b/library/cpp/streams/zc_memory_input/ya.make @@ -0,0 +1,12 @@ +LIBRARY() + +OWNER( + pg + g:util +) + +SRCS( + zc_memory_input.cpp +) + +END() diff --git a/library/cpp/streams/zc_memory_input/zc_memory_input.cpp b/library/cpp/streams/zc_memory_input/zc_memory_input.cpp new file mode 100644 index 0000000000..682099a239 --- /dev/null +++ b/library/cpp/streams/zc_memory_input/zc_memory_input.cpp @@ -0,0 +1 @@ +#include "zc_memory_input.h" diff --git a/library/cpp/streams/zc_memory_input/zc_memory_input.h b/library/cpp/streams/zc_memory_input/zc_memory_input.h new file mode 100644 index 0000000000..c939d8e426 --- /dev/null +++ b/library/cpp/streams/zc_memory_input/zc_memory_input.h @@ -0,0 +1,48 @@ +#pragma once + +#include <util/stream/mem.h> +#include <util/system/defaults.h> +#include <util/generic/yexception.h> + +/// Zero-copy memory input with fixed read +class TZCMemoryInput: public TMemoryInput { +public: + TZCMemoryInput() { + } + + TZCMemoryInput(const char* dataPtr, size_t size) + : TMemoryInput(dataPtr, size) + { + } + + TZCMemoryInput(TMemoryInput& rhs) + : TMemoryInput(rhs.Buf(), rhs.Avail()) + { + } + + /// if there's 'size' data read it, otherwise just return false + Y_FORCE_INLINE bool ReadFixed(const char*& buf, size_t size) { + if (Avail() >= size) { + buf = Buf(); + Reset(Buf() + size, Avail() - size); + return true; + } + return false; + } + + template <class T> + Y_FORCE_INLINE T LoadPOD() { + const char* buf = nullptr; + if (!ReadFixed(buf, sizeof(T))) + ythrow yexception() << "TZCMemoryInput::LoadPOD failed: not enough data (" + << Avail() << " of " << sizeof(T) << " bytes)"; + T res; + memcpy(&res, buf, sizeof(T)); + return res; + } + + template <class T> + Y_FORCE_INLINE void ReadPOD(T& x) { + x = LoadPOD<T>(); + } +}; diff --git a/library/cpp/streams/zstd/ut/ya.make b/library/cpp/streams/zstd/ut/ya.make new file mode 100644 index 0000000000..1b98f0ad5e --- /dev/null +++ b/library/cpp/streams/zstd/ut/ya.make @@ -0,0 +1,12 @@ +UNITTEST_FOR(library/cpp/streams/zstd) + +OWNER( + bulatman + g:util +) + +SRCS( + zstd_ut.cpp +) + +END() diff --git a/library/cpp/streams/zstd/ya.make b/library/cpp/streams/zstd/ya.make new file mode 100644 index 0000000000..c284deeeff --- /dev/null +++ b/library/cpp/streams/zstd/ya.make @@ -0,0 +1,16 @@ +LIBRARY() + +OWNER( + bulatman + g:util +) + +PEERDIR( + contrib/libs/zstd +) + +SRCS( + zstd.cpp +) + +END() diff --git a/library/cpp/streams/zstd/zstd.cpp b/library/cpp/streams/zstd/zstd.cpp new file mode 100644 index 0000000000..29816f6d4c --- /dev/null +++ b/library/cpp/streams/zstd/zstd.cpp @@ -0,0 +1,173 @@ +#include "zstd.h" + +#include <util/generic/buffer.h> +#include <util/generic/yexception.h> + +#define ZSTD_STATIC_LINKING_ONLY +#include <contrib/libs/zstd/include/zstd.h> + +namespace { + inline void CheckError(const char* op, size_t code) { + if (::ZSTD_isError(code)) { + ythrow yexception() << op << TStringBuf(" zstd error: ") << ::ZSTD_getErrorName(code); + } + } + + struct DestroyZCStream { + static void Destroy(::ZSTD_CStream* p) noexcept { + ::ZSTD_freeCStream(p); + } + }; + + struct DestroyZDStream { + static void Destroy(::ZSTD_DStream* p) noexcept { + ::ZSTD_freeDStream(p); + } + }; +} + +class TZstdCompress::TImpl { +public: + TImpl(IOutputStream* slave, int quality) + : Slave_(slave) + , ZCtx_(::ZSTD_createCStream()) + , Buffer_(::ZSTD_CStreamOutSize()) // do reserve + { + Y_ENSURE(nullptr != ZCtx_.Get(), "Failed to allocate ZSTD_CStream"); + Y_ENSURE(0 != Buffer_.Capacity(), "ZSTD_CStreamOutSize was too small"); + CheckError("init", ZSTD_initCStream(ZCtx_.Get(), quality)); + } + + void Write(const void* buffer, size_t size) { + ::ZSTD_inBuffer zIn{buffer, size, 0}; + auto zOut = OutBuffer(); + + while (0 != zIn.size) { + CheckError("compress", ::ZSTD_compressStream(ZCtx_.Get(), &zOut, &zIn)); + DoWrite(zOut); + // forget about the data we already compressed + zIn.src = static_cast<const unsigned char*>(zIn.src) + zIn.pos; + zIn.size -= zIn.pos; + zIn.pos = 0; + } + } + + void Flush() { + auto zOut = OutBuffer(); + CheckError("flush", ::ZSTD_flushStream(ZCtx_.Get(), &zOut)); + DoWrite(zOut); + } + + void Finish() { + auto zOut = OutBuffer(); + size_t returnCode; + do { + returnCode = ::ZSTD_endStream(ZCtx_.Get(), &zOut); + CheckError("finish", returnCode); + DoWrite(zOut); + } while (0 != returnCode); // zero means there is no more bytes to flush + } + +private: + ::ZSTD_outBuffer OutBuffer() { + return {Buffer_.Data(), Buffer_.Capacity(), 0}; + } + + void DoWrite(::ZSTD_outBuffer& buffer) { + Slave_->Write(buffer.dst, buffer.pos); + buffer.pos = 0; + } +private: + IOutputStream* Slave_; + THolder<::ZSTD_CStream, DestroyZCStream> ZCtx_; + TBuffer Buffer_; +}; + +TZstdCompress::TZstdCompress(IOutputStream* slave, int quality) + : Impl_(new TImpl(slave, quality)) { +} + +TZstdCompress::~TZstdCompress() { + try { + Finish(); + } catch (...) { + } +} + +void TZstdCompress::DoWrite(const void* buffer, size_t size) { + Y_ENSURE(Impl_, "Cannot use stream after finish."); + Impl_->Write(buffer, size); +} + +void TZstdCompress::DoFlush() { + Y_ENSURE(Impl_, "Cannot use stream after finish."); + Impl_->Flush(); +} + +void TZstdCompress::DoFinish() { + // Finish should be idempotent + if (Impl_) { + auto impl = std::move(Impl_); + impl->Finish(); + } +} + +//////////////////////////////////////////////////////////////////////////////// + +class TZstdDecompress::TImpl { +public: + TImpl(IInputStream* slave, size_t bufferSize) + : Slave_(slave) + , ZCtx_(::ZSTD_createDStream()) + , Buffer_(bufferSize) // do reserve + , Offset_(0) + { + Y_ENSURE(nullptr != ZCtx_.Get(), "Failed to allocate ZSTD_DStream"); + Y_ENSURE(0 != Buffer_.Capacity(), "Buffer size was too small"); + } + + size_t Read(void* buffer, size_t size) { + Y_ASSERT(size > 0); + + ::ZSTD_outBuffer zOut{buffer, size, 0}; + ::ZSTD_inBuffer zIn{Buffer_.Data(), Buffer_.Size(), Offset_}; + + size_t returnCode = 0; + while (zOut.pos != zOut.size) { + if (zIn.pos == zIn.size) { + zIn.size = Slave_->Read(Buffer_.Data(), Buffer_.Capacity()); + Buffer_.Resize(zIn.size); + zIn.pos = Offset_ = 0; + if (0 == zIn.size) { + // end of stream, need to check that there is no uncompleted blocks + Y_ENSURE(0 == returnCode, "Incomplete block"); + break; + } + } + returnCode = ::ZSTD_decompressStream(ZCtx_.Get(), &zOut, &zIn); + CheckError("decompress", returnCode); + if (0 == returnCode) { + // The frame is over, prepare to (maybe) start a new frame + ZSTD_initDStream(ZCtx_.Get()); + } + } + Offset_ = zIn.pos; + return zOut.pos; + } + +private: + IInputStream* Slave_; + THolder<::ZSTD_DStream, DestroyZDStream> ZCtx_; + TBuffer Buffer_; + size_t Offset_; +}; + +TZstdDecompress::TZstdDecompress(IInputStream* slave, size_t bufferSize) + : Impl_(new TImpl(slave, bufferSize)) { +} + +TZstdDecompress::~TZstdDecompress() = default; + +size_t TZstdDecompress::DoRead(void* buffer, size_t size) { + return Impl_->Read(buffer, size); +} diff --git a/library/cpp/streams/zstd/zstd.h b/library/cpp/streams/zstd/zstd.h new file mode 100644 index 0000000000..667a0494b7 --- /dev/null +++ b/library/cpp/streams/zstd/zstd.h @@ -0,0 +1,53 @@ +#pragma once + +#include <util/generic/ptr.h> +#include <util/stream/input.h> +#include <util/stream/output.h> + +/** + * @addtogroup Streams_Archs + * @{ + */ + +// @brief Stream to compress into zstd archive +class TZstdCompress: public IOutputStream { +public: + /** + @param slave stream to write compressed data to + @param quality, higher quality - slower but better compression. + 0 is default compression (see constant ZSTD_CLEVEL_DEFAULT(3)) + max compression is ZSTD_MAX_CLEVEL (22) + */ + explicit TZstdCompress(IOutputStream* slave, int quality = 0); + ~TZstdCompress() override; +private: + void DoWrite(const void* buffer, size_t size) override; + void DoFlush() override; + void DoFinish() override; + +public: + class TImpl; + THolder<TImpl> Impl_; +}; + +//////////////////////////////////////////////////////////////////////////////// + +// @brief Buffered stream to decompress from zstd archive +class TZstdDecompress: public IInputStream { +public: + /** + @param slave stream to read compressed data from + @param bufferSize approximate size of buffer compressed data is read in + */ + explicit TZstdDecompress(IInputStream* slave, size_t bufferSize = 8 * 1024); + ~TZstdDecompress() override; + +private: + size_t DoRead(void* buffer, size_t size) override; + +private: + class TImpl; + THolder<TImpl> Impl_; +}; + +/** @} */ diff --git a/library/cpp/streams/zstd/zstd_ut.cpp b/library/cpp/streams/zstd/zstd_ut.cpp new file mode 100644 index 0000000000..ef479fdd97 --- /dev/null +++ b/library/cpp/streams/zstd/zstd_ut.cpp @@ -0,0 +1,94 @@ +#include "zstd.h" + +#include <library/cpp/testing/unittest/registar.h> + +#include <util/random/fast.h> +#include <util/stream/null.h> +#include <util/stream/str.h> + +Y_UNIT_TEST_SUITE(TZstdTestSuite) { + TString Compress(TString data, int quality = -1) { + TString compressed; + TStringOutput output(compressed); + TZstdCompress compressStream(&output, quality); + compressStream.Write(data.data(), data.size()); + compressStream.Finish(); + output.Finish(); + return compressed; + } + + TString Decompress(TString data) { + TStringInput input(data); + TZstdDecompress decompressStream(&input); + return decompressStream.ReadAll(); + } + + void TestCase(const TString& s) { + UNIT_ASSERT_VALUES_EQUAL(s, Decompress(Compress(s, -1))); + UNIT_ASSERT_VALUES_EQUAL(s, Decompress(Compress(s, 0))); + UNIT_ASSERT_VALUES_EQUAL(s, Decompress(Compress(s, 22))); + UNIT_ASSERT_VALUES_EQUAL(s, Decompress(Compress(s, 11))); + UNIT_ASSERT_VALUES_EQUAL(s, Decompress(Compress(s, 100500))); + } + + TString GenerateRandomString(size_t size) { + TReallyFastRng32 rng(42); + TString result; + result.reserve(size + sizeof(ui64)); + while (result.size() < size) { + ui64 value = rng.GenRand64(); + result += TStringBuf(reinterpret_cast<const char*>(&value), sizeof(value)); + } + result.resize(size); + return result; + } + + Y_UNIT_TEST(TestHelloWorld) { + TestCase("hello world"); + } + + Y_UNIT_TEST(TestSeveralStreamsWithSameQuality) { + auto s1 = GenerateRandomString(1 << 15); + auto s2 = GenerateRandomString(1 << 15); + auto c1 = Compress(s1); + auto c2 = Compress(s2); + UNIT_ASSERT_VALUES_EQUAL(s1 + s2, Decompress(c1 + c2)); + } + + Y_UNIT_TEST(TestSeveralStreamsWithDifferentQuality) { + auto s1 = GenerateRandomString(1 << 15); + auto s2 = GenerateRandomString(1 << 15); + auto c1 = Compress(s1, 1); + auto c2 = Compress(s2, 2); + UNIT_ASSERT_VALUES_EQUAL(s1 + s2, Decompress(c1 + c2)); + } + + Y_UNIT_TEST(TestIncompleteStream) { + TString manyAs(64 * 1024, 'a'); + auto compressed = Compress(manyAs); + TString truncated(compressed.data(), compressed.size() - 1); + UNIT_CHECK_GENERATED_EXCEPTION(Decompress(truncated), std::exception); + } + + Y_UNIT_TEST(Test64KB) { + auto manyAs = TString(64 * 1024, 'a'); + TString str("Hello from the Matrix!@#% How are you?}{\n\t\a"); + TestCase(manyAs + str + manyAs); + } + + Y_UNIT_TEST(Test1MB) { + TestCase(GenerateRandomString(1 * 1024 * 1024)); + } + + Y_UNIT_TEST(TestEmpty) { + TestCase(""); + } + + Y_UNIT_TEST(TestWriteAfterFinish) { + TNullOutput output; + TZstdCompress compressStream(&output); + compressStream.Finish(); + UNIT_ASSERT_EXCEPTION_CONTAINS(compressStream.Write("a", 1), std::exception, "Cannot use stream after finish."); + UNIT_ASSERT_EXCEPTION_CONTAINS(compressStream.Flush(), std::exception, "Cannot use stream after finish."); + } +} |