diff options
author | qrort <qrort@yandex-team.com> | 2022-11-30 23:47:12 +0300 |
---|---|---|
committer | qrort <qrort@yandex-team.com> | 2022-11-30 23:47:12 +0300 |
commit | 22f8ae0e3f5d68b92aecccdf96c1d841a0334311 (patch) | |
tree | bffa27765faf54126ad44bcafa89fadecb7a73d7 /library/cpp/streams | |
parent | 332b99e2173f0425444abb759eebcb2fafaa9209 (diff) | |
download | ydb-22f8ae0e3f5d68b92aecccdf96c1d841a0334311.tar.gz |
validate canons without yatest_common
Diffstat (limited to 'library/cpp/streams')
22 files changed, 2542 insertions, 0 deletions
diff --git a/library/cpp/streams/base64/base64stream.cpp b/library/cpp/streams/base64/base64stream.cpp new file mode 100644 index 00000000000..b49d238829c --- /dev/null +++ b/library/cpp/streams/base64/base64stream.cpp @@ -0,0 +1 @@ +#include "base64stream.h" diff --git a/library/cpp/streams/base64/base64stream.h b/library/cpp/streams/base64/base64stream.h new file mode 100644 index 00000000000..b156c468958 --- /dev/null +++ b/library/cpp/streams/base64/base64stream.h @@ -0,0 +1,129 @@ +#pragma once + +#include <library/cpp/string_utils/base64/base64.h> + +#include <util/generic/ptr.h> +#include <util/stream/input.h> +#include <util/stream/output.h> + +class TStringDataInputStream: public IInputStream { +protected: + const char* (*GetLine)(size_t); + size_t BufferSize; + TArrayHolder<char> OutBuf; + size_t Line; + size_t LineLength; + size_t Cursor; + +private: + bool NextLine() { + Cursor = 0; + const TStringBuf str(GetLine(Line)); + if (str.empty()) { + LineLength = 0; + return false; + } + + ++Line; + + const size_t size = Base64DecodeBufSize(str.size()); + + if (BufferSize < size) { + OutBuf.Reset(new char[size]); + BufferSize = size; + } + + LineLength = Base64Decode(str, OutBuf.Get()).size(); + Y_ASSERT(LineLength <= BufferSize); + + return LineLength > 0; + } + + size_t ReadDataFromLine(char* buf, size_t len) { + size_t n = Min(len, LineLength - Cursor); + memcpy(static_cast<void*>(buf), static_cast<void*>(OutBuf.Get() + Cursor), n); + Cursor += n; + return n; + } + +protected: + size_t DoRead(void* buf, size_t len) override { + size_t readed = 0; + while (readed < len) { + readed += ReadDataFromLine(static_cast<char*>(buf) + readed, len - readed); + if (readed < len && !NextLine()) + break; + } + return readed; + } + +public: + TStringDataInputStream(const char* (*getLine)(size_t), size_t bufferSize = 0) + : GetLine(getLine) + , BufferSize(bufferSize) + , OutBuf(new char[BufferSize]) + , Line(0) + , LineLength(0) + , Cursor(0) + { + } +}; + +class TStringDataOutputStream: public IOutputStream { +public: + TStringDataOutputStream(IOutputStream* out, const int maxOutStrLen) + : mStream(out) + , MaxReadLen(Base64DecodeBufSize(maxOutStrLen)) + , BufRead(new unsigned char[MaxReadLen]) + , BufReadOffset(0) + , BufOut(new char[maxOutStrLen + 1]){}; + + ~TStringDataOutputStream() override { + try { + Finish(); + } catch (...) { + } + } + +private: + IOutputStream* mStream; + size_t MaxReadLen; + TArrayHolder<unsigned char> BufRead; + size_t BufReadOffset; + TArrayHolder<char> BufOut; + + void WriteLine() { + if (BufReadOffset > 0) { + mStream->Write(" \""); + Y_ASSERT(BufReadOffset <= MaxReadLen); + char* b = BufOut.Get(); + char* e = Base64Encode(b, BufRead.Get(), BufReadOffset); + mStream->Write(b, e - b); + mStream->Write("\",\n"); + BufReadOffset = 0; + } + } + + void DoWrite(const void* buf, size_t size) override { + size_t res = Min(MaxReadLen - BufReadOffset, size); + size_t buf_offset = 0; + while (res > 0) { + memcpy(BufRead.Get() + BufReadOffset, (const char*)buf + buf_offset, res); + BufReadOffset += res; + if (BufReadOffset < MaxReadLen) + return; + WriteLine(); + buf_offset += res; + size -= res; + res = Min(MaxReadLen - BufReadOffset, size); + } + } + + void DoFlush() override { + WriteLine(); + } + + void DoFinish() override { + DoFlush(); + } +}; diff --git a/library/cpp/streams/base64/static_data.cpp b/library/cpp/streams/base64/static_data.cpp new file mode 100644 index 00000000000..8fe346da6ce --- /dev/null +++ b/library/cpp/streams/base64/static_data.cpp @@ -0,0 +1 @@ +#include "static_data.h" diff --git a/library/cpp/streams/base64/static_data.h b/library/cpp/streams/base64/static_data.h new file mode 100644 index 00000000000..584ac182995 --- /dev/null +++ b/library/cpp/streams/base64/static_data.h @@ -0,0 +1,43 @@ +#pragma once + +#include "base64stream.h" + +#include <util/memory/blob.h> +#include <util/stream/zlib.h> +#include <util/generic/ptr.h> + +class TCompressedStaticData { +public: + using TDataFunction = const char* (*)(size_t); + using TData = TBlob; + + class TDataInputStream: public IInputStream { + public: + TDataInputStream(TDataFunction function) + : Compressed(function) + , Decompressed(&Compressed) + { + } + + private: + size_t DoRead(void* buf, size_t len) override { + return Decompressed.Read(buf, len); + } + + private: + TStringDataInputStream Compressed; + TZLibDecompress Decompressed; + }; + + TCompressedStaticData(TDataFunction function) { + TDataInputStream inp(function); + Data = TBlob::FromStream(inp); + } + + const TData& GetData() const { + return Data; + } + +private: + TData Data; +}; diff --git a/library/cpp/streams/growing_file_input/growing_file_input.cpp b/library/cpp/streams/growing_file_input/growing_file_input.cpp new file mode 100644 index 00000000000..0bbfa5ade9e --- /dev/null +++ b/library/cpp/streams/growing_file_input/growing_file_input.cpp @@ -0,0 +1,40 @@ +#include "growing_file_input.h" + +#include <util/datetime/base.h> +#include <util/generic/yexception.h> + +TGrowingFileInput::TGrowingFileInput(const TString& path) + : File_(path, OpenExisting | RdOnly | Seq) +{ + if (!File_.IsOpen()) { + ythrow TIoException() << "file " << path << " not open"; + } + + File_.Seek(0, sEnd); +} + +TGrowingFileInput::TGrowingFileInput(const TFile& file) + : File_(file) +{ + if (!File_.IsOpen()) { + ythrow TIoException() << "file (" << file.GetName() << ") not open"; + } + + File_.Seek(0, sEnd); +} + +size_t TGrowingFileInput::DoRead(void* buf, size_t len) { + for (int sleepTime = 1;;) { + size_t rr = File_.Read(buf, len); + + if (rr != 0) { + return rr; + } + + NanoSleep((ui64)sleepTime * 1000000); + + if (sleepTime < 2000) { + sleepTime <<= 1; + } + } +} diff --git a/library/cpp/streams/growing_file_input/growing_file_input.h b/library/cpp/streams/growing_file_input/growing_file_input.h new file mode 100644 index 00000000000..9054a5f3dac --- /dev/null +++ b/library/cpp/streams/growing_file_input/growing_file_input.h @@ -0,0 +1,23 @@ +#pragma once + +#include <util/stream/input.h> +#include <util/system/file.h> + +/** + * Growing file input stream. + * + * File descriptor offsets to the end of the file, when the object is created. + * + * Read function waites for reading at least one byte. + */ +class TGrowingFileInput: public IInputStream { +public: + TGrowingFileInput(const TFile& file); + TGrowingFileInput(const TString& path); + +private: + size_t DoRead(void* buf, size_t len) override; + +private: + TFile File_; +}; diff --git a/library/cpp/streams/lz/common/compressor.h b/library/cpp/streams/lz/common/compressor.h new file mode 100644 index 00000000000..ffa1b198ae7 --- /dev/null +++ b/library/cpp/streams/lz/common/compressor.h @@ -0,0 +1,349 @@ +#pragma once + +#include <util/system/yassert.h> +#include <util/system/byteorder.h> +#include <util/memory/addstorage.h> +#include <util/generic/buffer.h> +#include <util/generic/utility.h> +#include <util/generic/singleton.h> +#include <util/stream/mem.h> + +#include "error.h" + +static inline ui8 HostToLittle(ui8 t) noexcept { + return t; +} + +static inline ui8 LittleToHost(ui8 t) noexcept { + return t; +} + +struct TCommonData { + static const size_t overhead = sizeof(ui16) + sizeof(ui8); +}; + +const size_t SIGNATURE_SIZE = 4; + +template <class TCompressor, class TBase> +class TCompressorBase: public TAdditionalStorage<TCompressorBase<TCompressor, TBase>>, public TCompressor, public TCommonData { +public: + inline TCompressorBase(IOutputStream* slave, ui16 blockSize) + : Slave_(slave) + , BlockSize_(blockSize) + { + /* + * save signature + */ + static_assert(sizeof(TCompressor::signature) - 1 == SIGNATURE_SIZE, "expect sizeof(TCompressor::signature) - 1 == SIGNATURE_SIZE"); + Slave_->Write(TCompressor::signature, sizeof(TCompressor::signature) - 1); + + /* + * save version + */ + this->Save((ui32)1); + + /* + * save block size + */ + this->Save(BlockSize()); + } + + inline ~TCompressorBase() { + } + + inline void Write(const char* buf, size_t len) { + while (len) { + const ui16 toWrite = (ui16)Min<size_t>(len, this->BlockSize()); + + this->WriteBlock(buf, toWrite); + + buf += toWrite; + len -= toWrite; + } + } + + inline void Flush() { + } + + inline void Finish() { + this->Flush(); + this->WriteBlock(nullptr, 0); + } + + template <class T> + static inline void Save(T t, IOutputStream* out) { + t = HostToLittle(t); + + out->Write(&t, sizeof(t)); + } + + template <class T> + inline void Save(T t) { + Save(t, Slave_); + } + +private: + inline void* Block() const noexcept { + return this->AdditionalData(); + } + + inline ui16 BlockSize() const noexcept { + return BlockSize_; + } + + inline void WriteBlock(const void* ptr, ui16 len) { + Y_ASSERT(len <= this->BlockSize()); + + ui8 compressed = false; + + if (len) { + const size_t out = this->Compress((const char*)ptr, len, (char*)Block(), this->AdditionalDataLength()); + // catch compressor buffer overrun (e.g. SEARCH-2043) + //Y_VERIFY(out <= this->Hint(this->BlockSize())); + + if (out < len || TCompressor::SaveIncompressibleChunks()) { + compressed = true; + ptr = Block(); + len = (ui16)out; + } + } + + char tmp[overhead]; + TMemoryOutput header(tmp, sizeof(tmp)); + + this->Save(len, &header); + this->Save(compressed, &header); + + using TPart = IOutputStream::TPart; + if (ptr) { + const TPart parts[] = { + TPart(tmp, sizeof(tmp)), + TPart(ptr, len), + }; + + Slave_->Write(parts, sizeof(parts) / sizeof(*parts)); + } else { + Slave_->Write(tmp, sizeof(tmp)); + } + } + +private: + IOutputStream* Slave_; + const ui16 BlockSize_; +}; + +template <class T> +static inline T GLoad(IInputStream* input) { + T t; + + if (input->Load(&t, sizeof(t)) != sizeof(t)) { + ythrow TDecompressorError() << "stream error"; + } + + return LittleToHost(t); +} + +class TDecompressSignature { +public: + inline TDecompressSignature(IInputStream* input) { + if (input->Load(Buffer_, SIGNATURE_SIZE) != SIGNATURE_SIZE) { + ythrow TDecompressorError() << "can not load stream signature"; + } + } + + template <class TDecompressor> + inline bool Check() const { + static_assert(sizeof(TDecompressor::signature) - 1 == SIGNATURE_SIZE, "expect sizeof(TDecompressor::signature) - 1 == SIGNATURE_SIZE"); + return memcmp(TDecompressor::signature, Buffer_, SIGNATURE_SIZE) == 0; + } + +private: + char Buffer_[SIGNATURE_SIZE]; +}; + +template <class TDecompressor> +static inline IInputStream* ConsumeSignature(IInputStream* input) { + TDecompressSignature sign(input); + if (!sign.Check<TDecompressor>()) { + ythrow TDecompressorError() << "incorrect signature"; + } + return input; +} + +template <class TDecompressor> +class TDecompressorBaseImpl: public TDecompressor, public TCommonData { +public: + static inline ui32 CheckVer(ui32 v) { + if (v != 1) { + ythrow yexception() << TStringBuf("incorrect stream version: ") << v; + } + + return v; + } + + inline TDecompressorBaseImpl(IInputStream* slave) + : Slave_(slave) + , Input_(nullptr, 0) + , Eof_(false) + , Version_(CheckVer(Load<ui32>())) + , BlockSize_(Load<ui16>()) + , OutBufSize_(TDecompressor::Hint(BlockSize_)) + , Tmp_(2 * OutBufSize_) + , In_(Tmp_.Data()) + , Out_(In_ + OutBufSize_) + { + this->InitFromStream(Slave_); + } + + inline ~TDecompressorBaseImpl() { + } + + inline size_t Read(void* buf, size_t len) { + size_t ret = Input_.Read(buf, len); + + if (ret) { + return ret; + } + + if (Eof_) { + return 0; + } + + this->FillNextBlock(); + + ret = Input_.Read(buf, len); + + if (ret) { + return ret; + } + + Eof_ = true; + + return 0; + } + + inline void FillNextBlock() { + char tmp[overhead]; + + if (Slave_->Load(tmp, sizeof(tmp)) != sizeof(tmp)) { + ythrow TDecompressorError() << "can not read block header"; + } + + TMemoryInput header(tmp, sizeof(tmp)); + + const ui16 len = GLoad<ui16>(&header); + if (len > Tmp_.Capacity()) { + ythrow TDecompressorError() << "invalid len inside block header"; + } + const ui8 compressed = GLoad<ui8>(&header); + + if (compressed > 1) { + ythrow TDecompressorError() << "broken header"; + } + + if (Slave_->Load(In_, len) != len) { + ythrow TDecompressorError() << "can not read data"; + } + + if (compressed) { + const size_t ret = this->Decompress(In_, len, Out_, OutBufSize_); + + Input_.Reset(Out_, ret); + } else { + Input_.Reset(In_, len); + } + } + + template <class T> + inline T Load() { + return GLoad<T>(Slave_); + } + +protected: + IInputStream* Slave_; + TMemoryInput Input_; + bool Eof_; + const ui32 Version_; + const ui16 BlockSize_; + const size_t OutBufSize_; + TBuffer Tmp_; + char* In_; + char* Out_; +}; + +template <class TDecompressor, class TBase> +class TDecompressorBase: public TDecompressorBaseImpl<TDecompressor> { +public: + inline TDecompressorBase(IInputStream* slave) + : TDecompressorBaseImpl<TDecompressor>(ConsumeSignature<TDecompressor>(slave)) + { + } + + inline ~TDecompressorBase() { + } +}; + +#define DEF_COMPRESSOR_COMMON(rname, name) \ + rname::~rname() { \ + try { \ + Finish(); \ + } catch (...) { \ + } \ + } \ + \ + void rname::DoWrite(const void* buf, size_t len) { \ + if (!Impl_) { \ + ythrow yexception() << "can not write to finalized stream"; \ + } \ + \ + Impl_->Write((const char*)buf, len); \ + } \ + \ + void rname::DoFlush() { \ + if (!Impl_) { \ + ythrow yexception() << "can not flush finalized stream"; \ + } \ + \ + Impl_->Flush(); \ + } \ + \ + void rname::DoFinish() { \ + THolder<TImpl> impl(Impl_.Release()); \ + \ + if (impl) { \ + impl->Finish(); \ + } \ + } + +#define DEF_COMPRESSOR(rname, name) \ + class rname::TImpl: public TCompressorBase<name, TImpl> { \ + public: \ + inline TImpl(IOutputStream* out, ui16 blockSize) \ + : TCompressorBase<name, TImpl>(out, blockSize) { \ + } \ + }; \ + \ + rname::rname(IOutputStream* slave, ui16 blockSize) \ + : Impl_(new (TImpl::Hint(blockSize)) TImpl(slave, blockSize)) { \ + } \ + \ + DEF_COMPRESSOR_COMMON(rname, name) + +#define DEF_DECOMPRESSOR(rname, name) \ + class rname::TImpl: public TDecompressorBase<name, TImpl> { \ + public: \ + inline TImpl(IInputStream* in) \ + : TDecompressorBase<name, TImpl>(in) { \ + } \ + }; \ + \ + rname::rname(IInputStream* slave) \ + : Impl_(new TImpl(slave)) { \ + } \ + \ + rname::~rname() { \ + } \ + \ + size_t rname::DoRead(void* buf, size_t len) { \ + return Impl_->Read(buf, len); \ + } diff --git a/library/cpp/streams/lz/lz.cpp b/library/cpp/streams/lz/lz.cpp new file mode 100644 index 00000000000..63c6e964875 --- /dev/null +++ b/library/cpp/streams/lz/lz.cpp @@ -0,0 +1,324 @@ +#include "lz.h" + +#include <util/system/yassert.h> +#include <util/system/byteorder.h> +#include <util/memory/addstorage.h> +#include <util/generic/buffer.h> +#include <util/generic/utility.h> +#include <util/generic/singleton.h> +#include <util/generic/yexception.h> +#include <util/stream/mem.h> + +#include <library/cpp/streams/lz/common/compressor.h> + +#include <library/cpp/streams/lz/lz4/block.h> +#include <library/cpp/streams/lz/snappy/block.h> + +#include <contrib/libs/fastlz/fastlz.h> +#include <contrib/libs/quicklz/quicklz.h> +#include <contrib/libs/minilzo/minilzo.h> + +/* + * MiniLzo + */ +class TMiniLzo { + class TInit { + public: + inline TInit() { + if (lzo_init() != LZO_E_OK) { + ythrow yexception() << "can not init lzo engine"; + } + } + }; + +public: + static const char signature[]; + + inline TMiniLzo() { + Singleton<TInit>(); + } + + inline ~TMiniLzo() { + } + + static inline size_t Hint(size_t len) noexcept { + // see SEARCH-2043 and, e.g. examples at + // http://stackoverflow.com/questions/4235019/how-to-get-lzo-to-work-with-a-file-stream + return len + (len / 16) + 64 + 3; + } + + static inline bool SaveIncompressibleChunks() noexcept { + return false; + } +}; + +const char TMiniLzo::signature[] = "YLZO"; + +template <size_t N> +class TFixedArray { +public: + inline TFixedArray() noexcept { + memset(WorkMem_, 0, sizeof(WorkMem_)); + } + +protected: + char WorkMem_[N]; +}; + +class TMiniLzoCompressor: public TMiniLzo, public TFixedArray<LZO1X_MEM_COMPRESS + 1> { +public: + inline size_t Compress(const char* data, size_t len, char* ptr, size_t /*dstMaxSize*/) { + lzo_uint out = 0; + lzo1x_1_compress((const lzo_bytep)data, len, (lzo_bytep)ptr, &out, WorkMem_); + + return out; + } +}; + +class TMiniLzoDecompressor: public TMiniLzo, public TFixedArray<LZO1X_MEM_DECOMPRESS + 1> { +public: + inline size_t Decompress(const char* data, size_t len, char* ptr, size_t /*max*/) { + lzo_uint ret = 0; + + lzo1x_decompress((const lzo_bytep)data, len, (lzo_bytep)ptr, &ret, WorkMem_); + + return ret; + } + + inline void InitFromStream(IInputStream*) const noexcept { + } +}; + +DEF_COMPRESSOR(TLzoCompress, TMiniLzoCompressor) +DEF_DECOMPRESSOR(TLzoDecompress, TMiniLzoDecompressor) + +/* + * FastLZ + */ +class TFastLZ { +public: + static const char signature[]; + + static inline size_t Hint(size_t len) noexcept { + return Max<size_t>((size_t)(len * 1.06), 100); + } + + inline size_t Compress(const char* data, size_t len, char* ptr, size_t /*dstMaxSize*/) { + return fastlz_compress(data, len, ptr); + } + + inline size_t Decompress(const char* data, size_t len, char* ptr, size_t max) { + return fastlz_decompress(data, len, ptr, max); + } + + inline void InitFromStream(IInputStream*) const noexcept { + } + + static inline bool SaveIncompressibleChunks() noexcept { + return false; + } +}; + +const char TFastLZ::signature[] = "YLZF"; + +DEF_COMPRESSOR(TLzfCompress, TFastLZ) +DEF_DECOMPRESSOR(TLzfDecompress, TFastLZ) + +/* + * QuickLZ + */ +class TQuickLZBase { +public: + static const char signature[]; + + static inline size_t Hint(size_t len) noexcept { + return len + 500; + } + + inline TQuickLZBase() + : Table_(nullptr) + { + } + + inline void Init(unsigned ver, unsigned lev, unsigned mod, unsigned type) { + Table_ = LzqTable(ver, lev, mod); + + if (!Table_) { + ythrow yexception() << "unsupported lzq stream(" << ver << ", " << lev << ", " << mod << ")"; + } + + const size_t size = Table_->Setting(3) + Table_->Setting(type); + + Mem_.Reset(::operator new(size)); + memset(Mem_.Get(), 0, size); + } + + inline bool SaveIncompressibleChunks() const noexcept { + // we must save incompressible chunks "as is" + // after compressor run in streaming mode + return Table_->Setting(3); + } + +protected: + const TQuickLZMethods* Table_; + THolder<void> Mem_; +}; + +const char TQuickLZBase::signature[] = "YLZQ"; + +class TQuickLZCompress: public TQuickLZBase { +public: + inline size_t Compress(const char* data, size_t len, char* ptr, size_t /*dstMaxSize*/) { + return Table_->Compress(data, ptr, len, (char*)Mem_.Get()); + } +}; + +class TQuickLZDecompress: public TQuickLZBase { +public: + inline size_t Decompress(const char* data, size_t /*len*/, char* ptr, size_t /*max*/) { + return Table_->Decompress(data, ptr, (char*)Mem_.Get()); + } + + inline void InitFromStream(IInputStream* in) { + const ui8 ver = ::GLoad<ui8>(in); + const ui8 lev = ::GLoad<ui8>(in); + const ui8 mod = ::GLoad<ui8>(in); + + Init(ver, lev, mod, 2); + } +}; + +class TLzqCompress::TImpl: public TCompressorBase<TQuickLZCompress, TImpl> { +public: + inline TImpl(IOutputStream* out, ui16 blockSize, EVersion ver, unsigned level, EMode mode) + : TCompressorBase<TQuickLZCompress, TImpl>(out, blockSize) + { + memset(AdditionalData(), 0, AdditionalDataLength()); + + Init(ver, level, mode, 1); + + Save((ui8)ver); + Save((ui8)level); + Save((ui8)mode); + } +}; + +TLzqCompress::TLzqCompress(IOutputStream* slave, ui16 blockSize, EVersion ver, unsigned level, EMode mode) + : Impl_(new (TImpl::Hint(blockSize)) TImpl(slave, blockSize, ver, level, mode)) +{ +} + +DEF_COMPRESSOR_COMMON(TLzqCompress, TQuickLZCompress) +DEF_DECOMPRESSOR(TLzqDecompress, TQuickLZDecompress) + +namespace { + template <class T> + struct TInputHolder { + static inline T Set(T t) noexcept { + return t; + } + }; + + template <class T> + struct TInputHolder<TAutoPtr<T>> { + inline T* Set(TAutoPtr<T> v) noexcept { + V_ = v; + + return V_.Get(); + } + + TAutoPtr<T> V_; + }; + + // Decompressing input streams without signature verification + template <class TInput, class TDecompressor> + class TLzDecompressInput: public TInputHolder<TInput>, public IInputStream { + public: + inline TLzDecompressInput(TInput in) + : Impl_(this->Set(in)) + { + } + + private: + size_t DoRead(void* buf, size_t len) override { + return Impl_.Read(buf, len); + } + + private: + TDecompressorBaseImpl<TDecompressor> Impl_; + }; +} + +template <class T> +static TAutoPtr<IInputStream> TryOpenLzDecompressorX(const TDecompressSignature& s, T input) { + if (s.Check<TLZ4>()) + return new TLzDecompressInput<T, TLZ4>(input); + + if (s.Check<TSnappy>()) + return new TLzDecompressInput<T, TSnappy>(input); + + if (s.Check<TMiniLzo>()) + return new TLzDecompressInput<T, TMiniLzoDecompressor>(input); + + if (s.Check<TFastLZ>()) + return new TLzDecompressInput<T, TFastLZ>(input); + + if (s.Check<TQuickLZDecompress>()) + return new TLzDecompressInput<T, TQuickLZDecompress>(input); + + return nullptr; +} + +template <class T> +static inline TAutoPtr<IInputStream> TryOpenLzDecompressorImpl(const TStringBuf& signature, T input) { + if (signature.size() == SIGNATURE_SIZE) { + TMemoryInput mem(signature.data(), signature.size()); + TDecompressSignature s(&mem); + + return TryOpenLzDecompressorX(s, input); + } + + return nullptr; +} + +template <class T> +static inline TAutoPtr<IInputStream> TryOpenLzDecompressorImpl(T input) { + TDecompressSignature s(&*input); + + return TryOpenLzDecompressorX(s, input); +} + +template <class T> +static inline TAutoPtr<IInputStream> OpenLzDecompressorImpl(T input) { + TAutoPtr<IInputStream> ret = TryOpenLzDecompressorImpl(input); + + if (!ret) { + ythrow TDecompressorError() << "Unknown compression format"; + } + + return ret; +} + +TAutoPtr<IInputStream> OpenLzDecompressor(IInputStream* input) { + return OpenLzDecompressorImpl(input); +} + +TAutoPtr<IInputStream> TryOpenLzDecompressor(IInputStream* input) { + return TryOpenLzDecompressorImpl(input); +} + +TAutoPtr<IInputStream> TryOpenLzDecompressor(const TStringBuf& signature, IInputStream* input) { + return TryOpenLzDecompressorImpl(signature, input); +} + +TAutoPtr<IInputStream> OpenOwnedLzDecompressor(TAutoPtr<IInputStream> input) { + return OpenLzDecompressorImpl(input); +} + +TAutoPtr<IInputStream> TryOpenOwnedLzDecompressor(TAutoPtr<IInputStream> input) { + return TryOpenLzDecompressorImpl(input); +} + +TAutoPtr<IInputStream> TryOpenOwnedLzDecompressor(const TStringBuf& signature, TAutoPtr<IInputStream> input) { + return TryOpenLzDecompressorImpl(signature, input); +} diff --git a/library/cpp/streams/lz/lz4/block.h b/library/cpp/streams/lz/lz4/block.h new file mode 100644 index 00000000000..9a912c0be20 --- /dev/null +++ b/library/cpp/streams/lz/lz4/block.h @@ -0,0 +1,35 @@ +#pragma once + +#include <library/cpp/streams/lz/common/compressor.h> + +#include <contrib/libs/lz4/lz4.h> + +/* + * LZ4 + */ +class TLZ4 { +public: + static constexpr char signature[]= "LZ.4"; + + static inline size_t Hint(size_t len) noexcept { + return Max<size_t>((size_t)(len * 1.06), 100); + } + + inline size_t Compress(const char* data, size_t len, char* ptr, size_t dstMaxSize) { + return LZ4_compress_default(data, ptr, len, dstMaxSize); + } + + inline size_t Decompress(const char* data, size_t len, char* ptr, size_t max) { + int res = LZ4_decompress_safe(data, ptr, len, max); + if (res < 0) + ythrow TDecompressorError(); + return res; + } + + inline void InitFromStream(IInputStream*) const noexcept { + } + + static inline bool SaveIncompressibleChunks() noexcept { + return false; + } +}; diff --git a/library/cpp/streams/lz/lz4/lz4.cpp b/library/cpp/streams/lz/lz4/lz4.cpp new file mode 100644 index 00000000000..220e358c076 --- /dev/null +++ b/library/cpp/streams/lz/lz4/lz4.cpp @@ -0,0 +1,5 @@ +#include "lz4.h" +#include "block.h" + +DEF_COMPRESSOR(TLz4Compress, TLZ4) +DEF_DECOMPRESSOR(TLz4Decompress, TLZ4) diff --git a/library/cpp/streams/lz/lz_ut.cpp b/library/cpp/streams/lz/lz_ut.cpp new file mode 100644 index 00000000000..6876f070fc0 --- /dev/null +++ b/library/cpp/streams/lz/lz_ut.cpp @@ -0,0 +1,287 @@ +#include "lz.h" + +#include <library/cpp/testing/unittest/registar.h> +#include <library/cpp/resource/resource.h> + +#include <util/stream/file.h> +#include <util/generic/vector.h> +#include <util/system/tempfile.h> +#include <util/generic/singleton.h> + +#define LDATA "./ldata" +#define LDATA_RANDOM "./ldata.random" + +static const TString data = "aa aaa aa aaa aa aaa bb bbb bb bbb bb bbb"; + +namespace { + /** + * Produces well-formed random crap + **/ + TString RandomString(size_t size) { + TString entropy(NResource::Find("/random.data")); + TString result; + size_t seed = 1; + size_t j = 0; + for (size_t i = 0; i < size; ++i) { + seed *= 3; + char sym; + do { + sym = char((seed ^ i) % 256); + if (!sym) { + seed += 1; + } + } while (!sym); + Y_ASSERT(sym); + j = (j + 1) % entropy.size(); + result += char(sym + entropy[j]); + } + return result; + } + + TVector<TString> InitRandomData() { + static const TVector<size_t> sizes = { + 0, + 1, + 127, + 2017, + 32767, + }; + + TVector<TString> result; + for (auto size : sizes) { + result.push_back(RandomString(size)); + } + result.push_back(NResource::Find("/request.data")); + return result; + } + + TString TestFileName(const TString& d, size_t bufferSize) { + return LDATA_RANDOM + TString(".") + ToString(d.size()) + TString(".") + ToString(bufferSize); + } + + struct TRandomData: public TVector<TString> { + inline TRandomData() { + InitRandomData().swap(*this); + } + }; +} + +static const TVector<size_t> bufferSizes = { + 127, + 1024, + 32768, +}; + +namespace { + template <TLzqCompress::EVersion Ver, int Level, TLzqCompress::EMode Mode> + struct TLzqCompressX: public TLzqCompress { + inline TLzqCompressX(IOutputStream* out, size_t bufLen) + : TLzqCompress(out, bufLen, Ver, Level, Mode) + { + } + }; +} + +template <class C> +static inline void TestGoodDataCompress() { + TFixedBufferFileOutput o(LDATA); + C c(&o, 1024); + + TString d = data; + + for (size_t i = 0; i < 10; ++i) { + c.Write(d.data(), d.size()); + c << Endl; + d = d + d; + } + + c.Finish(); + o.Finish(); +} + +template <class C> +static inline void TestIncompressibleDataCompress(const TString& d, size_t bufferSize) { + TString testFileName = TestFileName(d, bufferSize); + TFixedBufferFileOutput o(testFileName); + C c(&o, bufferSize); + c.Write(d.data(), d.size()); + c.Finish(); + o.Finish(); +} + +template <class C> +static inline void TestCompress() { + TestGoodDataCompress<C>(); + for (auto bufferSize : bufferSizes) { + for (auto rd : *Singleton<TRandomData>()) { + TestIncompressibleDataCompress<C>(rd, bufferSize); + } + } +} + +template <class D> +static inline void TestGoodDataDecompress() { + TTempFile tmpFile(LDATA); + + { + TFileInput i1(LDATA); + D ld(&i1); + + TString d = data; + + for (size_t i2 = 0; i2 < 10; ++i2) { + UNIT_ASSERT_EQUAL(ld.ReadLine(), d); + + d = d + d; + } + } +} + +template <class D> +static inline void TestIncompressibleDataDecompress(const TString& d, size_t bufferSize) { + TString testFileName = TestFileName(d, bufferSize); + TTempFile tmpFile(testFileName); + + { + TFileInput i(testFileName); + D ld(&i); + + UNIT_ASSERT_EQUAL(ld.ReadAll(), d); + } +} + +template <class D> +static inline void TestDecompress() { + TestGoodDataDecompress<D>(); + for (auto bufferSize : bufferSizes) { + for (auto rd : *Singleton<TRandomData>()) { + TestIncompressibleDataDecompress<D>(rd, bufferSize); + } + } +} + +class TMixedDecompress: public IInputStream { +public: + TMixedDecompress(IInputStream* input) + : Slave_(OpenLzDecompressor(input).Release()) + { + } + +private: + size_t DoRead(void* buf, size_t len) override { + return Slave_->Read(buf, len); + } + +private: + THolder<IInputStream> Slave_; +}; + +template <class C> +static inline void TestMixedDecompress() { + TestCompress<C>(); + TestDecompress<TMixedDecompress>(); +} + +template <class D, class C> +static inline void TestDecompressError() { + TestCompress<C>(); + UNIT_ASSERT_EXCEPTION(TestDecompress<D>(), TDecompressorError); +} + +Y_UNIT_TEST_SUITE(TLzTest) { + Y_UNIT_TEST(TestLzo) { + TestCompress<TLzoCompress>(); + TestDecompress<TLzoDecompress>(); + } + + Y_UNIT_TEST(TestLzf) { + TestCompress<TLzfCompress>(); + TestDecompress<TLzfDecompress>(); + } + + Y_UNIT_TEST(TestLzq) { + TestCompress<TLzqCompress>(); + TestDecompress<TLzqDecompress>(); + } + + Y_UNIT_TEST(TestLzq151_1) { + TestCompress<TLzqCompressX<TLzqCompress::V_1_51, 1, TLzqCompress::M_0>>(); + TestDecompress<TLzqDecompress>(); + } + + Y_UNIT_TEST(TestLzq151_2) { + TestCompress<TLzqCompressX<TLzqCompress::V_1_51, 2, TLzqCompress::M_100000>>(); + TestDecompress<TLzqDecompress>(); + } + + Y_UNIT_TEST(TestLzq151_3) { + TestCompress<TLzqCompressX<TLzqCompress::V_1_51, 3, TLzqCompress::M_1000000>>(); + TestDecompress<TLzqDecompress>(); + } + + Y_UNIT_TEST(TestLzq140_1) { + TestCompress<TLzqCompressX<TLzqCompress::V_1_40, 1, TLzqCompress::M_0>>(); + TestDecompress<TLzqDecompress>(); + } + + Y_UNIT_TEST(TestLzq140_2) { + TestCompress<TLzqCompressX<TLzqCompress::V_1_40, 2, TLzqCompress::M_100000>>(); + TestDecompress<TLzqDecompress>(); + } + + Y_UNIT_TEST(TestLzq140_3) { + TestCompress<TLzqCompressX<TLzqCompress::V_1_40, 3, TLzqCompress::M_1000000>>(); + TestDecompress<TLzqDecompress>(); + } + + Y_UNIT_TEST(TestLz4) { + TestCompress<TLz4Compress>(); + TestDecompress<TLz4Decompress>(); + } + + Y_UNIT_TEST(TestSnappy) { + TestCompress<TSnappyCompress>(); + TestDecompress<TSnappyDecompress>(); + } + + Y_UNIT_TEST(TestGeneric) { + TestMixedDecompress<TLzoCompress>(); + TestMixedDecompress<TLzfCompress>(); + TestMixedDecompress<TLzqCompress>(); + TestMixedDecompress<TLz4Compress>(); + TestMixedDecompress<TSnappyCompress>(); + } + + Y_UNIT_TEST(TestDecompressorError) { + TestDecompressError<TLzoDecompress, TLzfCompress>(); + TestDecompressError<TLzfDecompress, TLzqCompress>(); + TestDecompressError<TLzqDecompress, TLz4Compress>(); + TestDecompressError<TLz4Decompress, TSnappyCompress>(); + TestDecompressError<TSnappyDecompress, TBufferedOutput>(); + TestDecompressError<TMixedDecompress, TBufferedOutput>(); + } + + Y_UNIT_TEST(TestFactory) { + TStringStream ss; + + { + TLz4Compress c(&ss); + + c.Write("123456789", 9); + c.Finish(); + } + + TAutoPtr<IInputStream> is(OpenOwnedLzDecompressor(new TStringInput(ss.Str()))); + + UNIT_ASSERT_EQUAL(is->ReadAll(), "123456789"); + } + + Y_UNIT_TEST(TestYQ609) { + auto data = NResource::Find("/yq_609.data"); + + TMemoryInput input(data.Data(), data.Size()); + + TLz4Decompress d(&input); + UNIT_ASSERT_EXCEPTION(d.ReadAll(), TDecompressorError); + } +} diff --git a/library/cpp/streams/lz/snappy/block.h b/library/cpp/streams/lz/snappy/block.h new file mode 100644 index 00000000000..2a09a17dad5 --- /dev/null +++ b/library/cpp/streams/lz/snappy/block.h @@ -0,0 +1,37 @@ +#pragma once + +#include <library/cpp/streams/lz/common/compressor.h> + +#include <contrib/libs/snappy/snappy.h> + +/* + * Snappy + */ +class TSnappy { +public: + static constexpr char signature[] = "Snap"; + + static inline size_t Hint(size_t len) noexcept { + return Max<size_t>(snappy::MaxCompressedLength(len), 100); + } + + inline size_t Compress(const char* data, size_t len, char* ptr, size_t /*dstMaxSize*/) { + size_t reslen = 0; + snappy::RawCompress(data, len, ptr, &reslen); + return reslen; + } + + inline size_t Decompress(const char* data, size_t len, char* ptr, size_t) { + size_t srclen = 0; + if (!snappy::GetUncompressedLength(data, len, &srclen) || !snappy::RawUncompress(data, len, ptr)) + ythrow TDecompressorError(); + return srclen; + } + + inline void InitFromStream(IInputStream*) const noexcept { + } + + static inline bool SaveIncompressibleChunks() noexcept { + return false; + } +}; diff --git a/library/cpp/streams/lz/snappy/snappy.cpp b/library/cpp/streams/lz/snappy/snappy.cpp new file mode 100644 index 00000000000..bdb27db9b66 --- /dev/null +++ b/library/cpp/streams/lz/snappy/snappy.cpp @@ -0,0 +1,5 @@ +#include "snappy.h" +#include "block.h" + +DEF_COMPRESSOR(TSnappyCompress, TSnappy) +DEF_DECOMPRESSOR(TSnappyDecompress, TSnappy) diff --git a/library/cpp/streams/lz/ut/random.data b/library/cpp/streams/lz/ut/random.data Binary files differnew file mode 100644 index 00000000000..6b3c159b58f --- /dev/null +++ b/library/cpp/streams/lz/ut/random.data diff --git a/library/cpp/streams/lz/ut/request.data b/library/cpp/streams/lz/ut/request.data new file mode 100644 index 00000000000..6a29bfe6af3 --- /dev/null +++ b/library/cpp/streams/lz/ut/request.data @@ -0,0 +1 @@ +pron=snipfactors&pron=grmult_QUICK_SAMOHOD_ON_MIDDLE%3Ad%3A0.1%3A0.1&pron=umrfull&pron=usemiddleownerdata&pron=onlyfastrank&pron=sortbyfastmn&pron=rtb2000000&pron=tbs120000&pron=multbs0.63&pron=versiontbs2&pron=wffrm_ru.fast.download&pron=whrm_ru.hast.download&pron=rcy-lzo&qtree=cHic7V0NdFXVlb7nvpuXxyFhXgNx4Z0ljURpYFkbwZlxaR2s80apf0VmlrVvqgNIgBRI4kuE6KxZTYJABIZg_VlqUSnIj2hfIAiEAAEELag479k1SrV1KuMvrXUW2pnxr875ufe-87PvvS_JKyrGtQzvvrP3vufs_e2zz88-5-ErcEmsKB4daVSgKrPaKDNsY7QxzhhvXFgSM-IG-d6oMqqNS4omFU02rjemRmejlch4ABlrkNGFjD3IIP8dQkYGzbBfQfgHmLPFCBsVVzztlqb6m6bNsFGFJ7ZYEGtMMqjY2Y8fOd0V67Io0quNC9ClDTEUN2yXYrRRRfiq0SQ0-S6T1M2YjVJXYLc0jkayd1xgTEHtSXNdenqM8NIXD5-OySfzG43kc2T6kLhpI_ejYaNv0I-zjWbzNjNmzi5uKG5BBqmbfbAUz1JaNyzbmunJbM4uIf-3ZZfTRqKKgFa-id1GKpxQW58pYY1VKEmb38SkyebktW6TVxVhhShuek03SFtN28x0sQbGicBYtj2zObM108u-YaXZDvZ5GPkczWyTyzLbnM-IfN4pfN7tfCaazC4UZK0QeLtzNPnI8eipnEU5Gqd-JeR7K7vQeUNJ3LItUtvdzpNBytq8t0QIV5v2di6hPbs8R-XUBeDP7BL4Nws17hY-i5rao9HHCU0ssydQ370F0beqG_q0rAA655roFb5fJryhjWlyuGJ5w9U98SKUei6Kyzx4dme6vrIQFYHB4Aa2kNeq14-qIPDgJcsK2vKT7XIiAFW7egDkDpi94ySgidWH4LvQGMIEQ04Jc6dqobPvJtrrpnGmCjmeNIxoMco9jDrm7FizQcIYEsLY00W4XgljZZktmW1E6N7MLspK-qteKZQNAULZymI3lAHcUDg7aLFwBlCTkLayWA5p_4IBOtJlIDegtyYNoifEvAvSuOtDw6dXxIfYI4mYXk0gwMk0fMzEI1Ri0pfuFLos1JKMpHfsJ1WIsCrsESCwJ9TFOV1Yxb9Mzl8Rx31U8njVwhqUXW32-gB5E8L_oACZKV-CrglAd7HpQpfR62C1Lv17hlVWTNC52HTRiaaaBJ2VmJU4eEAXmFPQEoJHb3hJG6hUdnEUGDy2EJu0MmfuzewOHTw-Pio3eJQ4IW9bXuQMHiVKOmAeJXvaOIxzNIr-qQZ6GQqY_lOtJhYFdqsuYW1Kd-9jf_c7XSvc7f45Au9Awubw_DyH4fZRFKCDSPu-w0lzU1prPe8nChEc4DGj1gpW13G4RLS_ZF3-rpxtx2EFKxJ1DtqU9hwcl2mJL8jUKEetOML_mYAjtGV6icAW2pFm20Id4eDUnCNInJAj_Ak5jiBREkc4OFV2hLXUsBKRbNhnkubGNDSN9MYpRh6GzEGRR69eZ9Qrw8AUwOgNuZlJqeqlSvZB9UsQ_p6ienPCeKC7RD7qJtSQisdQNpsUjh5a7f43YXzV3dVEwUwE1e9thlCTByJ4ilKTyPTaJqkqEcDyb6cjbl0oPVSZbSazNy0mRiYM8vLAZZiWxFFuJNGvpYHUVZqcO5Lm6q1JtEUQxXg4-1Bil0hTzQznwbQjtXWzPGk_wBaRViOuWBBxa7aS_7tyJuXyanLVadSkkYcIfU-dJ3o0jhLR82rmKDCJSDAR1zm6EP6-Ypxo5onMrmxHqH0e8MzjcECx9bvMQA4BsdEDETm6no2dsr7E11dNfelpQW3djPoFjaH9yfFzvKUnhwUC1gnekbgkpOLHz5Gx9V0c5YUqvDq9qkdgeA1X4XW1IEqAhLU6vbVXFQaAioiLOJKZuAnYrbZQNbO9M4nWqXD1W_1aDXQezQ22ETzUSh_xVgsJNaTWiUyrpJAolFDLGh2FSQFTARSIlCqOpguZeKRZYVUVVRujS2M_jrcg8hjJ41FmVZ8QQSJMGSSGNi_5MkFNuWFMS9iHEP6hosAYXamcUdM4x0ZjR6MqR48xQI9rHq509egxQdpMMm16JKOp4iSN_hX2CgVome1pGQoCSnUsjLZoK6qs5BbLbdzDFp6EKQRMVvWgheTopChrkv8ycgbhGb6aGlIx5ltV51WcN7biS6Qx-y2Ea9Qm1dbNr6lrqk9R4-daFdCkT372757SPG7_VXKPRG_TldgrLPAq-WgOAuoQVjXtu5GCmbVWLF5-91VTEvadhcTMS7p3WdRsIF4sQLkfPenFLsYI6bWWzwppsa7TSZgV5KtPNsiryWPb4QWEk-pYiW2oAKiBYvLLH47wxkw-uyoz-ZCJ76ggbcBU-L2UUJQsicRi5ce2zk3YfzRxrTpfIBOrLjoKpqtgmSekzhOK79sPPeYBR-GFtPGhM2OQKR3NSDNnhSRgxvQzhOMysbN8gJzpxV4yvVjfKYz1CjG1EGaY0tqPPrVA6oDKsYSRPI6Iv7I-_qgOxCjfapMsEAUs0H3fsZW5oSFnglQ_lw8NOYGORTLA4kXCkKAfcEQ-Ac1IHiA6KL-otDlhbzH1aE3GT3Vz67nrBcae54_m-mmXCWrukxxpHo3e4snYKwyZa4QPBfnI8kpYYiGc2qJOTZy3h455Lty8IGFvRPo8r2HGTEmBUJ_V9bCnQEoP6S7B-yxSqmvtLEy_F_qsSPu6tDKB8Kv872nl_7OBQOBlHe_FJGCmSNgM7XLuXW95cwqHB2pEHZ9SOBR6Q67AblmhO2C3uetGkOZ-uIrY6q4RBQzHR4rwgjw2GaDgXJDNhgNBmw1ocK9hcK-hb3sNr5biZg3PO-gCL10IJZ-6ef4HMCaD8LwvFyAAMRCgf1niAFqj1gG9fygW6bZrG-40fmw5MH1w131w131w1z10193rQr9K6T0cVwuF9u-QfPbU6zWYxfuc8tWX5DP-bp4wJfsnC-3ZDiFpqhprXb0SsgRpbIp5rs5BOn3fKaky3fOfkPNx4p5SMk68dPX8hP1YKW4ICIXb3VTIscFDOp8QuN0_H3KjHgK3ezmRSgh8Mo8QGNly8MhgDByMgQOKgb6pj4MxcDAGful6jZMZA1X_1GPg-Uof3o9sRy9-raXxa97q-oR9ZyluUuLXCPYaJUFZimAYiGAtr5a6EQwUAMWwpTyGgfR6FDs6FIOUQhwz7zhC5nIHifaKCD43n4KYHEzz_6Kk-auWF2NdhLSf4fOUR-DgWGpwLDXgLH6fKHUoQqIUvp7MsrZH9CglH6Ih3Q2pZmiUOvFf3jwLFABFqU6eQwfS61HqU4RBSil9cr9_-qSQ9CgBBNq9FIEavifKhxTLPWMGvK1_bwgw5fvFxJSfPEYGHK8Va2vHp-nnobKtNL9OMOZQwJivvzTBNaaPCMicL0aZOX04dIOuiWIfWtGkdxx8nu5AnkaAPSzT5Xb1lMpxSsuOkuGc-1xGnjF3UpmGrst7z3Q4t1N53i08E0dnecfiO1Z4z3HyTMOQv0QeqHaToWSr56YUI-rTcuF9i2RpQpvY-6iLC99EifxtQp3ZG2n36EmV38jL27l-nbcUKXU2FQlxQhFjOxm5mtJ2divPvYHvbPPeOVzQpNjyNo9jBOEocTvgL7mNuRX7YtVhxKqcYnmI199OO_CL1y5I2CdMLYmtxHOqHnpKJSRnInOkzPV1iRHy8I95IoFEp_v1WoQlCmGPkyegkM7Znef3CNGwh8VwtbsOT0MRY35eKSlOhnuukr3akRShPkGzPTNWVV59XhOZ7Zn6Jn7jnGk3zZ4Wvom_8q2R3ia-wwMp_xEnMdgh0Xfxz8duGcu4QOEJhKjZpHv_OleBsl2olozkJpqKN_OREU_a91n435Cipgr5SDl0RFDQXxzKu3p_8ddcBYZKgzT7v3wcEsqrQ30bwqFcDP4ot8V_bjxuj6H7yiGM0F76uPhwe3Q_-IQ9eJqDlboEh0hhuQHyNkBOAt0GuCS44X3d-WZgsYhLHbFiZ5a_89T1CXuXpaVGDZle09gwd1pTXb2QMQKt-i99-m9cROR4INM_F2GmzxHpbvXXOFfaF8caB_NpOUKI5eGDtP3J7ETs1Ejfxfl6darKR1qL1gw_ylaN8lofypCMsyEE08BxFLDzsQienFS7z35562CqXf_Styzmlb-hWvz6AqLFZ038j2oC9MxUTY2kQSjx-eB-T4OMAdLe61x7rFzX3IWYFYhtlCDIm5LKdX6NTYKONF6_g1SuwpGvwoGKtOZZER_97qcnHJaMuTlhd1p4qqJfnOtaJS0XAVp-cdVcV8sCG6Tr3bznE6j08PYwwkK5EMi-EOcUaSBb5VdDq_2n9Izs51y9KlwixkX_I5Qsy9uj1KJvjLzDb0vdxdByiqE5V5M5wUcRfIu29aAOE7ozO0MXdZ57_21h7wGQAB5gc1Z1IHodYvuAxETvtLMXaIo2prsJ4qx15B_FLIVa8wxf3x3ez7W5FdIMpMOv3v1KO4QS6jkcHqRw-OzYDQl7mZUXHHpC4bDi6ftmBMGhB4LDwogvHHp0OKxDIBx65L6nkI4NzR4DnZw59qN51bPQfWS_6uqDj-N04fDpi-cn7KPF-DZ14VBrG5sRhC4cZt64d5l364WPEAgkzzpLhzCHDpMdCPvQiiGg5aF0dzKykZph3T5nBrL5pG2WaV0GA05XXjVvX-VgXL3gwVlwA65lkKvHl6L0Hbs-92Gs0h_kU2m0NGltTBNlR9al9wXey3HyNyr71-wt-TTb646-EKbycfZP6GmTrxu1Cfs4AsaXT5C2tfDLYULGl7uXluXGlx4b5NLNzvDSI9K9eAw7gU-KQ-6DGYsFMeqJiYDT-l7z_4dOX0p_PCthv27iuUrz48S8XSzc9pK3qAtO0GRw1bve_YoaM6SKN_i0RqPVNbIIYY1KbfCoeIx0lHz9SCSE1n3s-FC7LA86cXXHSQuVucLWg3wU30oHIYcua0rY70XwPFXxom_lpfjtn-70zjtp3JDm3-NjUY1W1_zP6clJhUrwcrNlZdLc1Jk0N3TSA5RJ89FOeuEC7ev2sm7vcDKyPr2P_tlPQg79s47-2USKIxvon_X0z6P0zyZKt4mWbqCfNnAOUrCx8xmhC6XU7A8t7jwsLaCg1NTA6jqzH1rhTWQAsrGTD0Ro5TekeQPWp3kjHk2zhkiDFPaGavkN_VjR4xh4njrf7QfJQGO3GYwBemo1FAOPPLYWgxig3OBEF-kYoLQ6Bs7VWrxZwTwSMZ_6Z42-VzpsSzejB2ICH4Wm6ebPlbdPTdirIkBvtoigYVe2g3ivPmaDFPrmi9fkejOFGdLng45PqbS6Pt-jvZlCJeaUtXSfSiMG__DzAo2-H9zbnLD3IXy9erY5dUtjY2342eb7fuF1fQ4LZJzr-MlmTqAvoU3ATtFALuWIxWOkUfdQtz5_y60JeyGw98XfUhe69_XRf3vntV0eqFn3OntfDonesDHYLfNdHmRLd1eAdIVZKuV6eYF657VWPTF2BDdquTUtbM-6lS5xuBe7hc67X_12LrUG4IcU1uOm1gD0uqO2mxiklE-HSoswA7t3r-9LKSch0Q30YG7UDgr2TAfx4E-Bo-YEUaFHzXs_zh01J_SQze7jIKfFOsC_hen3QftPyq7N5SpDPzeTfGD-LoX5d5qnJ-xXIrhV3dS1SXe_jcTEbnpvn3yNoaSpUujEzkPHPVUFyIE0-DRHfQCXjv0VJg6gH_QAx97vUA-IbLkxYb9k4pvVg1lcaSwVpo2pcXPowaxjv3jXu4of4AcPJ3MHAajBJBSATsoVdFJR3HV41wonMQnFR9d_oLp-6T8aEvavTSCnz23V7bxN2cXkc3hOX9f7V-dy-kARkMqzyMnpAzl0tT-IsA8tpHr3tgMfReWdppmXuQIUzjaJZ6L5bJP4BnWT2GlPT-gW57H0wtwuscsFaXUv1SrNtnNodD3ej7BXKs0plLXvCQJoKwUtVApaqBS0UCmAtlIAbaW0nlwpdB2VAmgr3VnJVCgkrKQhYd5bsxL2n4CtdrdfDdXi8baxihKzrZASO3hn79EEKTHbKiWhfc47g3SFZYxwH3LQvqBI1-tP5wPst-n6y9Gz5ibsFy3cokVp8XZdMkXbkV2WXQLMGqEovXqZl3MVIAZMkeAbQgFcuiWXRnAAfUCQHhMvtUexxTKQG1wMq4j_hT0SpP9zXIryeZ7bKMwkl0PtMPlQ_tpPahP2Dgv_EzQfnFM_L3Q--ObOD6LShJAwQSA6HMlNCAmJPl6u5DPdOfWSz7Ch7TzPuc7HroC-JHZVQVxgWtcEWP66JEqrA3cpIVNn6mfq1vdAWf3MNaICx4AC9UwwmE7LAwMnFwa9HCtWvvSBU_9yLBzHpLkP0cD5_bduSdjLI3o2d2YLO5Gqd8vQKtHKp8Z72dwiI9Twe3gAlej0rvd3CEsUX60FvGHxYaRv-zhKDHT_3U0J-50onq1dK78w0-OMeNXzvBAsW1YmcvfKS6yQkf7g_sCCRKmb6RwskYRc4nF3EVYkSkduDhzhvYMpTYxOvROXX5i7WJzDIe1f3ZsPArzvAI5Z5T0VNWR2hgt4d-ABU3NlPH3aTXNmpepvqZsB3hkIHnx8dai3I55jh1z5EJ9LC1R6sLkaC8UDXqJOTfET1_90Y-Xa4BcRTilaHFpb11Qzd27tLBqkBTVqV9ZByxSL77jOVacoB9LnFKZOkUrX5zgslosBXNsbUBr2ip4pEGtKTatrnFmTCmwVtMd2d_tp3pzWFQI16ddO-rhLA16p7RYOHB4_goX1BRzshyBSMGwsf9hsNfF07bxUTWp-7U014O2G4N7RPV6qmcsL6bTXPTfFScABnlM2cI1eA8oKTjX3O9QQcp110JN0sxbtRY3kE_Sq6zcWkTH1moi-CclHeKG7GL9tieV-eYOxQBrfwQeWDgWYVu6UyasyTrrk-s7PfXXmLBxzxrwhazOcKjhfOydNuygtaAWHm40M0mPl437TyLZZtaFnpivTTUYGT9AGKktrkMf0_n5M7oJyiRUyY7t7QblEqZvzBqyQaEu9_U6CgAYFZfEyopc9FM4398xN2I9FgJvbvfrQO-pC9fL49l_FAcVQ3oB7CxRKcA9CofkiHqG4MaCSVvtP9w4wj8jHfr10uejOjtkJ-3Er2H554Prl5_eaoP1gYG-O6PaDgD0OKyQBN-8fNXGZTOxNlr1Fkkh6x75c1yCZ-su7htj32cCo-BCewCjrK-_zDRxAR-gPBuCLfvV39i6Eb1QPijo_Q9TcELoPuXTn5d5RUY8LQs2V_KSoR6MPJM7GudJ4vr8mRNszgsxyVtM9qAefXZCwVwT9xlSIJ-T3G1NrtN-Y0n8BoVA_MTXw34QiKhrBVLST5lHd-vOmhL0B6acQs4uyLZJ-oB-KOvHpmd4pRMoAaec7_BAiLdb7hArMCgTHls2rw5XX_YRFJrGrRtYl7N8W8vdoFptaOne0uSlVM68GnMBCS4YHX77MG9NxVkgpd3HIOBQgYnjRwBetz4ZEaUvMyoTiBMI_UuehDTWpmfWpedPqfCYV0Pzzg3e9E28iP6SS-Xz-KVDparkGi-WFXkD2H_NT0BnUYWLlh38y389hMntDHabtrssFh8nsDXSYzF4fh8ns7aPDGORpKToTs19MKTs9huxy4j9fSxdPPOMv0ZUXVRjfpHWodmiIFhya08svfu22iWcYRstEXxq7nBZzGvxN-k81cmkshwaXU94zxuMbLtDkWN67ai--AnyXpcq5y06qcixBzv0N_xoix7_OVjzmyel48o_fPuPYvrGqnKhA89RzV4HvytH4vwvHsUMTKz990e_-FpKTozm9_J1ru0No_N9Fpj-hNPF43HsX03Pzm7ep7yKjBQ8_2bXVE8-Y1ZpVbUEisEfz4WcTQZoRAk3VDzsuVmnKHGzgWGxKNIbKzOsm8y-pYr0vJ5Mvh5MvS0jrhpIvi8mXkWsm1_BvaT3Eb4k__D_2jF58&rdba=clon&relev=ad_cat%3DO;ad_filtr%3D10;cl%3D2;cm2%3D0.0181118;cm%3D0.0869565;country%3Dru;ct%3D520-87%3A11886-13;dmoz_th%3D174-83%7C179-12;dnorm%3D32+autocad+bit+windows+xp+%D1%80%D1%83%D1%81%D1%81%D0%BA%D0%B8%D0%B9+%D1%82%D0%BE%D1%80%D1%80%D0%B5%D0%BD%D1%82;dnorm_nav%3D32+autocad+bit+windows+xp+%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D1%82%D0%BD%D0%BE+%D1%80%D1%83%D1%81%D1%81%D0%BA%D0%B8%D0%B9+%D1%81%D0%BA%D0%B0%D1%87%D0%B0%D1%82%D1%8C+%D1%82%D0%BE%D1%80%D1%80%D0%B5%D0%BD%D1%82;dnorm_old%3D32+autocad+bit+windows+xp+%D1%80%D1%83%D1%81%D1%81%D0%BA%D0%B8%D0%B9+%D1%82%D0%BE%D1%80%D1%80%D0%B5%D0%BD%D1%82;dnorm_w%3Dautoc+%D1%81%D0%BA%D0%B0%D1%87%D0%B0+%D0%B1%D0%B5%D1%81%D0%BF%D0%BB+%D0%BD%D0%B0+%D1%80%D1%83%D1%81%D1%81%D0%BA+%D1%82%D0%BE%D1%80%D1%80%D0%B5+32+bit+%D0%B4%D0%BB%D1%8F+windo+xp;fe%3D6.48773;fir%3D0.267373;forum%3D0.59255;fqwg%3D5.218048216e-15;frqcls%3Dstrong_tail;fsm%3D177;fsml%3D11;geov%3D-0.000436704;hpq%3D0;iad%3D1;iad_vw%3D-33.57460785;il%3D0;ilp%3D0.001022;im%3D536870912;imgintent%3D0.56;is_music_ext%3D0.000;isorg%3D0.460;issite%3D0.460;issoft%3D0.750;ldatopicru%3D197;mum%3D0;mut%3D15;navmx%3D0.268605;norm%3Dautocad+%D1%81%D0%BA%D0%B0%D1%87%D0%B0%D1%82%D1%8C+%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D1%82%D0%BD%D0%BE+%D0%BD%D0%B0+%D1%80%D1%83%D1%81%D1%81%D0%BA%D0%BE%D0%BC+%D1%82%D0%BE%D1%80%D1%80%D0%B5%D0%BD%D1%82+32+bit+%D0%B4%D0%BB%D1%8F+windows+xp;ow%3D52;prel%3D3;qc%3D3;qlu%3D1;qmpl%3Dru;qr2r%3D0;qr2u%3D1;qr2v%3D0;qrr%3D0;qru%3D1;qrv%3D0;qt%3Dru_nloc;relev_locale%3Dru;synnorm%3D32bit+autcad+pyc+winxp+%D1%82%D0%BE%D1%80%D0%B5%D0%BD%D1%82;syq%3D1.136661591e-07;th3561%3D0.0179;th3973%3D0.0682;topicality%3D36.0447;tvm%3D0.00222546;uil%3Dru;utq%3Ddownload+here+upon+russian+for+autodesk+inventor+auto+cad+%D0%B0%D0%B2%D1%82%D0%BE%D0%BA%D0%B0%D0%B4+autoca+pdf+torrent+%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D1%82%D0%BD%D0%BE+%D0%B7%D0%B0%D0%B3%D1%80%D1%83%D0%B7%D0%B8%D1%82%D1%8C+feed+load+%D0%B7%D0%B0%D0%B3%D1%80%D1%83%D0%B6%D0%B0%D1%82%D1%8C+%D0%B7%D0%B0%D0%BA%D0%B0%D1%87%D0%B8%D0%B2%D0%B0%D1%82%D1%8C+%D1%81%D0%BA%D0%B0%D1%87%D0%B8%D0%B2%D0%B0%D0%BD%D0%B8%D0%B5+%D1%81%D0%BA%D0%B0%D1%87%D0%B8%D0%B2%D0%B0%D1%82%D1%8C%D1%81%D1%8F+%D1%81%D0%BA%D0%B0%D1%87%D0%BA%D0%B0+galloping+skachat+besplatno+free+%D0%B1%D0%B5%D1%81%D0%BF%D0%BB+%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D1%82%D0%BD%D0%B8%D0%B9+%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D1%82%D0%BD%D0%B8%D0%BA+%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D1%82%D0%BD%D0%BD%D1%8B%D0%B9+%D0%B4%D0%B0%D1%80%D0%BE%D0%BC+gift+%D1%81%D0%B2%D0%BE%D0%B1%D0%BE%D0%B4%D0%BD%D0%BE+freely+%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D0%BD%D0%BE+%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D1%82%D0%BE+%D1%85%D0%B0%D0%BB%D1%8F%D0%B2%D0%BD%D1%8B%D0%B9+russia+%D1%80%D0%BE%D1%81%D1%81%D0%B8%D0%B9%D1%81%D0%BA%D0%B8%D0%B9+rus+%D0%B2%D0%B5%D0%BB%D0%B8%D0%BA%D0%BE%D1%80%D1%83%D1%81%D1%81%D0%BA%D0%B8%D0%B9+%D1%80%D1%83%D1%81%D0%B8%D1%81%D1%82%D0%B8%D0%BA%D0%B0+%D1%80%D1%83%D1%81%D0%B8%D1%84%D0%B8%D0%BA%D0%B0%D1%86%D0%B8%D1%8F+russification+%D1%80%D1%83%D1%81%D0%BA+%D1%80%D1%83%D1%81%D1%81+%D1%80%D1%83%D1%81%D1%81%D0%BA%D0%BE%D1%8F%D0%B7%D1%8B%D1%87%D0%BD%D1%8B%D0%B9+russkom+%D0%B1%D0%B8%D1%82%D0%BD%D1%8B%D0%B9+%D1%83%D0%BA%D1%83%D1%81%D0%B8%D1%82%D1%8C+background+intelligent+transfer+service+%D0%B1%D0%B8%D1%82+bit+%D0%B2%D0%B8%D0%BD%D0%B4%D0%BE%D0%B2%D1%81+%D0%B2%D0%B8%D0%BD%D0%B4%D0%BE%D1%83%D0%B7+%D0%B2%D0%B8%D0%BD%D0%B4%D0%BE%D1%83%D1%81+windowsxp+windows+xtreme+performance+%D1%81%D0%BA%D0%B0%D1%87%D0%B0%D1%82%D1%8C%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D1%82%D0%BD%D0%BE;vcomm%3D0.248625;vintent%3D0.95;vm%3D268435456;vnorm%3Dautocad+32+bit+%D0%B4%D0%BB%D1%8F+windows+xp;vnorm_dopp%3D32+autocad+bit+windows+xp;vnorm_syn%3D32bit+autcad+winxp;vqi2tvec%3D1%7C0rkjACu_CrE81xPoE93gAB5X2uKgUsx_JvATChj78_A%2C;vqsemvec%3D371364212%091370421533%093362250591%092760956004;wizqbundle%3DehRMWi40AQAAAACAUzAB-F8KZipZehRMWi40AQAAAACARwAB8wYSXBJGEgdhdXRvY2FkGAAgAioMCggPAHNzEAAqDQoJDgAQJw8AJAsKLAAnEAEcAIRzJxAAIJMCKh0AoDICCAI4AUAAIAAAAAAwo7-XscucoK7PAQr0ByrmB2oA8P9S1AMB9g8SjxcShgkSDtGB0LrQsNGH0LDRgtGMGAAgASoQCgwWAIfQuRAAKhYKEhIAfbLRiNC40YUYAMe90L3QvtC1EAAqEgpYAE3QtdGCLAABRAAt0LwYAH290L3Rg9GORAA9vdC-LABRtdC80YJwADsYChSgAAAcAB2DMgACdgAP0gADH7WOAAMh0L4CAQrSAD290YtcAAD-AD2w0Y-6AD-70LiiAAQRs9QACloAP7vQsFoAAS_Ri54AAy_RiwABAACKAB--AAEFA14BCjIBAjIAApAACv4BHbuOAABEARG1dgAKKgAdveYAAMwBMYvQvAABCuYAD2wBAQBYAB-4WAAAH7KCAAIRuGoBCYIAATYCCtgAAlQAD4IAACHRjpQCCZYAAAADLxAB7AIAAtYBNxQKEGQCX7XRiNGM8gABI9CwSAEMMAACqgIKSgMCeAEABgMJigAARgAN9gABLAAB5LYA8HMB5AANigAAFgAEigANdAAh0Ys-AQ8YAAEA1gMKYAATuWQDCi4AAGAAH7UaAwARu9QCDywAAAG6AAoYAAIgAna5EAAS6A0SYQQyuNCyjQEBjQQ1GgoWdwECGgAzjtGJgQII7QEBHAAAawQBQwINNgABsQACyQENHABP0Y7RiRcDAANSlAHwZdMAAhoAAusADU4AAGoAA04ACEkBAzQADzIAAQCAAA8NBQABMgABlQIKOQECGAACwQI_HAoYHgEABPMCD5wAAgOLAQ6cAAAXBQ8yAAMC7QUNbAAA6gAFxwQPOgADAiMCDToAAfIABDoADx4ABQLBBA5YAAAaRwDw2Q0cAAFYAA_mAQQBrgACYQQNcgAA5gEAYwUBSQMNVgAAHgADBwYOcgIfuXICBgGoAAEjAw9wAAIFKQQOVAABHgAPKgIHAXsHDsABACkFDWoAAhIBA5MEDtoAABYDEbhbBg1QAAGxAw0yAAFQAAJDBw0cAAJiAQ-aAQgCtQUOOAAAKgEC9QcOHAAAogAP7gMHDoUFAR4DALIBAaMHDVAAA8AAAKUICI0EAjYAAn0HDTQAAk4AASYBDWABAfQAAcAAC4MFAlIAAL4AAP8HDzgAAw-hBQECOAACNQgPigAEAQACDhwAAWwAAXAJAOAAUgICaQYOHAAAOAACkXYA8GcBOAABCgIOIAIA6wgPMgAED4QFBAAoAQKxCA4QAQFqAAOoAg6aAQKgAA42AAC8AATuBA4eAAJUAA9IBAQBCgICWAANxgACHAAB-AAPjAAAATsIDTQAACYCDxYBBQJQAACcAUcgowIqHAsA3wWgMgIIATgBQAAgRQT4AjDk_fqeq-3Q-OMBCqgCKpoC9wPwCAgBAf0XEr0GEqABEhLQsdC10YHQv9C7-QMA1gPxARgBIAEqHAoY0L_Qvh4An7UABPAlGAoUGgAELxYKUgAAHxBQAAkftVAABn21EAAS-AQSawBv0YvQuRgApQAKD1UABR--vwAFTwME-4W_AAKdtdC9EAAqGgoW8QBfsNCz0L69AAYAJwEPDQEFDz8BAT--0LyAAAI_vdGLZAADT9GL0Y8aAAYfhRoABQEaAQ_OAAFfvtC80YM2AAYAnAAPtgABEbCEAA_oAAEfsEwAAyHQvoIAD18CHgD0AQ-IAQQfvogBCBG-1gEP8AAAb9GL0LzQuKQABAFAAk8gowIqzAIAKwHovMHs0NOEgNjdAQpIKjspAfEAKQAB8QISJxIUEgTQvdCwFAXBCAoMAGEQASCjAioLbAUAdQESAWwF-AHYpuGipb-7vI8BCu0BKt8BTADxBM0AAfcPEvEEEuwCEg7RgNGD0YFyBRC4NwEEbAUhaL5RBREoAAWRFAA-vtGOFACIZwXgFAoQTgBN0LzRgxYAe7jVAPAfEAB6AAASAAw-AB-zkAAAP7jQuXwAADq8EAG4ACHRg6QAClIAH7BSAAQAugAK9iAA8Cm8ZgAADnoAH7geAQCPuNGFEAAS5AFvAQIKkQAPIQELDCwAD70AAw6VAAIPAQo-AA_7ACsP5wAGSO0AYy4C0L7QvPAAA2UC-QDW3teX_ICvr_gBCqMBKpbwAPEMhAAB-g8SmAIS-gESDtGC0L7RgNGA0LXQvdGC8ACiFgoSFgCg0L7Qst0A8hMMAAYuAE_RixAALgAAP7DQvC4AAS_QsC4AAy_RhS4AAnq1hgJQcgBv0LD_APIESAACH752AAN70YMQACoSCugAa5cBFRWXAQOnANjLxM6v-_j5l1oKRyo6lAH0GCgAAXQSJxIZEgswAQDwDTMyGAAgACoGCgIzMhABIAAqAjMyMgA4AksH-QCA877ojIGuoecBCqQBKpfvAPFthQAB9hESvQESMhIDYml0GAEgAioICgRiaXRzEAAqCQoFYml0JwsAcHMnEAAqBworAEYQARI1NADwAQsKB2JpdHRpbmcQACoKCgYNAEdlZBAATQAFNwAgQBJgADVlGABsABdlNQAVbgwAAk0AAoQAEGWEAARCAEAgkwIqg_AAB_MH2J-xsZ7qkK_2ewpJKj3wAPECKwAB8wQSLRIYEgbQtNC70Y-KAVIKCg4AYzsBEQ1LAAfSAunHnYGH9bv932YKkQEqhPIA8QNyAAHyBhLFARJBEgZ3aW5kb3f1AGEMCggOAHPnAPEaCgocAGMQACoLCgcaACYQAScAZSdzEAASQkMAPwAgAjUABzINCglCAFb8APEKRAB0ZWQQABIoEl8AD4gAAAh8AEQgkwIqK5QAB98A2MbplNn6pa_fAwpDKjffALElAAASIRIQEgJ4cDIJcAYKAnhwEAFFADoCeHAXCdiHlbXAt_bOjnUKVypKRQCwOAAB9AcSQxIsEgh-CUFkZXNrTQBgDgoKEAB1wABiKgwKIABlWAAcD50A-QDmvbXihY6InIYBCpgBKosyAR95WwAcJPEHtADwHxJiEksSCGludmVudG9yRQA0DQoJEAAARAA0DgoKDwAEVAAEEAAgEAFyAAQOAEF9AUdkAAQRnAAAWgr5ADC5xvmm0_qipSoKoAEqkpsAsIAAAfADEkwSORIE9gAB8gBhCgoGDABx8gBwCAoYAGAQAcoBQBYAVGVhAPUAIwCBZXMQACCTAiokAPECqgAQRAID8A5jYWRNAGEJCgVjYWQ1ADAICgQLACMQABUAAWEAIP4CfCAQAUcAAAqiAK_p2fTEnaq4t64BowAC4PwREpoCEvwBEhDQsNCyVQQAswoj0LSbBWAYChQYAJwcBAFTBIEyAB-1FgABjCAEURYKEkYASgQCVgTwAAI_vtGOGAADL9C5RgABingE8AIqEgoOdAAvEAGIAAGL0LwQAPcEES2jAAASAwNNBOjw7e2q0ZnMoLYBCmUqWDoCcEYAAfIFElYQAwBEASFjYUYBARADEEYyATEOAHNNATAKCipwAwBNARAoIAMAQwERGmcAB8QD6JGI7ved-9fU6wEKfypzZwDzBWEAAfUHErABEkwSCGRvd25sb2FkegNQEABkEAEfDNEeAHRzEAAqDwoLDwCUXgNxKg4KChEAb2ID0EtOABkHPQAqcydNAIWTAACCAByMggCeq4T0m6vIyclDNgWhABIkEhISA3BkZuMAIAcKCwACEQMACgAEDAICFQfosbOQvtqmg6CnAQpjKlbKABZE6QxxdG9ycmVudE8AAekMAcAAAekMwBknDwAQJx4AJAsKO-AAAK0AHA6tAPgB_Ma8_MTw4JzsAQqrCiqdCmcALwsF7wjrJPANtgfgEpoYEvgXEhLQt9Cw0LOFCODQtkIDJdGMmwIJGgBj0N8N8Du1qwAAkQAGHgAzt9C4HgAANQIKHABzttCw0LXQvLkBOxQKEHAAAAMCAH8BBk4AMLbQtd8ADS4AU7fQuNGCQwEILgABYAADKQEAl4YM8AIFKwENmgBl0Y7RidC4AwIGOi8L8GMF6QEGGAAAbgAACAEEOwMPUgAEAesBOBIKDuwAAZ0CCuoAAWYATtGMEAEYAS_Qu-oAAQKwAAFpAwzqADG10L2dAgLTAwqYAASgARG40gAKbAAAHAAAHQMNmgAAUAUNhAAALgAAAAEC4gAMhAADigEDJAK2CvAWHgARu28ED5wAAxW16QIJbgEC8gABDwQKOAAAUAABigABHAAK1AYA8WQNQAIAFgAAoAIOmAARtVgCDBgAAFIBAAUECngABQIBDzYCAyHQu5UDCjQAAkwABM0DD2gAAQCHBQ1OAQH8AARqAgweAACeAB-92gIGAx4AAaQAD4wAAgOQAwwcAANUAQFiAgAvBQaMAQX4AA2MAAQaAAQchgDwB1QAE7VMAw5uAAIaBACWAQ8cAAIDVQa7APAbcAAfvFIABQE0ARG1wAAPMgECAOwED4YCAAMyAQNfBwqOAzG30Y8gAg2iXwHw9AJUAw0cAAMQBQStBgZIAQEQAS_QvPoEAgAQBD2I0Yx2AwSeAAI0AA1sAAGIAA98BQUAJAMCHAAMeAIBfgEPbAAAA1AAAZADD0QBAwRrBwqKAACYAQBuAAKmAA5uAA9QBAEDhgADeAEK6AEBHAADEAEMiAAE2AAG8wcKYgEfuSACAgSiAALyBArmAQFkAQ3cAgRqABO4bQcK8gABNAABNgIA6AEN8gAv0Y44AgMAAAIDnAMKJAYRt14DDGQAAuwAAlYBChwABdAAAQQFCiQBBBwABEAECjoABFYAAvwGChwABDoAAr4CClYABR4CD0YDDABSBgpgAQBWAA_UBgIAKgMDvQoUAvAeMgAAUAIEfAEPHgIAAEgBDcICAzgAA1QACtwAAPgAADYCAmEKDbIBD74FAgNQjgLwAw1UBgEcAA2CAABmAAK6AA_aBBcBQQA0Ag4XAvGyD1IAAQKWAwHCBw_UAAQPoAAAAbABD04AAQDOAAPLCg0GAQGUCA-OAQUAPgEAZwsKrAEA8gAB1gQPGAIEAtgAApgFDr4AAPQAAgAICowAAF4BAjQIDW4AAw4BAowADlIAARoAD_oAAgIaBwItDQ5-AQHEAA8wAQEBcAAChAMPNgADD8IHAgNmAQ-mAAMCigAP_gYDAxwAD9gDAAHSAQC2Cg-SBAkDDAINGAEAOgACGAECHQwGNQUAewbQgtGMMgIIAV0GEAEUEvgBi5Xsua2Avp3ZAQqeCCqRCH4SX_8DAfoTKQQBAB8OIdGMMgpyHAoYGgD3ATAEANENAt8NBDQEYx25HAD3AjMEJNGLvwwQOMkCIWqPTRKxhgBdtdC90YsuAF85BPIJYAAFb9C-0Y4QALYAAAJ8AD-80Lg6AAFvOQQghADYAvEKAbIADwgBAxOwCAEKugABNgAv0YxqAAcBIDsE8QgEAS3Rg-oAADQBH7tGAAgfvOoAAHO10KgSYZgAD6ABAy0EwIQBALwBAFAADcoAT_MOwGgAAAFSAQ9oAA4A6DEN8Q4AOgAfuzwCCQHAAQq4AAAgAgKiAR-FDgEBAXQCAiIEwQBsAB-yVgECD5gAAx8EEbAfBPENAgEKlAAAMAEACgMPNgIEL9C7agICAEwAE72kAiEEglQDDWoCAZ4CIQRDAVIADyAEs9oCCsAABRwAAQwCIgSAEgI3GAoUqAIiBC8UAiIEASOeAyIEZi_QvooABB8EGTYfBLwWASHQvqIDCyIDISAEogFIAR-1ZAEEP7UdBJBABB-IQAQCAdAYEgccBBCs9wAAHAR1iAAPWAMBAhwE8QAxvtC8tgQKigAECAIP2gAjBPQEUAQKNAAAGAAADgECeAEPOgIBAiAEbwKIAA44ACAEBPIAH7hCBgQCmAERvugBDWABHAT2BAIQBwGcAwoMBhG3XgMOIgEAxAEcBBkgHARoBdgAAYYAHAT5AASSBQTsAwo6AAYQAQBABCEEUS_Rg24CIQTzAvoABHwBDuYBAAACACgICoYAIwQQHCMEZwE2AAEyASMEIrYBIwT6BA9EAwEA1gAD8AcOggAC1gAAEAkjBIpqAg5SAAHUACUE8AgPuAIGTILRjBACCgHqAAIGAQH8Aw0YAgoCNW4CCyAEphG1qAAKkgEAdAAhBCAABFEAoBoAAVIADBwAAbL6A_YADn4BAIwEDvQHAUYBI9GJJASCARQBDy4FAwQkBPEDAqYAAqYCD_wAAwQWAw_gAAMCZwJwAU4JAvwASuMKW3ULAfcBFBHp16ff677V8PFICo8IKoEhBNHvAwH3FRLkGBLAGBIUSggCNxY3uNCyIwRwFAoQHAC3sFASAh0EEBZiFBBBRQiQHAA9FgoSHABoWBbwABgKZgAASgC9vdCw0Y8QAHgSoDIAAU4AATgACmyPAZAh0LNsAAxUAF-dCHC4VAAMHbzAJBNkE75UAA9wHBIgxABdBPMDEbWmAD8eChrEAAQCOgAP5AAAQQTwET4ABgBcABOwlAAP6AADH4U8AQQAOgARuOgAD5YABQRYBwDxDAAByAEPmAEKIdGLXAAK6AF_sNC50YLQtZIAB8oSUEIBAFG9tATgOAAPrAABA8oAL9C4tAF9CPGKE7UgAQ_IAAQh0YskATcSCg6cAQKOAg8yAAcPkgIFAnAAAJICDBQDEbugAAqkAQEUAwBGAy0QAYAAH7sIAQAAXgACdgEOXgAesBYAT9GO0YJsAQMBVgIRtVIBCpIAAVQDA6QBDNYDEbXSAQwwAAa2AQ0wAA9-AAUByAACggMP0AEFBLQBDG4AABAEEb4OAw-cAQQh0L7WAA8ekwDxIjwADNoAEbsoBAysAgGmAA2GAQG0AQNwAQ-eAAADTgIKIAQCNAATuVgBCjYAAhwAAiwJAPCyggAftVYECg9MAwQD4AEA3gMPCgEBAdoAAQoBD3IAAQT4AwwQAR-1wAAJARwCCjIAAtoAH7LmAQQA2AATi7YDChIBBKAAD9wACwBkAww4AAIeAgA4AAwaAAbyAA-kAAEAbAMLFAIBlgMOIAECigQfvBgCBQNyAwx-AAM2AA9-AAMBigYPCAMAAcQBE7giAQ_GAQUC5AELPAQPRAUHD2wABQGSAwCSAQDABg8UAgEPiAAIAqgAAUQFD6YAAAEeABG1kv8A8poAA3gAALIBCnYAAjwCAJIAHYuCAwFQAQOkBw84AQUAOgAAzAAPigEBAmQGCggCAN4CEYjyAQ9SAAAA3gIF-gcPUgAAIdC-4AALQgIADAQPfgUDAFAAA9AGCk4AAvYAAioGDxABBAHoBQvwAQ9kAwUCfAED_gYNngABSgAKgAABWgEDJAUPCAEAAVAABB4EDIQAH73sAAgCcgEP0gACA1QEDxwCAAFuAAIGSRfQsAEPOgAEA0oIDx4AAHEG8glGBw8eAAUCsAIMHAEARAQCGgAPOAAFAq7cAPAOzAAAkAUPzAABAFoAAL4ID3IAAgE4BA9UAAYB0gNDAPAMADgAA5gCClYABGQCAKILChoAA4gAAXAACh4DiwnwAuQCD2oAAQG-AA8ABAoBbgFIDgRZ3QsCVQAzCBAAMwj4AZy27ceGip-f2QEK4wEq1gEzCNHEAAH9FBKMBBLoAxIU1RQARxoBEARh0L3QuNC1NQiSGAocAK7RjNGOEARQNgBv0Y9mD1A2AAEAUncPQDYABY-UFAAEBBBSmg_wAbhwAAUv0LVwAAgANgAPwABxA_AAwAAFL9GMwAAJL9GP2gADGQDwARoAA0_QuNC5GgADIdGM1ADYAvART9GM0LXuAAMAGgAB7gAPegEAAIQAD0QBBQAcAAJEAU3qAEzPAQBF-AT4AJi7qpKGyID3PQrsAyrfA-YA2s0BAfsXErYLEpALEhbmAAAZCSPRgU8TYB4KGh4AcPEMYbXQtSIAauUEwBQgACPQuxoADToAjzcJABsXQCAACU9OG_IIIAAKM7DRjyAAKxoKuADr0LvQvtGB0YwJAfAGlgACXAACHgAPOgABBBwAPyAKHDoABw3wBrNcAA_YAAIBEgEzuNC5mAAP-AACT18F8BwgAAwPQAALH7hAAAcBgAAGQAANkgEBcgEAFgEPmgADFbhSAQ0gAAJaAAFAAQfwEg8cAQUzvNGD3AAOWgEztdGCHAANXgAAEAIHXgANIAABfloAYA8EWAENnjIA8lUfsLoBCQKCAAXMAg9CAAMxuNC8HgIAAgEPpAAED7QCBwKEAAViAQ1iAAGCAA9iABAFpAANYgEl0Y5iAQ72AjG50YI6AAD-AQ0YAgWyAg5UAAMCBB8QAAQAA7IAD64CDQQgBA1cmgnwDqYDBhe1sAANLAEBTAEA8AEEbgIPIgAFBqwDDSIAtQLwHBQCCgBmAAemBA5UATK10YggBQ_-BAEB6gEAbgEPKgIFD74EBwDaAAFmAkvvAVvbBASTBecG6enM4IPgrJuYeQqUASqH1QLUdQAB8AES7gES0gESDO8BMQYAeB4ZYRAKFAB3uHcWsQ4mAD2-0LkUAJewvQLxChQKECgAMbDQvD4AClAAK7AQYgAt0YNOAD3MF-I9sNC8TAAbtRIAmLXQumMSRMEAsLCiDwIzEdiW1aqNnOWslW4KcCpjhALxBFEAAfMHEocBEikSB3NrYWNoYXT7EWANCgkPAHTLEfASKgsKHgBKEAESRisAMwwKCCsAACoABjkAKXMnDwANSABEPxEcVj8R-AC80eakh8SnqcABCosBKn9zANRtAAHwERKkAhL0ARIgCAEC8wIGOxpAEAD_AesHAR0BMCoKJjsaMywADDwa8AwmCiIoABIPVAAVH7VUABQAKAAvJArQAA5vEAGZATsnAA6XAen5xY6Ulab6tjMKrAEqnpcB8QaMAAH1CRKhAxIvEgliZXNwbGF0bm8EAWEPCgsRAHYEAfERDQoiAEwQARIeMQALIAAcUiAACFEAEGVRADUQCgxiAF_BFPAldAAOH1BUAAkBQgA1DgoKVAAAZAAIdQAPxgAhD-YAHgjVADtpbmeDACxlZBEAAKUACjoBRj0BEWo0BAerE-iJnJWDwt20i4EBCnwqcDwB8QFeAAHwBBKkARIrEgRmcmVlqAByCQoFDABwcu0XUQsAcXN0YRWxIwBOEAESNy0AcGTGAUEHLQBUuhOwRQAZczkAD2YAGUF_ABxxvAHp5MjfwIHK0NUTCpoBKo0tAdZ7AAH2CxK5AxLOARIKrgEBpgGBEAoMEgCG0YsrFTAQEgATEgI6FXAOCjoAZxABDxWwJgBavtC8EABMAE3GDVAmAByyJs8CwCYAH7BMAAAesCYAT_YCQNEAvkfLARFoHAEHLhXpy6ShvsiwpJVTCvoBKu2dANbbAAH_FBLQBBKsBBIUnQAAWwIBNgYVuTYGIAG9_BkDkhxQOgARtR4HB2ACL9GFGgDADBC5BwCRuNC8GgAGfbkQWxqAggAfuDIABS9LHDQEnbV1BvAPTAB_tdC80YMQAGgAAQ9OAAUh0Y5OAA8CAQERtc4Apw1QP7jQvLgYAMCPagAPcAEHADwBD1RIChCP6A7yBvIABQGmAQ82AAAj0LU2AA9aAQF_tcYDWhMCAbC5xwOfxY7vtvKou7MXxwICj_8UEp8FErwC_QACEbqkAQAzBzECbxBxD0A2AAFvghtgEAAeAAh_NRtgGgoWPAACgACBKh4KGhwAAl_CAfESPAAIDxwACC_RgzgACB-wrgAIP7DRhR4ACQDqAA8_Af8tBx07YgICdQP4AcaI8fa1rMGLsQEKpgIqmALJAt0GAQH_FhKnCxK4AhIWrAERvfwd4AEgASoaCh4AAz-8EAAclAeAvtC5HAAIDziNACCOOFUBkIMcAD4cChiqAMsAELPNARAFJwBgtRwABQDkBAiyVgAGP7zRgzoABY8RDPAXErw7ASIAOwAPOwEDD1cBCS_Ri8kABwNXAQ_JAAUCHwEPHgACAHIaCBAeLQiwsFsBCQ8_ASef0YuYAvEuEsgDegIhDz8BIw-WAgkPWwEQDwYDAg-yAi4PkwEpDzwAAgAJAg-2AggPDAMPD-gAAgKvAQ-zAgMPCgP_MygBPGYFA9cB6Pnp99egktmV6QEKTSpBJwGwLwAB9wgSORIgEgqVGCDRgLEBAcoBMA4KElQDACQDApcZByQDn8HXhZeluLalHj8EAPUH-xEStAESlAESENGB0LLQvtCx0L7QtGsFYhYKEhgA3aIfMBoKFncFMBwAbS4D8gMUCkwAHxBKAAMALgAPSgAFfbmnAhFLPwQHfgDY6IT9ysKFy7o5ClcqS80AuDkAAf0OEnkSLBIQ8gEFfAAwFAoYsgExLgAbTgkcRVkA2N3H666P0qrwTApTKkdZABE1WQAcS1kAAM8ZAfMDAFkAEm0sARwXVQDpn_CL1IKJ3ethCrwBKq5QBPAInAAB_RASiAMS6AISENGF0LDQu9GP0LKyDQNjHwBZAPAKHxAWAAE9tRAALAA_g9GOFgAAPovRjxYAL7EGoQEv0L5YAAGr0L70IMAWChKyAD680LgYAG7KAUDRg3IAnyCQhS4AAj6z0L4u7CPxBw8WAAARsMwAC4oAL9CwRAADINGLRAATARxTDgXorN7M1fKtjPSPAQpUKkgUARE2PBq0OxImEgZydXNzaWEKGQY1GhAcNRoAAgYCpR0HgQbY2YSlusPR2boqCm0qYFYAsk4AAfQFEngSGhIHVgCSbhgBIAIqCwoPIwhyHAAzDQoJKz4mAmEmoykQADkAByoAhCfdGw0sCPkA-6qwpt32g5SJAQrUASrGhAGgtAAB8AkS_AMS2AoMMYDQvuUgMLjQuTkIMQgAfuABANYEEV_uD_EQGgACT7jQuRA0AAM_vtGONAAGH7UaAAMv0YM0AAh9vCoUQBK4AH4WIWAaChYYAE-FAWE0AAQfuIAhDPAF6AAHL9GFaAAGEbOEAA84AQERvrYADBABmApghgAPHAABAiFiCg88AQSu1gEAgwMgvwEwDAqtBa-q4bqf8pLDtqYBdQkAsfALEmQSHBIDcnVzQgFQCAoEcnXUG_AAKgcKFQBGEAESNB4AeQoKvQFAZSAAQfEfEBXyHDQLAIBJASBAAHEFB7cB-QDj_o6a2pz4xI8BCt0BKs9JAfEIvQAB_xoS7gQSxAQSGtCy0LXQu9C40LpGIRGDVQEQusMAAVABYR4KIgAFn6gHgSogChxCAAR_ZiPAEABCAAoftSAACj-4XwwQC5ML8AKiAAkfuaIAChO-ogA_HAoYxE0BoUAADD-80YOiAAnqEPAMIAAJL9C4AgEMH75AAA0v0YVgAAw_uRABpAEIWAEQ5DEHkQcPwgELELCiASYFMCUCBY8AClIB-QDxkqauxviIwsEBCqIBKpXgANCDAAH-ExK0AhKSAhISiRqB0YHQuNGB0YLmAAJqI1EYChQaACMOADwkUBoAAX7QcwswFgpQYxRQD04AAozkBaAqFAoQTgAPMAAERAJhYgAEH7UYRALTGAADP9CwEOAAA56-0P8EHcVsC-nkupf52uG8oTcKqQEqm6UAgIkAAf8XEsoC_goVFqUAEoSjAEPRhtC4QSFxHAoYHgADj40AAf4NIwOfhR7xIhoKXAACARwAD1oABE_QtdGOHgAHAFoAD1gAB0_RjxABdAAFAVYAD5AABR-5cgAHr9DLBQCjAmrLAALA0Y9RAfkAsJKumbn30cyuAQqTASqGrADTdAAB9AkS0AESuAESCKwAEbofAmAQCgwQAGo0I6IQABIAK9GFEgCVdiNgDgoKNgBlVAHxAgwKVgAaEFQAEb5UADUSCg4wwgzyATQAB0QAK9GDVAAbtRAAdbCaABFlyAIHQga_-oj2o4emraw2CpVFDQAq9QiWACHRgREHBK0eZBAKDB4AaZ4AEpiZAGESCg4kAITuAfAIDgoKFAAr0Ys2ADuw0YUiABuDIgARvmqaAHMq0LUyAIXQlwAcq5cA-QD_-tm7xIeol4cBCqECKpTQCCACAV8DcrIGEogGEhouAQCiBKfQvtGP0LfRi9GHNgYAXwMSBt0B0BwKGEIABa8QACoiCh5tB-AiAANv0LXQtRAAYgAKf8AUYCAKHEAAAw4ncLzQuEIACy-IBPEXEB-5RAAJDyAADhGwpgAPKAEGIdC-hAAPIAAGAGoBLxAB5gAHb9C1AzBqAQn3KDBgAAefBvU14AAKEb4gAA_IAQU_tdC9PgAKAsQBD6QBBgD-AC_Rg0IACSHRiwYCD4AABT-90LA-AAkh0L4-AA-AAAkCPgEPIAEGr9D8AkzpAgbApAPpiY66wtOj_skWCpwBKo9SAhF9bA4gzgJsDgA-BjFrb238BA9sDgCRGisACRwAGigccw7QRwANRgAaRioABnEATCUF8RWAAA9IAAgaR0gAMw4KCnIAOWluZ0kAL2VkSQALD9cACQ8cAAiYDhEdMQcHIwXp1JyM4sqUsu1pCr4BKrCfAPAAngAB9QwSxAISqAISDNCxfh4HtQEAoQ8QpLQHAk4CwiYAatC-0LPQvhQAiBIo8BUQADoATNGL0LUSACrRhTgAI9CwTAAHcgAATAANOAAbjzgAW75tBvALJgA6g9GOEgAs0L6CAC3QviQAEbBcAAeAAEpPBmNIAAAYAWWsCDwBABOEAvkAgone2__i_ODlAQr7AyrtUBLzAtsBAfQOEv4IEuAIEg7Rg9C6BQUEfS1gEgoWAJeIMicA4wJxFAADKgBmEPMJoD4AIdCyLgA3tRBrGTESANvlGgHsAGEYChQwAFlKAmAyAANKAElpEsF2AAJKAAosAAIUAFmsC2BEAAMYACtvCUADXACryAQCRwUwjgB0DgQB9y1gSdGP0YKEIhLwjz-I0YzKAAITtcoAClYAMbzRghIBBvoAACoAAxIBKdGDtgAEngARtfoADzIAAQ8sAQQh0LDQAAe6ACHQuHABNA4KCswAAdwADVQAT9GL0LwaAAMFmgEGDAIAoAARuy4AD_gBAQDgAQYYAAPmACHRi5oADEQAAMYBBpwAG4gIAQU8AAE6AQ9UAAAB3AAGGAAAmAAAfAI_uNGFmgECE7jkdgAAahLyKwQCfAIJUAELCgIBpAAp0YvAAQEUAAB2AgHAAQZuAAUYAAFuAAaEAQS4ABO1CgIJcAAt0Lt6AgJKAgrWDvAIIdC7rgMGfgAGZgAPJAMDAjwCBjAAAJB_F_MLAU4BBhgABEgAEbhUAwaGAAEwAAkIAQYqAHQlJF5NBAA_AzQV-QDDn9HZ2s_81sUBCt0CKtCCBPEHPgEB9goSxgESVRIKYmFja2dyb3VuZGEDcg8KCxIAdnOJAXARAB0nEgB3RwPwEioOCkcATRABElZXADYRCg1GAF9pbmcQAGoADC5lZFgARx0KRGkA8gxEA-ESPBIiEgtpbnRlbGxpZ18j8xDJAAcTAAI9AAglAAU-AOHTARJLEgh0cmFuc2ZlcgUBWCYg8QC2AfAqDwABAQElDAovACAQAf0ABB4AAA0BKxJvTQA0EAoMIAAScvoAD18ABwH2AApfADQPCgs_AAI-AAcRIQHAABYBBZAABdYA1KMByjDyAXNlcnZpY2XVAAMPAACnAERuCoICkwADDgAB4t0jYEoQARJFSK4S8AIAowEFOQACnQAGHQAAOQEIRzQAIQRVXwMCHwQByB6fkNmj0NC6popBvxMCIPIHDQdCogESBh8EA3gGQQgOAIIzKhAO-Sgik9BzBhQKSjEgV7BJA3E6ABmLOgA4OQ9gKdCwDgBzVQ1AEAoMZl4DoLzQuD4AEL5cAEOrBxF5lwAHEQfY4__2qtGk64MoClIqRZAL8AQzAAH7DBJFEigSDtCy0LjQvdC0fwAh0YGpBwDxAw_DKgWvrNbfha7H8saFAacIAoj6DxLCAhL6AVgAQdGD0LfzAAFqKxKbYwfxBCoSCi4AHxAsAAERviwAD0QAAIy3CPAHFAoQXAAPFgABL9GDFgACH4sWAAGM0PQAQBgKFFh4C2HQuIoAAn8tAVASKPwAAl4bAVgVHPn_APkA8omD97a-ic6qAQqCAir1tgXW4wAB_xIS5AQSxgQSEgQBAdQKAEUQEQCwACQaADcXMjIAjxgHMBAAMp0AQIUYAAIZBwAHAECL0LUYJCwgtRj7CVDQvDAAAZ4No48YAAEh0LAYAHsSD_MWxgACZAA7FAoQGAAA2gAOFgAfsGAAAhG-2AA6EgoORAAuEAE2AbIHckYAAw9oAQRVHHEyAAEATAAPUADyC744AQ6yARO-sgEN2AAv0YsgAQgfuWQAAgEYBgEc9QYB2ODPj6GB5K3DQApMKkAFArIuAAH2BxI2Eh4SCTcrE3OvKjANChHiBQDGAxEQTgAC6gL4AwAAAAAw-tq7zK_BqvY-CnIqZk4AEVSGKx-BhisaASkWQRIoEhsBAjEIOAB9BxwrdAC_0omXtNqDmvtTCkcXLwZB0YXRgL0CDxcvAgO9AOmOm_CVjYH8lQMKrwEqohACEZD4DqJwEkESBnh0cmVtIxVgCwoHDgBjAgUwCgob9w4ALgSQGQA2JxAADgBpUQ2AEhhDAAc2AEMtAUVDAPcS8wT3J3QSWhILcGVyZm9ybWFuY2UYACACKhEKDRMAAFUACxMAAXYANxAKDCYAAKIAKA8KSwACcQAIEvsAAVkEn4-3-KiRx_W-bxIwCE_RhdC_-wAO8BHs9vjQu-y-q9sBEpIPCAsQABoQCgYqBBCszRkSAggAGgQA0A8KBSoDEOYKEgIIARoEAAIRAHCQBxICCAIaBAACEQBwhgISAggDGgQAAhEAcKweEgIIBBoEAAIRAHDEYBICCAUaBAAHEQAgBhoEAAJ4AIDorgMSAggHGgQAAiMAEJoRACAIGgQAAhEAcOosEgIICRoEAAI0AICuzQESAggKGgQAgB0KEwgBEAEhlQLhACCMQCoEEKKfIxICCAvLADoeChQfAFAFEPyhzz8AEQwgAAI_AJAQIXyXoXOSSYo_AHHa-RQSAggNHwACPwCQCCFY7o9JfJaNPwAwtsirPwARDiAAMR8KFT8ABX4AYAYQuJjikMIAEQ8hAAJgAJABIV24LWmnAY9gACDO2EAAERBZAQQfAHDuyi08B_aOHwAgsp8fABkRHwBhB0ZK1hsOPgB5lqYEEgIIEh8AYc2MfSSf8z4AEcBdABkTHwBnMTCaOAjUHwAZFB8AcBUHTKcN9I18AHmA3g0SAggVHwBhZmEqRwHLPgAg8uObABkWHwBhMKIgeX_1PgB55NczEgIIFx8AYawhD5xn9D4Ad9DNEhICCBgfAIEoIWSP8_0lhh8AcZDnGBICCBkfABEgtgHzDBAhbBwBtIyUQsAqBRC284YREgIIGhoECAEQAjkBkCEhxLnuH70Yi70AcYrGNxICCBugAgIfAIEBIZNP1el_OjkBBncBBh8AYTE2JtPjLh8AIMTB3AAZHD4AYXDHgNvQJJ4AcdacbBICCB0fAAIVAvABASHyO1hVremOQCoFEMzz6DoBGR4gAHASbDrXX7yLIACBjsaYZBICCB8gAAI1ApABIbYJAsO4x441AkDQ5ZaLTQMRICEABb4AUSGu6wMgXAEgvIo9ARkhHwBhMMdbmuHv8wIgnO2tAxEiHwAEnwBhrl2R_tiInwAwtv25NgIZIyAAYcP9JXiQ2CAAMKSro0AAESQgAAR-AGGLK-daiRBfAHnm1k4SAgglHwBh7EZcgN4SPAEgmMdeABEm-AMEPgBhRGexjSY2HwAg-vAQAxEnHwAxHAoSkAPxAtC9FxyK4I1AKgMQ3jwSAggoHgAEPQBh9t5A8_kRPQAgvvuYAREpHwARHhcCkAEhlLRkGAhPjFkBMMCe6iAAGSogAGFpZIFpINZ5ATDix-3YBBErIAAEXwBhtIzr7boa2AFxsvNNEgIILB8ABF8AYYY1tLkswBkBMOKug18AES0gAAQ_AGHSNJus7fwZAXnqgikSAgguHwBh8NvF-JqEtgEgoIu0AhEvHwACXgCBCCGBc64oXDJLBIHmufQGEgIIMCAAAl4AgQghOBnAMVET_AAG7AMTBfECgQEh7slu6-q8vABxkMUyEgIIMTsFAj4AgQEhMPynFpOSHwB3gJBEEgIIMh8AgQQh8KgtOBekuwB3holWEgIIMx8AgQghAReekbwQfAB54IAIEgIINB8AYdq5_0ule10Acb7sJhICCDWVBQTaAGFGheYVTpfaAImsttkQEgIINiAAYeB_xxzyESAAMNrOv-8CETcgABEfwwWRECEQW242YSd7zAPFuUcSAgg4GgQICRAKNQJwB8SqrXP5jjUCARQGGTkCBmGCqTQor5WeAHfy_SESAgg6HwCBBCHoEquIE8OXAXfE2kYSAgg7HwCBCCEMRqorWdI-ACCCk30AEDwfAC0iAAIAIQkJ7QXxCADwPy0AAIA_MgYSBBCc-QcyBRIDEKgIBwAhzAYHACGGAgcAIaARBwAhkFIHACHEYDIAIriRHQAhmgMWAJCULDIFEgMQhiQAAAA%2C;wmaxone%3D1.000;wminone%3D1.000;ydngd_mean%3D0.261882;ydngi%3D0.5063;ynhg_mean%3D0.1904;zsset%3DeNptV0ty20YQvUrvsrJLkiuu5DggMCRh4ucBYIlekZJtKbFd3Lkqt6AkgoIpkroCULlQ3usBTMpOSWj09Pz687ob9CIj7by9bJbtJf4W8urk5emZjEqTFxKbUuIyKiQzNk8Tae6autk1m3YuzbZZNpumbhfNWk5fc8/p69fy%2Bx8nJ%2BL5RYjVXhCIF1rxkrQYG7xtWiYQ5fjP8R9iyblnAxl4/gQkikCKAgoNjFeADFMLPko5mw7E9xI%2BvxWgeNI0AimTwtgXeWHDiRHfeu%2BnEnh2AlKUsQTGC0iKMWicShAaCdIRnoQPjgrSc7DW5Dlo%2BA7TZTEVMxwavxATi7E2tWJy38uMmHcww1zI0PMNiMWTj2nqMByNC1DoCwLPDSPMCAzgwjKRkRcbGYVDPDaSURoFBsI0DWRkvYSUJo%2B9aChj6gx/pTIOCwljb2QkzCOuCotc3mQjeVPGmUxCOGwSJqMgjSXyBiaSCAqChgPr2alEKQ6K0mQksWdDDxRGxvB6nAbG8gVzQeCTdDgU6CRpEk0lzfBvrKdBTEtE3yv8sWS%2BZKFflLAxi7ypZDg4S8ME8ymCOUDcMpv69GSWv5K3iiBLT8G2COdak5WDKPTFhjmE1M6m0N3CIgsf5R4Pyz1olRvfGvBjD/GRPIwznJ5nxgSgqcVMYQ0XlFBU8mk8CAEOoGxYkE5B4BhSwKtICy8CLWFEAb%2BUmSCOUwH28NA0UBxzPg5xy3kY4klUECYXIMWYTrg4EyD%2BjnmCZ/ZsgERYUvCIfJgr62TNStn2Uppbld02tWPnmK6w%2BbbZN49OdNPskH5gq2at2fjFDXZ81Vh7x5NveUnzgCys2k%2Bi%2B2/dS7OSzJbHMVcrXs0LeBUFYHFNhfld%2B7n5DhEUqTiP83a4dEHRZ71v5nRAbYDwXk/ZuyP0CrWY4l3H4q4r6LXXGrGRZqWLKzIV2fbaLeRwy31k15DP%2BOaAx7qtvNytxlYM3KErVbPmmRWurpsnZ1eFGSx9wJ57%2Btzt5JD3bES12jrf11SU0x3j3OwGupa%2B1SrXLvq5rV40Y60Tnt1%2B4AEbdfCcCmzgkyUN2mhwNmRgIHxLh3/RIfZvGcXDUOPCCGho9xBDWOGuPUW6joLuim6Dns4I3D9j4dxKMVUzqlut0rWo/eobx3RBx4CwaD/xCKJv234GLNRrW5h9LYqFGUr7yrFzVXXlfK8Ch6v2S3/mToNcuU4iehsFTInNM4GzbN88AS0Oy/QVQXeYVEgo8J0rFADSQQhmPenKazK1Tt479hNMWpLdPw8vBUy3mbJYBMjj%2Bu9u%2BBUR19A%2BOf9r4F1Q1F6Ka0Jdbb7rsuVoascTsUMzX0U/wtEJ9kRjx0K%2B7dlL7L7TwzYqghfWDoZ0zwPCMlP926suPpoYD0w1CJx1auZcAbLs7lQ7iArnvLrLoEM%2Bq2BNJ3eGVopWB59fJvdyQCveXKE1Qq%2BaE/kadLLQeof0vVQNNk70USufzrssXnbugwDJhS2PdDnDThEi034UzZ1Nn/rtX245wzXrvabAg4p0yfrYuF5U/biHCnUvLdnOO4teVHUoPQwB7NvngsWva1yBgODzEdjoCtZE0WtYVREGceYhkTsV5pRd0RdMM7fxStdeK8r6BgBfXGuf6Kz9oIVo%2B4NRUOtgwVfdw%2BSD1vwb4W6tX7VWXIJGs64fcLe%2BFIJ0M5QgACcDKQI5OX15cionZ6RnZ6/k7E9%2B5/lGu2OArx2/wMcRvj/QufktVMhoIOGFTN5LPM3fRvgyyMoIbdaizeclzvQzKZOwQJs1gzhN0Km1j95ru2In09cLhdV317Rql0eO0aLPSqStSq28Zop0Vd01kco1F1c8rrBjd6iplWujD3rtXgvwUemheO9eTxqoG8Va3ybY1RGSS%2BaKtgKubD%2B6qucawE%2BiQ1F2vYGJdimaS7dqSMXKxuGG9ZhWPbbfiJ0OUVfOsO1RFi9cPX%2Bg14jwtVItwlxw33WXVS/QOXwm0BjMdBWbgr%2BBvbnW7q6g7bu0n7uavXb5evgOcI1QG%2B7BLf8j7Fer27vKq1l51/dAN3xgV9J62WnlqqVGk78wDoVyr33isRd0FRfRXR61IYXNSvSj5t4BgTnXta%2Bjoq5W37sq476Tbg7l4qoP3k/lHpXM2VO7zdddI9zpd077VdnnrRhlY6eAa79BlRsXiidNwJVEU4uP4UnIX1exF4XJRJr1v/8wHtKdsDoCZ/uNv2HM5D%2Bak0fD;relevgeo%3D225;geochanged%3D1;;udbts%3D1459797057;downerprotobundle%3DCAwVQbwCVw%2C%2C&reqid=1461253060957657-600715567702129748010289-myt1-1909&snip=exps%3Dmarker_ratings_info%2Cmarker_movie_rating%3D0;report%3Dwww;reps%3Dall_on;snip_width%3D550;uil%3Dru;dqsigml%3D4&snip=t%3D0&text=%28autocad%3A%3A209750+^+autodesk%3A%3A288721+^+%28%28autodesk%3A%3A288721+%26/%281+1%29+inventor%3A%3A1697918%29%29+^+%28%28auto%3A%3A28861+%26/%281+1%29+cad%3A%3A171629%29%29+^+%D0%B0%D0%B2%D1%82%D0%BE%D0%BA%D0%B0%D0%B4%3A%3A1405467+^+autoca%3A%3A420234780%29+%26%26/%28-3+5%29+%28%28%D1%81%D0%BA%D0%B0%D1%87%D0%B0%D1%82%D1%8C%3A%3A691+^+download%3A%3A13863+^+pdf%3A%3A10201+^+torrent%3A%3A35211+^+%28%28%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D1%82%D0%BD%D0%BE%3A%3A456+%26/%281+1%29+%D0%B7%D0%B0%D0%B3%D1%80%D1%83%D0%B7%D0%B8%D1%82%D1%8C%3A%3A13856%29%29+^+%D0%B7%D0%B0%D0%B3%D1%80%D1%83%D0%B6%D0%B0%D1%82%D1%8C%3A%3A13856+^+%D0%B7%D0%B0%D0%BA%D0%B0%D1%87%D0%B8%D0%B2%D0%B0%D1%82%D1%8C%3A%3A112512+^+%D1%81%D0%BA%D0%B0%D1%87%D0%B8%D0%B2%D0%B0%D0%BD%D0%B8%D0%B5%3A%3A14585+^+%D1%81%D0%BA%D0%B0%D1%87%D0%B8%D0%B2%D0%B0%D1%82%D1%8C%D1%81%D1%8F%3A%3A423410+^+%D1%81%D0%BA%D0%B0%D1%87%D0%BA%D0%B0%3A%3A152424+^+skachat%3A%3A203208%29+%26/%28-64+64%29+%28%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D1%82%D0%BD%D0%BE%3A%3A456+^+besplatno%3A%3A455045+^+download%3A%3A13863+^+free%3A%3A12386+^+%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%3A%3A886571+^+%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D1%82%D0%BD%D0%B8%D0%B9%3A%3A14490854+^+%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D1%82%D0%BD%D0%B8%D0%BA%3A%3A105058695+^+%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D1%82%D0%BD%D0%BD%D1%8B%D0%B9%3A%3A280156520+^+%D0%B4%D0%B0%D1%80%D0%BE%D0%BC%3A%3A148126+^+%D1%81%D0%B2%D0%BE%D0%B1%D0%BE%D0%B4%D0%BD%D0%BE%3A%3A88910+^+%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D0%BD%D0%BE%3A%3A4669275+^+%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D1%82%D0%BE%3A%3A10775250+^+%D1%85%D0%B0%D0%BB%D1%8F%D0%B2%D0%BD%D1%8B%D0%B9%3A%3A644531%29%29+^+%D1%81%D0%BA%D0%B0%D1%87%D0%B0%D1%82%D1%8C%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D1%82%D0%BD%D0%BE%3A%3A17882331+%26/%28-64+64%29+%D0%BD%D0%B0%3A%3A131+%26/%28-64+64%29+%28%D1%80%D1%83%D1%81%D1%81%D0%BA%D0%BE%D0%BC%3A%3A1942+^+russia%3A%3A37324+^+russian%3A%3A31805+^+%D1%80%D0%BE%D1%81%D1%81%D0%B8%D0%B9%D1%81%D0%BA%D0%B8%D0%B9%3A%3A3887+^+rus%3A%3A16095+^+%D0%B2%D0%B5%D0%BB%D0%B8%D0%BA%D0%BE%D1%80%D1%83%D1%81%D1%81%D0%BA%D0%B8%D0%B9%3A%3A1918880+^+%D1%80%D1%83%D1%81%D0%B8%D1%81%D1%82%D0%B8%D0%BA%D0%B0%3A%3A27111921+^+%D1%80%D1%83%D1%81%D0%B8%D1%84%D0%B8%D0%BA%D0%B0%D1%86%D0%B8%D1%8F%3A%3A638169+^+!!%D1%80%D1%83%D1%81%D0%BA%3A%3A1076145+^+%D1%80%D1%83%D1%81%D1%81%3A%3A336053+^+%D1%80%D1%83%D1%81%D1%81%D0%BA%D0%BE%D1%8F%D0%B7%D1%8B%D1%87%D0%BD%D1%8B%D0%B9%3A%3A139984+^+russkom%3A%3A7245427%29+%26%26/%28-3+5%29+%28%D1%82%D0%BE%D1%80%D1%80%D0%B5%D0%BD%D1%82%3A%3A6178+^+torrent%3A%3A35211%29+%26%26/%28-3+5%29+32%3A%3A6178+%26/%28-1+1%29+%28bit%3A%3A27572+^+%D0%B1%D0%B8%D1%82%D0%BD%D1%8B%D0%B9%3A%3A414024+^+%D1%83%D0%BA%D1%83%D1%81%D0%B8%D1%82%D1%8C%3A%3A558080+^+%28%28background%3A%3A96185+%26/%281+1%29+intelligent%3A%3A705091+%26/%281+1%29+transfer%3A%3A180746+%26/%281+1%29+service%3A%3A17789%29%29+^+%D0%B1%D0%B8%D1%82%3A%3A65584%29+%26%26/%28-3+5%29+%D0%B4%D0%BB%D1%8F%3A%3A205+%26/%28-64+64%29+%28%28windows%3A%3A2869+^+%D0%B2%D0%B8%D0%BD%D0%B4%D0%BE%D0%B2%D1%81%3A%3A318239+^+%D0%B2%D0%B8%D0%BD%D0%B4%D0%BE%D1%83%D0%B7%3A%3A17509782+^+%D0%B2%D0%B8%D0%BD%D0%B4%D0%BE%D1%83%D1%81%3A%3A2618285%29+%26%26/%28-3+5%29+%28xp%3A%3A13143+^+windows%3A%3A2869+^+%D1%85%D1%80%3A%3A278393+^+%28%28xtreme%3A%3A579234+%26/%281+1%29+performance%3A%3A113530%29%29+^+%D1%85%D0%BF%3A%3A582849%29%29+^+windowsxp%3A%3A585285+softness%3A6&tld=ru&uil=ru&user_request=autocad+%D1%81%D0%BA%D0%B0%D1%87%D0%B0%D1%82%D1%8C+%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D1%82%D0%BD%D0%BE+%D0%BD%D0%B0+%D1%80%D1%83%D1%81%D1%81%D0%BA%D0%BE%D0%BC+%D1%82%D0%BE%D1%80%D1%80%D0%B5%D0%BD%D1%82+32+bit+%D0%B4%D0%BB%D1%8F+windows+xp
\ No newline at end of file diff --git a/library/cpp/streams/lz/ut/yq_609.data b/library/cpp/streams/lz/ut/yq_609.data Binary files differnew file mode 100644 index 00000000000..9694b0c97f1 --- /dev/null +++ b/library/cpp/streams/lz/ut/yq_609.data diff --git a/library/cpp/streams/lzop/lzop.cpp b/library/cpp/streams/lzop/lzop.cpp new file mode 100644 index 00000000000..5d1ad33f700 --- /dev/null +++ b/library/cpp/streams/lzop/lzop.cpp @@ -0,0 +1,827 @@ +#include "lzop.h" + +#include <util/generic/buffer.h> + +#include <util/system/info.h> + +#include <contrib/libs/minilzo/minilzo.h> + +// See https://svn.yandex.ru/statbox/packages/yandex/statbox-binaries/include/Statbox/LZOP.h +// https://github.yandex-team.ru/logbroker/push-client/blob/c820971a769df920d6ea9152a053474e75986914/src/lzo.c +// https://github.yandex-team.ru/logbroker/push-client/blob/c820971a769df920d6ea9152a053474e75986914/src/lzo.h +// As the source for the inspiration. +//////////////////////////////////////////////////////////////////////////////// + +namespace NPrivate { + namespace NLzop { + static unsigned const char MAGIC[9] = + { + 0x89, 0x4c, 0x5a, 0x4f, + 0x00, + 0x0d, 0x0a, 0x1a, 0x0a}; + + // 32-bit Version. + inline unsigned int RoundUpToPow2(unsigned int x) { + x -= 1; + x |= (x >> 1); + x |= (x >> 2); + x |= (x >> 4); + x |= (x >> 8); + x |= (x >> 16); + return x + 1; + } + + inline unsigned char* Get8(unsigned char* p, unsigned* v, ui32* adler32, ui32* crc32) { + *v = 0; + *v |= (unsigned)(*p++); + + *adler32 = lzo_adler32(*adler32, (const unsigned char*)(p - 1), 1); + *crc32 = lzo_crc32(*crc32, (const unsigned char*)(p - 1), 1); + + return p; + } + + inline unsigned char* Put8(unsigned char* p, unsigned v, ui32* adler32, ui32* crc32) { + *p++ = v & 0xff; + + *adler32 = lzo_adler32(*adler32, (const unsigned char*)(p - 1), 1); + *crc32 = lzo_crc32(*crc32, (const unsigned char*)(p - 1), 1); + + return p; + } + + inline unsigned char* Get16(unsigned char* p, unsigned* v, ui32* adler32, ui32* crc32) { + *v = 0; + *v |= (unsigned)(*p++) << 8; + *v |= (unsigned)(*p++); + + *adler32 = lzo_adler32(*adler32, (const unsigned char*)(p - 2), 2); + *crc32 = lzo_crc32(*crc32, (const unsigned char*)(p - 2), 2); + + return p; + } + + inline unsigned char* Put16(unsigned char* p, unsigned v, ui32* adler32, ui32* crc32) { + *p++ = (v >> 8) & 0xff; + *p++ = (v)&0xff; + + *adler32 = lzo_adler32(*adler32, (const unsigned char*)(p - 2), 2); + *crc32 = lzo_crc32(*crc32, (const unsigned char*)(p - 2), 2); + + return p; + } + + inline unsigned char* Get32(unsigned char* p, unsigned* v, ui32* adler32, ui32* crc32) { + *v = 0; + *v |= (unsigned)(*p++) << 24; + *v |= (unsigned)(*p++) << 16; + *v |= (unsigned)(*p++) << 8; + *v |= (unsigned)(*p++); + + *adler32 = lzo_adler32(*adler32, (const unsigned char*)(p - 4), 4); + *crc32 = lzo_crc32(*crc32, (const unsigned char*)(p - 4), 4); + + return p; + } + + inline unsigned char* Put32(unsigned char* p, unsigned v, ui32* adler32, ui32* crc32) { + *p++ = (v >> 24) & 0xff; + *p++ = (v >> 16) & 0xff; + *p++ = (v >> 8) & 0xff; + *p++ = (v)&0xff; + + *adler32 = lzo_adler32(*adler32, (const unsigned char*)(p - 4), 4); + *crc32 = lzo_crc32(*crc32, (const unsigned char*)(p - 4), 4); + + return p; + } + + enum ELzoFlag { + LZO_ADLER32_D = 0x00000001L, + LZO_ADLER32_C = 0x00000002L, + LZO_STDIN = 0x00000004L, + LZO_STDOUT = 0x00000008L, + LZO_NAME_DEFAULT = 0x00000010L, + LZO_DOSISH = 0x00000020L, + LZO_H_EXTRA_FIELD = 0x00000040L, + LZO_H_GMTDIFF = 0x00000080L, + LZO_CRC32_D = 0x00000100L, + LZO_CRC32_C = 0x00000200L, + LZO_MULTIPART = 0x00000400L, + LZO_H_FILTER = 0x00000800L, + LZO_H_CRC32 = 0x00001000L, + LZO_H_PATH = 0x00002000L, + LZO_MASK = 0x00003FFFL + }; + + enum ELzoResult { + LZO_END_OF_STREAM = 0, + LZO_MORE_DATA = 1, + LZO_OK = 2, + LZO_WRONG_MAGIC = -12, + LZO_VERSION_TOO_LOW = -13, + LZO_EXTRACT_VERSION_TOO_HIGH = -14, + LZO_EXTRACT_VERSION_TOO_LOW = -15, + LZO_WRONG_CHECKSUM = -16, + LZO_WRONG_METHOD = -18, + LZO_COMPRESS_ERROR = -1, + LZO_WRONG_DST_LEN = -2, + LZO_DST_LEN_TOO_BIG = -3, + LZO_WRONG_SRC_LEN = -4, + LZO_INVALID_SRC_ADLER32 = -5, + LZO_INVALID_SRC_CRC32 = -6, + LZO_DECOMPRESS_ERROR = -7, + LZO_INVALID_DST_ADLER32 = -8, + LZO_INVALID_DST_CRC32 = -9, + }; + + // XXX(sandello): I don't really know where this comes from. + struct THeader { + unsigned Version; + unsigned LibVersion; + unsigned VersionNeededToExtract; + unsigned char Method; + unsigned char Level; + + ui32 Flags; + ui32 Filter; + ui32 Mode; + ui32 MTimeLow; + ui32 MTimeHigh; + + ui32 HeaderChecksum; + + ui32 ExtraFieldLen; + ui32 ExtraFieldChecksum; + + const unsigned char* MethodName; + + char Name[255 + 1]; + }; + + } +} + +//////////////////////////////////////////////////////////////////////////////// + +class TLzopCompress::TImpl + : public IOutputStream { +public: + inline TImpl(IOutputStream* slave, ui16 blockSize) + : Slave(slave) + , HeaderWasWritten(false) + , UncompressedBuffer(blockSize) + , CompressedBuffer(8 + 4 * blockSize) + { + ResetHeader(); + } + +protected: + void DoWrite(const void* buffer, size_t length) override; + void DoFlush() override; + void DoFinish() override; + +private: + IOutputStream* Slave; + + NPrivate::NLzop::THeader Header; + bool HeaderWasWritten; + + TBuffer UncompressedBuffer; + TBuffer CompressedBuffer; + + void EnsureCompressedSpace(size_t demand); + void EnsureUncompressedSpace(size_t demand); + + void ProduceHeader(); + void ProduceData(); + void ProduceTrailer(); + void ResetHeader(); +}; + +void TLzopCompress::TImpl::DoWrite(const void* buffer, size_t length) { + const char* data = (const char*)buffer; + while (length > 0) { + size_t bytesToFit = Min(UncompressedBuffer.Capacity(), length); + size_t bytesToWrite = Min(UncompressedBuffer.Avail(), length); + if (bytesToWrite > 0) { + UncompressedBuffer.Append(data, bytesToWrite); + data += bytesToWrite; + length -= bytesToWrite; + } else { + EnsureUncompressedSpace(bytesToFit); + } + } +} + +void TLzopCompress::TImpl::DoFlush() { + EnsureUncompressedSpace(UncompressedBuffer.Capacity()); + EnsureCompressedSpace(CompressedBuffer.Capacity()); +} + +void TLzopCompress::TImpl::DoFinish() { + EnsureUncompressedSpace(UncompressedBuffer.Capacity()); + ProduceTrailer(); + Flush(); +} + +void TLzopCompress::TImpl::EnsureCompressedSpace(size_t demand) { + Y_ASSERT(demand <= CompressedBuffer.Capacity()); + if (CompressedBuffer.Avail() < demand) { + Slave->Write(CompressedBuffer.Data(), CompressedBuffer.Size()); + CompressedBuffer.Clear(); + } + Y_ASSERT(demand <= CompressedBuffer.Avail()); +} + +void TLzopCompress::TImpl::EnsureUncompressedSpace(size_t demand) { + Y_ASSERT(demand <= UncompressedBuffer.Capacity()); + if (UncompressedBuffer.Avail() < demand) { + ProduceData(); + } + Y_ASSERT(demand <= UncompressedBuffer.Avail()); +} + +void TLzopCompress::TImpl::ResetHeader() { + ::memset(&Header, 0, sizeof(Header)); +} + +void TLzopCompress::TImpl::ProduceHeader() { + using namespace NPrivate::NLzop; + + ui32 adler32 = 1; + ui32 crc32 = 0; + + unsigned char* p; + unsigned char* pb; + + EnsureCompressedSpace(sizeof(MAGIC) + sizeof(Header)); + pb = p = (unsigned char*)CompressedBuffer.Pos(); + + // Magic. + ::memcpy(p, MAGIC, sizeof(MAGIC)); + p += sizeof(MAGIC); + + // .Version + p = Put16(p, 0x1030U, &adler32, &crc32); + // .LibVersion + p = Put16(p, lzo_version() & 0xFFFFU, &adler32, &crc32); + // .VersionNeededToExtract + p = Put16(p, 0x0900, &adler32, &crc32); + // .Method + // XXX(sandello): Method deviates from Statbox' implementation. + // In compatibility we trust. + p = Put8(p, 2, &adler32, &crc32); // 1 = LZO1X_1, 2 = LZO1X_1_15 + // .Level + p = Put8(p, 3, &adler32, &crc32); + // .Flags + p = Put32(p, 0, &adler32, &crc32); + // .Mode + p = Put32(p, 0644, &adler32, &crc32); + // .MTimeLow + p = Put32(p, 0, &adler32, &crc32); + // .MTimeHigh + p = Put32(p, 0, &adler32, &crc32); + // .Name + p = Put8(p, 0, &adler32, &crc32); + // .HeaderChecksum + p = Put32(p, adler32, &adler32, &crc32); + + CompressedBuffer.Proceed(CompressedBuffer.Size() + (p - pb)); +} + +void TLzopCompress::TImpl::ProduceTrailer() { + using namespace NPrivate::NLzop; + + ui32 adler32 = 1; + ui32 crc32 = 0; + + unsigned char* p; + unsigned char* pb; + + EnsureCompressedSpace(4); + pb = p = (unsigned char*)CompressedBuffer.Pos(); + + p = Put32(p, 0, &adler32, &crc32); + + CompressedBuffer.Proceed(CompressedBuffer.Size() + (p - pb)); +} + +void TLzopCompress::TImpl::ProduceData() { + using namespace NPrivate::NLzop; + + ui32 srcLen = (ui32)UncompressedBuffer.Size(); + ui32 dstLen; + + ui32 adler32 = 1; + ui32 crc32 = 0; + + unsigned char* p; + unsigned char* pb; + + lzo_uint result; + + // See include/lzo/lzo1x.h from lzo-2.06. + // const size_t LZO1X_1_MEM_COMPRESS = (lzo_uint32)(16384L * lzo_sizeof_dict_t); + unsigned char scratch[LZO1X_1_MEM_COMPRESS]; + + if (!HeaderWasWritten) { + ProduceHeader(); + HeaderWasWritten = true; + } + + EnsureCompressedSpace(8 + 4 * srcLen); + pb = p = (unsigned char*)CompressedBuffer.Pos(); + + p = Put32(p, srcLen, &adler32, &crc32); + p += 4; + + // XXX(sandello): Used compression Method deviates from Statbox's implementation. + // Here we use |lzo1x_1_compress| (implemented in minilzo) whilst Statbox + // uses |lzo1x_1_15_compress|. + if (lzo1x_1_compress( + (unsigned char*)UncompressedBuffer.Data(), + UncompressedBuffer.Size(), + p, + &result, + scratch) != LZO_E_OK) + { + ythrow yexception() << "LZOP Error: " << (int)LZO_COMPRESS_ERROR; + } + + dstLen = result; + + if (dstLen < srcLen) { + Put32(pb + 4, dstLen, &adler32, &crc32); + /**/ + result = dstLen; + } else { + Put32(pb + 4, srcLen, &adler32, &crc32); + ::memcpy(p, UncompressedBuffer.Data(), UncompressedBuffer.Size()); + result = srcLen; + } + + result += 4 + 4; // srcLen + dstLen + (adler32|crc32, disabled) + + UncompressedBuffer.Clear(); + CompressedBuffer.Proceed(CompressedBuffer.Size() + result); +} + +TLzopCompress::TLzopCompress(IOutputStream* slave, ui16 maxBlockSize) + : Impl_(new TImpl(slave, maxBlockSize)) +{ +} + +TLzopCompress::~TLzopCompress() { + try { + Finish(); + } catch (...) { + } +} + +void TLzopCompress::DoWrite(const void* buffer, size_t length) { + if (!Impl_) { + ythrow yexception() << "Stream is dead"; + } + Impl_->Write((const char*)buffer, length); +} + +void TLzopCompress::DoFlush() { + if (!Impl_) { + ythrow yexception() << "Stream is dead"; + } + Impl_->Flush(); +} + +void TLzopCompress::DoFinish() { + THolder<TImpl> impl(Impl_.Release()); + if (!!impl) { + impl->Finish(); + } +} + +//////////////////////////////////////////////////////////////////////////////// + +class TLzopDecompress::TImpl + : public IInputStream { +public: + inline TImpl(IInputStream* slave, ui32 initialBufferSize) + : Slave(slave) + , Exhausted(false) + , Hint(0) + , InputData(NPrivate::NLzop::RoundUpToPow2(initialBufferSize)) + , OutputData(NPrivate::NLzop::RoundUpToPow2(initialBufferSize)) + , InputOffset(0) + , OutputOffset(0) + { + ResetHeader(); + } + +protected: + size_t DoRead(void* buffer, size_t length) override; + +private: + IInputStream* Slave; + bool Exhausted; + unsigned int Hint; + + NPrivate::NLzop::THeader Header; + + TBuffer InputData; + TBuffer OutputData; + + size_t InputOffset; + size_t OutputOffset; + + void Trim(TBuffer& buffer, size_t& length); + + int ConsumeHeader(); + int ConsumeData(); + void ResetHeader(); +}; + +size_t TLzopDecompress::TImpl::DoRead(void* buffer, size_t length) { + size_t bytesRead = 0; + size_t bytesAvailable = 0; + + do { + bytesAvailable = Min(OutputData.Size() - OutputOffset, length); + if (!bytesAvailable && !Exhausted) { + int rv; + while ((rv = ConsumeData()) == NPrivate::NLzop::LZO_MORE_DATA) { + if (Hint) { + InputData.Reserve(NPrivate::NLzop::RoundUpToPow2(Hint)); + Hint = 0; + } else if (InputData.Avail() == 0) { + InputData.Reserve(2 * InputData.Capacity()); + } + + size_t tmp = Slave->Load(InputData.Pos(), InputData.Avail()); + if (tmp) { + InputData.Advance(tmp); + } else { + Exhausted = true; + break; + } + } + + Trim(InputData, InputOffset); + + switch (rv) { + case NPrivate::NLzop::LZO_OK: + case NPrivate::NLzop::LZO_MORE_DATA: + break; + case NPrivate::NLzop::LZO_END_OF_STREAM: + ResetHeader(); + break; + default: + ythrow yexception() << "LZOP Error: " << rv; + break; + } + } else if (bytesAvailable) { + ::memcpy( + (char*)buffer + bytesRead, + OutputData.Data() + OutputOffset, + bytesAvailable); + bytesRead += bytesAvailable; + OutputOffset += bytesAvailable; + + Trim(OutputData, OutputOffset); + } else { + break; + } + } while (!bytesRead); + + return bytesRead; +} + +void TLzopDecompress::TImpl::ResetHeader() { + ::memset(&Header, 0, sizeof(Header)); +} + +void TLzopDecompress::TImpl::Trim(TBuffer& buffer, size_t& length) { + size_t remaining = buffer.Size() - length; + ::memmove( + buffer.Data(), + buffer.Data() + length, + remaining); + buffer.Resize(remaining); + length = 0; +} + +int TLzopDecompress::TImpl::ConsumeHeader() { + using namespace NPrivate::NLzop; + + THeader* h = &Header; + + ui32 adler32 = 1; + ui32 crc32 = 0; + ui32 checksum; + + unsigned tmp; + + unsigned char* p; + unsigned char* pb; + unsigned char* pe; + + pb = p = (unsigned char*)InputData.Data() + InputOffset; + pe = (unsigned char*)InputData.Pos(); + + // Magic. + if (pe < p + sizeof(MAGIC)) + return LZO_MORE_DATA; + if (memcmp(MAGIC, p, sizeof(MAGIC)) != 0) { + return LZO_WRONG_MAGIC; + } + p += sizeof(MAGIC); + + // .Version + if (pe - p < 2) + return LZO_MORE_DATA; + p = Get16(p, &h->Version, &adler32, &crc32); + if (h->Version < 0x0900) { + return LZO_VERSION_TOO_LOW; + } + + // .LibVersion, .VersionNeededToExtract + if (pe - p < 2) + return LZO_MORE_DATA; + p = Get16(p, &h->LibVersion, &adler32, &crc32); + if (h->Version >= 0x0940) { + if (pe - p < 2) + return LZO_MORE_DATA; + p = Get16(p, &h->VersionNeededToExtract, &adler32, &crc32); + if (h->VersionNeededToExtract > 0x1030) { + return LZO_EXTRACT_VERSION_TOO_HIGH; + } + if (h->VersionNeededToExtract < 0x0900) { + return LZO_EXTRACT_VERSION_TOO_LOW; + } + } + + // .Method, .Level + if (pe - p < 1) + return LZO_MORE_DATA; + p = Get8(p, &tmp, &adler32, &crc32); + h->Method = tmp; + if (h->Version >= 0x0940) { + if (pe - p < 1) + return LZO_MORE_DATA; + p = Get8(p, &tmp, &adler32, &crc32); + h->Level = tmp; + } + + // .Flags + if (pe - p < 4) + return LZO_MORE_DATA; + p = Get32(p, &h->Flags, &adler32, &crc32); + + // .Filter + if (h->Flags & LZO_H_FILTER) { + if (pe - p < 4) + return LZO_MORE_DATA; + p = Get32(p, &h->Filter, &adler32, &crc32); + } + + // .Mode + if (pe - p < 4) + return LZO_MORE_DATA; + p = Get32(p, &h->Mode, &adler32, &crc32); + + // .MTimeLow + if (pe - p < 4) + return LZO_MORE_DATA; + p = Get32(p, &h->MTimeLow, &adler32, &crc32); + + // .MTimeHigh + if (h->Version >= 0x0940) { + if (pe - p < 4) + return LZO_MORE_DATA; + p = Get32(p, &h->MTimeHigh, &adler32, &crc32); + } + if (h->Version < 0x0120) { + if (h->MTimeLow == 0xffffffffUL) { + h->MTimeLow = 0; + } + h->MTimeHigh = 0; + } + + // .Name + if (pe - p < 1) + return LZO_MORE_DATA; + p = Get8(p, &tmp, &adler32, &crc32); + if (tmp > 0) { + if (pe - p < tmp) + return LZO_MORE_DATA; + adler32 = lzo_adler32(adler32, p, tmp); + crc32 = lzo_crc32(crc32, p, tmp); + + ::memcpy(h->Name, p, tmp); + p += tmp; + } + + if (h->Flags & LZO_H_CRC32) { + checksum = crc32; + } else { + checksum = adler32; + } + + // .HeaderChecksum + if (pe - p < 4) + return LZO_MORE_DATA; + p = Get32(p, &h->HeaderChecksum, &adler32, &crc32); + if (h->HeaderChecksum != checksum) { + return LZO_WRONG_CHECKSUM; + } + + // XXX(sandello): This is internal Statbox constraint. + // XXX(aozeritsky): Statbox uses Method = 2, Java uses Method = 1 + // XXX(aozeritsky): Both methods use the same decompression function + if (!(h->Method == 1 || h->Method == 2)) { + return LZO_WRONG_METHOD; + } + + if (h->Flags & LZO_H_EXTRA_FIELD) { + if (pe - p < 4) + return LZO_MORE_DATA; + p = Get32(p, &h->ExtraFieldLen, &adler32, &crc32); + if (pe - p < h->ExtraFieldLen) + return LZO_MORE_DATA; + p += h->ExtraFieldLen; + } + + // OK + InputOffset += p - pb; + return LZO_OK; +} + +int TLzopDecompress::TImpl::ConsumeData() { + using namespace NPrivate::NLzop; + + THeader* h = &Header; + + ui32 adler32 = 1; + ui32 crc32 = 0; + + ui32 dAdler32 = 1; + ui32 dCrc32 = 0; + ui32 cAdler32 = 1; + ui32 cCrc32 = 0; + + ui32 dstLen; + ui32 srcLen; + + unsigned char* p; + unsigned char* pb; + unsigned char* pe; + + if (h->Version == 0) { + return ConsumeHeader(); + } + + pb = p = (unsigned char*)InputData.Data() + InputOffset; + pe = (unsigned char*)InputData.Pos(); + + // dstLen + if (pe - p < 4) + return LZO_MORE_DATA; + p = Get32(p, &dstLen, &adler32, &crc32); + + if (dstLen == 0) { + InputOffset += p - pb; + return LZO_END_OF_STREAM; + } + if (dstLen == 0xffffffffUL) { + return LZO_WRONG_DST_LEN; + } + if (dstLen > 64 * 1024 * 1024) { + return LZO_DST_LEN_TOO_BIG; + } + + // srcLen + if (pe - p < 4) + return LZO_MORE_DATA; + p = Get32(p, &srcLen, &adler32, &crc32); + + if (srcLen <= 0 || srcLen > dstLen) { + return LZO_WRONG_SRC_LEN; + } + + if (h->Flags & LZO_ADLER32_D) { + if (pe - p < 4) + return LZO_MORE_DATA; + p = Get32(p, &dAdler32, &adler32, &crc32); + } + if (h->Flags & LZO_CRC32_D) { + if (pe - p < 4) + return LZO_MORE_DATA; + p = Get32(p, &dCrc32, &adler32, &crc32); + } + + if (h->Flags & LZO_ADLER32_C) { + if (srcLen < dstLen) { + if (pe - p < 4) + return LZO_MORE_DATA; + p = Get32(p, &cAdler32, &adler32, &crc32); + } else { + if (!(h->Flags & LZO_ADLER32_D)) + ythrow yexception() << "h->Flags & LZO_ADLER32_C & ~LZO_ADLER32_D"; + cAdler32 = dAdler32; + } + } + if (h->Flags & LZO_CRC32_C) { + if (srcLen < dstLen) { + if (pe - p < 4) + return LZO_MORE_DATA; + p = Get32(p, &cCrc32, &adler32, &crc32); + } else { + if (!(h->Flags & LZO_CRC32_D)) + ythrow yexception() << "h->Flags & LZO_CRC32_C & ~LZO_CRC32_D"; + cCrc32 = dCrc32; + } + } + + // Rock'n'roll! Check'n'consume! + if (pe - p < srcLen) { + Hint = (p - pb) + srcLen; + return LZO_MORE_DATA; + } + + if (h->Flags & LZO_ADLER32_C) { + ui32 checksum; + checksum = lzo_adler32(1, p, srcLen); + if (checksum != cAdler32) { + return LZO_INVALID_SRC_ADLER32; + } + } + if (h->Flags & LZO_CRC32_C) { + ui32 checksum; + checksum = lzo_crc32(1, p, srcLen); + if (checksum != cCrc32) { + return LZO_INVALID_SRC_CRC32; + } + } + + if (OutputData.Avail() < dstLen) { + OutputData.Reserve(RoundUpToPow2(2 * (OutputData.Size() + dstLen))); + } + + unsigned char* output = (unsigned char*)OutputData.Pos(); + OutputData.Advance(dstLen); + + if (srcLen < dstLen) { + lzo_uint tmp; + int rv; + + tmp = dstLen; + rv = lzo1x_decompress_safe( + p, + srcLen, + output, + &tmp, + 0); + + if (rv != LZO_E_OK || tmp != dstLen) { + return LZO_DECOMPRESS_ERROR; + } + } else { + if (!(dstLen == srcLen)) { + ythrow yexception() << "dstLen == srcLen"; + } + ::memcpy(output, p, srcLen); + } + + p += srcLen; + + // Check again. + if (h->Flags & LZO_ADLER32_D) { + ui32 checksum; + checksum = lzo_adler32(1, output, dstLen); + if (checksum != dAdler32) { + return LZO_INVALID_DST_ADLER32; + } + } + if (h->Flags & LZO_CRC32_D) { + ui32 checksum; + checksum = lzo_crc32(1, output, dstLen); + if (checksum != dCrc32) { + return LZO_INVALID_DST_CRC32; + } + } + + // OK + InputOffset += p - pb; + return LZO_OK; +} + +TLzopDecompress::TLzopDecompress(IInputStream* slave, ui32 initialBufferSize) + : Impl_(new TImpl(slave, initialBufferSize)) +{ +} + +TLzopDecompress::~TLzopDecompress() { +} + +size_t TLzopDecompress::DoRead(void* buffer, size_t length) { + return Impl_->Read(buffer, length); +} diff --git a/library/cpp/streams/lzop/lzop.h b/library/cpp/streams/lzop/lzop.h new file mode 100644 index 00000000000..523e5ad01bc --- /dev/null +++ b/library/cpp/streams/lzop/lzop.h @@ -0,0 +1,34 @@ +#pragma once + +#include <util/generic/ptr.h> +#include <util/generic/yexception.h> +#include <util/stream/input.h> +#include <util/stream/output.h> + +class TLzopCompress: public IOutputStream { +public: + TLzopCompress(IOutputStream* slave, ui16 maxBlockSize = 1 << 15); + ~TLzopCompress() override; + +private: + void DoWrite(const void* buf, size_t len) override; + void DoFlush() override; + void DoFinish() override; + +private: + class TImpl; + THolder<TImpl> Impl_; +}; + +class TLzopDecompress: public IInputStream { +public: + TLzopDecompress(IInputStream* slave, ui32 initialBufferSize = 1 << 16); + ~TLzopDecompress() override; + +private: + size_t DoRead(void* buf, size_t len) override; + +private: + class TImpl; + THolder<TImpl> Impl_; +}; diff --git a/library/cpp/streams/xz/README.md b/library/cpp/streams/xz/README.md new file mode 100644 index 00000000000..edb344c0fb4 --- /dev/null +++ b/library/cpp/streams/xz/README.md @@ -0,0 +1,9 @@ +XZ +=== +`TXzDecompress` supports file formats: +1) `.xz` - could be generated with CLI-tool `xz`. This format allows concatenating compressed files as is: +``` + echo foo | xz > foobar.xz + echo bar | xz >> foobar.xz +``` +2) `.lzma` - could be generated with CLI-tool `lzma` - it is legacy: https://fossies.org/linux/xz/README diff --git a/library/cpp/streams/xz/decompress.cpp b/library/cpp/streams/xz/decompress.cpp new file mode 100644 index 00000000000..361b3cff14c --- /dev/null +++ b/library/cpp/streams/xz/decompress.cpp @@ -0,0 +1,326 @@ +#include "decompress.h" + +#include <contrib/libs/lzma/liblzma/api/lzma.h> + +#include <util/generic/yexception.h> +#include <util/stream/output.h> +#include <util/stream/str.h> +#include <util/stream/zerocopy.h> + +// Based on https://fossies.org/linux/xz/doc/examples/02_decompress.c + +/////////////////////////////////////////////////////////////////////////////// +// +/// \file 02_decompress.c +/// \brief Decompress .xz files to stdout +/// +/// Usage: ./02_decompress INPUT_FILES... > OUTFILE +/// +/// Example: ./02_decompress foo.xz bar.xz > foobar +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +namespace { + class IInput { + public: + virtual ~IInput() = default; + virtual size_t Next(const ui8*& ptr) = 0; + }; + + class TCopyInput: public IInput { + public: + TCopyInput(IInputStream* slave) + : Slave_(slave) + { + } + + size_t Next(const ui8*& ptr) override { + ptr = Inbuf_; + return Slave_->Read(Inbuf_, sizeof(Inbuf_)); + } + + private: + IInputStream* Slave_; + ui8 Inbuf_[4096]; + }; + + class TZeroCopy: public IInput { + public: + TZeroCopy(IZeroCopyInput* slave) + : Slave_(slave) + { + } + + size_t Next(const ui8*& ptr) override { + return Slave_->Next(&ptr); + } + + private: + IZeroCopyInput* Slave_; + }; + + std::unique_ptr<IInput> createInput(IInputStream* slave) { + return std::make_unique<TCopyInput>(slave); + } + + std::unique_ptr<IInput> createInput(IZeroCopyInput* slave) { + return std::make_unique<TZeroCopy>(slave); + } +} + +class TUnbufferedXzDecompress::TImpl { +public: + template <class T> + TImpl(T* slave) + : Input_(createInput(slave)) + , Strm_(LZMA_STREAM_INIT) + { + TString err; + Y_ENSURE(initDecoder(&Strm_, err), + "Error initializing the decoder: " << err); + Strm_.next_in = NULL; + Strm_.avail_in = 0; + } + + ~TImpl() { + // Free the memory allocated for the decoder + lzma_end(&Strm_); + } + + size_t DoRead(void* buf, size_t len) { + if (IsOutFinished_) { + return 0; + } + + size_t res; + TString err; + + Y_ENSURE(decompress(buf, len, res, err), + "lzma decoder error: " << err); + + return res; + } + +private: + bool decompress(void* buf, size_t len, size_t& outLen, TString& err) { + // When LZMA_CONCATENATED flag was used when initializing the decoder, + // we need to tell lzma_code() when there will be no more input. + // This is done by setting action to LZMA_FINISH instead of LZMA_RUN + // in the same way as it is done when encoding. + // + // When LZMA_CONCATENATED isn't used, there is no need to use + // LZMA_FINISH to tell when all the input has been read, but it + // is still OK to use it if you want. When LZMA_CONCATENATED isn't + // used, the decoder will stop after the first .xz stream. In that + // case some unused data may be left in strm->next_in. + lzma_action action = LZMA_RUN; + + Strm_.next_out = (ui8*)buf; + Strm_.avail_out = len; + + while (true) { + if (Strm_.avail_in == 0 && !IsInFinished_) { + size_t size = Input_->Next(Strm_.next_in); + + if (size == 0) { + IsInFinished_ = true; + } else { + Strm_.avail_in = size; + } + + // Once the end of the input file has been reached, + // we need to tell lzma_code() that no more input + // will be coming. As said before, this isn't required + // if the LZMA_CONCATENATED flag isn't used when + // initializing the decoder. + if (IsInFinished_) + action = LZMA_FINISH; + } + + lzma_ret ret = lzma_code(&Strm_, action); + + if (ret == LZMA_STREAM_END) { + // Once everything has been decoded successfully, the + // return value of lzma_code() will be LZMA_STREAM_END. + // + // It is important to check for LZMA_STREAM_END. Do not + // assume that getting ret != LZMA_OK would mean that + // everything has gone well or that when you aren't + // getting more output it must have successfully + // decoded everything. + IsOutFinished_ = true; + } + + if (Strm_.avail_out == 0 || ret == LZMA_STREAM_END) { + outLen = len - Strm_.avail_out; + return true; + } + + if (ret != LZMA_OK) { + // It's not LZMA_OK nor LZMA_STREAM_END, + // so it must be an error code. See lzma/base.h + // (src/liblzma/api/lzma/base.h in the source package + // or e.g. /usr/include/lzma/base.h depending on the + // install prefix) for the list and documentation of + // possible values. Many values listen in lzma_ret + // enumeration aren't possible in this example, but + // can be made possible by enabling memory usage limit + // or adding flags to the decoder initialization. + switch (ret) { + case LZMA_MEM_ERROR: + err = "Memory allocation failed"; + break; + + case LZMA_FORMAT_ERROR: + // .xz magic bytes weren't found. + err = "The input is not in the .xz format"; + break; + + case LZMA_OPTIONS_ERROR: + // For example, the headers specify a filter + // that isn't supported by this liblzma + // version (or it hasn't been enabled when + // building liblzma, but no-one sane does + // that unless building liblzma for an + // embedded system). Upgrading to a newer + // liblzma might help. + // + // Note that it is unlikely that the file has + // accidentally became corrupt if you get this + // error. The integrity of the .xz headers is + // always verified with a CRC32, so + // unintentionally corrupt files can be + // distinguished from unsupported files. + err = "Unsupported compression options"; + break; + + case LZMA_DATA_ERROR: + err = "Compressed file is corrupt"; + break; + + case LZMA_BUF_ERROR: + // Typically this error means that a valid + // file has got truncated, but it might also + // be a damaged part in the file that makes + // the decoder think the file is truncated. + // If you prefer, you can use the same error + // message for this as for LZMA_DATA_ERROR. + err = "Compressed file is truncated or " + "otherwise corrupt"; + break; + + default: + // This is most likely LZMA_PROG_ERROR. + err = "Unknown error, possibly a bug"; + break; + } + + TStringOutput out(err); + out << "[" << (int)ret << "]"; + return false; + } + } + } + + static bool initDecoder(lzma_stream* strm, TString& err) { + // Initialize a .xz decoder. The decoder supports a memory usage limit + // and a set of flags. + // + // The memory usage of the decompressor depends on the settings used + // to compress a .xz file. It can vary from less than a megabyte to + // a few gigabytes, but in practice (at least for now) it rarely + // exceeds 65 MiB because that's how much memory is required to + // decompress files created with "xz -9". Settings requiring more + // memory take extra effort to use and don't (at least for now) + // provide significantly better compression in most cases. + // + // Memory usage limit is useful if it is important that the + // decompressor won't consume gigabytes of memory. The need + // for limiting depends on the application. In this example, + // no memory usage limiting is used. This is done by setting + // the limit to UINT64_MAX. + // + // The .xz format allows concatenating compressed files as is: + // + // echo foo | xz > foobar.xz + // echo bar | xz >> foobar.xz + // + // When decompressing normal standalone .xz files, LZMA_CONCATENATED + // should always be used to support decompression of concatenated + // .xz files. If LZMA_CONCATENATED isn't used, the decoder will stop + // after the first .xz stream. This can be useful when .xz data has + // been embedded inside another file format. + // + // Flags other than LZMA_CONCATENATED are supported too, and can + // be combined with bitwise-or. See lzma/container.h + // (src/liblzma/api/lzma/container.h in the source package or e.g. + // /usr/include/lzma/container.h depending on the install prefix) + // for details. + lzma_ret ret = lzma_auto_decoder( + strm, UINT64_MAX, LZMA_CONCATENATED); + + // Return successfully if the initialization went fine. + if (ret == LZMA_OK) + return true; + + // Something went wrong. The possible errors are documented in + // lzma/container.h (src/liblzma/api/lzma/container.h in the source + // package or e.g. /usr/include/lzma/container.h depending on the + // install prefix). + // + // Note that LZMA_MEMLIMIT_ERROR is never possible here. If you + // specify a very tiny limit, the error will be delayed until + // the first headers have been parsed by a call to lzma_code(). + switch (ret) { + case LZMA_MEM_ERROR: + err = "Memory allocation failed"; + break; + + case LZMA_OPTIONS_ERROR: + err = "Unsupported decompressor flags"; + break; + + default: + // This is most likely LZMA_PROG_ERROR indicating a bug in + // this program or in liblzma. It is inconvenient to have a + // separate error message for errors that should be impossible + // to occur, but knowing the error code is important for + // debugging. That's why it is good to print the error code + // at least when there is no good error message to show. + err = "Unknown error, possibly a bug"; + break; + } + + TStringOutput out(err); + out << "[" << (int)ret << "]"; + return false; + } + +private: + std::unique_ptr<IInput> Input_; + lzma_stream Strm_; + + bool IsInFinished_ = false; + bool IsOutFinished_ = false; +}; + +TUnbufferedXzDecompress::TUnbufferedXzDecompress(IInputStream* slave) + : Impl_(std::make_unique<TImpl>(slave)) +{ +} + +TUnbufferedXzDecompress::TUnbufferedXzDecompress(IZeroCopyInput* slave) + : Impl_(std::make_unique<TImpl>(slave)) +{ +} + +TUnbufferedXzDecompress::~TUnbufferedXzDecompress() = default; + +size_t TUnbufferedXzDecompress::DoRead(void* buf, size_t len) { + return Impl_->DoRead(buf, len); +} diff --git a/library/cpp/streams/xz/decompress.h b/library/cpp/streams/xz/decompress.h new file mode 100644 index 00000000000..8389cbdaf91 --- /dev/null +++ b/library/cpp/streams/xz/decompress.h @@ -0,0 +1,40 @@ +#pragma once + +#include <util/stream/buffered.h> +#include <util/stream/input.h> + +class IZeroCopyInput; + +/** + * Unbuffered decompressing stream for .XZ and .LZMA files. + * + * Do not use it for reading in small pieces. + */ +class TUnbufferedXzDecompress: public IInputStream { +public: + TUnbufferedXzDecompress(IInputStream* slave); + TUnbufferedXzDecompress(IZeroCopyInput* slave); + ~TUnbufferedXzDecompress() override; + +private: + size_t DoRead(void* buf, size_t len) override; + +private: + class TImpl; + std::unique_ptr<TImpl> Impl_; +}; + +/** + * Buffered decompressing stream for .XZ and .LZMA files. + * + * Supports efficient `ReadLine` calls and similar "reading in small pieces" + * usage patterns. + */ +class TXzDecompress: public TBuffered<TUnbufferedXzDecompress> { +public: + template <class T> + inline TXzDecompress(T&& t, size_t buf = 1 << 13) + : TBuffered<TUnbufferedXzDecompress>(buf, std::forward<T>(t)) + { + } +}; diff --git a/library/cpp/streams/xz/ut/decompress_ut.cpp b/library/cpp/streams/xz/ut/decompress_ut.cpp new file mode 100644 index 00000000000..2ebeca4e85a --- /dev/null +++ b/library/cpp/streams/xz/ut/decompress_ut.cpp @@ -0,0 +1,26 @@ +#include <library/cpp/streams/xz/decompress.h> + +#include <library/cpp/string_utils/base64/base64.h> +#include <library/cpp/testing/unittest/registar.h> + +Y_UNIT_TEST_SUITE(XzDecompress) { + Y_UNIT_TEST(decompress) { + TStringStream in; + in << Base64Decode("/Td6WFoAAATm1rRGAgAhARYAAAB0L+Wj4ANQAUVdABkMHARbg7qMApbl/qwEvrgQKpvF7Rbp/QJJdquZ88M3I5x3ANhSSpxvtnSoyPDeC6M8vz0vNKiOCsbIqvsGIwxrx+6YNqT87gDxVS8S3fHeoAZTf+zbg1DpDtv7Xh7Q3ug24wxNbPMi2p+WAo3V0LAi+lGUQmA44nJlabRv0XZ5CWhwgYtEWrrbPxoFjONeCa4p5BoX+TVgWegToFQMeJhVXMbDGWOIFL56X/F7nDJ47pjAy2GJIHHI5W/wrGH6uB0TCwpudW96peQaEgwMSZE07PfPE+XkfEymxhkxTs5Mnpc2rmQCiZ+3I6PqP+Qj8fuqaxb0fAJPQrbWYsqqeXP/3VNOeDRk+Szr9H3TMGI6yepUgkrgqNpaIYYcbxTU43eofcnTdwdsgi8fpH99tx3rrKq4zveStkZgZQqeY+MCvineIAAAAAAA2X8RUfmPU3kAAeEC0QYAANTt6P6xxGf7AgAAAAAEWVo="); + + TXzDecompress xz(&in); + + UNIT_ASSERT_VALUES_EQUAL( + xz.ReadAll(), + "2020-08-27T18:22:02.332 INFO: Starting blackbox module\n" + "2020-08-27T18:22:02.850 INFO: Init libauth (root kspace=<yandex_ru>, sign with key #49)\n" + "2020-08-27T18:22:02.851 DEBUG: KeyRing: randoms: table name loaded. Took 0.000918s\n" + "2020-08-27T18:22:02.853 DEBUG: KeyRing: randoms: min-max key id loaded. Took 0.001249s\n" + "2020-08-27T18:22:02.863 DEBUG: KeyRing: randoms: new keys loaded. Took 0.010837s\n" + "2020-08-27T18:22:02.865 DEBUG: Loaded 2389 new key(s) for keyspace 'yandex_ru'. Key ids: 330589-335364\n" + "2020-08-27T18:22:02.866 INFO: Attempt to load second time for spacename yandex_ru\n" + "2020-08-27T18:22:02.867 DEBUG: KeyRing: randoms_ua: table name loaded. Took 0.000926s\n" + "2020-08-27T18:22:02.868 DEBUG: KeyRing: randoms_ua: min-max key id loaded. Took 0.001212s\n" + "2020-08-27T18:22:02.871 DEBUG: KeyRing: randoms_ua: new keys loaded. Took 0.003202s\n"); + } +} |