aboutsummaryrefslogtreecommitdiffstats
path: root/library/cpp/streams
diff options
context:
space:
mode:
authorDevtools Arcadia <arcadia-devtools@yandex-team.ru>2022-02-07 18:08:42 +0300
committerDevtools Arcadia <arcadia-devtools@mous.vla.yp-c.yandex.net>2022-02-07 18:08:42 +0300
commit1110808a9d39d4b808aef724c861a2e1a38d2a69 (patch)
treee26c9fed0de5d9873cce7e00bc214573dc2195b7 /library/cpp/streams
downloadydb-1110808a9d39d4b808aef724c861a2e1a38d2a69.tar.gz
intermediate changes
ref:cde9a383711a11544ce7e107a78147fb96cc4029
Diffstat (limited to 'library/cpp/streams')
-rw-r--r--library/cpp/streams/brotli/brotli.cpp231
-rw-r--r--library/cpp/streams/brotli/brotli.h52
-rw-r--r--library/cpp/streams/brotli/brotli_ut.cpp89
-rw-r--r--library/cpp/streams/brotli/ut/ya.make12
-rw-r--r--library/cpp/streams/brotli/ya.make17
-rw-r--r--library/cpp/streams/bzip2/bzip2.cpp204
-rw-r--r--library/cpp/streams/bzip2/bzip2.h53
-rw-r--r--library/cpp/streams/bzip2/bzip2_ut.cpp41
-rw-r--r--library/cpp/streams/bzip2/ut/ya.make12
-rw-r--r--library/cpp/streams/bzip2/ya.make16
-rw-r--r--library/cpp/streams/lz/lz.cpp731
-rw-r--r--library/cpp/streams/lz/lz.h242
-rw-r--r--library/cpp/streams/lz/lz_ut.cpp287
-rw-r--r--library/cpp/streams/lz/ut/random.databin0 -> 8196 bytes
-rw-r--r--library/cpp/streams/lz/ut/request.data1
-rw-r--r--library/cpp/streams/lz/ut/ya.make18
-rw-r--r--library/cpp/streams/lz/ut/yq_609.databin0 -> 721 bytes
-rw-r--r--library/cpp/streams/lz/ya.make20
-rw-r--r--library/cpp/streams/lzma/lzma.cpp520
-rw-r--r--library/cpp/streams/lzma/lzma.h37
-rw-r--r--library/cpp/streams/lzma/lzma_ut.cpp127
-rw-r--r--library/cpp/streams/lzma/ut/ya.make12
-rw-r--r--library/cpp/streams/lzma/ya.make16
-rw-r--r--library/cpp/streams/ya.make26
-rw-r--r--library/cpp/streams/zc_memory_input/ya.make12
-rw-r--r--library/cpp/streams/zc_memory_input/zc_memory_input.cpp1
-rw-r--r--library/cpp/streams/zc_memory_input/zc_memory_input.h48
-rw-r--r--library/cpp/streams/zstd/ut/ya.make12
-rw-r--r--library/cpp/streams/zstd/ya.make16
-rw-r--r--library/cpp/streams/zstd/zstd.cpp173
-rw-r--r--library/cpp/streams/zstd/zstd.h53
-rw-r--r--library/cpp/streams/zstd/zstd_ut.cpp94
32 files changed, 3173 insertions, 0 deletions
diff --git a/library/cpp/streams/brotli/brotli.cpp b/library/cpp/streams/brotli/brotli.cpp
new file mode 100644
index 0000000000..38052cb688
--- /dev/null
+++ b/library/cpp/streams/brotli/brotli.cpp
@@ -0,0 +1,231 @@
+#include "brotli.h"
+
+#include <contrib/libs/brotli/include/brotli/decode.h>
+#include <contrib/libs/brotli/include/brotli/encode.h>
+
+#include <util/generic/yexception.h>
+#include <util/memory/addstorage.h>
+
+namespace {
+ struct TAllocator {
+ static void* Allocate(void* /* opaque */, size_t size) {
+ return ::operator new(size);
+ }
+
+ static void Deallocate(void* /* opaque */, void* ptr) noexcept {
+ ::operator delete(ptr);
+ }
+ };
+
+}
+
+class TBrotliCompress::TImpl {
+public:
+ TImpl(IOutputStream* slave, int quality)
+ : Slave_(slave)
+ , EncoderState_(BrotliEncoderCreateInstance(&TAllocator::Allocate, &TAllocator::Deallocate, nullptr))
+ {
+ if (!EncoderState_) {
+ ythrow yexception() << "Brotli encoder initialization failed";
+ }
+
+ auto res = BrotliEncoderSetParameter(
+ EncoderState_,
+ BROTLI_PARAM_QUALITY,
+ quality);
+
+ if (!res) {
+ BrotliEncoderDestroyInstance(EncoderState_);
+ ythrow yexception() << "Failed to set brotli encoder quality to " << quality;
+ }
+ }
+
+ ~TImpl() {
+ BrotliEncoderDestroyInstance(EncoderState_);
+ }
+
+ void Write(const void* buffer, size_t size) {
+ DoWrite(buffer, size, BROTLI_OPERATION_PROCESS);
+ }
+
+ void Flush() {
+ DoWrite(nullptr, 0, BROTLI_OPERATION_FLUSH);
+ }
+
+ void Finish() {
+ Flush();
+ DoWrite(nullptr, 0, BROTLI_OPERATION_FINISH);
+ Y_VERIFY(BrotliEncoderIsFinished(EncoderState_));
+ }
+
+private:
+ IOutputStream* Slave_;
+ BrotliEncoderState* EncoderState_;
+
+ void DoWrite(const void* buffer, size_t size, BrotliEncoderOperation operation) {
+ size_t availableOut = 0;
+ ui8* outputBuffer = nullptr;
+
+ const ui8* uBuffer = static_cast<const ui8*>(buffer);
+
+ do {
+ auto result = BrotliEncoderCompressStream(
+ EncoderState_,
+ operation,
+ &size,
+ &uBuffer,
+ &availableOut,
+ &outputBuffer,
+ nullptr);
+
+ if (result == BROTLI_FALSE) {
+ ythrow yexception() << "Brotli encoder failed to process buffer";
+ }
+
+ size_t outputLength = 0;
+ const ui8* output = BrotliEncoderTakeOutput(EncoderState_, &outputLength);
+ if (outputLength > 0) {
+ Slave_->Write(output, outputLength);
+ }
+ } while (size > 0 || BrotliEncoderHasMoreOutput(EncoderState_));
+ }
+};
+
+TBrotliCompress::TBrotliCompress(IOutputStream* slave, int quality) {
+ Impl_.Reset(new TImpl(slave, quality));
+}
+
+TBrotliCompress::~TBrotliCompress() {
+ try {
+ Finish();
+ } catch (...) {
+ }
+}
+
+void TBrotliCompress::DoWrite(const void* buffer, size_t size) {
+ Impl_->Write(buffer, size);
+}
+
+void TBrotliCompress::DoFlush() {
+ if (Impl_) {
+ Impl_->Flush();
+ }
+}
+
+void TBrotliCompress::DoFinish() {
+ THolder<TImpl> impl(Impl_.Release());
+
+ if (impl) {
+ impl->Finish();
+ }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TBrotliDecompress::TImpl: public TAdditionalStorage<TImpl> {
+public:
+ TImpl(IInputStream* slave)
+ : Slave_(slave)
+ {
+ InitDecoder();
+ }
+
+ ~TImpl() {
+ FreeDecoder();
+ }
+
+ size_t Read(void* buffer, size_t size) {
+ Y_ASSERT(size > 0);
+
+ ui8* outBuffer = static_cast<ui8*>(buffer);
+ size_t availableOut = size;
+ size_t decompressedSize = 0;
+
+ BrotliDecoderResult result;
+ do {
+ if (InputAvailable_ == 0 && !InputExhausted_) {
+ InputBuffer_ = TmpBuf();
+ InputAvailable_ = Slave_->Read((void*)InputBuffer_, TmpBufLen());
+ if (InputAvailable_ == 0) {
+ InputExhausted_ = true;
+ }
+ }
+
+ if (SubstreamFinished_ && !InputExhausted_) {
+ ResetState();
+ }
+
+ result = BrotliDecoderDecompressStream(
+ DecoderState_,
+ &InputAvailable_,
+ &InputBuffer_,
+ &availableOut,
+ &outBuffer,
+ nullptr);
+
+ decompressedSize = size - availableOut;
+ SubstreamFinished_ = (result == BROTLI_DECODER_RESULT_SUCCESS);
+
+ if (result == BROTLI_DECODER_RESULT_ERROR) {
+ BrotliDecoderErrorCode code = BrotliDecoderGetErrorCode(DecoderState_);
+ ythrow yexception() << "Brotli decoder failed to decompress buffer: "
+ << BrotliDecoderErrorString(code);
+ } else if (result == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) {
+ Y_VERIFY(availableOut != size,
+ "Buffer passed to read in Brotli decoder is too small");
+ break;
+ }
+ } while (decompressedSize == 0 && result == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT && !InputExhausted_);
+
+ if (!SubstreamFinished_ && decompressedSize == 0) {
+ ythrow yexception() << "Input stream is incomplete";
+ }
+
+ return decompressedSize;
+ }
+
+private:
+ IInputStream* Slave_;
+ BrotliDecoderState* DecoderState_;
+
+ bool SubstreamFinished_ = false;
+ bool InputExhausted_ = false;
+ const ui8* InputBuffer_ = nullptr;
+ size_t InputAvailable_ = 0;
+
+ unsigned char* TmpBuf() noexcept {
+ return static_cast<unsigned char*>(AdditionalData());
+ }
+
+ size_t TmpBufLen() const noexcept {
+ return AdditionalDataLength();
+ }
+
+ void InitDecoder() {
+ DecoderState_ = BrotliDecoderCreateInstance(&TAllocator::Allocate, &TAllocator::Deallocate, nullptr);
+ if (!DecoderState_) {
+ ythrow yexception() << "Brotli decoder initialization failed";
+ }
+ }
+
+ void FreeDecoder() {
+ BrotliDecoderDestroyInstance(DecoderState_);
+ }
+
+ void ResetState() {
+ Y_VERIFY(BrotliDecoderIsFinished(DecoderState_));
+ FreeDecoder();
+ InitDecoder();
+ }
+};
+
+TBrotliDecompress::TBrotliDecompress(IInputStream* slave, size_t bufferSize)
+ : Impl_(new (bufferSize) TImpl(slave))
+{
+}
+
+TBrotliDecompress::~TBrotliDecompress() = default;
+
+size_t TBrotliDecompress::DoRead(void* buffer, size_t size) {
+ return Impl_->Read(buffer, size);
+}
diff --git a/library/cpp/streams/brotli/brotli.h b/library/cpp/streams/brotli/brotli.h
new file mode 100644
index 0000000000..b3af869e29
--- /dev/null
+++ b/library/cpp/streams/brotli/brotli.h
@@ -0,0 +1,52 @@
+#pragma once
+
+#include <util/generic/ptr.h>
+#include <util/stream/input.h>
+#include <util/stream/output.h>
+
+/**
+ * @addtogroup Streams_Archs
+ * @{
+ */
+
+class TBrotliCompress: public IOutputStream {
+public:
+ static constexpr int BEST_QUALITY = 11;
+
+ /**
+ @param slave stream to write compressed data to
+ @param quality the higher the quality, the slower and better the compression. Range is 0 to 11.
+ */
+ explicit TBrotliCompress(IOutputStream* slave, int quality = BEST_QUALITY);
+ ~TBrotliCompress() override;
+
+private:
+ void DoWrite(const void* buffer, size_t size) override;
+ void DoFlush() override;
+ void DoFinish() override;
+
+public:
+ class TImpl;
+ THolder<TImpl> Impl_;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TBrotliDecompress: public IInputStream {
+public:
+ /**
+ @param slave stream to read compressed data from
+ @param bufferSize approximate size of buffer compressed data is read in
+ */
+ explicit TBrotliDecompress(IInputStream* slave, size_t bufferSize = 8 * 1024);
+ ~TBrotliDecompress() override;
+
+private:
+ size_t DoRead(void* buffer, size_t size) override;
+
+private:
+ class TImpl;
+ THolder<TImpl> Impl_;
+};
+
+/** @} */
diff --git a/library/cpp/streams/brotli/brotli_ut.cpp b/library/cpp/streams/brotli/brotli_ut.cpp
new file mode 100644
index 0000000000..aeb2e284dc
--- /dev/null
+++ b/library/cpp/streams/brotli/brotli_ut.cpp
@@ -0,0 +1,89 @@
+#include "brotli.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <util/random/fast.h>
+
+Y_UNIT_TEST_SUITE(TBrotliTestSuite) {
+ TString Compress(TString data) {
+ TString compressed;
+ TStringOutput output(compressed);
+ TBrotliCompress compressStream(&output, 11);
+ compressStream.Write(data.data(), data.size());
+ compressStream.Finish();
+ output.Finish();
+ return compressed;
+ }
+
+ TString Decompress(TString data) {
+ TStringInput input(data);
+ TBrotliDecompress decompressStream(&input);
+ return decompressStream.ReadAll();
+ }
+
+ void TestCase(const TString& s) {
+ UNIT_ASSERT_VALUES_EQUAL(s, Decompress(Compress(s)));
+ }
+
+ TString GenerateRandomString(size_t size) {
+ TReallyFastRng32 rng(42);
+ TString result;
+ result.reserve(size + sizeof(ui64));
+ while (result.size() < size) {
+ ui64 value = rng.GenRand64();
+ result += TStringBuf(reinterpret_cast<const char*>(&value), sizeof(value));
+ }
+ result.resize(size);
+ return result;
+ }
+
+ Y_UNIT_TEST(TestHelloWorld) {
+ TestCase("hello world");
+ }
+
+ Y_UNIT_TEST(TestFlush) {
+ TStringStream ss;
+ TBrotliCompress compressStream(&ss);
+ TBrotliDecompress decompressStream(&ss);
+
+ for (size_t i = 0; i < 3; ++i) {
+ TString s = GenerateRandomString(1 << 15);
+ compressStream.Write(s.data(), s.size());
+ compressStream.Flush();
+
+ TString r(s.size(), '*');
+ decompressStream.Load((char*)r.data(), r.size());
+
+ UNIT_ASSERT_VALUES_EQUAL(s, r);
+ }
+ }
+
+ Y_UNIT_TEST(TestSeveralStreams) {
+ auto s1 = GenerateRandomString(1 << 15);
+ auto s2 = GenerateRandomString(1 << 15);
+ auto c1 = Compress(s1);
+ auto c2 = Compress(s2);
+ UNIT_ASSERT_VALUES_EQUAL(s1 + s2, Decompress(c1 + c2));
+ }
+
+ Y_UNIT_TEST(TestIncompleteStream) {
+ TString manyAs(64 * 1024, 'a');
+ auto compressed = Compress(manyAs);
+ TString truncated(compressed.data(), compressed.size() - 1);
+ UNIT_CHECK_GENERATED_EXCEPTION(Decompress(truncated), std::exception);
+ }
+
+ Y_UNIT_TEST(Test64KB) {
+ auto manyAs = TString(64 * 1024, 'a');
+ TString str("Hello from the Matrix!@#% How are you?}{\n\t\a");
+ TestCase(manyAs + str + manyAs);
+ }
+
+ Y_UNIT_TEST(Test1MB) {
+ TestCase(GenerateRandomString(1 * 1024 * 1024));
+ }
+
+ Y_UNIT_TEST(TestEmpty) {
+ TestCase("");
+ }
+}
diff --git a/library/cpp/streams/brotli/ut/ya.make b/library/cpp/streams/brotli/ut/ya.make
new file mode 100644
index 0000000000..243462f1b2
--- /dev/null
+++ b/library/cpp/streams/brotli/ut/ya.make
@@ -0,0 +1,12 @@
+UNITTEST_FOR(library/cpp/streams/brotli)
+
+OWNER(
+ levysotsky
+ g:util
+)
+
+SRCS(
+ brotli_ut.cpp
+)
+
+END()
diff --git a/library/cpp/streams/brotli/ya.make b/library/cpp/streams/brotli/ya.make
new file mode 100644
index 0000000000..fa2bfec9cc
--- /dev/null
+++ b/library/cpp/streams/brotli/ya.make
@@ -0,0 +1,17 @@
+LIBRARY()
+
+OWNER(
+ levysotsky
+ g:util
+)
+
+PEERDIR(
+ contrib/libs/brotli/enc
+ contrib/libs/brotli/dec
+)
+
+SRCS(
+ brotli.cpp
+)
+
+END()
diff --git a/library/cpp/streams/bzip2/bzip2.cpp b/library/cpp/streams/bzip2/bzip2.cpp
new file mode 100644
index 0000000000..bccc5c6807
--- /dev/null
+++ b/library/cpp/streams/bzip2/bzip2.cpp
@@ -0,0 +1,204 @@
+#include "bzip2.h"
+
+#include <util/memory/addstorage.h>
+#include <util/generic/scope.h>
+
+#include <contrib/libs/libbz2/bzlib.h>
+
+class TBZipDecompress::TImpl: public TAdditionalStorage<TImpl> {
+public:
+ inline TImpl(IInputStream* input)
+ : Stream_(input)
+ {
+ Zero(BzStream_);
+ Init();
+ }
+
+ inline ~TImpl() {
+ Clear();
+ }
+
+ inline void Init() {
+ if (BZ2_bzDecompressInit(&BzStream_, 0, 0) != BZ_OK) {
+ ythrow TBZipDecompressError() << "can not init bzip engine";
+ }
+ }
+
+ inline void Clear() noexcept {
+ BZ2_bzDecompressEnd(&BzStream_);
+ }
+
+ inline size_t Read(void* buf, size_t size) {
+ BzStream_.next_out = (char*)buf;
+ BzStream_.avail_out = size;
+
+ while (true) {
+ if (BzStream_.avail_in == 0) {
+ if (FillInputBuffer() == 0) {
+ return 0;
+ }
+ }
+
+ switch (BZ2_bzDecompress(&BzStream_)) {
+ case BZ_STREAM_END: {
+ Clear();
+ Init();
+ [[fallthrough]];
+ }
+
+ case BZ_OK: {
+ const size_t processed = size - BzStream_.avail_out;
+
+ if (processed) {
+ return processed;
+ }
+
+ break;
+ }
+
+ default:
+ ythrow TBZipDecompressError() << "bzip error";
+ }
+ }
+ }
+
+ inline size_t FillInputBuffer() {
+ BzStream_.next_in = (char*)AdditionalData();
+ BzStream_.avail_in = Stream_->Read(BzStream_.next_in, AdditionalDataLength());
+
+ return BzStream_.avail_in;
+ }
+
+private:
+ IInputStream* Stream_;
+ bz_stream BzStream_;
+};
+
+TBZipDecompress::TBZipDecompress(IInputStream* input, size_t bufLen)
+ : Impl_(new (bufLen) TImpl(input))
+{
+}
+
+TBZipDecompress::~TBZipDecompress() {
+}
+
+size_t TBZipDecompress::DoRead(void* buf, size_t size) {
+ return Impl_->Read(buf, size);
+}
+
+class TBZipCompress::TImpl: public TAdditionalStorage<TImpl> {
+public:
+ inline TImpl(IOutputStream* stream, size_t level)
+ : Stream_(stream)
+ {
+ Zero(BzStream_);
+
+ if (BZ2_bzCompressInit(&BzStream_, level, 0, 0) != BZ_OK) {
+ ythrow TBZipCompressError() << "can not init bzip engine";
+ }
+
+ BzStream_.next_out = TmpBuf();
+ BzStream_.avail_out = TmpBufLen();
+ }
+
+ inline ~TImpl() {
+ BZ2_bzCompressEnd(&BzStream_);
+ }
+
+ inline void Write(const void* buf, size_t size) {
+ BzStream_.next_in = (char*)buf;
+ BzStream_.avail_in = size;
+
+ Y_DEFER {
+ BzStream_.next_in = 0;
+ BzStream_.avail_in = 0;
+ };
+
+ while (BzStream_.avail_in) {
+ const int ret = BZ2_bzCompress(&BzStream_, BZ_RUN);
+
+ switch (ret) {
+ case BZ_RUN_OK:
+ continue;
+
+ case BZ_PARAM_ERROR:
+ case BZ_OUTBUFF_FULL:
+ Stream_->Write(TmpBuf(), TmpBufLen() - BzStream_.avail_out);
+ BzStream_.next_out = TmpBuf();
+ BzStream_.avail_out = TmpBufLen();
+
+ break;
+
+ default:
+ ythrow TBZipCompressError() << "bzip error(" << ret << ", " << BzStream_.avail_out << ")";
+ }
+ }
+ }
+
+ inline void Flush() {
+ /*
+ * TODO ?
+ */
+ }
+
+ inline void Finish() {
+ int ret = BZ2_bzCompress(&BzStream_, BZ_FINISH);
+
+ while (ret != BZ_STREAM_END) {
+ Stream_->Write(TmpBuf(), TmpBufLen() - BzStream_.avail_out);
+ BzStream_.next_out = TmpBuf();
+ BzStream_.avail_out = TmpBufLen();
+
+ ret = BZ2_bzCompress(&BzStream_, BZ_FINISH);
+ }
+
+ Stream_->Write(TmpBuf(), TmpBufLen() - BzStream_.avail_out);
+ }
+
+private:
+ inline char* TmpBuf() noexcept {
+ return (char*)AdditionalData();
+ }
+
+ inline size_t TmpBufLen() const noexcept {
+ return AdditionalDataLength();
+ }
+
+private:
+ IOutputStream* Stream_;
+ bz_stream BzStream_;
+};
+
+TBZipCompress::TBZipCompress(IOutputStream* out, size_t compressionLevel, size_t bufLen)
+ : Impl_(new (bufLen) TImpl(out, compressionLevel))
+{
+}
+
+TBZipCompress::~TBZipCompress() {
+ try {
+ Finish();
+ } catch (...) {
+ }
+}
+
+void TBZipCompress::DoWrite(const void* buf, size_t size) {
+ if (!Impl_) {
+ ythrow TBZipCompressError() << "can not write to finished bzip stream";
+ }
+
+ Impl_->Write(buf, size);
+}
+
+void TBZipCompress::DoFlush() {
+ if (Impl_) {
+ Impl_->Flush();
+ }
+}
+
+void TBZipCompress::DoFinish() {
+ THolder<TImpl> impl(Impl_.Release());
+
+ if (impl) {
+ impl->Finish();
+ }
+}
diff --git a/library/cpp/streams/bzip2/bzip2.h b/library/cpp/streams/bzip2/bzip2.h
new file mode 100644
index 0000000000..2322277ef6
--- /dev/null
+++ b/library/cpp/streams/bzip2/bzip2.h
@@ -0,0 +1,53 @@
+#pragma once
+
+#include <util/stream/input.h>
+#include <util/stream/output.h>
+#include <util/generic/ptr.h>
+#include <util/generic/yexception.h>
+
+#define BZIP_BUF_LEN (8 * 1024)
+#define BZIP_COMPRESSION_LEVEL 6
+
+/**
+ * @addtogroup Streams_Archs
+ * @{
+ */
+
+class TBZipException: public yexception {
+};
+
+class TBZipDecompressError: public TBZipException {
+};
+
+class TBZipCompressError: public TBZipException {
+};
+
+class TBZipDecompress: public IInputStream {
+public:
+ TBZipDecompress(IInputStream* input, size_t bufLen = BZIP_BUF_LEN);
+ ~TBZipDecompress() override;
+
+private:
+ size_t DoRead(void* buf, size_t size) override;
+
+private:
+ class TImpl;
+ THolder<TImpl> Impl_;
+};
+
+class TBZipCompress: public IOutputStream {
+public:
+ TBZipCompress(IOutputStream* out, size_t compressionLevel = BZIP_COMPRESSION_LEVEL, size_t bufLen = BZIP_BUF_LEN);
+ ~TBZipCompress() override;
+
+private:
+ void DoWrite(const void* buf, size_t size) override;
+ void DoFlush() override;
+ void DoFinish() override;
+
+public:
+ class TImpl;
+ THolder<TImpl> Impl_;
+};
+
+/** @} */
diff --git a/library/cpp/streams/bzip2/bzip2_ut.cpp b/library/cpp/streams/bzip2/bzip2_ut.cpp
new file mode 100644
index 0000000000..69a98f296c
--- /dev/null
+++ b/library/cpp/streams/bzip2/bzip2_ut.cpp
@@ -0,0 +1,41 @@
+#include "bzip2.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <util/stream/file.h>
+#include <util/system/tempfile.h>
+
+#define ZDATA "./zdata"
+
+Y_UNIT_TEST_SUITE(TBZipTest) {
+ static const TString data = "8s7d5vc6s5vc67sa4c65ascx6asd4xcv76adsfxv76s";
+
+ Y_UNIT_TEST(TestCompress) {
+ TUnbufferedFileOutput o(ZDATA);
+ TBZipCompress c(&o);
+
+ c.Write(data.data(), data.size());
+ c.Finish();
+ o.Finish();
+ }
+
+ Y_UNIT_TEST(TestDecompress) {
+ TTempFile tmp(ZDATA);
+
+ {
+ TUnbufferedFileInput i(ZDATA);
+ TBZipDecompress d(&i);
+
+ UNIT_ASSERT_EQUAL(d.ReadLine(), data);
+ }
+ }
+
+ Y_UNIT_TEST(TestCorrupted) {
+ TMemoryInput i("blablabla", 10);
+ TBZipDecompress d(&i);
+
+ UNIT_ASSERT_EXCEPTION(d.ReadLine(), TBZipDecompressError);
+ }
+}
+
+#undef ZDATA
diff --git a/library/cpp/streams/bzip2/ut/ya.make b/library/cpp/streams/bzip2/ut/ya.make
new file mode 100644
index 0000000000..5ef91498ca
--- /dev/null
+++ b/library/cpp/streams/bzip2/ut/ya.make
@@ -0,0 +1,12 @@
+UNITTEST_FOR(library/cpp/streams/bzip2)
+
+OWNER(
+ pg
+ g:util
+)
+
+SRCS(
+ bzip2_ut.cpp
+)
+
+END()
diff --git a/library/cpp/streams/bzip2/ya.make b/library/cpp/streams/bzip2/ya.make
new file mode 100644
index 0000000000..122a35837c
--- /dev/null
+++ b/library/cpp/streams/bzip2/ya.make
@@ -0,0 +1,16 @@
+LIBRARY()
+
+OWNER(
+ pg
+ g:util
+)
+
+PEERDIR(
+ contrib/libs/libbz2
+)
+
+SRCS(
+ bzip2.cpp
+)
+
+END()
diff --git a/library/cpp/streams/lz/lz.cpp b/library/cpp/streams/lz/lz.cpp
new file mode 100644
index 0000000000..b65bb3ed96
--- /dev/null
+++ b/library/cpp/streams/lz/lz.cpp
@@ -0,0 +1,731 @@
+#include "lz.h"
+
+#include <util/system/yassert.h>
+#include <util/system/byteorder.h>
+#include <util/memory/addstorage.h>
+#include <util/generic/buffer.h>
+#include <util/generic/utility.h>
+#include <util/generic/singleton.h>
+#include <util/generic/yexception.h>
+#include <util/stream/mem.h>
+
+#include <contrib/libs/lz4/lz4.h>
+#include <contrib/libs/fastlz/fastlz.h>
+#include <contrib/libs/snappy/snappy.h>
+#include <contrib/libs/quicklz/quicklz.h>
+#include <contrib/libs/minilzo/minilzo.h>
+
+static inline ui8 HostToLittle(ui8 t) noexcept {
+ return t;
+}
+
+static inline ui8 LittleToHost(ui8 t) noexcept {
+ return t;
+}
+
+struct TCommonData {
+ static const size_t overhead = sizeof(ui16) + sizeof(ui8);
+};
+
+const size_t SIGNATURE_SIZE = 4;
+
+template <class TCompressor, class TBase>
+class TCompressorBase: public TAdditionalStorage<TCompressorBase<TCompressor, TBase>>, public TCompressor, public TCommonData {
+public:
+ inline TCompressorBase(IOutputStream* slave, ui16 blockSize)
+ : Slave_(slave)
+ , BlockSize_(blockSize)
+ {
+ /*
+ * save signature
+ */
+ static_assert(sizeof(TCompressor::signature) - 1 == SIGNATURE_SIZE, "expect sizeof(TCompressor::signature) - 1 == SIGNATURE_SIZE");
+ Slave_->Write(TCompressor::signature, sizeof(TCompressor::signature) - 1);
+
+ /*
+ * save version
+ */
+ this->Save((ui32)1);
+
+ /*
+ * save block size
+ */
+ this->Save(BlockSize());
+ }
+
+ inline ~TCompressorBase() {
+ }
+
+ inline void Write(const char* buf, size_t len) {
+ while (len) {
+ const ui16 toWrite = (ui16)Min<size_t>(len, this->BlockSize());
+
+ this->WriteBlock(buf, toWrite);
+
+ buf += toWrite;
+ len -= toWrite;
+ }
+ }
+
+ inline void Flush() {
+ }
+
+ inline void Finish() {
+ this->Flush();
+ this->WriteBlock(nullptr, 0);
+ }
+
+ template <class T>
+ static inline void Save(T t, IOutputStream* out) {
+ t = HostToLittle(t);
+
+ out->Write(&t, sizeof(t));
+ }
+
+ template <class T>
+ inline void Save(T t) {
+ Save(t, Slave_);
+ }
+
+private:
+ inline void* Block() const noexcept {
+ return this->AdditionalData();
+ }
+
+ inline ui16 BlockSize() const noexcept {
+ return BlockSize_;
+ }
+
+ inline void WriteBlock(const void* ptr, ui16 len) {
+ Y_ASSERT(len <= this->BlockSize());
+
+ ui8 compressed = false;
+
+ if (len) {
+ const size_t out = this->Compress((const char*)ptr, len, (char*)Block(), this->AdditionalDataLength());
+ // catch compressor buffer overrun (e.g. SEARCH-2043)
+ //Y_VERIFY(out <= this->Hint(this->BlockSize()));
+
+ if (out < len || TCompressor::SaveIncompressibleChunks()) {
+ compressed = true;
+ ptr = Block();
+ len = (ui16)out;
+ }
+ }
+
+ char tmp[overhead];
+ TMemoryOutput header(tmp, sizeof(tmp));
+
+ this->Save(len, &header);
+ this->Save(compressed, &header);
+
+ using TPart = IOutputStream::TPart;
+ if (ptr) {
+ const TPart parts[] = {
+ TPart(tmp, sizeof(tmp)),
+ TPart(ptr, len),
+ };
+
+ Slave_->Write(parts, sizeof(parts) / sizeof(*parts));
+ } else {
+ Slave_->Write(tmp, sizeof(tmp));
+ }
+ }
+
+private:
+ IOutputStream* Slave_;
+ const ui16 BlockSize_;
+};
+
+template <class T>
+static inline T GLoad(IInputStream* input) {
+ T t;
+
+ if (input->Load(&t, sizeof(t)) != sizeof(t)) {
+ ythrow TDecompressorError() << "stream error";
+ }
+
+ return LittleToHost(t);
+}
+
+class TDecompressSignature {
+public:
+ inline TDecompressSignature(IInputStream* input) {
+ if (input->Load(Buffer_, SIGNATURE_SIZE) != SIGNATURE_SIZE) {
+ ythrow TDecompressorError() << "can not load stream signature";
+ }
+ }
+
+ template <class TDecompressor>
+ inline bool Check() const {
+ static_assert(sizeof(TDecompressor::signature) - 1 == SIGNATURE_SIZE, "expect sizeof(TDecompressor::signature) - 1 == SIGNATURE_SIZE");
+ return memcmp(TDecompressor::signature, Buffer_, SIGNATURE_SIZE) == 0;
+ }
+
+private:
+ char Buffer_[SIGNATURE_SIZE];
+};
+
+template <class TDecompressor>
+static inline IInputStream* ConsumeSignature(IInputStream* input) {
+ TDecompressSignature sign(input);
+ if (!sign.Check<TDecompressor>()) {
+ ythrow TDecompressorError() << "incorrect signature";
+ }
+ return input;
+}
+
+template <class TDecompressor>
+class TDecompressorBaseImpl: public TDecompressor, public TCommonData {
+public:
+ static inline ui32 CheckVer(ui32 v) {
+ if (v != 1) {
+ ythrow yexception() << TStringBuf("incorrect stream version: ") << v;
+ }
+
+ return v;
+ }
+
+ inline TDecompressorBaseImpl(IInputStream* slave)
+ : Slave_(slave)
+ , Input_(nullptr, 0)
+ , Eof_(false)
+ , Version_(CheckVer(Load<ui32>()))
+ , BlockSize_(Load<ui16>())
+ , OutBufSize_(TDecompressor::Hint(BlockSize_))
+ , Tmp_(2 * OutBufSize_)
+ , In_(Tmp_.Data())
+ , Out_(In_ + OutBufSize_)
+ {
+ this->InitFromStream(Slave_);
+ }
+
+ inline ~TDecompressorBaseImpl() {
+ }
+
+ inline size_t Read(void* buf, size_t len) {
+ size_t ret = Input_.Read(buf, len);
+
+ if (ret) {
+ return ret;
+ }
+
+ if (Eof_) {
+ return 0;
+ }
+
+ this->FillNextBlock();
+
+ ret = Input_.Read(buf, len);
+
+ if (ret) {
+ return ret;
+ }
+
+ Eof_ = true;
+
+ return 0;
+ }
+
+ inline void FillNextBlock() {
+ char tmp[overhead];
+
+ if (Slave_->Load(tmp, sizeof(tmp)) != sizeof(tmp)) {
+ ythrow TDecompressorError() << "can not read block header";
+ }
+
+ TMemoryInput header(tmp, sizeof(tmp));
+
+ const ui16 len = GLoad<ui16>(&header);
+ if (len > Tmp_.Capacity()) {
+ ythrow TDecompressorError() << "invalid len inside block header";
+ }
+ const ui8 compressed = GLoad<ui8>(&header);
+
+ if (compressed > 1) {
+ ythrow TDecompressorError() << "broken header";
+ }
+
+ if (Slave_->Load(In_, len) != len) {
+ ythrow TDecompressorError() << "can not read data";
+ }
+
+ if (compressed) {
+ const size_t ret = this->Decompress(In_, len, Out_, OutBufSize_);
+
+ Input_.Reset(Out_, ret);
+ } else {
+ Input_.Reset(In_, len);
+ }
+ }
+
+ template <class T>
+ inline T Load() {
+ return GLoad<T>(Slave_);
+ }
+
+protected:
+ IInputStream* Slave_;
+ TMemoryInput Input_;
+ bool Eof_;
+ const ui32 Version_;
+ const ui16 BlockSize_;
+ const size_t OutBufSize_;
+ TBuffer Tmp_;
+ char* In_;
+ char* Out_;
+};
+
+template <class TDecompressor, class TBase>
+class TDecompressorBase: public TDecompressorBaseImpl<TDecompressor> {
+public:
+ inline TDecompressorBase(IInputStream* slave)
+ : TDecompressorBaseImpl<TDecompressor>(ConsumeSignature<TDecompressor>(slave))
+ {
+ }
+
+ inline ~TDecompressorBase() {
+ }
+};
+
+#define DEF_COMPRESSOR_COMMON(rname, name) \
+ rname::~rname() { \
+ try { \
+ Finish(); \
+ } catch (...) { \
+ } \
+ } \
+ \
+ void rname::DoWrite(const void* buf, size_t len) { \
+ if (!Impl_) { \
+ ythrow yexception() << "can not write to finalized stream"; \
+ } \
+ \
+ Impl_->Write((const char*)buf, len); \
+ } \
+ \
+ void rname::DoFlush() { \
+ if (!Impl_) { \
+ ythrow yexception() << "can not flush finalized stream"; \
+ } \
+ \
+ Impl_->Flush(); \
+ } \
+ \
+ void rname::DoFinish() { \
+ THolder<TImpl> impl(Impl_.Release()); \
+ \
+ if (impl) { \
+ impl->Finish(); \
+ } \
+ }
+
+#define DEF_COMPRESSOR(rname, name) \
+ class rname::TImpl: public TCompressorBase<name, TImpl> { \
+ public: \
+ inline TImpl(IOutputStream* out, ui16 blockSize) \
+ : TCompressorBase<name, TImpl>(out, blockSize) { \
+ } \
+ }; \
+ \
+ rname::rname(IOutputStream* slave, ui16 blockSize) \
+ : Impl_(new (TImpl::Hint(blockSize)) TImpl(slave, blockSize)) { \
+ } \
+ \
+ DEF_COMPRESSOR_COMMON(rname, name)
+
+#define DEF_DECOMPRESSOR(rname, name) \
+ class rname::TImpl: public TDecompressorBase<name, TImpl> { \
+ public: \
+ inline TImpl(IInputStream* in) \
+ : TDecompressorBase<name, TImpl>(in) { \
+ } \
+ }; \
+ \
+ rname::rname(IInputStream* slave) \
+ : Impl_(new TImpl(slave)) { \
+ } \
+ \
+ rname::~rname() { \
+ } \
+ \
+ size_t rname::DoRead(void* buf, size_t len) { \
+ return Impl_->Read(buf, len); \
+ }
+
+/*
+ * MiniLzo
+ */
+class TMiniLzo {
+ class TInit {
+ public:
+ inline TInit() {
+ if (lzo_init() != LZO_E_OK) {
+ ythrow yexception() << "can not init lzo engine";
+ }
+ }
+ };
+
+public:
+ static const char signature[];
+
+ inline TMiniLzo() {
+ Singleton<TInit>();
+ }
+
+ inline ~TMiniLzo() {
+ }
+
+ static inline size_t Hint(size_t len) noexcept {
+ // see SEARCH-2043 and, e.g. examples at
+ // http://stackoverflow.com/questions/4235019/how-to-get-lzo-to-work-with-a-file-stream
+ return len + (len / 16) + 64 + 3;
+ }
+
+ static inline bool SaveIncompressibleChunks() noexcept {
+ return false;
+ }
+};
+
+const char TMiniLzo::signature[] = "YLZO";
+
+template <size_t N>
+class TFixedArray {
+public:
+ inline TFixedArray() noexcept {
+ memset(WorkMem_, 0, sizeof(WorkMem_));
+ }
+
+protected:
+ char WorkMem_[N];
+};
+
+class TMiniLzoCompressor: public TMiniLzo, public TFixedArray<LZO1X_MEM_COMPRESS + 1> {
+public:
+ inline size_t Compress(const char* data, size_t len, char* ptr, size_t /*dstMaxSize*/) {
+ lzo_uint out = 0;
+ lzo1x_1_compress((const lzo_bytep)data, len, (lzo_bytep)ptr, &out, WorkMem_);
+
+ return out;
+ }
+};
+
+class TMiniLzoDecompressor: public TMiniLzo, public TFixedArray<LZO1X_MEM_DECOMPRESS + 1> {
+public:
+ inline size_t Decompress(const char* data, size_t len, char* ptr, size_t /*max*/) {
+ lzo_uint ret = 0;
+
+ lzo1x_decompress((const lzo_bytep)data, len, (lzo_bytep)ptr, &ret, WorkMem_);
+
+ return ret;
+ }
+
+ inline void InitFromStream(IInputStream*) const noexcept {
+ }
+};
+
+DEF_COMPRESSOR(TLzoCompress, TMiniLzoCompressor)
+DEF_DECOMPRESSOR(TLzoDecompress, TMiniLzoDecompressor)
+
+/*
+ * FastLZ
+ */
+class TFastLZ {
+public:
+ static const char signature[];
+
+ static inline size_t Hint(size_t len) noexcept {
+ return Max<size_t>((size_t)(len * 1.06), 100);
+ }
+
+ inline size_t Compress(const char* data, size_t len, char* ptr, size_t /*dstMaxSize*/) {
+ return fastlz_compress(data, len, ptr);
+ }
+
+ inline size_t Decompress(const char* data, size_t len, char* ptr, size_t max) {
+ return fastlz_decompress(data, len, ptr, max);
+ }
+
+ inline void InitFromStream(IInputStream*) const noexcept {
+ }
+
+ static inline bool SaveIncompressibleChunks() noexcept {
+ return false;
+ }
+};
+
+const char TFastLZ::signature[] = "YLZF";
+
+DEF_COMPRESSOR(TLzfCompress, TFastLZ)
+DEF_DECOMPRESSOR(TLzfDecompress, TFastLZ)
+
+/*
+ * LZ4
+ */
+class TLZ4 {
+public:
+ static const char signature[];
+
+ static inline size_t Hint(size_t len) noexcept {
+ return Max<size_t>((size_t)(len * 1.06), 100);
+ }
+
+ inline size_t Compress(const char* data, size_t len, char* ptr, size_t dstMaxSize) {
+ return LZ4_compress_default(data, ptr, len, dstMaxSize);
+ }
+
+ inline size_t Decompress(const char* data, size_t len, char* ptr, size_t max) {
+ int res = LZ4_decompress_safe(data, ptr, len, max);
+ if (res < 0)
+ ythrow TDecompressorError();
+ return res;
+ }
+
+ inline void InitFromStream(IInputStream*) const noexcept {
+ }
+
+ static inline bool SaveIncompressibleChunks() noexcept {
+ return false;
+ }
+};
+
+const char TLZ4::signature[] = "LZ.4";
+
+DEF_COMPRESSOR(TLz4Compress, TLZ4)
+DEF_DECOMPRESSOR(TLz4Decompress, TLZ4)
+
+/*
+ * Snappy
+ */
+class TSnappy {
+public:
+ static const char signature[];
+
+ static inline size_t Hint(size_t len) noexcept {
+ return Max<size_t>(snappy::MaxCompressedLength(len), 100);
+ }
+
+ inline size_t Compress(const char* data, size_t len, char* ptr, size_t /*dstMaxSize*/) {
+ size_t reslen = 0;
+ snappy::RawCompress(data, len, ptr, &reslen);
+ return reslen;
+ }
+
+ inline size_t Decompress(const char* data, size_t len, char* ptr, size_t) {
+ size_t srclen = 0;
+ if (!snappy::GetUncompressedLength(data, len, &srclen) || !snappy::RawUncompress(data, len, ptr))
+ ythrow TDecompressorError();
+ return srclen;
+ }
+
+ inline void InitFromStream(IInputStream*) const noexcept {
+ }
+
+ static inline bool SaveIncompressibleChunks() noexcept {
+ return false;
+ }
+};
+
+const char TSnappy::signature[] = "Snap";
+
+DEF_COMPRESSOR(TSnappyCompress, TSnappy)
+DEF_DECOMPRESSOR(TSnappyDecompress, TSnappy)
+
+/*
+ * QuickLZ
+ */
+class TQuickLZBase {
+public:
+ static const char signature[];
+
+ static inline size_t Hint(size_t len) noexcept {
+ return len + 500;
+ }
+
+ inline TQuickLZBase()
+ : Table_(nullptr)
+ {
+ }
+
+ inline void Init(unsigned ver, unsigned lev, unsigned mod, unsigned type) {
+ Table_ = LzqTable(ver, lev, mod);
+
+ if (!Table_) {
+ ythrow yexception() << "unsupported lzq stream(" << ver << ", " << lev << ", " << mod << ")";
+ }
+
+ const size_t size = Table_->Setting(3) + Table_->Setting(type);
+
+ Mem_.Reset(::operator new(size));
+ memset(Mem_.Get(), 0, size);
+ }
+
+ inline bool SaveIncompressibleChunks() const noexcept {
+ // we must save incompressible chunks "as is"
+ // after compressor run in streaming mode
+ return Table_->Setting(3);
+ }
+
+protected:
+ const TQuickLZMethods* Table_;
+ THolder<void> Mem_;
+};
+
+const char TQuickLZBase::signature[] = "YLZQ";
+
+class TQuickLZCompress: public TQuickLZBase {
+public:
+ inline size_t Compress(const char* data, size_t len, char* ptr, size_t /*dstMaxSize*/) {
+ return Table_->Compress(data, ptr, len, (char*)Mem_.Get());
+ }
+};
+
+class TQuickLZDecompress: public TQuickLZBase {
+public:
+ inline size_t Decompress(const char* data, size_t /*len*/, char* ptr, size_t /*max*/) {
+ return Table_->Decompress(data, ptr, (char*)Mem_.Get());
+ }
+
+ inline void InitFromStream(IInputStream* in) {
+ const ui8 ver = ::GLoad<ui8>(in);
+ const ui8 lev = ::GLoad<ui8>(in);
+ const ui8 mod = ::GLoad<ui8>(in);
+
+ Init(ver, lev, mod, 2);
+ }
+};
+
+class TLzqCompress::TImpl: public TCompressorBase<TQuickLZCompress, TImpl> {
+public:
+ inline TImpl(IOutputStream* out, ui16 blockSize, EVersion ver, unsigned level, EMode mode)
+ : TCompressorBase<TQuickLZCompress, TImpl>(out, blockSize)
+ {
+ memset(AdditionalData(), 0, AdditionalDataLength());
+
+ Init(ver, level, mode, 1);
+
+ Save((ui8)ver);
+ Save((ui8)level);
+ Save((ui8)mode);
+ }
+};
+
+TLzqCompress::TLzqCompress(IOutputStream* slave, ui16 blockSize, EVersion ver, unsigned level, EMode mode)
+ : Impl_(new (TImpl::Hint(blockSize)) TImpl(slave, blockSize, ver, level, mode))
+{
+}
+
+DEF_COMPRESSOR_COMMON(TLzqCompress, TQuickLZCompress)
+DEF_DECOMPRESSOR(TLzqDecompress, TQuickLZDecompress)
+
+namespace {
+ template <class T>
+ struct TInputHolder {
+ static inline T Set(T t) noexcept {
+ return t;
+ }
+ };
+
+ template <class T>
+ struct TInputHolder<TAutoPtr<T>> {
+ inline T* Set(TAutoPtr<T> v) noexcept {
+ V_ = v;
+
+ return V_.Get();
+ }
+
+ TAutoPtr<T> V_;
+ };
+
+ // Decompressing input streams without signature verification
+ template <class TInput, class TDecompressor>
+ class TLzDecompressInput: public TInputHolder<TInput>, public IInputStream {
+ public:
+ inline TLzDecompressInput(TInput in)
+ : Impl_(this->Set(in))
+ {
+ }
+
+ private:
+ size_t DoRead(void* buf, size_t len) override {
+ return Impl_.Read(buf, len);
+ }
+
+ private:
+ TDecompressorBaseImpl<TDecompressor> Impl_;
+ };
+}
+
+template <class T>
+static TAutoPtr<IInputStream> TryOpenLzDecompressorX(const TDecompressSignature& s, T input) {
+ if (s.Check<TLZ4>())
+ return new TLzDecompressInput<T, TLZ4>(input);
+
+ if (s.Check<TSnappy>())
+ return new TLzDecompressInput<T, TSnappy>(input);
+
+ if (s.Check<TMiniLzo>())
+ return new TLzDecompressInput<T, TMiniLzoDecompressor>(input);
+
+ if (s.Check<TFastLZ>())
+ return new TLzDecompressInput<T, TFastLZ>(input);
+
+ if (s.Check<TQuickLZDecompress>())
+ return new TLzDecompressInput<T, TQuickLZDecompress>(input);
+
+ return nullptr;
+}
+
+template <class T>
+static inline TAutoPtr<IInputStream> TryOpenLzDecompressorImpl(const TStringBuf& signature, T input) {
+ if (signature.size() == SIGNATURE_SIZE) {
+ TMemoryInput mem(signature.data(), signature.size());
+ TDecompressSignature s(&mem);
+
+ return TryOpenLzDecompressorX(s, input);
+ }
+
+ return nullptr;
+}
+
+template <class T>
+static inline TAutoPtr<IInputStream> TryOpenLzDecompressorImpl(T input) {
+ TDecompressSignature s(&*input);
+
+ return TryOpenLzDecompressorX(s, input);
+}
+
+template <class T>
+static inline TAutoPtr<IInputStream> OpenLzDecompressorImpl(T input) {
+ TAutoPtr<IInputStream> ret = TryOpenLzDecompressorImpl(input);
+
+ if (!ret) {
+ ythrow TDecompressorError() << "Unknown compression format";
+ }
+
+ return ret;
+}
+
+TAutoPtr<IInputStream> OpenLzDecompressor(IInputStream* input) {
+ return OpenLzDecompressorImpl(input);
+}
+
+TAutoPtr<IInputStream> TryOpenLzDecompressor(IInputStream* input) {
+ return TryOpenLzDecompressorImpl(input);
+}
+
+TAutoPtr<IInputStream> TryOpenLzDecompressor(const TStringBuf& signature, IInputStream* input) {
+ return TryOpenLzDecompressorImpl(signature, input);
+}
+
+TAutoPtr<IInputStream> OpenOwnedLzDecompressor(TAutoPtr<IInputStream> input) {
+ return OpenLzDecompressorImpl(input);
+}
+
+TAutoPtr<IInputStream> TryOpenOwnedLzDecompressor(TAutoPtr<IInputStream> input) {
+ return TryOpenLzDecompressorImpl(input);
+}
+
+TAutoPtr<IInputStream> TryOpenOwnedLzDecompressor(const TStringBuf& signature, TAutoPtr<IInputStream> input) {
+ return TryOpenLzDecompressorImpl(signature, input);
+}
diff --git a/library/cpp/streams/lz/lz.h b/library/cpp/streams/lz/lz.h
new file mode 100644
index 0000000000..3a2eaad88b
--- /dev/null
+++ b/library/cpp/streams/lz/lz.h
@@ -0,0 +1,242 @@
+#pragma once
+
+#include <util/stream/output.h>
+#include <util/stream/input.h>
+#include <util/generic/ptr.h>
+#include <util/generic/yexception.h>
+
+/**
+ * @file
+ *
+ * All lz compressors compress blocks. `Write` method splits input data into
+ * blocks, compresses each block and then writes each compressed block to the
+ * underlying output stream. Thus compression classes are not buffered.
+ * MaxBlockSize parameter specified max allowed block size.
+ *
+ * See http://altdevblogaday.com/2011/04/22/survey-of-fast-compression-algorithms-part-1/
+ * for some comparisons.
+ */
+
+struct TDecompressorError: public yexception {
+};
+
+/**
+ * @addtogroup Streams_Archs
+ * @{
+ */
+
+/**
+ * Lz4 compressing stream.
+ *
+ * @see http://code.google.com/p/lz4/
+ */
+class TLz4Compress: public IOutputStream {
+public:
+ TLz4Compress(IOutputStream* slave, ui16 maxBlockSize = 1 << 15);
+ ~TLz4Compress() override;
+
+private:
+ void DoWrite(const void* buf, size_t len) override;
+ void DoFlush() override;
+ void DoFinish() override;
+
+private:
+ class TImpl;
+ THolder<TImpl> Impl_;
+};
+
+/**
+ * Lz4 decompressing stream.
+ *
+ * @see http://code.google.com/p/lz4/
+ */
+class TLz4Decompress: public IInputStream {
+public:
+ TLz4Decompress(IInputStream* slave);
+ ~TLz4Decompress() override;
+
+private:
+ size_t DoRead(void* buf, size_t len) override;
+
+private:
+ class TImpl;
+ THolder<TImpl> Impl_;
+};
+
+/**
+ * Snappy compressing stream.
+ *
+ * @see http://code.google.com/p/snappy/
+ */
+class TSnappyCompress: public IOutputStream {
+public:
+ TSnappyCompress(IOutputStream* slave, ui16 maxBlockSize = 1 << 15);
+ ~TSnappyCompress() override;
+
+private:
+ void DoWrite(const void* buf, size_t len) override;
+ void DoFlush() override;
+ void DoFinish() override;
+
+private:
+ class TImpl;
+ THolder<TImpl> Impl_;
+};
+
+/**
+ * Snappy decompressing stream.
+ *
+ * @see http://code.google.com/p/snappy/
+ */
+class TSnappyDecompress: public IInputStream {
+public:
+ TSnappyDecompress(IInputStream* slave);
+ ~TSnappyDecompress() override;
+
+private:
+ size_t DoRead(void* buf, size_t len) override;
+
+private:
+ class TImpl;
+ THolder<TImpl> Impl_;
+};
+
+/**
+ * MiniLZO compressing stream.
+ */
+class TLzoCompress: public IOutputStream {
+public:
+ TLzoCompress(IOutputStream* slave, ui16 maxBlockSize = 1 << 15);
+ ~TLzoCompress() override;
+
+private:
+ void DoWrite(const void* buf, size_t len) override;
+ void DoFlush() override;
+ void DoFinish() override;
+
+private:
+ class TImpl;
+ THolder<TImpl> Impl_;
+};
+
+/**
+ * MiniLZO decompressing stream.
+ */
+class TLzoDecompress: public IInputStream {
+public:
+ TLzoDecompress(IInputStream* slave);
+ ~TLzoDecompress() override;
+
+private:
+ size_t DoRead(void* buf, size_t len) override;
+
+private:
+ class TImpl;
+ THolder<TImpl> Impl_;
+};
+
+/**
+ * FastLZ compressing stream.
+ */
+class TLzfCompress: public IOutputStream {
+public:
+ TLzfCompress(IOutputStream* slave, ui16 maxBlockSize = 1 << 15);
+ ~TLzfCompress() override;
+
+private:
+ void DoWrite(const void* buf, size_t len) override;
+ void DoFlush() override;
+ void DoFinish() override;
+
+private:
+ class TImpl;
+ THolder<TImpl> Impl_;
+};
+
+/**
+ * FastLZ decompressing stream.
+ */
+class TLzfDecompress: public IInputStream {
+public:
+ TLzfDecompress(IInputStream* slave);
+ ~TLzfDecompress() override;
+
+private:
+ size_t DoRead(void* buf, size_t len) override;
+
+private:
+ class TImpl;
+ THolder<TImpl> Impl_;
+};
+
+/**
+ * QuickLZ compressing stream.
+ */
+class TLzqCompress: public IOutputStream {
+public:
+ enum EVersion {
+ V_1_31 = 0,
+ V_1_40 = 1,
+ V_1_51 = 2
+ };
+
+ /*
+ * streaming mode - actually, backlog size
+ */
+ enum EMode {
+ M_0 = 0,
+ M_100000 = 1,
+ M_1000000 = 2
+ };
+
+ TLzqCompress(IOutputStream* slave, ui16 maxBlockSize = 1 << 15,
+ EVersion ver = V_1_31,
+ unsigned level = 0,
+ EMode mode = M_0);
+ ~TLzqCompress() override;
+
+private:
+ void DoWrite(const void* buf, size_t len) override;
+ void DoFlush() override;
+ void DoFinish() override;
+
+private:
+ class TImpl;
+ THolder<TImpl> Impl_;
+};
+
+/**
+ * QuickLZ decompressing stream.
+ */
+class TLzqDecompress: public IInputStream {
+public:
+ TLzqDecompress(IInputStream* slave);
+ ~TLzqDecompress() override;
+
+private:
+ size_t DoRead(void* buf, size_t len) override;
+
+private:
+ class TImpl;
+ THolder<TImpl> Impl_;
+};
+
+/** @} */
+
+/**
+ * Reads a compression signature from the provided input stream and returns a
+ * corresponding decompressing stream.
+ *
+ * Note that returned stream doesn't own the provided input stream, thus it's
+ * up to the user to free them both.
+ *
+ * @param input Stream to decompress.
+ * @return Decompressing proxy input stream.
+ */
+TAutoPtr<IInputStream> OpenLzDecompressor(IInputStream* input);
+TAutoPtr<IInputStream> TryOpenLzDecompressor(IInputStream* input);
+TAutoPtr<IInputStream> TryOpenLzDecompressor(const TStringBuf& signature, IInputStream* input);
+
+TAutoPtr<IInputStream> OpenOwnedLzDecompressor(TAutoPtr<IInputStream> input);
+TAutoPtr<IInputStream> TryOpenOwnedLzDecompressor(TAutoPtr<IInputStream> input);
+TAutoPtr<IInputStream> TryOpenOwnedLzDecompressor(const TStringBuf& signature, TAutoPtr<IInputStream> input);
diff --git a/library/cpp/streams/lz/lz_ut.cpp b/library/cpp/streams/lz/lz_ut.cpp
new file mode 100644
index 0000000000..6876f070fc
--- /dev/null
+++ b/library/cpp/streams/lz/lz_ut.cpp
@@ -0,0 +1,287 @@
+#include "lz.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+#include <library/cpp/resource/resource.h>
+
+#include <util/stream/file.h>
+#include <util/generic/vector.h>
+#include <util/system/tempfile.h>
+#include <util/generic/singleton.h>
+
+#define LDATA "./ldata"
+#define LDATA_RANDOM "./ldata.random"
+
+static const TString data = "aa aaa aa aaa aa aaa bb bbb bb bbb bb bbb";
+
+namespace {
+ /**
+ * Produces well-formed random crap
+ **/
+ TString RandomString(size_t size) {
+ TString entropy(NResource::Find("/random.data"));
+ TString result;
+ size_t seed = 1;
+ size_t j = 0;
+ for (size_t i = 0; i < size; ++i) {
+ seed *= 3;
+ char sym;
+ do {
+ sym = char((seed ^ i) % 256);
+ if (!sym) {
+ seed += 1;
+ }
+ } while (!sym);
+ Y_ASSERT(sym);
+ j = (j + 1) % entropy.size();
+ result += char(sym + entropy[j]);
+ }
+ return result;
+ }
+
+ TVector<TString> InitRandomData() {
+ static const TVector<size_t> sizes = {
+ 0,
+ 1,
+ 127,
+ 2017,
+ 32767,
+ };
+
+ TVector<TString> result;
+ for (auto size : sizes) {
+ result.push_back(RandomString(size));
+ }
+ result.push_back(NResource::Find("/request.data"));
+ return result;
+ }
+
+ TString TestFileName(const TString& d, size_t bufferSize) {
+ return LDATA_RANDOM + TString(".") + ToString(d.size()) + TString(".") + ToString(bufferSize);
+ }
+
+ struct TRandomData: public TVector<TString> {
+ inline TRandomData() {
+ InitRandomData().swap(*this);
+ }
+ };
+}
+
+static const TVector<size_t> bufferSizes = {
+ 127,
+ 1024,
+ 32768,
+};
+
+namespace {
+ template <TLzqCompress::EVersion Ver, int Level, TLzqCompress::EMode Mode>
+ struct TLzqCompressX: public TLzqCompress {
+ inline TLzqCompressX(IOutputStream* out, size_t bufLen)
+ : TLzqCompress(out, bufLen, Ver, Level, Mode)
+ {
+ }
+ };
+}
+
+template <class C>
+static inline void TestGoodDataCompress() {
+ TFixedBufferFileOutput o(LDATA);
+ C c(&o, 1024);
+
+ TString d = data;
+
+ for (size_t i = 0; i < 10; ++i) {
+ c.Write(d.data(), d.size());
+ c << Endl;
+ d = d + d;
+ }
+
+ c.Finish();
+ o.Finish();
+}
+
+template <class C>
+static inline void TestIncompressibleDataCompress(const TString& d, size_t bufferSize) {
+ TString testFileName = TestFileName(d, bufferSize);
+ TFixedBufferFileOutput o(testFileName);
+ C c(&o, bufferSize);
+ c.Write(d.data(), d.size());
+ c.Finish();
+ o.Finish();
+}
+
+template <class C>
+static inline void TestCompress() {
+ TestGoodDataCompress<C>();
+ for (auto bufferSize : bufferSizes) {
+ for (auto rd : *Singleton<TRandomData>()) {
+ TestIncompressibleDataCompress<C>(rd, bufferSize);
+ }
+ }
+}
+
+template <class D>
+static inline void TestGoodDataDecompress() {
+ TTempFile tmpFile(LDATA);
+
+ {
+ TFileInput i1(LDATA);
+ D ld(&i1);
+
+ TString d = data;
+
+ for (size_t i2 = 0; i2 < 10; ++i2) {
+ UNIT_ASSERT_EQUAL(ld.ReadLine(), d);
+
+ d = d + d;
+ }
+ }
+}
+
+template <class D>
+static inline void TestIncompressibleDataDecompress(const TString& d, size_t bufferSize) {
+ TString testFileName = TestFileName(d, bufferSize);
+ TTempFile tmpFile(testFileName);
+
+ {
+ TFileInput i(testFileName);
+ D ld(&i);
+
+ UNIT_ASSERT_EQUAL(ld.ReadAll(), d);
+ }
+}
+
+template <class D>
+static inline void TestDecompress() {
+ TestGoodDataDecompress<D>();
+ for (auto bufferSize : bufferSizes) {
+ for (auto rd : *Singleton<TRandomData>()) {
+ TestIncompressibleDataDecompress<D>(rd, bufferSize);
+ }
+ }
+}
+
+class TMixedDecompress: public IInputStream {
+public:
+ TMixedDecompress(IInputStream* input)
+ : Slave_(OpenLzDecompressor(input).Release())
+ {
+ }
+
+private:
+ size_t DoRead(void* buf, size_t len) override {
+ return Slave_->Read(buf, len);
+ }
+
+private:
+ THolder<IInputStream> Slave_;
+};
+
+template <class C>
+static inline void TestMixedDecompress() {
+ TestCompress<C>();
+ TestDecompress<TMixedDecompress>();
+}
+
+template <class D, class C>
+static inline void TestDecompressError() {
+ TestCompress<C>();
+ UNIT_ASSERT_EXCEPTION(TestDecompress<D>(), TDecompressorError);
+}
+
+Y_UNIT_TEST_SUITE(TLzTest) {
+ Y_UNIT_TEST(TestLzo) {
+ TestCompress<TLzoCompress>();
+ TestDecompress<TLzoDecompress>();
+ }
+
+ Y_UNIT_TEST(TestLzf) {
+ TestCompress<TLzfCompress>();
+ TestDecompress<TLzfDecompress>();
+ }
+
+ Y_UNIT_TEST(TestLzq) {
+ TestCompress<TLzqCompress>();
+ TestDecompress<TLzqDecompress>();
+ }
+
+ Y_UNIT_TEST(TestLzq151_1) {
+ TestCompress<TLzqCompressX<TLzqCompress::V_1_51, 1, TLzqCompress::M_0>>();
+ TestDecompress<TLzqDecompress>();
+ }
+
+ Y_UNIT_TEST(TestLzq151_2) {
+ TestCompress<TLzqCompressX<TLzqCompress::V_1_51, 2, TLzqCompress::M_100000>>();
+ TestDecompress<TLzqDecompress>();
+ }
+
+ Y_UNIT_TEST(TestLzq151_3) {
+ TestCompress<TLzqCompressX<TLzqCompress::V_1_51, 3, TLzqCompress::M_1000000>>();
+ TestDecompress<TLzqDecompress>();
+ }
+
+ Y_UNIT_TEST(TestLzq140_1) {
+ TestCompress<TLzqCompressX<TLzqCompress::V_1_40, 1, TLzqCompress::M_0>>();
+ TestDecompress<TLzqDecompress>();
+ }
+
+ Y_UNIT_TEST(TestLzq140_2) {
+ TestCompress<TLzqCompressX<TLzqCompress::V_1_40, 2, TLzqCompress::M_100000>>();
+ TestDecompress<TLzqDecompress>();
+ }
+
+ Y_UNIT_TEST(TestLzq140_3) {
+ TestCompress<TLzqCompressX<TLzqCompress::V_1_40, 3, TLzqCompress::M_1000000>>();
+ TestDecompress<TLzqDecompress>();
+ }
+
+ Y_UNIT_TEST(TestLz4) {
+ TestCompress<TLz4Compress>();
+ TestDecompress<TLz4Decompress>();
+ }
+
+ Y_UNIT_TEST(TestSnappy) {
+ TestCompress<TSnappyCompress>();
+ TestDecompress<TSnappyDecompress>();
+ }
+
+ Y_UNIT_TEST(TestGeneric) {
+ TestMixedDecompress<TLzoCompress>();
+ TestMixedDecompress<TLzfCompress>();
+ TestMixedDecompress<TLzqCompress>();
+ TestMixedDecompress<TLz4Compress>();
+ TestMixedDecompress<TSnappyCompress>();
+ }
+
+ Y_UNIT_TEST(TestDecompressorError) {
+ TestDecompressError<TLzoDecompress, TLzfCompress>();
+ TestDecompressError<TLzfDecompress, TLzqCompress>();
+ TestDecompressError<TLzqDecompress, TLz4Compress>();
+ TestDecompressError<TLz4Decompress, TSnappyCompress>();
+ TestDecompressError<TSnappyDecompress, TBufferedOutput>();
+ TestDecompressError<TMixedDecompress, TBufferedOutput>();
+ }
+
+ Y_UNIT_TEST(TestFactory) {
+ TStringStream ss;
+
+ {
+ TLz4Compress c(&ss);
+
+ c.Write("123456789", 9);
+ c.Finish();
+ }
+
+ TAutoPtr<IInputStream> is(OpenOwnedLzDecompressor(new TStringInput(ss.Str())));
+
+ UNIT_ASSERT_EQUAL(is->ReadAll(), "123456789");
+ }
+
+ Y_UNIT_TEST(TestYQ609) {
+ auto data = NResource::Find("/yq_609.data");
+
+ TMemoryInput input(data.Data(), data.Size());
+
+ TLz4Decompress d(&input);
+ UNIT_ASSERT_EXCEPTION(d.ReadAll(), TDecompressorError);
+ }
+}
diff --git a/library/cpp/streams/lz/ut/random.data b/library/cpp/streams/lz/ut/random.data
new file mode 100644
index 0000000000..6b3c159b58
--- /dev/null
+++ b/library/cpp/streams/lz/ut/random.data
Binary files differ
diff --git a/library/cpp/streams/lz/ut/request.data b/library/cpp/streams/lz/ut/request.data
new file mode 100644
index 0000000000..6a29bfe6af
--- /dev/null
+++ b/library/cpp/streams/lz/ut/request.data
@@ -0,0 +1 @@
+pron=snipfactors&pron=grmult_QUICK_SAMOHOD_ON_MIDDLE%3Ad%3A0.1%3A0.1&pron=umrfull&pron=usemiddleownerdata&pron=onlyfastrank&pron=sortbyfastmn&pron=rtb2000000&pron=tbs120000&pron=multbs0.63&pron=versiontbs2&pron=wffrm_ru.fast.download&pron=whrm_ru.hast.download&pron=rcy-lzo&qtree=cHic7V0NdFXVlb7nvpuXxyFhXgNx4Z0ljURpYFkbwZlxaR2s80apf0VmlrVvqgNIgBRI4kuE6KxZTYJABIZg_VlqUSnIj2hfIAiEAAEELag479k1SrV1KuMvrXUW2pnxr875ufe-87PvvS_JKyrGtQzvvrP3vufs_e2zz88-5-ErcEmsKB4daVSgKrPaKDNsY7QxzhhvXFgSM-IG-d6oMqqNS4omFU02rjemRmejlch4ABlrkNGFjD3IIP8dQkYGzbBfQfgHmLPFCBsVVzztlqb6m6bNsFGFJ7ZYEGtMMqjY2Y8fOd0V67Io0quNC9ClDTEUN2yXYrRRRfiq0SQ0-S6T1M2YjVJXYLc0jkayd1xgTEHtSXNdenqM8NIXD5-OySfzG43kc2T6kLhpI_ejYaNv0I-zjWbzNjNmzi5uKG5BBqmbfbAUz1JaNyzbmunJbM4uIf-3ZZfTRqKKgFa-id1GKpxQW58pYY1VKEmb38SkyebktW6TVxVhhShuek03SFtN28x0sQbGicBYtj2zObM108u-YaXZDvZ5GPkczWyTyzLbnM-IfN4pfN7tfCaazC4UZK0QeLtzNPnI8eipnEU5Gqd-JeR7K7vQeUNJ3LItUtvdzpNBytq8t0QIV5v2di6hPbs8R-XUBeDP7BL4Nws17hY-i5rao9HHCU0ssydQ370F0beqG_q0rAA655roFb5fJryhjWlyuGJ5w9U98SKUei6Kyzx4dme6vrIQFYHB4Aa2kNeq14-qIPDgJcsK2vKT7XIiAFW7egDkDpi94ySgidWH4LvQGMIEQ04Jc6dqobPvJtrrpnGmCjmeNIxoMco9jDrm7FizQcIYEsLY00W4XgljZZktmW1E6N7MLspK-qteKZQNAULZymI3lAHcUDg7aLFwBlCTkLayWA5p_4IBOtJlIDegtyYNoifEvAvSuOtDw6dXxIfYI4mYXk0gwMk0fMzEI1Ri0pfuFLos1JKMpHfsJ1WIsCrsESCwJ9TFOV1Yxb9Mzl8Rx31U8njVwhqUXW32-gB5E8L_oACZKV-CrglAd7HpQpfR62C1Lv17hlVWTNC52HTRiaaaBJ2VmJU4eEAXmFPQEoJHb3hJG6hUdnEUGDy2EJu0MmfuzewOHTw-Pio3eJQ4IW9bXuQMHiVKOmAeJXvaOIxzNIr-qQZ6GQqY_lOtJhYFdqsuYW1Kd-9jf_c7XSvc7f45Au9Awubw_DyH4fZRFKCDSPu-w0lzU1prPe8nChEc4DGj1gpW13G4RLS_ZF3-rpxtx2EFKxJ1DtqU9hwcl2mJL8jUKEetOML_mYAjtGV6icAW2pFm20Id4eDUnCNInJAj_Ak5jiBREkc4OFV2hLXUsBKRbNhnkubGNDSN9MYpRh6GzEGRR69eZ9Qrw8AUwOgNuZlJqeqlSvZB9UsQ_p6ienPCeKC7RD7qJtSQisdQNpsUjh5a7f43YXzV3dVEwUwE1e9thlCTByJ4ilKTyPTaJqkqEcDyb6cjbl0oPVSZbSazNy0mRiYM8vLAZZiWxFFuJNGvpYHUVZqcO5Lm6q1JtEUQxXg4-1Bil0hTzQznwbQjtXWzPGk_wBaRViOuWBBxa7aS_7tyJuXyanLVadSkkYcIfU-dJ3o0jhLR82rmKDCJSDAR1zm6EP6-Ypxo5onMrmxHqH0e8MzjcECx9bvMQA4BsdEDETm6no2dsr7E11dNfelpQW3djPoFjaH9yfFzvKUnhwUC1gnekbgkpOLHz5Gx9V0c5YUqvDq9qkdgeA1X4XW1IEqAhLU6vbVXFQaAioiLOJKZuAnYrbZQNbO9M4nWqXD1W_1aDXQezQ22ETzUSh_xVgsJNaTWiUyrpJAolFDLGh2FSQFTARSIlCqOpguZeKRZYVUVVRujS2M_jrcg8hjJ41FmVZ8QQSJMGSSGNi_5MkFNuWFMS9iHEP6hosAYXamcUdM4x0ZjR6MqR48xQI9rHq509egxQdpMMm16JKOp4iSN_hX2CgVome1pGQoCSnUsjLZoK6qs5BbLbdzDFp6EKQRMVvWgheTopChrkv8ycgbhGb6aGlIx5ltV51WcN7biS6Qx-y2Ea9Qm1dbNr6lrqk9R4-daFdCkT372757SPG7_VXKPRG_TldgrLPAq-WgOAuoQVjXtu5GCmbVWLF5-91VTEvadhcTMS7p3WdRsIF4sQLkfPenFLsYI6bWWzwppsa7TSZgV5KtPNsiryWPb4QWEk-pYiW2oAKiBYvLLH47wxkw-uyoz-ZCJ76ggbcBU-L2UUJQsicRi5ce2zk3YfzRxrTpfIBOrLjoKpqtgmSekzhOK79sPPeYBR-GFtPGhM2OQKR3NSDNnhSRgxvQzhOMysbN8gJzpxV4yvVjfKYz1CjG1EGaY0tqPPrVA6oDKsYSRPI6Iv7I-_qgOxCjfapMsEAUs0H3fsZW5oSFnglQ_lw8NOYGORTLA4kXCkKAfcEQ-Ac1IHiA6KL-otDlhbzH1aE3GT3Vz67nrBcae54_m-mmXCWrukxxpHo3e4snYKwyZa4QPBfnI8kpYYiGc2qJOTZy3h455Lty8IGFvRPo8r2HGTEmBUJ_V9bCnQEoP6S7B-yxSqmvtLEy_F_qsSPu6tDKB8Kv872nl_7OBQOBlHe_FJGCmSNgM7XLuXW95cwqHB2pEHZ9SOBR6Q67AblmhO2C3uetGkOZ-uIrY6q4RBQzHR4rwgjw2GaDgXJDNhgNBmw1ocK9hcK-hb3sNr5biZg3PO-gCL10IJZ-6ef4HMCaD8LwvFyAAMRCgf1niAFqj1gG9fygW6bZrG-40fmw5MH1w131w131w1z10193rQr9K6T0cVwuF9u-QfPbU6zWYxfuc8tWX5DP-bp4wJfsnC-3ZDiFpqhprXb0SsgRpbIp5rs5BOn3fKaky3fOfkPNx4p5SMk68dPX8hP1YKW4ICIXb3VTIscFDOp8QuN0_H3KjHgK3ezmRSgh8Mo8QGNly8MhgDByMgQOKgb6pj4MxcDAGful6jZMZA1X_1GPg-Uof3o9sRy9-raXxa97q-oR9ZyluUuLXCPYaJUFZimAYiGAtr5a6EQwUAMWwpTyGgfR6FDs6FIOUQhwz7zhC5nIHifaKCD43n4KYHEzz_6Kk-auWF2NdhLSf4fOUR-DgWGpwLDXgLH6fKHUoQqIUvp7MsrZH9CglH6Ih3Q2pZmiUOvFf3jwLFABFqU6eQwfS61HqU4RBSil9cr9_-qSQ9CgBBNq9FIEavifKhxTLPWMGvK1_bwgw5fvFxJSfPEYGHK8Va2vHp-nnobKtNL9OMOZQwJivvzTBNaaPCMicL0aZOX04dIOuiWIfWtGkdxx8nu5AnkaAPSzT5Xb1lMpxSsuOkuGc-1xGnjF3UpmGrst7z3Q4t1N53i08E0dnecfiO1Z4z3HyTMOQv0QeqHaToWSr56YUI-rTcuF9i2RpQpvY-6iLC99EifxtQp3ZG2n36EmV38jL27l-nbcUKXU2FQlxQhFjOxm5mtJ2divPvYHvbPPeOVzQpNjyNo9jBOEocTvgL7mNuRX7YtVhxKqcYnmI199OO_CL1y5I2CdMLYmtxHOqHnpKJSRnInOkzPV1iRHy8I95IoFEp_v1WoQlCmGPkyegkM7Znef3CNGwh8VwtbsOT0MRY35eKSlOhnuukr3akRShPkGzPTNWVV59XhOZ7Zn6Jn7jnGk3zZ4Wvom_8q2R3ia-wwMp_xEnMdgh0Xfxz8duGcu4QOEJhKjZpHv_OleBsl2olozkJpqKN_OREU_a91n435Cipgr5SDl0RFDQXxzKu3p_8ddcBYZKgzT7v3wcEsqrQ30bwqFcDP4ot8V_bjxuj6H7yiGM0F76uPhwe3Q_-IQ9eJqDlboEh0hhuQHyNkBOAt0GuCS44X3d-WZgsYhLHbFiZ5a_89T1CXuXpaVGDZle09gwd1pTXb2QMQKt-i99-m9cROR4INM_F2GmzxHpbvXXOFfaF8caB_NpOUKI5eGDtP3J7ETs1Ejfxfl6darKR1qL1gw_ylaN8lofypCMsyEE08BxFLDzsQienFS7z35562CqXf_Styzmlb-hWvz6AqLFZ038j2oC9MxUTY2kQSjx-eB-T4OMAdLe61x7rFzX3IWYFYhtlCDIm5LKdX6NTYKONF6_g1SuwpGvwoGKtOZZER_97qcnHJaMuTlhd1p4qqJfnOtaJS0XAVp-cdVcV8sCG6Tr3bznE6j08PYwwkK5EMi-EOcUaSBb5VdDq_2n9Izs51y9KlwixkX_I5Qsy9uj1KJvjLzDb0vdxdByiqE5V5M5wUcRfIu29aAOE7ozO0MXdZ57_21h7wGQAB5gc1Z1IHodYvuAxETvtLMXaIo2prsJ4qx15B_FLIVa8wxf3x3ez7W5FdIMpMOv3v1KO4QS6jkcHqRw-OzYDQl7mZUXHHpC4bDi6ftmBMGhB4LDwogvHHp0OKxDIBx65L6nkI4NzR4DnZw59qN51bPQfWS_6uqDj-N04fDpi-cn7KPF-DZ14VBrG5sRhC4cZt64d5l364WPEAgkzzpLhzCHDpMdCPvQiiGg5aF0dzKykZph3T5nBrL5pG2WaV0GA05XXjVvX-VgXL3gwVlwA65lkKvHl6L0Hbs-92Gs0h_kU2m0NGltTBNlR9al9wXey3HyNyr71-wt-TTb646-EKbycfZP6GmTrxu1Cfs4AsaXT5C2tfDLYULGl7uXluXGlx4b5NLNzvDSI9K9eAw7gU-KQ-6DGYsFMeqJiYDT-l7z_4dOX0p_PCthv27iuUrz48S8XSzc9pK3qAtO0GRw1bve_YoaM6SKN_i0RqPVNbIIYY1KbfCoeIx0lHz9SCSE1n3s-FC7LA86cXXHSQuVucLWg3wU30oHIYcua0rY70XwPFXxom_lpfjtn-70zjtp3JDm3-NjUY1W1_zP6clJhUrwcrNlZdLc1Jk0N3TSA5RJ89FOeuEC7ev2sm7vcDKyPr2P_tlPQg79s47-2USKIxvon_X0z6P0zyZKt4mWbqCfNnAOUrCx8xmhC6XU7A8t7jwsLaCg1NTA6jqzH1rhTWQAsrGTD0Ro5TekeQPWp3kjHk2zhkiDFPaGavkN_VjR4xh4njrf7QfJQGO3GYwBemo1FAOPPLYWgxig3OBEF-kYoLQ6Bs7VWrxZwTwSMZ_6Z42-VzpsSzejB2ICH4Wm6ebPlbdPTdirIkBvtoigYVe2g3ivPmaDFPrmi9fkejOFGdLng45PqbS6Pt-jvZlCJeaUtXSfSiMG__DzAo2-H9zbnLD3IXy9erY5dUtjY2342eb7fuF1fQ4LZJzr-MlmTqAvoU3ATtFALuWIxWOkUfdQtz5_y60JeyGw98XfUhe69_XRf3vntV0eqFn3OntfDonesDHYLfNdHmRLd1eAdIVZKuV6eYF657VWPTF2BDdquTUtbM-6lS5xuBe7hc67X_12LrUG4IcU1uOm1gD0uqO2mxiklE-HSoswA7t3r-9LKSch0Q30YG7UDgr2TAfx4E-Bo-YEUaFHzXs_zh01J_SQze7jIKfFOsC_hen3QftPyq7N5SpDPzeTfGD-LoX5d5qnJ-xXIrhV3dS1SXe_jcTEbnpvn3yNoaSpUujEzkPHPVUFyIE0-DRHfQCXjv0VJg6gH_QAx97vUA-IbLkxYb9k4pvVg1lcaSwVpo2pcXPowaxjv3jXu4of4AcPJ3MHAajBJBSATsoVdFJR3HV41wonMQnFR9d_oLp-6T8aEvavTSCnz23V7bxN2cXkc3hOX9f7V-dy-kARkMqzyMnpAzl0tT-IsA8tpHr3tgMfReWdppmXuQIUzjaJZ6L5bJP4BnWT2GlPT-gW57H0wtwuscsFaXUv1SrNtnNodD3ej7BXKs0plLXvCQJoKwUtVApaqBS0UCmAtlIAbaW0nlwpdB2VAmgr3VnJVCgkrKQhYd5bsxL2n4CtdrdfDdXi8baxihKzrZASO3hn79EEKTHbKiWhfc47g3SFZYxwH3LQvqBI1-tP5wPst-n6y9Gz5ibsFy3cokVp8XZdMkXbkV2WXQLMGqEovXqZl3MVIAZMkeAbQgFcuiWXRnAAfUCQHhMvtUexxTKQG1wMq4j_hT0SpP9zXIryeZ7bKMwkl0PtMPlQ_tpPahP2Dgv_EzQfnFM_L3Q--ObOD6LShJAwQSA6HMlNCAmJPl6u5DPdOfWSz7Ch7TzPuc7HroC-JHZVQVxgWtcEWP66JEqrA3cpIVNn6mfq1vdAWf3MNaICx4AC9UwwmE7LAwMnFwa9HCtWvvSBU_9yLBzHpLkP0cD5_bduSdjLI3o2d2YLO5Gqd8vQKtHKp8Z72dwiI9Twe3gAlej0rvd3CEsUX60FvGHxYaRv-zhKDHT_3U0J-50onq1dK78w0-OMeNXzvBAsW1YmcvfKS6yQkf7g_sCCRKmb6RwskYRc4nF3EVYkSkduDhzhvYMpTYxOvROXX5i7WJzDIe1f3ZsPArzvAI5Z5T0VNWR2hgt4d-ABU3NlPH3aTXNmpepvqZsB3hkIHnx8dai3I55jh1z5EJ9LC1R6sLkaC8UDXqJOTfET1_90Y-Xa4BcRTilaHFpb11Qzd27tLBqkBTVqV9ZByxSL77jOVacoB9LnFKZOkUrX5zgslosBXNsbUBr2ip4pEGtKTatrnFmTCmwVtMd2d_tp3pzWFQI16ddO-rhLA16p7RYOHB4_goX1BRzshyBSMGwsf9hsNfF07bxUTWp-7U014O2G4N7RPV6qmcsL6bTXPTfFScABnlM2cI1eA8oKTjX3O9QQcp110JN0sxbtRY3kE_Sq6zcWkTH1moi-CclHeKG7GL9tieV-eYOxQBrfwQeWDgWYVu6UyasyTrrk-s7PfXXmLBxzxrwhazOcKjhfOydNuygtaAWHm40M0mPl437TyLZZtaFnpivTTUYGT9AGKktrkMf0_n5M7oJyiRUyY7t7QblEqZvzBqyQaEu9_U6CgAYFZfEyopc9FM4398xN2I9FgJvbvfrQO-pC9fL49l_FAcVQ3oB7CxRKcA9CofkiHqG4MaCSVvtP9w4wj8jHfr10uejOjtkJ-3Er2H554Prl5_eaoP1gYG-O6PaDgD0OKyQBN-8fNXGZTOxNlr1Fkkh6x75c1yCZ-su7htj32cCo-BCewCjrK-_zDRxAR-gPBuCLfvV39i6Eb1QPijo_Q9TcELoPuXTn5d5RUY8LQs2V_KSoR6MPJM7GudJ4vr8mRNszgsxyVtM9qAefXZCwVwT9xlSIJ-T3G1NrtN-Y0n8BoVA_MTXw34QiKhrBVLST5lHd-vOmhL0B6acQs4uyLZJ-oB-KOvHpmd4pRMoAaec7_BAiLdb7hArMCgTHls2rw5XX_YRFJrGrRtYl7N8W8vdoFptaOne0uSlVM68GnMBCS4YHX77MG9NxVkgpd3HIOBQgYnjRwBetz4ZEaUvMyoTiBMI_UuehDTWpmfWpedPqfCYV0Pzzg3e9E28iP6SS-Xz-KVDparkGi-WFXkD2H_NT0BnUYWLlh38y389hMntDHabtrssFh8nsDXSYzF4fh8ns7aPDGORpKToTs19MKTs9huxy4j9fSxdPPOMv0ZUXVRjfpHWodmiIFhya08svfu22iWcYRstEXxq7nBZzGvxN-k81cmkshwaXU94zxuMbLtDkWN67ai--AnyXpcq5y06qcixBzv0N_xoix7_OVjzmyel48o_fPuPYvrGqnKhA89RzV4HvytH4vwvHsUMTKz990e_-FpKTozm9_J1ru0No_N9Fpj-hNPF43HsX03Pzm7ep7yKjBQ8_2bXVE8-Y1ZpVbUEisEfz4WcTQZoRAk3VDzsuVmnKHGzgWGxKNIbKzOsm8y-pYr0vJ5Mvh5MvS0jrhpIvi8mXkWsm1_BvaT3Eb4k__D_2jF58&rdba=clon&relev=ad_cat%3DO;ad_filtr%3D10;cl%3D2;cm2%3D0.0181118;cm%3D0.0869565;country%3Dru;ct%3D520-87%3A11886-13;dmoz_th%3D174-83%7C179-12;dnorm%3D32+autocad+bit+windows+xp+%D1%80%D1%83%D1%81%D1%81%D0%BA%D0%B8%D0%B9+%D1%82%D0%BE%D1%80%D1%80%D0%B5%D0%BD%D1%82;dnorm_nav%3D32+autocad+bit+windows+xp+%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D1%82%D0%BD%D0%BE+%D1%80%D1%83%D1%81%D1%81%D0%BA%D0%B8%D0%B9+%D1%81%D0%BA%D0%B0%D1%87%D0%B0%D1%82%D1%8C+%D1%82%D0%BE%D1%80%D1%80%D0%B5%D0%BD%D1%82;dnorm_old%3D32+autocad+bit+windows+xp+%D1%80%D1%83%D1%81%D1%81%D0%BA%D0%B8%D0%B9+%D1%82%D0%BE%D1%80%D1%80%D0%B5%D0%BD%D1%82;dnorm_w%3Dautoc+%D1%81%D0%BA%D0%B0%D1%87%D0%B0+%D0%B1%D0%B5%D1%81%D0%BF%D0%BB+%D0%BD%D0%B0+%D1%80%D1%83%D1%81%D1%81%D0%BA+%D1%82%D0%BE%D1%80%D1%80%D0%B5+32+bit+%D0%B4%D0%BB%D1%8F+windo+xp;fe%3D6.48773;fir%3D0.267373;forum%3D0.59255;fqwg%3D5.218048216e-15;frqcls%3Dstrong_tail;fsm%3D177;fsml%3D11;geov%3D-0.000436704;hpq%3D0;iad%3D1;iad_vw%3D-33.57460785;il%3D0;ilp%3D0.001022;im%3D536870912;imgintent%3D0.56;is_music_ext%3D0.000;isorg%3D0.460;issite%3D0.460;issoft%3D0.750;ldatopicru%3D197;mum%3D0;mut%3D15;navmx%3D0.268605;norm%3Dautocad+%D1%81%D0%BA%D0%B0%D1%87%D0%B0%D1%82%D1%8C+%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D1%82%D0%BD%D0%BE+%D0%BD%D0%B0+%D1%80%D1%83%D1%81%D1%81%D0%BA%D0%BE%D0%BC+%D1%82%D0%BE%D1%80%D1%80%D0%B5%D0%BD%D1%82+32+bit+%D0%B4%D0%BB%D1%8F+windows+xp;ow%3D52;prel%3D3;qc%3D3;qlu%3D1;qmpl%3Dru;qr2r%3D0;qr2u%3D1;qr2v%3D0;qrr%3D0;qru%3D1;qrv%3D0;qt%3Dru_nloc;relev_locale%3Dru;synnorm%3D32bit+autcad+pyc+winxp+%D1%82%D0%BE%D1%80%D0%B5%D0%BD%D1%82;syq%3D1.136661591e-07;th3561%3D0.0179;th3973%3D0.0682;topicality%3D36.0447;tvm%3D0.00222546;uil%3Dru;utq%3Ddownload+here+upon+russian+for+autodesk+inventor+auto+cad+%D0%B0%D0%B2%D1%82%D0%BE%D0%BA%D0%B0%D0%B4+autoca+pdf+torrent+%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D1%82%D0%BD%D0%BE+%D0%B7%D0%B0%D0%B3%D1%80%D1%83%D0%B7%D0%B8%D1%82%D1%8C+feed+load+%D0%B7%D0%B0%D0%B3%D1%80%D1%83%D0%B6%D0%B0%D1%82%D1%8C+%D0%B7%D0%B0%D0%BA%D0%B0%D1%87%D0%B8%D0%B2%D0%B0%D1%82%D1%8C+%D1%81%D0%BA%D0%B0%D1%87%D0%B8%D0%B2%D0%B0%D0%BD%D0%B8%D0%B5+%D1%81%D0%BA%D0%B0%D1%87%D0%B8%D0%B2%D0%B0%D1%82%D1%8C%D1%81%D1%8F+%D1%81%D0%BA%D0%B0%D1%87%D0%BA%D0%B0+galloping+skachat+besplatno+free+%D0%B1%D0%B5%D1%81%D0%BF%D0%BB+%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D1%82%D0%BD%D0%B8%D0%B9+%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D1%82%D0%BD%D0%B8%D0%BA+%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D1%82%D0%BD%D0%BD%D1%8B%D0%B9+%D0%B4%D0%B0%D1%80%D0%BE%D0%BC+gift+%D1%81%D0%B2%D0%BE%D0%B1%D0%BE%D0%B4%D0%BD%D0%BE+freely+%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D0%BD%D0%BE+%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D1%82%D0%BE+%D1%85%D0%B0%D0%BB%D1%8F%D0%B2%D0%BD%D1%8B%D0%B9+russia+%D1%80%D0%BE%D1%81%D1%81%D0%B8%D0%B9%D1%81%D0%BA%D0%B8%D0%B9+rus+%D0%B2%D0%B5%D0%BB%D0%B8%D0%BA%D0%BE%D1%80%D1%83%D1%81%D1%81%D0%BA%D0%B8%D0%B9+%D1%80%D1%83%D1%81%D0%B8%D1%81%D1%82%D0%B8%D0%BA%D0%B0+%D1%80%D1%83%D1%81%D0%B8%D1%84%D0%B8%D0%BA%D0%B0%D1%86%D0%B8%D1%8F+russification+%D1%80%D1%83%D1%81%D0%BA+%D1%80%D1%83%D1%81%D1%81+%D1%80%D1%83%D1%81%D1%81%D0%BA%D0%BE%D1%8F%D0%B7%D1%8B%D1%87%D0%BD%D1%8B%D0%B9+russkom+%D0%B1%D0%B8%D1%82%D0%BD%D1%8B%D0%B9+%D1%83%D0%BA%D1%83%D1%81%D0%B8%D1%82%D1%8C+background+intelligent+transfer+service+%D0%B1%D0%B8%D1%82+bit+%D0%B2%D0%B8%D0%BD%D0%B4%D0%BE%D0%B2%D1%81+%D0%B2%D0%B8%D0%BD%D0%B4%D0%BE%D1%83%D0%B7+%D0%B2%D0%B8%D0%BD%D0%B4%D0%BE%D1%83%D1%81+windowsxp+windows+xtreme+performance+%D1%81%D0%BA%D0%B0%D1%87%D0%B0%D1%82%D1%8C%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D1%82%D0%BD%D0%BE;vcomm%3D0.248625;vintent%3D0.95;vm%3D268435456;vnorm%3Dautocad+32+bit+%D0%B4%D0%BB%D1%8F+windows+xp;vnorm_dopp%3D32+autocad+bit+windows+xp;vnorm_syn%3D32bit+autcad+winxp;vqi2tvec%3D1%7C0rkjACu_CrE81xPoE93gAB5X2uKgUsx_JvATChj78_A%2C;vqsemvec%3D371364212%091370421533%093362250591%092760956004;wizqbundle%3DehRMWi40AQAAAACAUzAB-F8KZipZehRMWi40AQAAAACARwAB8wYSXBJGEgdhdXRvY2FkGAAgAioMCggPAHNzEAAqDQoJDgAQJw8AJAsKLAAnEAEcAIRzJxAAIJMCKh0AoDICCAI4AUAAIAAAAAAwo7-XscucoK7PAQr0ByrmB2oA8P9S1AMB9g8SjxcShgkSDtGB0LrQsNGH0LDRgtGMGAAgASoQCgwWAIfQuRAAKhYKEhIAfbLRiNC40YUYAMe90L3QvtC1EAAqEgpYAE3QtdGCLAABRAAt0LwYAH290L3Rg9GORAA9vdC-LABRtdC80YJwADsYChSgAAAcAB2DMgACdgAP0gADH7WOAAMh0L4CAQrSAD290YtcAAD-AD2w0Y-6AD-70LiiAAQRs9QACloAP7vQsFoAAS_Ri54AAy_RiwABAACKAB--AAEFA14BCjIBAjIAApAACv4BHbuOAABEARG1dgAKKgAdveYAAMwBMYvQvAABCuYAD2wBAQBYAB-4WAAAH7KCAAIRuGoBCYIAATYCCtgAAlQAD4IAACHRjpQCCZYAAAADLxAB7AIAAtYBNxQKEGQCX7XRiNGM8gABI9CwSAEMMAACqgIKSgMCeAEABgMJigAARgAN9gABLAAB5LYA8HMB5AANigAAFgAEigANdAAh0Ys-AQ8YAAEA1gMKYAATuWQDCi4AAGAAH7UaAwARu9QCDywAAAG6AAoYAAIgAna5EAAS6A0SYQQyuNCyjQEBjQQ1GgoWdwECGgAzjtGJgQII7QEBHAAAawQBQwINNgABsQACyQENHABP0Y7RiRcDAANSlAHwZdMAAhoAAusADU4AAGoAA04ACEkBAzQADzIAAQCAAA8NBQABMgABlQIKOQECGAACwQI_HAoYHgEABPMCD5wAAgOLAQ6cAAAXBQ8yAAMC7QUNbAAA6gAFxwQPOgADAiMCDToAAfIABDoADx4ABQLBBA5YAAAaRwDw2Q0cAAFYAA_mAQQBrgACYQQNcgAA5gEAYwUBSQMNVgAAHgADBwYOcgIfuXICBgGoAAEjAw9wAAIFKQQOVAABHgAPKgIHAXsHDsABACkFDWoAAhIBA5MEDtoAABYDEbhbBg1QAAGxAw0yAAFQAAJDBw0cAAJiAQ-aAQgCtQUOOAAAKgEC9QcOHAAAogAP7gMHDoUFAR4DALIBAaMHDVAAA8AAAKUICI0EAjYAAn0HDTQAAk4AASYBDWABAfQAAcAAC4MFAlIAAL4AAP8HDzgAAw-hBQECOAACNQgPigAEAQACDhwAAWwAAXAJAOAAUgICaQYOHAAAOAACkXYA8GcBOAABCgIOIAIA6wgPMgAED4QFBAAoAQKxCA4QAQFqAAOoAg6aAQKgAA42AAC8AATuBA4eAAJUAA9IBAQBCgICWAANxgACHAAB-AAPjAAAATsIDTQAACYCDxYBBQJQAACcAUcgowIqHAsA3wWgMgIIATgBQAAgRQT4AjDk_fqeq-3Q-OMBCqgCKpoC9wPwCAgBAf0XEr0GEqABEhLQsdC10YHQv9C7-QMA1gPxARgBIAEqHAoY0L_Qvh4An7UABPAlGAoUGgAELxYKUgAAHxBQAAkftVAABn21EAAS-AQSawBv0YvQuRgApQAKD1UABR--vwAFTwME-4W_AAKdtdC9EAAqGgoW8QBfsNCz0L69AAYAJwEPDQEFDz8BAT--0LyAAAI_vdGLZAADT9GL0Y8aAAYfhRoABQEaAQ_OAAFfvtC80YM2AAYAnAAPtgABEbCEAA_oAAEfsEwAAyHQvoIAD18CHgD0AQ-IAQQfvogBCBG-1gEP8AAAb9GL0LzQuKQABAFAAk8gowIqzAIAKwHovMHs0NOEgNjdAQpIKjspAfEAKQAB8QISJxIUEgTQvdCwFAXBCAoMAGEQASCjAioLbAUAdQESAWwF-AHYpuGipb-7vI8BCu0BKt8BTADxBM0AAfcPEvEEEuwCEg7RgNGD0YFyBRC4NwEEbAUhaL5RBREoAAWRFAA-vtGOFACIZwXgFAoQTgBN0LzRgxYAe7jVAPAfEAB6AAASAAw-AB-zkAAAP7jQuXwAADq8EAG4ACHRg6QAClIAH7BSAAQAugAK9iAA8Cm8ZgAADnoAH7geAQCPuNGFEAAS5AFvAQIKkQAPIQELDCwAD70AAw6VAAIPAQo-AA_7ACsP5wAGSO0AYy4C0L7QvPAAA2UC-QDW3teX_ICvr_gBCqMBKpbwAPEMhAAB-g8SmAIS-gESDtGC0L7RgNGA0LXQvdGC8ACiFgoSFgCg0L7Qst0A8hMMAAYuAE_RixAALgAAP7DQvC4AAS_QsC4AAy_RhS4AAnq1hgJQcgBv0LD_APIESAACH752AAN70YMQACoSCugAa5cBFRWXAQOnANjLxM6v-_j5l1oKRyo6lAH0GCgAAXQSJxIZEgswAQDwDTMyGAAgACoGCgIzMhABIAAqAjMyMgA4AksH-QCA877ojIGuoecBCqQBKpfvAPFthQAB9hESvQESMhIDYml0GAEgAioICgRiaXRzEAAqCQoFYml0JwsAcHMnEAAqBworAEYQARI1NADwAQsKB2JpdHRpbmcQACoKCgYNAEdlZBAATQAFNwAgQBJgADVlGABsABdlNQAVbgwAAk0AAoQAEGWEAARCAEAgkwIqg_AAB_MH2J-xsZ7qkK_2ewpJKj3wAPECKwAB8wQSLRIYEgbQtNC70Y-KAVIKCg4AYzsBEQ1LAAfSAunHnYGH9bv932YKkQEqhPIA8QNyAAHyBhLFARJBEgZ3aW5kb3f1AGEMCggOAHPnAPEaCgocAGMQACoLCgcaACYQAScAZSdzEAASQkMAPwAgAjUABzINCglCAFb8APEKRAB0ZWQQABIoEl8AD4gAAAh8AEQgkwIqK5QAB98A2MbplNn6pa_fAwpDKjffALElAAASIRIQEgJ4cDIJcAYKAnhwEAFFADoCeHAXCdiHlbXAt_bOjnUKVypKRQCwOAAB9AcSQxIsEgh-CUFkZXNrTQBgDgoKEAB1wABiKgwKIABlWAAcD50A-QDmvbXihY6InIYBCpgBKosyAR95WwAcJPEHtADwHxJiEksSCGludmVudG9yRQA0DQoJEAAARAA0DgoKDwAEVAAEEAAgEAFyAAQOAEF9AUdkAAQRnAAAWgr5ADC5xvmm0_qipSoKoAEqkpsAsIAAAfADEkwSORIE9gAB8gBhCgoGDABx8gBwCAoYAGAQAcoBQBYAVGVhAPUAIwCBZXMQACCTAiokAPECqgAQRAID8A5jYWRNAGEJCgVjYWQ1ADAICgQLACMQABUAAWEAIP4CfCAQAUcAAAqiAK_p2fTEnaq4t64BowAC4PwREpoCEvwBEhDQsNCyVQQAswoj0LSbBWAYChQYAJwcBAFTBIEyAB-1FgABjCAEURYKEkYASgQCVgTwAAI_vtGOGAADL9C5RgABingE8AIqEgoOdAAvEAGIAAGL0LwQAPcEES2jAAASAwNNBOjw7e2q0ZnMoLYBCmUqWDoCcEYAAfIFElYQAwBEASFjYUYBARADEEYyATEOAHNNATAKCipwAwBNARAoIAMAQwERGmcAB8QD6JGI7ved-9fU6wEKfypzZwDzBWEAAfUHErABEkwSCGRvd25sb2FkegNQEABkEAEfDNEeAHRzEAAqDwoLDwCUXgNxKg4KChEAb2ID0EtOABkHPQAqcydNAIWTAACCAByMggCeq4T0m6vIyclDNgWhABIkEhISA3BkZuMAIAcKCwACEQMACgAEDAICFQfosbOQvtqmg6CnAQpjKlbKABZE6QxxdG9ycmVudE8AAekMAcAAAekMwBknDwAQJx4AJAsKO-AAAK0AHA6tAPgB_Ma8_MTw4JzsAQqrCiqdCmcALwsF7wjrJPANtgfgEpoYEvgXEhLQt9Cw0LOFCODQtkIDJdGMmwIJGgBj0N8N8Du1qwAAkQAGHgAzt9C4HgAANQIKHABzttCw0LXQvLkBOxQKEHAAAAMCAH8BBk4AMLbQtd8ADS4AU7fQuNGCQwEILgABYAADKQEAl4YM8AIFKwENmgBl0Y7RidC4AwIGOi8L8GMF6QEGGAAAbgAACAEEOwMPUgAEAesBOBIKDuwAAZ0CCuoAAWYATtGMEAEYAS_Qu-oAAQKwAAFpAwzqADG10L2dAgLTAwqYAASgARG40gAKbAAAHAAAHQMNmgAAUAUNhAAALgAAAAEC4gAMhAADigEDJAK2CvAWHgARu28ED5wAAxW16QIJbgEC8gABDwQKOAAAUAABigABHAAK1AYA8WQNQAIAFgAAoAIOmAARtVgCDBgAAFIBAAUECngABQIBDzYCAyHQu5UDCjQAAkwABM0DD2gAAQCHBQ1OAQH8AARqAgweAACeAB-92gIGAx4AAaQAD4wAAgOQAwwcAANUAQFiAgAvBQaMAQX4AA2MAAQaAAQchgDwB1QAE7VMAw5uAAIaBACWAQ8cAAIDVQa7APAbcAAfvFIABQE0ARG1wAAPMgECAOwED4YCAAMyAQNfBwqOAzG30Y8gAg2iXwHw9AJUAw0cAAMQBQStBgZIAQEQAS_QvPoEAgAQBD2I0Yx2AwSeAAI0AA1sAAGIAA98BQUAJAMCHAAMeAIBfgEPbAAAA1AAAZADD0QBAwRrBwqKAACYAQBuAAKmAA5uAA9QBAEDhgADeAEK6AEBHAADEAEMiAAE2AAG8wcKYgEfuSACAgSiAALyBArmAQFkAQ3cAgRqABO4bQcK8gABNAABNgIA6AEN8gAv0Y44AgMAAAIDnAMKJAYRt14DDGQAAuwAAlYBChwABdAAAQQFCiQBBBwABEAECjoABFYAAvwGChwABDoAAr4CClYABR4CD0YDDABSBgpgAQBWAA_UBgIAKgMDvQoUAvAeMgAAUAIEfAEPHgIAAEgBDcICAzgAA1QACtwAAPgAADYCAmEKDbIBD74FAgNQjgLwAw1UBgEcAA2CAABmAAK6AA_aBBcBQQA0Ag4XAvGyD1IAAQKWAwHCBw_UAAQPoAAAAbABD04AAQDOAAPLCg0GAQGUCA-OAQUAPgEAZwsKrAEA8gAB1gQPGAIEAtgAApgFDr4AAPQAAgAICowAAF4BAjQIDW4AAw4BAowADlIAARoAD_oAAgIaBwItDQ5-AQHEAA8wAQEBcAAChAMPNgADD8IHAgNmAQ-mAAMCigAP_gYDAxwAD9gDAAHSAQC2Cg-SBAkDDAINGAEAOgACGAECHQwGNQUAewbQgtGMMgIIAV0GEAEUEvgBi5Xsua2Avp3ZAQqeCCqRCH4SX_8DAfoTKQQBAB8OIdGMMgpyHAoYGgD3ATAEANENAt8NBDQEYx25HAD3AjMEJNGLvwwQOMkCIWqPTRKxhgBdtdC90YsuAF85BPIJYAAFb9C-0Y4QALYAAAJ8AD-80Lg6AAFvOQQghADYAvEKAbIADwgBAxOwCAEKugABNgAv0YxqAAcBIDsE8QgEAS3Rg-oAADQBH7tGAAgfvOoAAHO10KgSYZgAD6ABAy0EwIQBALwBAFAADcoAT_MOwGgAAAFSAQ9oAA4A6DEN8Q4AOgAfuzwCCQHAAQq4AAAgAgKiAR-FDgEBAXQCAiIEwQBsAB-yVgECD5gAAx8EEbAfBPENAgEKlAAAMAEACgMPNgIEL9C7agICAEwAE72kAiEEglQDDWoCAZ4CIQRDAVIADyAEs9oCCsAABRwAAQwCIgSAEgI3GAoUqAIiBC8UAiIEASOeAyIEZi_QvooABB8EGTYfBLwWASHQvqIDCyIDISAEogFIAR-1ZAEEP7UdBJBABB-IQAQCAdAYEgccBBCs9wAAHAR1iAAPWAMBAhwE8QAxvtC8tgQKigAECAIP2gAjBPQEUAQKNAAAGAAADgECeAEPOgIBAiAEbwKIAA44ACAEBPIAH7hCBgQCmAERvugBDWABHAT2BAIQBwGcAwoMBhG3XgMOIgEAxAEcBBkgHARoBdgAAYYAHAT5AASSBQTsAwo6AAYQAQBABCEEUS_Rg24CIQTzAvoABHwBDuYBAAACACgICoYAIwQQHCMEZwE2AAEyASMEIrYBIwT6BA9EAwEA1gAD8AcOggAC1gAAEAkjBIpqAg5SAAHUACUE8AgPuAIGTILRjBACCgHqAAIGAQH8Aw0YAgoCNW4CCyAEphG1qAAKkgEAdAAhBCAABFEAoBoAAVIADBwAAbL6A_YADn4BAIwEDvQHAUYBI9GJJASCARQBDy4FAwQkBPEDAqYAAqYCD_wAAwQWAw_gAAMCZwJwAU4JAvwASuMKW3ULAfcBFBHp16ff677V8PFICo8IKoEhBNHvAwH3FRLkGBLAGBIUSggCNxY3uNCyIwRwFAoQHAC3sFASAh0EEBZiFBBBRQiQHAA9FgoSHABoWBbwABgKZgAASgC9vdCw0Y8QAHgSoDIAAU4AATgACmyPAZAh0LNsAAxUAF-dCHC4VAAMHbzAJBNkE75UAA9wHBIgxABdBPMDEbWmAD8eChrEAAQCOgAP5AAAQQTwET4ABgBcABOwlAAP6AADH4U8AQQAOgARuOgAD5YABQRYBwDxDAAByAEPmAEKIdGLXAAK6AF_sNC50YLQtZIAB8oSUEIBAFG9tATgOAAPrAABA8oAL9C4tAF9CPGKE7UgAQ_IAAQh0YskATcSCg6cAQKOAg8yAAcPkgIFAnAAAJICDBQDEbugAAqkAQEUAwBGAy0QAYAAH7sIAQAAXgACdgEOXgAesBYAT9GO0YJsAQMBVgIRtVIBCpIAAVQDA6QBDNYDEbXSAQwwAAa2AQ0wAA9-AAUByAACggMP0AEFBLQBDG4AABAEEb4OAw-cAQQh0L7WAA8ekwDxIjwADNoAEbsoBAysAgGmAA2GAQG0AQNwAQ-eAAADTgIKIAQCNAATuVgBCjYAAhwAAiwJAPCyggAftVYECg9MAwQD4AEA3gMPCgEBAdoAAQoBD3IAAQT4AwwQAR-1wAAJARwCCjIAAtoAH7LmAQQA2AATi7YDChIBBKAAD9wACwBkAww4AAIeAgA4AAwaAAbyAA-kAAEAbAMLFAIBlgMOIAECigQfvBgCBQNyAwx-AAM2AA9-AAMBigYPCAMAAcQBE7giAQ_GAQUC5AELPAQPRAUHD2wABQGSAwCSAQDABg8UAgEPiAAIAqgAAUQFD6YAAAEeABG1kv8A8poAA3gAALIBCnYAAjwCAJIAHYuCAwFQAQOkBw84AQUAOgAAzAAPigEBAmQGCggCAN4CEYjyAQ9SAAAA3gIF-gcPUgAAIdC-4AALQgIADAQPfgUDAFAAA9AGCk4AAvYAAioGDxABBAHoBQvwAQ9kAwUCfAED_gYNngABSgAKgAABWgEDJAUPCAEAAVAABB4EDIQAH73sAAgCcgEP0gACA1QEDxwCAAFuAAIGSRfQsAEPOgAEA0oIDx4AAHEG8glGBw8eAAUCsAIMHAEARAQCGgAPOAAFAq7cAPAOzAAAkAUPzAABAFoAAL4ID3IAAgE4BA9UAAYB0gNDAPAMADgAA5gCClYABGQCAKILChoAA4gAAXAACh4DiwnwAuQCD2oAAQG-AA8ABAoBbgFIDgRZ3QsCVQAzCBAAMwj4AZy27ceGip-f2QEK4wEq1gEzCNHEAAH9FBKMBBLoAxIU1RQARxoBEARh0L3QuNC1NQiSGAocAK7RjNGOEARQNgBv0Y9mD1A2AAEAUncPQDYABY-UFAAEBBBSmg_wAbhwAAUv0LVwAAgANgAPwABxA_AAwAAFL9GMwAAJL9GP2gADGQDwARoAA0_QuNC5GgADIdGM1ADYAvART9GM0LXuAAMAGgAB7gAPegEAAIQAD0QBBQAcAAJEAU3qAEzPAQBF-AT4AJi7qpKGyID3PQrsAyrfA-YA2s0BAfsXErYLEpALEhbmAAAZCSPRgU8TYB4KGh4AcPEMYbXQtSIAauUEwBQgACPQuxoADToAjzcJABsXQCAACU9OG_IIIAAKM7DRjyAAKxoKuADr0LvQvtGB0YwJAfAGlgACXAACHgAPOgABBBwAPyAKHDoABw3wBrNcAA_YAAIBEgEzuNC5mAAP-AACT18F8BwgAAwPQAALH7hAAAcBgAAGQAANkgEBcgEAFgEPmgADFbhSAQ0gAAJaAAFAAQfwEg8cAQUzvNGD3AAOWgEztdGCHAANXgAAEAIHXgANIAABfloAYA8EWAENnjIA8lUfsLoBCQKCAAXMAg9CAAMxuNC8HgIAAgEPpAAED7QCBwKEAAViAQ1iAAGCAA9iABAFpAANYgEl0Y5iAQ72AjG50YI6AAD-AQ0YAgWyAg5UAAMCBB8QAAQAA7IAD64CDQQgBA1cmgnwDqYDBhe1sAANLAEBTAEA8AEEbgIPIgAFBqwDDSIAtQLwHBQCCgBmAAemBA5UATK10YggBQ_-BAEB6gEAbgEPKgIFD74EBwDaAAFmAkvvAVvbBASTBecG6enM4IPgrJuYeQqUASqH1QLUdQAB8AES7gES0gESDO8BMQYAeB4ZYRAKFAB3uHcWsQ4mAD2-0LkUAJewvQLxChQKECgAMbDQvD4AClAAK7AQYgAt0YNOAD3MF-I9sNC8TAAbtRIAmLXQumMSRMEAsLCiDwIzEdiW1aqNnOWslW4KcCpjhALxBFEAAfMHEocBEikSB3NrYWNoYXT7EWANCgkPAHTLEfASKgsKHgBKEAESRisAMwwKCCsAACoABjkAKXMnDwANSABEPxEcVj8R-AC80eakh8SnqcABCosBKn9zANRtAAHwERKkAhL0ARIgCAEC8wIGOxpAEAD_AesHAR0BMCoKJjsaMywADDwa8AwmCiIoABIPVAAVH7VUABQAKAAvJArQAA5vEAGZATsnAA6XAen5xY6Ulab6tjMKrAEqnpcB8QaMAAH1CRKhAxIvEgliZXNwbGF0bm8EAWEPCgsRAHYEAfERDQoiAEwQARIeMQALIAAcUiAACFEAEGVRADUQCgxiAF_BFPAldAAOH1BUAAkBQgA1DgoKVAAAZAAIdQAPxgAhD-YAHgjVADtpbmeDACxlZBEAAKUACjoBRj0BEWo0BAerE-iJnJWDwt20i4EBCnwqcDwB8QFeAAHwBBKkARIrEgRmcmVlqAByCQoFDABwcu0XUQsAcXN0YRWxIwBOEAESNy0AcGTGAUEHLQBUuhOwRQAZczkAD2YAGUF_ABxxvAHp5MjfwIHK0NUTCpoBKo0tAdZ7AAH2CxK5AxLOARIKrgEBpgGBEAoMEgCG0YsrFTAQEgATEgI6FXAOCjoAZxABDxWwJgBavtC8EABMAE3GDVAmAByyJs8CwCYAH7BMAAAesCYAT_YCQNEAvkfLARFoHAEHLhXpy6ShvsiwpJVTCvoBKu2dANbbAAH_FBLQBBKsBBIUnQAAWwIBNgYVuTYGIAG9_BkDkhxQOgARtR4HB2ACL9GFGgDADBC5BwCRuNC8GgAGfbkQWxqAggAfuDIABS9LHDQEnbV1BvAPTAB_tdC80YMQAGgAAQ9OAAUh0Y5OAA8CAQERtc4Apw1QP7jQvLgYAMCPagAPcAEHADwBD1RIChCP6A7yBvIABQGmAQ82AAAj0LU2AA9aAQF_tcYDWhMCAbC5xwOfxY7vtvKou7MXxwICj_8UEp8FErwC_QACEbqkAQAzBzECbxBxD0A2AAFvghtgEAAeAAh_NRtgGgoWPAACgACBKh4KGhwAAl_CAfESPAAIDxwACC_RgzgACB-wrgAIP7DRhR4ACQDqAA8_Af8tBx07YgICdQP4AcaI8fa1rMGLsQEKpgIqmALJAt0GAQH_FhKnCxK4AhIWrAERvfwd4AEgASoaCh4AAz-8EAAclAeAvtC5HAAIDziNACCOOFUBkIMcAD4cChiqAMsAELPNARAFJwBgtRwABQDkBAiyVgAGP7zRgzoABY8RDPAXErw7ASIAOwAPOwEDD1cBCS_Ri8kABwNXAQ_JAAUCHwEPHgACAHIaCBAeLQiwsFsBCQ8_ASef0YuYAvEuEsgDegIhDz8BIw-WAgkPWwEQDwYDAg-yAi4PkwEpDzwAAgAJAg-2AggPDAMPD-gAAgKvAQ-zAgMPCgP_MygBPGYFA9cB6Pnp99egktmV6QEKTSpBJwGwLwAB9wgSORIgEgqVGCDRgLEBAcoBMA4KElQDACQDApcZByQDn8HXhZeluLalHj8EAPUH-xEStAESlAESENGB0LLQvtCx0L7QtGsFYhYKEhgA3aIfMBoKFncFMBwAbS4D8gMUCkwAHxBKAAMALgAPSgAFfbmnAhFLPwQHfgDY6IT9ysKFy7o5ClcqS80AuDkAAf0OEnkSLBIQ8gEFfAAwFAoYsgExLgAbTgkcRVkA2N3H666P0qrwTApTKkdZABE1WQAcS1kAAM8ZAfMDAFkAEm0sARwXVQDpn_CL1IKJ3ethCrwBKq5QBPAInAAB_RASiAMS6AISENGF0LDQu9GP0LKyDQNjHwBZAPAKHxAWAAE9tRAALAA_g9GOFgAAPovRjxYAL7EGoQEv0L5YAAGr0L70IMAWChKyAD680LgYAG7KAUDRg3IAnyCQhS4AAj6z0L4u7CPxBw8WAAARsMwAC4oAL9CwRAADINGLRAATARxTDgXorN7M1fKtjPSPAQpUKkgUARE2PBq0OxImEgZydXNzaWEKGQY1GhAcNRoAAgYCpR0HgQbY2YSlusPR2boqCm0qYFYAsk4AAfQFEngSGhIHVgCSbhgBIAIqCwoPIwhyHAAzDQoJKz4mAmEmoykQADkAByoAhCfdGw0sCPkA-6qwpt32g5SJAQrUASrGhAGgtAAB8AkS_AMS2AoMMYDQvuUgMLjQuTkIMQgAfuABANYEEV_uD_EQGgACT7jQuRA0AAM_vtGONAAGH7UaAAMv0YM0AAh9vCoUQBK4AH4WIWAaChYYAE-FAWE0AAQfuIAhDPAF6AAHL9GFaAAGEbOEAA84AQERvrYADBABmApghgAPHAABAiFiCg88AQSu1gEAgwMgvwEwDAqtBa-q4bqf8pLDtqYBdQkAsfALEmQSHBIDcnVzQgFQCAoEcnXUG_AAKgcKFQBGEAESNB4AeQoKvQFAZSAAQfEfEBXyHDQLAIBJASBAAHEFB7cB-QDj_o6a2pz4xI8BCt0BKs9JAfEIvQAB_xoS7gQSxAQSGtCy0LXQu9C40LpGIRGDVQEQusMAAVABYR4KIgAFn6gHgSogChxCAAR_ZiPAEABCAAoftSAACj-4XwwQC5ML8AKiAAkfuaIAChO-ogA_HAoYxE0BoUAADD-80YOiAAnqEPAMIAAJL9C4AgEMH75AAA0v0YVgAAw_uRABpAEIWAEQ5DEHkQcPwgELELCiASYFMCUCBY8AClIB-QDxkqauxviIwsEBCqIBKpXgANCDAAH-ExK0AhKSAhISiRqB0YHQuNGB0YLmAAJqI1EYChQaACMOADwkUBoAAX7QcwswFgpQYxRQD04AAozkBaAqFAoQTgAPMAAERAJhYgAEH7UYRALTGAADP9CwEOAAA56-0P8EHcVsC-nkupf52uG8oTcKqQEqm6UAgIkAAf8XEsoC_goVFqUAEoSjAEPRhtC4QSFxHAoYHgADj40AAf4NIwOfhR7xIhoKXAACARwAD1oABE_QtdGOHgAHAFoAD1gAB0_RjxABdAAFAVYAD5AABR-5cgAHr9DLBQCjAmrLAALA0Y9RAfkAsJKumbn30cyuAQqTASqGrADTdAAB9AkS0AESuAESCKwAEbofAmAQCgwQAGo0I6IQABIAK9GFEgCVdiNgDgoKNgBlVAHxAgwKVgAaEFQAEb5UADUSCg4wwgzyATQAB0QAK9GDVAAbtRAAdbCaABFlyAIHQga_-oj2o4emraw2CpVFDQAq9QiWACHRgREHBK0eZBAKDB4AaZ4AEpiZAGESCg4kAITuAfAIDgoKFAAr0Ys2ADuw0YUiABuDIgARvmqaAHMq0LUyAIXQlwAcq5cA-QD_-tm7xIeol4cBCqECKpTQCCACAV8DcrIGEogGEhouAQCiBKfQvtGP0LfRi9GHNgYAXwMSBt0B0BwKGEIABa8QACoiCh5tB-AiAANv0LXQtRAAYgAKf8AUYCAKHEAAAw4ncLzQuEIACy-IBPEXEB-5RAAJDyAADhGwpgAPKAEGIdC-hAAPIAAGAGoBLxAB5gAHb9C1AzBqAQn3KDBgAAefBvU14AAKEb4gAA_IAQU_tdC9PgAKAsQBD6QBBgD-AC_Rg0IACSHRiwYCD4AABT-90LA-AAkh0L4-AA-AAAkCPgEPIAEGr9D8AkzpAgbApAPpiY66wtOj_skWCpwBKo9SAhF9bA4gzgJsDgA-BjFrb238BA9sDgCRGisACRwAGigccw7QRwANRgAaRioABnEATCUF8RWAAA9IAAgaR0gAMw4KCnIAOWluZ0kAL2VkSQALD9cACQ8cAAiYDhEdMQcHIwXp1JyM4sqUsu1pCr4BKrCfAPAAngAB9QwSxAISqAISDNCxfh4HtQEAoQ8QpLQHAk4CwiYAatC-0LPQvhQAiBIo8BUQADoATNGL0LUSACrRhTgAI9CwTAAHcgAATAANOAAbjzgAW75tBvALJgA6g9GOEgAs0L6CAC3QviQAEbBcAAeAAEpPBmNIAAAYAWWsCDwBABOEAvkAgone2__i_ODlAQr7AyrtUBLzAtsBAfQOEv4IEuAIEg7Rg9C6BQUEfS1gEgoWAJeIMicA4wJxFAADKgBmEPMJoD4AIdCyLgA3tRBrGTESANvlGgHsAGEYChQwAFlKAmAyAANKAElpEsF2AAJKAAosAAIUAFmsC2BEAAMYACtvCUADXACryAQCRwUwjgB0DgQB9y1gSdGP0YKEIhLwjz-I0YzKAAITtcoAClYAMbzRghIBBvoAACoAAxIBKdGDtgAEngARtfoADzIAAQ8sAQQh0LDQAAe6ACHQuHABNA4KCswAAdwADVQAT9GL0LwaAAMFmgEGDAIAoAARuy4AD_gBAQDgAQYYAAPmACHRi5oADEQAAMYBBpwAG4gIAQU8AAE6AQ9UAAAB3AAGGAAAmAAAfAI_uNGFmgECE7jkdgAAahLyKwQCfAIJUAELCgIBpAAp0YvAAQEUAAB2AgHAAQZuAAUYAAFuAAaEAQS4ABO1CgIJcAAt0Lt6AgJKAgrWDvAIIdC7rgMGfgAGZgAPJAMDAjwCBjAAAJB_F_MLAU4BBhgABEgAEbhUAwaGAAEwAAkIAQYqAHQlJF5NBAA_AzQV-QDDn9HZ2s_81sUBCt0CKtCCBPEHPgEB9goSxgESVRIKYmFja2dyb3VuZGEDcg8KCxIAdnOJAXARAB0nEgB3RwPwEioOCkcATRABElZXADYRCg1GAF9pbmcQAGoADC5lZFgARx0KRGkA8gxEA-ESPBIiEgtpbnRlbGxpZ18j8xDJAAcTAAI9AAglAAU-AOHTARJLEgh0cmFuc2ZlcgUBWCYg8QC2AfAqDwABAQElDAovACAQAf0ABB4AAA0BKxJvTQA0EAoMIAAScvoAD18ABwH2AApfADQPCgs_AAI-AAcRIQHAABYBBZAABdYA1KMByjDyAXNlcnZpY2XVAAMPAACnAERuCoICkwADDgAB4t0jYEoQARJFSK4S8AIAowEFOQACnQAGHQAAOQEIRzQAIQRVXwMCHwQByB6fkNmj0NC6popBvxMCIPIHDQdCogESBh8EA3gGQQgOAIIzKhAO-Sgik9BzBhQKSjEgV7BJA3E6ABmLOgA4OQ9gKdCwDgBzVQ1AEAoMZl4DoLzQuD4AEL5cAEOrBxF5lwAHEQfY4__2qtGk64MoClIqRZAL8AQzAAH7DBJFEigSDtCy0LjQvdC0fwAh0YGpBwDxAw_DKgWvrNbfha7H8saFAacIAoj6DxLCAhL6AVgAQdGD0LfzAAFqKxKbYwfxBCoSCi4AHxAsAAERviwAD0QAAIy3CPAHFAoQXAAPFgABL9GDFgACH4sWAAGM0PQAQBgKFFh4C2HQuIoAAn8tAVASKPwAAl4bAVgVHPn_APkA8omD97a-ic6qAQqCAir1tgXW4wAB_xIS5AQSxgQSEgQBAdQKAEUQEQCwACQaADcXMjIAjxgHMBAAMp0AQIUYAAIZBwAHAECL0LUYJCwgtRj7CVDQvDAAAZ4No48YAAEh0LAYAHsSD_MWxgACZAA7FAoQGAAA2gAOFgAfsGAAAhG-2AA6EgoORAAuEAE2AbIHckYAAw9oAQRVHHEyAAEATAAPUADyC744AQ6yARO-sgEN2AAv0YsgAQgfuWQAAgEYBgEc9QYB2ODPj6GB5K3DQApMKkAFArIuAAH2BxI2Eh4SCTcrE3OvKjANChHiBQDGAxEQTgAC6gL4AwAAAAAw-tq7zK_BqvY-CnIqZk4AEVSGKx-BhisaASkWQRIoEhsBAjEIOAB9BxwrdAC_0omXtNqDmvtTCkcXLwZB0YXRgL0CDxcvAgO9AOmOm_CVjYH8lQMKrwEqohACEZD4DqJwEkESBnh0cmVtIxVgCwoHDgBjAgUwCgob9w4ALgSQGQA2JxAADgBpUQ2AEhhDAAc2AEMtAUVDAPcS8wT3J3QSWhILcGVyZm9ybWFuY2UYACACKhEKDRMAAFUACxMAAXYANxAKDCYAAKIAKA8KSwACcQAIEvsAAVkEn4-3-KiRx_W-bxIwCE_RhdC_-wAO8BHs9vjQu-y-q9sBEpIPCAsQABoQCgYqBBCszRkSAggAGgQA0A8KBSoDEOYKEgIIARoEAAIRAHCQBxICCAIaBAACEQBwhgISAggDGgQAAhEAcKweEgIIBBoEAAIRAHDEYBICCAUaBAAHEQAgBhoEAAJ4AIDorgMSAggHGgQAAiMAEJoRACAIGgQAAhEAcOosEgIICRoEAAI0AICuzQESAggKGgQAgB0KEwgBEAEhlQLhACCMQCoEEKKfIxICCAvLADoeChQfAFAFEPyhzz8AEQwgAAI_AJAQIXyXoXOSSYo_AHHa-RQSAggNHwACPwCQCCFY7o9JfJaNPwAwtsirPwARDiAAMR8KFT8ABX4AYAYQuJjikMIAEQ8hAAJgAJABIV24LWmnAY9gACDO2EAAERBZAQQfAHDuyi08B_aOHwAgsp8fABkRHwBhB0ZK1hsOPgB5lqYEEgIIEh8AYc2MfSSf8z4AEcBdABkTHwBnMTCaOAjUHwAZFB8AcBUHTKcN9I18AHmA3g0SAggVHwBhZmEqRwHLPgAg8uObABkWHwBhMKIgeX_1PgB55NczEgIIFx8AYawhD5xn9D4Ad9DNEhICCBgfAIEoIWSP8_0lhh8AcZDnGBICCBkfABEgtgHzDBAhbBwBtIyUQsAqBRC284YREgIIGhoECAEQAjkBkCEhxLnuH70Yi70AcYrGNxICCBugAgIfAIEBIZNP1el_OjkBBncBBh8AYTE2JtPjLh8AIMTB3AAZHD4AYXDHgNvQJJ4AcdacbBICCB0fAAIVAvABASHyO1hVremOQCoFEMzz6DoBGR4gAHASbDrXX7yLIACBjsaYZBICCB8gAAI1ApABIbYJAsO4x441AkDQ5ZaLTQMRICEABb4AUSGu6wMgXAEgvIo9ARkhHwBhMMdbmuHv8wIgnO2tAxEiHwAEnwBhrl2R_tiInwAwtv25NgIZIyAAYcP9JXiQ2CAAMKSro0AAESQgAAR-AGGLK-daiRBfAHnm1k4SAgglHwBh7EZcgN4SPAEgmMdeABEm-AMEPgBhRGexjSY2HwAg-vAQAxEnHwAxHAoSkAPxAtC9FxyK4I1AKgMQ3jwSAggoHgAEPQBh9t5A8_kRPQAgvvuYAREpHwARHhcCkAEhlLRkGAhPjFkBMMCe6iAAGSogAGFpZIFpINZ5ATDix-3YBBErIAAEXwBhtIzr7boa2AFxsvNNEgIILB8ABF8AYYY1tLkswBkBMOKug18AES0gAAQ_AGHSNJus7fwZAXnqgikSAgguHwBh8NvF-JqEtgEgoIu0AhEvHwACXgCBCCGBc64oXDJLBIHmufQGEgIIMCAAAl4AgQghOBnAMVET_AAG7AMTBfECgQEh7slu6-q8vABxkMUyEgIIMTsFAj4AgQEhMPynFpOSHwB3gJBEEgIIMh8AgQQh8KgtOBekuwB3holWEgIIMx8AgQghAReekbwQfAB54IAIEgIINB8AYdq5_0ule10Acb7sJhICCDWVBQTaAGFGheYVTpfaAImsttkQEgIINiAAYeB_xxzyESAAMNrOv-8CETcgABEfwwWRECEQW242YSd7zAPFuUcSAgg4GgQICRAKNQJwB8SqrXP5jjUCARQGGTkCBmGCqTQor5WeAHfy_SESAgg6HwCBBCHoEquIE8OXAXfE2kYSAgg7HwCBCCEMRqorWdI-ACCCk30AEDwfAC0iAAIAIQkJ7QXxCADwPy0AAIA_MgYSBBCc-QcyBRIDEKgIBwAhzAYHACGGAgcAIaARBwAhkFIHACHEYDIAIriRHQAhmgMWAJCULDIFEgMQhiQAAAA%2C;wmaxone%3D1.000;wminone%3D1.000;ydngd_mean%3D0.261882;ydngi%3D0.5063;ynhg_mean%3D0.1904;zsset%3DeNptV0ty20YQvUrvsrJLkiuu5DggMCRh4ucBYIlekZJtKbFd3Lkqt6AkgoIpkroCULlQ3usBTMpOSWj09Pz687ob9CIj7by9bJbtJf4W8urk5emZjEqTFxKbUuIyKiQzNk8Tae6autk1m3YuzbZZNpumbhfNWk5fc8/p69fy%2Bx8nJ%2BL5RYjVXhCIF1rxkrQYG7xtWiYQ5fjP8R9iyblnAxl4/gQkikCKAgoNjFeADFMLPko5mw7E9xI%2BvxWgeNI0AimTwtgXeWHDiRHfeu%2BnEnh2AlKUsQTGC0iKMWicShAaCdIRnoQPjgrSc7DW5Dlo%2BA7TZTEVMxwavxATi7E2tWJy38uMmHcww1zI0PMNiMWTj2nqMByNC1DoCwLPDSPMCAzgwjKRkRcbGYVDPDaSURoFBsI0DWRkvYSUJo%2B9aChj6gx/pTIOCwljb2QkzCOuCotc3mQjeVPGmUxCOGwSJqMgjSXyBiaSCAqChgPr2alEKQ6K0mQksWdDDxRGxvB6nAbG8gVzQeCTdDgU6CRpEk0lzfBvrKdBTEtE3yv8sWS%2BZKFflLAxi7ypZDg4S8ME8ymCOUDcMpv69GSWv5K3iiBLT8G2COdak5WDKPTFhjmE1M6m0N3CIgsf5R4Pyz1olRvfGvBjD/GRPIwznJ5nxgSgqcVMYQ0XlFBU8mk8CAEOoGxYkE5B4BhSwKtICy8CLWFEAb%2BUmSCOUwH28NA0UBxzPg5xy3kY4klUECYXIMWYTrg4EyD%2BjnmCZ/ZsgERYUvCIfJgr62TNStn2Uppbld02tWPnmK6w%2BbbZN49OdNPskH5gq2at2fjFDXZ81Vh7x5NveUnzgCys2k%2Bi%2B2/dS7OSzJbHMVcrXs0LeBUFYHFNhfld%2B7n5DhEUqTiP83a4dEHRZ71v5nRAbYDwXk/ZuyP0CrWY4l3H4q4r6LXXGrGRZqWLKzIV2fbaLeRwy31k15DP%2BOaAx7qtvNytxlYM3KErVbPmmRWurpsnZ1eFGSx9wJ57%2Btzt5JD3bES12jrf11SU0x3j3OwGupa%2B1SrXLvq5rV40Y60Tnt1%2B4AEbdfCcCmzgkyUN2mhwNmRgIHxLh3/RIfZvGcXDUOPCCGho9xBDWOGuPUW6joLuim6Dns4I3D9j4dxKMVUzqlut0rWo/eobx3RBx4CwaD/xCKJv234GLNRrW5h9LYqFGUr7yrFzVXXlfK8Ch6v2S3/mToNcuU4iehsFTInNM4GzbN88AS0Oy/QVQXeYVEgo8J0rFADSQQhmPenKazK1Tt479hNMWpLdPw8vBUy3mbJYBMjj%2Bu9u%2BBUR19A%2BOf9r4F1Q1F6Ka0Jdbb7rsuVoascTsUMzX0U/wtEJ9kRjx0K%2B7dlL7L7TwzYqghfWDoZ0zwPCMlP926suPpoYD0w1CJx1auZcAbLs7lQ7iArnvLrLoEM%2Bq2BNJ3eGVopWB59fJvdyQCveXKE1Qq%2BaE/kadLLQeof0vVQNNk70USufzrssXnbugwDJhS2PdDnDThEi034UzZ1Nn/rtX245wzXrvabAg4p0yfrYuF5U/biHCnUvLdnOO4teVHUoPQwB7NvngsWva1yBgODzEdjoCtZE0WtYVREGceYhkTsV5pRd0RdMM7fxStdeK8r6BgBfXGuf6Kz9oIVo%2B4NRUOtgwVfdw%2BSD1vwb4W6tX7VWXIJGs64fcLe%2BFIJ0M5QgACcDKQI5OX15cionZ6RnZ6/k7E9%2B5/lGu2OArx2/wMcRvj/QufktVMhoIOGFTN5LPM3fRvgyyMoIbdaizeclzvQzKZOwQJs1gzhN0Km1j95ru2In09cLhdV317Rql0eO0aLPSqStSq28Zop0Vd01kco1F1c8rrBjd6iplWujD3rtXgvwUemheO9eTxqoG8Va3ybY1RGSS%2BaKtgKubD%2B6qucawE%2BiQ1F2vYGJdimaS7dqSMXKxuGG9ZhWPbbfiJ0OUVfOsO1RFi9cPX%2Bg14jwtVItwlxw33WXVS/QOXwm0BjMdBWbgr%2BBvbnW7q6g7bu0n7uavXb5evgOcI1QG%2B7BLf8j7Fer27vKq1l51/dAN3xgV9J62WnlqqVGk78wDoVyr33isRd0FRfRXR61IYXNSvSj5t4BgTnXta%2Bjoq5W37sq476Tbg7l4qoP3k/lHpXM2VO7zdddI9zpd077VdnnrRhlY6eAa79BlRsXiidNwJVEU4uP4UnIX1exF4XJRJr1v/8wHtKdsDoCZ/uNv2HM5D%2Bak0fD;relevgeo%3D225;geochanged%3D1;;udbts%3D1459797057;downerprotobundle%3DCAwVQbwCVw%2C%2C&reqid=1461253060957657-600715567702129748010289-myt1-1909&snip=exps%3Dmarker_ratings_info%2Cmarker_movie_rating%3D0;report%3Dwww;reps%3Dall_on;snip_width%3D550;uil%3Dru;dqsigml%3D4&snip=t%3D0&text=%28autocad%3A%3A209750+^+autodesk%3A%3A288721+^+%28%28autodesk%3A%3A288721+%26/%281+1%29+inventor%3A%3A1697918%29%29+^+%28%28auto%3A%3A28861+%26/%281+1%29+cad%3A%3A171629%29%29+^+%D0%B0%D0%B2%D1%82%D0%BE%D0%BA%D0%B0%D0%B4%3A%3A1405467+^+autoca%3A%3A420234780%29+%26%26/%28-3+5%29+%28%28%D1%81%D0%BA%D0%B0%D1%87%D0%B0%D1%82%D1%8C%3A%3A691+^+download%3A%3A13863+^+pdf%3A%3A10201+^+torrent%3A%3A35211+^+%28%28%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D1%82%D0%BD%D0%BE%3A%3A456+%26/%281+1%29+%D0%B7%D0%B0%D0%B3%D1%80%D1%83%D0%B7%D0%B8%D1%82%D1%8C%3A%3A13856%29%29+^+%D0%B7%D0%B0%D0%B3%D1%80%D1%83%D0%B6%D0%B0%D1%82%D1%8C%3A%3A13856+^+%D0%B7%D0%B0%D0%BA%D0%B0%D1%87%D0%B8%D0%B2%D0%B0%D1%82%D1%8C%3A%3A112512+^+%D1%81%D0%BA%D0%B0%D1%87%D0%B8%D0%B2%D0%B0%D0%BD%D0%B8%D0%B5%3A%3A14585+^+%D1%81%D0%BA%D0%B0%D1%87%D0%B8%D0%B2%D0%B0%D1%82%D1%8C%D1%81%D1%8F%3A%3A423410+^+%D1%81%D0%BA%D0%B0%D1%87%D0%BA%D0%B0%3A%3A152424+^+skachat%3A%3A203208%29+%26/%28-64+64%29+%28%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D1%82%D0%BD%D0%BE%3A%3A456+^+besplatno%3A%3A455045+^+download%3A%3A13863+^+free%3A%3A12386+^+%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%3A%3A886571+^+%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D1%82%D0%BD%D0%B8%D0%B9%3A%3A14490854+^+%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D1%82%D0%BD%D0%B8%D0%BA%3A%3A105058695+^+%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D1%82%D0%BD%D0%BD%D1%8B%D0%B9%3A%3A280156520+^+%D0%B4%D0%B0%D1%80%D0%BE%D0%BC%3A%3A148126+^+%D1%81%D0%B2%D0%BE%D0%B1%D0%BE%D0%B4%D0%BD%D0%BE%3A%3A88910+^+%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D0%BD%D0%BE%3A%3A4669275+^+%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D1%82%D0%BE%3A%3A10775250+^+%D1%85%D0%B0%D0%BB%D1%8F%D0%B2%D0%BD%D1%8B%D0%B9%3A%3A644531%29%29+^+%D1%81%D0%BA%D0%B0%D1%87%D0%B0%D1%82%D1%8C%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D1%82%D0%BD%D0%BE%3A%3A17882331+%26/%28-64+64%29+%D0%BD%D0%B0%3A%3A131+%26/%28-64+64%29+%28%D1%80%D1%83%D1%81%D1%81%D0%BA%D0%BE%D0%BC%3A%3A1942+^+russia%3A%3A37324+^+russian%3A%3A31805+^+%D1%80%D0%BE%D1%81%D1%81%D0%B8%D0%B9%D1%81%D0%BA%D0%B8%D0%B9%3A%3A3887+^+rus%3A%3A16095+^+%D0%B2%D0%B5%D0%BB%D0%B8%D0%BA%D0%BE%D1%80%D1%83%D1%81%D1%81%D0%BA%D0%B8%D0%B9%3A%3A1918880+^+%D1%80%D1%83%D1%81%D0%B8%D1%81%D1%82%D0%B8%D0%BA%D0%B0%3A%3A27111921+^+%D1%80%D1%83%D1%81%D0%B8%D1%84%D0%B8%D0%BA%D0%B0%D1%86%D0%B8%D1%8F%3A%3A638169+^+!!%D1%80%D1%83%D1%81%D0%BA%3A%3A1076145+^+%D1%80%D1%83%D1%81%D1%81%3A%3A336053+^+%D1%80%D1%83%D1%81%D1%81%D0%BA%D0%BE%D1%8F%D0%B7%D1%8B%D1%87%D0%BD%D1%8B%D0%B9%3A%3A139984+^+russkom%3A%3A7245427%29+%26%26/%28-3+5%29+%28%D1%82%D0%BE%D1%80%D1%80%D0%B5%D0%BD%D1%82%3A%3A6178+^+torrent%3A%3A35211%29+%26%26/%28-3+5%29+32%3A%3A6178+%26/%28-1+1%29+%28bit%3A%3A27572+^+%D0%B1%D0%B8%D1%82%D0%BD%D1%8B%D0%B9%3A%3A414024+^+%D1%83%D0%BA%D1%83%D1%81%D0%B8%D1%82%D1%8C%3A%3A558080+^+%28%28background%3A%3A96185+%26/%281+1%29+intelligent%3A%3A705091+%26/%281+1%29+transfer%3A%3A180746+%26/%281+1%29+service%3A%3A17789%29%29+^+%D0%B1%D0%B8%D1%82%3A%3A65584%29+%26%26/%28-3+5%29+%D0%B4%D0%BB%D1%8F%3A%3A205+%26/%28-64+64%29+%28%28windows%3A%3A2869+^+%D0%B2%D0%B8%D0%BD%D0%B4%D0%BE%D0%B2%D1%81%3A%3A318239+^+%D0%B2%D0%B8%D0%BD%D0%B4%D0%BE%D1%83%D0%B7%3A%3A17509782+^+%D0%B2%D0%B8%D0%BD%D0%B4%D0%BE%D1%83%D1%81%3A%3A2618285%29+%26%26/%28-3+5%29+%28xp%3A%3A13143+^+windows%3A%3A2869+^+%D1%85%D1%80%3A%3A278393+^+%28%28xtreme%3A%3A579234+%26/%281+1%29+performance%3A%3A113530%29%29+^+%D1%85%D0%BF%3A%3A582849%29%29+^+windowsxp%3A%3A585285+softness%3A6&tld=ru&uil=ru&user_request=autocad+%D1%81%D0%BA%D0%B0%D1%87%D0%B0%D1%82%D1%8C+%D0%B1%D0%B5%D1%81%D0%BF%D0%BB%D0%B0%D1%82%D0%BD%D0%BE+%D0%BD%D0%B0+%D1%80%D1%83%D1%81%D1%81%D0%BA%D0%BE%D0%BC+%D1%82%D0%BE%D1%80%D1%80%D0%B5%D0%BD%D1%82+32+bit+%D0%B4%D0%BB%D1%8F+windows+xp \ No newline at end of file
diff --git a/library/cpp/streams/lz/ut/ya.make b/library/cpp/streams/lz/ut/ya.make
new file mode 100644
index 0000000000..18288c8ac9
--- /dev/null
+++ b/library/cpp/streams/lz/ut/ya.make
@@ -0,0 +1,18 @@
+UNITTEST_FOR(library/cpp/streams/lz)
+
+OWNER(
+ pg
+ g:util
+)
+
+RESOURCE(
+ random.data /random.data
+ request.data /request.data
+ yq_609.data /yq_609.data
+)
+
+SRCS(
+ lz_ut.cpp
+)
+
+END()
diff --git a/library/cpp/streams/lz/ut/yq_609.data b/library/cpp/streams/lz/ut/yq_609.data
new file mode 100644
index 0000000000..9694b0c97f
--- /dev/null
+++ b/library/cpp/streams/lz/ut/yq_609.data
Binary files differ
diff --git a/library/cpp/streams/lz/ya.make b/library/cpp/streams/lz/ya.make
new file mode 100644
index 0000000000..e5eea0b096
--- /dev/null
+++ b/library/cpp/streams/lz/ya.make
@@ -0,0 +1,20 @@
+LIBRARY()
+
+OWNER(
+ pg
+ g:util
+)
+
+PEERDIR(
+ contrib/libs/fastlz
+ contrib/libs/lz4
+ contrib/libs/minilzo
+ contrib/libs/quicklz
+ contrib/libs/snappy
+)
+
+SRCS(
+ lz.cpp
+)
+
+END()
diff --git a/library/cpp/streams/lzma/lzma.cpp b/library/cpp/streams/lzma/lzma.cpp
new file mode 100644
index 0000000000..f1942fa546
--- /dev/null
+++ b/library/cpp/streams/lzma/lzma.cpp
@@ -0,0 +1,520 @@
+#include "lzma.h"
+
+#include <util/stream/mem.h>
+#include <util/system/context.h>
+#include <util/generic/cast.h>
+#include <util/memory/addstorage.h>
+#include <util/generic/ptr.h>
+#include <util/generic/intrlist.h>
+#include <util/generic/scope.h>
+
+extern "C" {
+#include <contrib/libs/lzmasdk/LzmaEnc.h>
+#include <contrib/libs/lzmasdk/LzmaDec.h>
+}
+
+namespace {
+ class TMemoryGc {
+ class TAllocation: public TIntrusiveListItem<TAllocation>, public TAdditionalStorage<TAllocation> {
+ };
+
+ public:
+ inline void* Allocate(size_t len) {
+ if (len > 1024 * 1024 * 1024) {
+ return nullptr;
+ }
+
+ TAllocation* ret = new (len) TAllocation;
+
+ Allocs_.PushBack(ret);
+
+ return ret->AdditionalData();
+ }
+
+ inline void Deallocate(void* ptr) noexcept {
+ if (ptr) {
+ delete TAllocation::ObjectFromData(ptr);
+ }
+ }
+
+ private:
+ TIntrusiveListWithAutoDelete<TAllocation, TDelete> Allocs_;
+ };
+
+ template <class T>
+ class TInverseFilter {
+ class TTrampoLine: public ITrampoLine {
+ public:
+ inline TTrampoLine(TInverseFilter* parent)
+ : Parent_(parent)
+ {
+ }
+
+ void DoRun() override {
+ Parent_->RunFilter();
+ }
+
+ private:
+ TInverseFilter* Parent_;
+ };
+
+ class TInput: public IInputStream {
+ public:
+ inline TInput(TInverseFilter* parent)
+ : Parent_(parent)
+ {
+ }
+
+ ~TInput() override {
+ }
+
+ size_t DoRead(void* ptr, size_t len) override {
+ return Parent_->ReadImpl(ptr, len);
+ }
+
+ private:
+ TInverseFilter* Parent_;
+ };
+
+ class TOutput: public IOutputStream {
+ public:
+ inline TOutput(TInverseFilter* parent)
+ : Parent_(parent)
+ {
+ }
+
+ ~TOutput() override {
+ }
+
+ void DoWrite(const void* ptr, size_t len) override {
+ Parent_->WriteImpl(ptr, len);
+ }
+
+ private:
+ TInverseFilter* Parent_;
+ };
+
+ public:
+ inline TInverseFilter(IOutputStream* slave, T* filter)
+ : Slave_(slave)
+ , Filter_(filter)
+ , TrampoLine_(this)
+ , FilterCtx_(FilterClosure())
+ , Finished_(false)
+ , In_(nullptr, 0)
+ {
+ }
+
+ virtual ~TInverseFilter() {
+ if (!UncaughtException()) {
+ try {
+ Finish();
+ } catch (...) {
+ }
+ } else {
+ //rely on gc
+ }
+ }
+
+ inline void Write(const void* ptr, size_t len) {
+ In_.Reset(ptr, len);
+
+ Y_DEFER {
+ In_.Reset(0, 0);
+ };
+
+ while (In_.Avail()) {
+ SwitchTo();
+ }
+ }
+
+ inline void Finish() {
+ if (!Finished_) {
+ Finished_ = true;
+ SwitchTo();
+ }
+ }
+
+ private:
+ inline void RunFilter() {
+ try {
+ TInput in(this);
+ TOutput out(this);
+
+ (*Filter_)(&in, &out);
+ } catch (...) {
+ Err_ = std::current_exception();
+ }
+
+ SwitchFrom();
+ }
+
+ inline TContClosure FilterClosure() {
+ return {&TrampoLine_, TArrayRef(Stack_, sizeof(Stack_))};
+ }
+
+ inline size_t ReadImpl(void* ptr, size_t len) {
+ while (!Finished_) {
+ const size_t ret = In_.Read(ptr, len);
+
+ if (ret) {
+ return ret;
+ }
+
+ SwitchFrom();
+ }
+
+ return 0;
+ }
+
+ inline void WriteImpl(const void* ptr, size_t len) {
+ Y_ASSERT(!Out_.Avail());
+
+ Out_.Reset(ptr, len);
+
+ while (Out_.Avail()) {
+ SwitchFrom();
+ }
+ }
+
+ inline bool FlushImpl() {
+ if (Out_.Avail()) {
+ TransferData(&Out_, Slave_);
+ Out_.Reset(nullptr, 0);
+
+ return true;
+ }
+
+ return false;
+ }
+
+ inline void SwitchTo() {
+ do {
+ CurrentCtx_.SwitchTo(&FilterCtx_);
+
+ if (Err_) {
+ Finished_ = true;
+
+ std::rethrow_exception(Err_);
+ }
+ } while (FlushImpl());
+ }
+
+ inline void SwitchFrom() {
+ FilterCtx_.SwitchTo(&CurrentCtx_);
+ }
+
+ private:
+ IOutputStream* Slave_;
+ T* Filter_;
+ TTrampoLine TrampoLine_;
+ char Stack_[16 * 1024];
+ TContMachineContext FilterCtx_;
+ TContMachineContext CurrentCtx_;
+ bool Finished_;
+ TMemoryInput In_;
+ TMemoryInput Out_;
+ std::exception_ptr Err_;
+ };
+
+ class TLzma {
+ public:
+ class TLzmaInput: public ISeqInStream {
+ public:
+ inline TLzmaInput(IInputStream* slave)
+ : Slave_(slave)
+ {
+ Read = ReadFunc;
+ }
+
+ private:
+ static inline SRes ReadFunc(const ISeqInStream* p, void* ptr, size_t* len) {
+ *len = const_cast<TLzmaInput*>(static_cast<const TLzmaInput*>(p))->Slave_->Read(ptr, *len);
+
+ return SZ_OK;
+ }
+
+ private:
+ IInputStream* Slave_;
+ };
+
+ class TLzmaOutput: public ISeqOutStream {
+ public:
+ inline TLzmaOutput(IOutputStream* slave)
+ : Slave_(slave)
+ {
+ Write = WriteFunc;
+ }
+
+ private:
+ static inline size_t WriteFunc(const ISeqOutStream* p, const void* ptr, size_t len) {
+ const_cast<TLzmaOutput*>(static_cast<const TLzmaOutput*>(p))->Slave_->Write(ptr, len);
+
+ return len;
+ }
+
+ private:
+ IOutputStream* Slave_;
+ };
+
+ class TAlloc: public ISzAlloc {
+ public:
+ inline TAlloc() {
+ Alloc = AllocFunc;
+ Free = FreeFunc;
+ }
+
+ private:
+ static void* AllocFunc(const ISzAlloc* t, size_t len) {
+ return static_cast<TAlloc*>(((ISzAlloc*)t))->Gc_.Allocate(len);
+ }
+
+ static void FreeFunc(const ISzAlloc* t, void* p) {
+ static_cast<TAlloc*>(((ISzAlloc*)t))->Gc_.Deallocate(p);
+ }
+
+ private:
+ TMemoryGc Gc_;
+ };
+
+ inline ISzAlloc* Alloc() noexcept {
+ return &Alloc_;
+ }
+
+ static inline void Check(SRes r) {
+ if (r != SZ_OK) {
+ ythrow yexception() << "lzma error(" << r << ")";
+ }
+ }
+
+ private:
+ TAlloc Alloc_;
+ };
+
+ class TLzmaCompressBase: public TLzma {
+ public:
+ inline TLzmaCompressBase(size_t level)
+ : H_(LzmaEnc_Create(Alloc()))
+ {
+ if (!H_) {
+ ythrow yexception() << "can not init lzma engine";
+ }
+
+ LzmaEncProps_Init(&Props_);
+
+ Props_.level = level;
+ Props_.dictSize = 0;
+ Props_.lc = -1;
+ Props_.lp = -1;
+ Props_.pb = -1;
+ Props_.fb = -1;
+ Props_.numThreads = -1;
+ Props_.writeEndMark = 1;
+
+ Check(LzmaEnc_SetProps(H_, &Props_));
+ size_t bufLen = sizeof(PropsBuf_);
+ Zero(PropsBuf_);
+ Check(LzmaEnc_WriteProperties(H_, PropsBuf_, &bufLen));
+ }
+
+ inline ~TLzmaCompressBase() {
+ LzmaEnc_Destroy(H_, Alloc(), Alloc());
+ }
+
+ inline void operator()(IInputStream* in, IOutputStream* out) {
+ TLzmaInput input(in);
+ TLzmaOutput output(out);
+
+ out->Write(PropsBuf_, sizeof(PropsBuf_));
+
+ Check(LzmaEnc_Encode(H_, &output, &input, nullptr, Alloc(), Alloc()));
+ }
+
+ private:
+ CLzmaEncHandle H_;
+ CLzmaEncProps Props_;
+ Byte PropsBuf_[LZMA_PROPS_SIZE];
+ };
+}
+
+class TLzmaCompress::TImpl: public TLzmaCompressBase, public TInverseFilter<TLzmaCompressBase> {
+public:
+ inline TImpl(IOutputStream* slave, size_t level)
+ : TLzmaCompressBase(level)
+ , TInverseFilter<TLzmaCompressBase>(slave, this)
+ {
+ }
+};
+
+class TLzmaDecompress::TImpl: public TLzma {
+public:
+ inline TImpl()
+ : InBegin_(nullptr)
+ , InEnd_(nullptr)
+ {
+ LzmaDec_Construct(&H_);
+ }
+ inline virtual ~TImpl() {
+ LzmaDec_Free(&H_, Alloc());
+ }
+
+ inline size_t Read(void* ptr, size_t len) {
+ Byte* pos = (Byte*)ptr;
+ Byte* end = pos + len;
+
+ retry:
+ size_t availLen = InEnd_ - InBegin_;
+ size_t bufLen = end - pos;
+ ELzmaStatus status;
+
+ Check(LzmaDec_DecodeToBuf(&H_, pos, &bufLen, (Byte*)InBegin_, &availLen, LZMA_FINISH_ANY, &status));
+
+ InBegin_ += availLen;
+ pos += bufLen;
+
+ if (status == LZMA_STATUS_NEEDS_MORE_INPUT) {
+ Y_ASSERT(InEnd_ == InBegin_);
+ if (!Fill()) {
+ ythrow yexception() << "incomplete lzma stream";
+ }
+
+ goto retry;
+ }
+
+ return pos - (Byte*)ptr;
+ }
+
+private:
+ virtual bool Fill() = 0;
+
+protected:
+ CLzmaDec H_;
+ char* InBegin_;
+ char* InEnd_;
+};
+
+class TLzmaDecompress::TImplStream: public TImpl {
+public:
+ inline TImplStream(IInputStream* slave)
+ : Slave_(slave)
+ {
+ Byte buf[LZMA_PROPS_SIZE];
+
+ if (Slave_->Load(buf, sizeof(buf)) != sizeof(buf))
+ ythrow yexception() << "can't read lzma header";
+
+ Check(LzmaDec_Allocate(&H_, buf, sizeof(buf), Alloc()));
+ LzmaDec_Init(&H_);
+ }
+
+private:
+ bool Fill() override {
+ size_t size = Slave_->Read(In_, sizeof(In_));
+ InBegin_ = In_;
+ InEnd_ = In_ + size;
+
+ return size;
+ }
+
+private:
+ IInputStream* Slave_;
+ char In_[4096];
+};
+
+class TLzmaDecompress::TImplZeroCopy: public TLzmaDecompress::TImpl {
+public:
+ inline TImplZeroCopy(IZeroCopyInput* in)
+ : Input_(in)
+ {
+ if (!Fill())
+ ythrow yexception() << "can't read lzma header";
+
+ char buf[LZMA_PROPS_SIZE];
+ char* header;
+ if (InEnd_ - InBegin_ >= LZMA_PROPS_SIZE) {
+ header = InBegin_;
+ InBegin_ += LZMA_PROPS_SIZE;
+ } else {
+ //bad luck, first part is less than header
+ //try to copy header part by part to the local buffer
+ const char* end = buf + sizeof(buf);
+ char* pos = buf;
+ while (1) {
+ size_t left = end - pos;
+ size_t avail = InEnd_ - InBegin_;
+ if (left < avail) {
+ memcpy(pos, InBegin_, left);
+ InBegin_ += left;
+ break;
+ } else {
+ memcpy(pos, InBegin_, avail);
+ pos += avail;
+ if (!Fill()) {
+ ythrow yexception() << "can't read lzma header";
+ }
+ }
+ }
+ header = buf;
+ }
+
+ Check(LzmaDec_Allocate(&H_, (Byte*)header, LZMA_PROPS_SIZE, Alloc()));
+
+ LzmaDec_Init(&H_);
+ }
+
+private:
+ bool Fill() override {
+ size_t size = Input_->Next(&InBegin_);
+
+ if (size) {
+ InEnd_ = InBegin_ + size;
+
+ return true;
+ }
+
+ return false;
+ }
+
+ IZeroCopyInput* Input_;
+};
+
+TLzmaCompress::TLzmaCompress(IOutputStream* slave, size_t level)
+ : Impl_(new TImpl(slave, level))
+{
+}
+
+TLzmaCompress::~TLzmaCompress() {
+}
+
+void TLzmaCompress::DoWrite(const void* buf, size_t len) {
+ if (!Impl_) {
+ ythrow yexception() << "can not write to finished lzma stream";
+ }
+
+ Impl_->Write(buf, len);
+}
+
+void TLzmaCompress::DoFinish() {
+ THolder<TImpl> impl(Impl_.Release());
+
+ if (impl) {
+ impl->Finish();
+ }
+}
+
+TLzmaDecompress::TLzmaDecompress(IInputStream* slave)
+ : Impl_(new TImplStream(slave))
+{
+}
+
+TLzmaDecompress::TLzmaDecompress(IZeroCopyInput* input)
+ : Impl_(new TImplZeroCopy(input))
+{
+}
+
+TLzmaDecompress::~TLzmaDecompress() {
+}
+
+size_t TLzmaDecompress::DoRead(void* buf, size_t len) {
+ return Impl_->Read(buf, len);
+}
diff --git a/library/cpp/streams/lzma/lzma.h b/library/cpp/streams/lzma/lzma.h
new file mode 100644
index 0000000000..ca1e06e9ef
--- /dev/null
+++ b/library/cpp/streams/lzma/lzma.h
@@ -0,0 +1,37 @@
+#pragma once
+
+#include <util/stream/input.h>
+#include <util/stream/output.h>
+#include <util/stream/zerocopy.h>
+
+#include <util/generic/ptr.h>
+
+class TLzmaCompress: public IOutputStream {
+public:
+ TLzmaCompress(IOutputStream* slave, size_t level = 7);
+ ~TLzmaCompress() override;
+
+private:
+ void DoWrite(const void* buf, size_t len) override;
+ void DoFinish() override;
+
+private:
+ class TImpl;
+ THolder<TImpl> Impl_;
+};
+
+class TLzmaDecompress: public IInputStream {
+public:
+ TLzmaDecompress(IInputStream* slave);
+ TLzmaDecompress(IZeroCopyInput* input);
+ ~TLzmaDecompress() override;
+
+private:
+ size_t DoRead(void* buf, size_t len) override;
+
+private:
+ class TImpl;
+ class TImplStream;
+ class TImplZeroCopy;
+ THolder<TImpl> Impl_;
+};
diff --git a/library/cpp/streams/lzma/lzma_ut.cpp b/library/cpp/streams/lzma/lzma_ut.cpp
new file mode 100644
index 0000000000..847e98d1ca
--- /dev/null
+++ b/library/cpp/streams/lzma/lzma_ut.cpp
@@ -0,0 +1,127 @@
+#include "lzma.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <util/stream/mem.h>
+#include <util/random/fast.h>
+#include <util/random/random.h>
+
+class TStrokaByOneByte: public IZeroCopyInput {
+public:
+ TStrokaByOneByte(const TString& s)
+ : Data(s)
+ , Pos(s.data())
+ {
+ }
+
+private:
+ size_t DoNext(const void** ptr, size_t len) override {
+ if (Pos < Data.end()) {
+ len = Min(len, static_cast<size_t>(1));
+ *ptr = Pos;
+ Pos += len;
+ return len;
+ } else {
+ return 0;
+ }
+ }
+
+ TString Data;
+ const char* Pos;
+};
+
+class TLzmaTest: public TTestBase {
+ UNIT_TEST_SUITE(TLzmaTest);
+ UNIT_TEST(Test1)
+ UNIT_TEST(Test2)
+ UNIT_TEST_SUITE_END();
+
+private:
+ inline TString GenData() {
+ TString data;
+ TReallyFastRng32 rnd(RandomNumber<ui64>());
+
+ for (size_t i = 0; i < 50000; ++i) {
+ const char ch = rnd.Uniform(256);
+ const size_t len = 1 + rnd.Uniform(10);
+
+ data += TString(len, ch);
+ }
+
+ return data;
+ }
+
+ inline void Test2() {
+ class TExcOutput: public IOutputStream {
+ public:
+ ~TExcOutput() override {
+ }
+
+ void DoWrite(const void*, size_t) override {
+ throw 12345;
+ }
+ };
+
+ TString data(GenData());
+ TMemoryInput mi(data.data(), data.size());
+ TExcOutput out;
+
+ try {
+ TLzmaCompress c(&out);
+
+ TransferData(&mi, &c);
+ } catch (int i) {
+ UNIT_ASSERT_EQUAL(i, 12345);
+ }
+ }
+
+ inline void Test1() {
+ TString data(GenData());
+ TString data1;
+ TString res;
+
+ {
+ TMemoryInput mi(data.data(), data.size());
+ TStringOutput so(res);
+ TLzmaCompress c(&so);
+
+ TransferData(&mi, &c);
+
+ c.Finish();
+ }
+
+ {
+ TMemoryInput mi(res.data(), res.size());
+ TStringOutput so(data1);
+ TLzmaDecompress d((IInputStream*)&mi);
+
+ TransferData(&d, &so);
+ }
+
+ UNIT_ASSERT_EQUAL(data, data1);
+
+ data1.clear();
+ {
+ TMemoryInput mi(res.data(), res.size());
+ TStringOutput so(data1);
+ TLzmaDecompress d(&mi);
+
+ TransferData(&d, &so);
+ }
+
+ UNIT_ASSERT_EQUAL(data, data1);
+
+ data1.clear();
+ {
+ TStrokaByOneByte mi(res);
+ TStringOutput so(data1);
+ TLzmaDecompress d(&mi);
+
+ TransferData(&d, &so);
+ }
+
+ UNIT_ASSERT_EQUAL(data, data1);
+ }
+};
+
+UNIT_TEST_SUITE_REGISTRATION(TLzmaTest);
diff --git a/library/cpp/streams/lzma/ut/ya.make b/library/cpp/streams/lzma/ut/ya.make
new file mode 100644
index 0000000000..01624f0259
--- /dev/null
+++ b/library/cpp/streams/lzma/ut/ya.make
@@ -0,0 +1,12 @@
+UNITTEST_FOR(library/cpp/streams/lzma)
+
+OWNER(
+ pg
+ g:util
+)
+
+SRCS(
+ lzma_ut.cpp
+)
+
+END()
diff --git a/library/cpp/streams/lzma/ya.make b/library/cpp/streams/lzma/ya.make
new file mode 100644
index 0000000000..38c05145c4
--- /dev/null
+++ b/library/cpp/streams/lzma/ya.make
@@ -0,0 +1,16 @@
+LIBRARY()
+
+OWNER(
+ pg
+ g:util
+)
+
+PEERDIR(
+ contrib/libs/lzmasdk
+)
+
+SRCS(
+ lzma.cpp
+)
+
+END()
diff --git a/library/cpp/streams/ya.make b/library/cpp/streams/ya.make
new file mode 100644
index 0000000000..7426a874ee
--- /dev/null
+++ b/library/cpp/streams/ya.make
@@ -0,0 +1,26 @@
+RECURSE(
+ brotli
+ brotli/ut
+ base64
+ brotli
+ bzip2
+ bzip2/ut
+ factory
+ factory/ut
+ fields_io
+ fields_io/ut
+ growing_file_input
+ growing_file_input/ut
+ lz
+ lz/ut
+ lzma
+ lzma/ut
+ lzop
+ lzop/ut
+ special
+ special/ut
+ xz
+ zc_memory_input
+ zstd
+ zstd/ut
+)
diff --git a/library/cpp/streams/zc_memory_input/ya.make b/library/cpp/streams/zc_memory_input/ya.make
new file mode 100644
index 0000000000..bc94d6f1ed
--- /dev/null
+++ b/library/cpp/streams/zc_memory_input/ya.make
@@ -0,0 +1,12 @@
+LIBRARY()
+
+OWNER(
+ pg
+ g:util
+)
+
+SRCS(
+ zc_memory_input.cpp
+)
+
+END()
diff --git a/library/cpp/streams/zc_memory_input/zc_memory_input.cpp b/library/cpp/streams/zc_memory_input/zc_memory_input.cpp
new file mode 100644
index 0000000000..682099a239
--- /dev/null
+++ b/library/cpp/streams/zc_memory_input/zc_memory_input.cpp
@@ -0,0 +1 @@
+#include "zc_memory_input.h"
diff --git a/library/cpp/streams/zc_memory_input/zc_memory_input.h b/library/cpp/streams/zc_memory_input/zc_memory_input.h
new file mode 100644
index 0000000000..c939d8e426
--- /dev/null
+++ b/library/cpp/streams/zc_memory_input/zc_memory_input.h
@@ -0,0 +1,48 @@
+#pragma once
+
+#include <util/stream/mem.h>
+#include <util/system/defaults.h>
+#include <util/generic/yexception.h>
+
+/// Zero-copy memory input with fixed read
+class TZCMemoryInput: public TMemoryInput {
+public:
+ TZCMemoryInput() {
+ }
+
+ TZCMemoryInput(const char* dataPtr, size_t size)
+ : TMemoryInput(dataPtr, size)
+ {
+ }
+
+ TZCMemoryInput(TMemoryInput& rhs)
+ : TMemoryInput(rhs.Buf(), rhs.Avail())
+ {
+ }
+
+ /// if there's 'size' data read it, otherwise just return false
+ Y_FORCE_INLINE bool ReadFixed(const char*& buf, size_t size) {
+ if (Avail() >= size) {
+ buf = Buf();
+ Reset(Buf() + size, Avail() - size);
+ return true;
+ }
+ return false;
+ }
+
+ template <class T>
+ Y_FORCE_INLINE T LoadPOD() {
+ const char* buf = nullptr;
+ if (!ReadFixed(buf, sizeof(T)))
+ ythrow yexception() << "TZCMemoryInput::LoadPOD failed: not enough data ("
+ << Avail() << " of " << sizeof(T) << " bytes)";
+ T res;
+ memcpy(&res, buf, sizeof(T));
+ return res;
+ }
+
+ template <class T>
+ Y_FORCE_INLINE void ReadPOD(T& x) {
+ x = LoadPOD<T>();
+ }
+};
diff --git a/library/cpp/streams/zstd/ut/ya.make b/library/cpp/streams/zstd/ut/ya.make
new file mode 100644
index 0000000000..1b98f0ad5e
--- /dev/null
+++ b/library/cpp/streams/zstd/ut/ya.make
@@ -0,0 +1,12 @@
+UNITTEST_FOR(library/cpp/streams/zstd)
+
+OWNER(
+ bulatman
+ g:util
+)
+
+SRCS(
+ zstd_ut.cpp
+)
+
+END()
diff --git a/library/cpp/streams/zstd/ya.make b/library/cpp/streams/zstd/ya.make
new file mode 100644
index 0000000000..c284deeeff
--- /dev/null
+++ b/library/cpp/streams/zstd/ya.make
@@ -0,0 +1,16 @@
+LIBRARY()
+
+OWNER(
+ bulatman
+ g:util
+)
+
+PEERDIR(
+ contrib/libs/zstd
+)
+
+SRCS(
+ zstd.cpp
+)
+
+END()
diff --git a/library/cpp/streams/zstd/zstd.cpp b/library/cpp/streams/zstd/zstd.cpp
new file mode 100644
index 0000000000..29816f6d4c
--- /dev/null
+++ b/library/cpp/streams/zstd/zstd.cpp
@@ -0,0 +1,173 @@
+#include "zstd.h"
+
+#include <util/generic/buffer.h>
+#include <util/generic/yexception.h>
+
+#define ZSTD_STATIC_LINKING_ONLY
+#include <contrib/libs/zstd/include/zstd.h>
+
+namespace {
+ inline void CheckError(const char* op, size_t code) {
+ if (::ZSTD_isError(code)) {
+ ythrow yexception() << op << TStringBuf(" zstd error: ") << ::ZSTD_getErrorName(code);
+ }
+ }
+
+ struct DestroyZCStream {
+ static void Destroy(::ZSTD_CStream* p) noexcept {
+ ::ZSTD_freeCStream(p);
+ }
+ };
+
+ struct DestroyZDStream {
+ static void Destroy(::ZSTD_DStream* p) noexcept {
+ ::ZSTD_freeDStream(p);
+ }
+ };
+}
+
+class TZstdCompress::TImpl {
+public:
+ TImpl(IOutputStream* slave, int quality)
+ : Slave_(slave)
+ , ZCtx_(::ZSTD_createCStream())
+ , Buffer_(::ZSTD_CStreamOutSize()) // do reserve
+ {
+ Y_ENSURE(nullptr != ZCtx_.Get(), "Failed to allocate ZSTD_CStream");
+ Y_ENSURE(0 != Buffer_.Capacity(), "ZSTD_CStreamOutSize was too small");
+ CheckError("init", ZSTD_initCStream(ZCtx_.Get(), quality));
+ }
+
+ void Write(const void* buffer, size_t size) {
+ ::ZSTD_inBuffer zIn{buffer, size, 0};
+ auto zOut = OutBuffer();
+
+ while (0 != zIn.size) {
+ CheckError("compress", ::ZSTD_compressStream(ZCtx_.Get(), &zOut, &zIn));
+ DoWrite(zOut);
+ // forget about the data we already compressed
+ zIn.src = static_cast<const unsigned char*>(zIn.src) + zIn.pos;
+ zIn.size -= zIn.pos;
+ zIn.pos = 0;
+ }
+ }
+
+ void Flush() {
+ auto zOut = OutBuffer();
+ CheckError("flush", ::ZSTD_flushStream(ZCtx_.Get(), &zOut));
+ DoWrite(zOut);
+ }
+
+ void Finish() {
+ auto zOut = OutBuffer();
+ size_t returnCode;
+ do {
+ returnCode = ::ZSTD_endStream(ZCtx_.Get(), &zOut);
+ CheckError("finish", returnCode);
+ DoWrite(zOut);
+ } while (0 != returnCode); // zero means there is no more bytes to flush
+ }
+
+private:
+ ::ZSTD_outBuffer OutBuffer() {
+ return {Buffer_.Data(), Buffer_.Capacity(), 0};
+ }
+
+ void DoWrite(::ZSTD_outBuffer& buffer) {
+ Slave_->Write(buffer.dst, buffer.pos);
+ buffer.pos = 0;
+ }
+private:
+ IOutputStream* Slave_;
+ THolder<::ZSTD_CStream, DestroyZCStream> ZCtx_;
+ TBuffer Buffer_;
+};
+
+TZstdCompress::TZstdCompress(IOutputStream* slave, int quality)
+ : Impl_(new TImpl(slave, quality)) {
+}
+
+TZstdCompress::~TZstdCompress() {
+ try {
+ Finish();
+ } catch (...) {
+ }
+}
+
+void TZstdCompress::DoWrite(const void* buffer, size_t size) {
+ Y_ENSURE(Impl_, "Cannot use stream after finish.");
+ Impl_->Write(buffer, size);
+}
+
+void TZstdCompress::DoFlush() {
+ Y_ENSURE(Impl_, "Cannot use stream after finish.");
+ Impl_->Flush();
+}
+
+void TZstdCompress::DoFinish() {
+ // Finish should be idempotent
+ if (Impl_) {
+ auto impl = std::move(Impl_);
+ impl->Finish();
+ }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+class TZstdDecompress::TImpl {
+public:
+ TImpl(IInputStream* slave, size_t bufferSize)
+ : Slave_(slave)
+ , ZCtx_(::ZSTD_createDStream())
+ , Buffer_(bufferSize) // do reserve
+ , Offset_(0)
+ {
+ Y_ENSURE(nullptr != ZCtx_.Get(), "Failed to allocate ZSTD_DStream");
+ Y_ENSURE(0 != Buffer_.Capacity(), "Buffer size was too small");
+ }
+
+ size_t Read(void* buffer, size_t size) {
+ Y_ASSERT(size > 0);
+
+ ::ZSTD_outBuffer zOut{buffer, size, 0};
+ ::ZSTD_inBuffer zIn{Buffer_.Data(), Buffer_.Size(), Offset_};
+
+ size_t returnCode = 0;
+ while (zOut.pos != zOut.size) {
+ if (zIn.pos == zIn.size) {
+ zIn.size = Slave_->Read(Buffer_.Data(), Buffer_.Capacity());
+ Buffer_.Resize(zIn.size);
+ zIn.pos = Offset_ = 0;
+ if (0 == zIn.size) {
+ // end of stream, need to check that there is no uncompleted blocks
+ Y_ENSURE(0 == returnCode, "Incomplete block");
+ break;
+ }
+ }
+ returnCode = ::ZSTD_decompressStream(ZCtx_.Get(), &zOut, &zIn);
+ CheckError("decompress", returnCode);
+ if (0 == returnCode) {
+ // The frame is over, prepare to (maybe) start a new frame
+ ZSTD_initDStream(ZCtx_.Get());
+ }
+ }
+ Offset_ = zIn.pos;
+ return zOut.pos;
+ }
+
+private:
+ IInputStream* Slave_;
+ THolder<::ZSTD_DStream, DestroyZDStream> ZCtx_;
+ TBuffer Buffer_;
+ size_t Offset_;
+};
+
+TZstdDecompress::TZstdDecompress(IInputStream* slave, size_t bufferSize)
+ : Impl_(new TImpl(slave, bufferSize)) {
+}
+
+TZstdDecompress::~TZstdDecompress() = default;
+
+size_t TZstdDecompress::DoRead(void* buffer, size_t size) {
+ return Impl_->Read(buffer, size);
+}
diff --git a/library/cpp/streams/zstd/zstd.h b/library/cpp/streams/zstd/zstd.h
new file mode 100644
index 0000000000..667a0494b7
--- /dev/null
+++ b/library/cpp/streams/zstd/zstd.h
@@ -0,0 +1,53 @@
+#pragma once
+
+#include <util/generic/ptr.h>
+#include <util/stream/input.h>
+#include <util/stream/output.h>
+
+/**
+ * @addtogroup Streams_Archs
+ * @{
+ */
+
+// @brief Stream to compress into zstd archive
+class TZstdCompress: public IOutputStream {
+public:
+ /**
+ @param slave stream to write compressed data to
+ @param quality, higher quality - slower but better compression.
+ 0 is default compression (see constant ZSTD_CLEVEL_DEFAULT(3))
+ max compression is ZSTD_MAX_CLEVEL (22)
+ */
+ explicit TZstdCompress(IOutputStream* slave, int quality = 0);
+ ~TZstdCompress() override;
+private:
+ void DoWrite(const void* buffer, size_t size) override;
+ void DoFlush() override;
+ void DoFinish() override;
+
+public:
+ class TImpl;
+ THolder<TImpl> Impl_;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+// @brief Buffered stream to decompress from zstd archive
+class TZstdDecompress: public IInputStream {
+public:
+ /**
+ @param slave stream to read compressed data from
+ @param bufferSize approximate size of buffer compressed data is read in
+ */
+ explicit TZstdDecompress(IInputStream* slave, size_t bufferSize = 8 * 1024);
+ ~TZstdDecompress() override;
+
+private:
+ size_t DoRead(void* buffer, size_t size) override;
+
+private:
+ class TImpl;
+ THolder<TImpl> Impl_;
+};
+
+/** @} */
diff --git a/library/cpp/streams/zstd/zstd_ut.cpp b/library/cpp/streams/zstd/zstd_ut.cpp
new file mode 100644
index 0000000000..ef479fdd97
--- /dev/null
+++ b/library/cpp/streams/zstd/zstd_ut.cpp
@@ -0,0 +1,94 @@
+#include "zstd.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <util/random/fast.h>
+#include <util/stream/null.h>
+#include <util/stream/str.h>
+
+Y_UNIT_TEST_SUITE(TZstdTestSuite) {
+ TString Compress(TString data, int quality = -1) {
+ TString compressed;
+ TStringOutput output(compressed);
+ TZstdCompress compressStream(&output, quality);
+ compressStream.Write(data.data(), data.size());
+ compressStream.Finish();
+ output.Finish();
+ return compressed;
+ }
+
+ TString Decompress(TString data) {
+ TStringInput input(data);
+ TZstdDecompress decompressStream(&input);
+ return decompressStream.ReadAll();
+ }
+
+ void TestCase(const TString& s) {
+ UNIT_ASSERT_VALUES_EQUAL(s, Decompress(Compress(s, -1)));
+ UNIT_ASSERT_VALUES_EQUAL(s, Decompress(Compress(s, 0)));
+ UNIT_ASSERT_VALUES_EQUAL(s, Decompress(Compress(s, 22)));
+ UNIT_ASSERT_VALUES_EQUAL(s, Decompress(Compress(s, 11)));
+ UNIT_ASSERT_VALUES_EQUAL(s, Decompress(Compress(s, 100500)));
+ }
+
+ TString GenerateRandomString(size_t size) {
+ TReallyFastRng32 rng(42);
+ TString result;
+ result.reserve(size + sizeof(ui64));
+ while (result.size() < size) {
+ ui64 value = rng.GenRand64();
+ result += TStringBuf(reinterpret_cast<const char*>(&value), sizeof(value));
+ }
+ result.resize(size);
+ return result;
+ }
+
+ Y_UNIT_TEST(TestHelloWorld) {
+ TestCase("hello world");
+ }
+
+ Y_UNIT_TEST(TestSeveralStreamsWithSameQuality) {
+ auto s1 = GenerateRandomString(1 << 15);
+ auto s2 = GenerateRandomString(1 << 15);
+ auto c1 = Compress(s1);
+ auto c2 = Compress(s2);
+ UNIT_ASSERT_VALUES_EQUAL(s1 + s2, Decompress(c1 + c2));
+ }
+
+ Y_UNIT_TEST(TestSeveralStreamsWithDifferentQuality) {
+ auto s1 = GenerateRandomString(1 << 15);
+ auto s2 = GenerateRandomString(1 << 15);
+ auto c1 = Compress(s1, 1);
+ auto c2 = Compress(s2, 2);
+ UNIT_ASSERT_VALUES_EQUAL(s1 + s2, Decompress(c1 + c2));
+ }
+
+ Y_UNIT_TEST(TestIncompleteStream) {
+ TString manyAs(64 * 1024, 'a');
+ auto compressed = Compress(manyAs);
+ TString truncated(compressed.data(), compressed.size() - 1);
+ UNIT_CHECK_GENERATED_EXCEPTION(Decompress(truncated), std::exception);
+ }
+
+ Y_UNIT_TEST(Test64KB) {
+ auto manyAs = TString(64 * 1024, 'a');
+ TString str("Hello from the Matrix!@#% How are you?}{\n\t\a");
+ TestCase(manyAs + str + manyAs);
+ }
+
+ Y_UNIT_TEST(Test1MB) {
+ TestCase(GenerateRandomString(1 * 1024 * 1024));
+ }
+
+ Y_UNIT_TEST(TestEmpty) {
+ TestCase("");
+ }
+
+ Y_UNIT_TEST(TestWriteAfterFinish) {
+ TNullOutput output;
+ TZstdCompress compressStream(&output);
+ compressStream.Finish();
+ UNIT_ASSERT_EXCEPTION_CONTAINS(compressStream.Write("a", 1), std::exception, "Cannot use stream after finish.");
+ UNIT_ASSERT_EXCEPTION_CONTAINS(compressStream.Flush(), std::exception, "Cannot use stream after finish.");
+ }
+}