diff options
author | asaitgalin <asaitgalin@yandex-team.ru> | 2022-02-10 16:47:28 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:47:28 +0300 |
commit | cb85b6b6ed9608dfa24ee0362fd286dd27fd40ae (patch) | |
tree | 9814fbd1c3effac9b8377c5d604b367b14e2db55 /library/cpp/streams/brotli | |
parent | c0780d05ad256f75dc8e0fa36aee5dbce402e8f6 (diff) | |
download | ydb-cb85b6b6ed9608dfa24ee0362fd286dd27fd40ae.tar.gz |
Restoring authorship annotation for <asaitgalin@yandex-team.ru>. Commit 2 of 2.
Diffstat (limited to 'library/cpp/streams/brotli')
-rw-r--r-- | library/cpp/streams/brotli/brotli.cpp | 262 | ||||
-rw-r--r-- | library/cpp/streams/brotli/brotli_ut.cpp | 74 |
2 files changed, 168 insertions, 168 deletions
diff --git a/library/cpp/streams/brotli/brotli.cpp b/library/cpp/streams/brotli/brotli.cpp index a7923c2b39..38052cb688 100644 --- a/library/cpp/streams/brotli/brotli.cpp +++ b/library/cpp/streams/brotli/brotli.cpp @@ -1,98 +1,98 @@ #include "brotli.h" -#include <contrib/libs/brotli/include/brotli/decode.h> -#include <contrib/libs/brotli/include/brotli/encode.h> +#include <contrib/libs/brotli/include/brotli/decode.h> +#include <contrib/libs/brotli/include/brotli/encode.h> -#include <util/generic/yexception.h> +#include <util/generic/yexception.h> #include <util/memory/addstorage.h> -namespace { - struct TAllocator { - static void* Allocate(void* /* opaque */, size_t size) { - return ::operator new(size); - } - - static void Deallocate(void* /* opaque */, void* ptr) noexcept { - ::operator delete(ptr); - } - }; - -} - +namespace { + struct TAllocator { + static void* Allocate(void* /* opaque */, size_t size) { + return ::operator new(size); + } + + static void Deallocate(void* /* opaque */, void* ptr) noexcept { + ::operator delete(ptr); + } + }; + +} + class TBrotliCompress::TImpl { public: - TImpl(IOutputStream* slave, int quality) + TImpl(IOutputStream* slave, int quality) : Slave_(slave) - , EncoderState_(BrotliEncoderCreateInstance(&TAllocator::Allocate, &TAllocator::Deallocate, nullptr)) + , EncoderState_(BrotliEncoderCreateInstance(&TAllocator::Allocate, &TAllocator::Deallocate, nullptr)) { - if (!EncoderState_) { - ythrow yexception() << "Brotli encoder initialization failed"; - } - - auto res = BrotliEncoderSetParameter( - EncoderState_, - BROTLI_PARAM_QUALITY, - quality); - - if (!res) { - BrotliEncoderDestroyInstance(EncoderState_); - ythrow yexception() << "Failed to set brotli encoder quality to " << quality; + if (!EncoderState_) { + ythrow yexception() << "Brotli encoder initialization failed"; } + + auto res = BrotliEncoderSetParameter( + EncoderState_, + BROTLI_PARAM_QUALITY, + quality); + + if (!res) { + BrotliEncoderDestroyInstance(EncoderState_); + ythrow yexception() << "Failed to set brotli encoder quality to " << quality; + } + } + + ~TImpl() { + BrotliEncoderDestroyInstance(EncoderState_); + } + + void Write(const void* buffer, size_t size) { + DoWrite(buffer, size, BROTLI_OPERATION_PROCESS); } - ~TImpl() { - BrotliEncoderDestroyInstance(EncoderState_); - } - - void Write(const void* buffer, size_t size) { - DoWrite(buffer, size, BROTLI_OPERATION_PROCESS); - } - void Flush() { - DoWrite(nullptr, 0, BROTLI_OPERATION_FLUSH); + DoWrite(nullptr, 0, BROTLI_OPERATION_FLUSH); + } + + void Finish() { + Flush(); + DoWrite(nullptr, 0, BROTLI_OPERATION_FINISH); + Y_VERIFY(BrotliEncoderIsFinished(EncoderState_)); } - void Finish() { - Flush(); - DoWrite(nullptr, 0, BROTLI_OPERATION_FINISH); - Y_VERIFY(BrotliEncoderIsFinished(EncoderState_)); - } - private: IOutputStream* Slave_; - BrotliEncoderState* EncoderState_; - - void DoWrite(const void* buffer, size_t size, BrotliEncoderOperation operation) { - size_t availableOut = 0; - ui8* outputBuffer = nullptr; - - const ui8* uBuffer = static_cast<const ui8*>(buffer); - - do { - auto result = BrotliEncoderCompressStream( - EncoderState_, - operation, - &size, - &uBuffer, - &availableOut, - &outputBuffer, - nullptr); - - if (result == BROTLI_FALSE) { - ythrow yexception() << "Brotli encoder failed to process buffer"; - } - - size_t outputLength = 0; - const ui8* output = BrotliEncoderTakeOutput(EncoderState_, &outputLength); - if (outputLength > 0) { - Slave_->Write(output, outputLength); - } - } while (size > 0 || BrotliEncoderHasMoreOutput(EncoderState_)); - } + BrotliEncoderState* EncoderState_; + + void DoWrite(const void* buffer, size_t size, BrotliEncoderOperation operation) { + size_t availableOut = 0; + ui8* outputBuffer = nullptr; + + const ui8* uBuffer = static_cast<const ui8*>(buffer); + + do { + auto result = BrotliEncoderCompressStream( + EncoderState_, + operation, + &size, + &uBuffer, + &availableOut, + &outputBuffer, + nullptr); + + if (result == BROTLI_FALSE) { + ythrow yexception() << "Brotli encoder failed to process buffer"; + } + + size_t outputLength = 0; + const ui8* output = BrotliEncoderTakeOutput(EncoderState_, &outputLength); + if (outputLength > 0) { + Slave_->Write(output, outputLength); + } + } while (size > 0 || BrotliEncoderHasMoreOutput(EncoderState_)); + } }; TBrotliCompress::TBrotliCompress(IOutputStream* slave, int quality) { - Impl_.Reset(new TImpl(slave, quality)); + Impl_.Reset(new TImpl(slave, quality)); } TBrotliCompress::~TBrotliCompress() { @@ -116,7 +116,7 @@ void TBrotliCompress::DoFinish() { THolder<TImpl> impl(Impl_.Release()); if (impl) { - impl->Finish(); + impl->Finish(); } } @@ -127,71 +127,71 @@ public: TImpl(IInputStream* slave) : Slave_(slave) { - InitDecoder(); + InitDecoder(); } ~TImpl() { - FreeDecoder(); + FreeDecoder(); } size_t Read(void* buffer, size_t size) { - Y_ASSERT(size > 0); + Y_ASSERT(size > 0); - ui8* outBuffer = static_cast<ui8*>(buffer); + ui8* outBuffer = static_cast<ui8*>(buffer); size_t availableOut = size; size_t decompressedSize = 0; - - BrotliDecoderResult result; + + BrotliDecoderResult result; do { - if (InputAvailable_ == 0 && !InputExhausted_) { + if (InputAvailable_ == 0 && !InputExhausted_) { InputBuffer_ = TmpBuf(); - InputAvailable_ = Slave_->Read((void*)InputBuffer_, TmpBufLen()); - if (InputAvailable_ == 0) { - InputExhausted_ = true; + InputAvailable_ = Slave_->Read((void*)InputBuffer_, TmpBufLen()); + if (InputAvailable_ == 0) { + InputExhausted_ = true; } } - if (SubstreamFinished_ && !InputExhausted_) { - ResetState(); - } - - result = BrotliDecoderDecompressStream( - DecoderState_, - &InputAvailable_, + if (SubstreamFinished_ && !InputExhausted_) { + ResetState(); + } + + result = BrotliDecoderDecompressStream( + DecoderState_, + &InputAvailable_, &InputBuffer_, &availableOut, - &outBuffer, - nullptr); - + &outBuffer, + nullptr); + decompressedSize = size - availableOut; - SubstreamFinished_ = (result == BROTLI_DECODER_RESULT_SUCCESS); - - if (result == BROTLI_DECODER_RESULT_ERROR) { - BrotliDecoderErrorCode code = BrotliDecoderGetErrorCode(DecoderState_); - ythrow yexception() << "Brotli decoder failed to decompress buffer: " - << BrotliDecoderErrorString(code); - } else if (result == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) { - Y_VERIFY(availableOut != size, - "Buffer passed to read in Brotli decoder is too small"); + SubstreamFinished_ = (result == BROTLI_DECODER_RESULT_SUCCESS); + + if (result == BROTLI_DECODER_RESULT_ERROR) { + BrotliDecoderErrorCode code = BrotliDecoderGetErrorCode(DecoderState_); + ythrow yexception() << "Brotli decoder failed to decompress buffer: " + << BrotliDecoderErrorString(code); + } else if (result == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) { + Y_VERIFY(availableOut != size, + "Buffer passed to read in Brotli decoder is too small"); break; } } while (decompressedSize == 0 && result == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT && !InputExhausted_); - if (!SubstreamFinished_ && decompressedSize == 0) { - ythrow yexception() << "Input stream is incomplete"; - } - - return decompressedSize; + if (!SubstreamFinished_ && decompressedSize == 0) { + ythrow yexception() << "Input stream is incomplete"; + } + + return decompressedSize; } private: IInputStream* Slave_; - BrotliDecoderState* DecoderState_; + BrotliDecoderState* DecoderState_; - bool SubstreamFinished_ = false; - bool InputExhausted_ = false; + bool SubstreamFinished_ = false; + bool InputExhausted_ = false; const ui8* InputBuffer_ = nullptr; - size_t InputAvailable_ = 0; + size_t InputAvailable_ = 0; unsigned char* TmpBuf() noexcept { return static_cast<unsigned char*>(AdditionalData()); @@ -200,23 +200,23 @@ private: size_t TmpBufLen() const noexcept { return AdditionalDataLength(); } - - void InitDecoder() { - DecoderState_ = BrotliDecoderCreateInstance(&TAllocator::Allocate, &TAllocator::Deallocate, nullptr); - if (!DecoderState_) { - ythrow yexception() << "Brotli decoder initialization failed"; - } - } - - void FreeDecoder() { - BrotliDecoderDestroyInstance(DecoderState_); - } - - void ResetState() { - Y_VERIFY(BrotliDecoderIsFinished(DecoderState_)); - FreeDecoder(); - InitDecoder(); - } + + void InitDecoder() { + DecoderState_ = BrotliDecoderCreateInstance(&TAllocator::Allocate, &TAllocator::Deallocate, nullptr); + if (!DecoderState_) { + ythrow yexception() << "Brotli decoder initialization failed"; + } + } + + void FreeDecoder() { + BrotliDecoderDestroyInstance(DecoderState_); + } + + void ResetState() { + Y_VERIFY(BrotliDecoderIsFinished(DecoderState_)); + FreeDecoder(); + InitDecoder(); + } }; TBrotliDecompress::TBrotliDecompress(IInputStream* slave, size_t bufferSize) diff --git a/library/cpp/streams/brotli/brotli_ut.cpp b/library/cpp/streams/brotli/brotli_ut.cpp index fff37e3684..aeb2e284dc 100644 --- a/library/cpp/streams/brotli/brotli_ut.cpp +++ b/library/cpp/streams/brotli/brotli_ut.cpp @@ -5,28 +5,28 @@ #include <util/random/fast.h> Y_UNIT_TEST_SUITE(TBrotliTestSuite) { - TString Compress(TString data) { - TString compressed; - TStringOutput output(compressed); - TBrotliCompress compressStream(&output, 11); + TString Compress(TString data) { + TString compressed; + TStringOutput output(compressed); + TBrotliCompress compressStream(&output, 11); compressStream.Write(data.data(), data.size()); - compressStream.Finish(); - output.Finish(); - return compressed; - } - - TString Decompress(TString data) { - TStringInput input(data); - TBrotliDecompress decompressStream(&input); - return decompressStream.ReadAll(); - } - - void TestCase(const TString& s) { - UNIT_ASSERT_VALUES_EQUAL(s, Decompress(Compress(s))); - } - - TString GenerateRandomString(size_t size) { - TReallyFastRng32 rng(42); + compressStream.Finish(); + output.Finish(); + return compressed; + } + + TString Decompress(TString data) { + TStringInput input(data); + TBrotliDecompress decompressStream(&input); + return decompressStream.ReadAll(); + } + + void TestCase(const TString& s) { + UNIT_ASSERT_VALUES_EQUAL(s, Decompress(Compress(s))); + } + + TString GenerateRandomString(size_t size) { + TReallyFastRng32 rng(42); TString result; result.reserve(size + sizeof(ui64)); while (result.size() < size) { @@ -38,7 +38,7 @@ Y_UNIT_TEST_SUITE(TBrotliTestSuite) { } Y_UNIT_TEST(TestHelloWorld) { - TestCase("hello world"); + TestCase("hello world"); } Y_UNIT_TEST(TestFlush) { @@ -59,31 +59,31 @@ Y_UNIT_TEST_SUITE(TBrotliTestSuite) { } Y_UNIT_TEST(TestSeveralStreams) { - auto s1 = GenerateRandomString(1 << 15); - auto s2 = GenerateRandomString(1 << 15); - auto c1 = Compress(s1); - auto c2 = Compress(s2); - UNIT_ASSERT_VALUES_EQUAL(s1 + s2, Decompress(c1 + c2)); + auto s1 = GenerateRandomString(1 << 15); + auto s2 = GenerateRandomString(1 << 15); + auto c1 = Compress(s1); + auto c2 = Compress(s2); + UNIT_ASSERT_VALUES_EQUAL(s1 + s2, Decompress(c1 + c2)); } Y_UNIT_TEST(TestIncompleteStream) { - TString manyAs(64 * 1024, 'a'); - auto compressed = Compress(manyAs); + TString manyAs(64 * 1024, 'a'); + auto compressed = Compress(manyAs); TString truncated(compressed.data(), compressed.size() - 1); - UNIT_CHECK_GENERATED_EXCEPTION(Decompress(truncated), std::exception); + UNIT_CHECK_GENERATED_EXCEPTION(Decompress(truncated), std::exception); } Y_UNIT_TEST(Test64KB) { - auto manyAs = TString(64 * 1024, 'a'); - TString str("Hello from the Matrix!@#% How are you?}{\n\t\a"); - TestCase(manyAs + str + manyAs); + auto manyAs = TString(64 * 1024, 'a'); + TString str("Hello from the Matrix!@#% How are you?}{\n\t\a"); + TestCase(manyAs + str + manyAs); } Y_UNIT_TEST(Test1MB) { - TestCase(GenerateRandomString(1 * 1024 * 1024)); - } - + TestCase(GenerateRandomString(1 * 1024 * 1024)); + } + Y_UNIT_TEST(TestEmpty) { - TestCase(""); + TestCase(""); } } |