aboutsummaryrefslogtreecommitdiffstats
path: root/library/cpp/streams/brotli
diff options
context:
space:
mode:
authorasaitgalin <asaitgalin@yandex-team.ru>2022-02-10 16:47:28 +0300
committerDaniil Cherednik <dcherednik@yandex-team.ru>2022-02-10 16:47:28 +0300
commitcb85b6b6ed9608dfa24ee0362fd286dd27fd40ae (patch)
tree9814fbd1c3effac9b8377c5d604b367b14e2db55 /library/cpp/streams/brotli
parentc0780d05ad256f75dc8e0fa36aee5dbce402e8f6 (diff)
downloadydb-cb85b6b6ed9608dfa24ee0362fd286dd27fd40ae.tar.gz
Restoring authorship annotation for <asaitgalin@yandex-team.ru>. Commit 2 of 2.
Diffstat (limited to 'library/cpp/streams/brotli')
-rw-r--r--library/cpp/streams/brotli/brotli.cpp262
-rw-r--r--library/cpp/streams/brotli/brotli_ut.cpp74
2 files changed, 168 insertions, 168 deletions
diff --git a/library/cpp/streams/brotli/brotli.cpp b/library/cpp/streams/brotli/brotli.cpp
index a7923c2b39..38052cb688 100644
--- a/library/cpp/streams/brotli/brotli.cpp
+++ b/library/cpp/streams/brotli/brotli.cpp
@@ -1,98 +1,98 @@
#include "brotli.h"
-#include <contrib/libs/brotli/include/brotli/decode.h>
-#include <contrib/libs/brotli/include/brotli/encode.h>
+#include <contrib/libs/brotli/include/brotli/decode.h>
+#include <contrib/libs/brotli/include/brotli/encode.h>
-#include <util/generic/yexception.h>
+#include <util/generic/yexception.h>
#include <util/memory/addstorage.h>
-namespace {
- struct TAllocator {
- static void* Allocate(void* /* opaque */, size_t size) {
- return ::operator new(size);
- }
-
- static void Deallocate(void* /* opaque */, void* ptr) noexcept {
- ::operator delete(ptr);
- }
- };
-
-}
-
+namespace {
+ struct TAllocator {
+ static void* Allocate(void* /* opaque */, size_t size) {
+ return ::operator new(size);
+ }
+
+ static void Deallocate(void* /* opaque */, void* ptr) noexcept {
+ ::operator delete(ptr);
+ }
+ };
+
+}
+
class TBrotliCompress::TImpl {
public:
- TImpl(IOutputStream* slave, int quality)
+ TImpl(IOutputStream* slave, int quality)
: Slave_(slave)
- , EncoderState_(BrotliEncoderCreateInstance(&TAllocator::Allocate, &TAllocator::Deallocate, nullptr))
+ , EncoderState_(BrotliEncoderCreateInstance(&TAllocator::Allocate, &TAllocator::Deallocate, nullptr))
{
- if (!EncoderState_) {
- ythrow yexception() << "Brotli encoder initialization failed";
- }
-
- auto res = BrotliEncoderSetParameter(
- EncoderState_,
- BROTLI_PARAM_QUALITY,
- quality);
-
- if (!res) {
- BrotliEncoderDestroyInstance(EncoderState_);
- ythrow yexception() << "Failed to set brotli encoder quality to " << quality;
+ if (!EncoderState_) {
+ ythrow yexception() << "Brotli encoder initialization failed";
}
+
+ auto res = BrotliEncoderSetParameter(
+ EncoderState_,
+ BROTLI_PARAM_QUALITY,
+ quality);
+
+ if (!res) {
+ BrotliEncoderDestroyInstance(EncoderState_);
+ ythrow yexception() << "Failed to set brotli encoder quality to " << quality;
+ }
+ }
+
+ ~TImpl() {
+ BrotliEncoderDestroyInstance(EncoderState_);
+ }
+
+ void Write(const void* buffer, size_t size) {
+ DoWrite(buffer, size, BROTLI_OPERATION_PROCESS);
}
- ~TImpl() {
- BrotliEncoderDestroyInstance(EncoderState_);
- }
-
- void Write(const void* buffer, size_t size) {
- DoWrite(buffer, size, BROTLI_OPERATION_PROCESS);
- }
-
void Flush() {
- DoWrite(nullptr, 0, BROTLI_OPERATION_FLUSH);
+ DoWrite(nullptr, 0, BROTLI_OPERATION_FLUSH);
+ }
+
+ void Finish() {
+ Flush();
+ DoWrite(nullptr, 0, BROTLI_OPERATION_FINISH);
+ Y_VERIFY(BrotliEncoderIsFinished(EncoderState_));
}
- void Finish() {
- Flush();
- DoWrite(nullptr, 0, BROTLI_OPERATION_FINISH);
- Y_VERIFY(BrotliEncoderIsFinished(EncoderState_));
- }
-
private:
IOutputStream* Slave_;
- BrotliEncoderState* EncoderState_;
-
- void DoWrite(const void* buffer, size_t size, BrotliEncoderOperation operation) {
- size_t availableOut = 0;
- ui8* outputBuffer = nullptr;
-
- const ui8* uBuffer = static_cast<const ui8*>(buffer);
-
- do {
- auto result = BrotliEncoderCompressStream(
- EncoderState_,
- operation,
- &size,
- &uBuffer,
- &availableOut,
- &outputBuffer,
- nullptr);
-
- if (result == BROTLI_FALSE) {
- ythrow yexception() << "Brotli encoder failed to process buffer";
- }
-
- size_t outputLength = 0;
- const ui8* output = BrotliEncoderTakeOutput(EncoderState_, &outputLength);
- if (outputLength > 0) {
- Slave_->Write(output, outputLength);
- }
- } while (size > 0 || BrotliEncoderHasMoreOutput(EncoderState_));
- }
+ BrotliEncoderState* EncoderState_;
+
+ void DoWrite(const void* buffer, size_t size, BrotliEncoderOperation operation) {
+ size_t availableOut = 0;
+ ui8* outputBuffer = nullptr;
+
+ const ui8* uBuffer = static_cast<const ui8*>(buffer);
+
+ do {
+ auto result = BrotliEncoderCompressStream(
+ EncoderState_,
+ operation,
+ &size,
+ &uBuffer,
+ &availableOut,
+ &outputBuffer,
+ nullptr);
+
+ if (result == BROTLI_FALSE) {
+ ythrow yexception() << "Brotli encoder failed to process buffer";
+ }
+
+ size_t outputLength = 0;
+ const ui8* output = BrotliEncoderTakeOutput(EncoderState_, &outputLength);
+ if (outputLength > 0) {
+ Slave_->Write(output, outputLength);
+ }
+ } while (size > 0 || BrotliEncoderHasMoreOutput(EncoderState_));
+ }
};
TBrotliCompress::TBrotliCompress(IOutputStream* slave, int quality) {
- Impl_.Reset(new TImpl(slave, quality));
+ Impl_.Reset(new TImpl(slave, quality));
}
TBrotliCompress::~TBrotliCompress() {
@@ -116,7 +116,7 @@ void TBrotliCompress::DoFinish() {
THolder<TImpl> impl(Impl_.Release());
if (impl) {
- impl->Finish();
+ impl->Finish();
}
}
@@ -127,71 +127,71 @@ public:
TImpl(IInputStream* slave)
: Slave_(slave)
{
- InitDecoder();
+ InitDecoder();
}
~TImpl() {
- FreeDecoder();
+ FreeDecoder();
}
size_t Read(void* buffer, size_t size) {
- Y_ASSERT(size > 0);
+ Y_ASSERT(size > 0);
- ui8* outBuffer = static_cast<ui8*>(buffer);
+ ui8* outBuffer = static_cast<ui8*>(buffer);
size_t availableOut = size;
size_t decompressedSize = 0;
-
- BrotliDecoderResult result;
+
+ BrotliDecoderResult result;
do {
- if (InputAvailable_ == 0 && !InputExhausted_) {
+ if (InputAvailable_ == 0 && !InputExhausted_) {
InputBuffer_ = TmpBuf();
- InputAvailable_ = Slave_->Read((void*)InputBuffer_, TmpBufLen());
- if (InputAvailable_ == 0) {
- InputExhausted_ = true;
+ InputAvailable_ = Slave_->Read((void*)InputBuffer_, TmpBufLen());
+ if (InputAvailable_ == 0) {
+ InputExhausted_ = true;
}
}
- if (SubstreamFinished_ && !InputExhausted_) {
- ResetState();
- }
-
- result = BrotliDecoderDecompressStream(
- DecoderState_,
- &InputAvailable_,
+ if (SubstreamFinished_ && !InputExhausted_) {
+ ResetState();
+ }
+
+ result = BrotliDecoderDecompressStream(
+ DecoderState_,
+ &InputAvailable_,
&InputBuffer_,
&availableOut,
- &outBuffer,
- nullptr);
-
+ &outBuffer,
+ nullptr);
+
decompressedSize = size - availableOut;
- SubstreamFinished_ = (result == BROTLI_DECODER_RESULT_SUCCESS);
-
- if (result == BROTLI_DECODER_RESULT_ERROR) {
- BrotliDecoderErrorCode code = BrotliDecoderGetErrorCode(DecoderState_);
- ythrow yexception() << "Brotli decoder failed to decompress buffer: "
- << BrotliDecoderErrorString(code);
- } else if (result == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) {
- Y_VERIFY(availableOut != size,
- "Buffer passed to read in Brotli decoder is too small");
+ SubstreamFinished_ = (result == BROTLI_DECODER_RESULT_SUCCESS);
+
+ if (result == BROTLI_DECODER_RESULT_ERROR) {
+ BrotliDecoderErrorCode code = BrotliDecoderGetErrorCode(DecoderState_);
+ ythrow yexception() << "Brotli decoder failed to decompress buffer: "
+ << BrotliDecoderErrorString(code);
+ } else if (result == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) {
+ Y_VERIFY(availableOut != size,
+ "Buffer passed to read in Brotli decoder is too small");
break;
}
} while (decompressedSize == 0 && result == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT && !InputExhausted_);
- if (!SubstreamFinished_ && decompressedSize == 0) {
- ythrow yexception() << "Input stream is incomplete";
- }
-
- return decompressedSize;
+ if (!SubstreamFinished_ && decompressedSize == 0) {
+ ythrow yexception() << "Input stream is incomplete";
+ }
+
+ return decompressedSize;
}
private:
IInputStream* Slave_;
- BrotliDecoderState* DecoderState_;
+ BrotliDecoderState* DecoderState_;
- bool SubstreamFinished_ = false;
- bool InputExhausted_ = false;
+ bool SubstreamFinished_ = false;
+ bool InputExhausted_ = false;
const ui8* InputBuffer_ = nullptr;
- size_t InputAvailable_ = 0;
+ size_t InputAvailable_ = 0;
unsigned char* TmpBuf() noexcept {
return static_cast<unsigned char*>(AdditionalData());
@@ -200,23 +200,23 @@ private:
size_t TmpBufLen() const noexcept {
return AdditionalDataLength();
}
-
- void InitDecoder() {
- DecoderState_ = BrotliDecoderCreateInstance(&TAllocator::Allocate, &TAllocator::Deallocate, nullptr);
- if (!DecoderState_) {
- ythrow yexception() << "Brotli decoder initialization failed";
- }
- }
-
- void FreeDecoder() {
- BrotliDecoderDestroyInstance(DecoderState_);
- }
-
- void ResetState() {
- Y_VERIFY(BrotliDecoderIsFinished(DecoderState_));
- FreeDecoder();
- InitDecoder();
- }
+
+ void InitDecoder() {
+ DecoderState_ = BrotliDecoderCreateInstance(&TAllocator::Allocate, &TAllocator::Deallocate, nullptr);
+ if (!DecoderState_) {
+ ythrow yexception() << "Brotli decoder initialization failed";
+ }
+ }
+
+ void FreeDecoder() {
+ BrotliDecoderDestroyInstance(DecoderState_);
+ }
+
+ void ResetState() {
+ Y_VERIFY(BrotliDecoderIsFinished(DecoderState_));
+ FreeDecoder();
+ InitDecoder();
+ }
};
TBrotliDecompress::TBrotliDecompress(IInputStream* slave, size_t bufferSize)
diff --git a/library/cpp/streams/brotli/brotli_ut.cpp b/library/cpp/streams/brotli/brotli_ut.cpp
index fff37e3684..aeb2e284dc 100644
--- a/library/cpp/streams/brotli/brotli_ut.cpp
+++ b/library/cpp/streams/brotli/brotli_ut.cpp
@@ -5,28 +5,28 @@
#include <util/random/fast.h>
Y_UNIT_TEST_SUITE(TBrotliTestSuite) {
- TString Compress(TString data) {
- TString compressed;
- TStringOutput output(compressed);
- TBrotliCompress compressStream(&output, 11);
+ TString Compress(TString data) {
+ TString compressed;
+ TStringOutput output(compressed);
+ TBrotliCompress compressStream(&output, 11);
compressStream.Write(data.data(), data.size());
- compressStream.Finish();
- output.Finish();
- return compressed;
- }
-
- TString Decompress(TString data) {
- TStringInput input(data);
- TBrotliDecompress decompressStream(&input);
- return decompressStream.ReadAll();
- }
-
- void TestCase(const TString& s) {
- UNIT_ASSERT_VALUES_EQUAL(s, Decompress(Compress(s)));
- }
-
- TString GenerateRandomString(size_t size) {
- TReallyFastRng32 rng(42);
+ compressStream.Finish();
+ output.Finish();
+ return compressed;
+ }
+
+ TString Decompress(TString data) {
+ TStringInput input(data);
+ TBrotliDecompress decompressStream(&input);
+ return decompressStream.ReadAll();
+ }
+
+ void TestCase(const TString& s) {
+ UNIT_ASSERT_VALUES_EQUAL(s, Decompress(Compress(s)));
+ }
+
+ TString GenerateRandomString(size_t size) {
+ TReallyFastRng32 rng(42);
TString result;
result.reserve(size + sizeof(ui64));
while (result.size() < size) {
@@ -38,7 +38,7 @@ Y_UNIT_TEST_SUITE(TBrotliTestSuite) {
}
Y_UNIT_TEST(TestHelloWorld) {
- TestCase("hello world");
+ TestCase("hello world");
}
Y_UNIT_TEST(TestFlush) {
@@ -59,31 +59,31 @@ Y_UNIT_TEST_SUITE(TBrotliTestSuite) {
}
Y_UNIT_TEST(TestSeveralStreams) {
- auto s1 = GenerateRandomString(1 << 15);
- auto s2 = GenerateRandomString(1 << 15);
- auto c1 = Compress(s1);
- auto c2 = Compress(s2);
- UNIT_ASSERT_VALUES_EQUAL(s1 + s2, Decompress(c1 + c2));
+ auto s1 = GenerateRandomString(1 << 15);
+ auto s2 = GenerateRandomString(1 << 15);
+ auto c1 = Compress(s1);
+ auto c2 = Compress(s2);
+ UNIT_ASSERT_VALUES_EQUAL(s1 + s2, Decompress(c1 + c2));
}
Y_UNIT_TEST(TestIncompleteStream) {
- TString manyAs(64 * 1024, 'a');
- auto compressed = Compress(manyAs);
+ TString manyAs(64 * 1024, 'a');
+ auto compressed = Compress(manyAs);
TString truncated(compressed.data(), compressed.size() - 1);
- UNIT_CHECK_GENERATED_EXCEPTION(Decompress(truncated), std::exception);
+ UNIT_CHECK_GENERATED_EXCEPTION(Decompress(truncated), std::exception);
}
Y_UNIT_TEST(Test64KB) {
- auto manyAs = TString(64 * 1024, 'a');
- TString str("Hello from the Matrix!@#% How are you?}{\n\t\a");
- TestCase(manyAs + str + manyAs);
+ auto manyAs = TString(64 * 1024, 'a');
+ TString str("Hello from the Matrix!@#% How are you?}{\n\t\a");
+ TestCase(manyAs + str + manyAs);
}
Y_UNIT_TEST(Test1MB) {
- TestCase(GenerateRandomString(1 * 1024 * 1024));
- }
-
+ TestCase(GenerateRandomString(1 * 1024 * 1024));
+ }
+
Y_UNIT_TEST(TestEmpty) {
- TestCase("");
+ TestCase("");
}
}