diff options
Diffstat (limited to 'library/cpp/blockcodecs')
| -rw-r--r-- | library/cpp/blockcodecs/codecs/zstd/README.md | 40 | ||||
| -rw-r--r-- | library/cpp/blockcodecs/codecs/zstd/zstd.cpp | 29 | ||||
| -rw-r--r-- | library/cpp/blockcodecs/codecs_ut.cpp | 3 |
3 files changed, 60 insertions, 12 deletions
diff --git a/library/cpp/blockcodecs/codecs/zstd/README.md b/library/cpp/blockcodecs/codecs/zstd/README.md new file mode 100644 index 00000000000..98236f837d7 --- /dev/null +++ b/library/cpp/blockcodecs/codecs/zstd/README.md @@ -0,0 +1,40 @@ +Zstd codecs +============= + +This library registers zstd compression codecs as `zstd_1`, ..., `zstd_22`. +Fast levels are also registered as `zstd_fast_1`, ..., `zstd_fast_7`. + +Measured codec performance on every level. Values below are provided just for reference, exact numbers may vary depending on CPU model and type of data being compressed. + +| Codec | Comp. Ratio | Comp. Speed (MBps) | Decomp. Speed (MBps) | +|--------------|-------------|---------------------|----------------------| +| lz4 | 0.5876 | 913 | 4100 | +| zstd_fast_7 | 0.5783 | 1066 | 2887 | +| zstd_fast_6 | 0.5733 | 1050 | 2870 | +| zstd_fast_5 | 0.5528 | 942 | 2594 | +| zstd_fast_4 | 0.5529 | 918 | 2659 | +| zstd_fast_3 | 0.5408 | 885 | 2519 | +| zstd_fast_2 | 0.5132 | 769 | 2374 | +| zstd_fast_1 | 0.5119 | 707 | 2386 | +| zstd_1 | 0.4691 | 690 | 1692 | +| zstd_2 | 0.4083 | 467 | 1496 | +| zstd_3 | 0.3505 | 358 | 1801 | +| zstd_4 | 0.3356 | 310 | 1932 | +| zstd_5 | 0.3175 | 218 | 1832 | +| zstd_6 | 0.3168 | 173 | 1920 | +| zstd_7 | 0.3081 | 149 | 1965 | +| zstd_8 | 0.3077 | 125 | 2005 | +| zstd_9 | 0.272 | 128 | 2179 | +| zstd_10 | 0.2693 | 98.5 | 2228 | +| zstd_11 | 0.2684 | 78.6 | 2185 | +| zstd_12 | 0.2682 | 71.1 | 2231 | +| zstd_13 | 0.2687 | 27.8 | 2102 | +| zstd_14 | 0.2676 | 24 | 2024 | +| zstd_15 | 0.2663 | 18.8 | 2225 | +| zstd_16 | 0.257 | 15.2 | 2093 | +| zstd_17 | 0.2521 | 12.3 | 2072 | +| zstd_18 | 0.241 | 9.68 | 1696 | +| zstd_19 | 0.2395 | 7.97 | 1709 | +| zstd_20 | 0.2337 | 6.26 | 1520 | +| zstd_21 | 0.2255 | 5.22 | 1442 | +| zstd_22 | 0.2037 | 3.88 | 1490 | diff --git a/library/cpp/blockcodecs/codecs/zstd/zstd.cpp b/library/cpp/blockcodecs/codecs/zstd/zstd.cpp index 95299b3f6d3..3846ab85802 100644 --- a/library/cpp/blockcodecs/codecs/zstd/zstd.cpp +++ b/library/cpp/blockcodecs/codecs/zstd/zstd.cpp @@ -5,36 +5,38 @@ #define ZSTD_STATIC_LINKING_ONLY #include <contrib/libs/zstd/include/zstd.h> +#include <utility> + using namespace NBlockCodecs; namespace { struct TZStd08Codec: public TAddLengthCodec<TZStd08Codec> { - inline TZStd08Codec(unsigned level) + TZStd08Codec(int level, TString name) : Level(level) - , MyName(TStringBuf("zstd08_") + ToString(Level)) + , MyName(std::move(name)) { } - static inline size_t CheckError(size_t ret, const char* what) { - if (ZSTD_isError(ret)) { + static size_t CheckError(size_t ret, const char* what) { + if (Y_UNLIKELY(ZSTD_isError(ret))) { ythrow yexception() << what << TStringBuf(" zstd error: ") << ZSTD_getErrorName(ret); } return ret; } - static inline size_t DoMaxCompressedLength(size_t l) noexcept { + static size_t DoMaxCompressedLength(size_t l) noexcept { return ZSTD_compressBound(l); } - inline size_t DoCompress(const TData& in, void* out) const { + size_t DoCompress(const TData& in, void* out) const { return CheckError(ZSTD_compress(out, DoMaxCompressedLength(in.size()), in.data(), in.size(), Level), "compress"); } - inline void DoDecompress(const TData& in, void* out, size_t dsize) const { + static void DoDecompress(const TData& in, void* out, size_t dsize) { const size_t res = CheckError(ZSTD_decompress(out, dsize, in.data(), in.size()), "decompress"); - if (res != dsize) { + if (Y_UNLIKELY(res != dsize)) { ythrow TDecompressError(dsize, res); } } @@ -43,15 +45,20 @@ namespace { return MyName; } - const unsigned Level; + const int Level; const TString MyName; }; struct TZStd08Registrar { TZStd08Registrar() { for (int i = 1; i <= ZSTD_maxCLevel(); ++i) { - RegisterCodec(MakeHolder<TZStd08Codec>(i)); - RegisterAlias("zstd_" + ToString(i), "zstd08_" + ToString(i)); + const TString name = "zstd08_"sv + ToString(i); + RegisterCodec(MakeHolder<TZStd08Codec>(i, name)); + RegisterAlias("zstd_"sv + ToString(i), name); + } + + for (int i = 1; i <= 7; ++i) { + RegisterCodec(MakeHolder<TZStd08Codec>(-i, "zstd_fast_"sv + ToString(i))); } } }; diff --git a/library/cpp/blockcodecs/codecs_ut.cpp b/library/cpp/blockcodecs/codecs_ut.cpp index bfe5a236909..e167816680e 100644 --- a/library/cpp/blockcodecs/codecs_ut.cpp +++ b/library/cpp/blockcodecs/codecs_ut.cpp @@ -320,7 +320,8 @@ Y_UNIT_TEST_SUITE(TBlockCodecsTest) { "zstd08_1,zstd08_10,zstd08_11,zstd08_12,zstd08_13,zstd08_14,zstd08_15,zstd08_16,zstd08_17,zstd08_18," "zstd08_19,zstd08_2,zstd08_20,zstd08_21,zstd08_22,zstd08_3,zstd08_4,zstd08_5,zstd08_6,zstd08_7,zstd08_8," "zstd08_9,zstd_1,zstd_10,zstd_11,zstd_12,zstd_13,zstd_14,zstd_15,zstd_16,zstd_17,zstd_18,zstd_19,zstd_2," - "zstd_20,zstd_21,zstd_22,zstd_3,zstd_4,zstd_5,zstd_6,zstd_7,zstd_8,zstd_9"; + "zstd_20,zstd_21,zstd_22,zstd_3,zstd_4,zstd_5,zstd_6,zstd_7,zstd_8,zstd_9," + "zstd_fast_1,zstd_fast_2,zstd_fast_3,zstd_fast_4,zstd_fast_5,zstd_fast_6,zstd_fast_7"; UNIT_ASSERT_VALUES_EQUAL(ALL_CODECS, JoinSeq(",", ListAllCodecs())); } |