diff options
author | Ruslan Kovalev <ruslan.a.kovalev@gmail.com> | 2022-02-10 16:46:45 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:46:45 +0300 |
commit | 9123176b341b6f2658cff5132482b8237c1416c8 (patch) | |
tree | 49e222ea1c5804306084bb3ae065bb702625360f /contrib/libs/zstd06 | |
parent | 59e19371de37995fcb36beb16cd6ec030af960bc (diff) | |
download | ydb-9123176b341b6f2658cff5132482b8237c1416c8.tar.gz |
Restoring authorship annotation for Ruslan Kovalev <ruslan.a.kovalev@gmail.com>. Commit 2 of 2.
Diffstat (limited to 'contrib/libs/zstd06')
48 files changed, 16922 insertions, 16922 deletions
diff --git a/contrib/libs/zstd06/common/bitstream.h b/contrib/libs/zstd06/common/bitstream.h index ae1071a6d3..97fc621579 100644 --- a/contrib/libs/zstd06/common/bitstream.h +++ b/contrib/libs/zstd06/common/bitstream.h @@ -1,4 +1,4 @@ -#include <contrib/libs/zstd06/renames.h> +#include <contrib/libs/zstd06/renames.h> /* ****************************************************************** bitstream Part of FSE library @@ -85,7 +85,7 @@ MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC); /* Start with initCStream, providing the size of buffer to write into. * bitStream will never write outside of this buffer. -* `dstCapacity` must be >= sizeof(bitD->bitContainer), otherwise @return will be an error code. +* `dstCapacity` must be >= sizeof(bitD->bitContainer), otherwise @return will be an error code. * * bits are first added to a local register. * Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems. @@ -129,7 +129,7 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD); * Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t). * You can then retrieve bitFields stored into the local register, **in reverse order**. * Local register is explicitly reloaded from memory by the BIT_reloadDStream() method. -* A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished. +* A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished. * Otherwise, it can be less than that, so proceed accordingly. * Checking if DStream has reached its end can be performed with BIT_endOfDStream(). */ @@ -264,9 +264,9 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si { if (srcSize < 1) { memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); } - if (srcSize >= sizeof(bitD->bitContainer)) { /* normal case */ + if (srcSize >= sizeof(bitD->bitContainer)) { /* normal case */ bitD->start = (const char*)srcBuffer; - bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer); + bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer); bitD->bitContainer = MEM_readLEST(bitD->ptr); { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ @@ -277,58 +277,58 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si bitD->bitContainer = *(const BYTE*)(bitD->start); switch(srcSize) { - case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16); - case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24); - case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32); - case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24; - case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16; - case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8; + case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16); + case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24); + case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32); + case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24; + case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16; + case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8; default:; } { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ bitD->bitsConsumed = 8 - BIT_highbit32(lastByte); } - bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize)*8; + bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize)*8; } return srcSize; } -MEM_STATIC size_t BIT_getUpperBits(size_t bitContainer, U32 const start) +MEM_STATIC size_t BIT_getUpperBits(size_t bitContainer, U32 const start) { - return bitContainer >> start; + return bitContainer >> start; } -MEM_STATIC size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits) +MEM_STATIC size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits) { #if defined(__BMI__) && defined(__GNUC__) /* experimental */ -# if defined(__x86_64__) - if (sizeof(bitContainer)==8) - return _bextr_u64(bitContainer, start, nbBits); - else -# endif - return _bextr_u32(bitContainer, start, nbBits); +# if defined(__x86_64__) + if (sizeof(bitContainer)==8) + return _bextr_u64(bitContainer, start, nbBits); + else +# endif + return _bextr_u32(bitContainer, start, nbBits); #else - return (bitContainer >> start) & BIT_mask[nbBits]; + return (bitContainer >> start) & BIT_mask[nbBits]; #endif } -MEM_STATIC size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits) +MEM_STATIC size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits) { - return bitContainer & BIT_mask[nbBits]; + return bitContainer & BIT_mask[nbBits]; } /*! BIT_lookBits() : * Provides next n bits from local register. - * local register is not modified. + * local register is not modified. * On 32-bits, maxNbBits==24. * On 64-bits, maxNbBits==56. * @return : value extracted */ MEM_STATIC size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits) { -#if defined(__BMI__) && defined(__GNUC__) /* experimental; fails if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8 */ - return BIT_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits); +#if defined(__BMI__) && defined(__GNUC__) /* experimental; fails if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8 */ + return BIT_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits); #else U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1; return ((bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> 1) >> ((bitMask-nbBits) & bitMask); @@ -373,7 +373,7 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits) * Refill `BIT_DStream_t` from src buffer previously defined (see BIT_initDStream() ). * This function is safe, it guarantees it will not read beyond src buffer. * @return : status of `BIT_DStream_t` internal register. - if status == unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */ + if status == unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */ MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD) { if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should never happen */ diff --git a/contrib/libs/zstd06/common/entropy_common.c b/contrib/libs/zstd06/common/entropy_common.c index f8c957736a..b4c366e61a 100644 --- a/contrib/libs/zstd06/common/entropy_common.c +++ b/contrib/libs/zstd06/common/entropy_common.c @@ -1,158 +1,158 @@ -/* - Common functions of New Generation Entropy library - Copyright (C) 2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy - - Public forum : https://groups.google.com/forum/#!forum/lz4c -*************************************************************************** */ - -/* ************************************* -* Dependencies -***************************************/ -#include <stdlib.h> -#include "mem.h" -#include "fse_static.h" /* FSE_MIN_TABLELOG */ -#include "error_private.h" -#include "fse.h" /* declaration of FSE_isError, FSE_getErrorName */ -#include "huf.h" /* declaration of HUF_isError, HUF_getErrorName */ - - - -/*-**************************************** -* FSE Error Management -******************************************/ -unsigned FSE_isError(size_t code) { return ERR_isError(code); } - -const char* FSE_getErrorName(size_t code) { return ERR_getErrorName(code); } - - -/* ************************************************************** -* HUF Error Management -****************************************************************/ -unsigned HUF_isError(size_t code) { return ERR_isError(code); } - -const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); } - - -/*-************************************************************** -* FSE NCount encoding-decoding -****************************************************************/ -static short FSE_abs(short a) { return a<0 ? -a : a; } - -size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, - const void* headerBuffer, size_t hbSize) -{ - const BYTE* const istart = (const BYTE*) headerBuffer; - const BYTE* const iend = istart + hbSize; - const BYTE* ip = istart; - int nbBits; - int remaining; - int threshold; - U32 bitStream; - int bitCount; - unsigned charnum = 0; - int previous0 = 0; - - if (hbSize < 4) return ERROR(srcSize_wrong); - bitStream = MEM_readLE32(ip); - nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG; /* extract tableLog */ - if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge); - bitStream >>= 4; - bitCount = 4; - *tableLogPtr = nbBits; - remaining = (1<<nbBits)+1; - threshold = 1<<nbBits; - nbBits++; - - while ((remaining>1) && (charnum<=*maxSVPtr)) { - if (previous0) { - unsigned n0 = charnum; - while ((bitStream & 0xFFFF) == 0xFFFF) { - n0+=24; - if (ip < iend-5) { - ip+=2; - bitStream = MEM_readLE32(ip) >> bitCount; - } else { - bitStream >>= 16; - bitCount+=16; - } } - while ((bitStream & 3) == 3) { - n0+=3; - bitStream>>=2; - bitCount+=2; - } - n0 += bitStream & 3; - bitCount += 2; - if (n0 > *maxSVPtr) return ERROR(maxSymbolValue_tooSmall); - while (charnum < n0) normalizedCounter[charnum++] = 0; - if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { - ip += bitCount>>3; - bitCount &= 7; - bitStream = MEM_readLE32(ip) >> bitCount; - } - else - bitStream >>= 2; - } - { short const max = (short)((2*threshold-1)-remaining); - short count; - - if ((bitStream & (threshold-1)) < (U32)max) { - count = (short)(bitStream & (threshold-1)); - bitCount += nbBits-1; - } else { - count = (short)(bitStream & (2*threshold-1)); - if (count >= threshold) count -= max; - bitCount += nbBits; - } - - count--; /* extra accuracy */ - remaining -= FSE_abs(count); - normalizedCounter[charnum++] = count; - previous0 = !count; - while (remaining < threshold) { - nbBits--; - threshold >>= 1; - } - - if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { - ip += bitCount>>3; - bitCount &= 7; - } else { - bitCount -= (int)(8 * (iend - 4 - ip)); - ip = iend - 4; - } - bitStream = MEM_readLE32(ip) >> (bitCount & 31); - } } /* while ((remaining>1) && (charnum<=*maxSVPtr)) */ - if (remaining != 1) return ERROR(GENERIC); - *maxSVPtr = charnum-1; - - ip += (bitCount+7)>>3; - if ((size_t)(ip-istart) > hbSize) return ERROR(srcSize_wrong); - return ip-istart; -} +/* + Common functions of New Generation Entropy library + Copyright (C) 2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy + - Public forum : https://groups.google.com/forum/#!forum/lz4c +*************************************************************************** */ + +/* ************************************* +* Dependencies +***************************************/ +#include <stdlib.h> +#include "mem.h" +#include "fse_static.h" /* FSE_MIN_TABLELOG */ +#include "error_private.h" +#include "fse.h" /* declaration of FSE_isError, FSE_getErrorName */ +#include "huf.h" /* declaration of HUF_isError, HUF_getErrorName */ + + + +/*-**************************************** +* FSE Error Management +******************************************/ +unsigned FSE_isError(size_t code) { return ERR_isError(code); } + +const char* FSE_getErrorName(size_t code) { return ERR_getErrorName(code); } + + +/* ************************************************************** +* HUF Error Management +****************************************************************/ +unsigned HUF_isError(size_t code) { return ERR_isError(code); } + +const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); } + + +/*-************************************************************** +* FSE NCount encoding-decoding +****************************************************************/ +static short FSE_abs(short a) { return a<0 ? -a : a; } + +size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize) +{ + const BYTE* const istart = (const BYTE*) headerBuffer; + const BYTE* const iend = istart + hbSize; + const BYTE* ip = istart; + int nbBits; + int remaining; + int threshold; + U32 bitStream; + int bitCount; + unsigned charnum = 0; + int previous0 = 0; + + if (hbSize < 4) return ERROR(srcSize_wrong); + bitStream = MEM_readLE32(ip); + nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG; /* extract tableLog */ + if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge); + bitStream >>= 4; + bitCount = 4; + *tableLogPtr = nbBits; + remaining = (1<<nbBits)+1; + threshold = 1<<nbBits; + nbBits++; + + while ((remaining>1) && (charnum<=*maxSVPtr)) { + if (previous0) { + unsigned n0 = charnum; + while ((bitStream & 0xFFFF) == 0xFFFF) { + n0+=24; + if (ip < iend-5) { + ip+=2; + bitStream = MEM_readLE32(ip) >> bitCount; + } else { + bitStream >>= 16; + bitCount+=16; + } } + while ((bitStream & 3) == 3) { + n0+=3; + bitStream>>=2; + bitCount+=2; + } + n0 += bitStream & 3; + bitCount += 2; + if (n0 > *maxSVPtr) return ERROR(maxSymbolValue_tooSmall); + while (charnum < n0) normalizedCounter[charnum++] = 0; + if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { + ip += bitCount>>3; + bitCount &= 7; + bitStream = MEM_readLE32(ip) >> bitCount; + } + else + bitStream >>= 2; + } + { short const max = (short)((2*threshold-1)-remaining); + short count; + + if ((bitStream & (threshold-1)) < (U32)max) { + count = (short)(bitStream & (threshold-1)); + bitCount += nbBits-1; + } else { + count = (short)(bitStream & (2*threshold-1)); + if (count >= threshold) count -= max; + bitCount += nbBits; + } + + count--; /* extra accuracy */ + remaining -= FSE_abs(count); + normalizedCounter[charnum++] = count; + previous0 = !count; + while (remaining < threshold) { + nbBits--; + threshold >>= 1; + } + + if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { + ip += bitCount>>3; + bitCount &= 7; + } else { + bitCount -= (int)(8 * (iend - 4 - ip)); + ip = iend - 4; + } + bitStream = MEM_readLE32(ip) >> (bitCount & 31); + } } /* while ((remaining>1) && (charnum<=*maxSVPtr)) */ + if (remaining != 1) return ERROR(GENERIC); + *maxSVPtr = charnum-1; + + ip += (bitCount+7)>>3; + if ((size_t)(ip-istart) > hbSize) return ERROR(srcSize_wrong); + return ip-istart; +} diff --git a/contrib/libs/zstd06/common/error_private.h b/contrib/libs/zstd06/common/error_private.h index c423dddfe9..d8d1ef24b3 100644 --- a/contrib/libs/zstd06/common/error_private.h +++ b/contrib/libs/zstd06/common/error_private.h @@ -1,4 +1,4 @@ -#include <contrib/libs/zstd06/renames.h> +#include <contrib/libs/zstd06/renames.h> /* ****************************************************************** Error codes and messages Copyright (C) 2013-2016, Yann Collet @@ -79,17 +79,17 @@ typedef ZSTD_ErrorCode ERR_enum; ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); } -ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); } +ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); } /*-**************************************** * Error Strings ******************************************/ -ERR_STATIC const char* ERR_getErrorString(ERR_enum code) +ERR_STATIC const char* ERR_getErrorString(ERR_enum code) { static const char* notErrorCode = "Unspecified error code"; - switch( code ) + switch( code ) { case PREFIX(no_error): return "No error detected"; case PREFIX(GENERIC): return "Error (generic)"; @@ -108,14 +108,14 @@ ERR_STATIC const char* ERR_getErrorString(ERR_enum code) case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small"; case PREFIX(dictionary_corrupted): return "Dictionary is corrupted"; case PREFIX(maxCode): - default: return notErrorCode; + default: return notErrorCode; } } -ERR_STATIC const char* ERR_getErrorName(size_t code) -{ - return ERR_getErrorString(ERR_getErrorCode(code)); -} +ERR_STATIC const char* ERR_getErrorName(size_t code) +{ + return ERR_getErrorString(ERR_getErrorCode(code)); +} #if defined (__cplusplus) } diff --git a/contrib/libs/zstd06/common/error_public.h b/contrib/libs/zstd06/common/error_public.h index e9ce5a9f09..1d8f03995f 100644 --- a/contrib/libs/zstd06/common/error_public.h +++ b/contrib/libs/zstd06/common/error_public.h @@ -1,4 +1,4 @@ -#include <contrib/libs/zstd06/renames.h> +#include <contrib/libs/zstd06/renames.h> /* ****************************************************************** Error codes list Copyright (C) 2016, Yann Collet diff --git a/contrib/libs/zstd06/common/fse.h b/contrib/libs/zstd06/common/fse.h index 942e8d79a2..1cc6992951 100644 --- a/contrib/libs/zstd06/common/fse.h +++ b/contrib/libs/zstd06/common/fse.h @@ -1,4 +1,4 @@ -#include <contrib/libs/zstd06/renames.h> +#include <contrib/libs/zstd06/renames.h> /* ****************************************************************** FSE : Finite State Entropy codec Public Prototypes declaration diff --git a/contrib/libs/zstd06/common/fse_decompress.c b/contrib/libs/zstd06/common/fse_decompress.c index db6378a610..601edd7417 100644 --- a/contrib/libs/zstd06/common/fse_decompress.c +++ b/contrib/libs/zstd06/common/fse_decompress.c @@ -1,330 +1,330 @@ -/* ****************************************************************** - FSE : Finite State Entropy decoder - Copyright (C) 2013-2015, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy - - Public forum : https://groups.google.com/forum/#!forum/lz4c -****************************************************************** */ - - -/* ************************************************************** -* Compiler specifics -****************************************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# define FORCE_INLINE static __forceinline -# include <intrin.h> /* For Visual 2005 */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */ -#else -# ifdef __GNUC__ -# define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -# define FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define FORCE_INLINE static inline -# endif -#endif - - -/* ************************************************************** -* Includes -****************************************************************/ -#include <stdlib.h> /* malloc, free, qsort */ -#include <string.h> /* memcpy, memset */ -#include <stdio.h> /* printf (debug) */ -#include "bitstream.h" -#include "fse_static.h" - - -/* ************************************************************** -* Error Management -****************************************************************/ -#define FSE_isError ERR_isError -#define FSE_STATIC_ASSERT(c) { enum { FSE_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ - - -/* ************************************************************** -* Complex types -****************************************************************/ -typedef U32 DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)]; - - -/* ************************************************************** -* Templates -****************************************************************/ -/* - designed to be included - for type-specific functions (template emulation in C) - Objective is to write these functions only once, for improved maintenance -*/ - -/* safety checks */ -#ifndef FSE_FUNCTION_EXTENSION -# error "FSE_FUNCTION_EXTENSION must be defined" -#endif -#ifndef FSE_FUNCTION_TYPE -# error "FSE_FUNCTION_TYPE must be defined" -#endif - -/* Function names */ -#define FSE_CAT(X,Y) X##Y -#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y) -#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y) - - -/* Function templates */ -FSE_DTable* FSE_createDTable (unsigned tableLog) -{ - if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX; - return (FSE_DTable*)malloc( FSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32) ); -} - -void FSE_freeDTable (FSE_DTable* dt) -{ - free(dt); -} - -size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) -{ - void* const tdPtr = dt+1; /* because *dt is unsigned, 32-bits aligned on 32-bits */ - FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (tdPtr); - U16 symbolNext[FSE_MAX_SYMBOL_VALUE+1]; - - U32 const maxSV1 = maxSymbolValue + 1; - U32 const tableSize = 1 << tableLog; - U32 highThreshold = tableSize-1; - - /* Sanity Checks */ - if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge); - if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); - - /* Init, lay down lowprob symbols */ - { FSE_DTableHeader DTableH; - DTableH.tableLog = (U16)tableLog; - DTableH.fastMode = 1; - { S16 const largeLimit= (S16)(1 << (tableLog-1)); - U32 s; - for (s=0; s<maxSV1; s++) { - if (normalizedCounter[s]==-1) { - tableDecode[highThreshold--].symbol = (FSE_FUNCTION_TYPE)s; - symbolNext[s] = 1; - } else { - if (normalizedCounter[s] >= largeLimit) DTableH.fastMode=0; - symbolNext[s] = normalizedCounter[s]; - } } } - memcpy(dt, &DTableH, sizeof(DTableH)); - } - - /* Spread symbols */ - { U32 const tableMask = tableSize-1; - U32 const step = FSE_TABLESTEP(tableSize); - U32 s, position = 0; - for (s=0; s<maxSV1; s++) { - int i; - for (i=0; i<normalizedCounter[s]; i++) { - tableDecode[position].symbol = (FSE_FUNCTION_TYPE)s; - position = (position + step) & tableMask; - while (position > highThreshold) position = (position + step) & tableMask; /* lowprob area */ - } } - - if (position!=0) return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ - } - - /* Build Decoding table */ - { U32 u; - for (u=0; u<tableSize; u++) { - FSE_FUNCTION_TYPE const symbol = (FSE_FUNCTION_TYPE)(tableDecode[u].symbol); - U16 nextState = symbolNext[symbol]++; - tableDecode[u].nbBits = (BYTE) (tableLog - BIT_highbit32 ((U32)nextState) ); - tableDecode[u].newState = (U16) ( (nextState << tableDecode[u].nbBits) - tableSize); - } } - - return 0; -} - - - -#ifndef FSE_COMMONDEFS_ONLY - -/*-******************************************************* -* Decompression (Byte symbols) -*********************************************************/ -size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue) -{ - void* ptr = dt; - FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr; - void* dPtr = dt + 1; - FSE_decode_t* const cell = (FSE_decode_t*)dPtr; - - DTableH->tableLog = 0; - DTableH->fastMode = 0; - - cell->newState = 0; - cell->symbol = symbolValue; - cell->nbBits = 0; - - return 0; -} - - -size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits) -{ - void* ptr = dt; - FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr; - void* dPtr = dt + 1; - FSE_decode_t* const dinfo = (FSE_decode_t*)dPtr; - const unsigned tableSize = 1 << nbBits; - const unsigned tableMask = tableSize - 1; - const unsigned maxSV1 = tableMask+1; - unsigned s; - - /* Sanity checks */ - if (nbBits < 1) return ERROR(GENERIC); /* min size */ - - /* Build Decoding Table */ - DTableH->tableLog = (U16)nbBits; - DTableH->fastMode = 1; - for (s=0; s<maxSV1; s++) { - dinfo[s].newState = 0; - dinfo[s].symbol = (BYTE)s; - dinfo[s].nbBits = (BYTE)nbBits; - } - - return 0; -} - -FORCE_INLINE size_t FSE_decompress_usingDTable_generic( - void* dst, size_t maxDstSize, - const void* cSrc, size_t cSrcSize, - const FSE_DTable* dt, const unsigned fast) -{ - BYTE* const ostart = (BYTE*) dst; - BYTE* op = ostart; - BYTE* const omax = op + maxDstSize; - BYTE* const olimit = omax-3; - - BIT_DStream_t bitD; - FSE_DState_t state1; - FSE_DState_t state2; - - /* Init */ - { size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize); /* replaced last arg by maxCompressed Size */ - if (FSE_isError(errorCode)) return errorCode; } - - FSE_initDState(&state1, &bitD, dt); - FSE_initDState(&state2, &bitD, dt); - -#define FSE_GETSYMBOL(statePtr) fast ? FSE_decodeSymbolFast(statePtr, &bitD) : FSE_decodeSymbol(statePtr, &bitD) - - /* 4 symbols per loop */ - for ( ; (BIT_reloadDStream(&bitD)==BIT_DStream_unfinished) && (op<olimit) ; op+=4) { - op[0] = FSE_GETSYMBOL(&state1); - - if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ - BIT_reloadDStream(&bitD); - - op[1] = FSE_GETSYMBOL(&state2); - - if (FSE_MAX_TABLELOG*4+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ - { if (BIT_reloadDStream(&bitD) > BIT_DStream_unfinished) { op+=2; break; } } - - op[2] = FSE_GETSYMBOL(&state1); - - if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ - BIT_reloadDStream(&bitD); - - op[3] = FSE_GETSYMBOL(&state2); - } - - /* tail */ - /* note : BIT_reloadDStream(&bitD) >= FSE_DStream_partiallyFilled; Ends at exactly BIT_DStream_completed */ - while (1) { - if (op>(omax-2)) return ERROR(dstSize_tooSmall); - - *op++ = FSE_GETSYMBOL(&state1); - - if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) { - *op++ = FSE_GETSYMBOL(&state2); - break; - } - - if (op>(omax-2)) return ERROR(dstSize_tooSmall); - - *op++ = FSE_GETSYMBOL(&state2); - - if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) { - *op++ = FSE_GETSYMBOL(&state1); - break; - } } - - return op-ostart; -} - - -size_t FSE_decompress_usingDTable(void* dst, size_t originalSize, - const void* cSrc, size_t cSrcSize, - const FSE_DTable* dt) -{ - const void* ptr = dt; - const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr; - const U32 fastMode = DTableH->fastMode; - - /* select fast mode (static) */ - if (fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1); - return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0); -} - - -size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize) -{ - const BYTE* const istart = (const BYTE*)cSrc; - const BYTE* ip = istart; - short counting[FSE_MAX_SYMBOL_VALUE+1]; - DTable_max_t dt; /* Static analyzer seems unable to understand this table will be properly initialized later */ - unsigned tableLog; - unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE; - - if (cSrcSize<2) return ERROR(srcSize_wrong); /* too small input size */ - - /* normal FSE decoding mode */ - { size_t const NCountLength = FSE_readNCount (counting, &maxSymbolValue, &tableLog, istart, cSrcSize); - if (FSE_isError(NCountLength)) return NCountLength; - if (NCountLength >= cSrcSize) return ERROR(srcSize_wrong); /* too small input size */ - ip += NCountLength; - cSrcSize -= NCountLength; - } - - { size_t const errorCode = FSE_buildDTable (dt, counting, maxSymbolValue, tableLog); - if (FSE_isError(errorCode)) return errorCode; } - - return FSE_decompress_usingDTable (dst, maxDstSize, ip, cSrcSize, dt); /* always return, even if it is an error code */ -} - - - -#endif /* FSE_COMMONDEFS_ONLY */ +/* ****************************************************************** + FSE : Finite State Entropy decoder + Copyright (C) 2013-2015, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + - Public forum : https://groups.google.com/forum/#!forum/lz4c +****************************************************************** */ + + +/* ************************************************************** +* Compiler specifics +****************************************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# define FORCE_INLINE static __forceinline +# include <intrin.h> /* For Visual 2005 */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */ +#else +# ifdef __GNUC__ +# define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif +#endif + + +/* ************************************************************** +* Includes +****************************************************************/ +#include <stdlib.h> /* malloc, free, qsort */ +#include <string.h> /* memcpy, memset */ +#include <stdio.h> /* printf (debug) */ +#include "bitstream.h" +#include "fse_static.h" + + +/* ************************************************************** +* Error Management +****************************************************************/ +#define FSE_isError ERR_isError +#define FSE_STATIC_ASSERT(c) { enum { FSE_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ + + +/* ************************************************************** +* Complex types +****************************************************************/ +typedef U32 DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)]; + + +/* ************************************************************** +* Templates +****************************************************************/ +/* + designed to be included + for type-specific functions (template emulation in C) + Objective is to write these functions only once, for improved maintenance +*/ + +/* safety checks */ +#ifndef FSE_FUNCTION_EXTENSION +# error "FSE_FUNCTION_EXTENSION must be defined" +#endif +#ifndef FSE_FUNCTION_TYPE +# error "FSE_FUNCTION_TYPE must be defined" +#endif + +/* Function names */ +#define FSE_CAT(X,Y) X##Y +#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y) +#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y) + + +/* Function templates */ +FSE_DTable* FSE_createDTable (unsigned tableLog) +{ + if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX; + return (FSE_DTable*)malloc( FSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32) ); +} + +void FSE_freeDTable (FSE_DTable* dt) +{ + free(dt); +} + +size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) +{ + void* const tdPtr = dt+1; /* because *dt is unsigned, 32-bits aligned on 32-bits */ + FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (tdPtr); + U16 symbolNext[FSE_MAX_SYMBOL_VALUE+1]; + + U32 const maxSV1 = maxSymbolValue + 1; + U32 const tableSize = 1 << tableLog; + U32 highThreshold = tableSize-1; + + /* Sanity Checks */ + if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge); + if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); + + /* Init, lay down lowprob symbols */ + { FSE_DTableHeader DTableH; + DTableH.tableLog = (U16)tableLog; + DTableH.fastMode = 1; + { S16 const largeLimit= (S16)(1 << (tableLog-1)); + U32 s; + for (s=0; s<maxSV1; s++) { + if (normalizedCounter[s]==-1) { + tableDecode[highThreshold--].symbol = (FSE_FUNCTION_TYPE)s; + symbolNext[s] = 1; + } else { + if (normalizedCounter[s] >= largeLimit) DTableH.fastMode=0; + symbolNext[s] = normalizedCounter[s]; + } } } + memcpy(dt, &DTableH, sizeof(DTableH)); + } + + /* Spread symbols */ + { U32 const tableMask = tableSize-1; + U32 const step = FSE_TABLESTEP(tableSize); + U32 s, position = 0; + for (s=0; s<maxSV1; s++) { + int i; + for (i=0; i<normalizedCounter[s]; i++) { + tableDecode[position].symbol = (FSE_FUNCTION_TYPE)s; + position = (position + step) & tableMask; + while (position > highThreshold) position = (position + step) & tableMask; /* lowprob area */ + } } + + if (position!=0) return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ + } + + /* Build Decoding table */ + { U32 u; + for (u=0; u<tableSize; u++) { + FSE_FUNCTION_TYPE const symbol = (FSE_FUNCTION_TYPE)(tableDecode[u].symbol); + U16 nextState = symbolNext[symbol]++; + tableDecode[u].nbBits = (BYTE) (tableLog - BIT_highbit32 ((U32)nextState) ); + tableDecode[u].newState = (U16) ( (nextState << tableDecode[u].nbBits) - tableSize); + } } + + return 0; +} + + + +#ifndef FSE_COMMONDEFS_ONLY + +/*-******************************************************* +* Decompression (Byte symbols) +*********************************************************/ +size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue) +{ + void* ptr = dt; + FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr; + void* dPtr = dt + 1; + FSE_decode_t* const cell = (FSE_decode_t*)dPtr; + + DTableH->tableLog = 0; + DTableH->fastMode = 0; + + cell->newState = 0; + cell->symbol = symbolValue; + cell->nbBits = 0; + + return 0; +} + + +size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits) +{ + void* ptr = dt; + FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr; + void* dPtr = dt + 1; + FSE_decode_t* const dinfo = (FSE_decode_t*)dPtr; + const unsigned tableSize = 1 << nbBits; + const unsigned tableMask = tableSize - 1; + const unsigned maxSV1 = tableMask+1; + unsigned s; + + /* Sanity checks */ + if (nbBits < 1) return ERROR(GENERIC); /* min size */ + + /* Build Decoding Table */ + DTableH->tableLog = (U16)nbBits; + DTableH->fastMode = 1; + for (s=0; s<maxSV1; s++) { + dinfo[s].newState = 0; + dinfo[s].symbol = (BYTE)s; + dinfo[s].nbBits = (BYTE)nbBits; + } + + return 0; +} + +FORCE_INLINE size_t FSE_decompress_usingDTable_generic( + void* dst, size_t maxDstSize, + const void* cSrc, size_t cSrcSize, + const FSE_DTable* dt, const unsigned fast) +{ + BYTE* const ostart = (BYTE*) dst; + BYTE* op = ostart; + BYTE* const omax = op + maxDstSize; + BYTE* const olimit = omax-3; + + BIT_DStream_t bitD; + FSE_DState_t state1; + FSE_DState_t state2; + + /* Init */ + { size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize); /* replaced last arg by maxCompressed Size */ + if (FSE_isError(errorCode)) return errorCode; } + + FSE_initDState(&state1, &bitD, dt); + FSE_initDState(&state2, &bitD, dt); + +#define FSE_GETSYMBOL(statePtr) fast ? FSE_decodeSymbolFast(statePtr, &bitD) : FSE_decodeSymbol(statePtr, &bitD) + + /* 4 symbols per loop */ + for ( ; (BIT_reloadDStream(&bitD)==BIT_DStream_unfinished) && (op<olimit) ; op+=4) { + op[0] = FSE_GETSYMBOL(&state1); + + if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ + BIT_reloadDStream(&bitD); + + op[1] = FSE_GETSYMBOL(&state2); + + if (FSE_MAX_TABLELOG*4+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ + { if (BIT_reloadDStream(&bitD) > BIT_DStream_unfinished) { op+=2; break; } } + + op[2] = FSE_GETSYMBOL(&state1); + + if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ + BIT_reloadDStream(&bitD); + + op[3] = FSE_GETSYMBOL(&state2); + } + + /* tail */ + /* note : BIT_reloadDStream(&bitD) >= FSE_DStream_partiallyFilled; Ends at exactly BIT_DStream_completed */ + while (1) { + if (op>(omax-2)) return ERROR(dstSize_tooSmall); + + *op++ = FSE_GETSYMBOL(&state1); + + if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) { + *op++ = FSE_GETSYMBOL(&state2); + break; + } + + if (op>(omax-2)) return ERROR(dstSize_tooSmall); + + *op++ = FSE_GETSYMBOL(&state2); + + if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) { + *op++ = FSE_GETSYMBOL(&state1); + break; + } } + + return op-ostart; +} + + +size_t FSE_decompress_usingDTable(void* dst, size_t originalSize, + const void* cSrc, size_t cSrcSize, + const FSE_DTable* dt) +{ + const void* ptr = dt; + const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr; + const U32 fastMode = DTableH->fastMode; + + /* select fast mode (static) */ + if (fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1); + return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0); +} + + +size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize) +{ + const BYTE* const istart = (const BYTE*)cSrc; + const BYTE* ip = istart; + short counting[FSE_MAX_SYMBOL_VALUE+1]; + DTable_max_t dt; /* Static analyzer seems unable to understand this table will be properly initialized later */ + unsigned tableLog; + unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE; + + if (cSrcSize<2) return ERROR(srcSize_wrong); /* too small input size */ + + /* normal FSE decoding mode */ + { size_t const NCountLength = FSE_readNCount (counting, &maxSymbolValue, &tableLog, istart, cSrcSize); + if (FSE_isError(NCountLength)) return NCountLength; + if (NCountLength >= cSrcSize) return ERROR(srcSize_wrong); /* too small input size */ + ip += NCountLength; + cSrcSize -= NCountLength; + } + + { size_t const errorCode = FSE_buildDTable (dt, counting, maxSymbolValue, tableLog); + if (FSE_isError(errorCode)) return errorCode; } + + return FSE_decompress_usingDTable (dst, maxDstSize, ip, cSrcSize, dt); /* always return, even if it is an error code */ +} + + + +#endif /* FSE_COMMONDEFS_ONLY */ diff --git a/contrib/libs/zstd06/common/fse_static.h b/contrib/libs/zstd06/common/fse_static.h index d499f45df4..d09589efee 100644 --- a/contrib/libs/zstd06/common/fse_static.h +++ b/contrib/libs/zstd06/common/fse_static.h @@ -1,4 +1,4 @@ -#include <contrib/libs/zstd06/renames.h> +#include <contrib/libs/zstd06/renames.h> /* ****************************************************************** FSE : Finite State Entropy coder header file for static linking (only) @@ -335,54 +335,54 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr) } - -#ifndef FSE_COMMONDEFS_ONLY - -/* ************************************************************** -* Tuning parameters -****************************************************************/ -/*!MEMORY_USAGE : -* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) -* Increasing memory usage improves compression ratio -* Reduced memory usage can improve speed, due to cache effect -* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ -#define FSE_MAX_MEMORY_USAGE 14 -#define FSE_DEFAULT_MEMORY_USAGE 13 - -/*!FSE_MAX_SYMBOL_VALUE : -* Maximum symbol value authorized. -* Required for proper stack allocation */ -#define FSE_MAX_SYMBOL_VALUE 255 - - -/* ************************************************************** -* template functions type & suffix -****************************************************************/ -#define FSE_FUNCTION_TYPE BYTE -#define FSE_FUNCTION_EXTENSION -#define FSE_DECODE_TYPE FSE_decode_t - - -#endif /* !FSE_COMMONDEFS_ONLY */ - - -/* *************************************************************** -* Constants -*****************************************************************/ -#define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE-2) -#define FSE_MAX_TABLESIZE (1U<<FSE_MAX_TABLELOG) -#define FSE_MAXTABLESIZE_MASK (FSE_MAX_TABLESIZE-1) -#define FSE_DEFAULT_TABLELOG (FSE_DEFAULT_MEMORY_USAGE-2) -#define FSE_MIN_TABLELOG 5 - -#define FSE_TABLELOG_ABSOLUTE_MAX 15 -#if FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX -#error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported" -#endif - -#define FSE_TABLESTEP(tableSize) ((tableSize>>1) + (tableSize>>3) + 3) - - + +#ifndef FSE_COMMONDEFS_ONLY + +/* ************************************************************** +* Tuning parameters +****************************************************************/ +/*!MEMORY_USAGE : +* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) +* Increasing memory usage improves compression ratio +* Reduced memory usage can improve speed, due to cache effect +* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ +#define FSE_MAX_MEMORY_USAGE 14 +#define FSE_DEFAULT_MEMORY_USAGE 13 + +/*!FSE_MAX_SYMBOL_VALUE : +* Maximum symbol value authorized. +* Required for proper stack allocation */ +#define FSE_MAX_SYMBOL_VALUE 255 + + +/* ************************************************************** +* template functions type & suffix +****************************************************************/ +#define FSE_FUNCTION_TYPE BYTE +#define FSE_FUNCTION_EXTENSION +#define FSE_DECODE_TYPE FSE_decode_t + + +#endif /* !FSE_COMMONDEFS_ONLY */ + + +/* *************************************************************** +* Constants +*****************************************************************/ +#define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE-2) +#define FSE_MAX_TABLESIZE (1U<<FSE_MAX_TABLELOG) +#define FSE_MAXTABLESIZE_MASK (FSE_MAX_TABLESIZE-1) +#define FSE_DEFAULT_TABLELOG (FSE_DEFAULT_MEMORY_USAGE-2) +#define FSE_MIN_TABLELOG 5 + +#define FSE_TABLELOG_ABSOLUTE_MAX 15 +#if FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX +#error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported" +#endif + +#define FSE_TABLESTEP(tableSize) ((tableSize>>1) + (tableSize>>3) + 3) + + #if defined (__cplusplus) } #endif diff --git a/contrib/libs/zstd06/common/huf.h b/contrib/libs/zstd06/common/huf.h index f375711625..d520a779ea 100644 --- a/contrib/libs/zstd06/common/huf.h +++ b/contrib/libs/zstd06/common/huf.h @@ -1,98 +1,98 @@ -#include <contrib/libs/zstd06/renames.h> -/* ****************************************************************** - Huffman coder, part of New Generation Entropy library - header file - Copyright (C) 2013-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Source repository : https://github.com/Cyan4973/FiniteStateEntropy -****************************************************************** */ -#ifndef HUF_H -#define HUF_H - -#if defined (__cplusplus) -extern "C" { -#endif - - -/* **************************************** -* Dependency -******************************************/ -#include <stddef.h> /* size_t */ - - -/* **************************************** -* HUF simple functions -******************************************/ -size_t HUF_compress(void* dst, size_t dstCapacity, - const void* src, size_t srcSize); -size_t HUF_decompress(void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize); -/* -HUF_compress() : - Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'. - 'dst' buffer must be already allocated. Compression runs faster if dstCapacity >= HUF_compressBound(srcSize). - Note : srcSize must be <= 128 KB - @return : size of compressed data (<= dstCapacity) - Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!! - if return == 1, srcData is a single repeated byte symbol (RLE compression). - if HUF_isError(return), compression failed (more details using HUF_getErrorName()) - -HUF_decompress() : - Decompress HUF data from buffer 'cSrc', of size 'cSrcSize', - into already allocated destination buffer 'dst', of size 'dstSize'. - `dstSize` : must be the **exact** size of original (uncompressed) data. - Note : in contrast with FSE, HUF_decompress can regenerate - RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data, - because it knows size to regenerate. - @return : size of regenerated data (== dstSize) - or an error code, which can be tested using HUF_isError() -*/ - - -/* **************************************** -* Tool functions -******************************************/ -size_t HUF_compressBound(size_t size); /**< maximum compressed size */ - -/* Error Management */ -unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */ -const char* HUF_getErrorName(size_t code); /**< provides error code string (useful for debugging) */ - - -/* **************************************** -* Advanced functions -******************************************/ -size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); - - -#if defined (__cplusplus) -} -#endif - -#endif /* HUF_H */ +#include <contrib/libs/zstd06/renames.h> +/* ****************************************************************** + Huffman coder, part of New Generation Entropy library + header file + Copyright (C) 2013-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - Source repository : https://github.com/Cyan4973/FiniteStateEntropy +****************************************************************** */ +#ifndef HUF_H +#define HUF_H + +#if defined (__cplusplus) +extern "C" { +#endif + + +/* **************************************** +* Dependency +******************************************/ +#include <stddef.h> /* size_t */ + + +/* **************************************** +* HUF simple functions +******************************************/ +size_t HUF_compress(void* dst, size_t dstCapacity, + const void* src, size_t srcSize); +size_t HUF_decompress(void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize); +/* +HUF_compress() : + Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'. + 'dst' buffer must be already allocated. Compression runs faster if dstCapacity >= HUF_compressBound(srcSize). + Note : srcSize must be <= 128 KB + @return : size of compressed data (<= dstCapacity) + Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!! + if return == 1, srcData is a single repeated byte symbol (RLE compression). + if HUF_isError(return), compression failed (more details using HUF_getErrorName()) + +HUF_decompress() : + Decompress HUF data from buffer 'cSrc', of size 'cSrcSize', + into already allocated destination buffer 'dst', of size 'dstSize'. + `dstSize` : must be the **exact** size of original (uncompressed) data. + Note : in contrast with FSE, HUF_decompress can regenerate + RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data, + because it knows size to regenerate. + @return : size of regenerated data (== dstSize) + or an error code, which can be tested using HUF_isError() +*/ + + +/* **************************************** +* Tool functions +******************************************/ +size_t HUF_compressBound(size_t size); /**< maximum compressed size */ + +/* Error Management */ +unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */ +const char* HUF_getErrorName(size_t code); /**< provides error code string (useful for debugging) */ + + +/* **************************************** +* Advanced functions +******************************************/ +size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); + + +#if defined (__cplusplus) +} +#endif + +#endif /* HUF_H */ diff --git a/contrib/libs/zstd06/common/huf_static.h b/contrib/libs/zstd06/common/huf_static.h index 68a238e3ab..8c81b6a503 100644 --- a/contrib/libs/zstd06/common/huf_static.h +++ b/contrib/libs/zstd06/common/huf_static.h @@ -1,228 +1,228 @@ -#include <contrib/libs/zstd06/renames.h> -/* ****************************************************************** - Huffman codec, part of New Generation Entropy library - header file, for static linking only - Copyright (C) 2013-2016, Yann Collet - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Source repository : https://github.com/Cyan4973/FiniteStateEntropy -****************************************************************** */ -#ifndef HUF_STATIC_H -#define HUF_STATIC_H - -#if defined (__cplusplus) -extern "C" { -#endif - - -/* **************************************** -* Dependency -******************************************/ -#include "huf.h" -#include "fse.h" -#include "bitstream.h" - - -/* **************************************** -* Static allocation -******************************************/ -/* HUF buffer bounds */ -#define HUF_CTABLEBOUND 129 -#define HUF_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true if incompressible pre-filtered with fast heuristic */ -#define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ - -/* static allocation of HUF's Compression Table */ -#define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \ - U32 name##hb[maxSymbolValue+1]; \ - void* name##hv = &(name##hb); \ - HUF_CElt* name = (HUF_CElt*)(name##hv) /* no final ; */ - -/* static allocation of HUF's DTable */ -#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1<<maxTableLog)) -#define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \ - unsigned short DTable[HUF_DTABLE_SIZE(maxTableLog)] = { maxTableLog } -#define HUF_CREATE_STATIC_DTABLEX4(DTable, maxTableLog) \ - unsigned int DTable[HUF_DTABLE_SIZE(maxTableLog)] = { maxTableLog } -#define HUF_CREATE_STATIC_DTABLEX6(DTable, maxTableLog) \ - unsigned int DTable[HUF_DTABLE_SIZE(maxTableLog) * 3 / 2] = { maxTableLog } - - -/* **************************************** -* Advanced decompression functions -******************************************/ -size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */ -size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbols decoder */ -size_t HUF_decompress4X6 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* quad-symbols decoder, only works for dstSize >= 64 */ - - -/* **************************************** -* HUF detailed API -******************************************/ -/*! -HUF_compress() does the following: -1. count symbol occurrence from source[] into table count[] using FSE_count() -2. build Huffman table from count using HUF_buildCTable() -3. save Huffman table to memory buffer using HUF_writeCTable() -4. encode the data stream using HUF_compress4X_usingCTable() - -The following API allows targeting specific sub-functions for advanced tasks. -For example, it's possible to compress several blocks using the same 'CTable', -or to save and regenerate 'CTable' using external methods. -*/ -/* FSE_count() : find it within "fse.h" */ -typedef struct HUF_CElt_s HUF_CElt; /* incomplete type */ -size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits); -size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog); -size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); - - -/*! -HUF_decompress() does the following: -1. select the decompression algorithm (X2, X4, X6) based on pre-computed heuristics -2. build Huffman table from save, using HUF_readDTableXn() -3. decode 1 or 4 segments in parallel using HUF_decompressSXn_usingDTable -*/ -size_t HUF_readDTableX2 (unsigned short* DTable, const void* src, size_t srcSize); -size_t HUF_readDTableX4 (unsigned* DTable, const void* src, size_t srcSize); -size_t HUF_readDTableX6 (unsigned* DTable, const void* src, size_t srcSize); - -size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned short* DTable); -size_t HUF_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable); -size_t HUF_decompress4X6_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable); - - -/* single stream variants */ - -size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); -size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); - -size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */ -size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */ -size_t HUF_decompress1X6 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* quad-symbols decoder, only works for dstSize >= 64 */ - -size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned short* DTable); -size_t HUF_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable); -size_t HUF_decompress1X6_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable); - - -/* Loading a CTable saved with HUF_writeCTable() */ - -size_t HUF_readCTable (HUF_CElt* CTable, unsigned maxSymbolValue, const void* src, size_t srcSize); - - -/* ************************************************************** -* Constants -****************************************************************/ -#define HUF_ABSOLUTEMAX_TABLELOG 16 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */ -#define HUF_MAX_TABLELOG 12 /* max configured tableLog (for static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */ -#define HUF_DEFAULT_TABLELOG HUF_MAX_TABLELOG /* tableLog by default, when not specified */ -#define HUF_MAX_SYMBOL_VALUE 255 -#if (HUF_MAX_TABLELOG > HUF_ABSOLUTEMAX_TABLELOG) -# error "HUF_MAX_TABLELOG is too large !" -#endif - - - -/*! HUF_readStats() : - Read compact Huffman tree, saved by HUF_writeCTable(). - `huffWeight` is destination buffer. - @return : size read from `src` -*/ -MEM_STATIC size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, - U32* nbSymbolsPtr, U32* tableLogPtr, - const void* src, size_t srcSize) -{ - U32 weightTotal; - const BYTE* ip = (const BYTE*) src; - size_t iSize = ip[0]; - size_t oSize; - - //memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */ - - if (iSize >= 128) { /* special header */ - if (iSize >= (242)) { /* RLE */ - static U32 l[14] = { 1, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128 }; - oSize = l[iSize-242]; - memset(huffWeight, 1, hwSize); - iSize = 0; - } - else { /* Incompressible */ - oSize = iSize - 127; - iSize = ((oSize+1)/2); - if (iSize+1 > srcSize) return ERROR(srcSize_wrong); - if (oSize >= hwSize) return ERROR(corruption_detected); - ip += 1; - { U32 n; - for (n=0; n<oSize; n+=2) { - huffWeight[n] = ip[n/2] >> 4; - huffWeight[n+1] = ip[n/2] & 15; - } } } } - else { /* header compressed with FSE (normal case) */ - if (iSize+1 > srcSize) return ERROR(srcSize_wrong); - oSize = FSE_decompress(huffWeight, hwSize-1, ip+1, iSize); /* max (hwSize-1) values decoded, as last one is implied */ - if (FSE_isError(oSize)) return oSize; - } - - /* collect weight stats */ - memset(rankStats, 0, (HUF_ABSOLUTEMAX_TABLELOG + 1) * sizeof(U32)); - weightTotal = 0; - { U32 n; for (n=0; n<oSize; n++) { - if (huffWeight[n] >= HUF_ABSOLUTEMAX_TABLELOG) return ERROR(corruption_detected); - rankStats[huffWeight[n]]++; - weightTotal += (1 << huffWeight[n]) >> 1; - } } - - /* get last non-null symbol weight (implied, total must be 2^n) */ - { U32 const tableLog = BIT_highbit32(weightTotal) + 1; - if (tableLog > HUF_ABSOLUTEMAX_TABLELOG) return ERROR(corruption_detected); - *tableLogPtr = tableLog; - /* determine last weight */ - { U32 const total = 1 << tableLog; - U32 const rest = total - weightTotal; - U32 const verif = 1 << BIT_highbit32(rest); - U32 const lastWeight = BIT_highbit32(rest) + 1; - if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */ - huffWeight[oSize] = (BYTE)lastWeight; - rankStats[lastWeight]++; - } } - - /* check tree construction validity */ - if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */ - - /* results */ - *nbSymbolsPtr = (U32)(oSize+1); - return iSize+1; -} - - - -#if defined (__cplusplus) -} -#endif - -#endif /* HUF_STATIC_H */ +#include <contrib/libs/zstd06/renames.h> +/* ****************************************************************** + Huffman codec, part of New Generation Entropy library + header file, for static linking only + Copyright (C) 2013-2016, Yann Collet + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - Source repository : https://github.com/Cyan4973/FiniteStateEntropy +****************************************************************** */ +#ifndef HUF_STATIC_H +#define HUF_STATIC_H + +#if defined (__cplusplus) +extern "C" { +#endif + + +/* **************************************** +* Dependency +******************************************/ +#include "huf.h" +#include "fse.h" +#include "bitstream.h" + + +/* **************************************** +* Static allocation +******************************************/ +/* HUF buffer bounds */ +#define HUF_CTABLEBOUND 129 +#define HUF_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true if incompressible pre-filtered with fast heuristic */ +#define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ + +/* static allocation of HUF's Compression Table */ +#define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \ + U32 name##hb[maxSymbolValue+1]; \ + void* name##hv = &(name##hb); \ + HUF_CElt* name = (HUF_CElt*)(name##hv) /* no final ; */ + +/* static allocation of HUF's DTable */ +#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1<<maxTableLog)) +#define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \ + unsigned short DTable[HUF_DTABLE_SIZE(maxTableLog)] = { maxTableLog } +#define HUF_CREATE_STATIC_DTABLEX4(DTable, maxTableLog) \ + unsigned int DTable[HUF_DTABLE_SIZE(maxTableLog)] = { maxTableLog } +#define HUF_CREATE_STATIC_DTABLEX6(DTable, maxTableLog) \ + unsigned int DTable[HUF_DTABLE_SIZE(maxTableLog) * 3 / 2] = { maxTableLog } + + +/* **************************************** +* Advanced decompression functions +******************************************/ +size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */ +size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbols decoder */ +size_t HUF_decompress4X6 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* quad-symbols decoder, only works for dstSize >= 64 */ + + +/* **************************************** +* HUF detailed API +******************************************/ +/*! +HUF_compress() does the following: +1. count symbol occurrence from source[] into table count[] using FSE_count() +2. build Huffman table from count using HUF_buildCTable() +3. save Huffman table to memory buffer using HUF_writeCTable() +4. encode the data stream using HUF_compress4X_usingCTable() + +The following API allows targeting specific sub-functions for advanced tasks. +For example, it's possible to compress several blocks using the same 'CTable', +or to save and regenerate 'CTable' using external methods. +*/ +/* FSE_count() : find it within "fse.h" */ +typedef struct HUF_CElt_s HUF_CElt; /* incomplete type */ +size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits); +size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog); +size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); + + +/*! +HUF_decompress() does the following: +1. select the decompression algorithm (X2, X4, X6) based on pre-computed heuristics +2. build Huffman table from save, using HUF_readDTableXn() +3. decode 1 or 4 segments in parallel using HUF_decompressSXn_usingDTable +*/ +size_t HUF_readDTableX2 (unsigned short* DTable, const void* src, size_t srcSize); +size_t HUF_readDTableX4 (unsigned* DTable, const void* src, size_t srcSize); +size_t HUF_readDTableX6 (unsigned* DTable, const void* src, size_t srcSize); + +size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned short* DTable); +size_t HUF_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable); +size_t HUF_decompress4X6_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable); + + +/* single stream variants */ + +size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); +size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); + +size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */ +size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */ +size_t HUF_decompress1X6 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* quad-symbols decoder, only works for dstSize >= 64 */ + +size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned short* DTable); +size_t HUF_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable); +size_t HUF_decompress1X6_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const unsigned* DTable); + + +/* Loading a CTable saved with HUF_writeCTable() */ + +size_t HUF_readCTable (HUF_CElt* CTable, unsigned maxSymbolValue, const void* src, size_t srcSize); + + +/* ************************************************************** +* Constants +****************************************************************/ +#define HUF_ABSOLUTEMAX_TABLELOG 16 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */ +#define HUF_MAX_TABLELOG 12 /* max configured tableLog (for static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */ +#define HUF_DEFAULT_TABLELOG HUF_MAX_TABLELOG /* tableLog by default, when not specified */ +#define HUF_MAX_SYMBOL_VALUE 255 +#if (HUF_MAX_TABLELOG > HUF_ABSOLUTEMAX_TABLELOG) +# error "HUF_MAX_TABLELOG is too large !" +#endif + + + +/*! HUF_readStats() : + Read compact Huffman tree, saved by HUF_writeCTable(). + `huffWeight` is destination buffer. + @return : size read from `src` +*/ +MEM_STATIC size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize) +{ + U32 weightTotal; + const BYTE* ip = (const BYTE*) src; + size_t iSize = ip[0]; + size_t oSize; + + //memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */ + + if (iSize >= 128) { /* special header */ + if (iSize >= (242)) { /* RLE */ + static U32 l[14] = { 1, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128 }; + oSize = l[iSize-242]; + memset(huffWeight, 1, hwSize); + iSize = 0; + } + else { /* Incompressible */ + oSize = iSize - 127; + iSize = ((oSize+1)/2); + if (iSize+1 > srcSize) return ERROR(srcSize_wrong); + if (oSize >= hwSize) return ERROR(corruption_detected); + ip += 1; + { U32 n; + for (n=0; n<oSize; n+=2) { + huffWeight[n] = ip[n/2] >> 4; + huffWeight[n+1] = ip[n/2] & 15; + } } } } + else { /* header compressed with FSE (normal case) */ + if (iSize+1 > srcSize) return ERROR(srcSize_wrong); + oSize = FSE_decompress(huffWeight, hwSize-1, ip+1, iSize); /* max (hwSize-1) values decoded, as last one is implied */ + if (FSE_isError(oSize)) return oSize; + } + + /* collect weight stats */ + memset(rankStats, 0, (HUF_ABSOLUTEMAX_TABLELOG + 1) * sizeof(U32)); + weightTotal = 0; + { U32 n; for (n=0; n<oSize; n++) { + if (huffWeight[n] >= HUF_ABSOLUTEMAX_TABLELOG) return ERROR(corruption_detected); + rankStats[huffWeight[n]]++; + weightTotal += (1 << huffWeight[n]) >> 1; + } } + + /* get last non-null symbol weight (implied, total must be 2^n) */ + { U32 const tableLog = BIT_highbit32(weightTotal) + 1; + if (tableLog > HUF_ABSOLUTEMAX_TABLELOG) return ERROR(corruption_detected); + *tableLogPtr = tableLog; + /* determine last weight */ + { U32 const total = 1 << tableLog; + U32 const rest = total - weightTotal; + U32 const verif = 1 << BIT_highbit32(rest); + U32 const lastWeight = BIT_highbit32(rest) + 1; + if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */ + huffWeight[oSize] = (BYTE)lastWeight; + rankStats[lastWeight]++; + } } + + /* check tree construction validity */ + if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */ + + /* results */ + *nbSymbolsPtr = (U32)(oSize+1); + return iSize+1; +} + + + +#if defined (__cplusplus) +} +#endif + +#endif /* HUF_STATIC_H */ diff --git a/contrib/libs/zstd06/common/mem.h b/contrib/libs/zstd06/common/mem.h index 67a0f66842..85d78c84c1 100644 --- a/contrib/libs/zstd06/common/mem.h +++ b/contrib/libs/zstd06/common/mem.h @@ -1,4 +1,4 @@ -#include <contrib/libs/zstd06/renames.h> +#include <contrib/libs/zstd06/renames.h> /* ****************************************************************** mem.h low-level memory access routines @@ -43,17 +43,17 @@ extern "C" { /*-**************************************** * Dependencies ******************************************/ -#include <stddef.h> /* size_t, ptrdiff_t */ -#include <string.h> /* memcpy */ +#include <stddef.h> /* size_t, ptrdiff_t */ +#include <string.h> /* memcpy */ /*-**************************************** * Compiler specifics ******************************************/ -#if defined(_MSC_VER) /* Visual Studio */ -# include <stdlib.h> /* _byteswap_ulong */ -# include <intrin.h> /* _byteswap_* */ -#endif +#if defined(_MSC_VER) /* Visual Studio */ +# include <stdlib.h> /* _byteswap_ulong */ +# include <intrin.h> /* _byteswap_* */ +#endif #if defined(__GNUC__) # define MEM_STATIC static __attribute__((unused)) #elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) @@ -64,15 +64,15 @@ extern "C" { # define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ #endif -/* code only tested on 32 and 64 bits systems */ -#define MEM_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; } -MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); } +/* code only tested on 32 and 64 bits systems */ +#define MEM_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; } +MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); } + - /*-************************************************************** * Basic Types *****************************************************************/ -#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) # include <stdint.h> typedef uint8_t BYTE; typedef uint16_t U16; @@ -117,8 +117,8 @@ MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (size # endif #endif -MEM_STATIC unsigned MEM_32bits(void) { return sizeof(size_t)==4; } -MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; } +MEM_STATIC unsigned MEM_32bits(void) { return sizeof(size_t)==4; } +MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; } MEM_STATIC unsigned MEM_isLittleEndian(void) { @@ -196,48 +196,48 @@ MEM_STATIC void MEM_write64(void* memPtr, U64 value) #endif /* MEM_FORCE_MEMORY_ACCESS */ -MEM_STATIC U32 MEM_swap32(U32 in) -{ -#if defined(_MSC_VER) /* Visual Studio */ - return _byteswap_ulong(in); -#elif defined (__GNUC__) - return __builtin_bswap32(in); -#else - return ((in << 24) & 0xff000000 ) | - ((in << 8) & 0x00ff0000 ) | - ((in >> 8) & 0x0000ff00 ) | - ((in >> 24) & 0x000000ff ); -#endif -} - -MEM_STATIC U64 MEM_swap64(U64 in) -{ -#if defined(_MSC_VER) /* Visual Studio */ - return _byteswap_uint64(in); -#elif defined (__GNUC__) - return __builtin_bswap64(in); -#else - return ((in << 56) & 0xff00000000000000ULL) | - ((in << 40) & 0x00ff000000000000ULL) | - ((in << 24) & 0x0000ff0000000000ULL) | - ((in << 8) & 0x000000ff00000000ULL) | - ((in >> 8) & 0x00000000ff000000ULL) | - ((in >> 24) & 0x0000000000ff0000ULL) | - ((in >> 40) & 0x000000000000ff00ULL) | - ((in >> 56) & 0x00000000000000ffULL); -#endif -} - -MEM_STATIC size_t MEM_swapST(size_t in) -{ - if (MEM_32bits()) - return (size_t)MEM_swap32((U32)in); - else - return (size_t)MEM_swap64((U64)in); -} - -/*=== Little endian r/w ===*/ - +MEM_STATIC U32 MEM_swap32(U32 in) +{ +#if defined(_MSC_VER) /* Visual Studio */ + return _byteswap_ulong(in); +#elif defined (__GNUC__) + return __builtin_bswap32(in); +#else + return ((in << 24) & 0xff000000 ) | + ((in << 8) & 0x00ff0000 ) | + ((in >> 8) & 0x0000ff00 ) | + ((in >> 24) & 0x000000ff ); +#endif +} + +MEM_STATIC U64 MEM_swap64(U64 in) +{ +#if defined(_MSC_VER) /* Visual Studio */ + return _byteswap_uint64(in); +#elif defined (__GNUC__) + return __builtin_bswap64(in); +#else + return ((in << 56) & 0xff00000000000000ULL) | + ((in << 40) & 0x00ff000000000000ULL) | + ((in << 24) & 0x0000ff0000000000ULL) | + ((in << 8) & 0x000000ff00000000ULL) | + ((in >> 8) & 0x00000000ff000000ULL) | + ((in >> 24) & 0x0000000000ff0000ULL) | + ((in >> 40) & 0x000000000000ff00ULL) | + ((in >> 56) & 0x00000000000000ffULL); +#endif +} + +MEM_STATIC size_t MEM_swapST(size_t in) +{ + if (MEM_32bits()) + return (size_t)MEM_swap32((U32)in); + else + return (size_t)MEM_swap64((U64)in); +} + +/*=== Little endian r/w ===*/ + MEM_STATIC U16 MEM_readLE16(const void* memPtr) { if (MEM_isLittleEndian()) @@ -259,47 +259,47 @@ MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val) } } -MEM_STATIC U32 MEM_readLE24(const void* memPtr) -{ - return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16); -} - -MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val) -{ - MEM_writeLE16(memPtr, (U16)val); - ((BYTE*)memPtr)[2] = (BYTE)(val>>16); -} - +MEM_STATIC U32 MEM_readLE24(const void* memPtr) +{ + return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16); +} + +MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val) +{ + MEM_writeLE16(memPtr, (U16)val); + ((BYTE*)memPtr)[2] = (BYTE)(val>>16); +} + MEM_STATIC U32 MEM_readLE32(const void* memPtr) { if (MEM_isLittleEndian()) return MEM_read32(memPtr); - else - return MEM_swap32(MEM_read32(memPtr)); + else + return MEM_swap32(MEM_read32(memPtr)); } MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32) { - if (MEM_isLittleEndian()) + if (MEM_isLittleEndian()) MEM_write32(memPtr, val32); - else - MEM_write32(memPtr, MEM_swap32(val32)); + else + MEM_write32(memPtr, MEM_swap32(val32)); } MEM_STATIC U64 MEM_readLE64(const void* memPtr) { if (MEM_isLittleEndian()) return MEM_read64(memPtr); - else - return MEM_swap64(MEM_read64(memPtr)); + else + return MEM_swap64(MEM_read64(memPtr)); } MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64) { - if (MEM_isLittleEndian()) + if (MEM_isLittleEndian()) MEM_write64(memPtr, val64); - else - MEM_write64(memPtr, MEM_swap64(val64)); + else + MEM_write64(memPtr, MEM_swap64(val64)); } MEM_STATIC size_t MEM_readLEST(const void* memPtr) @@ -318,58 +318,58 @@ MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val) MEM_writeLE64(memPtr, (U64)val); } -/*=== Big endian r/w ===*/ - -MEM_STATIC U32 MEM_readBE32(const void* memPtr) -{ - if (MEM_isLittleEndian()) - return MEM_swap32(MEM_read32(memPtr)); - else - return MEM_read32(memPtr); -} - -MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32) -{ - if (MEM_isLittleEndian()) - MEM_write32(memPtr, MEM_swap32(val32)); - else - MEM_write32(memPtr, val32); -} - -MEM_STATIC U64 MEM_readBE64(const void* memPtr) -{ - if (MEM_isLittleEndian()) - return MEM_swap64(MEM_read64(memPtr)); - else - return MEM_read64(memPtr); -} - -MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64) -{ - if (MEM_isLittleEndian()) - MEM_write64(memPtr, MEM_swap64(val64)); - else - MEM_write64(memPtr, val64); -} - -MEM_STATIC size_t MEM_readBEST(const void* memPtr) -{ - if (MEM_32bits()) - return (size_t)MEM_readBE32(memPtr); - else - return (size_t)MEM_readBE64(memPtr); -} - -MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val) -{ - if (MEM_32bits()) - MEM_writeBE32(memPtr, (U32)val); - else - MEM_writeBE64(memPtr, (U64)val); -} - - -/* function safe only for comparisons */ +/*=== Big endian r/w ===*/ + +MEM_STATIC U32 MEM_readBE32(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_swap32(MEM_read32(memPtr)); + else + return MEM_read32(memPtr); +} + +MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32) +{ + if (MEM_isLittleEndian()) + MEM_write32(memPtr, MEM_swap32(val32)); + else + MEM_write32(memPtr, val32); +} + +MEM_STATIC U64 MEM_readBE64(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_swap64(MEM_read64(memPtr)); + else + return MEM_read64(memPtr); +} + +MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64) +{ + if (MEM_isLittleEndian()) + MEM_write64(memPtr, MEM_swap64(val64)); + else + MEM_write64(memPtr, val64); +} + +MEM_STATIC size_t MEM_readBEST(const void* memPtr) +{ + if (MEM_32bits()) + return (size_t)MEM_readBE32(memPtr); + else + return (size_t)MEM_readBE64(memPtr); +} + +MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val) +{ + if (MEM_32bits()) + MEM_writeBE32(memPtr, (U32)val); + else + MEM_writeBE64(memPtr, (U64)val); +} + + +/* function safe only for comparisons */ MEM_STATIC U32 MEM_readMINMATCH(const void* memPtr, U32 length) { switch (length) diff --git a/contrib/libs/zstd06/common/xxhash.c b/contrib/libs/zstd06/common/xxhash.c index ab1be712e1..e175ae96eb 100644 --- a/contrib/libs/zstd06/common/xxhash.c +++ b/contrib/libs/zstd06/common/xxhash.c @@ -1,854 +1,854 @@ -/* -* xxHash - Fast Hash algorithm -* Copyright (C) 2012-2016, Yann Collet -* -* BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) -* -* Redistribution and use in source and binary forms, with or without -* modification, are permitted provided that the following conditions are -* met: -* -* * Redistributions of source code must retain the above copyright -* notice, this list of conditions and the following disclaimer. -* * Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following disclaimer -* in the documentation and/or other materials provided with the -* distribution. -* -* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -* -* You can contact the author at : -* - xxHash homepage: http://www.xxhash.com -* - xxHash source repository : https://github.com/Cyan4973/xxHash -*/ - - -/* ************************************* -* Tuning parameters -***************************************/ -/*!XXH_FORCE_MEMORY_ACCESS : - * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. - * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. - * The below switch allow to select different access method for improved performance. - * Method 0 (default) : use `memcpy()`. Safe and portable. - * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). - * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. - * Method 2 : direct access. This method doesn't depend on compiler but violate C standard. - * It can generate buggy code on targets which do not support unaligned memory accesses. - * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) - * See http://stackoverflow.com/a/32095106/646947 for details. - * Prefer these methods in priority order (0 > 1 > 2) - */ -#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ -# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) -# define XXH_FORCE_MEMORY_ACCESS 2 -# elif defined(__INTEL_COMPILER) || \ - (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) -# define XXH_FORCE_MEMORY_ACCESS 1 -# endif -#endif - -/*!XXH_ACCEPT_NULL_INPUT_POINTER : - * If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer. - * When this option is enabled, xxHash output for null input pointers will be the same as a null-length input. - * By default, this option is disabled. To enable it, uncomment below define : - */ -/* #define XXH_ACCEPT_NULL_INPUT_POINTER 1 */ - -/*!XXH_FORCE_NATIVE_FORMAT : - * By default, xxHash library provides endian-independant Hash values, based on little-endian convention. - * Results are therefore identical for little-endian and big-endian CPU. - * This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format. - * Should endian-independance be of no importance for your application, you may set the #define below to 1, - * to improve speed for Big-endian CPU. - * This option has no impact on Little_Endian CPU. - */ -#ifndef XXH_FORCE_NATIVE_FORMAT /* can be defined externally */ -# define XXH_FORCE_NATIVE_FORMAT 0 -#endif - -/*!XXH_FORCE_ALIGN_CHECK : - * This is a minor performance trick, only useful with lots of very small keys. - * It means : check for aligned/unaligned input. - * The check costs one initial branch per hash; set to 0 when the input data - * is guaranteed to be aligned. - */ -#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ -# if defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64) -# define XXH_FORCE_ALIGN_CHECK 0 -# else -# define XXH_FORCE_ALIGN_CHECK 1 -# endif -#endif - - -/* ************************************* -* Includes & Memory related functions -***************************************/ -/* Modify the local functions below should you wish to use some other memory routines */ -/* for malloc(), free() */ -#include <stdlib.h> -static void* XXH_malloc(size_t s) { return malloc(s); } -static void XXH_free (void* p) { free(p); } -/* for memcpy() */ -#include <string.h> -static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); } - -#define XXH_STATIC_LINKING_ONLY -#include "xxhash.h" - - -/* ************************************* -* Compiler Specific Options -***************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -# define FORCE_INLINE static __forceinline -#else -# if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ -# ifdef __GNUC__ -# define FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define FORCE_INLINE static inline -# endif -# else -# define FORCE_INLINE static -# endif /* __STDC_VERSION__ */ -#endif - - -/* ************************************* -* Basic Types -***************************************/ -#ifndef MEM_MODULE -# define MEM_MODULE -# if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ -# include <stdint.h> - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; -# else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef unsigned int U32; - typedef signed int S32; - typedef unsigned long long U64; -# endif -#endif - - -#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) - -/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ -static U32 XXH_read32(const void* memPtr) { return *(const U32*) memPtr; } -static U64 XXH_read64(const void* memPtr) { return *(const U64*) memPtr; } - -#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) - -/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ -/* currently only defined for gcc and icc */ -typedef union { U32 u32; U64 u64; } __attribute__((packed)) unalign; - -static U32 XXH_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } -static U64 XXH_read64(const void* ptr) { return ((const unalign*)ptr)->u64; } - -#else - -/* portable and safe solution. Generally efficient. - * see : http://stackoverflow.com/a/32095106/646947 - */ - -static U32 XXH_read32(const void* memPtr) -{ - U32 val; - memcpy(&val, memPtr, sizeof(val)); - return val; -} - -static U64 XXH_read64(const void* memPtr) -{ - U64 val; - memcpy(&val, memPtr, sizeof(val)); - return val; -} - -#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ - - -/* **************************************** -* Compiler-specific Functions and Macros -******************************************/ -#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) - -/* Note : although _rotl exists for minGW (GCC under windows), performance seems poor */ -#if defined(_MSC_VER) -# define XXH_rotl32(x,r) _rotl(x,r) -# define XXH_rotl64(x,r) _rotl64(x,r) -#else -# define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r))) -# define XXH_rotl64(x,r) ((x << r) | (x >> (64 - r))) -#endif - -#if defined(_MSC_VER) /* Visual Studio */ -# define XXH_swap32 _byteswap_ulong -# define XXH_swap64 _byteswap_uint64 -#elif GCC_VERSION >= 403 -# define XXH_swap32 __builtin_bswap32 -# define XXH_swap64 __builtin_bswap64 -#else -static U32 XXH_swap32 (U32 x) -{ - return ((x << 24) & 0xff000000 ) | - ((x << 8) & 0x00ff0000 ) | - ((x >> 8) & 0x0000ff00 ) | - ((x >> 24) & 0x000000ff ); -} -static U64 XXH_swap64 (U64 x) -{ - return ((x << 56) & 0xff00000000000000ULL) | - ((x << 40) & 0x00ff000000000000ULL) | - ((x << 24) & 0x0000ff0000000000ULL) | - ((x << 8) & 0x000000ff00000000ULL) | - ((x >> 8) & 0x00000000ff000000ULL) | - ((x >> 24) & 0x0000000000ff0000ULL) | - ((x >> 40) & 0x000000000000ff00ULL) | - ((x >> 56) & 0x00000000000000ffULL); -} -#endif - - -/* ************************************* -* Architecture Macros -***************************************/ -typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; - -/* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example on the compiler command line */ -#ifndef XXH_CPU_LITTLE_ENDIAN - static const int g_one = 1; -# define XXH_CPU_LITTLE_ENDIAN (*(const char*)(&g_one)) -#endif - - -/* *************************** -* Memory reads -*****************************/ -typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; - -FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align) -{ - if (align==XXH_unaligned) - return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); - else - return endian==XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr); -} - -FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian) -{ - return XXH_readLE32_align(ptr, endian, XXH_unaligned); -} - -static U32 XXH_readBE32(const void* ptr) -{ - return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr); -} - -FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align) -{ - if (align==XXH_unaligned) - return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); - else - return endian==XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr); -} - -FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian) -{ - return XXH_readLE64_align(ptr, endian, XXH_unaligned); -} - -static U64 XXH_readBE64(const void* ptr) -{ - return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr); -} - - -/* ************************************* -* Macros -***************************************/ -#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ - - -/* ************************************* -* Constants -***************************************/ -static const U32 PRIME32_1 = 2654435761U; -static const U32 PRIME32_2 = 2246822519U; -static const U32 PRIME32_3 = 3266489917U; -static const U32 PRIME32_4 = 668265263U; -static const U32 PRIME32_5 = 374761393U; - -static const U64 PRIME64_1 = 11400714785074694791ULL; -static const U64 PRIME64_2 = 14029467366897019727ULL; -static const U64 PRIME64_3 = 1609587929392839161ULL; -static const U64 PRIME64_4 = 9650029242287828579ULL; -static const U64 PRIME64_5 = 2870177450012600261ULL; - -XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } - - -/* *************************** -* Simple Hash Functions -*****************************/ - -static U32 XXH32_round(U32 seed, U32 input) -{ - seed += input * PRIME32_2; - seed = XXH_rotl32(seed, 13); - seed *= PRIME32_1; - return seed; -} - -FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align) -{ - const BYTE* p = (const BYTE*)input; - const BYTE* bEnd = p + len; - U32 h32; -#define XXH_get32bits(p) XXH_readLE32_align(p, endian, align) - -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (p==NULL) { - len=0; - bEnd=p=(const BYTE*)(size_t)16; - } -#endif - - if (len>=16) { - const BYTE* const limit = bEnd - 16; - U32 v1 = seed + PRIME32_1 + PRIME32_2; - U32 v2 = seed + PRIME32_2; - U32 v3 = seed + 0; - U32 v4 = seed - PRIME32_1; - - do { - v1 = XXH32_round(v1, XXH_get32bits(p)); p+=4; - v2 = XXH32_round(v2, XXH_get32bits(p)); p+=4; - v3 = XXH32_round(v3, XXH_get32bits(p)); p+=4; - v4 = XXH32_round(v4, XXH_get32bits(p)); p+=4; - } while (p<=limit); - - h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); - } else { - h32 = seed + PRIME32_5; - } - - h32 += (U32) len; - - while (p+4<=bEnd) { - h32 += XXH_get32bits(p) * PRIME32_3; - h32 = XXH_rotl32(h32, 17) * PRIME32_4 ; - p+=4; - } - - while (p<bEnd) { - h32 += (*p) * PRIME32_5; - h32 = XXH_rotl32(h32, 11) * PRIME32_1 ; - p++; - } - - h32 ^= h32 >> 15; - h32 *= PRIME32_2; - h32 ^= h32 >> 13; - h32 *= PRIME32_3; - h32 ^= h32 >> 16; - - return h32; -} - - -XXH_PUBLIC_API unsigned int XXH32 (const void* input, size_t len, unsigned int seed) -{ -#if 0 - /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ - XXH32_CREATESTATE_STATIC(state); - XXH32_reset(state, seed); - XXH32_update(state, input, len); - return XXH32_digest(state); -#else - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if (XXH_FORCE_ALIGN_CHECK) { - if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */ - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); - else - return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); - } } - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); - else - return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); -#endif -} - - -static U64 XXH64_round(U64 acc, U64 input) -{ - acc += input * PRIME64_2; - acc = XXH_rotl64(acc, 31); - acc *= PRIME64_1; - return acc; -} - -static U64 XXH64_mergeRound(U64 acc, U64 val) -{ - val = XXH64_round(0, val); - acc ^= val; - acc = acc * PRIME64_1 + PRIME64_4; - return acc; -} - -FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align) -{ - const BYTE* p = (const BYTE*)input; - const BYTE* const bEnd = p + len; - U64 h64; -#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align) - -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (p==NULL) { - len=0; - bEnd=p=(const BYTE*)(size_t)32; - } -#endif - - if (len>=32) { - const BYTE* const limit = bEnd - 32; - U64 v1 = seed + PRIME64_1 + PRIME64_2; - U64 v2 = seed + PRIME64_2; - U64 v3 = seed + 0; - U64 v4 = seed - PRIME64_1; - - do { - v1 = XXH64_round(v1, XXH_get64bits(p)); p+=8; - v2 = XXH64_round(v2, XXH_get64bits(p)); p+=8; - v3 = XXH64_round(v3, XXH_get64bits(p)); p+=8; - v4 = XXH64_round(v4, XXH_get64bits(p)); p+=8; - } while (p<=limit); - - h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); - h64 = XXH64_mergeRound(h64, v1); - h64 = XXH64_mergeRound(h64, v2); - h64 = XXH64_mergeRound(h64, v3); - h64 = XXH64_mergeRound(h64, v4); - - } else { - h64 = seed + PRIME64_5; - } - - h64 += (U64) len; - - while (p+8<=bEnd) { - U64 const k1 = XXH64_round(0, XXH_get64bits(p)); - h64 ^= k1; - h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; - p+=8; - } - - if (p+4<=bEnd) { - h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1; - h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; - p+=4; - } - - while (p<bEnd) { - h64 ^= (*p) * PRIME64_5; - h64 = XXH_rotl64(h64, 11) * PRIME64_1; - p++; - } - - h64 ^= h64 >> 33; - h64 *= PRIME64_2; - h64 ^= h64 >> 29; - h64 *= PRIME64_3; - h64 ^= h64 >> 32; - - return h64; -} - - -XXH_PUBLIC_API unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed) -{ -#if 0 - /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ - XXH64_CREATESTATE_STATIC(state); - XXH64_reset(state, seed); - XXH64_update(state, input, len); - return XXH64_digest(state); -#else - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if (XXH_FORCE_ALIGN_CHECK) { - if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */ - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); - else - return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); - } } - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); - else - return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); -#endif -} - - -/* ************************************************** -* Advanced Hash Functions -****************************************************/ - -XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void) -{ - return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); -} -XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) -{ - XXH_free(statePtr); - return XXH_OK; -} - -XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void) -{ - return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); -} -XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) -{ - XXH_free(statePtr); - return XXH_OK; -} - - -/*** Hash feed ***/ - -XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, unsigned int seed) -{ - XXH32_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ - memset(&state, 0, sizeof(state)); - state.seed = seed; - state.v1 = seed + PRIME32_1 + PRIME32_2; - state.v2 = seed + PRIME32_2; - state.v3 = seed + 0; - state.v4 = seed - PRIME32_1; - memcpy(statePtr, &state, sizeof(state)); - return XXH_OK; -} - - -XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, unsigned long long seed) -{ - XXH64_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ - memset(&state, 0, sizeof(state)); - state.seed = seed; - state.v1 = seed + PRIME64_1 + PRIME64_2; - state.v2 = seed + PRIME64_2; - state.v3 = seed + 0; - state.v4 = seed - PRIME64_1; - memcpy(statePtr, &state, sizeof(state)); - return XXH_OK; -} - - -FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state, const void* input, size_t len, XXH_endianess endian) -{ - const BYTE* p = (const BYTE*)input; - const BYTE* const bEnd = p + len; - -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (input==NULL) return XXH_ERROR; -#endif - - state->total_len += len; - - if (state->memsize + len < 16) { /* fill in tmp buffer */ - XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len); - state->memsize += (U32)len; - return XXH_OK; - } - - if (state->memsize) { /* some data left from previous update */ - XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize); - { const U32* p32 = state->mem32; - state->v1 = XXH32_round(state->v1, XXH_readLE32(p32, endian)); p32++; - state->v2 = XXH32_round(state->v2, XXH_readLE32(p32, endian)); p32++; - state->v3 = XXH32_round(state->v3, XXH_readLE32(p32, endian)); p32++; - state->v4 = XXH32_round(state->v4, XXH_readLE32(p32, endian)); p32++; - } - p += 16-state->memsize; - state->memsize = 0; - } - - if (p <= bEnd-16) { - const BYTE* const limit = bEnd - 16; - U32 v1 = state->v1; - U32 v2 = state->v2; - U32 v3 = state->v3; - U32 v4 = state->v4; - - do { - v1 = XXH32_round(v1, XXH_readLE32(p, endian)); p+=4; - v2 = XXH32_round(v2, XXH_readLE32(p, endian)); p+=4; - v3 = XXH32_round(v3, XXH_readLE32(p, endian)); p+=4; - v4 = XXH32_round(v4, XXH_readLE32(p, endian)); p+=4; - } while (p<=limit); - - state->v1 = v1; - state->v2 = v2; - state->v3 = v3; - state->v4 = v4; - } - - if (p < bEnd) { - XXH_memcpy(state->mem32, p, bEnd-p); - state->memsize = (int)(bEnd-p); - } - - return XXH_OK; -} - -XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_update_endian(state_in, input, len, XXH_littleEndian); - else - return XXH32_update_endian(state_in, input, len, XXH_bigEndian); -} - - - -FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian) -{ - const BYTE * p = (const BYTE*)state->mem32; - const BYTE* const bEnd = (const BYTE*)(state->mem32) + state->memsize; - U32 h32; - - if (state->total_len >= 16) { - h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18); - } else { - h32 = state->seed + PRIME32_5; - } - - h32 += (U32) state->total_len; - - while (p+4<=bEnd) { - h32 += XXH_readLE32(p, endian) * PRIME32_3; - h32 = XXH_rotl32(h32, 17) * PRIME32_4; - p+=4; - } - - while (p<bEnd) { - h32 += (*p) * PRIME32_5; - h32 = XXH_rotl32(h32, 11) * PRIME32_1; - p++; - } - - h32 ^= h32 >> 15; - h32 *= PRIME32_2; - h32 ^= h32 >> 13; - h32 *= PRIME32_3; - h32 ^= h32 >> 16; - - return h32; -} - - -XXH_PUBLIC_API unsigned int XXH32_digest (const XXH32_state_t* state_in) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_digest_endian(state_in, XXH_littleEndian); - else - return XXH32_digest_endian(state_in, XXH_bigEndian); -} - - - -/* **** XXH64 **** */ - -FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian) -{ - const BYTE* p = (const BYTE*)input; - const BYTE* const bEnd = p + len; - -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (input==NULL) return XXH_ERROR; -#endif - - state->total_len += len; - - if (state->memsize + len < 32) { /* fill in tmp buffer */ - XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len); - state->memsize += (U32)len; - return XXH_OK; - } - - if (state->memsize) { /* tmp buffer is full */ - XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize); - state->v1 = XXH64_round(state->v1, XXH_readLE64(state->mem64+0, endian)); - state->v2 = XXH64_round(state->v2, XXH_readLE64(state->mem64+1, endian)); - state->v3 = XXH64_round(state->v3, XXH_readLE64(state->mem64+2, endian)); - state->v4 = XXH64_round(state->v4, XXH_readLE64(state->mem64+3, endian)); - p += 32-state->memsize; - state->memsize = 0; - } - - if (p+32 <= bEnd) { - const BYTE* const limit = bEnd - 32; - U64 v1 = state->v1; - U64 v2 = state->v2; - U64 v3 = state->v3; - U64 v4 = state->v4; - - do { - v1 = XXH64_round(v1, XXH_readLE64(p, endian)); p+=8; - v2 = XXH64_round(v2, XXH_readLE64(p, endian)); p+=8; - v3 = XXH64_round(v3, XXH_readLE64(p, endian)); p+=8; - v4 = XXH64_round(v4, XXH_readLE64(p, endian)); p+=8; - } while (p<=limit); - - state->v1 = v1; - state->v2 = v2; - state->v3 = v3; - state->v4 = v4; - } - - if (p < bEnd) { - XXH_memcpy(state->mem64, p, bEnd-p); - state->memsize = (int)(bEnd-p); - } - - return XXH_OK; -} - -XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t len) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH64_update_endian(state_in, input, len, XXH_littleEndian); - else - return XXH64_update_endian(state_in, input, len, XXH_bigEndian); -} - - - -FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian) -{ - const BYTE * p = (const BYTE*)state->mem64; - const BYTE* const bEnd = (const BYTE*)state->mem64 + state->memsize; - U64 h64; - - if (state->total_len >= 32) { - U64 const v1 = state->v1; - U64 const v2 = state->v2; - U64 const v3 = state->v3; - U64 const v4 = state->v4; - - h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); - h64 = XXH64_mergeRound(h64, v1); - h64 = XXH64_mergeRound(h64, v2); - h64 = XXH64_mergeRound(h64, v3); - h64 = XXH64_mergeRound(h64, v4); - } else { - h64 = state->seed + PRIME64_5; - } - - h64 += (U64) state->total_len; - - while (p+8<=bEnd) { - U64 const k1 = XXH64_round(0, XXH_readLE64(p, endian)); - h64 ^= k1; - h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; - p+=8; - } - - if (p+4<=bEnd) { - h64 ^= (U64)(XXH_readLE32(p, endian)) * PRIME64_1; - h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; - p+=4; - } - - while (p<bEnd) { - h64 ^= (*p) * PRIME64_5; - h64 = XXH_rotl64(h64, 11) * PRIME64_1; - p++; - } - - h64 ^= h64 >> 33; - h64 *= PRIME64_2; - h64 ^= h64 >> 29; - h64 *= PRIME64_3; - h64 ^= h64 >> 32; - - return h64; -} - - -XXH_PUBLIC_API unsigned long long XXH64_digest (const XXH64_state_t* state_in) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH64_digest_endian(state_in, XXH_littleEndian); - else - return XXH64_digest_endian(state_in, XXH_bigEndian); -} - - -/* ************************** -* Canonical representation -****************************/ - -/*! Default XXH result types are basic unsigned 32 and 64 bits. -* The canonical representation follows human-readable write convention, aka big-endian (large digits first). -* These functions allow transformation of hash result into and from its canonical format. -* This way, hash values can be written into a file or buffer, and remain comparable across different systems and programs. -*/ - -XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash) -{ - XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); - if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash); - memcpy(dst, &hash, sizeof(*dst)); -} - -XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash) -{ - XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); - if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash); - memcpy(dst, &hash, sizeof(*dst)); -} - -XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src) -{ - return XXH_readBE32(src); -} - -XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src) -{ - return XXH_readBE64(src); -} +/* +* xxHash - Fast Hash algorithm +* Copyright (C) 2012-2016, Yann Collet +* +* BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following conditions are +* met: +* +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* * Redistributions in binary form must reproduce the above +* copyright notice, this list of conditions and the following disclaimer +* in the documentation and/or other materials provided with the +* distribution. +* +* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +* +* You can contact the author at : +* - xxHash homepage: http://www.xxhash.com +* - xxHash source repository : https://github.com/Cyan4973/xxHash +*/ + + +/* ************************************* +* Tuning parameters +***************************************/ +/*!XXH_FORCE_MEMORY_ACCESS : + * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. + * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. + * The below switch allow to select different access method for improved performance. + * Method 0 (default) : use `memcpy()`. Safe and portable. + * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). + * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. + * Method 2 : direct access. This method doesn't depend on compiler but violate C standard. + * It can generate buggy code on targets which do not support unaligned memory accesses. + * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) + * See http://stackoverflow.com/a/32095106/646947 for details. + * Prefer these methods in priority order (0 > 1 > 2) + */ +#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ +# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) +# define XXH_FORCE_MEMORY_ACCESS 2 +# elif defined(__INTEL_COMPILER) || \ + (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) +# define XXH_FORCE_MEMORY_ACCESS 1 +# endif +#endif + +/*!XXH_ACCEPT_NULL_INPUT_POINTER : + * If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer. + * When this option is enabled, xxHash output for null input pointers will be the same as a null-length input. + * By default, this option is disabled. To enable it, uncomment below define : + */ +/* #define XXH_ACCEPT_NULL_INPUT_POINTER 1 */ + +/*!XXH_FORCE_NATIVE_FORMAT : + * By default, xxHash library provides endian-independant Hash values, based on little-endian convention. + * Results are therefore identical for little-endian and big-endian CPU. + * This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format. + * Should endian-independance be of no importance for your application, you may set the #define below to 1, + * to improve speed for Big-endian CPU. + * This option has no impact on Little_Endian CPU. + */ +#ifndef XXH_FORCE_NATIVE_FORMAT /* can be defined externally */ +# define XXH_FORCE_NATIVE_FORMAT 0 +#endif + +/*!XXH_FORCE_ALIGN_CHECK : + * This is a minor performance trick, only useful with lots of very small keys. + * It means : check for aligned/unaligned input. + * The check costs one initial branch per hash; set to 0 when the input data + * is guaranteed to be aligned. + */ +#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ +# if defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64) +# define XXH_FORCE_ALIGN_CHECK 0 +# else +# define XXH_FORCE_ALIGN_CHECK 1 +# endif +#endif + + +/* ************************************* +* Includes & Memory related functions +***************************************/ +/* Modify the local functions below should you wish to use some other memory routines */ +/* for malloc(), free() */ +#include <stdlib.h> +static void* XXH_malloc(size_t s) { return malloc(s); } +static void XXH_free (void* p) { free(p); } +/* for memcpy() */ +#include <string.h> +static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); } + +#define XXH_STATIC_LINKING_ONLY +#include "xxhash.h" + + +/* ************************************* +* Compiler Specific Options +***************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# define FORCE_INLINE static __forceinline +#else +# if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# ifdef __GNUC__ +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif +# else +# define FORCE_INLINE static +# endif /* __STDC_VERSION__ */ +#endif + + +/* ************************************* +* Basic Types +***************************************/ +#ifndef MEM_MODULE +# define MEM_MODULE +# if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# include <stdint.h> + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; +# else + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; +# endif +#endif + + +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) + +/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ +static U32 XXH_read32(const void* memPtr) { return *(const U32*) memPtr; } +static U64 XXH_read64(const void* memPtr) { return *(const U64*) memPtr; } + +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) + +/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ +/* currently only defined for gcc and icc */ +typedef union { U32 u32; U64 u64; } __attribute__((packed)) unalign; + +static U32 XXH_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } +static U64 XXH_read64(const void* ptr) { return ((const unalign*)ptr)->u64; } + +#else + +/* portable and safe solution. Generally efficient. + * see : http://stackoverflow.com/a/32095106/646947 + */ + +static U32 XXH_read32(const void* memPtr) +{ + U32 val; + memcpy(&val, memPtr, sizeof(val)); + return val; +} + +static U64 XXH_read64(const void* memPtr) +{ + U64 val; + memcpy(&val, memPtr, sizeof(val)); + return val; +} + +#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ + + +/* **************************************** +* Compiler-specific Functions and Macros +******************************************/ +#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + +/* Note : although _rotl exists for minGW (GCC under windows), performance seems poor */ +#if defined(_MSC_VER) +# define XXH_rotl32(x,r) _rotl(x,r) +# define XXH_rotl64(x,r) _rotl64(x,r) +#else +# define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r))) +# define XXH_rotl64(x,r) ((x << r) | (x >> (64 - r))) +#endif + +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap32 _byteswap_ulong +# define XXH_swap64 _byteswap_uint64 +#elif GCC_VERSION >= 403 +# define XXH_swap32 __builtin_bswap32 +# define XXH_swap64 __builtin_bswap64 +#else +static U32 XXH_swap32 (U32 x) +{ + return ((x << 24) & 0xff000000 ) | + ((x << 8) & 0x00ff0000 ) | + ((x >> 8) & 0x0000ff00 ) | + ((x >> 24) & 0x000000ff ); +} +static U64 XXH_swap64 (U64 x) +{ + return ((x << 56) & 0xff00000000000000ULL) | + ((x << 40) & 0x00ff000000000000ULL) | + ((x << 24) & 0x0000ff0000000000ULL) | + ((x << 8) & 0x000000ff00000000ULL) | + ((x >> 8) & 0x00000000ff000000ULL) | + ((x >> 24) & 0x0000000000ff0000ULL) | + ((x >> 40) & 0x000000000000ff00ULL) | + ((x >> 56) & 0x00000000000000ffULL); +} +#endif + + +/* ************************************* +* Architecture Macros +***************************************/ +typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; + +/* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example on the compiler command line */ +#ifndef XXH_CPU_LITTLE_ENDIAN + static const int g_one = 1; +# define XXH_CPU_LITTLE_ENDIAN (*(const char*)(&g_one)) +#endif + + +/* *************************** +* Memory reads +*****************************/ +typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; + +FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align) +{ + if (align==XXH_unaligned) + return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); + else + return endian==XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr); +} + +FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian) +{ + return XXH_readLE32_align(ptr, endian, XXH_unaligned); +} + +static U32 XXH_readBE32(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr); +} + +FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align) +{ + if (align==XXH_unaligned) + return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); + else + return endian==XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr); +} + +FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian) +{ + return XXH_readLE64_align(ptr, endian, XXH_unaligned); +} + +static U64 XXH_readBE64(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr); +} + + +/* ************************************* +* Macros +***************************************/ +#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ + + +/* ************************************* +* Constants +***************************************/ +static const U32 PRIME32_1 = 2654435761U; +static const U32 PRIME32_2 = 2246822519U; +static const U32 PRIME32_3 = 3266489917U; +static const U32 PRIME32_4 = 668265263U; +static const U32 PRIME32_5 = 374761393U; + +static const U64 PRIME64_1 = 11400714785074694791ULL; +static const U64 PRIME64_2 = 14029467366897019727ULL; +static const U64 PRIME64_3 = 1609587929392839161ULL; +static const U64 PRIME64_4 = 9650029242287828579ULL; +static const U64 PRIME64_5 = 2870177450012600261ULL; + +XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } + + +/* *************************** +* Simple Hash Functions +*****************************/ + +static U32 XXH32_round(U32 seed, U32 input) +{ + seed += input * PRIME32_2; + seed = XXH_rotl32(seed, 13); + seed *= PRIME32_1; + return seed; +} + +FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align) +{ + const BYTE* p = (const BYTE*)input; + const BYTE* bEnd = p + len; + U32 h32; +#define XXH_get32bits(p) XXH_readLE32_align(p, endian, align) + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (p==NULL) { + len=0; + bEnd=p=(const BYTE*)(size_t)16; + } +#endif + + if (len>=16) { + const BYTE* const limit = bEnd - 16; + U32 v1 = seed + PRIME32_1 + PRIME32_2; + U32 v2 = seed + PRIME32_2; + U32 v3 = seed + 0; + U32 v4 = seed - PRIME32_1; + + do { + v1 = XXH32_round(v1, XXH_get32bits(p)); p+=4; + v2 = XXH32_round(v2, XXH_get32bits(p)); p+=4; + v3 = XXH32_round(v3, XXH_get32bits(p)); p+=4; + v4 = XXH32_round(v4, XXH_get32bits(p)); p+=4; + } while (p<=limit); + + h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); + } else { + h32 = seed + PRIME32_5; + } + + h32 += (U32) len; + + while (p+4<=bEnd) { + h32 += XXH_get32bits(p) * PRIME32_3; + h32 = XXH_rotl32(h32, 17) * PRIME32_4 ; + p+=4; + } + + while (p<bEnd) { + h32 += (*p) * PRIME32_5; + h32 = XXH_rotl32(h32, 11) * PRIME32_1 ; + p++; + } + + h32 ^= h32 >> 15; + h32 *= PRIME32_2; + h32 ^= h32 >> 13; + h32 *= PRIME32_3; + h32 ^= h32 >> 16; + + return h32; +} + + +XXH_PUBLIC_API unsigned int XXH32 (const void* input, size_t len, unsigned int seed) +{ +#if 0 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH32_CREATESTATE_STATIC(state); + XXH32_reset(state, seed); + XXH32_update(state, input, len); + return XXH32_digest(state); +#else + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if (XXH_FORCE_ALIGN_CHECK) { + if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */ + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); + else + return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); + } } + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); + else + return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); +#endif +} + + +static U64 XXH64_round(U64 acc, U64 input) +{ + acc += input * PRIME64_2; + acc = XXH_rotl64(acc, 31); + acc *= PRIME64_1; + return acc; +} + +static U64 XXH64_mergeRound(U64 acc, U64 val) +{ + val = XXH64_round(0, val); + acc ^= val; + acc = acc * PRIME64_1 + PRIME64_4; + return acc; +} + +FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align) +{ + const BYTE* p = (const BYTE*)input; + const BYTE* const bEnd = p + len; + U64 h64; +#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align) + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (p==NULL) { + len=0; + bEnd=p=(const BYTE*)(size_t)32; + } +#endif + + if (len>=32) { + const BYTE* const limit = bEnd - 32; + U64 v1 = seed + PRIME64_1 + PRIME64_2; + U64 v2 = seed + PRIME64_2; + U64 v3 = seed + 0; + U64 v4 = seed - PRIME64_1; + + do { + v1 = XXH64_round(v1, XXH_get64bits(p)); p+=8; + v2 = XXH64_round(v2, XXH_get64bits(p)); p+=8; + v3 = XXH64_round(v3, XXH_get64bits(p)); p+=8; + v4 = XXH64_round(v4, XXH_get64bits(p)); p+=8; + } while (p<=limit); + + h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); + h64 = XXH64_mergeRound(h64, v1); + h64 = XXH64_mergeRound(h64, v2); + h64 = XXH64_mergeRound(h64, v3); + h64 = XXH64_mergeRound(h64, v4); + + } else { + h64 = seed + PRIME64_5; + } + + h64 += (U64) len; + + while (p+8<=bEnd) { + U64 const k1 = XXH64_round(0, XXH_get64bits(p)); + h64 ^= k1; + h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; + p+=8; + } + + if (p+4<=bEnd) { + h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1; + h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; + p+=4; + } + + while (p<bEnd) { + h64 ^= (*p) * PRIME64_5; + h64 = XXH_rotl64(h64, 11) * PRIME64_1; + p++; + } + + h64 ^= h64 >> 33; + h64 *= PRIME64_2; + h64 ^= h64 >> 29; + h64 *= PRIME64_3; + h64 ^= h64 >> 32; + + return h64; +} + + +XXH_PUBLIC_API unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed) +{ +#if 0 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH64_CREATESTATE_STATIC(state); + XXH64_reset(state, seed); + XXH64_update(state, input, len); + return XXH64_digest(state); +#else + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if (XXH_FORCE_ALIGN_CHECK) { + if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */ + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); + else + return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); + } } + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); + else + return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); +#endif +} + + +/* ************************************************** +* Advanced Hash Functions +****************************************************/ + +XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void) +{ + return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); +} +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +} + +XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void) +{ + return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); +} +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +} + + +/*** Hash feed ***/ + +XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, unsigned int seed) +{ + XXH32_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ + memset(&state, 0, sizeof(state)); + state.seed = seed; + state.v1 = seed + PRIME32_1 + PRIME32_2; + state.v2 = seed + PRIME32_2; + state.v3 = seed + 0; + state.v4 = seed - PRIME32_1; + memcpy(statePtr, &state, sizeof(state)); + return XXH_OK; +} + + +XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, unsigned long long seed) +{ + XXH64_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ + memset(&state, 0, sizeof(state)); + state.seed = seed; + state.v1 = seed + PRIME64_1 + PRIME64_2; + state.v2 = seed + PRIME64_2; + state.v3 = seed + 0; + state.v4 = seed - PRIME64_1; + memcpy(statePtr, &state, sizeof(state)); + return XXH_OK; +} + + +FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state, const void* input, size_t len, XXH_endianess endian) +{ + const BYTE* p = (const BYTE*)input; + const BYTE* const bEnd = p + len; + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (input==NULL) return XXH_ERROR; +#endif + + state->total_len += len; + + if (state->memsize + len < 16) { /* fill in tmp buffer */ + XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len); + state->memsize += (U32)len; + return XXH_OK; + } + + if (state->memsize) { /* some data left from previous update */ + XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize); + { const U32* p32 = state->mem32; + state->v1 = XXH32_round(state->v1, XXH_readLE32(p32, endian)); p32++; + state->v2 = XXH32_round(state->v2, XXH_readLE32(p32, endian)); p32++; + state->v3 = XXH32_round(state->v3, XXH_readLE32(p32, endian)); p32++; + state->v4 = XXH32_round(state->v4, XXH_readLE32(p32, endian)); p32++; + } + p += 16-state->memsize; + state->memsize = 0; + } + + if (p <= bEnd-16) { + const BYTE* const limit = bEnd - 16; + U32 v1 = state->v1; + U32 v2 = state->v2; + U32 v3 = state->v3; + U32 v4 = state->v4; + + do { + v1 = XXH32_round(v1, XXH_readLE32(p, endian)); p+=4; + v2 = XXH32_round(v2, XXH_readLE32(p, endian)); p+=4; + v3 = XXH32_round(v3, XXH_readLE32(p, endian)); p+=4; + v4 = XXH32_round(v4, XXH_readLE32(p, endian)); p+=4; + } while (p<=limit); + + state->v1 = v1; + state->v2 = v2; + state->v3 = v3; + state->v4 = v4; + } + + if (p < bEnd) { + XXH_memcpy(state->mem32, p, bEnd-p); + state->memsize = (int)(bEnd-p); + } + + return XXH_OK; +} + +XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_update_endian(state_in, input, len, XXH_littleEndian); + else + return XXH32_update_endian(state_in, input, len, XXH_bigEndian); +} + + + +FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian) +{ + const BYTE * p = (const BYTE*)state->mem32; + const BYTE* const bEnd = (const BYTE*)(state->mem32) + state->memsize; + U32 h32; + + if (state->total_len >= 16) { + h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18); + } else { + h32 = state->seed + PRIME32_5; + } + + h32 += (U32) state->total_len; + + while (p+4<=bEnd) { + h32 += XXH_readLE32(p, endian) * PRIME32_3; + h32 = XXH_rotl32(h32, 17) * PRIME32_4; + p+=4; + } + + while (p<bEnd) { + h32 += (*p) * PRIME32_5; + h32 = XXH_rotl32(h32, 11) * PRIME32_1; + p++; + } + + h32 ^= h32 >> 15; + h32 *= PRIME32_2; + h32 ^= h32 >> 13; + h32 *= PRIME32_3; + h32 ^= h32 >> 16; + + return h32; +} + + +XXH_PUBLIC_API unsigned int XXH32_digest (const XXH32_state_t* state_in) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_digest_endian(state_in, XXH_littleEndian); + else + return XXH32_digest_endian(state_in, XXH_bigEndian); +} + + + +/* **** XXH64 **** */ + +FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian) +{ + const BYTE* p = (const BYTE*)input; + const BYTE* const bEnd = p + len; + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (input==NULL) return XXH_ERROR; +#endif + + state->total_len += len; + + if (state->memsize + len < 32) { /* fill in tmp buffer */ + XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len); + state->memsize += (U32)len; + return XXH_OK; + } + + if (state->memsize) { /* tmp buffer is full */ + XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize); + state->v1 = XXH64_round(state->v1, XXH_readLE64(state->mem64+0, endian)); + state->v2 = XXH64_round(state->v2, XXH_readLE64(state->mem64+1, endian)); + state->v3 = XXH64_round(state->v3, XXH_readLE64(state->mem64+2, endian)); + state->v4 = XXH64_round(state->v4, XXH_readLE64(state->mem64+3, endian)); + p += 32-state->memsize; + state->memsize = 0; + } + + if (p+32 <= bEnd) { + const BYTE* const limit = bEnd - 32; + U64 v1 = state->v1; + U64 v2 = state->v2; + U64 v3 = state->v3; + U64 v4 = state->v4; + + do { + v1 = XXH64_round(v1, XXH_readLE64(p, endian)); p+=8; + v2 = XXH64_round(v2, XXH_readLE64(p, endian)); p+=8; + v3 = XXH64_round(v3, XXH_readLE64(p, endian)); p+=8; + v4 = XXH64_round(v4, XXH_readLE64(p, endian)); p+=8; + } while (p<=limit); + + state->v1 = v1; + state->v2 = v2; + state->v3 = v3; + state->v4 = v4; + } + + if (p < bEnd) { + XXH_memcpy(state->mem64, p, bEnd-p); + state->memsize = (int)(bEnd-p); + } + + return XXH_OK; +} + +XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t len) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_update_endian(state_in, input, len, XXH_littleEndian); + else + return XXH64_update_endian(state_in, input, len, XXH_bigEndian); +} + + + +FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian) +{ + const BYTE * p = (const BYTE*)state->mem64; + const BYTE* const bEnd = (const BYTE*)state->mem64 + state->memsize; + U64 h64; + + if (state->total_len >= 32) { + U64 const v1 = state->v1; + U64 const v2 = state->v2; + U64 const v3 = state->v3; + U64 const v4 = state->v4; + + h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); + h64 = XXH64_mergeRound(h64, v1); + h64 = XXH64_mergeRound(h64, v2); + h64 = XXH64_mergeRound(h64, v3); + h64 = XXH64_mergeRound(h64, v4); + } else { + h64 = state->seed + PRIME64_5; + } + + h64 += (U64) state->total_len; + + while (p+8<=bEnd) { + U64 const k1 = XXH64_round(0, XXH_readLE64(p, endian)); + h64 ^= k1; + h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; + p+=8; + } + + if (p+4<=bEnd) { + h64 ^= (U64)(XXH_readLE32(p, endian)) * PRIME64_1; + h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; + p+=4; + } + + while (p<bEnd) { + h64 ^= (*p) * PRIME64_5; + h64 = XXH_rotl64(h64, 11) * PRIME64_1; + p++; + } + + h64 ^= h64 >> 33; + h64 *= PRIME64_2; + h64 ^= h64 >> 29; + h64 *= PRIME64_3; + h64 ^= h64 >> 32; + + return h64; +} + + +XXH_PUBLIC_API unsigned long long XXH64_digest (const XXH64_state_t* state_in) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_digest_endian(state_in, XXH_littleEndian); + else + return XXH64_digest_endian(state_in, XXH_bigEndian); +} + + +/* ************************** +* Canonical representation +****************************/ + +/*! Default XXH result types are basic unsigned 32 and 64 bits. +* The canonical representation follows human-readable write convention, aka big-endian (large digits first). +* These functions allow transformation of hash result into and from its canonical format. +* This way, hash values can be written into a file or buffer, and remain comparable across different systems and programs. +*/ + +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash); + memcpy(dst, &hash, sizeof(*dst)); +} + +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash); + memcpy(dst, &hash, sizeof(*dst)); +} + +XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src) +{ + return XXH_readBE32(src); +} + +XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src) +{ + return XXH_readBE64(src); +} diff --git a/contrib/libs/zstd06/common/xxhash.h b/contrib/libs/zstd06/common/xxhash.h index ae77e981b5..b940b4fa9a 100644 --- a/contrib/libs/zstd06/common/xxhash.h +++ b/contrib/libs/zstd06/common/xxhash.h @@ -1,274 +1,274 @@ -#include <contrib/libs/zstd06/renames.h> -/* - xxHash - Extremely Fast Hash algorithm - Header File - Copyright (C) 2012-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - xxHash source repository : https://github.com/Cyan4973/xxHash -*/ - -/* Notice extracted from xxHash homepage : - -xxHash is an extremely fast Hash algorithm, running at RAM speed limits. -It also successfully passes all tests from the SMHasher suite. - -Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz) - -Name Speed Q.Score Author -xxHash 5.4 GB/s 10 -CrapWow 3.2 GB/s 2 Andrew -MumurHash 3a 2.7 GB/s 10 Austin Appleby -SpookyHash 2.0 GB/s 10 Bob Jenkins -SBox 1.4 GB/s 9 Bret Mulvey -Lookup3 1.2 GB/s 9 Bob Jenkins -SuperFastHash 1.2 GB/s 1 Paul Hsieh -CityHash64 1.05 GB/s 10 Pike & Alakuijala -FNV 0.55 GB/s 5 Fowler, Noll, Vo -CRC32 0.43 GB/s 9 -MD5-32 0.33 GB/s 10 Ronald L. Rivest -SHA1-32 0.28 GB/s 10 - -Q.Score is a measure of quality of the hash function. -It depends on successfully passing SMHasher test set. -10 is a perfect score. - -A 64-bits version, named XXH64, is available since r35. -It offers much better speed, but for 64-bits applications only. -Name Speed on 64 bits Speed on 32 bits -XXH64 13.8 GB/s 1.9 GB/s -XXH32 6.8 GB/s 6.0 GB/s -*/ - -#ifndef XXHASH_H_5627135585666179 -#define XXHASH_H_5627135585666179 1 - -#if defined (__cplusplus) -extern "C" { -#endif - - -/* **************************** -* Definitions -******************************/ -#include <stddef.h> /* size_t */ -typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; - - -/* **************************** -* API modifier -******************************/ -/*!XXH_PRIVATE_API -* Transforms all publics symbols within `xxhash.c` into private ones. -* Methodology : -* instead of : #include "xxhash.h" -* do : -* #define XXH_PRIVATE_API -* #include "xxhash.c" // note the .c , instead of .h -* also : don't compile and link xxhash.c separately -*/ -#ifdef XXH_PRIVATE_API -# if defined(__GNUC__) -# define XXH_PUBLIC_API static __attribute__((unused)) -# elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -# define XXH_PUBLIC_API static inline -# elif defined(_MSC_VER) -# define XXH_PUBLIC_API static __inline -# else -# define XXH_PUBLIC_API static /* this version may generate warnings for unused static functions; disable the relevant warning */ -# endif -#else -# define XXH_PUBLIC_API /* do nothing */ -#endif - -/*!XXH_NAMESPACE, aka Namespace Emulation : - -If you want to include _and expose_ xxHash functions from within your own library, -but also want to avoid symbol collisions with another library which also includes xxHash, - -you can use XXH_NAMESPACE, to automatically prefix any public symbol from `xxhash.c` -with the value of XXH_NAMESPACE (so avoid to keep it NULL and avoid numeric values). - -Note that no change is required within the calling program as long as it also includes `xxhash.h` : -regular symbol name will be automatically translated by this header. -*/ -#ifdef XXH_NAMESPACE -# define XXH_CAT(A,B) A##B -# define XXH_NAME2(A,B) XXH_CAT(A,B) -# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) -# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) -# define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber) -# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) -# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) -# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) -# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) -# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) -# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) -# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) -# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) -# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) -# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) -#endif - - -/* ************************************* -* Version -***************************************/ -#define XXH_VERSION_MAJOR 0 -#define XXH_VERSION_MINOR 6 -#define XXH_VERSION_RELEASE 0 -#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE) -XXH_PUBLIC_API unsigned XXH_versionNumber (void); - - -/* **************************** -* Simple Hash Functions -******************************/ -typedef unsigned int XXH32_hash_t; -typedef unsigned long long XXH64_hash_t; - -XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, unsigned int seed); -XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t length, unsigned long long seed); - -/*! -XXH32() : - Calculate the 32-bits hash of sequence "length" bytes stored at memory address "input". - The memory between input & input+length must be valid (allocated and read-accessible). - "seed" can be used to alter the result predictably. - Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s -XXH64() : - Calculate the 64-bits hash of sequence of length "len" stored at memory address "input". - "seed" can be used to alter the result predictably. - This function runs faster on 64-bits systems, but slower on 32-bits systems (see benchmark). -*/ - - -/* **************************** -* Streaming Hash Functions -******************************/ -typedef struct XXH32_state_s XXH32_state_t; /* incomplete type */ -typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ - -/*! Dynamic allocation of states - Compatible with dynamic libraries */ - -XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void); -XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); - -XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void); -XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); - - -/* hash streaming */ - -XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, unsigned int seed); -XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); -XXH_PUBLIC_API XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr); - -XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, unsigned long long seed); -XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length); -XXH_PUBLIC_API XXH64_hash_t XXH64_digest (const XXH64_state_t* statePtr); - -/*! -These functions generate the xxHash of an input provided in multiple segments, -as opposed to provided as a single block. - -XXH state must first be allocated, using either static or dynamic method provided above. - -Start a new hash by initializing state with a seed, using XXHnn_reset(). - -Then, feed the hash state by calling XXHnn_update() as many times as necessary. -Obviously, input must be valid, hence allocated and read accessible. -The function returns an error code, with 0 meaning OK, and any other value meaning there is an error. - -Finally, a hash value can be produced anytime, by using XXHnn_digest(). -This function returns the nn-bits hash as an int or long long. - -It's still possible to continue inserting input into the hash state after a digest, -and later on generate some new hashes, by calling again XXHnn_digest(). - -When done, free XXH state space if it was allocated dynamically. -*/ - - -/* ************************** -* Canonical representation -****************************/ -typedef struct { unsigned char digest[4]; } XXH32_canonical_t; -typedef struct { unsigned char digest[8]; } XXH64_canonical_t; - -XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash); -XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash); - -XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); -XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src); - -/*! Default result type for XXH functions are primitive unsigned 32 and 64 bits. -* The canonical representation uses human-readable write convention, aka big-endian (large digits first). -* These functions allow transformation of hash result into and from its canonical format. -* This way, hash values can be written into a file / memory, and remain comparable on different systems and programs. -*/ - - -#ifdef XXH_STATIC_LINKING_ONLY - -/* This part contains definition which shall only be used with static linking. - The prototypes / types defined here are not guaranteed to remain stable. - They could change in a future version, becoming incompatible with a different version of the library */ - - struct XXH32_state_s { - unsigned long long total_len; - unsigned seed; - unsigned v1; - unsigned v2; - unsigned v3; - unsigned v4; - unsigned mem32[4]; /* buffer defined as U32 for alignment */ - unsigned memsize; - }; /* typedef'd to XXH32_state_t */ - - struct XXH64_state_s { - unsigned long long total_len; - unsigned long long seed; - unsigned long long v1; - unsigned long long v2; - unsigned long long v3; - unsigned long long v4; - unsigned long long mem64[4]; /* buffer defined as U64 for alignment */ - unsigned memsize; - }; /* typedef'd to XXH64_state_t */ - - -#endif - - -#if defined (__cplusplus) -} -#endif - -#endif /* XXHASH_H_5627135585666179 */ +#include <contrib/libs/zstd06/renames.h> +/* + xxHash - Extremely Fast Hash algorithm + Header File + Copyright (C) 2012-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - xxHash source repository : https://github.com/Cyan4973/xxHash +*/ + +/* Notice extracted from xxHash homepage : + +xxHash is an extremely fast Hash algorithm, running at RAM speed limits. +It also successfully passes all tests from the SMHasher suite. + +Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz) + +Name Speed Q.Score Author +xxHash 5.4 GB/s 10 +CrapWow 3.2 GB/s 2 Andrew +MumurHash 3a 2.7 GB/s 10 Austin Appleby +SpookyHash 2.0 GB/s 10 Bob Jenkins +SBox 1.4 GB/s 9 Bret Mulvey +Lookup3 1.2 GB/s 9 Bob Jenkins +SuperFastHash 1.2 GB/s 1 Paul Hsieh +CityHash64 1.05 GB/s 10 Pike & Alakuijala +FNV 0.55 GB/s 5 Fowler, Noll, Vo +CRC32 0.43 GB/s 9 +MD5-32 0.33 GB/s 10 Ronald L. Rivest +SHA1-32 0.28 GB/s 10 + +Q.Score is a measure of quality of the hash function. +It depends on successfully passing SMHasher test set. +10 is a perfect score. + +A 64-bits version, named XXH64, is available since r35. +It offers much better speed, but for 64-bits applications only. +Name Speed on 64 bits Speed on 32 bits +XXH64 13.8 GB/s 1.9 GB/s +XXH32 6.8 GB/s 6.0 GB/s +*/ + +#ifndef XXHASH_H_5627135585666179 +#define XXHASH_H_5627135585666179 1 + +#if defined (__cplusplus) +extern "C" { +#endif + + +/* **************************** +* Definitions +******************************/ +#include <stddef.h> /* size_t */ +typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; + + +/* **************************** +* API modifier +******************************/ +/*!XXH_PRIVATE_API +* Transforms all publics symbols within `xxhash.c` into private ones. +* Methodology : +* instead of : #include "xxhash.h" +* do : +* #define XXH_PRIVATE_API +* #include "xxhash.c" // note the .c , instead of .h +* also : don't compile and link xxhash.c separately +*/ +#ifdef XXH_PRIVATE_API +# if defined(__GNUC__) +# define XXH_PUBLIC_API static __attribute__((unused)) +# elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define XXH_PUBLIC_API static inline +# elif defined(_MSC_VER) +# define XXH_PUBLIC_API static __inline +# else +# define XXH_PUBLIC_API static /* this version may generate warnings for unused static functions; disable the relevant warning */ +# endif +#else +# define XXH_PUBLIC_API /* do nothing */ +#endif + +/*!XXH_NAMESPACE, aka Namespace Emulation : + +If you want to include _and expose_ xxHash functions from within your own library, +but also want to avoid symbol collisions with another library which also includes xxHash, + +you can use XXH_NAMESPACE, to automatically prefix any public symbol from `xxhash.c` +with the value of XXH_NAMESPACE (so avoid to keep it NULL and avoid numeric values). + +Note that no change is required within the calling program as long as it also includes `xxhash.h` : +regular symbol name will be automatically translated by this header. +*/ +#ifdef XXH_NAMESPACE +# define XXH_CAT(A,B) A##B +# define XXH_NAME2(A,B) XXH_CAT(A,B) +# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) +# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) +# define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber) +# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) +# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) +# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) +# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) +# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) +# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) +# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) +# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) +# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) +# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) +#endif + + +/* ************************************* +* Version +***************************************/ +#define XXH_VERSION_MAJOR 0 +#define XXH_VERSION_MINOR 6 +#define XXH_VERSION_RELEASE 0 +#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE) +XXH_PUBLIC_API unsigned XXH_versionNumber (void); + + +/* **************************** +* Simple Hash Functions +******************************/ +typedef unsigned int XXH32_hash_t; +typedef unsigned long long XXH64_hash_t; + +XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, unsigned int seed); +XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t length, unsigned long long seed); + +/*! +XXH32() : + Calculate the 32-bits hash of sequence "length" bytes stored at memory address "input". + The memory between input & input+length must be valid (allocated and read-accessible). + "seed" can be used to alter the result predictably. + Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s +XXH64() : + Calculate the 64-bits hash of sequence of length "len" stored at memory address "input". + "seed" can be used to alter the result predictably. + This function runs faster on 64-bits systems, but slower on 32-bits systems (see benchmark). +*/ + + +/* **************************** +* Streaming Hash Functions +******************************/ +typedef struct XXH32_state_s XXH32_state_t; /* incomplete type */ +typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ + +/*! Dynamic allocation of states + Compatible with dynamic libraries */ + +XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void); +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); + +XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void); +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); + + +/* hash streaming */ + +XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, unsigned int seed); +XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); +XXH_PUBLIC_API XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr); + +XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, unsigned long long seed); +XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length); +XXH_PUBLIC_API XXH64_hash_t XXH64_digest (const XXH64_state_t* statePtr); + +/*! +These functions generate the xxHash of an input provided in multiple segments, +as opposed to provided as a single block. + +XXH state must first be allocated, using either static or dynamic method provided above. + +Start a new hash by initializing state with a seed, using XXHnn_reset(). + +Then, feed the hash state by calling XXHnn_update() as many times as necessary. +Obviously, input must be valid, hence allocated and read accessible. +The function returns an error code, with 0 meaning OK, and any other value meaning there is an error. + +Finally, a hash value can be produced anytime, by using XXHnn_digest(). +This function returns the nn-bits hash as an int or long long. + +It's still possible to continue inserting input into the hash state after a digest, +and later on generate some new hashes, by calling again XXHnn_digest(). + +When done, free XXH state space if it was allocated dynamically. +*/ + + +/* ************************** +* Canonical representation +****************************/ +typedef struct { unsigned char digest[4]; } XXH32_canonical_t; +typedef struct { unsigned char digest[8]; } XXH64_canonical_t; + +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash); +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash); + +XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); +XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src); + +/*! Default result type for XXH functions are primitive unsigned 32 and 64 bits. +* The canonical representation uses human-readable write convention, aka big-endian (large digits first). +* These functions allow transformation of hash result into and from its canonical format. +* This way, hash values can be written into a file / memory, and remain comparable on different systems and programs. +*/ + + +#ifdef XXH_STATIC_LINKING_ONLY + +/* This part contains definition which shall only be used with static linking. + The prototypes / types defined here are not guaranteed to remain stable. + They could change in a future version, becoming incompatible with a different version of the library */ + + struct XXH32_state_s { + unsigned long long total_len; + unsigned seed; + unsigned v1; + unsigned v2; + unsigned v3; + unsigned v4; + unsigned mem32[4]; /* buffer defined as U32 for alignment */ + unsigned memsize; + }; /* typedef'd to XXH32_state_t */ + + struct XXH64_state_s { + unsigned long long total_len; + unsigned long long seed; + unsigned long long v1; + unsigned long long v2; + unsigned long long v3; + unsigned long long v4; + unsigned long long mem64[4]; /* buffer defined as U64 for alignment */ + unsigned memsize; + }; /* typedef'd to XXH64_state_t */ + + +#endif + + +#if defined (__cplusplus) +} +#endif + +#endif /* XXHASH_H_5627135585666179 */ diff --git a/contrib/libs/zstd06/common/zbuff.h b/contrib/libs/zstd06/common/zbuff.h index 5f49fe57ed..54c40b47ca 100644 --- a/contrib/libs/zstd06/common/zbuff.h +++ b/contrib/libs/zstd06/common/zbuff.h @@ -1,4 +1,4 @@ -#include <contrib/libs/zstd06/renames.h> +#include <contrib/libs/zstd06/renames.h> /* Buffered version of Zstd compression library Copyright (C) 2015-2016, Yann Collet. @@ -154,7 +154,7 @@ ZSTDLIB_API unsigned ZBUFF_isError(size_t errorCode); ZSTDLIB_API const char* ZBUFF_getErrorName(size_t errorCode); /** Functions below provide recommended buffer sizes for Compression or Decompression operations. -* These sizes are just hints, they tend to offer better latency */ +* These sizes are just hints, they tend to offer better latency */ ZSTDLIB_API size_t ZBUFF_recommendedCInSize(void); ZSTDLIB_API size_t ZBUFF_recommendedCOutSize(void); ZSTDLIB_API size_t ZBUFF_recommendedDInSize(void); diff --git a/contrib/libs/zstd06/common/zbuff_static.h b/contrib/libs/zstd06/common/zbuff_static.h index 2676bc4012..e06404f646 100644 --- a/contrib/libs/zstd06/common/zbuff_static.h +++ b/contrib/libs/zstd06/common/zbuff_static.h @@ -1,4 +1,4 @@ -#include <contrib/libs/zstd06/renames.h> +#include <contrib/libs/zstd06/renames.h> /* zstd - buffered version of compression library experimental complementary API, for static linking only @@ -47,7 +47,7 @@ extern "C" { ***************************************/ #include "zstd_static.h" /* ZSTD_parameters */ #include "zbuff.h" -#include "zstd_internal.h" /* MIN */ +#include "zstd_internal.h" /* MIN */ /* ************************************* @@ -57,14 +57,14 @@ ZSTDLIB_API size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, U64 pledgedSrcSize); -MEM_STATIC size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) -{ - size_t length = MIN(dstCapacity, srcSize); - memcpy(dst, src, length); - return length; -} +MEM_STATIC size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + size_t length = MIN(dstCapacity, srcSize); + memcpy(dst, src, length); + return length; +} + - #if defined (__cplusplus) } #endif diff --git a/contrib/libs/zstd06/common/zstd.h b/contrib/libs/zstd06/common/zstd.h index 44ee62586b..3574a82ec0 100644 --- a/contrib/libs/zstd06/common/zstd.h +++ b/contrib/libs/zstd06/common/zstd.h @@ -1,4 +1,4 @@ -#include <contrib/libs/zstd06/renames.h> +#include <contrib/libs/zstd06/renames.h> /* zstd - standard compression library Header File @@ -60,15 +60,15 @@ extern "C" { /* ************************************* * Version ***************************************/ -#define ZSTD_VERSION_MAJOR 0 -#define ZSTD_VERSION_MINOR 6 -#define ZSTD_VERSION_RELEASE 2 - -#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE -#define ZSTD_QUOTE(str) #str -#define ZSTD_EXPAND_AND_QUOTE(str) ZSTD_QUOTE(str) -#define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION) - +#define ZSTD_VERSION_MAJOR 0 +#define ZSTD_VERSION_MINOR 6 +#define ZSTD_VERSION_RELEASE 2 + +#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE +#define ZSTD_QUOTE(str) #str +#define ZSTD_EXPAND_AND_QUOTE(str) ZSTD_QUOTE(str) +#define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION) + #define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) ZSTDLIB_API unsigned ZSTD_versionNumber (void); diff --git a/contrib/libs/zstd06/common/zstd_common.c b/contrib/libs/zstd06/common/zstd_common.c index ea7b0dd025..8812c03885 100644 --- a/contrib/libs/zstd06/common/zstd_common.c +++ b/contrib/libs/zstd06/common/zstd_common.c @@ -1,72 +1,72 @@ -/* - Common functions of Zstd compression library - Copyright (C) 2015-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - zstd homepage : http://www.zstd.net/ -*/ - - -/*-************************************* -* Dependencies -***************************************/ -#include "error_private.h" -#include "zstd.h" /* declaration of ZSTD_isError, ZSTD_getErrorName */ -#include "zbuff.h" /* declaration of ZBUFF_isError, ZBUFF_getErrorName */ - - -/*-**************************************** -* Version -******************************************/ -unsigned ZSTD_versionNumber (void) { return ZSTD_VERSION_NUMBER; } - - -/*-**************************************** -* ZSTD Error Management -******************************************/ -/*! ZSTD_isError() : -* tells if a return value is an error code */ -unsigned ZSTD_isError(size_t code) { return ERR_isError(code); } - -/*! ZSTD_getErrorName() : -* provides error code string from function result (useful for debugging) */ -const char* ZSTD_getErrorName(size_t code) { return ERR_getErrorName(code); } - -/*! ZSTD_getError() : -* convert a `size_t` function result into a proper ZSTD_errorCode enum */ -ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); } - -/*! ZSTD_getErrorString() : -* provides error code string from enum */ -const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorName(code); } - - -/* ************************************************************** -* ZBUFF Error Management -****************************************************************/ -unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); } - -const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); } +/* + Common functions of Zstd compression library + Copyright (C) 2015-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - zstd homepage : http://www.zstd.net/ +*/ + + +/*-************************************* +* Dependencies +***************************************/ +#include "error_private.h" +#include "zstd.h" /* declaration of ZSTD_isError, ZSTD_getErrorName */ +#include "zbuff.h" /* declaration of ZBUFF_isError, ZBUFF_getErrorName */ + + +/*-**************************************** +* Version +******************************************/ +unsigned ZSTD_versionNumber (void) { return ZSTD_VERSION_NUMBER; } + + +/*-**************************************** +* ZSTD Error Management +******************************************/ +/*! ZSTD_isError() : +* tells if a return value is an error code */ +unsigned ZSTD_isError(size_t code) { return ERR_isError(code); } + +/*! ZSTD_getErrorName() : +* provides error code string from function result (useful for debugging) */ +const char* ZSTD_getErrorName(size_t code) { return ERR_getErrorName(code); } + +/*! ZSTD_getError() : +* convert a `size_t` function result into a proper ZSTD_errorCode enum */ +ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); } + +/*! ZSTD_getErrorString() : +* provides error code string from enum */ +const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorName(code); } + + +/* ************************************************************** +* ZBUFF Error Management +****************************************************************/ +unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); } + +const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); } diff --git a/contrib/libs/zstd06/common/zstd_internal.h b/contrib/libs/zstd06/common/zstd_internal.h index 663ccbe430..2ce2b18cf1 100644 --- a/contrib/libs/zstd06/common/zstd_internal.h +++ b/contrib/libs/zstd06/common/zstd_internal.h @@ -1,4 +1,4 @@ -#include <contrib/libs/zstd06/renames.h> +#include <contrib/libs/zstd06/renames.h> /* zstd_internal - common functions to include Header File for include @@ -200,15 +200,15 @@ typedef struct { U32 rep[ZSTD_REP_INIT]; } ZSTD_optimal_t; -//#if ZSTD_OPT_DEBUG == 3 -// #include ".debug/zstd_stats.h" -//#else +//#if ZSTD_OPT_DEBUG == 3 +// #include ".debug/zstd_stats.h" +//#else typedef struct { U32 unused; } ZSTD_stats_t; - MEM_STATIC void ZSTD_statsPrint(ZSTD_stats_t* stats, U32 searchLength) { (void)stats; (void)searchLength; } - MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats) { (void)stats; } - MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats) { (void)stats; } - MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) { (void)stats; (void)litLength; (void)literals; (void)offset; (void)matchLength; } -//#endif + MEM_STATIC void ZSTD_statsPrint(ZSTD_stats_t* stats, U32 searchLength) { (void)stats; (void)searchLength; } + MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats) { (void)stats; } + MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats) { (void)stats; } + MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) { (void)stats; (void)litLength; (void)literals; (void)offset; (void)matchLength; } +//#endif typedef struct { void* buffer; @@ -243,9 +243,9 @@ typedef struct { U32 log2litSum; U32 log2offCodeSum; U32 factor; - U32 cachedPrice; - U32 cachedLitLength; - const BYTE* cachedLiterals; + U32 cachedPrice; + U32 cachedLitLength; + const BYTE* cachedLiterals; ZSTD_stats_t stats; } seqStore_t; diff --git a/contrib/libs/zstd06/common/zstd_static.h b/contrib/libs/zstd06/common/zstd_static.h index 3118761987..a1dae42094 100644 --- a/contrib/libs/zstd06/common/zstd_static.h +++ b/contrib/libs/zstd06/common/zstd_static.h @@ -1,4 +1,4 @@ -#include <contrib/libs/zstd06/renames.h> +#include <contrib/libs/zstd06/renames.h> /* zstd - standard compression library Header File for static linking only @@ -104,7 +104,7 @@ ZSTDLIB_API unsigned ZSTD_maxCLevel (void); /*! ZSTD_getCParams() : * @return ZSTD_compressionParameters structure for a selected compression level and srcSize. * `srcSize` value is optional, select 0 if not known */ -ZSTDLIB_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, U64 srcSize, size_t dictSize); +ZSTDLIB_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, U64 srcSize, size_t dictSize); /*! ZSTD_checkParams() : * Ensure param values remain within authorized range */ @@ -251,8 +251,8 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ds */ #define ZSTD_BLOCKSIZE_MAX (128 * 1024) /* define, for static allocation */ -ZSTDLIB_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); -ZSTDLIB_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTDLIB_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTDLIB_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); /*-************************************* @@ -262,8 +262,8 @@ ZSTDLIB_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCa /*! ZSTD_getErrorCode() : convert a `size_t` function result into a `ZSTD_ErrorCode` enum type, which can be used to compare directly with enum list published into "error_public.h" */ -ZSTDLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult); -ZSTDLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code); +ZSTDLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult); +ZSTDLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code); #if defined (__cplusplus) diff --git a/contrib/libs/zstd06/compress/fse_compress.c b/contrib/libs/zstd06/compress/fse_compress.c index 38427e5ca3..dad0be7942 100644 --- a/contrib/libs/zstd06/compress/fse_compress.c +++ b/contrib/libs/zstd06/compress/fse_compress.c @@ -1,803 +1,803 @@ -/* ****************************************************************** - FSE : Finite State Entropy encoder - Copyright (C) 2013-2015, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy - - Public forum : https://groups.google.com/forum/#!forum/lz4c -****************************************************************** */ - -/* ************************************************************** -* Compiler specifics -****************************************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# define FORCE_INLINE static __forceinline -# include <intrin.h> /* For Visual 2005 */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */ -#else -# ifdef __GNUC__ -# define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -# define FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define FORCE_INLINE static inline -# endif -#endif - - -/* ************************************************************** -* Includes -****************************************************************/ -#include <stdlib.h> /* malloc, free, qsort */ -#include <string.h> /* memcpy, memset */ -#include <stdio.h> /* printf (debug) */ -#include "bitstream.h" -#include "fse_static.h" - - -/* ************************************************************** -* Error Management -****************************************************************/ -#define FSE_STATIC_ASSERT(c) { enum { FSE_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ - - -/* ************************************************************** -* Complex types -****************************************************************/ -typedef U32 CTable_max_t[FSE_CTABLE_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)]; - - -/* ************************************************************** -* Templates -****************************************************************/ -/* - designed to be included - for type-specific functions (template emulation in C) - Objective is to write these functions only once, for improved maintenance -*/ - -/* safety checks */ -#ifndef FSE_FUNCTION_EXTENSION -# error "FSE_FUNCTION_EXTENSION must be defined" -#endif -#ifndef FSE_FUNCTION_TYPE -# error "FSE_FUNCTION_TYPE must be defined" -#endif - -/* Function names */ -#define FSE_CAT(X,Y) X##Y -#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y) -#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y) - - -/* Function templates */ -size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) -{ - U32 const tableSize = 1 << tableLog; - U32 const tableMask = tableSize - 1; - void* const ptr = ct; - U16* const tableU16 = ( (U16*) ptr) + 2; - void* const FSCT = ((U32*)ptr) + 1 /* header */ + (tableLog ? tableSize>>1 : 1) ; - FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT); - U32 const step = FSE_TABLESTEP(tableSize); - U32 cumul[FSE_MAX_SYMBOL_VALUE+2]; - - FSE_FUNCTION_TYPE tableSymbol[FSE_MAX_TABLESIZE]; /* memset() is not necessary, even if static analyzer complain about it */ - U32 highThreshold = tableSize-1; - - /* CTable header */ - - - tableU16[-2] = (U16) tableLog; - tableU16[-1] = (U16) maxSymbolValue; - - /* For explanations on how to distribute symbol values over the table : - * http://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */ - - /* symbol start positions */ - { U32 u; - cumul[0] = 0; - for (u=1; u<=maxSymbolValue+1; u++) { - if (normalizedCounter[u-1]==-1) { /* Low proba symbol */ - cumul[u] = cumul[u-1] + 1; - tableSymbol[highThreshold--] = (FSE_FUNCTION_TYPE)(u-1); - } else { - cumul[u] = cumul[u-1] + normalizedCounter[u-1]; - } } - cumul[maxSymbolValue+1] = tableSize+1; - } - - /* Spread symbols */ - { U32 position = 0; - U32 symbol; - for (symbol=0; symbol<=maxSymbolValue; symbol++) { - int nbOccurences; - for (nbOccurences=0; nbOccurences<normalizedCounter[symbol]; nbOccurences++) { - tableSymbol[position] = (FSE_FUNCTION_TYPE)symbol; - position = (position + step) & tableMask; - while (position > highThreshold) position = (position + step) & tableMask; /* Low proba area */ - } } - - if (position!=0) return ERROR(GENERIC); /* Must have gone through all positions */ - } - - /* Build table */ - { U32 u; for (u=0; u<tableSize; u++) { - FSE_FUNCTION_TYPE s = tableSymbol[u]; /* note : static analyzer may not understand tableSymbol is properly initialized */ - tableU16[cumul[s]++] = (U16) (tableSize+u); /* TableU16 : sorted by symbol order; gives next state value */ - }} - - /* Build Symbol Transformation Table */ - { unsigned total = 0; - unsigned s; - for (s=0; s<=maxSymbolValue; s++) { - switch (normalizedCounter[s]) - { - case 0: break; - - case -1: - case 1: - symbolTT[s].deltaNbBits = (tableLog << 16) - (1<<tableLog); - symbolTT[s].deltaFindState = total - 1; - total ++; - break; - default : - { - U32 const maxBitsOut = tableLog - BIT_highbit32 (normalizedCounter[s]-1); - U32 const minStatePlus = normalizedCounter[s] << maxBitsOut; - symbolTT[s].deltaNbBits = (maxBitsOut << 16) - minStatePlus; - symbolTT[s].deltaFindState = total - normalizedCounter[s]; - total += normalizedCounter[s]; - } } } } - - return 0; -} - - - -#ifndef FSE_COMMONDEFS_ONLY - -/*-************************************************************** -* FSE NCount encoding-decoding -****************************************************************/ -size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog) -{ - size_t maxHeaderSize = (((maxSymbolValue+1) * tableLog) >> 3) + 3; - return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */ -} - -static short FSE_abs(short a) { return a<0 ? -a : a; } - -static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize, - const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, - unsigned writeIsSafe) -{ - BYTE* const ostart = (BYTE*) header; - BYTE* out = ostart; - BYTE* const oend = ostart + headerBufferSize; - int nbBits; - const int tableSize = 1 << tableLog; - int remaining; - int threshold; - U32 bitStream; - int bitCount; - unsigned charnum = 0; - int previous0 = 0; - - bitStream = 0; - bitCount = 0; - /* Table Size */ - bitStream += (tableLog-FSE_MIN_TABLELOG) << bitCount; - bitCount += 4; - - /* Init */ - remaining = tableSize+1; /* +1 for extra accuracy */ - threshold = tableSize; - nbBits = tableLog+1; - - while (remaining>1) { /* stops at 1 */ - if (previous0) { - unsigned start = charnum; - while (!normalizedCounter[charnum]) charnum++; - while (charnum >= start+24) { - start+=24; - bitStream += 0xFFFFU << bitCount; - if ((!writeIsSafe) && (out > oend-2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ - out[0] = (BYTE) bitStream; - out[1] = (BYTE)(bitStream>>8); - out+=2; - bitStream>>=16; - } - while (charnum >= start+3) { - start+=3; - bitStream += 3 << bitCount; - bitCount += 2; - } - bitStream += (charnum-start) << bitCount; - bitCount += 2; - if (bitCount>16) { - if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ - out[0] = (BYTE)bitStream; - out[1] = (BYTE)(bitStream>>8); - out += 2; - bitStream >>= 16; - bitCount -= 16; - } } - { short count = normalizedCounter[charnum++]; - const short max = (short)((2*threshold-1)-remaining); - remaining -= FSE_abs(count); - if (remaining<1) return ERROR(GENERIC); - count++; /* +1 for extra accuracy */ - if (count>=threshold) count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */ - bitStream += count << bitCount; - bitCount += nbBits; - bitCount -= (count<max); - previous0 = (count==1); - while (remaining<threshold) nbBits--, threshold>>=1; - } - if (bitCount>16) { - if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ - out[0] = (BYTE)bitStream; - out[1] = (BYTE)(bitStream>>8); - out += 2; - bitStream >>= 16; - bitCount -= 16; - } } - - /* flush remaining bitStream */ - if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ - out[0] = (BYTE)bitStream; - out[1] = (BYTE)(bitStream>>8); - out+= (bitCount+7) /8; - - if (charnum > maxSymbolValue + 1) return ERROR(GENERIC); - - return (out-ostart); -} - - -size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) -{ - if (tableLog > FSE_MAX_TABLELOG) return ERROR(GENERIC); /* Unsupported */ - if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported */ - - if (bufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog)) - return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 0); - - return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1); -} - - - -/*-************************************************************** -* Counting histogram -****************************************************************/ -/*! FSE_count_simple - This function just counts byte values within `src`, - and store the histogram into table `count`. - This function is unsafe : it doesn't check that all values within `src` can fit into `count`. - For this reason, prefer using a table `count` with 256 elements. - @return : count of most numerous element -*/ -static size_t FSE_count_simple(unsigned* count, unsigned* maxSymbolValuePtr, - const void* src, size_t srcSize) -{ - const BYTE* ip = (const BYTE*)src; - const BYTE* const end = ip + srcSize; - unsigned maxSymbolValue = *maxSymbolValuePtr; - unsigned max=0; - - - memset(count, 0, (maxSymbolValue+1)*sizeof(*count)); - if (srcSize==0) { *maxSymbolValuePtr = 0; return 0; } - - while (ip<end) count[*ip++]++; - - while (!count[maxSymbolValue]) maxSymbolValue--; - *maxSymbolValuePtr = maxSymbolValue; - - { U32 s; for (s=0; s<=maxSymbolValue; s++) if (count[s] > max) max = count[s]; } - - return (size_t)max; -} - - -static size_t FSE_count_parallel(unsigned* count, unsigned* maxSymbolValuePtr, - const void* source, size_t sourceSize, - unsigned checkMax) -{ - const BYTE* ip = (const BYTE*)source; - const BYTE* const iend = ip+sourceSize; - unsigned maxSymbolValue = *maxSymbolValuePtr; - unsigned max=0; - - - U32 Counting1[256] = { 0 }; - U32 Counting2[256] = { 0 }; - U32 Counting3[256] = { 0 }; - U32 Counting4[256] = { 0 }; - - /* safety checks */ - if (!sourceSize) { - memset(count, 0, maxSymbolValue + 1); - *maxSymbolValuePtr = 0; - return 0; - } - if (!maxSymbolValue) maxSymbolValue = 255; /* 0 == default */ - - /* by stripes of 16 bytes */ - { U32 cached = MEM_read32(ip); ip += 4; - while (ip < iend-15) { - U32 c = cached; cached = MEM_read32(ip); ip += 4; - Counting1[(BYTE) c ]++; - Counting2[(BYTE)(c>>8) ]++; - Counting3[(BYTE)(c>>16)]++; - Counting4[ c>>24 ]++; - c = cached; cached = MEM_read32(ip); ip += 4; - Counting1[(BYTE) c ]++; - Counting2[(BYTE)(c>>8) ]++; - Counting3[(BYTE)(c>>16)]++; - Counting4[ c>>24 ]++; - c = cached; cached = MEM_read32(ip); ip += 4; - Counting1[(BYTE) c ]++; - Counting2[(BYTE)(c>>8) ]++; - Counting3[(BYTE)(c>>16)]++; - Counting4[ c>>24 ]++; - c = cached; cached = MEM_read32(ip); ip += 4; - Counting1[(BYTE) c ]++; - Counting2[(BYTE)(c>>8) ]++; - Counting3[(BYTE)(c>>16)]++; - Counting4[ c>>24 ]++; - } - ip-=4; - } - - /* finish last symbols */ - while (ip<iend) Counting1[*ip++]++; - - if (checkMax) { /* verify stats will fit into destination table */ - U32 s; for (s=255; s>maxSymbolValue; s--) { - Counting1[s] += Counting2[s] + Counting3[s] + Counting4[s]; - if (Counting1[s]) return ERROR(maxSymbolValue_tooSmall); - } } - - { U32 s; for (s=0; s<=maxSymbolValue; s++) { - count[s] = Counting1[s] + Counting2[s] + Counting3[s] + Counting4[s]; - if (count[s] > max) max = count[s]; - }} - - while (!count[maxSymbolValue]) maxSymbolValue--; - *maxSymbolValuePtr = maxSymbolValue; - return (size_t)max; -} - -/* fast variant (unsafe : won't check if src contains values beyond count[] limit) */ -size_t FSE_countFast(unsigned* count, unsigned* maxSymbolValuePtr, - const void* source, size_t sourceSize) -{ - if (sourceSize < 1500) return FSE_count_simple(count, maxSymbolValuePtr, source, sourceSize); - return FSE_count_parallel(count, maxSymbolValuePtr, source, sourceSize, 0); -} - -size_t FSE_count(unsigned* count, unsigned* maxSymbolValuePtr, - const void* source, size_t sourceSize) -{ - if (*maxSymbolValuePtr <255) - return FSE_count_parallel(count, maxSymbolValuePtr, source, sourceSize, 1); - *maxSymbolValuePtr = 255; - return FSE_countFast(count, maxSymbolValuePtr, source, sourceSize); -} - - - -/*-************************************************************** -* FSE Compression Code -****************************************************************/ -/*! FSE_sizeof_CTable() : - FSE_CTable is a variable size structure which contains : - `U16 tableLog;` - `U16 maxSymbolValue;` - `U16 nextStateNumber[1 << tableLog];` // This size is variable - `FSE_symbolCompressionTransform symbolTT[maxSymbolValue+1];` // This size is variable -Allocation is manual (C standard does not support variable-size structures). -*/ - -size_t FSE_sizeof_CTable (unsigned maxSymbolValue, unsigned tableLog) -{ - size_t size; - FSE_STATIC_ASSERT((size_t)FSE_CTABLE_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)*4 >= sizeof(CTable_max_t)); /* A compilation error here means FSE_CTABLE_SIZE_U32 is not large enough */ - if (tableLog > FSE_MAX_TABLELOG) return ERROR(GENERIC); - size = FSE_CTABLE_SIZE_U32 (tableLog, maxSymbolValue) * sizeof(U32); - return size; -} - -FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog) -{ - size_t size; - if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX; - size = FSE_CTABLE_SIZE_U32 (tableLog, maxSymbolValue) * sizeof(U32); - return (FSE_CTable*)malloc(size); -} - -void FSE_freeCTable (FSE_CTable* ct) { free(ct); } - -/* provides the minimum logSize to safely represent a distribution */ -static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue) -{ - U32 minBitsSrc = BIT_highbit32((U32)(srcSize - 1)) + 1; - U32 minBitsSymbols = BIT_highbit32(maxSymbolValue) + 2; - U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols; - return minBits; -} - -unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue) -{ - U32 maxBitsSrc = BIT_highbit32((U32)(srcSize - 1)) - 2; - U32 tableLog = maxTableLog; - U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue); - if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG; - if (maxBitsSrc < tableLog) tableLog = maxBitsSrc; /* Accuracy can be reduced */ - if (minBits > tableLog) tableLog = minBits; /* Need a minimum to safely represent all symbol values */ - if (tableLog < FSE_MIN_TABLELOG) tableLog = FSE_MIN_TABLELOG; - if (tableLog > FSE_MAX_TABLELOG) tableLog = FSE_MAX_TABLELOG; - return tableLog; -} - - -/* Secondary normalization method. - To be used when primary method fails. */ - -static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, size_t total, U32 maxSymbolValue) -{ - U32 s; - U32 distributed = 0; - U32 ToDistribute; - - /* Init */ - U32 lowThreshold = (U32)(total >> tableLog); - U32 lowOne = (U32)((total * 3) >> (tableLog + 1)); - - for (s=0; s<=maxSymbolValue; s++) { - if (count[s] == 0) { - norm[s]=0; - continue; - } - if (count[s] <= lowThreshold) { - norm[s] = -1; - distributed++; - total -= count[s]; - continue; - } - if (count[s] <= lowOne) { - norm[s] = 1; - distributed++; - total -= count[s]; - continue; - } - norm[s]=-2; - } - ToDistribute = (1 << tableLog) - distributed; - - if ((total / ToDistribute) > lowOne) { - /* risk of rounding to zero */ - lowOne = (U32)((total * 3) / (ToDistribute * 2)); - for (s=0; s<=maxSymbolValue; s++) { - if ((norm[s] == -2) && (count[s] <= lowOne)) { - norm[s] = 1; - distributed++; - total -= count[s]; - continue; - } } - ToDistribute = (1 << tableLog) - distributed; - } - - if (distributed == maxSymbolValue+1) { - /* all values are pretty poor; - probably incompressible data (should have already been detected); - find max, then give all remaining points to max */ - U32 maxV = 0, maxC = 0; - for (s=0; s<=maxSymbolValue; s++) - if (count[s] > maxC) maxV=s, maxC=count[s]; - norm[maxV] += (short)ToDistribute; - return 0; - } - - { - U64 const vStepLog = 62 - tableLog; - U64 const mid = (1ULL << (vStepLog-1)) - 1; - U64 const rStep = ((((U64)1<<vStepLog) * ToDistribute) + mid) / total; /* scale on remaining */ - U64 tmpTotal = mid; - for (s=0; s<=maxSymbolValue; s++) { - if (norm[s]==-2) { - U64 end = tmpTotal + (count[s] * rStep); - U32 sStart = (U32)(tmpTotal >> vStepLog); - U32 sEnd = (U32)(end >> vStepLog); - U32 weight = sEnd - sStart; - if (weight < 1) - return ERROR(GENERIC); - norm[s] = (short)weight; - tmpTotal = end; - } } } - - return 0; -} - - -size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog, - const unsigned* count, size_t total, - unsigned maxSymbolValue) -{ - /* Sanity checks */ - if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG; - if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported size */ - if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Unsupported size */ - if (tableLog < FSE_minTableLog(total, maxSymbolValue)) return ERROR(GENERIC); /* Too small tableLog, compression potentially impossible */ - - { U32 const rtbTable[] = { 0, 473195, 504333, 520860, 550000, 700000, 750000, 830000 }; - - U64 const scale = 62 - tableLog; - U64 const step = ((U64)1<<62) / total; /* <== here, one division ! */ - U64 const vStep = 1ULL<<(scale-20); - int stillToDistribute = 1<<tableLog; - unsigned s; - unsigned largest=0; - short largestP=0; - U32 lowThreshold = (U32)(total >> tableLog); - - for (s=0; s<=maxSymbolValue; s++) { - if (count[s] == total) return 0; /* rle special case */ - if (count[s] == 0) { normalizedCounter[s]=0; continue; } - if (count[s] <= lowThreshold) { - normalizedCounter[s] = -1; - stillToDistribute--; - } else { - short proba = (short)((count[s]*step) >> scale); - if (proba<8) { - U64 restToBeat = vStep * rtbTable[proba]; - proba += (count[s]*step) - ((U64)proba<<scale) > restToBeat; - } - if (proba > largestP) largestP=proba, largest=s; - normalizedCounter[s] = proba; - stillToDistribute -= proba; - } } - if (-stillToDistribute >= (normalizedCounter[largest] >> 1)) { - /* corner case, need another normalization method */ - size_t errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue); - if (FSE_isError(errorCode)) return errorCode; - } - else normalizedCounter[largest] += (short)stillToDistribute; - } - -#if 0 - { /* Print Table (debug) */ - U32 s; - U32 nTotal = 0; - for (s=0; s<=maxSymbolValue; s++) - printf("%3i: %4i \n", s, normalizedCounter[s]); - for (s=0; s<=maxSymbolValue; s++) - nTotal += abs(normalizedCounter[s]); - if (nTotal != (1U<<tableLog)) - printf("Warning !!! Total == %u != %u !!!", nTotal, 1U<<tableLog); - getchar(); - } -#endif - - return tableLog; -} - - -/* fake FSE_CTable, for raw (uncompressed) input */ -size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits) -{ - const unsigned tableSize = 1 << nbBits; - const unsigned tableMask = tableSize - 1; - const unsigned maxSymbolValue = tableMask; - void* const ptr = ct; - U16* const tableU16 = ( (U16*) ptr) + 2; - void* const FSCT = ((U32*)ptr) + 1 /* header */ + (tableSize>>1); /* assumption : tableLog >= 1 */ - FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT); - unsigned s; - - /* Sanity checks */ - if (nbBits < 1) return ERROR(GENERIC); /* min size */ - - /* header */ - tableU16[-2] = (U16) nbBits; - tableU16[-1] = (U16) maxSymbolValue; - - /* Build table */ - for (s=0; s<tableSize; s++) - tableU16[s] = (U16)(tableSize + s); - - /* Build Symbol Transformation Table */ - { const U32 deltaNbBits = (nbBits << 16) - (1 << nbBits); - - for (s=0; s<=maxSymbolValue; s++) { - symbolTT[s].deltaNbBits = deltaNbBits; - symbolTT[s].deltaFindState = s-1; - } } - - - return 0; -} - -/* fake FSE_CTable, for rle (100% always same symbol) input */ -size_t FSE_buildCTable_rle (FSE_CTable* ct, BYTE symbolValue) -{ - void* ptr = ct; - U16* tableU16 = ( (U16*) ptr) + 2; - void* FSCTptr = (U32*)ptr + 2; - FSE_symbolCompressionTransform* symbolTT = (FSE_symbolCompressionTransform*) FSCTptr; - - /* header */ - tableU16[-2] = (U16) 0; - tableU16[-1] = (U16) symbolValue; - - /* Build table */ - tableU16[0] = 0; - tableU16[1] = 0; /* just in case */ - - /* Build Symbol Transformation Table */ - symbolTT[symbolValue].deltaNbBits = 0; - symbolTT[symbolValue].deltaFindState = 0; - - return 0; -} - - -static size_t FSE_compress_usingCTable_generic (void* dst, size_t dstSize, - const void* src, size_t srcSize, - const FSE_CTable* ct, const unsigned fast) -{ - const BYTE* const istart = (const BYTE*) src; - const BYTE* const iend = istart + srcSize; - const BYTE* ip=iend; - - - BIT_CStream_t bitC; - FSE_CState_t CState1, CState2; - - /* init */ - if (srcSize <= 2) return 0; - { size_t const errorCode = BIT_initCStream(&bitC, dst, dstSize); - if (FSE_isError(errorCode)) return 0; } - -#define FSE_FLUSHBITS(s) (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s)) - - if (srcSize & 1) { - FSE_initCState2(&CState1, ct, *--ip); - FSE_initCState2(&CState2, ct, *--ip); - FSE_encodeSymbol(&bitC, &CState1, *--ip); - FSE_FLUSHBITS(&bitC); - } else { - FSE_initCState2(&CState2, ct, *--ip); - FSE_initCState2(&CState1, ct, *--ip); - } - - /* join to mod 4 */ - srcSize -= 2; - if ((sizeof(bitC.bitContainer)*8 > FSE_MAX_TABLELOG*4+7 ) && (srcSize & 2)) { /* test bit 2 */ - FSE_encodeSymbol(&bitC, &CState2, *--ip); - FSE_encodeSymbol(&bitC, &CState1, *--ip); - FSE_FLUSHBITS(&bitC); - } - - /* 2 or 4 encoding per loop */ - for ( ; ip>istart ; ) { - - FSE_encodeSymbol(&bitC, &CState2, *--ip); - - if (sizeof(bitC.bitContainer)*8 < FSE_MAX_TABLELOG*2+7 ) /* this test must be static */ - FSE_FLUSHBITS(&bitC); - - FSE_encodeSymbol(&bitC, &CState1, *--ip); - - if (sizeof(bitC.bitContainer)*8 > FSE_MAX_TABLELOG*4+7 ) { /* this test must be static */ - FSE_encodeSymbol(&bitC, &CState2, *--ip); - FSE_encodeSymbol(&bitC, &CState1, *--ip); - } - - FSE_FLUSHBITS(&bitC); - } - - FSE_flushCState(&bitC, &CState2); - FSE_flushCState(&bitC, &CState1); - return BIT_closeCStream(&bitC); -} - -size_t FSE_compress_usingCTable (void* dst, size_t dstSize, - const void* src, size_t srcSize, - const FSE_CTable* ct) -{ - const unsigned fast = (dstSize >= FSE_BLOCKBOUND(srcSize)); - - if (fast) - return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 1); - else - return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 0); -} - - -size_t FSE_compressBound(size_t size) { return FSE_COMPRESSBOUND(size); } - -size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog) -{ - const BYTE* const istart = (const BYTE*) src; - const BYTE* ip = istart; - - BYTE* const ostart = (BYTE*) dst; - BYTE* op = ostart; - BYTE* const oend = ostart + dstSize; - - U32 count[FSE_MAX_SYMBOL_VALUE+1]; - S16 norm[FSE_MAX_SYMBOL_VALUE+1]; - CTable_max_t ct; - size_t errorCode; - - /* init conditions */ - if (srcSize <= 1) return 0; /* Uncompressible */ - if (!maxSymbolValue) maxSymbolValue = FSE_MAX_SYMBOL_VALUE; - if (!tableLog) tableLog = FSE_DEFAULT_TABLELOG; - - /* Scan input and build symbol stats */ - errorCode = FSE_count (count, &maxSymbolValue, ip, srcSize); - if (FSE_isError(errorCode)) return errorCode; - if (errorCode == srcSize) return 1; - if (errorCode == 1) return 0; /* each symbol only present once */ - if (errorCode < (srcSize >> 7)) return 0; /* Heuristic : not compressible enough */ - - tableLog = FSE_optimalTableLog(tableLog, srcSize, maxSymbolValue); - errorCode = FSE_normalizeCount (norm, tableLog, count, srcSize, maxSymbolValue); - if (FSE_isError(errorCode)) return errorCode; - - /* Write table description header */ - errorCode = FSE_writeNCount (op, oend-op, norm, maxSymbolValue, tableLog); - if (FSE_isError(errorCode)) return errorCode; - op += errorCode; - - /* Compress */ - errorCode = FSE_buildCTable (ct, norm, maxSymbolValue, tableLog); - if (FSE_isError(errorCode)) return errorCode; - errorCode = FSE_compress_usingCTable(op, oend - op, ip, srcSize, ct); - if (errorCode == 0) return 0; /* not enough space for compressed data */ - op += errorCode; - - /* check compressibility */ - if ( (size_t)(op-ostart) >= srcSize-1 ) - return 0; - - return op-ostart; -} - -size_t FSE_compress (void* dst, size_t dstSize, const void* src, size_t srcSize) -{ - return FSE_compress2(dst, dstSize, src, (U32)srcSize, FSE_MAX_SYMBOL_VALUE, FSE_DEFAULT_TABLELOG); -} - - -#endif /* FSE_COMMONDEFS_ONLY */ +/* ****************************************************************** + FSE : Finite State Entropy encoder + Copyright (C) 2013-2015, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + - Public forum : https://groups.google.com/forum/#!forum/lz4c +****************************************************************** */ + +/* ************************************************************** +* Compiler specifics +****************************************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# define FORCE_INLINE static __forceinline +# include <intrin.h> /* For Visual 2005 */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */ +#else +# ifdef __GNUC__ +# define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif +#endif + + +/* ************************************************************** +* Includes +****************************************************************/ +#include <stdlib.h> /* malloc, free, qsort */ +#include <string.h> /* memcpy, memset */ +#include <stdio.h> /* printf (debug) */ +#include "bitstream.h" +#include "fse_static.h" + + +/* ************************************************************** +* Error Management +****************************************************************/ +#define FSE_STATIC_ASSERT(c) { enum { FSE_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ + + +/* ************************************************************** +* Complex types +****************************************************************/ +typedef U32 CTable_max_t[FSE_CTABLE_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)]; + + +/* ************************************************************** +* Templates +****************************************************************/ +/* + designed to be included + for type-specific functions (template emulation in C) + Objective is to write these functions only once, for improved maintenance +*/ + +/* safety checks */ +#ifndef FSE_FUNCTION_EXTENSION +# error "FSE_FUNCTION_EXTENSION must be defined" +#endif +#ifndef FSE_FUNCTION_TYPE +# error "FSE_FUNCTION_TYPE must be defined" +#endif + +/* Function names */ +#define FSE_CAT(X,Y) X##Y +#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y) +#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y) + + +/* Function templates */ +size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) +{ + U32 const tableSize = 1 << tableLog; + U32 const tableMask = tableSize - 1; + void* const ptr = ct; + U16* const tableU16 = ( (U16*) ptr) + 2; + void* const FSCT = ((U32*)ptr) + 1 /* header */ + (tableLog ? tableSize>>1 : 1) ; + FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT); + U32 const step = FSE_TABLESTEP(tableSize); + U32 cumul[FSE_MAX_SYMBOL_VALUE+2]; + + FSE_FUNCTION_TYPE tableSymbol[FSE_MAX_TABLESIZE]; /* memset() is not necessary, even if static analyzer complain about it */ + U32 highThreshold = tableSize-1; + + /* CTable header */ + + + tableU16[-2] = (U16) tableLog; + tableU16[-1] = (U16) maxSymbolValue; + + /* For explanations on how to distribute symbol values over the table : + * http://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */ + + /* symbol start positions */ + { U32 u; + cumul[0] = 0; + for (u=1; u<=maxSymbolValue+1; u++) { + if (normalizedCounter[u-1]==-1) { /* Low proba symbol */ + cumul[u] = cumul[u-1] + 1; + tableSymbol[highThreshold--] = (FSE_FUNCTION_TYPE)(u-1); + } else { + cumul[u] = cumul[u-1] + normalizedCounter[u-1]; + } } + cumul[maxSymbolValue+1] = tableSize+1; + } + + /* Spread symbols */ + { U32 position = 0; + U32 symbol; + for (symbol=0; symbol<=maxSymbolValue; symbol++) { + int nbOccurences; + for (nbOccurences=0; nbOccurences<normalizedCounter[symbol]; nbOccurences++) { + tableSymbol[position] = (FSE_FUNCTION_TYPE)symbol; + position = (position + step) & tableMask; + while (position > highThreshold) position = (position + step) & tableMask; /* Low proba area */ + } } + + if (position!=0) return ERROR(GENERIC); /* Must have gone through all positions */ + } + + /* Build table */ + { U32 u; for (u=0; u<tableSize; u++) { + FSE_FUNCTION_TYPE s = tableSymbol[u]; /* note : static analyzer may not understand tableSymbol is properly initialized */ + tableU16[cumul[s]++] = (U16) (tableSize+u); /* TableU16 : sorted by symbol order; gives next state value */ + }} + + /* Build Symbol Transformation Table */ + { unsigned total = 0; + unsigned s; + for (s=0; s<=maxSymbolValue; s++) { + switch (normalizedCounter[s]) + { + case 0: break; + + case -1: + case 1: + symbolTT[s].deltaNbBits = (tableLog << 16) - (1<<tableLog); + symbolTT[s].deltaFindState = total - 1; + total ++; + break; + default : + { + U32 const maxBitsOut = tableLog - BIT_highbit32 (normalizedCounter[s]-1); + U32 const minStatePlus = normalizedCounter[s] << maxBitsOut; + symbolTT[s].deltaNbBits = (maxBitsOut << 16) - minStatePlus; + symbolTT[s].deltaFindState = total - normalizedCounter[s]; + total += normalizedCounter[s]; + } } } } + + return 0; +} + + + +#ifndef FSE_COMMONDEFS_ONLY + +/*-************************************************************** +* FSE NCount encoding-decoding +****************************************************************/ +size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog) +{ + size_t maxHeaderSize = (((maxSymbolValue+1) * tableLog) >> 3) + 3; + return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */ +} + +static short FSE_abs(short a) { return a<0 ? -a : a; } + +static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize, + const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, + unsigned writeIsSafe) +{ + BYTE* const ostart = (BYTE*) header; + BYTE* out = ostart; + BYTE* const oend = ostart + headerBufferSize; + int nbBits; + const int tableSize = 1 << tableLog; + int remaining; + int threshold; + U32 bitStream; + int bitCount; + unsigned charnum = 0; + int previous0 = 0; + + bitStream = 0; + bitCount = 0; + /* Table Size */ + bitStream += (tableLog-FSE_MIN_TABLELOG) << bitCount; + bitCount += 4; + + /* Init */ + remaining = tableSize+1; /* +1 for extra accuracy */ + threshold = tableSize; + nbBits = tableLog+1; + + while (remaining>1) { /* stops at 1 */ + if (previous0) { + unsigned start = charnum; + while (!normalizedCounter[charnum]) charnum++; + while (charnum >= start+24) { + start+=24; + bitStream += 0xFFFFU << bitCount; + if ((!writeIsSafe) && (out > oend-2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ + out[0] = (BYTE) bitStream; + out[1] = (BYTE)(bitStream>>8); + out+=2; + bitStream>>=16; + } + while (charnum >= start+3) { + start+=3; + bitStream += 3 << bitCount; + bitCount += 2; + } + bitStream += (charnum-start) << bitCount; + bitCount += 2; + if (bitCount>16) { + if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ + out[0] = (BYTE)bitStream; + out[1] = (BYTE)(bitStream>>8); + out += 2; + bitStream >>= 16; + bitCount -= 16; + } } + { short count = normalizedCounter[charnum++]; + const short max = (short)((2*threshold-1)-remaining); + remaining -= FSE_abs(count); + if (remaining<1) return ERROR(GENERIC); + count++; /* +1 for extra accuracy */ + if (count>=threshold) count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */ + bitStream += count << bitCount; + bitCount += nbBits; + bitCount -= (count<max); + previous0 = (count==1); + while (remaining<threshold) nbBits--, threshold>>=1; + } + if (bitCount>16) { + if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ + out[0] = (BYTE)bitStream; + out[1] = (BYTE)(bitStream>>8); + out += 2; + bitStream >>= 16; + bitCount -= 16; + } } + + /* flush remaining bitStream */ + if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ + out[0] = (BYTE)bitStream; + out[1] = (BYTE)(bitStream>>8); + out+= (bitCount+7) /8; + + if (charnum > maxSymbolValue + 1) return ERROR(GENERIC); + + return (out-ostart); +} + + +size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) +{ + if (tableLog > FSE_MAX_TABLELOG) return ERROR(GENERIC); /* Unsupported */ + if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported */ + + if (bufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog)) + return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 0); + + return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1); +} + + + +/*-************************************************************** +* Counting histogram +****************************************************************/ +/*! FSE_count_simple + This function just counts byte values within `src`, + and store the histogram into table `count`. + This function is unsafe : it doesn't check that all values within `src` can fit into `count`. + For this reason, prefer using a table `count` with 256 elements. + @return : count of most numerous element +*/ +static size_t FSE_count_simple(unsigned* count, unsigned* maxSymbolValuePtr, + const void* src, size_t srcSize) +{ + const BYTE* ip = (const BYTE*)src; + const BYTE* const end = ip + srcSize; + unsigned maxSymbolValue = *maxSymbolValuePtr; + unsigned max=0; + + + memset(count, 0, (maxSymbolValue+1)*sizeof(*count)); + if (srcSize==0) { *maxSymbolValuePtr = 0; return 0; } + + while (ip<end) count[*ip++]++; + + while (!count[maxSymbolValue]) maxSymbolValue--; + *maxSymbolValuePtr = maxSymbolValue; + + { U32 s; for (s=0; s<=maxSymbolValue; s++) if (count[s] > max) max = count[s]; } + + return (size_t)max; +} + + +static size_t FSE_count_parallel(unsigned* count, unsigned* maxSymbolValuePtr, + const void* source, size_t sourceSize, + unsigned checkMax) +{ + const BYTE* ip = (const BYTE*)source; + const BYTE* const iend = ip+sourceSize; + unsigned maxSymbolValue = *maxSymbolValuePtr; + unsigned max=0; + + + U32 Counting1[256] = { 0 }; + U32 Counting2[256] = { 0 }; + U32 Counting3[256] = { 0 }; + U32 Counting4[256] = { 0 }; + + /* safety checks */ + if (!sourceSize) { + memset(count, 0, maxSymbolValue + 1); + *maxSymbolValuePtr = 0; + return 0; + } + if (!maxSymbolValue) maxSymbolValue = 255; /* 0 == default */ + + /* by stripes of 16 bytes */ + { U32 cached = MEM_read32(ip); ip += 4; + while (ip < iend-15) { + U32 c = cached; cached = MEM_read32(ip); ip += 4; + Counting1[(BYTE) c ]++; + Counting2[(BYTE)(c>>8) ]++; + Counting3[(BYTE)(c>>16)]++; + Counting4[ c>>24 ]++; + c = cached; cached = MEM_read32(ip); ip += 4; + Counting1[(BYTE) c ]++; + Counting2[(BYTE)(c>>8) ]++; + Counting3[(BYTE)(c>>16)]++; + Counting4[ c>>24 ]++; + c = cached; cached = MEM_read32(ip); ip += 4; + Counting1[(BYTE) c ]++; + Counting2[(BYTE)(c>>8) ]++; + Counting3[(BYTE)(c>>16)]++; + Counting4[ c>>24 ]++; + c = cached; cached = MEM_read32(ip); ip += 4; + Counting1[(BYTE) c ]++; + Counting2[(BYTE)(c>>8) ]++; + Counting3[(BYTE)(c>>16)]++; + Counting4[ c>>24 ]++; + } + ip-=4; + } + + /* finish last symbols */ + while (ip<iend) Counting1[*ip++]++; + + if (checkMax) { /* verify stats will fit into destination table */ + U32 s; for (s=255; s>maxSymbolValue; s--) { + Counting1[s] += Counting2[s] + Counting3[s] + Counting4[s]; + if (Counting1[s]) return ERROR(maxSymbolValue_tooSmall); + } } + + { U32 s; for (s=0; s<=maxSymbolValue; s++) { + count[s] = Counting1[s] + Counting2[s] + Counting3[s] + Counting4[s]; + if (count[s] > max) max = count[s]; + }} + + while (!count[maxSymbolValue]) maxSymbolValue--; + *maxSymbolValuePtr = maxSymbolValue; + return (size_t)max; +} + +/* fast variant (unsafe : won't check if src contains values beyond count[] limit) */ +size_t FSE_countFast(unsigned* count, unsigned* maxSymbolValuePtr, + const void* source, size_t sourceSize) +{ + if (sourceSize < 1500) return FSE_count_simple(count, maxSymbolValuePtr, source, sourceSize); + return FSE_count_parallel(count, maxSymbolValuePtr, source, sourceSize, 0); +} + +size_t FSE_count(unsigned* count, unsigned* maxSymbolValuePtr, + const void* source, size_t sourceSize) +{ + if (*maxSymbolValuePtr <255) + return FSE_count_parallel(count, maxSymbolValuePtr, source, sourceSize, 1); + *maxSymbolValuePtr = 255; + return FSE_countFast(count, maxSymbolValuePtr, source, sourceSize); +} + + + +/*-************************************************************** +* FSE Compression Code +****************************************************************/ +/*! FSE_sizeof_CTable() : + FSE_CTable is a variable size structure which contains : + `U16 tableLog;` + `U16 maxSymbolValue;` + `U16 nextStateNumber[1 << tableLog];` // This size is variable + `FSE_symbolCompressionTransform symbolTT[maxSymbolValue+1];` // This size is variable +Allocation is manual (C standard does not support variable-size structures). +*/ + +size_t FSE_sizeof_CTable (unsigned maxSymbolValue, unsigned tableLog) +{ + size_t size; + FSE_STATIC_ASSERT((size_t)FSE_CTABLE_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)*4 >= sizeof(CTable_max_t)); /* A compilation error here means FSE_CTABLE_SIZE_U32 is not large enough */ + if (tableLog > FSE_MAX_TABLELOG) return ERROR(GENERIC); + size = FSE_CTABLE_SIZE_U32 (tableLog, maxSymbolValue) * sizeof(U32); + return size; +} + +FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog) +{ + size_t size; + if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX; + size = FSE_CTABLE_SIZE_U32 (tableLog, maxSymbolValue) * sizeof(U32); + return (FSE_CTable*)malloc(size); +} + +void FSE_freeCTable (FSE_CTable* ct) { free(ct); } + +/* provides the minimum logSize to safely represent a distribution */ +static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue) +{ + U32 minBitsSrc = BIT_highbit32((U32)(srcSize - 1)) + 1; + U32 minBitsSymbols = BIT_highbit32(maxSymbolValue) + 2; + U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols; + return minBits; +} + +unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue) +{ + U32 maxBitsSrc = BIT_highbit32((U32)(srcSize - 1)) - 2; + U32 tableLog = maxTableLog; + U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue); + if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG; + if (maxBitsSrc < tableLog) tableLog = maxBitsSrc; /* Accuracy can be reduced */ + if (minBits > tableLog) tableLog = minBits; /* Need a minimum to safely represent all symbol values */ + if (tableLog < FSE_MIN_TABLELOG) tableLog = FSE_MIN_TABLELOG; + if (tableLog > FSE_MAX_TABLELOG) tableLog = FSE_MAX_TABLELOG; + return tableLog; +} + + +/* Secondary normalization method. + To be used when primary method fails. */ + +static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, size_t total, U32 maxSymbolValue) +{ + U32 s; + U32 distributed = 0; + U32 ToDistribute; + + /* Init */ + U32 lowThreshold = (U32)(total >> tableLog); + U32 lowOne = (U32)((total * 3) >> (tableLog + 1)); + + for (s=0; s<=maxSymbolValue; s++) { + if (count[s] == 0) { + norm[s]=0; + continue; + } + if (count[s] <= lowThreshold) { + norm[s] = -1; + distributed++; + total -= count[s]; + continue; + } + if (count[s] <= lowOne) { + norm[s] = 1; + distributed++; + total -= count[s]; + continue; + } + norm[s]=-2; + } + ToDistribute = (1 << tableLog) - distributed; + + if ((total / ToDistribute) > lowOne) { + /* risk of rounding to zero */ + lowOne = (U32)((total * 3) / (ToDistribute * 2)); + for (s=0; s<=maxSymbolValue; s++) { + if ((norm[s] == -2) && (count[s] <= lowOne)) { + norm[s] = 1; + distributed++; + total -= count[s]; + continue; + } } + ToDistribute = (1 << tableLog) - distributed; + } + + if (distributed == maxSymbolValue+1) { + /* all values are pretty poor; + probably incompressible data (should have already been detected); + find max, then give all remaining points to max */ + U32 maxV = 0, maxC = 0; + for (s=0; s<=maxSymbolValue; s++) + if (count[s] > maxC) maxV=s, maxC=count[s]; + norm[maxV] += (short)ToDistribute; + return 0; + } + + { + U64 const vStepLog = 62 - tableLog; + U64 const mid = (1ULL << (vStepLog-1)) - 1; + U64 const rStep = ((((U64)1<<vStepLog) * ToDistribute) + mid) / total; /* scale on remaining */ + U64 tmpTotal = mid; + for (s=0; s<=maxSymbolValue; s++) { + if (norm[s]==-2) { + U64 end = tmpTotal + (count[s] * rStep); + U32 sStart = (U32)(tmpTotal >> vStepLog); + U32 sEnd = (U32)(end >> vStepLog); + U32 weight = sEnd - sStart; + if (weight < 1) + return ERROR(GENERIC); + norm[s] = (short)weight; + tmpTotal = end; + } } } + + return 0; +} + + +size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog, + const unsigned* count, size_t total, + unsigned maxSymbolValue) +{ + /* Sanity checks */ + if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG; + if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported size */ + if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Unsupported size */ + if (tableLog < FSE_minTableLog(total, maxSymbolValue)) return ERROR(GENERIC); /* Too small tableLog, compression potentially impossible */ + + { U32 const rtbTable[] = { 0, 473195, 504333, 520860, 550000, 700000, 750000, 830000 }; + + U64 const scale = 62 - tableLog; + U64 const step = ((U64)1<<62) / total; /* <== here, one division ! */ + U64 const vStep = 1ULL<<(scale-20); + int stillToDistribute = 1<<tableLog; + unsigned s; + unsigned largest=0; + short largestP=0; + U32 lowThreshold = (U32)(total >> tableLog); + + for (s=0; s<=maxSymbolValue; s++) { + if (count[s] == total) return 0; /* rle special case */ + if (count[s] == 0) { normalizedCounter[s]=0; continue; } + if (count[s] <= lowThreshold) { + normalizedCounter[s] = -1; + stillToDistribute--; + } else { + short proba = (short)((count[s]*step) >> scale); + if (proba<8) { + U64 restToBeat = vStep * rtbTable[proba]; + proba += (count[s]*step) - ((U64)proba<<scale) > restToBeat; + } + if (proba > largestP) largestP=proba, largest=s; + normalizedCounter[s] = proba; + stillToDistribute -= proba; + } } + if (-stillToDistribute >= (normalizedCounter[largest] >> 1)) { + /* corner case, need another normalization method */ + size_t errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue); + if (FSE_isError(errorCode)) return errorCode; + } + else normalizedCounter[largest] += (short)stillToDistribute; + } + +#if 0 + { /* Print Table (debug) */ + U32 s; + U32 nTotal = 0; + for (s=0; s<=maxSymbolValue; s++) + printf("%3i: %4i \n", s, normalizedCounter[s]); + for (s=0; s<=maxSymbolValue; s++) + nTotal += abs(normalizedCounter[s]); + if (nTotal != (1U<<tableLog)) + printf("Warning !!! Total == %u != %u !!!", nTotal, 1U<<tableLog); + getchar(); + } +#endif + + return tableLog; +} + + +/* fake FSE_CTable, for raw (uncompressed) input */ +size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits) +{ + const unsigned tableSize = 1 << nbBits; + const unsigned tableMask = tableSize - 1; + const unsigned maxSymbolValue = tableMask; + void* const ptr = ct; + U16* const tableU16 = ( (U16*) ptr) + 2; + void* const FSCT = ((U32*)ptr) + 1 /* header */ + (tableSize>>1); /* assumption : tableLog >= 1 */ + FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT); + unsigned s; + + /* Sanity checks */ + if (nbBits < 1) return ERROR(GENERIC); /* min size */ + + /* header */ + tableU16[-2] = (U16) nbBits; + tableU16[-1] = (U16) maxSymbolValue; + + /* Build table */ + for (s=0; s<tableSize; s++) + tableU16[s] = (U16)(tableSize + s); + + /* Build Symbol Transformation Table */ + { const U32 deltaNbBits = (nbBits << 16) - (1 << nbBits); + + for (s=0; s<=maxSymbolValue; s++) { + symbolTT[s].deltaNbBits = deltaNbBits; + symbolTT[s].deltaFindState = s-1; + } } + + + return 0; +} + +/* fake FSE_CTable, for rle (100% always same symbol) input */ +size_t FSE_buildCTable_rle (FSE_CTable* ct, BYTE symbolValue) +{ + void* ptr = ct; + U16* tableU16 = ( (U16*) ptr) + 2; + void* FSCTptr = (U32*)ptr + 2; + FSE_symbolCompressionTransform* symbolTT = (FSE_symbolCompressionTransform*) FSCTptr; + + /* header */ + tableU16[-2] = (U16) 0; + tableU16[-1] = (U16) symbolValue; + + /* Build table */ + tableU16[0] = 0; + tableU16[1] = 0; /* just in case */ + + /* Build Symbol Transformation Table */ + symbolTT[symbolValue].deltaNbBits = 0; + symbolTT[symbolValue].deltaFindState = 0; + + return 0; +} + + +static size_t FSE_compress_usingCTable_generic (void* dst, size_t dstSize, + const void* src, size_t srcSize, + const FSE_CTable* ct, const unsigned fast) +{ + const BYTE* const istart = (const BYTE*) src; + const BYTE* const iend = istart + srcSize; + const BYTE* ip=iend; + + + BIT_CStream_t bitC; + FSE_CState_t CState1, CState2; + + /* init */ + if (srcSize <= 2) return 0; + { size_t const errorCode = BIT_initCStream(&bitC, dst, dstSize); + if (FSE_isError(errorCode)) return 0; } + +#define FSE_FLUSHBITS(s) (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s)) + + if (srcSize & 1) { + FSE_initCState2(&CState1, ct, *--ip); + FSE_initCState2(&CState2, ct, *--ip); + FSE_encodeSymbol(&bitC, &CState1, *--ip); + FSE_FLUSHBITS(&bitC); + } else { + FSE_initCState2(&CState2, ct, *--ip); + FSE_initCState2(&CState1, ct, *--ip); + } + + /* join to mod 4 */ + srcSize -= 2; + if ((sizeof(bitC.bitContainer)*8 > FSE_MAX_TABLELOG*4+7 ) && (srcSize & 2)) { /* test bit 2 */ + FSE_encodeSymbol(&bitC, &CState2, *--ip); + FSE_encodeSymbol(&bitC, &CState1, *--ip); + FSE_FLUSHBITS(&bitC); + } + + /* 2 or 4 encoding per loop */ + for ( ; ip>istart ; ) { + + FSE_encodeSymbol(&bitC, &CState2, *--ip); + + if (sizeof(bitC.bitContainer)*8 < FSE_MAX_TABLELOG*2+7 ) /* this test must be static */ + FSE_FLUSHBITS(&bitC); + + FSE_encodeSymbol(&bitC, &CState1, *--ip); + + if (sizeof(bitC.bitContainer)*8 > FSE_MAX_TABLELOG*4+7 ) { /* this test must be static */ + FSE_encodeSymbol(&bitC, &CState2, *--ip); + FSE_encodeSymbol(&bitC, &CState1, *--ip); + } + + FSE_FLUSHBITS(&bitC); + } + + FSE_flushCState(&bitC, &CState2); + FSE_flushCState(&bitC, &CState1); + return BIT_closeCStream(&bitC); +} + +size_t FSE_compress_usingCTable (void* dst, size_t dstSize, + const void* src, size_t srcSize, + const FSE_CTable* ct) +{ + const unsigned fast = (dstSize >= FSE_BLOCKBOUND(srcSize)); + + if (fast) + return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 1); + else + return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 0); +} + + +size_t FSE_compressBound(size_t size) { return FSE_COMPRESSBOUND(size); } + +size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog) +{ + const BYTE* const istart = (const BYTE*) src; + const BYTE* ip = istart; + + BYTE* const ostart = (BYTE*) dst; + BYTE* op = ostart; + BYTE* const oend = ostart + dstSize; + + U32 count[FSE_MAX_SYMBOL_VALUE+1]; + S16 norm[FSE_MAX_SYMBOL_VALUE+1]; + CTable_max_t ct; + size_t errorCode; + + /* init conditions */ + if (srcSize <= 1) return 0; /* Uncompressible */ + if (!maxSymbolValue) maxSymbolValue = FSE_MAX_SYMBOL_VALUE; + if (!tableLog) tableLog = FSE_DEFAULT_TABLELOG; + + /* Scan input and build symbol stats */ + errorCode = FSE_count (count, &maxSymbolValue, ip, srcSize); + if (FSE_isError(errorCode)) return errorCode; + if (errorCode == srcSize) return 1; + if (errorCode == 1) return 0; /* each symbol only present once */ + if (errorCode < (srcSize >> 7)) return 0; /* Heuristic : not compressible enough */ + + tableLog = FSE_optimalTableLog(tableLog, srcSize, maxSymbolValue); + errorCode = FSE_normalizeCount (norm, tableLog, count, srcSize, maxSymbolValue); + if (FSE_isError(errorCode)) return errorCode; + + /* Write table description header */ + errorCode = FSE_writeNCount (op, oend-op, norm, maxSymbolValue, tableLog); + if (FSE_isError(errorCode)) return errorCode; + op += errorCode; + + /* Compress */ + errorCode = FSE_buildCTable (ct, norm, maxSymbolValue, tableLog); + if (FSE_isError(errorCode)) return errorCode; + errorCode = FSE_compress_usingCTable(op, oend - op, ip, srcSize, ct); + if (errorCode == 0) return 0; /* not enough space for compressed data */ + op += errorCode; + + /* check compressibility */ + if ( (size_t)(op-ostart) >= srcSize-1 ) + return 0; + + return op-ostart; +} + +size_t FSE_compress (void* dst, size_t dstSize, const void* src, size_t srcSize) +{ + return FSE_compress2(dst, dstSize, src, (U32)srcSize, FSE_MAX_SYMBOL_VALUE, FSE_DEFAULT_TABLELOG); +} + + +#endif /* FSE_COMMONDEFS_ONLY */ diff --git a/contrib/libs/zstd06/compress/huf_compress.c b/contrib/libs/zstd06/compress/huf_compress.c index a110d86719..d126305c6f 100644 --- a/contrib/libs/zstd06/compress/huf_compress.c +++ b/contrib/libs/zstd06/compress/huf_compress.c @@ -1,560 +1,560 @@ -/* ****************************************************************** - Huffman encoder, part of New Generation Entropy library - Copyright (C) 2013-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy - - Public forum : https://groups.google.com/forum/#!forum/lz4c -****************************************************************** */ - -/* ************************************************************** -* Compiler specifics -****************************************************************/ -#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -/* inline is defined */ -#elif defined(_MSC_VER) -# define inline __inline -#else -# define inline /* disable inline */ -#endif - - -#ifdef _MSC_VER /* Visual Studio */ -# define FORCE_INLINE static __forceinline -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -#else -# ifdef __GNUC__ -# define FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define FORCE_INLINE static inline -# endif -#endif - - -/* ************************************************************** -* Includes -****************************************************************/ -#include <stdlib.h> /* malloc, free, qsort */ -#include <string.h> /* memcpy, memset */ -#include <stdio.h> /* printf (debug) */ -#include "huf_static.h" -#include "bitstream.h" -#include "fse.h" /* header compression */ - - -/* ************************************************************** -* Error Management -****************************************************************/ -#define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ - - -/* ******************************************************* -* HUF : Huffman block compression -*********************************************************/ -struct HUF_CElt_s { - U16 val; - BYTE nbBits; -}; /* typedef'd to HUF_CElt within huf_static.h */ - -typedef struct nodeElt_s { - U32 count; - U16 parent; - BYTE byte; - BYTE nbBits; -} nodeElt; - -/*! HUF_writeCTable() : - `CTable` : huffman tree to save, using huf representation. - @return : size of saved CTable */ -size_t HUF_writeCTable (void* dst, size_t maxDstSize, - const HUF_CElt* CTable, U32 maxSymbolValue, U32 huffLog) -{ - BYTE bitsToWeight[HUF_MAX_TABLELOG + 1]; - BYTE huffWeight[HUF_MAX_SYMBOL_VALUE + 1]; - U32 n; - BYTE* op = (BYTE*)dst; - size_t size; - - /* check conditions */ - if (maxSymbolValue > HUF_MAX_SYMBOL_VALUE + 1) - return ERROR(GENERIC); - - /* convert to weight */ - bitsToWeight[0] = 0; - for (n=1; n<=huffLog; n++) - bitsToWeight[n] = (BYTE)(huffLog + 1 - n); - for (n=0; n<maxSymbolValue; n++) - huffWeight[n] = bitsToWeight[CTable[n].nbBits]; - - size = FSE_compress(op+1, maxDstSize-1, huffWeight, maxSymbolValue); /* don't need last symbol stat : implied */ - if (HUF_isError(size)) return size; - if (size >= 128) return ERROR(GENERIC); /* should never happen, since maxSymbolValue <= 255 */ - if ((size <= 1) || (size >= maxSymbolValue/2)) { - if (size==1) { /* RLE */ - /* only possible case : serie of 1 (because there are at least 2) */ - /* can only be 2^n or (2^n-1), otherwise not an huffman tree */ - BYTE code; - switch(maxSymbolValue) - { - case 1: code = 0; break; - case 2: code = 1; break; - case 3: code = 2; break; - case 4: code = 3; break; - case 7: code = 4; break; - case 8: code = 5; break; - case 15: code = 6; break; - case 16: code = 7; break; - case 31: code = 8; break; - case 32: code = 9; break; - case 63: code = 10; break; - case 64: code = 11; break; - case 127: code = 12; break; - case 128: code = 13; break; - default : return ERROR(corruption_detected); - } - op[0] = (BYTE)(255-13 + code); - return 1; - } - /* Not compressible */ - if (maxSymbolValue > (241-128)) return ERROR(GENERIC); /* not implemented (not possible with current format) */ - if (((maxSymbolValue+1)/2) + 1 > maxDstSize) return ERROR(dstSize_tooSmall); /* not enough space within dst buffer */ - op[0] = (BYTE)(128 /*special case*/ + 0 /* Not Compressible */ + (maxSymbolValue-1)); - huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause issue in final combination */ - for (n=0; n<maxSymbolValue; n+=2) - op[(n/2)+1] = (BYTE)((huffWeight[n] << 4) + huffWeight[n+1]); - return ((maxSymbolValue+1)/2) + 1; - } - - /* normal header case */ - op[0] = (BYTE)size; - return size+1; -} - - - -size_t HUF_readCTable (HUF_CElt* CTable, U32 maxSymbolValue, const void* src, size_t srcSize) -{ - BYTE huffWeight[HUF_MAX_SYMBOL_VALUE + 1]; - U32 rankVal[HUF_ABSOLUTEMAX_TABLELOG + 1]; /* large enough for values from 0 to 16 */ - U32 tableLog = 0; - size_t readSize; - U32 nbSymbols = 0; - //memset(huffWeight, 0, sizeof(huffWeight)); /* is not necessary, even though some analyzer complain ... */ - - /* get symbol weights */ - readSize = HUF_readStats(huffWeight, HUF_MAX_SYMBOL_VALUE+1, rankVal, &nbSymbols, &tableLog, src, srcSize); - if (HUF_isError(readSize)) return readSize; - - /* check result */ - if (tableLog > HUF_MAX_TABLELOG) return ERROR(tableLog_tooLarge); - if (nbSymbols > maxSymbolValue+1) return ERROR(maxSymbolValue_tooSmall); - - /* Prepare base value per rank */ - { U32 n, nextRankStart = 0; - for (n=1; n<=tableLog; n++) { - U32 current = nextRankStart; - nextRankStart += (rankVal[n] << (n-1)); - rankVal[n] = current; - } } - - /* fill nbBits */ - { U32 n; for (n=0; n<nbSymbols; n++) { - const U32 w = huffWeight[n]; - CTable[n].nbBits = (BYTE)(tableLog + 1 - w); - }} - - /* fill val */ - { U16 nbPerRank[HUF_MAX_TABLELOG+1] = {0}; - U16 valPerRank[HUF_MAX_TABLELOG+1] = {0}; - { U32 n; for (n=0; n<nbSymbols; n++) nbPerRank[CTable[n].nbBits]++; } - /* determine stating value per rank */ - { U16 min = 0; - U32 n; for (n=HUF_MAX_TABLELOG; n>0; n--) { - valPerRank[n] = min; /* get starting value within each rank */ - min += nbPerRank[n]; - min >>= 1; - } } - /* assign value within rank, symbol order */ - { U32 n; for (n=0; n<=maxSymbolValue; n++) CTable[n].val = valPerRank[CTable[n].nbBits]++; } - } - - return readSize; -} - - -static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits) -{ - const U32 largestBits = huffNode[lastNonNull].nbBits; - if (largestBits <= maxNbBits) return largestBits; /* early exit : no elt > maxNbBits */ - - /* there are several too large elements (at least >= 2) */ - { int totalCost = 0; - const U32 baseCost = 1 << (largestBits - maxNbBits); - U32 n = lastNonNull; - - while (huffNode[n].nbBits > maxNbBits) { - totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits)); - huffNode[n].nbBits = (BYTE)maxNbBits; - n --; - } /* n stops at huffNode[n].nbBits <= maxNbBits */ - while (huffNode[n].nbBits == maxNbBits) n--; /* n end at index of smallest symbol using < maxNbBits */ - - /* renorm totalCost */ - totalCost >>= (largestBits - maxNbBits); /* note : totalCost is necessarily a multiple of baseCost */ - - /* repay normalized cost */ - { U32 const noSymbol = 0xF0F0F0F0; - U32 rankLast[HUF_MAX_TABLELOG+1]; - int pos; - - /* Get pos of last (smallest) symbol per rank */ - memset(rankLast, 0xF0, sizeof(rankLast)); - { U32 currentNbBits = maxNbBits; - for (pos=n ; pos >= 0; pos--) { - if (huffNode[pos].nbBits >= currentNbBits) continue; - currentNbBits = huffNode[pos].nbBits; /* < maxNbBits */ - rankLast[maxNbBits-currentNbBits] = pos; - } } - - while (totalCost > 0) { - U32 nBitsToDecrease = BIT_highbit32(totalCost) + 1; - for ( ; nBitsToDecrease > 1; nBitsToDecrease--) { - U32 highPos = rankLast[nBitsToDecrease]; - U32 lowPos = rankLast[nBitsToDecrease-1]; - if (highPos == noSymbol) continue; - if (lowPos == noSymbol) break; - { U32 const highTotal = huffNode[highPos].count; - U32 const lowTotal = 2 * huffNode[lowPos].count; - if (highTotal <= lowTotal) break; - } } - /* only triggered when no more rank 1 symbol left => find closest one (note : there is necessarily at least one !) */ - while ((nBitsToDecrease<=HUF_MAX_TABLELOG) && (rankLast[nBitsToDecrease] == noSymbol)) /* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */ - nBitsToDecrease ++; - totalCost -= 1 << (nBitsToDecrease-1); - if (rankLast[nBitsToDecrease-1] == noSymbol) - rankLast[nBitsToDecrease-1] = rankLast[nBitsToDecrease]; /* this rank is no longer empty */ - huffNode[rankLast[nBitsToDecrease]].nbBits ++; - if (rankLast[nBitsToDecrease] == 0) /* special case, reached largest symbol */ - rankLast[nBitsToDecrease] = noSymbol; - else { - rankLast[nBitsToDecrease]--; - if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits-nBitsToDecrease) - rankLast[nBitsToDecrease] = noSymbol; /* this rank is now empty */ - } } /* while (totalCost > 0) */ - - while (totalCost < 0) { /* Sometimes, cost correction overshoot */ - if (rankLast[1] == noSymbol) { /* special case : no rank 1 symbol (using maxNbBits-1); let's create one from largest rank 0 (using maxNbBits) */ - while (huffNode[n].nbBits == maxNbBits) n--; - huffNode[n+1].nbBits--; - rankLast[1] = n+1; - totalCost++; - continue; - } - huffNode[ rankLast[1] + 1 ].nbBits--; - rankLast[1]++; - totalCost ++; - } } } /* there are several too large elements (at least >= 2) */ - - return maxNbBits; -} - - -typedef struct { - U32 base; - U32 current; -} rankPos; - -static void HUF_sort(nodeElt* huffNode, const U32* count, U32 maxSymbolValue) -{ - rankPos rank[32]; - U32 n; - - memset(rank, 0, sizeof(rank)); - for (n=0; n<=maxSymbolValue; n++) { - U32 r = BIT_highbit32(count[n] + 1); - rank[r].base ++; - } - for (n=30; n>0; n--) rank[n-1].base += rank[n].base; - for (n=0; n<32; n++) rank[n].current = rank[n].base; - for (n=0; n<=maxSymbolValue; n++) { - U32 const c = count[n]; - U32 const r = BIT_highbit32(c+1) + 1; - U32 pos = rank[r].current++; - while ((pos > rank[r].base) && (c > huffNode[pos-1].count)) huffNode[pos]=huffNode[pos-1], pos--; - huffNode[pos].count = c; - huffNode[pos].byte = (BYTE)n; - } -} - - -#define STARTNODE (HUF_MAX_SYMBOL_VALUE+1) -size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits) -{ - nodeElt huffNode0[2*HUF_MAX_SYMBOL_VALUE+1 +1]; - nodeElt* huffNode = huffNode0 + 1; - U32 n, nonNullRank; - int lowS, lowN; - U16 nodeNb = STARTNODE; - U32 nodeRoot; - - /* safety checks */ - if (maxNbBits == 0) maxNbBits = HUF_DEFAULT_TABLELOG; - if (maxSymbolValue > HUF_MAX_SYMBOL_VALUE) return ERROR(GENERIC); - memset(huffNode0, 0, sizeof(huffNode0)); - - /* sort, decreasing order */ - HUF_sort(huffNode, count, maxSymbolValue); - - /* init for parents */ - nonNullRank = maxSymbolValue; - while(huffNode[nonNullRank].count == 0) nonNullRank--; - lowS = nonNullRank; nodeRoot = nodeNb + lowS - 1; lowN = nodeNb; - huffNode[nodeNb].count = huffNode[lowS].count + huffNode[lowS-1].count; - huffNode[lowS].parent = huffNode[lowS-1].parent = nodeNb; - nodeNb++; lowS-=2; - for (n=nodeNb; n<=nodeRoot; n++) huffNode[n].count = (U32)(1U<<30); - huffNode0[0].count = (U32)(1U<<31); - - /* create parents */ - while (nodeNb <= nodeRoot) { - U32 n1 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++; - U32 n2 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++; - huffNode[nodeNb].count = huffNode[n1].count + huffNode[n2].count; - huffNode[n1].parent = huffNode[n2].parent = nodeNb; - nodeNb++; - } - - /* distribute weights (unlimited tree height) */ - huffNode[nodeRoot].nbBits = 0; - for (n=nodeRoot-1; n>=STARTNODE; n--) - huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1; - for (n=0; n<=nonNullRank; n++) - huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1; - - /* enforce maxTableLog */ - maxNbBits = HUF_setMaxHeight(huffNode, nonNullRank, maxNbBits); - - /* fill result into tree (val, nbBits) */ - { U16 nbPerRank[HUF_MAX_TABLELOG+1] = {0}; - U16 valPerRank[HUF_MAX_TABLELOG+1] = {0}; - if (maxNbBits > HUF_MAX_TABLELOG) return ERROR(GENERIC); /* check fit into table */ - for (n=0; n<=nonNullRank; n++) - nbPerRank[huffNode[n].nbBits]++; - /* determine stating value per rank */ - { U16 min = 0; - for (n=maxNbBits; n>0; n--) { - valPerRank[n] = min; /* get starting value within each rank */ - min += nbPerRank[n]; - min >>= 1; - } } - for (n=0; n<=maxSymbolValue; n++) - tree[huffNode[n].byte].nbBits = huffNode[n].nbBits; /* push nbBits per symbol, symbol order */ - for (n=0; n<=maxSymbolValue; n++) - tree[n].val = valPerRank[tree[n].nbBits]++; /* assign value within rank, symbol order */ - } - - return maxNbBits; -} - -static void HUF_encodeSymbol(BIT_CStream_t* bitCPtr, U32 symbol, const HUF_CElt* CTable) -{ - BIT_addBitsFast(bitCPtr, CTable[symbol].val, CTable[symbol].nbBits); -} - -size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); } - -#define HUF_FLUSHBITS(s) (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s)) - -#define HUF_FLUSHBITS_1(stream) \ - if (sizeof((stream)->bitContainer)*8 < HUF_MAX_TABLELOG*2+7) HUF_FLUSHBITS(stream) - -#define HUF_FLUSHBITS_2(stream) \ - if (sizeof((stream)->bitContainer)*8 < HUF_MAX_TABLELOG*4+7) HUF_FLUSHBITS(stream) - -size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable) -{ - const BYTE* ip = (const BYTE*) src; - BYTE* const ostart = (BYTE*)dst; - BYTE* const oend = ostart + dstSize; - BYTE* op = ostart; - size_t n; - const unsigned fast = (dstSize >= HUF_BLOCKBOUND(srcSize)); - BIT_CStream_t bitC; - - /* init */ - if (dstSize < 8) return 0; /* not enough space to compress */ - { size_t const errorCode = BIT_initCStream(&bitC, op, oend-op); - if (HUF_isError(errorCode)) return 0; } - - n = srcSize & ~3; /* join to mod 4 */ - switch (srcSize & 3) - { - case 3 : HUF_encodeSymbol(&bitC, ip[n+ 2], CTable); - HUF_FLUSHBITS_2(&bitC); - case 2 : HUF_encodeSymbol(&bitC, ip[n+ 1], CTable); - HUF_FLUSHBITS_1(&bitC); - case 1 : HUF_encodeSymbol(&bitC, ip[n+ 0], CTable); - HUF_FLUSHBITS(&bitC); - case 0 : - default: ; - } - - for (; n>0; n-=4) { /* note : n&3==0 at this stage */ - HUF_encodeSymbol(&bitC, ip[n- 1], CTable); - HUF_FLUSHBITS_1(&bitC); - HUF_encodeSymbol(&bitC, ip[n- 2], CTable); - HUF_FLUSHBITS_2(&bitC); - HUF_encodeSymbol(&bitC, ip[n- 3], CTable); - HUF_FLUSHBITS_1(&bitC); - HUF_encodeSymbol(&bitC, ip[n- 4], CTable); - HUF_FLUSHBITS(&bitC); - } - - return BIT_closeCStream(&bitC); -} - - -size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable) -{ - size_t segmentSize = (srcSize+3)/4; /* first 3 segments */ - const BYTE* ip = (const BYTE*) src; - const BYTE* const iend = ip + srcSize; - BYTE* const ostart = (BYTE*) dst; - BYTE* const oend = ostart + dstSize; - BYTE* op = ostart; - size_t errorCode; - - if (dstSize < 6 + 1 + 1 + 1 + 8) return 0; /* minimum space to compress successfully */ - if (srcSize < 12) return 0; /* no saving possible : too small input */ - op += 6; /* jumpTable */ - - errorCode = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable); - if (HUF_isError(errorCode)) return errorCode; - if (errorCode==0) return 0; - MEM_writeLE16(ostart, (U16)errorCode); - - ip += segmentSize; - op += errorCode; - errorCode = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable); - if (HUF_isError(errorCode)) return errorCode; - if (errorCode==0) return 0; - MEM_writeLE16(ostart+2, (U16)errorCode); - - ip += segmentSize; - op += errorCode; - errorCode = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable); - if (HUF_isError(errorCode)) return errorCode; - if (errorCode==0) return 0; - MEM_writeLE16(ostart+4, (U16)errorCode); - - ip += segmentSize; - op += errorCode; - errorCode = HUF_compress1X_usingCTable(op, oend-op, ip, iend-ip, CTable); - if (HUF_isError(errorCode)) return errorCode; - if (errorCode==0) return 0; - - op += errorCode; - return op-ostart; -} - - -static size_t HUF_compress_internal ( - void* dst, size_t dstSize, - const void* src, size_t srcSize, - unsigned maxSymbolValue, unsigned huffLog, - unsigned singleStream) -{ - BYTE* const ostart = (BYTE*)dst; - BYTE* const oend = ostart + dstSize; - BYTE* op = ostart; - - U32 count[HUF_MAX_SYMBOL_VALUE+1]; - HUF_CElt CTable[HUF_MAX_SYMBOL_VALUE+1]; - size_t errorCode; - - /* checks & inits */ - if (srcSize < 1) return 0; /* Uncompressed - note : 1 means rle, so first byte must be correct */ - if (dstSize < 1) return 0; /* not compressible within dst budget */ - if (srcSize > 128 * 1024) return ERROR(srcSize_wrong); /* current block size limit */ - if (huffLog > HUF_MAX_TABLELOG) return ERROR(tableLog_tooLarge); - if (!maxSymbolValue) maxSymbolValue = HUF_MAX_SYMBOL_VALUE; - if (!huffLog) huffLog = HUF_DEFAULT_TABLELOG; - - /* Scan input and build symbol stats */ - errorCode = FSE_count (count, &maxSymbolValue, (const BYTE*)src, srcSize); - if (HUF_isError(errorCode)) return errorCode; - if (errorCode == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; } - if (errorCode <= (srcSize >> 7)+1) return 0; /* Heuristic : not compressible enough */ - - /* Build Huffman Tree */ - errorCode = HUF_buildCTable (CTable, count, maxSymbolValue, huffLog); - if (HUF_isError(errorCode)) return errorCode; - huffLog = (U32)errorCode; - - /* Write table description header */ - errorCode = HUF_writeCTable (op, dstSize, CTable, maxSymbolValue, huffLog); - if (HUF_isError(errorCode)) return errorCode; - if (errorCode + 12 >= srcSize) return 0; /* not useful to try compression */ - op += errorCode; - - /* Compress */ - if (singleStream) - errorCode = HUF_compress1X_usingCTable(op, oend - op, src, srcSize, CTable); /* single segment */ - else - errorCode = HUF_compress4X_usingCTable(op, oend - op, src, srcSize, CTable); - if (HUF_isError(errorCode)) return errorCode; - if (errorCode==0) return 0; - op += errorCode; - - /* check compressibility */ - if ((size_t)(op-ostart) >= srcSize-1) - return 0; - - return op-ostart; -} - - -size_t HUF_compress1X (void* dst, size_t dstSize, - const void* src, size_t srcSize, - unsigned maxSymbolValue, unsigned huffLog) -{ - return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1); -} - -size_t HUF_compress2 (void* dst, size_t dstSize, - const void* src, size_t srcSize, - unsigned maxSymbolValue, unsigned huffLog) -{ - return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0); -} - - -size_t HUF_compress (void* dst, size_t maxDstSize, const void* src, size_t srcSize) -{ - return HUF_compress2(dst, maxDstSize, src, (U32)srcSize, 255, HUF_DEFAULT_TABLELOG); -} +/* ****************************************************************** + Huffman encoder, part of New Generation Entropy library + Copyright (C) 2013-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy + - Public forum : https://groups.google.com/forum/#!forum/lz4c +****************************************************************** */ + +/* ************************************************************** +* Compiler specifics +****************************************************************/ +#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +/* inline is defined */ +#elif defined(_MSC_VER) +# define inline __inline +#else +# define inline /* disable inline */ +#endif + + +#ifdef _MSC_VER /* Visual Studio */ +# define FORCE_INLINE static __forceinline +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#else +# ifdef __GNUC__ +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif +#endif + + +/* ************************************************************** +* Includes +****************************************************************/ +#include <stdlib.h> /* malloc, free, qsort */ +#include <string.h> /* memcpy, memset */ +#include <stdio.h> /* printf (debug) */ +#include "huf_static.h" +#include "bitstream.h" +#include "fse.h" /* header compression */ + + +/* ************************************************************** +* Error Management +****************************************************************/ +#define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ + + +/* ******************************************************* +* HUF : Huffman block compression +*********************************************************/ +struct HUF_CElt_s { + U16 val; + BYTE nbBits; +}; /* typedef'd to HUF_CElt within huf_static.h */ + +typedef struct nodeElt_s { + U32 count; + U16 parent; + BYTE byte; + BYTE nbBits; +} nodeElt; + +/*! HUF_writeCTable() : + `CTable` : huffman tree to save, using huf representation. + @return : size of saved CTable */ +size_t HUF_writeCTable (void* dst, size_t maxDstSize, + const HUF_CElt* CTable, U32 maxSymbolValue, U32 huffLog) +{ + BYTE bitsToWeight[HUF_MAX_TABLELOG + 1]; + BYTE huffWeight[HUF_MAX_SYMBOL_VALUE + 1]; + U32 n; + BYTE* op = (BYTE*)dst; + size_t size; + + /* check conditions */ + if (maxSymbolValue > HUF_MAX_SYMBOL_VALUE + 1) + return ERROR(GENERIC); + + /* convert to weight */ + bitsToWeight[0] = 0; + for (n=1; n<=huffLog; n++) + bitsToWeight[n] = (BYTE)(huffLog + 1 - n); + for (n=0; n<maxSymbolValue; n++) + huffWeight[n] = bitsToWeight[CTable[n].nbBits]; + + size = FSE_compress(op+1, maxDstSize-1, huffWeight, maxSymbolValue); /* don't need last symbol stat : implied */ + if (HUF_isError(size)) return size; + if (size >= 128) return ERROR(GENERIC); /* should never happen, since maxSymbolValue <= 255 */ + if ((size <= 1) || (size >= maxSymbolValue/2)) { + if (size==1) { /* RLE */ + /* only possible case : serie of 1 (because there are at least 2) */ + /* can only be 2^n or (2^n-1), otherwise not an huffman tree */ + BYTE code; + switch(maxSymbolValue) + { + case 1: code = 0; break; + case 2: code = 1; break; + case 3: code = 2; break; + case 4: code = 3; break; + case 7: code = 4; break; + case 8: code = 5; break; + case 15: code = 6; break; + case 16: code = 7; break; + case 31: code = 8; break; + case 32: code = 9; break; + case 63: code = 10; break; + case 64: code = 11; break; + case 127: code = 12; break; + case 128: code = 13; break; + default : return ERROR(corruption_detected); + } + op[0] = (BYTE)(255-13 + code); + return 1; + } + /* Not compressible */ + if (maxSymbolValue > (241-128)) return ERROR(GENERIC); /* not implemented (not possible with current format) */ + if (((maxSymbolValue+1)/2) + 1 > maxDstSize) return ERROR(dstSize_tooSmall); /* not enough space within dst buffer */ + op[0] = (BYTE)(128 /*special case*/ + 0 /* Not Compressible */ + (maxSymbolValue-1)); + huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause issue in final combination */ + for (n=0; n<maxSymbolValue; n+=2) + op[(n/2)+1] = (BYTE)((huffWeight[n] << 4) + huffWeight[n+1]); + return ((maxSymbolValue+1)/2) + 1; + } + + /* normal header case */ + op[0] = (BYTE)size; + return size+1; +} + + + +size_t HUF_readCTable (HUF_CElt* CTable, U32 maxSymbolValue, const void* src, size_t srcSize) +{ + BYTE huffWeight[HUF_MAX_SYMBOL_VALUE + 1]; + U32 rankVal[HUF_ABSOLUTEMAX_TABLELOG + 1]; /* large enough for values from 0 to 16 */ + U32 tableLog = 0; + size_t readSize; + U32 nbSymbols = 0; + //memset(huffWeight, 0, sizeof(huffWeight)); /* is not necessary, even though some analyzer complain ... */ + + /* get symbol weights */ + readSize = HUF_readStats(huffWeight, HUF_MAX_SYMBOL_VALUE+1, rankVal, &nbSymbols, &tableLog, src, srcSize); + if (HUF_isError(readSize)) return readSize; + + /* check result */ + if (tableLog > HUF_MAX_TABLELOG) return ERROR(tableLog_tooLarge); + if (nbSymbols > maxSymbolValue+1) return ERROR(maxSymbolValue_tooSmall); + + /* Prepare base value per rank */ + { U32 n, nextRankStart = 0; + for (n=1; n<=tableLog; n++) { + U32 current = nextRankStart; + nextRankStart += (rankVal[n] << (n-1)); + rankVal[n] = current; + } } + + /* fill nbBits */ + { U32 n; for (n=0; n<nbSymbols; n++) { + const U32 w = huffWeight[n]; + CTable[n].nbBits = (BYTE)(tableLog + 1 - w); + }} + + /* fill val */ + { U16 nbPerRank[HUF_MAX_TABLELOG+1] = {0}; + U16 valPerRank[HUF_MAX_TABLELOG+1] = {0}; + { U32 n; for (n=0; n<nbSymbols; n++) nbPerRank[CTable[n].nbBits]++; } + /* determine stating value per rank */ + { U16 min = 0; + U32 n; for (n=HUF_MAX_TABLELOG; n>0; n--) { + valPerRank[n] = min; /* get starting value within each rank */ + min += nbPerRank[n]; + min >>= 1; + } } + /* assign value within rank, symbol order */ + { U32 n; for (n=0; n<=maxSymbolValue; n++) CTable[n].val = valPerRank[CTable[n].nbBits]++; } + } + + return readSize; +} + + +static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits) +{ + const U32 largestBits = huffNode[lastNonNull].nbBits; + if (largestBits <= maxNbBits) return largestBits; /* early exit : no elt > maxNbBits */ + + /* there are several too large elements (at least >= 2) */ + { int totalCost = 0; + const U32 baseCost = 1 << (largestBits - maxNbBits); + U32 n = lastNonNull; + + while (huffNode[n].nbBits > maxNbBits) { + totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits)); + huffNode[n].nbBits = (BYTE)maxNbBits; + n --; + } /* n stops at huffNode[n].nbBits <= maxNbBits */ + while (huffNode[n].nbBits == maxNbBits) n--; /* n end at index of smallest symbol using < maxNbBits */ + + /* renorm totalCost */ + totalCost >>= (largestBits - maxNbBits); /* note : totalCost is necessarily a multiple of baseCost */ + + /* repay normalized cost */ + { U32 const noSymbol = 0xF0F0F0F0; + U32 rankLast[HUF_MAX_TABLELOG+1]; + int pos; + + /* Get pos of last (smallest) symbol per rank */ + memset(rankLast, 0xF0, sizeof(rankLast)); + { U32 currentNbBits = maxNbBits; + for (pos=n ; pos >= 0; pos--) { + if (huffNode[pos].nbBits >= currentNbBits) continue; + currentNbBits = huffNode[pos].nbBits; /* < maxNbBits */ + rankLast[maxNbBits-currentNbBits] = pos; + } } + + while (totalCost > 0) { + U32 nBitsToDecrease = BIT_highbit32(totalCost) + 1; + for ( ; nBitsToDecrease > 1; nBitsToDecrease--) { + U32 highPos = rankLast[nBitsToDecrease]; + U32 lowPos = rankLast[nBitsToDecrease-1]; + if (highPos == noSymbol) continue; + if (lowPos == noSymbol) break; + { U32 const highTotal = huffNode[highPos].count; + U32 const lowTotal = 2 * huffNode[lowPos].count; + if (highTotal <= lowTotal) break; + } } + /* only triggered when no more rank 1 symbol left => find closest one (note : there is necessarily at least one !) */ + while ((nBitsToDecrease<=HUF_MAX_TABLELOG) && (rankLast[nBitsToDecrease] == noSymbol)) /* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */ + nBitsToDecrease ++; + totalCost -= 1 << (nBitsToDecrease-1); + if (rankLast[nBitsToDecrease-1] == noSymbol) + rankLast[nBitsToDecrease-1] = rankLast[nBitsToDecrease]; /* this rank is no longer empty */ + huffNode[rankLast[nBitsToDecrease]].nbBits ++; + if (rankLast[nBitsToDecrease] == 0) /* special case, reached largest symbol */ + rankLast[nBitsToDecrease] = noSymbol; + else { + rankLast[nBitsToDecrease]--; + if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits-nBitsToDecrease) + rankLast[nBitsToDecrease] = noSymbol; /* this rank is now empty */ + } } /* while (totalCost > 0) */ + + while (totalCost < 0) { /* Sometimes, cost correction overshoot */ + if (rankLast[1] == noSymbol) { /* special case : no rank 1 symbol (using maxNbBits-1); let's create one from largest rank 0 (using maxNbBits) */ + while (huffNode[n].nbBits == maxNbBits) n--; + huffNode[n+1].nbBits--; + rankLast[1] = n+1; + totalCost++; + continue; + } + huffNode[ rankLast[1] + 1 ].nbBits--; + rankLast[1]++; + totalCost ++; + } } } /* there are several too large elements (at least >= 2) */ + + return maxNbBits; +} + + +typedef struct { + U32 base; + U32 current; +} rankPos; + +static void HUF_sort(nodeElt* huffNode, const U32* count, U32 maxSymbolValue) +{ + rankPos rank[32]; + U32 n; + + memset(rank, 0, sizeof(rank)); + for (n=0; n<=maxSymbolValue; n++) { + U32 r = BIT_highbit32(count[n] + 1); + rank[r].base ++; + } + for (n=30; n>0; n--) rank[n-1].base += rank[n].base; + for (n=0; n<32; n++) rank[n].current = rank[n].base; + for (n=0; n<=maxSymbolValue; n++) { + U32 const c = count[n]; + U32 const r = BIT_highbit32(c+1) + 1; + U32 pos = rank[r].current++; + while ((pos > rank[r].base) && (c > huffNode[pos-1].count)) huffNode[pos]=huffNode[pos-1], pos--; + huffNode[pos].count = c; + huffNode[pos].byte = (BYTE)n; + } +} + + +#define STARTNODE (HUF_MAX_SYMBOL_VALUE+1) +size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits) +{ + nodeElt huffNode0[2*HUF_MAX_SYMBOL_VALUE+1 +1]; + nodeElt* huffNode = huffNode0 + 1; + U32 n, nonNullRank; + int lowS, lowN; + U16 nodeNb = STARTNODE; + U32 nodeRoot; + + /* safety checks */ + if (maxNbBits == 0) maxNbBits = HUF_DEFAULT_TABLELOG; + if (maxSymbolValue > HUF_MAX_SYMBOL_VALUE) return ERROR(GENERIC); + memset(huffNode0, 0, sizeof(huffNode0)); + + /* sort, decreasing order */ + HUF_sort(huffNode, count, maxSymbolValue); + + /* init for parents */ + nonNullRank = maxSymbolValue; + while(huffNode[nonNullRank].count == 0) nonNullRank--; + lowS = nonNullRank; nodeRoot = nodeNb + lowS - 1; lowN = nodeNb; + huffNode[nodeNb].count = huffNode[lowS].count + huffNode[lowS-1].count; + huffNode[lowS].parent = huffNode[lowS-1].parent = nodeNb; + nodeNb++; lowS-=2; + for (n=nodeNb; n<=nodeRoot; n++) huffNode[n].count = (U32)(1U<<30); + huffNode0[0].count = (U32)(1U<<31); + + /* create parents */ + while (nodeNb <= nodeRoot) { + U32 n1 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++; + U32 n2 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++; + huffNode[nodeNb].count = huffNode[n1].count + huffNode[n2].count; + huffNode[n1].parent = huffNode[n2].parent = nodeNb; + nodeNb++; + } + + /* distribute weights (unlimited tree height) */ + huffNode[nodeRoot].nbBits = 0; + for (n=nodeRoot-1; n>=STARTNODE; n--) + huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1; + for (n=0; n<=nonNullRank; n++) + huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1; + + /* enforce maxTableLog */ + maxNbBits = HUF_setMaxHeight(huffNode, nonNullRank, maxNbBits); + + /* fill result into tree (val, nbBits) */ + { U16 nbPerRank[HUF_MAX_TABLELOG+1] = {0}; + U16 valPerRank[HUF_MAX_TABLELOG+1] = {0}; + if (maxNbBits > HUF_MAX_TABLELOG) return ERROR(GENERIC); /* check fit into table */ + for (n=0; n<=nonNullRank; n++) + nbPerRank[huffNode[n].nbBits]++; + /* determine stating value per rank */ + { U16 min = 0; + for (n=maxNbBits; n>0; n--) { + valPerRank[n] = min; /* get starting value within each rank */ + min += nbPerRank[n]; + min >>= 1; + } } + for (n=0; n<=maxSymbolValue; n++) + tree[huffNode[n].byte].nbBits = huffNode[n].nbBits; /* push nbBits per symbol, symbol order */ + for (n=0; n<=maxSymbolValue; n++) + tree[n].val = valPerRank[tree[n].nbBits]++; /* assign value within rank, symbol order */ + } + + return maxNbBits; +} + +static void HUF_encodeSymbol(BIT_CStream_t* bitCPtr, U32 symbol, const HUF_CElt* CTable) +{ + BIT_addBitsFast(bitCPtr, CTable[symbol].val, CTable[symbol].nbBits); +} + +size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); } + +#define HUF_FLUSHBITS(s) (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s)) + +#define HUF_FLUSHBITS_1(stream) \ + if (sizeof((stream)->bitContainer)*8 < HUF_MAX_TABLELOG*2+7) HUF_FLUSHBITS(stream) + +#define HUF_FLUSHBITS_2(stream) \ + if (sizeof((stream)->bitContainer)*8 < HUF_MAX_TABLELOG*4+7) HUF_FLUSHBITS(stream) + +size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable) +{ + const BYTE* ip = (const BYTE*) src; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstSize; + BYTE* op = ostart; + size_t n; + const unsigned fast = (dstSize >= HUF_BLOCKBOUND(srcSize)); + BIT_CStream_t bitC; + + /* init */ + if (dstSize < 8) return 0; /* not enough space to compress */ + { size_t const errorCode = BIT_initCStream(&bitC, op, oend-op); + if (HUF_isError(errorCode)) return 0; } + + n = srcSize & ~3; /* join to mod 4 */ + switch (srcSize & 3) + { + case 3 : HUF_encodeSymbol(&bitC, ip[n+ 2], CTable); + HUF_FLUSHBITS_2(&bitC); + case 2 : HUF_encodeSymbol(&bitC, ip[n+ 1], CTable); + HUF_FLUSHBITS_1(&bitC); + case 1 : HUF_encodeSymbol(&bitC, ip[n+ 0], CTable); + HUF_FLUSHBITS(&bitC); + case 0 : + default: ; + } + + for (; n>0; n-=4) { /* note : n&3==0 at this stage */ + HUF_encodeSymbol(&bitC, ip[n- 1], CTable); + HUF_FLUSHBITS_1(&bitC); + HUF_encodeSymbol(&bitC, ip[n- 2], CTable); + HUF_FLUSHBITS_2(&bitC); + HUF_encodeSymbol(&bitC, ip[n- 3], CTable); + HUF_FLUSHBITS_1(&bitC); + HUF_encodeSymbol(&bitC, ip[n- 4], CTable); + HUF_FLUSHBITS(&bitC); + } + + return BIT_closeCStream(&bitC); +} + + +size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable) +{ + size_t segmentSize = (srcSize+3)/4; /* first 3 segments */ + const BYTE* ip = (const BYTE*) src; + const BYTE* const iend = ip + srcSize; + BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + BYTE* op = ostart; + size_t errorCode; + + if (dstSize < 6 + 1 + 1 + 1 + 8) return 0; /* minimum space to compress successfully */ + if (srcSize < 12) return 0; /* no saving possible : too small input */ + op += 6; /* jumpTable */ + + errorCode = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable); + if (HUF_isError(errorCode)) return errorCode; + if (errorCode==0) return 0; + MEM_writeLE16(ostart, (U16)errorCode); + + ip += segmentSize; + op += errorCode; + errorCode = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable); + if (HUF_isError(errorCode)) return errorCode; + if (errorCode==0) return 0; + MEM_writeLE16(ostart+2, (U16)errorCode); + + ip += segmentSize; + op += errorCode; + errorCode = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable); + if (HUF_isError(errorCode)) return errorCode; + if (errorCode==0) return 0; + MEM_writeLE16(ostart+4, (U16)errorCode); + + ip += segmentSize; + op += errorCode; + errorCode = HUF_compress1X_usingCTable(op, oend-op, ip, iend-ip, CTable); + if (HUF_isError(errorCode)) return errorCode; + if (errorCode==0) return 0; + + op += errorCode; + return op-ostart; +} + + +static size_t HUF_compress_internal ( + void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned huffLog, + unsigned singleStream) +{ + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstSize; + BYTE* op = ostart; + + U32 count[HUF_MAX_SYMBOL_VALUE+1]; + HUF_CElt CTable[HUF_MAX_SYMBOL_VALUE+1]; + size_t errorCode; + + /* checks & inits */ + if (srcSize < 1) return 0; /* Uncompressed - note : 1 means rle, so first byte must be correct */ + if (dstSize < 1) return 0; /* not compressible within dst budget */ + if (srcSize > 128 * 1024) return ERROR(srcSize_wrong); /* current block size limit */ + if (huffLog > HUF_MAX_TABLELOG) return ERROR(tableLog_tooLarge); + if (!maxSymbolValue) maxSymbolValue = HUF_MAX_SYMBOL_VALUE; + if (!huffLog) huffLog = HUF_DEFAULT_TABLELOG; + + /* Scan input and build symbol stats */ + errorCode = FSE_count (count, &maxSymbolValue, (const BYTE*)src, srcSize); + if (HUF_isError(errorCode)) return errorCode; + if (errorCode == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; } + if (errorCode <= (srcSize >> 7)+1) return 0; /* Heuristic : not compressible enough */ + + /* Build Huffman Tree */ + errorCode = HUF_buildCTable (CTable, count, maxSymbolValue, huffLog); + if (HUF_isError(errorCode)) return errorCode; + huffLog = (U32)errorCode; + + /* Write table description header */ + errorCode = HUF_writeCTable (op, dstSize, CTable, maxSymbolValue, huffLog); + if (HUF_isError(errorCode)) return errorCode; + if (errorCode + 12 >= srcSize) return 0; /* not useful to try compression */ + op += errorCode; + + /* Compress */ + if (singleStream) + errorCode = HUF_compress1X_usingCTable(op, oend - op, src, srcSize, CTable); /* single segment */ + else + errorCode = HUF_compress4X_usingCTable(op, oend - op, src, srcSize, CTable); + if (HUF_isError(errorCode)) return errorCode; + if (errorCode==0) return 0; + op += errorCode; + + /* check compressibility */ + if ((size_t)(op-ostart) >= srcSize-1) + return 0; + + return op-ostart; +} + + +size_t HUF_compress1X (void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned huffLog) +{ + return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1); +} + +size_t HUF_compress2 (void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned huffLog) +{ + return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0); +} + + +size_t HUF_compress (void* dst, size_t maxDstSize, const void* src, size_t srcSize) +{ + return HUF_compress2(dst, maxDstSize, src, (U32)srcSize, 255, HUF_DEFAULT_TABLELOG); +} diff --git a/contrib/libs/zstd06/compress/zbuff_compress.c b/contrib/libs/zstd06/compress/zbuff_compress.c index ec9d0ada60..260aca0870 100644 --- a/contrib/libs/zstd06/compress/zbuff_compress.c +++ b/contrib/libs/zstd06/compress/zbuff_compress.c @@ -1,291 +1,291 @@ -/* - Buffered version of Zstd compression library - Copyright (C) 2015-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - zstd homepage : http://www.zstd.net/ -*/ - - -/* ************************************* -* Dependencies -***************************************/ -#include <stdlib.h> -#include "error_private.h" -#include "zstd_internal.h" /* MIN, ZSTD_blockHeaderSize */ -#include "zstd_static.h" /* ZSTD_BLOCKSIZE_MAX */ -#include "zbuff_static.h" - - -/* ************************************* -* Constants -***************************************/ -static size_t const ZBUFF_endFrameSize = ZSTD_BLOCKHEADERSIZE; - - -/*_************************************************** -* Streaming compression -* -* A ZBUFF_CCtx object is required to track streaming operation. -* Use ZBUFF_createCCtx() and ZBUFF_freeCCtx() to create/release resources. -* Use ZBUFF_compressInit() to start a new compression operation. -* ZBUFF_CCtx objects can be reused multiple times. -* -* Use ZBUFF_compressContinue() repetitively to consume your input. -* *srcSizePtr and *dstCapacityPtr can be any size. -* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr. -* Note that it may not consume the entire input, in which case it's up to the caller to call again the function with remaining input. -* The content of dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters or change dst . -* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency) -* or an error code, which can be tested using ZBUFF_isError(). -* -* ZBUFF_compressFlush() can be used to instruct ZBUFF to compress and output whatever remains within its buffer. -* Note that it will not output more than *dstCapacityPtr. -* Therefore, some content might still be left into its internal buffer if dst buffer is too small. -* @return : nb of bytes still present into internal buffer (0 if it's empty) -* or an error code, which can be tested using ZBUFF_isError(). -* -* ZBUFF_compressEnd() instructs to finish a frame. -* It will perform a flush and write frame epilogue. -* Similar to ZBUFF_compressFlush(), it may not be able to output the entire internal buffer content if *dstCapacityPtr is too small. -* @return : nb of bytes still present into internal buffer (0 if it's empty) -* or an error code, which can be tested using ZBUFF_isError(). -* -* Hint : recommended buffer sizes (not compulsory) -* input : ZSTD_BLOCKSIZE_MAX (128 KB), internal unit size, it improves latency to use this value. -* output : ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + ZBUFF_endFrameSize : ensures it's always possible to write/flush/end a full block at best speed. -* **************************************************/ - -typedef enum { ZBUFFcs_init, ZBUFFcs_load, ZBUFFcs_flush } ZBUFF_cStage; - -/* *** Ressources *** */ -struct ZBUFF_CCtx_s { - ZSTD_CCtx* zc; - char* inBuff; - size_t inBuffSize; - size_t inToCompress; - size_t inBuffPos; - size_t inBuffTarget; - size_t blockSize; - char* outBuff; - size_t outBuffSize; - size_t outBuffContentSize; - size_t outBuffFlushedSize; - ZBUFF_cStage stage; -}; /* typedef'd tp ZBUFF_CCtx within "zstd_buffered.h" */ - -ZBUFF_CCtx* ZBUFF_createCCtx(void) -{ - ZBUFF_CCtx* zbc = (ZBUFF_CCtx*)malloc(sizeof(ZBUFF_CCtx)); - if (zbc==NULL) return NULL; - memset(zbc, 0, sizeof(*zbc)); - zbc->zc = ZSTD_createCCtx(); - return zbc; -} - -size_t ZBUFF_freeCCtx(ZBUFF_CCtx* zbc) -{ - if (zbc==NULL) return 0; /* support free on NULL */ - ZSTD_freeCCtx(zbc->zc); - free(zbc->inBuff); - free(zbc->outBuff); - free(zbc); - return 0; -} - - -/* *** Initialization *** */ - -size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, - const void* dict, size_t dictSize, - ZSTD_parameters params, U64 pledgedSrcSize) -{ - /* allocate buffers */ - { size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog; - if (zbc->inBuffSize < neededInBuffSize) { - zbc->inBuffSize = neededInBuffSize; - free(zbc->inBuff); /* should not be necessary */ - zbc->inBuff = (char*)malloc(neededInBuffSize); - if (zbc->inBuff == NULL) return ERROR(memory_allocation); - } - zbc->blockSize = MIN(ZSTD_BLOCKSIZE_MAX, neededInBuffSize/2); - } - if (zbc->outBuffSize < ZSTD_compressBound(zbc->blockSize)+1) { - zbc->outBuffSize = ZSTD_compressBound(zbc->blockSize)+1; - free(zbc->outBuff); /* should not be necessary */ - zbc->outBuff = (char*)malloc(zbc->outBuffSize); - if (zbc->outBuff == NULL) return ERROR(memory_allocation); - } - - { size_t const errorCode = ZSTD_compressBegin_advanced(zbc->zc, dict, dictSize, params, pledgedSrcSize); - if (ZSTD_isError(errorCode)) return errorCode; } - - zbc->inToCompress = 0; - zbc->inBuffPos = 0; - zbc->inBuffTarget = zbc->blockSize; - zbc->outBuffFlushedSize = 0; - zbc->stage = ZBUFFcs_load; - return 0; /* ready to go */ -} - - -size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* zbc, const void* dict, size_t dictSize, int compressionLevel) -{ - ZSTD_parameters params; - params.cParams = ZSTD_getCParams(compressionLevel, 0, dictSize); - params.fParams.contentSizeFlag = 0; - ZSTD_adjustCParams(¶ms.cParams, 0, dictSize); - return ZBUFF_compressInit_advanced(zbc, dict, dictSize, params, 0); -} - -size_t ZBUFF_compressInit(ZBUFF_CCtx* zbc, int compressionLevel) -{ - return ZBUFF_compressInitDictionary(zbc, NULL, 0, compressionLevel); -} - - -/* *** Compression *** */ - -static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc, - void* dst, size_t* dstCapacityPtr, - const void* src, size_t* srcSizePtr, - int flush) /* aggregate : wait for full block before compressing */ -{ - U32 notDone = 1; - const char* const istart = (const char*)src; - const char* const iend = istart + *srcSizePtr; - const char* ip = istart; - char* const ostart = (char*)dst; - char* const oend = ostart + *dstCapacityPtr; - char* op = ostart; - - while (notDone) { - switch(zbc->stage) - { - case ZBUFFcs_init: return ERROR(init_missing); /* call ZBUFF_compressInit() first ! */ - - case ZBUFFcs_load: - /* complete inBuffer */ - { size_t const toLoad = zbc->inBuffTarget - zbc->inBuffPos; - size_t const loaded = ZBUFF_limitCopy(zbc->inBuff + zbc->inBuffPos, toLoad, ip, iend-ip); - zbc->inBuffPos += loaded; - ip += loaded; - if ( (zbc->inBuffPos==zbc->inToCompress) || (!flush && (toLoad != loaded)) ) { - notDone = 0; break; /* not enough input to get a full block : stop there, wait for more */ - } } - /* compress current block (note : this stage cannot be stopped in the middle) */ - { void* cDst; - size_t cSize; - size_t const iSize = zbc->inBuffPos - zbc->inToCompress; - size_t oSize = oend-op; - if (oSize >= ZSTD_compressBound(iSize)) - cDst = op; /* compress directly into output buffer (avoid flush stage) */ - else - cDst = zbc->outBuff, oSize = zbc->outBuffSize; - cSize = ZSTD_compressContinue(zbc->zc, cDst, oSize, zbc->inBuff + zbc->inToCompress, iSize); - if (ZSTD_isError(cSize)) return cSize; - /* prepare next block */ - zbc->inBuffTarget = zbc->inBuffPos + zbc->blockSize; - if (zbc->inBuffTarget > zbc->inBuffSize) - zbc->inBuffPos = 0, zbc->inBuffTarget = zbc->blockSize; /* note : inBuffSize >= blockSize */ - zbc->inToCompress = zbc->inBuffPos; - if (cDst == op) { op += cSize; break; } /* no need to flush */ - zbc->outBuffContentSize = cSize; - zbc->outBuffFlushedSize = 0; - zbc->stage = ZBUFFcs_flush; /* continue to flush stage */ - } - - case ZBUFFcs_flush: - /* flush into dst */ - { size_t const toFlush = zbc->outBuffContentSize - zbc->outBuffFlushedSize; - size_t const flushed = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outBuffFlushedSize, toFlush); - op += flushed; - zbc->outBuffFlushedSize += flushed; - if (toFlush!=flushed) { notDone = 0; break; } /* not enough space within dst to store compressed block : stop there */ - zbc->outBuffContentSize = 0; - zbc->outBuffFlushedSize = 0; - zbc->stage = ZBUFFcs_load; - break; - } - default: - return ERROR(GENERIC); /* impossible */ - } - } - - *srcSizePtr = ip - istart; - *dstCapacityPtr = op - ostart; - { size_t hintInSize = zbc->inBuffTarget - zbc->inBuffPos; - if (hintInSize==0) hintInSize = zbc->blockSize; - return hintInSize; - } -} - -size_t ZBUFF_compressContinue(ZBUFF_CCtx* zbc, - void* dst, size_t* dstCapacityPtr, - const void* src, size_t* srcSizePtr) -{ - return ZBUFF_compressContinue_generic(zbc, dst, dstCapacityPtr, src, srcSizePtr, 0); -} - - - -/* *** Finalize *** */ - -size_t ZBUFF_compressFlush(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr) -{ - size_t srcSize = 0; - ZBUFF_compressContinue_generic(zbc, dst, dstCapacityPtr, &srcSize, &srcSize, 1); /* use a valid src address instead of NULL */ - return zbc->outBuffContentSize - zbc->outBuffFlushedSize; -} - - -size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr) -{ - BYTE* const ostart = (BYTE*)dst; - BYTE* const oend = ostart + *dstCapacityPtr; - BYTE* op = ostart; - size_t outSize = *dstCapacityPtr; - size_t epilogueSize, remaining; - ZBUFF_compressFlush(zbc, dst, &outSize); /* flush any remaining inBuff */ - op += outSize; - epilogueSize = ZSTD_compressEnd(zbc->zc, zbc->outBuff + zbc->outBuffContentSize, zbc->outBuffSize - zbc->outBuffContentSize); /* epilogue into outBuff */ - zbc->outBuffContentSize += epilogueSize; - outSize = oend-op; - zbc->stage = ZBUFFcs_flush; - remaining = ZBUFF_compressFlush(zbc, op, &outSize); /* attempt to flush epilogue into dst */ - op += outSize; - if (!remaining) zbc->stage = ZBUFFcs_init; /* close only if nothing left to flush */ - *dstCapacityPtr = op-ostart; /* tells how many bytes were written */ - return remaining; -} - - - -/* ************************************* -* Tool functions -***************************************/ -size_t ZBUFF_recommendedCInSize(void) { return ZSTD_BLOCKSIZE_MAX; } -size_t ZBUFF_recommendedCOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + ZBUFF_endFrameSize; } +/* + Buffered version of Zstd compression library + Copyright (C) 2015-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - zstd homepage : http://www.zstd.net/ +*/ + + +/* ************************************* +* Dependencies +***************************************/ +#include <stdlib.h> +#include "error_private.h" +#include "zstd_internal.h" /* MIN, ZSTD_blockHeaderSize */ +#include "zstd_static.h" /* ZSTD_BLOCKSIZE_MAX */ +#include "zbuff_static.h" + + +/* ************************************* +* Constants +***************************************/ +static size_t const ZBUFF_endFrameSize = ZSTD_BLOCKHEADERSIZE; + + +/*_************************************************** +* Streaming compression +* +* A ZBUFF_CCtx object is required to track streaming operation. +* Use ZBUFF_createCCtx() and ZBUFF_freeCCtx() to create/release resources. +* Use ZBUFF_compressInit() to start a new compression operation. +* ZBUFF_CCtx objects can be reused multiple times. +* +* Use ZBUFF_compressContinue() repetitively to consume your input. +* *srcSizePtr and *dstCapacityPtr can be any size. +* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr. +* Note that it may not consume the entire input, in which case it's up to the caller to call again the function with remaining input. +* The content of dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters or change dst . +* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency) +* or an error code, which can be tested using ZBUFF_isError(). +* +* ZBUFF_compressFlush() can be used to instruct ZBUFF to compress and output whatever remains within its buffer. +* Note that it will not output more than *dstCapacityPtr. +* Therefore, some content might still be left into its internal buffer if dst buffer is too small. +* @return : nb of bytes still present into internal buffer (0 if it's empty) +* or an error code, which can be tested using ZBUFF_isError(). +* +* ZBUFF_compressEnd() instructs to finish a frame. +* It will perform a flush and write frame epilogue. +* Similar to ZBUFF_compressFlush(), it may not be able to output the entire internal buffer content if *dstCapacityPtr is too small. +* @return : nb of bytes still present into internal buffer (0 if it's empty) +* or an error code, which can be tested using ZBUFF_isError(). +* +* Hint : recommended buffer sizes (not compulsory) +* input : ZSTD_BLOCKSIZE_MAX (128 KB), internal unit size, it improves latency to use this value. +* output : ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + ZBUFF_endFrameSize : ensures it's always possible to write/flush/end a full block at best speed. +* **************************************************/ + +typedef enum { ZBUFFcs_init, ZBUFFcs_load, ZBUFFcs_flush } ZBUFF_cStage; + +/* *** Ressources *** */ +struct ZBUFF_CCtx_s { + ZSTD_CCtx* zc; + char* inBuff; + size_t inBuffSize; + size_t inToCompress; + size_t inBuffPos; + size_t inBuffTarget; + size_t blockSize; + char* outBuff; + size_t outBuffSize; + size_t outBuffContentSize; + size_t outBuffFlushedSize; + ZBUFF_cStage stage; +}; /* typedef'd tp ZBUFF_CCtx within "zstd_buffered.h" */ + +ZBUFF_CCtx* ZBUFF_createCCtx(void) +{ + ZBUFF_CCtx* zbc = (ZBUFF_CCtx*)malloc(sizeof(ZBUFF_CCtx)); + if (zbc==NULL) return NULL; + memset(zbc, 0, sizeof(*zbc)); + zbc->zc = ZSTD_createCCtx(); + return zbc; +} + +size_t ZBUFF_freeCCtx(ZBUFF_CCtx* zbc) +{ + if (zbc==NULL) return 0; /* support free on NULL */ + ZSTD_freeCCtx(zbc->zc); + free(zbc->inBuff); + free(zbc->outBuff); + free(zbc); + return 0; +} + + +/* *** Initialization *** */ + +size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, + const void* dict, size_t dictSize, + ZSTD_parameters params, U64 pledgedSrcSize) +{ + /* allocate buffers */ + { size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog; + if (zbc->inBuffSize < neededInBuffSize) { + zbc->inBuffSize = neededInBuffSize; + free(zbc->inBuff); /* should not be necessary */ + zbc->inBuff = (char*)malloc(neededInBuffSize); + if (zbc->inBuff == NULL) return ERROR(memory_allocation); + } + zbc->blockSize = MIN(ZSTD_BLOCKSIZE_MAX, neededInBuffSize/2); + } + if (zbc->outBuffSize < ZSTD_compressBound(zbc->blockSize)+1) { + zbc->outBuffSize = ZSTD_compressBound(zbc->blockSize)+1; + free(zbc->outBuff); /* should not be necessary */ + zbc->outBuff = (char*)malloc(zbc->outBuffSize); + if (zbc->outBuff == NULL) return ERROR(memory_allocation); + } + + { size_t const errorCode = ZSTD_compressBegin_advanced(zbc->zc, dict, dictSize, params, pledgedSrcSize); + if (ZSTD_isError(errorCode)) return errorCode; } + + zbc->inToCompress = 0; + zbc->inBuffPos = 0; + zbc->inBuffTarget = zbc->blockSize; + zbc->outBuffFlushedSize = 0; + zbc->stage = ZBUFFcs_load; + return 0; /* ready to go */ +} + + +size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* zbc, const void* dict, size_t dictSize, int compressionLevel) +{ + ZSTD_parameters params; + params.cParams = ZSTD_getCParams(compressionLevel, 0, dictSize); + params.fParams.contentSizeFlag = 0; + ZSTD_adjustCParams(¶ms.cParams, 0, dictSize); + return ZBUFF_compressInit_advanced(zbc, dict, dictSize, params, 0); +} + +size_t ZBUFF_compressInit(ZBUFF_CCtx* zbc, int compressionLevel) +{ + return ZBUFF_compressInitDictionary(zbc, NULL, 0, compressionLevel); +} + + +/* *** Compression *** */ + +static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc, + void* dst, size_t* dstCapacityPtr, + const void* src, size_t* srcSizePtr, + int flush) /* aggregate : wait for full block before compressing */ +{ + U32 notDone = 1; + const char* const istart = (const char*)src; + const char* const iend = istart + *srcSizePtr; + const char* ip = istart; + char* const ostart = (char*)dst; + char* const oend = ostart + *dstCapacityPtr; + char* op = ostart; + + while (notDone) { + switch(zbc->stage) + { + case ZBUFFcs_init: return ERROR(init_missing); /* call ZBUFF_compressInit() first ! */ + + case ZBUFFcs_load: + /* complete inBuffer */ + { size_t const toLoad = zbc->inBuffTarget - zbc->inBuffPos; + size_t const loaded = ZBUFF_limitCopy(zbc->inBuff + zbc->inBuffPos, toLoad, ip, iend-ip); + zbc->inBuffPos += loaded; + ip += loaded; + if ( (zbc->inBuffPos==zbc->inToCompress) || (!flush && (toLoad != loaded)) ) { + notDone = 0; break; /* not enough input to get a full block : stop there, wait for more */ + } } + /* compress current block (note : this stage cannot be stopped in the middle) */ + { void* cDst; + size_t cSize; + size_t const iSize = zbc->inBuffPos - zbc->inToCompress; + size_t oSize = oend-op; + if (oSize >= ZSTD_compressBound(iSize)) + cDst = op; /* compress directly into output buffer (avoid flush stage) */ + else + cDst = zbc->outBuff, oSize = zbc->outBuffSize; + cSize = ZSTD_compressContinue(zbc->zc, cDst, oSize, zbc->inBuff + zbc->inToCompress, iSize); + if (ZSTD_isError(cSize)) return cSize; + /* prepare next block */ + zbc->inBuffTarget = zbc->inBuffPos + zbc->blockSize; + if (zbc->inBuffTarget > zbc->inBuffSize) + zbc->inBuffPos = 0, zbc->inBuffTarget = zbc->blockSize; /* note : inBuffSize >= blockSize */ + zbc->inToCompress = zbc->inBuffPos; + if (cDst == op) { op += cSize; break; } /* no need to flush */ + zbc->outBuffContentSize = cSize; + zbc->outBuffFlushedSize = 0; + zbc->stage = ZBUFFcs_flush; /* continue to flush stage */ + } + + case ZBUFFcs_flush: + /* flush into dst */ + { size_t const toFlush = zbc->outBuffContentSize - zbc->outBuffFlushedSize; + size_t const flushed = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outBuffFlushedSize, toFlush); + op += flushed; + zbc->outBuffFlushedSize += flushed; + if (toFlush!=flushed) { notDone = 0; break; } /* not enough space within dst to store compressed block : stop there */ + zbc->outBuffContentSize = 0; + zbc->outBuffFlushedSize = 0; + zbc->stage = ZBUFFcs_load; + break; + } + default: + return ERROR(GENERIC); /* impossible */ + } + } + + *srcSizePtr = ip - istart; + *dstCapacityPtr = op - ostart; + { size_t hintInSize = zbc->inBuffTarget - zbc->inBuffPos; + if (hintInSize==0) hintInSize = zbc->blockSize; + return hintInSize; + } +} + +size_t ZBUFF_compressContinue(ZBUFF_CCtx* zbc, + void* dst, size_t* dstCapacityPtr, + const void* src, size_t* srcSizePtr) +{ + return ZBUFF_compressContinue_generic(zbc, dst, dstCapacityPtr, src, srcSizePtr, 0); +} + + + +/* *** Finalize *** */ + +size_t ZBUFF_compressFlush(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr) +{ + size_t srcSize = 0; + ZBUFF_compressContinue_generic(zbc, dst, dstCapacityPtr, &srcSize, &srcSize, 1); /* use a valid src address instead of NULL */ + return zbc->outBuffContentSize - zbc->outBuffFlushedSize; +} + + +size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr) +{ + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + *dstCapacityPtr; + BYTE* op = ostart; + size_t outSize = *dstCapacityPtr; + size_t epilogueSize, remaining; + ZBUFF_compressFlush(zbc, dst, &outSize); /* flush any remaining inBuff */ + op += outSize; + epilogueSize = ZSTD_compressEnd(zbc->zc, zbc->outBuff + zbc->outBuffContentSize, zbc->outBuffSize - zbc->outBuffContentSize); /* epilogue into outBuff */ + zbc->outBuffContentSize += epilogueSize; + outSize = oend-op; + zbc->stage = ZBUFFcs_flush; + remaining = ZBUFF_compressFlush(zbc, op, &outSize); /* attempt to flush epilogue into dst */ + op += outSize; + if (!remaining) zbc->stage = ZBUFFcs_init; /* close only if nothing left to flush */ + *dstCapacityPtr = op-ostart; /* tells how many bytes were written */ + return remaining; +} + + + +/* ************************************* +* Tool functions +***************************************/ +size_t ZBUFF_recommendedCInSize(void) { return ZSTD_BLOCKSIZE_MAX; } +size_t ZBUFF_recommendedCOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + ZBUFF_endFrameSize; } diff --git a/contrib/libs/zstd06/compress/zstd_compress.c b/contrib/libs/zstd06/compress/zstd_compress.c index 8072d19ae1..1bb75c68cc 100644 --- a/contrib/libs/zstd06/compress/zstd_compress.c +++ b/contrib/libs/zstd06/compress/zstd_compress.c @@ -55,7 +55,7 @@ #include <string.h> /* memset */ #include "mem.h" #include "fse_static.h" -#include "huf_static.h" +#include "huf_static.h" #include "zstd_internal.h" @@ -218,7 +218,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, const size_t maxNbSeq = blockSize / divider; const size_t tokenSpace = blockSize + 11*maxNbSeq; const size_t chainSize = (params.cParams.strategy == ZSTD_fast) ? 0 : (1 << params.cParams.chainLog); - const size_t hSize = ((size_t)1) << params.cParams.hashLog; + const size_t hSize = ((size_t)1) << params.cParams.hashLog; const size_t h3Size = (zc->hashLog3) ? 1 << zc->hashLog3 : 0; const size_t tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); @@ -291,7 +291,7 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx) /* copy tables */ { const size_t chainSize = (srcCCtx->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << srcCCtx->params.cParams.chainLog); - const size_t hSize = ((size_t)1) << srcCCtx->params.cParams.hashLog; + const size_t hSize = ((size_t)1) << srcCCtx->params.cParams.hashLog; const size_t h3Size = (srcCCtx->hashLog3) ? 1 << srcCCtx->hashLog3 : 0; const size_t tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); memcpy(dstCCtx->workSpace, srcCCtx->workSpace, tableSpace); @@ -852,7 +852,7 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const B static const BYTE* g_start = NULL; const U32 pos = (U32)(literals - g_start); if (g_start==NULL) g_start = literals; - if ((pos > 2587900) && (pos < 2588050)) + if ((pos > 2587900) && (pos < 2588050)) printf("Cpos %6u :%5u literals & match %3u bytes at distance %6u \n", pos, (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode); #endif @@ -1966,16 +1966,16 @@ static void ZSTD_compressBlock_btlazy2_extDict(ZSTD_CCtx* ctx, const void* src, ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 1, 2); } - - -/* The optimal parser */ -#include "zstd_opt.h" - -static void ZSTD_compressBlock_btopt(ZSTD_CCtx* ctx, const void* src, size_t srcSize) -{ - ZSTD_compressBlock_opt_generic(ctx, src, srcSize); -} - + + +/* The optimal parser */ +#include "zstd_opt.h" + +static void ZSTD_compressBlock_btopt(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +{ + ZSTD_compressBlock_opt_generic(ctx, src, srcSize); +} + static void ZSTD_compressBlock_btopt_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize) { ZSTD_compressBlock_opt_extDict_generic(ctx, src, srcSize); @@ -2432,8 +2432,8 @@ size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcS /*-===== Pre-defined compression levels =====-*/ -#define ZSTD_DEFAULT_CLEVEL 5 -#define ZSTD_MAX_CLEVEL 22 +#define ZSTD_DEFAULT_CLEVEL 5 +#define ZSTD_MAX_CLEVEL 22 unsigned ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; } static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEVEL+1] = { @@ -2552,8 +2552,8 @@ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, U64 srcSize, si size_t const addedSize = srcSize ? 0 : 500; U64 const rSize = srcSize+dictSize ? srcSize+dictSize+addedSize : (U64)-1; U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB); /* intentional underflow for srcSizeHint == 0 */ - if (compressionLevel < 0) compressionLevel = ZSTD_DEFAULT_CLEVEL; - if (compressionLevel==0) compressionLevel = 1; + if (compressionLevel < 0) compressionLevel = ZSTD_DEFAULT_CLEVEL; + if (compressionLevel==0) compressionLevel = 1; if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL; cp = ZSTD_defaultCParameters[tableID][compressionLevel]; if (MEM_32bits()) { /* auto-correction, for 32-bits mode */ diff --git a/contrib/libs/zstd06/compress/zstd_opt.h b/contrib/libs/zstd06/compress/zstd_opt.h index 1bcba2732e..200eaf02a3 100644 --- a/contrib/libs/zstd06/compress/zstd_opt.h +++ b/contrib/libs/zstd06/compress/zstd_opt.h @@ -1,1033 +1,1033 @@ -#include <contrib/libs/zstd06/renames.h> -/* - ZSTD Optimal mode - Copyright (C) 2016, Przemyslaw Skibinski, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Zstd source repository : https://www.zstd.net -*/ - -/* Note : this file is intended to be included within zstd_compress.c */ - - -#define ZSTD_FREQ_DIV 5 - -/*-************************************* -* Price functions for optimal parser -***************************************/ -FORCE_INLINE void ZSTD_setLog2Prices(seqStore_t* ssPtr) -{ - ssPtr->log2matchLengthSum = ZSTD_highbit(ssPtr->matchLengthSum+1); - ssPtr->log2litLengthSum = ZSTD_highbit(ssPtr->litLengthSum+1); - ssPtr->log2litSum = ZSTD_highbit(ssPtr->litSum+1); - ssPtr->log2offCodeSum = ZSTD_highbit(ssPtr->offCodeSum+1); - ssPtr->factor = 1 + ((ssPtr->litSum>>5) / ssPtr->litLengthSum) + ((ssPtr->litSum<<1) / (ssPtr->litSum + ssPtr->matchSum)); -} - - -MEM_STATIC void ZSTD_rescaleFreqs(seqStore_t* ssPtr) -{ - unsigned u; - - ssPtr->cachedLiterals = NULL; - ssPtr->cachedPrice = ssPtr->cachedLitLength = 0; - - if (ssPtr->litLengthSum == 0) { - ssPtr->litSum = (2<<Litbits); - ssPtr->litLengthSum = MaxLL+1; - ssPtr->matchLengthSum = MaxML+1; - ssPtr->offCodeSum = (MaxOff+1); - ssPtr->matchSum = (2<<Litbits); - - for (u=0; u<=MaxLit; u++) - ssPtr->litFreq[u] = 2; - for (u=0; u<=MaxLL; u++) - ssPtr->litLengthFreq[u] = 1; - for (u=0; u<=MaxML; u++) - ssPtr->matchLengthFreq[u] = 1; - for (u=0; u<=MaxOff; u++) - ssPtr->offCodeFreq[u] = 1; - } else { - ssPtr->matchLengthSum = 0; - ssPtr->litLengthSum = 0; - ssPtr->offCodeSum = 0; - ssPtr->matchSum = 0; - ssPtr->litSum = 0; - - for (u=0; u<=MaxLit; u++) { - ssPtr->litFreq[u] = 1 + (ssPtr->litFreq[u]>>ZSTD_FREQ_DIV); - ssPtr->litSum += ssPtr->litFreq[u]; - } - for (u=0; u<=MaxLL; u++) { - ssPtr->litLengthFreq[u] = 1 + (ssPtr->litLengthFreq[u]>>ZSTD_FREQ_DIV); - ssPtr->litLengthSum += ssPtr->litLengthFreq[u]; - } - for (u=0; u<=MaxML; u++) { - ssPtr->matchLengthFreq[u] = 1 + (ssPtr->matchLengthFreq[u]>>ZSTD_FREQ_DIV); - ssPtr->matchLengthSum += ssPtr->matchLengthFreq[u]; - ssPtr->matchSum += ssPtr->matchLengthFreq[u] * (u + 3); - } - for (u=0; u<=MaxOff; u++) { - ssPtr->offCodeFreq[u] = 1 + (ssPtr->offCodeFreq[u]>>ZSTD_FREQ_DIV); - ssPtr->offCodeSum += ssPtr->offCodeFreq[u]; - } - } - - ZSTD_setLog2Prices(ssPtr); -} - - -FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* ssPtr, U32 litLength, const BYTE* literals) -{ - U32 price, u; - - if (litLength == 0) - return ssPtr->log2litLengthSum - ZSTD_highbit(ssPtr->litLengthFreq[0]+1); - - /* literals */ - if (ssPtr->cachedLiterals == literals) { - U32 additional = litLength - ssPtr->cachedLitLength; - const BYTE* literals2 = ssPtr->cachedLiterals + ssPtr->cachedLitLength; - price = ssPtr->cachedPrice + additional * ssPtr->log2litSum; - for (u=0; u < additional; u++) - price -= ZSTD_highbit(ssPtr->litFreq[literals2[u]]+1); - ssPtr->cachedPrice = price; - ssPtr->cachedLitLength = litLength; - } else { - price = litLength * ssPtr->log2litSum; - for (u=0; u < litLength; u++) - price -= ZSTD_highbit(ssPtr->litFreq[literals[u]]+1); - - if (litLength >= 12) { - ssPtr->cachedLiterals = literals; - ssPtr->cachedPrice = price; - ssPtr->cachedLitLength = litLength; - } - } - - /* literal Length */ - { static const BYTE LL_Code[64] = { 0, 1, 2, 3, 4, 5, 6, 7, - 8, 9, 10, 11, 12, 13, 14, 15, - 16, 16, 17, 17, 18, 18, 19, 19, - 20, 20, 20, 20, 21, 21, 21, 21, - 22, 22, 22, 22, 22, 22, 22, 22, - 23, 23, 23, 23, 23, 23, 23, 23, - 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24 }; - const BYTE LL_deltaCode = 19; - const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit(litLength) + LL_deltaCode : LL_Code[litLength]; - price += LL_bits[llCode] + ssPtr->log2litLengthSum - ZSTD_highbit(ssPtr->litLengthFreq[llCode]+1); - } - - return price; -} - - -FORCE_INLINE U32 ZSTD_getPrice(seqStore_t* seqStorePtr, U32 litLength, const BYTE* literals, U32 offset, U32 matchLength) -{ - /* offset */ - BYTE offCode = (BYTE)ZSTD_highbit(offset+1); - U32 price = offCode + seqStorePtr->log2offCodeSum - ZSTD_highbit(seqStorePtr->offCodeFreq[offCode]+1); - - /* match Length */ - { static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, - 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, - 38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, - 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, - 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, - 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, - 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 }; - const BYTE ML_deltaCode = 36; - const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit(matchLength) + ML_deltaCode : ML_Code[matchLength]; - price += ML_bits[mlCode] + seqStorePtr->log2matchLengthSum - ZSTD_highbit(seqStorePtr->matchLengthFreq[mlCode]+1); - } - - return price + ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + seqStorePtr->factor; -} - - -MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const BYTE* literals, U32 offset, U32 matchLength) -{ - U32 u; - - /* literals */ - seqStorePtr->litSum += litLength; - for (u=0; u < litLength; u++) - seqStorePtr->litFreq[literals[u]]++; - - /* literal Length */ - { static const BYTE LL_Code[64] = { 0, 1, 2, 3, 4, 5, 6, 7, - 8, 9, 10, 11, 12, 13, 14, 15, - 16, 16, 17, 17, 18, 18, 19, 19, - 20, 20, 20, 20, 21, 21, 21, 21, - 22, 22, 22, 22, 22, 22, 22, 22, - 23, 23, 23, 23, 23, 23, 23, 23, - 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24 }; - const BYTE LL_deltaCode = 19; - const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit(litLength) + LL_deltaCode : LL_Code[litLength]; - seqStorePtr->litLengthFreq[llCode]++; - seqStorePtr->litLengthSum++; - } - - /* match offset */ - { BYTE offCode = (BYTE)ZSTD_highbit(offset+1); - seqStorePtr->offCodeSum++; - seqStorePtr->offCodeFreq[offCode]++; - } - - /* match Length */ - { static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, - 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, - 38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, - 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, - 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, - 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, - 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 }; - const BYTE ML_deltaCode = 36; - const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit(matchLength) + ML_deltaCode : ML_Code[matchLength]; - seqStorePtr->matchLengthFreq[mlCode]++; - seqStorePtr->matchLengthSum++; - } - - ZSTD_setLog2Prices(seqStorePtr); -} - - -#define SET_PRICE(pos, mlen_, offset_, litlen_, price_) \ - { \ - while (last_pos < pos) { opt[last_pos+1].price = 1<<30; last_pos++; } \ - opt[pos].mlen = mlen_; \ - opt[pos].off = offset_; \ - opt[pos].litlen = litlen_; \ - opt[pos].price = price_; \ - ZSTD_LOG_PARSER("%d: SET price[%d/%d]=%d litlen=%d len=%d off=%d\n", (int)(inr-base), (int)pos, (int)last_pos, opt[pos].price, opt[pos].litlen, opt[pos].mlen, opt[pos].off); \ - } - - - - -/* Update hashTable3 up to ip (excluded) - Assumption : always within prefix (ie. not within extDict) */ -FORCE_INLINE -U32 ZSTD_insertAndFindFirstIndexHash3 (ZSTD_CCtx* zc, const BYTE* ip) -{ - U32* const hashTable3 = zc->hashTable3; - U32 const hashLog3 = zc->hashLog3; - const BYTE* const base = zc->base; - U32 idx = zc->nextToUpdate3; - const U32 target = zc->nextToUpdate3 = (U32)(ip - base); - const size_t hash3 = ZSTD_hash3Ptr(ip, hashLog3); - - while(idx < target) { - hashTable3[ZSTD_hash3Ptr(base+idx, hashLog3)] = idx; - idx++; - } - - return hashTable3[hash3]; -} - - -/*-************************************* -* Binary Tree search -***************************************/ -static U32 ZSTD_insertBtAndGetAllMatches ( - ZSTD_CCtx* zc, - const BYTE* const ip, const BYTE* const iLimit, - U32 nbCompares, const U32 mls, - U32 extDict, ZSTD_match_t* matches, const U32 minMatchLen) -{ - const BYTE* const base = zc->base; - const U32 current = (U32)(ip-base); - const U32 hashLog = zc->params.cParams.hashLog; - const size_t h = ZSTD_hashPtr(ip, hashLog, mls); - U32* const hashTable = zc->hashTable; - U32 matchIndex = hashTable[h]; - U32* const bt = zc->chainTable; - const U32 btLog = zc->params.cParams.chainLog - 1; - const U32 btMask= (1U << btLog) - 1; - size_t commonLengthSmaller=0, commonLengthLarger=0; - const BYTE* const dictBase = zc->dictBase; - const U32 dictLimit = zc->dictLimit; - const BYTE* const dictEnd = dictBase + dictLimit; - const BYTE* const prefixStart = base + dictLimit; - const U32 btLow = btMask >= current ? 0 : current - btMask; - const U32 windowLow = zc->lowLimit; - U32* smallerPtr = bt + 2*(current&btMask); - U32* largerPtr = bt + 2*(current&btMask) + 1; - U32 matchEndIdx = current+8; - U32 dummy32; /* to be nullified at the end */ - U32 mnum = 0; - - const U32 minMatch = (mls == 3) ? 3 : 4; - size_t bestLength = minMatchLen-1; - - if (minMatch == 3) { /* HC3 match finder */ - U32 const matchIndex3 = ZSTD_insertAndFindFirstIndexHash3 (zc, ip); - if (matchIndex3>windowLow && (current - matchIndex3 < (1<<18))) { - const BYTE* match; - size_t currentMl=0; - if ((!extDict) || matchIndex3 >= dictLimit) { - match = base + matchIndex3; - if (match[bestLength] == ip[bestLength]) currentMl = ZSTD_count(ip, match, iLimit); - } else { - match = dictBase + matchIndex3; - if (MEM_readMINMATCH(match, MINMATCH) == MEM_readMINMATCH(ip, MINMATCH)) /* assumption : matchIndex3 <= dictLimit-4 (by table construction) */ - currentMl = ZSTD_count_2segments(ip+MINMATCH, match+MINMATCH, iLimit, dictEnd, prefixStart) + MINMATCH; - } - - /* save best solution */ - if (currentMl > bestLength) { - bestLength = currentMl; - matches[mnum].off = ZSTD_REP_MOVE + current - matchIndex3; - matches[mnum].len = (U32)currentMl; - mnum++; - if (currentMl > ZSTD_OPT_NUM) goto update; - if (ip+currentMl == iLimit) goto update; /* best possible, and avoid read overflow*/ - } - } - } - - hashTable[h] = current; /* Update Hash Table */ - - while (nbCompares-- && (matchIndex > windowLow)) { - U32* nextPtr = bt + 2*(matchIndex & btMask); - size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ - const BYTE* match; - - if ((!extDict) || (matchIndex+matchLength >= dictLimit)) { - match = base + matchIndex; - if (match[matchLength] == ip[matchLength]) { -#if ZSTD_OPT_DEBUG >= 5 - size_t ml; - if (matchIndex < dictLimit) - ml = ZSTD_count_2segments(ip, dictBase + matchIndex, iLimit, dictEnd, prefixStart); - else - ml = ZSTD_count(ip, match, ip+matchLength); - if (ml < matchLength) - printf("%d: ERROR_NOEXT: offset=%d matchLength=%d matchIndex=%d dictLimit=%d ml=%d\n", current, (int)(current - matchIndex), (int)matchLength, (int)matchIndex, (int)dictLimit, (int)ml), exit(0); -#endif - matchLength += ZSTD_count(ip+matchLength+1, match+matchLength+1, iLimit) +1; - } - } else { - match = dictBase + matchIndex; -#if ZSTD_OPT_DEBUG >= 5 - if (memcmp(match, ip, matchLength) != 0) - printf("%d: ERROR_EXT: matchLength=%d ZSTD_count=%d\n", current, (int)matchLength, (int)ZSTD_count_2segments(ip+matchLength, match+matchLength, iLimit, dictEnd, prefixStart)), exit(0); -#endif - matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iLimit, dictEnd, prefixStart); - ZSTD_LOG_PARSER("%d: ZSTD_INSERTBTANDGETALLMATCHES=%d offset=%d dictBase=%p dictEnd=%p prefixStart=%p ip=%p match=%p\n", (int)current, (int)matchLength, (int)(current - matchIndex), dictBase, dictEnd, prefixStart, ip, match); - if (matchIndex+matchLength >= dictLimit) - match = base + matchIndex; /* to prepare for next usage of match[matchLength] */ - } - - if (matchLength > bestLength) { - if (matchLength > matchEndIdx - matchIndex) matchEndIdx = matchIndex + (U32)matchLength; - bestLength = matchLength; - matches[mnum].off = ZSTD_REP_MOVE + current - matchIndex; - matches[mnum].len = (U32)matchLength; - mnum++; - if (matchLength > ZSTD_OPT_NUM) break; - if (ip+matchLength == iLimit) /* equal : no way to know if inf or sup */ - break; /* drop, to guarantee consistency (miss a little bit of compression) */ - } - - if (match[matchLength] < ip[matchLength]) { - /* match is smaller than current */ - *smallerPtr = matchIndex; /* update smaller idx */ - commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ - if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */ - smallerPtr = nextPtr+1; /* new "smaller" => larger of match */ - matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ - } else { - /* match is larger than current */ - *largerPtr = matchIndex; - commonLengthLarger = matchLength; - if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */ - largerPtr = nextPtr; - matchIndex = nextPtr[0]; - } } - - *smallerPtr = *largerPtr = 0; - -update: - zc->nextToUpdate = (matchEndIdx > current + 8) ? matchEndIdx - 8 : current+1; - return mnum; -} - - -/** Tree updater, providing best match */ -static U32 ZSTD_BtGetAllMatches ( - ZSTD_CCtx* zc, - const BYTE* const ip, const BYTE* const iLimit, - const U32 maxNbAttempts, const U32 mls, ZSTD_match_t* matches, const U32 minMatchLen) -{ - if (ip < zc->base + zc->nextToUpdate) return 0; /* skipped area */ - ZSTD_updateTree(zc, ip, iLimit, maxNbAttempts, mls); - return ZSTD_insertBtAndGetAllMatches(zc, ip, iLimit, maxNbAttempts, mls, 0, matches, minMatchLen); -} - - -static U32 ZSTD_BtGetAllMatches_selectMLS ( - ZSTD_CCtx* zc, /* Index table will be updated */ - const BYTE* ip, const BYTE* const iHighLimit, - const U32 maxNbAttempts, const U32 matchLengthSearch, ZSTD_match_t* matches, const U32 minMatchLen) -{ - switch(matchLengthSearch) - { - case 3 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 3, matches, minMatchLen); - default : - case 4 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 4, matches, minMatchLen); - case 5 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 5, matches, minMatchLen); - case 6 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 6, matches, minMatchLen); - } -} - -/** Tree updater, providing best match */ -static U32 ZSTD_BtGetAllMatches_extDict ( - ZSTD_CCtx* zc, - const BYTE* const ip, const BYTE* const iLimit, - const U32 maxNbAttempts, const U32 mls, ZSTD_match_t* matches, const U32 minMatchLen) -{ - if (ip < zc->base + zc->nextToUpdate) return 0; /* skipped area */ - ZSTD_updateTree_extDict(zc, ip, iLimit, maxNbAttempts, mls); - return ZSTD_insertBtAndGetAllMatches(zc, ip, iLimit, maxNbAttempts, mls, 1, matches, minMatchLen); -} - - -static U32 ZSTD_BtGetAllMatches_selectMLS_extDict ( - ZSTD_CCtx* zc, /* Index table will be updated */ - const BYTE* ip, const BYTE* const iHighLimit, - const U32 maxNbAttempts, const U32 matchLengthSearch, ZSTD_match_t* matches, const U32 minMatchLen) -{ - switch(matchLengthSearch) - { - case 3 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 3, matches, minMatchLen); - default : - case 4 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 4, matches, minMatchLen); - case 5 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 5, matches, minMatchLen); - case 6 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 6, matches, minMatchLen); - } -} - - -/*-******************************* -* Optimal parser -*********************************/ -FORCE_INLINE -void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, - const void* src, size_t srcSize) -{ - seqStore_t* seqStorePtr = &(ctx->seqStore); - const BYTE* const istart = (const BYTE*)src; - const BYTE* ip = istart; - const BYTE* anchor = istart; - const BYTE* const iend = istart + srcSize; - const BYTE* const ilimit = iend - 8; - const BYTE* const base = ctx->base; - const BYTE* const prefixStart = base + ctx->dictLimit; - - const U32 maxSearches = 1U << ctx->params.cParams.searchLog; - const U32 sufficient_len = ctx->params.cParams.targetLength; - const U32 mls = ctx->params.cParams.searchLength; - const U32 minMatch = (ctx->params.cParams.searchLength == 3) ? 3 : 4; - - ZSTD_optimal_t* opt = seqStorePtr->priceTable; - ZSTD_match_t* matches = seqStorePtr->matchTable; - const BYTE* inr; - - /* init */ - U32 offset, rep[ZSTD_REP_INIT]; - { U32 i; for (i=0; i<ZSTD_REP_INIT; i++) rep[i]=REPCODE_STARTVALUE; } - - ctx->nextToUpdate3 = ctx->nextToUpdate; - ZSTD_resetSeqStore(seqStorePtr); - ZSTD_rescaleFreqs(seqStorePtr); - if ((ip-prefixStart) < REPCODE_STARTVALUE) ip = prefixStart + REPCODE_STARTVALUE; - - ZSTD_LOG_BLOCK("%d: COMPBLOCK_OPT_GENERIC srcSz=%d maxSrch=%d mls=%d sufLen=%d\n", (int)(ip-base), (int)srcSize, maxSearches, mls, sufficient_len); - - /* Match Loop */ - while (ip < ilimit) { - U32 cur, match_num, last_pos, litlen, price; - U32 u, mlen, best_mlen, best_off, litLength; - memset(opt, 0, sizeof(ZSTD_optimal_t)); - last_pos = 0; - litlen = (U32)(ip - anchor); - - /* check repCode */ - { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) - if (MEM_readMINMATCH(ip, minMatch) == MEM_readMINMATCH(ip - rep[i], minMatch)) { - /* repcode : we take it */ - mlen = (U32)ZSTD_count(ip+minMatch, ip+minMatch-rep[i], iend) + minMatch; - ZSTD_LOG_PARSER("%d: start try REP rep[%d]=%d mlen=%d\n", (int)(ip-base), i, (int)rep[i], (int)mlen); - if (mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) { - best_mlen = mlen; best_off = i; cur = 0; last_pos = 1; - goto _storeSequence; - } - best_off = (i<=1 && ip == anchor) ? 1-i : i; - do { - price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH); - if (mlen > last_pos || price < opt[mlen].price) - SET_PRICE(mlen, mlen, i, litlen, price); /* note : macro modifies last_pos */ - mlen--; - } while (mlen >= minMatch); - } } - - match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, ip, iend, maxSearches, mls, matches, minMatch); - - ZSTD_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-base), match_num, last_pos); - if (!last_pos && !match_num) { ip++; continue; } - - if (match_num && (matches[match_num-1].len > sufficient_len || matches[match_num-1].len >= ZSTD_OPT_NUM)) { - best_mlen = matches[match_num-1].len; - best_off = matches[match_num-1].off; - cur = 0; - last_pos = 1; - goto _storeSequence; - } - - /* set prices using matches at position = 0 */ - best_mlen = (last_pos) ? last_pos : minMatch; - for (u = 0; u < match_num; u++) { - mlen = (u>0) ? matches[u-1].len+1 : best_mlen; - best_mlen = matches[u].len; - ZSTD_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-base), matches[u].len, matches[u].off, (int)best_mlen, (int)last_pos); - while (mlen <= best_mlen) { - price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off, mlen - MINMATCH); - if (mlen > last_pos || price < opt[mlen].price) - SET_PRICE(mlen, mlen, matches[u].off, litlen, price); /* note : macro modifies last_pos */ - mlen++; - } } - - if (last_pos < minMatch) { ip++; continue; } - - /* initialize opt[0] */ - { U32 i ; for (i=0; i<ZSTD_REP_INIT; i++) opt[0].rep[i] = rep[i]; } - opt[0].mlen = 1; - opt[0].litlen = litlen; - - /* check further positions */ - for (cur = 1; cur <= last_pos; cur++) { - inr = ip + cur; - - if (opt[cur-1].mlen == 1) { - litlen = opt[cur-1].litlen + 1; - if (cur > litlen) { - price = opt[cur - litlen].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-litlen); - } else - price = ZSTD_getLiteralPrice(seqStorePtr, litlen, anchor); - } else { - litlen = 1; - price = opt[cur - 1].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-1); - } - - if (cur > last_pos || price <= opt[cur].price) // || ((price == opt[cur].price) && (opt[cur-1].mlen == 1) && (cur != litlen))) - SET_PRICE(cur, 1, 0, litlen, price); - - if (cur == last_pos) break; - - if (inr > ilimit) /* last match must start at a minimum distance of 8 from oend */ - continue; - - mlen = opt[cur].mlen; - if (opt[cur].off >= ZSTD_REP_NUM) { - opt[cur].rep[2] = opt[cur-mlen].rep[1]; - opt[cur].rep[1] = opt[cur-mlen].rep[0]; - opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE; - ZSTD_LOG_ENCODE("%d: COPYREP_OFF cur=%d mlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep[0], opt[cur].rep[1]); - } else { - opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur-mlen].rep[1] : opt[cur-mlen].rep[2]; - opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur-mlen].rep[0] : opt[cur-mlen].rep[1]; - opt[cur].rep[0] = opt[cur-mlen].rep[opt[cur].off]; - ZSTD_LOG_ENCODE("%d: COPYREP_NOR cur=%d mlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep[0], opt[cur].rep[1]); - } - - ZSTD_LOG_PARSER("%d: CURRENT_NoExt price[%d/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep[0], opt[cur].rep[1]); - - best_mlen = minMatch; - { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) - if (MEM_readMINMATCH(inr, minMatch) == MEM_readMINMATCH(inr - opt[cur].rep[i], minMatch)) { /* check rep */ - mlen = (U32)ZSTD_count(inr+minMatch, inr+minMatch - opt[cur].rep[i], iend) + minMatch; - ZSTD_LOG_PARSER("%d: Found REP %d/%d mlen=%d off=%d rep=%d opt[%d].off=%d\n", (int)(inr-base), i, ZSTD_REP_NUM, mlen, i, opt[cur].rep[i], cur, opt[cur].off); - - if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) { - ZSTD_LOG_PARSER("%d: REP sufficient_len=%d best_mlen=%d best_off=%d last_pos=%d\n", (int)(inr-base), sufficient_len, best_mlen, best_off, last_pos); - best_mlen = mlen; best_off = i; last_pos = cur + 1; - goto _storeSequence; - } - - best_off = (i<=1 && opt[cur].mlen != 1) ? 1-i : i; - if (opt[cur].mlen == 1) { - litlen = opt[cur].litlen; - if (cur > litlen) { - price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, best_off, mlen - MINMATCH); - } else - price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH); - } else { - litlen = 0; - price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH); - } - - if (mlen > best_mlen) best_mlen = mlen; - ZSTD_LOG_PARSER("%d: Found REP mlen=%d off=%d price=%d litlen=%d\n", (int)(inr-base), mlen, best_off, price, litlen); - - do { - if (cur + mlen > last_pos || price <= opt[cur + mlen].price) - SET_PRICE(cur + mlen, mlen, i, litlen, price); - mlen--; - } while (mlen >= minMatch); - } } - - match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, inr, iend, maxSearches, mls, matches, best_mlen); - ZSTD_LOG_PARSER("%d: ZSTD_GetAllMatches match_num=%d\n", (int)(inr-base), match_num); - - if (match_num > 0 && (matches[match_num-1].len > sufficient_len || cur + matches[match_num-1].len >= ZSTD_OPT_NUM)) { - best_mlen = matches[match_num-1].len; - best_off = matches[match_num-1].off; - last_pos = cur + 1; - goto _storeSequence; - } - - /* set prices using matches at position = cur */ - for (u = 0; u < match_num; u++) { - mlen = (u>0) ? matches[u-1].len+1 : best_mlen; - best_mlen = matches[u].len; - - // ZSTD_LOG_PARSER("%d: Found1 cur=%d mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(inr-base), cur, matches[u].len, matches[u].off, best_mlen, last_pos); - while (mlen <= best_mlen) { - if (opt[cur].mlen == 1) { - litlen = opt[cur].litlen; - if (cur > litlen) - price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off, mlen - MINMATCH); - else - price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off, mlen - MINMATCH); - } else { - litlen = 0; - price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off, mlen - MINMATCH); - } - - // ZSTD_LOG_PARSER("%d: Found2 mlen=%d best_mlen=%d off=%d price=%d litlen=%d\n", (int)(inr-base), mlen, best_mlen, matches[u].off, price, litlen); - if (cur + mlen > last_pos || (price < opt[cur + mlen].price)) - SET_PRICE(cur + mlen, mlen, matches[u].off, litlen, price); - - mlen++; - } } } // for (cur = 1; cur <= last_pos; cur++) - - best_mlen = opt[last_pos].mlen; - best_off = opt[last_pos].off; - cur = last_pos - best_mlen; - - /* store sequence */ -_storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ - for (u = 1; u <= last_pos; u++) - ZSTD_LOG_PARSER("%d: price[%d/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base+u), u, last_pos, opt[u].price, opt[u].off, opt[u].mlen, opt[u].litlen, opt[u].rep[0], opt[u].rep[1]); - ZSTD_LOG_PARSER("%d: cur=%d/%d best_mlen=%d best_off=%d rep[0]=%d\n", (int)(ip-base+cur), (int)cur, (int)last_pos, (int)best_mlen, (int)best_off, opt[cur].rep[0]); - - opt[0].mlen = 1; - - while (1) { - mlen = opt[cur].mlen; - offset = opt[cur].off; - opt[cur].mlen = best_mlen; - opt[cur].off = best_off; - best_mlen = mlen; - best_off = offset; - if (mlen > cur) break; - cur -= mlen; - } - - for (u = 0; u <= last_pos;) { - ZSTD_LOG_PARSER("%d: price2[%d/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base+u), u, last_pos, opt[u].price, opt[u].off, opt[u].mlen, opt[u].litlen, opt[u].rep[0], opt[u].rep[1]); - u += opt[u].mlen; - } - - for (cur=0; cur < last_pos; ) { - ZSTD_LOG_PARSER("%d: price3[%d/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base+cur), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep[0], opt[cur].rep[1]); - mlen = opt[cur].mlen; - if (mlen == 1) { ip++; cur++; continue; } - offset = opt[cur].off; - cur += mlen; - litLength = (U32)(ip - anchor); - // ZSTD_LOG_ENCODE("%d/%d: ENCODE literals=%d mlen=%d off=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep[0], (int)rep[1]); - - if (offset >= ZSTD_REP_NUM) { - rep[2] = rep[1]; - rep[1] = rep[0]; - rep[0] = offset - ZSTD_REP_MOVE; - } else { - if (offset != 0) { - best_off = rep[offset]; - if (offset != 1) rep[2] = rep[1]; - rep[1] = rep[0]; - rep[0] = best_off; - } - if (litLength == 0 && offset<=1) offset = 1-offset; - } - - ZSTD_LOG_ENCODE("%d/%d: ENCODE literals=%d mlen=%d off=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep[0], (int)rep[1]); - -#if ZSTD_OPT_DEBUG >= 5 - U32 ml2; - if (offset >= ZSTD_REP_NUM) - ml2 = (U32)ZSTD_count(ip, ip-(offset-ZSTD_REP_MOVE), iend); - else - ml2 = (U32)ZSTD_count(ip, ip-rep[0], iend); - if ((offset >= 8) && (ml2 < mlen || ml2 < minMatch)) { - printf("%d: ERROR_NoExt iend=%d mlen=%d offset=%d ml2=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset, (int)ml2); exit(0); } - if (ip < anchor) { - printf("%d: ERROR_NoExt ip < anchor iend=%d mlen=%d offset=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset); exit(0); } - if (ip + mlen > iend) { - printf("%d: ERROR_NoExt ip + mlen >= iend iend=%d mlen=%d offset=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset); exit(0); } -#endif - - ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH); - ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH); - anchor = ip = ip + mlen; - } } /* for (cur=0; cur < last_pos; ) */ - - { /* Last Literals */ - size_t lastLLSize = iend - anchor; - ZSTD_LOG_ENCODE("%d: lastLLSize literals=%u\n", (int)(ip-base), (U32)lastLLSize); - memcpy(seqStorePtr->lit, anchor, lastLLSize); - seqStorePtr->lit += lastLLSize; - } -} - - -FORCE_INLINE -void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, - const void* src, size_t srcSize) -{ - seqStore_t* seqStorePtr = &(ctx->seqStore); - const BYTE* const istart = (const BYTE*)src; - const BYTE* ip = istart; - const BYTE* anchor = istart; - const BYTE* const iend = istart + srcSize; - const BYTE* const ilimit = iend - 8; - const BYTE* const base = ctx->base; - const U32 dictLimit = ctx->dictLimit; - const BYTE* const prefixStart = base + dictLimit; - const BYTE* const dictBase = ctx->dictBase; - const BYTE* const dictEnd = dictBase + dictLimit; - - const U32 maxSearches = 1U << ctx->params.cParams.searchLog; - const U32 sufficient_len = ctx->params.cParams.targetLength; - const U32 mls = ctx->params.cParams.searchLength; - const U32 minMatch = (ctx->params.cParams.searchLength == 3) ? 3 : 4; - - ZSTD_optimal_t* opt = seqStorePtr->priceTable; - ZSTD_match_t* matches = seqStorePtr->matchTable; - const BYTE* inr; - - /* init */ - U32 offset, rep[ZSTD_REP_INIT]; - { U32 i; for (i=0; i<ZSTD_REP_INIT; i++) rep[i]=REPCODE_STARTVALUE; } - - ctx->nextToUpdate3 = ctx->nextToUpdate; - ZSTD_resetSeqStore(seqStorePtr); - ZSTD_rescaleFreqs(seqStorePtr); - if ((ip - prefixStart) < REPCODE_STARTVALUE) ip += REPCODE_STARTVALUE; - - ZSTD_LOG_BLOCK("%d: COMPBLOCK_OPT_EXTDICT srcSz=%d maxSrch=%d mls=%d sufLen=%d\n", (int)(ip-base), (int)srcSize, maxSearches, mls, sufficient_len); - - /* Match Loop */ - while (ip < ilimit) { - U32 cur, match_num, last_pos, litlen, price; - U32 u, mlen, best_mlen, best_off, litLength; - U32 current = (U32)(ip-base); - memset(opt, 0, sizeof(ZSTD_optimal_t)); - last_pos = 0; - inr = ip; - opt[0].litlen = (U32)(ip - anchor); - - /* check repCode */ - { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) { - const U32 repIndex = (U32)(current - rep[i]); - const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; - const BYTE* const repMatch = repBase + repIndex; - if ( ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow */ - && (MEM_readMINMATCH(ip, minMatch) == MEM_readMINMATCH(repMatch, minMatch)) ) { - /* repcode detected we should take it */ - const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; - mlen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iend, repEnd, prefixStart) + minMatch; - - ZSTD_LOG_PARSER("%d: start try REP rep[%d]=%d mlen=%d\n", (int)(ip-base), i, (int)rep[i], (int)mlen); - if (mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) { - best_mlen = mlen; best_off = i; cur = 0; last_pos = 1; - goto _storeSequence; - } - - best_off = (i<=1 && ip == anchor) ? 1-i : i; - litlen = opt[0].litlen; - do { - price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH); - if (mlen > last_pos || price < opt[mlen].price) - SET_PRICE(mlen, mlen, i, litlen, price); /* note : macro modifies last_pos */ - mlen--; - } while (mlen >= minMatch); - } } } - - match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, ip, iend, maxSearches, mls, matches, minMatch); /* first search (depth 0) */ - - ZSTD_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-base), match_num, last_pos); - if (!last_pos && !match_num) { ip++; continue; } - - { U32 i; for (i=0; i<ZSTD_REP_INIT; i++) opt[0].rep[i] = rep[i]; } - opt[0].mlen = 1; - - if (match_num && (matches[match_num-1].len > sufficient_len || matches[match_num-1].len >= ZSTD_OPT_NUM)) { - best_mlen = matches[match_num-1].len; - best_off = matches[match_num-1].off; - cur = 0; - last_pos = 1; - goto _storeSequence; - } - - best_mlen = (last_pos) ? last_pos : minMatch; - - // set prices using matches at position = 0 - for (u = 0; u < match_num; u++) { - mlen = (u>0) ? matches[u-1].len+1 : best_mlen; - best_mlen = matches[u].len; - ZSTD_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-base), matches[u].len, matches[u].off, (int)best_mlen, (int)last_pos); - litlen = opt[0].litlen; - while (mlen <= best_mlen) { - price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off, mlen - MINMATCH); - if (mlen > last_pos || price < opt[mlen].price) - SET_PRICE(mlen, mlen, matches[u].off, litlen, price); - mlen++; - } } - - if (last_pos < minMatch) { - // ip += ((ip-anchor) >> g_searchStrength) + 1; /* jump faster over incompressible sections */ - ip++; continue; - } - - /* check further positions */ - for (cur = 1; cur <= last_pos; cur++) { - inr = ip + cur; - - if (opt[cur-1].mlen == 1) { - litlen = opt[cur-1].litlen + 1; - if (cur > litlen) { - price = opt[cur - litlen].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-litlen); - } else - price = ZSTD_getLiteralPrice(seqStorePtr, litlen, anchor); - } else { - litlen = 1; - price = opt[cur - 1].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-1); - } - - if (cur > last_pos || price <= opt[cur].price) // || ((price == opt[cur].price) && (opt[cur-1].mlen == 1) && (cur != litlen))) - SET_PRICE(cur, 1, 0, litlen, price); - - if (cur == last_pos) break; - - if (inr > ilimit) /* last match must start at a minimum distance of 8 from oend */ - continue; - - mlen = opt[cur].mlen; - if (opt[cur].off >= ZSTD_REP_NUM) { - opt[cur].rep[2] = opt[cur-mlen].rep[1]; - opt[cur].rep[1] = opt[cur-mlen].rep[0]; - opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE; - ZSTD_LOG_ENCODE("%d: COPYREP_OFF cur=%d mlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep[0], opt[cur].rep[1]); - } else { - opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur-mlen].rep[1] : opt[cur-mlen].rep[2]; - opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur-mlen].rep[0] : opt[cur-mlen].rep[1]; - opt[cur].rep[0] = opt[cur-mlen].rep[opt[cur].off]; - ZSTD_LOG_ENCODE("%d: COPYREP_NOR cur=%d mlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep[0], opt[cur].rep[1]); - } - - ZSTD_LOG_PARSER("%d: CURRENT_Ext price[%d/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep[0], opt[cur].rep[1]); - best_mlen = 0; - - { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) { - const U32 repIndex = (U32)(current+cur - opt[cur].rep[i]); - const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; - const BYTE* const repMatch = repBase + repIndex; - if ( ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow */ - && (MEM_readMINMATCH(inr, minMatch) == MEM_readMINMATCH(repMatch, minMatch)) ) { - /* repcode detected */ - const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; - mlen = (U32)ZSTD_count_2segments(inr+minMatch, repMatch+minMatch, iend, repEnd, prefixStart) + minMatch; - ZSTD_LOG_PARSER("%d: Found REP %d/%d mlen=%d off=%d rep=%d opt[%d].off=%d\n", (int)(inr-base), i, ZSTD_REP_NUM, mlen, i, opt[cur].rep[i], cur, opt[cur].off); - - if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) { - ZSTD_LOG_PARSER("%d: REP sufficient_len=%d best_mlen=%d best_off=%d last_pos=%d\n", (int)(inr-base), sufficient_len, best_mlen, best_off, last_pos); - best_mlen = mlen; best_off = i; last_pos = cur + 1; - goto _storeSequence; - } - - best_off = (i<=1 && opt[cur].mlen != 1) ? 1-i : i; - if (opt[cur].mlen == 1) { - litlen = opt[cur].litlen; - if (cur > litlen) { - price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, best_off, mlen - MINMATCH); - } else - price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH); - } else { - litlen = 0; - price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH); - } - - best_mlen = mlen; - ZSTD_LOG_PARSER("%d: Found REP mlen=%d off=%d price=%d litlen=%d\n", (int)(inr-base), mlen, best_off, price, litlen); - - do { - if (cur + mlen > last_pos || price <= opt[cur + mlen].price) - SET_PRICE(cur + mlen, mlen, i, litlen, price); - mlen--; - } while (mlen >= minMatch); - } } } - - match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, inr, iend, maxSearches, mls, matches, minMatch); - ZSTD_LOG_PARSER("%d: ZSTD_GetAllMatches match_num=%d\n", (int)(inr-base), match_num); - - if (match_num > 0 && matches[match_num-1].len > sufficient_len) { - best_mlen = matches[match_num-1].len; - best_off = matches[match_num-1].off; - last_pos = cur + 1; - goto _storeSequence; - } - - best_mlen = (best_mlen > minMatch) ? best_mlen : minMatch; - - /* set prices using matches at position = cur */ - for (u = 0; u < match_num; u++) { - mlen = (u>0) ? matches[u-1].len+1 : best_mlen; - best_mlen = (cur + matches[u].len < ZSTD_OPT_NUM) ? matches[u].len : ZSTD_OPT_NUM - cur; - - // ZSTD_LOG_PARSER("%d: Found1 cur=%d mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(inr-base), cur, matches[u].len, matches[u].off, best_mlen, last_pos); - while (mlen <= best_mlen) { - if (opt[cur].mlen == 1) { - litlen = opt[cur].litlen; - if (cur > litlen) - price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off, mlen - MINMATCH); - else - price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off, mlen - MINMATCH); - } else { - litlen = 0; - price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off, mlen - MINMATCH); - } - - // ZSTD_LOG_PARSER("%d: Found2 mlen=%d best_mlen=%d off=%d price=%d litlen=%d\n", (int)(inr-base), mlen, best_mlen, matches[u].off, price, litlen); - if (cur + mlen > last_pos || (price < opt[cur + mlen].price)) - SET_PRICE(cur + mlen, mlen, matches[u].off, litlen, price); - - mlen++; - } } } /* for (cur = 1; cur <= last_pos; cur++) */ - - best_mlen = opt[last_pos].mlen; - best_off = opt[last_pos].off; - cur = last_pos - best_mlen; - - /* store sequence */ -_storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ - for (u = 1; u <= last_pos; u++) - ZSTD_LOG_PARSER("%d: price[%u/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base+u), u, last_pos, opt[u].price, opt[u].off, opt[u].mlen, opt[u].litlen, opt[u].rep[0], opt[u].rep[1]); - ZSTD_LOG_PARSER("%d: cur=%d/%d best_mlen=%d best_off=%d rep[0]=%d\n", (int)(ip-base+cur), (int)cur, (int)last_pos, (int)best_mlen, (int)best_off, opt[cur].rep[0]); - - opt[0].mlen = 1; - - while (1) { - mlen = opt[cur].mlen; - offset = opt[cur].off; - opt[cur].mlen = best_mlen; - opt[cur].off = best_off; - best_mlen = mlen; - best_off = offset; - if (mlen > cur) break; - cur -= mlen; - } - - for (u = 0; u <= last_pos; ) { - ZSTD_LOG_PARSER("%d: price2[%d/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base+u), u, last_pos, opt[u].price, opt[u].off, opt[u].mlen, opt[u].litlen, opt[u].rep[0], opt[u].rep[1]); - u += opt[u].mlen; - } - - for (cur=0; cur < last_pos; ) { - ZSTD_LOG_PARSER("%d: price3[%d/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base+cur), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep[0], opt[cur].rep[1]); - mlen = opt[cur].mlen; - if (mlen == 1) { ip++; cur++; continue; } - offset = opt[cur].off; - cur += mlen; - litLength = (U32)(ip - anchor); - // ZSTD_LOG_ENCODE("%d/%d: ENCODE1 literals=%d mlen=%d off=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep[0], (int)rep[1]); - - if (offset >= ZSTD_REP_NUM) { - rep[2] = rep[1]; - rep[1] = rep[0]; - rep[0] = offset - ZSTD_REP_MOVE; - } else { - if (offset != 0) { - best_off = rep[offset]; - if (offset != 1) rep[2] = rep[1]; - rep[1] = rep[0]; - rep[0] = best_off; - } - if (litLength == 0 && offset<=1) offset = 1-offset; - } - - ZSTD_LOG_ENCODE("%d/%d: ENCODE literals=%d mlen=%d off=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep[0], (int)rep[1]); - -#if ZSTD_OPT_DEBUG >= 5 - U32 ml2; - if (offset >= ZSTD_REP_NUM) { - best_off = offset - ZSTD_REP_MOVE; - if (best_off > (size_t)(ip - prefixStart)) { - const BYTE* match = dictEnd - (best_off - (ip - prefixStart)); - ml2 = ZSTD_count_2segments(ip, match, iend, dictEnd, prefixStart); - ZSTD_LOG_PARSER("%d: ZSTD_count_2segments=%d offset=%d dictBase=%p dictEnd=%p prefixStart=%p ip=%p match=%p\n", (int)current, (int)ml2, (int)best_off, dictBase, dictEnd, prefixStart, ip, match); - } - else ml2 = (U32)ZSTD_count(ip, ip-offset, iend); - } - else ml2 = (U32)ZSTD_count(ip, ip-rep[0], iend); - if ((offset >= 8) && (ml2 < mlen || ml2 < minMatch)) { - printf("%d: ERROR_Ext iend=%d mlen=%d offset=%d ml2=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset, (int)ml2); exit(0); } - if (ip < anchor) { - printf("%d: ERROR_Ext ip < anchor iend=%d mlen=%d offset=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset); exit(0); } - if (ip + mlen > iend) { - printf("%d: ERROR_Ext ip + mlen >= iend iend=%d mlen=%d offset=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset); exit(0); } -#endif - - ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH); - ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH); - anchor = ip = ip + mlen; - } } /* for (cur=0; cur < last_pos; ) */ - - { /* Last Literals */ - size_t lastLLSize = iend - anchor; - ZSTD_LOG_ENCODE("%d: lastLLSize literals=%u\n", (int)(ip-base), (U32)(lastLLSize)); - memcpy(seqStorePtr->lit, anchor, lastLLSize); - seqStorePtr->lit += lastLLSize; - } -} +#include <contrib/libs/zstd06/renames.h> +/* + ZSTD Optimal mode + Copyright (C) 2016, Przemyslaw Skibinski, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - Zstd source repository : https://www.zstd.net +*/ + +/* Note : this file is intended to be included within zstd_compress.c */ + + +#define ZSTD_FREQ_DIV 5 + +/*-************************************* +* Price functions for optimal parser +***************************************/ +FORCE_INLINE void ZSTD_setLog2Prices(seqStore_t* ssPtr) +{ + ssPtr->log2matchLengthSum = ZSTD_highbit(ssPtr->matchLengthSum+1); + ssPtr->log2litLengthSum = ZSTD_highbit(ssPtr->litLengthSum+1); + ssPtr->log2litSum = ZSTD_highbit(ssPtr->litSum+1); + ssPtr->log2offCodeSum = ZSTD_highbit(ssPtr->offCodeSum+1); + ssPtr->factor = 1 + ((ssPtr->litSum>>5) / ssPtr->litLengthSum) + ((ssPtr->litSum<<1) / (ssPtr->litSum + ssPtr->matchSum)); +} + + +MEM_STATIC void ZSTD_rescaleFreqs(seqStore_t* ssPtr) +{ + unsigned u; + + ssPtr->cachedLiterals = NULL; + ssPtr->cachedPrice = ssPtr->cachedLitLength = 0; + + if (ssPtr->litLengthSum == 0) { + ssPtr->litSum = (2<<Litbits); + ssPtr->litLengthSum = MaxLL+1; + ssPtr->matchLengthSum = MaxML+1; + ssPtr->offCodeSum = (MaxOff+1); + ssPtr->matchSum = (2<<Litbits); + + for (u=0; u<=MaxLit; u++) + ssPtr->litFreq[u] = 2; + for (u=0; u<=MaxLL; u++) + ssPtr->litLengthFreq[u] = 1; + for (u=0; u<=MaxML; u++) + ssPtr->matchLengthFreq[u] = 1; + for (u=0; u<=MaxOff; u++) + ssPtr->offCodeFreq[u] = 1; + } else { + ssPtr->matchLengthSum = 0; + ssPtr->litLengthSum = 0; + ssPtr->offCodeSum = 0; + ssPtr->matchSum = 0; + ssPtr->litSum = 0; + + for (u=0; u<=MaxLit; u++) { + ssPtr->litFreq[u] = 1 + (ssPtr->litFreq[u]>>ZSTD_FREQ_DIV); + ssPtr->litSum += ssPtr->litFreq[u]; + } + for (u=0; u<=MaxLL; u++) { + ssPtr->litLengthFreq[u] = 1 + (ssPtr->litLengthFreq[u]>>ZSTD_FREQ_DIV); + ssPtr->litLengthSum += ssPtr->litLengthFreq[u]; + } + for (u=0; u<=MaxML; u++) { + ssPtr->matchLengthFreq[u] = 1 + (ssPtr->matchLengthFreq[u]>>ZSTD_FREQ_DIV); + ssPtr->matchLengthSum += ssPtr->matchLengthFreq[u]; + ssPtr->matchSum += ssPtr->matchLengthFreq[u] * (u + 3); + } + for (u=0; u<=MaxOff; u++) { + ssPtr->offCodeFreq[u] = 1 + (ssPtr->offCodeFreq[u]>>ZSTD_FREQ_DIV); + ssPtr->offCodeSum += ssPtr->offCodeFreq[u]; + } + } + + ZSTD_setLog2Prices(ssPtr); +} + + +FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* ssPtr, U32 litLength, const BYTE* literals) +{ + U32 price, u; + + if (litLength == 0) + return ssPtr->log2litLengthSum - ZSTD_highbit(ssPtr->litLengthFreq[0]+1); + + /* literals */ + if (ssPtr->cachedLiterals == literals) { + U32 additional = litLength - ssPtr->cachedLitLength; + const BYTE* literals2 = ssPtr->cachedLiterals + ssPtr->cachedLitLength; + price = ssPtr->cachedPrice + additional * ssPtr->log2litSum; + for (u=0; u < additional; u++) + price -= ZSTD_highbit(ssPtr->litFreq[literals2[u]]+1); + ssPtr->cachedPrice = price; + ssPtr->cachedLitLength = litLength; + } else { + price = litLength * ssPtr->log2litSum; + for (u=0; u < litLength; u++) + price -= ZSTD_highbit(ssPtr->litFreq[literals[u]]+1); + + if (litLength >= 12) { + ssPtr->cachedLiterals = literals; + ssPtr->cachedPrice = price; + ssPtr->cachedLitLength = litLength; + } + } + + /* literal Length */ + { static const BYTE LL_Code[64] = { 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 16, 17, 17, 18, 18, 19, 19, + 20, 20, 20, 20, 21, 21, 21, 21, + 22, 22, 22, 22, 22, 22, 22, 22, + 23, 23, 23, 23, 23, 23, 23, 23, + 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24 }; + const BYTE LL_deltaCode = 19; + const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit(litLength) + LL_deltaCode : LL_Code[litLength]; + price += LL_bits[llCode] + ssPtr->log2litLengthSum - ZSTD_highbit(ssPtr->litLengthFreq[llCode]+1); + } + + return price; +} + + +FORCE_INLINE U32 ZSTD_getPrice(seqStore_t* seqStorePtr, U32 litLength, const BYTE* literals, U32 offset, U32 matchLength) +{ + /* offset */ + BYTE offCode = (BYTE)ZSTD_highbit(offset+1); + U32 price = offCode + seqStorePtr->log2offCodeSum - ZSTD_highbit(seqStorePtr->offCodeFreq[offCode]+1); + + /* match Length */ + { static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, + 38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, + 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, + 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 }; + const BYTE ML_deltaCode = 36; + const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit(matchLength) + ML_deltaCode : ML_Code[matchLength]; + price += ML_bits[mlCode] + seqStorePtr->log2matchLengthSum - ZSTD_highbit(seqStorePtr->matchLengthFreq[mlCode]+1); + } + + return price + ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + seqStorePtr->factor; +} + + +MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const BYTE* literals, U32 offset, U32 matchLength) +{ + U32 u; + + /* literals */ + seqStorePtr->litSum += litLength; + for (u=0; u < litLength; u++) + seqStorePtr->litFreq[literals[u]]++; + + /* literal Length */ + { static const BYTE LL_Code[64] = { 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 16, 17, 17, 18, 18, 19, 19, + 20, 20, 20, 20, 21, 21, 21, 21, + 22, 22, 22, 22, 22, 22, 22, 22, + 23, 23, 23, 23, 23, 23, 23, 23, + 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24 }; + const BYTE LL_deltaCode = 19; + const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit(litLength) + LL_deltaCode : LL_Code[litLength]; + seqStorePtr->litLengthFreq[llCode]++; + seqStorePtr->litLengthSum++; + } + + /* match offset */ + { BYTE offCode = (BYTE)ZSTD_highbit(offset+1); + seqStorePtr->offCodeSum++; + seqStorePtr->offCodeFreq[offCode]++; + } + + /* match Length */ + { static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, + 38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, + 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, + 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 }; + const BYTE ML_deltaCode = 36; + const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit(matchLength) + ML_deltaCode : ML_Code[matchLength]; + seqStorePtr->matchLengthFreq[mlCode]++; + seqStorePtr->matchLengthSum++; + } + + ZSTD_setLog2Prices(seqStorePtr); +} + + +#define SET_PRICE(pos, mlen_, offset_, litlen_, price_) \ + { \ + while (last_pos < pos) { opt[last_pos+1].price = 1<<30; last_pos++; } \ + opt[pos].mlen = mlen_; \ + opt[pos].off = offset_; \ + opt[pos].litlen = litlen_; \ + opt[pos].price = price_; \ + ZSTD_LOG_PARSER("%d: SET price[%d/%d]=%d litlen=%d len=%d off=%d\n", (int)(inr-base), (int)pos, (int)last_pos, opt[pos].price, opt[pos].litlen, opt[pos].mlen, opt[pos].off); \ + } + + + + +/* Update hashTable3 up to ip (excluded) + Assumption : always within prefix (ie. not within extDict) */ +FORCE_INLINE +U32 ZSTD_insertAndFindFirstIndexHash3 (ZSTD_CCtx* zc, const BYTE* ip) +{ + U32* const hashTable3 = zc->hashTable3; + U32 const hashLog3 = zc->hashLog3; + const BYTE* const base = zc->base; + U32 idx = zc->nextToUpdate3; + const U32 target = zc->nextToUpdate3 = (U32)(ip - base); + const size_t hash3 = ZSTD_hash3Ptr(ip, hashLog3); + + while(idx < target) { + hashTable3[ZSTD_hash3Ptr(base+idx, hashLog3)] = idx; + idx++; + } + + return hashTable3[hash3]; +} + + +/*-************************************* +* Binary Tree search +***************************************/ +static U32 ZSTD_insertBtAndGetAllMatches ( + ZSTD_CCtx* zc, + const BYTE* const ip, const BYTE* const iLimit, + U32 nbCompares, const U32 mls, + U32 extDict, ZSTD_match_t* matches, const U32 minMatchLen) +{ + const BYTE* const base = zc->base; + const U32 current = (U32)(ip-base); + const U32 hashLog = zc->params.cParams.hashLog; + const size_t h = ZSTD_hashPtr(ip, hashLog, mls); + U32* const hashTable = zc->hashTable; + U32 matchIndex = hashTable[h]; + U32* const bt = zc->chainTable; + const U32 btLog = zc->params.cParams.chainLog - 1; + const U32 btMask= (1U << btLog) - 1; + size_t commonLengthSmaller=0, commonLengthLarger=0; + const BYTE* const dictBase = zc->dictBase; + const U32 dictLimit = zc->dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const prefixStart = base + dictLimit; + const U32 btLow = btMask >= current ? 0 : current - btMask; + const U32 windowLow = zc->lowLimit; + U32* smallerPtr = bt + 2*(current&btMask); + U32* largerPtr = bt + 2*(current&btMask) + 1; + U32 matchEndIdx = current+8; + U32 dummy32; /* to be nullified at the end */ + U32 mnum = 0; + + const U32 minMatch = (mls == 3) ? 3 : 4; + size_t bestLength = minMatchLen-1; + + if (minMatch == 3) { /* HC3 match finder */ + U32 const matchIndex3 = ZSTD_insertAndFindFirstIndexHash3 (zc, ip); + if (matchIndex3>windowLow && (current - matchIndex3 < (1<<18))) { + const BYTE* match; + size_t currentMl=0; + if ((!extDict) || matchIndex3 >= dictLimit) { + match = base + matchIndex3; + if (match[bestLength] == ip[bestLength]) currentMl = ZSTD_count(ip, match, iLimit); + } else { + match = dictBase + matchIndex3; + if (MEM_readMINMATCH(match, MINMATCH) == MEM_readMINMATCH(ip, MINMATCH)) /* assumption : matchIndex3 <= dictLimit-4 (by table construction) */ + currentMl = ZSTD_count_2segments(ip+MINMATCH, match+MINMATCH, iLimit, dictEnd, prefixStart) + MINMATCH; + } + + /* save best solution */ + if (currentMl > bestLength) { + bestLength = currentMl; + matches[mnum].off = ZSTD_REP_MOVE + current - matchIndex3; + matches[mnum].len = (U32)currentMl; + mnum++; + if (currentMl > ZSTD_OPT_NUM) goto update; + if (ip+currentMl == iLimit) goto update; /* best possible, and avoid read overflow*/ + } + } + } + + hashTable[h] = current; /* Update Hash Table */ + + while (nbCompares-- && (matchIndex > windowLow)) { + U32* nextPtr = bt + 2*(matchIndex & btMask); + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + const BYTE* match; + + if ((!extDict) || (matchIndex+matchLength >= dictLimit)) { + match = base + matchIndex; + if (match[matchLength] == ip[matchLength]) { +#if ZSTD_OPT_DEBUG >= 5 + size_t ml; + if (matchIndex < dictLimit) + ml = ZSTD_count_2segments(ip, dictBase + matchIndex, iLimit, dictEnd, prefixStart); + else + ml = ZSTD_count(ip, match, ip+matchLength); + if (ml < matchLength) + printf("%d: ERROR_NOEXT: offset=%d matchLength=%d matchIndex=%d dictLimit=%d ml=%d\n", current, (int)(current - matchIndex), (int)matchLength, (int)matchIndex, (int)dictLimit, (int)ml), exit(0); +#endif + matchLength += ZSTD_count(ip+matchLength+1, match+matchLength+1, iLimit) +1; + } + } else { + match = dictBase + matchIndex; +#if ZSTD_OPT_DEBUG >= 5 + if (memcmp(match, ip, matchLength) != 0) + printf("%d: ERROR_EXT: matchLength=%d ZSTD_count=%d\n", current, (int)matchLength, (int)ZSTD_count_2segments(ip+matchLength, match+matchLength, iLimit, dictEnd, prefixStart)), exit(0); +#endif + matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iLimit, dictEnd, prefixStart); + ZSTD_LOG_PARSER("%d: ZSTD_INSERTBTANDGETALLMATCHES=%d offset=%d dictBase=%p dictEnd=%p prefixStart=%p ip=%p match=%p\n", (int)current, (int)matchLength, (int)(current - matchIndex), dictBase, dictEnd, prefixStart, ip, match); + if (matchIndex+matchLength >= dictLimit) + match = base + matchIndex; /* to prepare for next usage of match[matchLength] */ + } + + if (matchLength > bestLength) { + if (matchLength > matchEndIdx - matchIndex) matchEndIdx = matchIndex + (U32)matchLength; + bestLength = matchLength; + matches[mnum].off = ZSTD_REP_MOVE + current - matchIndex; + matches[mnum].len = (U32)matchLength; + mnum++; + if (matchLength > ZSTD_OPT_NUM) break; + if (ip+matchLength == iLimit) /* equal : no way to know if inf or sup */ + break; /* drop, to guarantee consistency (miss a little bit of compression) */ + } + + if (match[matchLength] < ip[matchLength]) { + /* match is smaller than current */ + *smallerPtr = matchIndex; /* update smaller idx */ + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + smallerPtr = nextPtr+1; /* new "smaller" => larger of match */ + matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ + } else { + /* match is larger than current */ + *largerPtr = matchIndex; + commonLengthLarger = matchLength; + if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + largerPtr = nextPtr; + matchIndex = nextPtr[0]; + } } + + *smallerPtr = *largerPtr = 0; + +update: + zc->nextToUpdate = (matchEndIdx > current + 8) ? matchEndIdx - 8 : current+1; + return mnum; +} + + +/** Tree updater, providing best match */ +static U32 ZSTD_BtGetAllMatches ( + ZSTD_CCtx* zc, + const BYTE* const ip, const BYTE* const iLimit, + const U32 maxNbAttempts, const U32 mls, ZSTD_match_t* matches, const U32 minMatchLen) +{ + if (ip < zc->base + zc->nextToUpdate) return 0; /* skipped area */ + ZSTD_updateTree(zc, ip, iLimit, maxNbAttempts, mls); + return ZSTD_insertBtAndGetAllMatches(zc, ip, iLimit, maxNbAttempts, mls, 0, matches, minMatchLen); +} + + +static U32 ZSTD_BtGetAllMatches_selectMLS ( + ZSTD_CCtx* zc, /* Index table will be updated */ + const BYTE* ip, const BYTE* const iHighLimit, + const U32 maxNbAttempts, const U32 matchLengthSearch, ZSTD_match_t* matches, const U32 minMatchLen) +{ + switch(matchLengthSearch) + { + case 3 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 3, matches, minMatchLen); + default : + case 4 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 4, matches, minMatchLen); + case 5 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 5, matches, minMatchLen); + case 6 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 6, matches, minMatchLen); + } +} + +/** Tree updater, providing best match */ +static U32 ZSTD_BtGetAllMatches_extDict ( + ZSTD_CCtx* zc, + const BYTE* const ip, const BYTE* const iLimit, + const U32 maxNbAttempts, const U32 mls, ZSTD_match_t* matches, const U32 minMatchLen) +{ + if (ip < zc->base + zc->nextToUpdate) return 0; /* skipped area */ + ZSTD_updateTree_extDict(zc, ip, iLimit, maxNbAttempts, mls); + return ZSTD_insertBtAndGetAllMatches(zc, ip, iLimit, maxNbAttempts, mls, 1, matches, minMatchLen); +} + + +static U32 ZSTD_BtGetAllMatches_selectMLS_extDict ( + ZSTD_CCtx* zc, /* Index table will be updated */ + const BYTE* ip, const BYTE* const iHighLimit, + const U32 maxNbAttempts, const U32 matchLengthSearch, ZSTD_match_t* matches, const U32 minMatchLen) +{ + switch(matchLengthSearch) + { + case 3 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 3, matches, minMatchLen); + default : + case 4 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 4, matches, minMatchLen); + case 5 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 5, matches, minMatchLen); + case 6 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 6, matches, minMatchLen); + } +} + + +/*-******************************* +* Optimal parser +*********************************/ +FORCE_INLINE +void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, + const void* src, size_t srcSize) +{ + seqStore_t* seqStorePtr = &(ctx->seqStore); + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - 8; + const BYTE* const base = ctx->base; + const BYTE* const prefixStart = base + ctx->dictLimit; + + const U32 maxSearches = 1U << ctx->params.cParams.searchLog; + const U32 sufficient_len = ctx->params.cParams.targetLength; + const U32 mls = ctx->params.cParams.searchLength; + const U32 minMatch = (ctx->params.cParams.searchLength == 3) ? 3 : 4; + + ZSTD_optimal_t* opt = seqStorePtr->priceTable; + ZSTD_match_t* matches = seqStorePtr->matchTable; + const BYTE* inr; + + /* init */ + U32 offset, rep[ZSTD_REP_INIT]; + { U32 i; for (i=0; i<ZSTD_REP_INIT; i++) rep[i]=REPCODE_STARTVALUE; } + + ctx->nextToUpdate3 = ctx->nextToUpdate; + ZSTD_resetSeqStore(seqStorePtr); + ZSTD_rescaleFreqs(seqStorePtr); + if ((ip-prefixStart) < REPCODE_STARTVALUE) ip = prefixStart + REPCODE_STARTVALUE; + + ZSTD_LOG_BLOCK("%d: COMPBLOCK_OPT_GENERIC srcSz=%d maxSrch=%d mls=%d sufLen=%d\n", (int)(ip-base), (int)srcSize, maxSearches, mls, sufficient_len); + + /* Match Loop */ + while (ip < ilimit) { + U32 cur, match_num, last_pos, litlen, price; + U32 u, mlen, best_mlen, best_off, litLength; + memset(opt, 0, sizeof(ZSTD_optimal_t)); + last_pos = 0; + litlen = (U32)(ip - anchor); + + /* check repCode */ + { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) + if (MEM_readMINMATCH(ip, minMatch) == MEM_readMINMATCH(ip - rep[i], minMatch)) { + /* repcode : we take it */ + mlen = (U32)ZSTD_count(ip+minMatch, ip+minMatch-rep[i], iend) + minMatch; + ZSTD_LOG_PARSER("%d: start try REP rep[%d]=%d mlen=%d\n", (int)(ip-base), i, (int)rep[i], (int)mlen); + if (mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) { + best_mlen = mlen; best_off = i; cur = 0; last_pos = 1; + goto _storeSequence; + } + best_off = (i<=1 && ip == anchor) ? 1-i : i; + do { + price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH); + if (mlen > last_pos || price < opt[mlen].price) + SET_PRICE(mlen, mlen, i, litlen, price); /* note : macro modifies last_pos */ + mlen--; + } while (mlen >= minMatch); + } } + + match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, ip, iend, maxSearches, mls, matches, minMatch); + + ZSTD_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-base), match_num, last_pos); + if (!last_pos && !match_num) { ip++; continue; } + + if (match_num && (matches[match_num-1].len > sufficient_len || matches[match_num-1].len >= ZSTD_OPT_NUM)) { + best_mlen = matches[match_num-1].len; + best_off = matches[match_num-1].off; + cur = 0; + last_pos = 1; + goto _storeSequence; + } + + /* set prices using matches at position = 0 */ + best_mlen = (last_pos) ? last_pos : minMatch; + for (u = 0; u < match_num; u++) { + mlen = (u>0) ? matches[u-1].len+1 : best_mlen; + best_mlen = matches[u].len; + ZSTD_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-base), matches[u].len, matches[u].off, (int)best_mlen, (int)last_pos); + while (mlen <= best_mlen) { + price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off, mlen - MINMATCH); + if (mlen > last_pos || price < opt[mlen].price) + SET_PRICE(mlen, mlen, matches[u].off, litlen, price); /* note : macro modifies last_pos */ + mlen++; + } } + + if (last_pos < minMatch) { ip++; continue; } + + /* initialize opt[0] */ + { U32 i ; for (i=0; i<ZSTD_REP_INIT; i++) opt[0].rep[i] = rep[i]; } + opt[0].mlen = 1; + opt[0].litlen = litlen; + + /* check further positions */ + for (cur = 1; cur <= last_pos; cur++) { + inr = ip + cur; + + if (opt[cur-1].mlen == 1) { + litlen = opt[cur-1].litlen + 1; + if (cur > litlen) { + price = opt[cur - litlen].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-litlen); + } else + price = ZSTD_getLiteralPrice(seqStorePtr, litlen, anchor); + } else { + litlen = 1; + price = opt[cur - 1].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-1); + } + + if (cur > last_pos || price <= opt[cur].price) // || ((price == opt[cur].price) && (opt[cur-1].mlen == 1) && (cur != litlen))) + SET_PRICE(cur, 1, 0, litlen, price); + + if (cur == last_pos) break; + + if (inr > ilimit) /* last match must start at a minimum distance of 8 from oend */ + continue; + + mlen = opt[cur].mlen; + if (opt[cur].off >= ZSTD_REP_NUM) { + opt[cur].rep[2] = opt[cur-mlen].rep[1]; + opt[cur].rep[1] = opt[cur-mlen].rep[0]; + opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE; + ZSTD_LOG_ENCODE("%d: COPYREP_OFF cur=%d mlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep[0], opt[cur].rep[1]); + } else { + opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur-mlen].rep[1] : opt[cur-mlen].rep[2]; + opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur-mlen].rep[0] : opt[cur-mlen].rep[1]; + opt[cur].rep[0] = opt[cur-mlen].rep[opt[cur].off]; + ZSTD_LOG_ENCODE("%d: COPYREP_NOR cur=%d mlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep[0], opt[cur].rep[1]); + } + + ZSTD_LOG_PARSER("%d: CURRENT_NoExt price[%d/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep[0], opt[cur].rep[1]); + + best_mlen = minMatch; + { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) + if (MEM_readMINMATCH(inr, minMatch) == MEM_readMINMATCH(inr - opt[cur].rep[i], minMatch)) { /* check rep */ + mlen = (U32)ZSTD_count(inr+minMatch, inr+minMatch - opt[cur].rep[i], iend) + minMatch; + ZSTD_LOG_PARSER("%d: Found REP %d/%d mlen=%d off=%d rep=%d opt[%d].off=%d\n", (int)(inr-base), i, ZSTD_REP_NUM, mlen, i, opt[cur].rep[i], cur, opt[cur].off); + + if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) { + ZSTD_LOG_PARSER("%d: REP sufficient_len=%d best_mlen=%d best_off=%d last_pos=%d\n", (int)(inr-base), sufficient_len, best_mlen, best_off, last_pos); + best_mlen = mlen; best_off = i; last_pos = cur + 1; + goto _storeSequence; + } + + best_off = (i<=1 && opt[cur].mlen != 1) ? 1-i : i; + if (opt[cur].mlen == 1) { + litlen = opt[cur].litlen; + if (cur > litlen) { + price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, best_off, mlen - MINMATCH); + } else + price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH); + } else { + litlen = 0; + price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH); + } + + if (mlen > best_mlen) best_mlen = mlen; + ZSTD_LOG_PARSER("%d: Found REP mlen=%d off=%d price=%d litlen=%d\n", (int)(inr-base), mlen, best_off, price, litlen); + + do { + if (cur + mlen > last_pos || price <= opt[cur + mlen].price) + SET_PRICE(cur + mlen, mlen, i, litlen, price); + mlen--; + } while (mlen >= minMatch); + } } + + match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, inr, iend, maxSearches, mls, matches, best_mlen); + ZSTD_LOG_PARSER("%d: ZSTD_GetAllMatches match_num=%d\n", (int)(inr-base), match_num); + + if (match_num > 0 && (matches[match_num-1].len > sufficient_len || cur + matches[match_num-1].len >= ZSTD_OPT_NUM)) { + best_mlen = matches[match_num-1].len; + best_off = matches[match_num-1].off; + last_pos = cur + 1; + goto _storeSequence; + } + + /* set prices using matches at position = cur */ + for (u = 0; u < match_num; u++) { + mlen = (u>0) ? matches[u-1].len+1 : best_mlen; + best_mlen = matches[u].len; + + // ZSTD_LOG_PARSER("%d: Found1 cur=%d mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(inr-base), cur, matches[u].len, matches[u].off, best_mlen, last_pos); + while (mlen <= best_mlen) { + if (opt[cur].mlen == 1) { + litlen = opt[cur].litlen; + if (cur > litlen) + price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off, mlen - MINMATCH); + else + price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off, mlen - MINMATCH); + } else { + litlen = 0; + price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off, mlen - MINMATCH); + } + + // ZSTD_LOG_PARSER("%d: Found2 mlen=%d best_mlen=%d off=%d price=%d litlen=%d\n", (int)(inr-base), mlen, best_mlen, matches[u].off, price, litlen); + if (cur + mlen > last_pos || (price < opt[cur + mlen].price)) + SET_PRICE(cur + mlen, mlen, matches[u].off, litlen, price); + + mlen++; + } } } // for (cur = 1; cur <= last_pos; cur++) + + best_mlen = opt[last_pos].mlen; + best_off = opt[last_pos].off; + cur = last_pos - best_mlen; + + /* store sequence */ +_storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ + for (u = 1; u <= last_pos; u++) + ZSTD_LOG_PARSER("%d: price[%d/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base+u), u, last_pos, opt[u].price, opt[u].off, opt[u].mlen, opt[u].litlen, opt[u].rep[0], opt[u].rep[1]); + ZSTD_LOG_PARSER("%d: cur=%d/%d best_mlen=%d best_off=%d rep[0]=%d\n", (int)(ip-base+cur), (int)cur, (int)last_pos, (int)best_mlen, (int)best_off, opt[cur].rep[0]); + + opt[0].mlen = 1; + + while (1) { + mlen = opt[cur].mlen; + offset = opt[cur].off; + opt[cur].mlen = best_mlen; + opt[cur].off = best_off; + best_mlen = mlen; + best_off = offset; + if (mlen > cur) break; + cur -= mlen; + } + + for (u = 0; u <= last_pos;) { + ZSTD_LOG_PARSER("%d: price2[%d/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base+u), u, last_pos, opt[u].price, opt[u].off, opt[u].mlen, opt[u].litlen, opt[u].rep[0], opt[u].rep[1]); + u += opt[u].mlen; + } + + for (cur=0; cur < last_pos; ) { + ZSTD_LOG_PARSER("%d: price3[%d/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base+cur), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep[0], opt[cur].rep[1]); + mlen = opt[cur].mlen; + if (mlen == 1) { ip++; cur++; continue; } + offset = opt[cur].off; + cur += mlen; + litLength = (U32)(ip - anchor); + // ZSTD_LOG_ENCODE("%d/%d: ENCODE literals=%d mlen=%d off=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep[0], (int)rep[1]); + + if (offset >= ZSTD_REP_NUM) { + rep[2] = rep[1]; + rep[1] = rep[0]; + rep[0] = offset - ZSTD_REP_MOVE; + } else { + if (offset != 0) { + best_off = rep[offset]; + if (offset != 1) rep[2] = rep[1]; + rep[1] = rep[0]; + rep[0] = best_off; + } + if (litLength == 0 && offset<=1) offset = 1-offset; + } + + ZSTD_LOG_ENCODE("%d/%d: ENCODE literals=%d mlen=%d off=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep[0], (int)rep[1]); + +#if ZSTD_OPT_DEBUG >= 5 + U32 ml2; + if (offset >= ZSTD_REP_NUM) + ml2 = (U32)ZSTD_count(ip, ip-(offset-ZSTD_REP_MOVE), iend); + else + ml2 = (U32)ZSTD_count(ip, ip-rep[0], iend); + if ((offset >= 8) && (ml2 < mlen || ml2 < minMatch)) { + printf("%d: ERROR_NoExt iend=%d mlen=%d offset=%d ml2=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset, (int)ml2); exit(0); } + if (ip < anchor) { + printf("%d: ERROR_NoExt ip < anchor iend=%d mlen=%d offset=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset); exit(0); } + if (ip + mlen > iend) { + printf("%d: ERROR_NoExt ip + mlen >= iend iend=%d mlen=%d offset=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset); exit(0); } +#endif + + ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH); + ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH); + anchor = ip = ip + mlen; + } } /* for (cur=0; cur < last_pos; ) */ + + { /* Last Literals */ + size_t lastLLSize = iend - anchor; + ZSTD_LOG_ENCODE("%d: lastLLSize literals=%u\n", (int)(ip-base), (U32)lastLLSize); + memcpy(seqStorePtr->lit, anchor, lastLLSize); + seqStorePtr->lit += lastLLSize; + } +} + + +FORCE_INLINE +void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, + const void* src, size_t srcSize) +{ + seqStore_t* seqStorePtr = &(ctx->seqStore); + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - 8; + const BYTE* const base = ctx->base; + const U32 dictLimit = ctx->dictLimit; + const BYTE* const prefixStart = base + dictLimit; + const BYTE* const dictBase = ctx->dictBase; + const BYTE* const dictEnd = dictBase + dictLimit; + + const U32 maxSearches = 1U << ctx->params.cParams.searchLog; + const U32 sufficient_len = ctx->params.cParams.targetLength; + const U32 mls = ctx->params.cParams.searchLength; + const U32 minMatch = (ctx->params.cParams.searchLength == 3) ? 3 : 4; + + ZSTD_optimal_t* opt = seqStorePtr->priceTable; + ZSTD_match_t* matches = seqStorePtr->matchTable; + const BYTE* inr; + + /* init */ + U32 offset, rep[ZSTD_REP_INIT]; + { U32 i; for (i=0; i<ZSTD_REP_INIT; i++) rep[i]=REPCODE_STARTVALUE; } + + ctx->nextToUpdate3 = ctx->nextToUpdate; + ZSTD_resetSeqStore(seqStorePtr); + ZSTD_rescaleFreqs(seqStorePtr); + if ((ip - prefixStart) < REPCODE_STARTVALUE) ip += REPCODE_STARTVALUE; + + ZSTD_LOG_BLOCK("%d: COMPBLOCK_OPT_EXTDICT srcSz=%d maxSrch=%d mls=%d sufLen=%d\n", (int)(ip-base), (int)srcSize, maxSearches, mls, sufficient_len); + + /* Match Loop */ + while (ip < ilimit) { + U32 cur, match_num, last_pos, litlen, price; + U32 u, mlen, best_mlen, best_off, litLength; + U32 current = (U32)(ip-base); + memset(opt, 0, sizeof(ZSTD_optimal_t)); + last_pos = 0; + inr = ip; + opt[0].litlen = (U32)(ip - anchor); + + /* check repCode */ + { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) { + const U32 repIndex = (U32)(current - rep[i]); + const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; + const BYTE* const repMatch = repBase + repIndex; + if ( ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow */ + && (MEM_readMINMATCH(ip, minMatch) == MEM_readMINMATCH(repMatch, minMatch)) ) { + /* repcode detected we should take it */ + const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; + mlen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iend, repEnd, prefixStart) + minMatch; + + ZSTD_LOG_PARSER("%d: start try REP rep[%d]=%d mlen=%d\n", (int)(ip-base), i, (int)rep[i], (int)mlen); + if (mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) { + best_mlen = mlen; best_off = i; cur = 0; last_pos = 1; + goto _storeSequence; + } + + best_off = (i<=1 && ip == anchor) ? 1-i : i; + litlen = opt[0].litlen; + do { + price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH); + if (mlen > last_pos || price < opt[mlen].price) + SET_PRICE(mlen, mlen, i, litlen, price); /* note : macro modifies last_pos */ + mlen--; + } while (mlen >= minMatch); + } } } + + match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, ip, iend, maxSearches, mls, matches, minMatch); /* first search (depth 0) */ + + ZSTD_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-base), match_num, last_pos); + if (!last_pos && !match_num) { ip++; continue; } + + { U32 i; for (i=0; i<ZSTD_REP_INIT; i++) opt[0].rep[i] = rep[i]; } + opt[0].mlen = 1; + + if (match_num && (matches[match_num-1].len > sufficient_len || matches[match_num-1].len >= ZSTD_OPT_NUM)) { + best_mlen = matches[match_num-1].len; + best_off = matches[match_num-1].off; + cur = 0; + last_pos = 1; + goto _storeSequence; + } + + best_mlen = (last_pos) ? last_pos : minMatch; + + // set prices using matches at position = 0 + for (u = 0; u < match_num; u++) { + mlen = (u>0) ? matches[u-1].len+1 : best_mlen; + best_mlen = matches[u].len; + ZSTD_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-base), matches[u].len, matches[u].off, (int)best_mlen, (int)last_pos); + litlen = opt[0].litlen; + while (mlen <= best_mlen) { + price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off, mlen - MINMATCH); + if (mlen > last_pos || price < opt[mlen].price) + SET_PRICE(mlen, mlen, matches[u].off, litlen, price); + mlen++; + } } + + if (last_pos < minMatch) { + // ip += ((ip-anchor) >> g_searchStrength) + 1; /* jump faster over incompressible sections */ + ip++; continue; + } + + /* check further positions */ + for (cur = 1; cur <= last_pos; cur++) { + inr = ip + cur; + + if (opt[cur-1].mlen == 1) { + litlen = opt[cur-1].litlen + 1; + if (cur > litlen) { + price = opt[cur - litlen].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-litlen); + } else + price = ZSTD_getLiteralPrice(seqStorePtr, litlen, anchor); + } else { + litlen = 1; + price = opt[cur - 1].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-1); + } + + if (cur > last_pos || price <= opt[cur].price) // || ((price == opt[cur].price) && (opt[cur-1].mlen == 1) && (cur != litlen))) + SET_PRICE(cur, 1, 0, litlen, price); + + if (cur == last_pos) break; + + if (inr > ilimit) /* last match must start at a minimum distance of 8 from oend */ + continue; + + mlen = opt[cur].mlen; + if (opt[cur].off >= ZSTD_REP_NUM) { + opt[cur].rep[2] = opt[cur-mlen].rep[1]; + opt[cur].rep[1] = opt[cur-mlen].rep[0]; + opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE; + ZSTD_LOG_ENCODE("%d: COPYREP_OFF cur=%d mlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep[0], opt[cur].rep[1]); + } else { + opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur-mlen].rep[1] : opt[cur-mlen].rep[2]; + opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur-mlen].rep[0] : opt[cur-mlen].rep[1]; + opt[cur].rep[0] = opt[cur-mlen].rep[opt[cur].off]; + ZSTD_LOG_ENCODE("%d: COPYREP_NOR cur=%d mlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep[0], opt[cur].rep[1]); + } + + ZSTD_LOG_PARSER("%d: CURRENT_Ext price[%d/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep[0], opt[cur].rep[1]); + best_mlen = 0; + + { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) { + const U32 repIndex = (U32)(current+cur - opt[cur].rep[i]); + const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; + const BYTE* const repMatch = repBase + repIndex; + if ( ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow */ + && (MEM_readMINMATCH(inr, minMatch) == MEM_readMINMATCH(repMatch, minMatch)) ) { + /* repcode detected */ + const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; + mlen = (U32)ZSTD_count_2segments(inr+minMatch, repMatch+minMatch, iend, repEnd, prefixStart) + minMatch; + ZSTD_LOG_PARSER("%d: Found REP %d/%d mlen=%d off=%d rep=%d opt[%d].off=%d\n", (int)(inr-base), i, ZSTD_REP_NUM, mlen, i, opt[cur].rep[i], cur, opt[cur].off); + + if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) { + ZSTD_LOG_PARSER("%d: REP sufficient_len=%d best_mlen=%d best_off=%d last_pos=%d\n", (int)(inr-base), sufficient_len, best_mlen, best_off, last_pos); + best_mlen = mlen; best_off = i; last_pos = cur + 1; + goto _storeSequence; + } + + best_off = (i<=1 && opt[cur].mlen != 1) ? 1-i : i; + if (opt[cur].mlen == 1) { + litlen = opt[cur].litlen; + if (cur > litlen) { + price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, best_off, mlen - MINMATCH); + } else + price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH); + } else { + litlen = 0; + price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH); + } + + best_mlen = mlen; + ZSTD_LOG_PARSER("%d: Found REP mlen=%d off=%d price=%d litlen=%d\n", (int)(inr-base), mlen, best_off, price, litlen); + + do { + if (cur + mlen > last_pos || price <= opt[cur + mlen].price) + SET_PRICE(cur + mlen, mlen, i, litlen, price); + mlen--; + } while (mlen >= minMatch); + } } } + + match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, inr, iend, maxSearches, mls, matches, minMatch); + ZSTD_LOG_PARSER("%d: ZSTD_GetAllMatches match_num=%d\n", (int)(inr-base), match_num); + + if (match_num > 0 && matches[match_num-1].len > sufficient_len) { + best_mlen = matches[match_num-1].len; + best_off = matches[match_num-1].off; + last_pos = cur + 1; + goto _storeSequence; + } + + best_mlen = (best_mlen > minMatch) ? best_mlen : minMatch; + + /* set prices using matches at position = cur */ + for (u = 0; u < match_num; u++) { + mlen = (u>0) ? matches[u-1].len+1 : best_mlen; + best_mlen = (cur + matches[u].len < ZSTD_OPT_NUM) ? matches[u].len : ZSTD_OPT_NUM - cur; + + // ZSTD_LOG_PARSER("%d: Found1 cur=%d mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(inr-base), cur, matches[u].len, matches[u].off, best_mlen, last_pos); + while (mlen <= best_mlen) { + if (opt[cur].mlen == 1) { + litlen = opt[cur].litlen; + if (cur > litlen) + price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off, mlen - MINMATCH); + else + price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off, mlen - MINMATCH); + } else { + litlen = 0; + price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off, mlen - MINMATCH); + } + + // ZSTD_LOG_PARSER("%d: Found2 mlen=%d best_mlen=%d off=%d price=%d litlen=%d\n", (int)(inr-base), mlen, best_mlen, matches[u].off, price, litlen); + if (cur + mlen > last_pos || (price < opt[cur + mlen].price)) + SET_PRICE(cur + mlen, mlen, matches[u].off, litlen, price); + + mlen++; + } } } /* for (cur = 1; cur <= last_pos; cur++) */ + + best_mlen = opt[last_pos].mlen; + best_off = opt[last_pos].off; + cur = last_pos - best_mlen; + + /* store sequence */ +_storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ + for (u = 1; u <= last_pos; u++) + ZSTD_LOG_PARSER("%d: price[%u/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base+u), u, last_pos, opt[u].price, opt[u].off, opt[u].mlen, opt[u].litlen, opt[u].rep[0], opt[u].rep[1]); + ZSTD_LOG_PARSER("%d: cur=%d/%d best_mlen=%d best_off=%d rep[0]=%d\n", (int)(ip-base+cur), (int)cur, (int)last_pos, (int)best_mlen, (int)best_off, opt[cur].rep[0]); + + opt[0].mlen = 1; + + while (1) { + mlen = opt[cur].mlen; + offset = opt[cur].off; + opt[cur].mlen = best_mlen; + opt[cur].off = best_off; + best_mlen = mlen; + best_off = offset; + if (mlen > cur) break; + cur -= mlen; + } + + for (u = 0; u <= last_pos; ) { + ZSTD_LOG_PARSER("%d: price2[%d/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base+u), u, last_pos, opt[u].price, opt[u].off, opt[u].mlen, opt[u].litlen, opt[u].rep[0], opt[u].rep[1]); + u += opt[u].mlen; + } + + for (cur=0; cur < last_pos; ) { + ZSTD_LOG_PARSER("%d: price3[%d/%d]=%d off=%d mlen=%d litlen=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base+cur), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep[0], opt[cur].rep[1]); + mlen = opt[cur].mlen; + if (mlen == 1) { ip++; cur++; continue; } + offset = opt[cur].off; + cur += mlen; + litLength = (U32)(ip - anchor); + // ZSTD_LOG_ENCODE("%d/%d: ENCODE1 literals=%d mlen=%d off=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep[0], (int)rep[1]); + + if (offset >= ZSTD_REP_NUM) { + rep[2] = rep[1]; + rep[1] = rep[0]; + rep[0] = offset - ZSTD_REP_MOVE; + } else { + if (offset != 0) { + best_off = rep[offset]; + if (offset != 1) rep[2] = rep[1]; + rep[1] = rep[0]; + rep[0] = best_off; + } + if (litLength == 0 && offset<=1) offset = 1-offset; + } + + ZSTD_LOG_ENCODE("%d/%d: ENCODE literals=%d mlen=%d off=%d rep[0]=%d rep[1]=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep[0], (int)rep[1]); + +#if ZSTD_OPT_DEBUG >= 5 + U32 ml2; + if (offset >= ZSTD_REP_NUM) { + best_off = offset - ZSTD_REP_MOVE; + if (best_off > (size_t)(ip - prefixStart)) { + const BYTE* match = dictEnd - (best_off - (ip - prefixStart)); + ml2 = ZSTD_count_2segments(ip, match, iend, dictEnd, prefixStart); + ZSTD_LOG_PARSER("%d: ZSTD_count_2segments=%d offset=%d dictBase=%p dictEnd=%p prefixStart=%p ip=%p match=%p\n", (int)current, (int)ml2, (int)best_off, dictBase, dictEnd, prefixStart, ip, match); + } + else ml2 = (U32)ZSTD_count(ip, ip-offset, iend); + } + else ml2 = (U32)ZSTD_count(ip, ip-rep[0], iend); + if ((offset >= 8) && (ml2 < mlen || ml2 < minMatch)) { + printf("%d: ERROR_Ext iend=%d mlen=%d offset=%d ml2=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset, (int)ml2); exit(0); } + if (ip < anchor) { + printf("%d: ERROR_Ext ip < anchor iend=%d mlen=%d offset=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset); exit(0); } + if (ip + mlen > iend) { + printf("%d: ERROR_Ext ip + mlen >= iend iend=%d mlen=%d offset=%d\n", (int)(ip - base), (int)(iend - ip), (int)mlen, (int)offset); exit(0); } +#endif + + ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH); + ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH); + anchor = ip = ip + mlen; + } } /* for (cur=0; cur < last_pos; ) */ + + { /* Last Literals */ + size_t lastLLSize = iend - anchor; + ZSTD_LOG_ENCODE("%d: lastLLSize literals=%u\n", (int)(ip-base), (U32)(lastLLSize)); + memcpy(seqStorePtr->lit, anchor, lastLLSize); + seqStorePtr->lit += lastLLSize; + } +} diff --git a/contrib/libs/zstd06/decompress/huf_decompress.c b/contrib/libs/zstd06/decompress/huf_decompress.c index 889b8e34a3..f6e6ec6c9a 100644 --- a/contrib/libs/zstd06/decompress/huf_decompress.c +++ b/contrib/libs/zstd06/decompress/huf_decompress.c @@ -1,758 +1,758 @@ -/* ****************************************************************** - Huffman decoder, part of New Generation Entropy library - Copyright (C) 2013-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy - - Public forum : https://groups.google.com/forum/#!forum/lz4c -****************************************************************** */ - -/* ************************************************************** -* Compiler specifics -****************************************************************/ -#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -/* inline is defined */ -#elif defined(_MSC_VER) -# define inline __inline -#else -# define inline /* disable inline */ -#endif - - -#ifdef _MSC_VER /* Visual Studio */ -# define FORCE_INLINE static __forceinline -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -#else -# ifdef __GNUC__ -# define FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define FORCE_INLINE static inline -# endif -#endif - - -/* ************************************************************** -* Includes -****************************************************************/ -#include <stdlib.h> /* malloc, free, qsort */ -#include <string.h> /* memcpy, memset */ -#include <stdio.h> /* printf (debug) */ -#include "huf_static.h" -#include "bitstream.h" -#include "fse.h" /* header compression */ - - - -/* ************************************************************** -* Error Management -****************************************************************/ -#define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ - - - -/* ******************************************************* -* HUF : Huffman block decompression -*********************************************************/ -typedef struct { BYTE byte; BYTE nbBits; } HUF_DEltX2; /* single-symbol decoding */ - -typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUF_DEltX4; /* double-symbols decoding */ - -typedef struct { BYTE symbol; BYTE weight; } sortedSymbol_t; - - - -/*-***************************/ -/* single-symbol decoding */ -/*-***************************/ - -size_t HUF_readDTableX2 (U16* DTable, const void* src, size_t srcSize) -{ - BYTE huffWeight[HUF_MAX_SYMBOL_VALUE + 1]; - U32 rankVal[HUF_ABSOLUTEMAX_TABLELOG + 1]; /* large enough for values from 0 to 16 */ - U32 tableLog = 0; - size_t iSize; - U32 nbSymbols = 0; - U32 n; - U32 nextRankStart; - void* const dtPtr = DTable + 1; - HUF_DEltX2* const dt = (HUF_DEltX2*)dtPtr; - - HUF_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(U16)); /* if compilation fails here, assertion is false */ - //memset(huffWeight, 0, sizeof(huffWeight)); /* is not necessary, even though some analyzer complain ... */ - - iSize = HUF_readStats(huffWeight, HUF_MAX_SYMBOL_VALUE + 1, rankVal, &nbSymbols, &tableLog, src, srcSize); - if (HUF_isError(iSize)) return iSize; - - /* check result */ - if (tableLog > DTable[0]) return ERROR(tableLog_tooLarge); /* DTable is too small */ - DTable[0] = (U16)tableLog; /* maybe should separate sizeof allocated DTable, from used size of DTable, in case of re-use */ - - /* Prepare ranks */ - nextRankStart = 0; - for (n=1; n<tableLog+1; n++) { - U32 current = nextRankStart; - nextRankStart += (rankVal[n] << (n-1)); - rankVal[n] = current; - } - - /* fill DTable */ - for (n=0; n<nbSymbols; n++) { - const U32 w = huffWeight[n]; - const U32 length = (1 << w) >> 1; - U32 i; - HUF_DEltX2 D; - D.byte = (BYTE)n; D.nbBits = (BYTE)(tableLog + 1 - w); - for (i = rankVal[w]; i < rankVal[w] + length; i++) - dt[i] = D; - rankVal[w] += length; - } - - return iSize; -} - - -static BYTE HUF_decodeSymbolX2(BIT_DStream_t* Dstream, const HUF_DEltX2* dt, const U32 dtLog) -{ - const size_t val = BIT_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */ - const BYTE c = dt[val].byte; - BIT_skipBits(Dstream, dt[val].nbBits); - return c; -} - -#define HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) \ - *ptr++ = HUF_decodeSymbolX2(DStreamPtr, dt, dtLog) - -#define HUF_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \ - if (MEM_64bits() || (HUF_MAX_TABLELOG<=12)) \ - HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) - -#define HUF_DECODE_SYMBOLX2_2(ptr, DStreamPtr) \ - if (MEM_64bits()) \ - HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) - -static inline size_t HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, const HUF_DEltX2* const dt, const U32 dtLog) -{ - BYTE* const pStart = p; - - /* up to 4 symbols at a time */ - while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p <= pEnd-4)) { - HUF_DECODE_SYMBOLX2_2(p, bitDPtr); - HUF_DECODE_SYMBOLX2_1(p, bitDPtr); - HUF_DECODE_SYMBOLX2_2(p, bitDPtr); - HUF_DECODE_SYMBOLX2_0(p, bitDPtr); - } - - /* closer to the end */ - while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p < pEnd)) - HUF_DECODE_SYMBOLX2_0(p, bitDPtr); - - /* no more data to retrieve from bitstream, hence no need to reload */ - while (p < pEnd) - HUF_DECODE_SYMBOLX2_0(p, bitDPtr); - - return pEnd-pStart; -} - -size_t HUF_decompress1X2_usingDTable( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const U16* DTable) -{ - BYTE* op = (BYTE*)dst; - BYTE* const oend = op + dstSize; - const U32 dtLog = DTable[0]; - const void* dtPtr = DTable; - const HUF_DEltX2* const dt = ((const HUF_DEltX2*)dtPtr)+1; - BIT_DStream_t bitD; - - { size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize); - if (HUF_isError(errorCode)) return errorCode; } - - HUF_decodeStreamX2(op, &bitD, oend, dt, dtLog); - - /* check */ - if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); - - return dstSize; -} - -size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_MAX_TABLELOG); - const BYTE* ip = (const BYTE*) cSrc; - - size_t const errorCode = HUF_readDTableX2 (DTable, cSrc, cSrcSize); - if (HUF_isError(errorCode)) return errorCode; - if (errorCode >= cSrcSize) return ERROR(srcSize_wrong); - ip += errorCode; - cSrcSize -= errorCode; - - return HUF_decompress1X2_usingDTable (dst, dstSize, ip, cSrcSize, DTable); -} - - -size_t HUF_decompress4X2_usingDTable( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const U16* DTable) -{ - /* Check */ - if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ - - { const BYTE* const istart = (const BYTE*) cSrc; - BYTE* const ostart = (BYTE*) dst; - BYTE* const oend = ostart + dstSize; - const void* const dtPtr = DTable; - const HUF_DEltX2* const dt = ((const HUF_DEltX2*)dtPtr) +1; - const U32 dtLog = DTable[0]; - size_t errorCode; - - /* Init */ - BIT_DStream_t bitD1; - BIT_DStream_t bitD2; - BIT_DStream_t bitD3; - BIT_DStream_t bitD4; - const size_t length1 = MEM_readLE16(istart); - const size_t length2 = MEM_readLE16(istart+2); - const size_t length3 = MEM_readLE16(istart+4); - size_t length4; - const BYTE* const istart1 = istart + 6; /* jumpTable */ - const BYTE* const istart2 = istart1 + length1; - const BYTE* const istart3 = istart2 + length2; - const BYTE* const istart4 = istart3 + length3; - const size_t segmentSize = (dstSize+3) / 4; - BYTE* const opStart2 = ostart + segmentSize; - BYTE* const opStart3 = opStart2 + segmentSize; - BYTE* const opStart4 = opStart3 + segmentSize; - BYTE* op1 = ostart; - BYTE* op2 = opStart2; - BYTE* op3 = opStart3; - BYTE* op4 = opStart4; - U32 endSignal; - - length4 = cSrcSize - (length1 + length2 + length3 + 6); - if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ - errorCode = BIT_initDStream(&bitD1, istart1, length1); - if (HUF_isError(errorCode)) return errorCode; - errorCode = BIT_initDStream(&bitD2, istart2, length2); - if (HUF_isError(errorCode)) return errorCode; - errorCode = BIT_initDStream(&bitD3, istart3, length3); - if (HUF_isError(errorCode)) return errorCode; - errorCode = BIT_initDStream(&bitD4, istart4, length4); - if (HUF_isError(errorCode)) return errorCode; - - /* 16-32 symbols per loop (4-8 symbols per stream) */ - endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); - for ( ; (endSignal==BIT_DStream_unfinished) && (op4<(oend-7)) ; ) { - HUF_DECODE_SYMBOLX2_2(op1, &bitD1); - HUF_DECODE_SYMBOLX2_2(op2, &bitD2); - HUF_DECODE_SYMBOLX2_2(op3, &bitD3); - HUF_DECODE_SYMBOLX2_2(op4, &bitD4); - HUF_DECODE_SYMBOLX2_1(op1, &bitD1); - HUF_DECODE_SYMBOLX2_1(op2, &bitD2); - HUF_DECODE_SYMBOLX2_1(op3, &bitD3); - HUF_DECODE_SYMBOLX2_1(op4, &bitD4); - HUF_DECODE_SYMBOLX2_2(op1, &bitD1); - HUF_DECODE_SYMBOLX2_2(op2, &bitD2); - HUF_DECODE_SYMBOLX2_2(op3, &bitD3); - HUF_DECODE_SYMBOLX2_2(op4, &bitD4); - HUF_DECODE_SYMBOLX2_0(op1, &bitD1); - HUF_DECODE_SYMBOLX2_0(op2, &bitD2); - HUF_DECODE_SYMBOLX2_0(op3, &bitD3); - HUF_DECODE_SYMBOLX2_0(op4, &bitD4); - endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); - } - - /* check corruption */ - if (op1 > opStart2) return ERROR(corruption_detected); - if (op2 > opStart3) return ERROR(corruption_detected); - if (op3 > opStart4) return ERROR(corruption_detected); - /* note : op4 supposed already verified within main loop */ - - /* finish bitStreams one by one */ - HUF_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog); - HUF_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog); - HUF_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog); - HUF_decodeStreamX2(op4, &bitD4, oend, dt, dtLog); - - /* check */ - endSignal = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); - if (!endSignal) return ERROR(corruption_detected); - - /* decoded size */ - return dstSize; - } -} - - -size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_MAX_TABLELOG); - const BYTE* ip = (const BYTE*) cSrc; - - size_t const errorCode = HUF_readDTableX2 (DTable, cSrc, cSrcSize); - if (HUF_isError(errorCode)) return errorCode; - if (errorCode >= cSrcSize) return ERROR(srcSize_wrong); - ip += errorCode; - cSrcSize -= errorCode; - - return HUF_decompress4X2_usingDTable (dst, dstSize, ip, cSrcSize, DTable); -} - - -/* *************************/ -/* double-symbols decoding */ -/* *************************/ - -static void HUF_fillDTableX4Level2(HUF_DEltX4* DTable, U32 sizeLog, const U32 consumed, - const U32* rankValOrigin, const int minWeight, - const sortedSymbol_t* sortedSymbols, const U32 sortedListSize, - U32 nbBitsBaseline, U16 baseSeq) -{ - HUF_DEltX4 DElt; - U32 rankVal[HUF_ABSOLUTEMAX_TABLELOG + 1]; - - /* get pre-calculated rankVal */ - memcpy(rankVal, rankValOrigin, sizeof(rankVal)); - - /* fill skipped values */ - if (minWeight>1) { - U32 i, skipSize = rankVal[minWeight]; - MEM_writeLE16(&(DElt.sequence), baseSeq); - DElt.nbBits = (BYTE)(consumed); - DElt.length = 1; - for (i = 0; i < skipSize; i++) - DTable[i] = DElt; - } - - /* fill DTable */ - { U32 s; for (s=0; s<sortedListSize; s++) { /* note : sortedSymbols already skipped */ - const U32 symbol = sortedSymbols[s].symbol; - const U32 weight = sortedSymbols[s].weight; - const U32 nbBits = nbBitsBaseline - weight; - const U32 length = 1 << (sizeLog-nbBits); - const U32 start = rankVal[weight]; - U32 i = start; - const U32 end = start + length; - - MEM_writeLE16(&(DElt.sequence), (U16)(baseSeq + (symbol << 8))); - DElt.nbBits = (BYTE)(nbBits + consumed); - DElt.length = 2; - do { DTable[i++] = DElt; } while (i<end); /* since length >= 1 */ - - rankVal[weight] += length; - }} -} - -typedef U32 rankVal_t[HUF_ABSOLUTEMAX_TABLELOG][HUF_ABSOLUTEMAX_TABLELOG + 1]; - -static void HUF_fillDTableX4(HUF_DEltX4* DTable, const U32 targetLog, - const sortedSymbol_t* sortedList, const U32 sortedListSize, - const U32* rankStart, rankVal_t rankValOrigin, const U32 maxWeight, - const U32 nbBitsBaseline) -{ - U32 rankVal[HUF_ABSOLUTEMAX_TABLELOG + 1]; - const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */ - const U32 minBits = nbBitsBaseline - maxWeight; - U32 s; - - memcpy(rankVal, rankValOrigin, sizeof(rankVal)); - - /* fill DTable */ - for (s=0; s<sortedListSize; s++) { - const U16 symbol = sortedList[s].symbol; - const U32 weight = sortedList[s].weight; - const U32 nbBits = nbBitsBaseline - weight; - const U32 start = rankVal[weight]; - const U32 length = 1 << (targetLog-nbBits); - - if (targetLog-nbBits >= minBits) { /* enough room for a second symbol */ - U32 sortedRank; - int minWeight = nbBits + scaleLog; - if (minWeight < 1) minWeight = 1; - sortedRank = rankStart[minWeight]; - HUF_fillDTableX4Level2(DTable+start, targetLog-nbBits, nbBits, - rankValOrigin[nbBits], minWeight, - sortedList+sortedRank, sortedListSize-sortedRank, - nbBitsBaseline, symbol); - } else { - HUF_DEltX4 DElt; - MEM_writeLE16(&(DElt.sequence), symbol); - DElt.nbBits = (BYTE)(nbBits); - DElt.length = 1; - { U32 u; - const U32 end = start + length; - for (u = start; u < end; u++) DTable[u] = DElt; - } } - rankVal[weight] += length; - } -} - -size_t HUF_readDTableX4 (U32* DTable, const void* src, size_t srcSize) -{ - BYTE weightList[HUF_MAX_SYMBOL_VALUE + 1]; - sortedSymbol_t sortedSymbol[HUF_MAX_SYMBOL_VALUE + 1]; - U32 rankStats[HUF_ABSOLUTEMAX_TABLELOG + 1] = { 0 }; - U32 rankStart0[HUF_ABSOLUTEMAX_TABLELOG + 2] = { 0 }; - U32* const rankStart = rankStart0+1; - rankVal_t rankVal; - U32 tableLog, maxW, sizeOfSort, nbSymbols; - const U32 memLog = DTable[0]; - size_t iSize; - void* dtPtr = DTable; - HUF_DEltX4* const dt = ((HUF_DEltX4*)dtPtr) + 1; - - HUF_STATIC_ASSERT(sizeof(HUF_DEltX4) == sizeof(U32)); /* if compilation fails here, assertion is false */ - if (memLog > HUF_ABSOLUTEMAX_TABLELOG) return ERROR(tableLog_tooLarge); - //memset(weightList, 0, sizeof(weightList)); /* is not necessary, even though some analyzer complain ... */ - - iSize = HUF_readStats(weightList, HUF_MAX_SYMBOL_VALUE + 1, rankStats, &nbSymbols, &tableLog, src, srcSize); - if (HUF_isError(iSize)) return iSize; - - /* check result */ - if (tableLog > memLog) return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */ - - /* find maxWeight */ - for (maxW = tableLog; rankStats[maxW]==0; maxW--) {} /* necessarily finds a solution before 0 */ - - /* Get start index of each weight */ - { U32 w, nextRankStart = 0; - for (w=1; w<maxW+1; w++) { - U32 current = nextRankStart; - nextRankStart += rankStats[w]; - rankStart[w] = current; - } - rankStart[0] = nextRankStart; /* put all 0w symbols at the end of sorted list*/ - sizeOfSort = nextRankStart; - } - - /* sort symbols by weight */ - { U32 s; - for (s=0; s<nbSymbols; s++) { - U32 const w = weightList[s]; - U32 const r = rankStart[w]++; - sortedSymbol[r].symbol = (BYTE)s; - sortedSymbol[r].weight = (BYTE)w; - } - rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */ - } - - /* Build rankVal */ - { U32* const rankVal0 = rankVal[0]; - { int const rescale = (memLog-tableLog) - 1; /* tableLog <= memLog */ - U32 nextRankVal = 0; - U32 w; - for (w=1; w<maxW+1; w++) { - U32 current = nextRankVal; - nextRankVal += rankStats[w] << (w+rescale); - rankVal0[w] = current; - } } - { U32 const minBits = tableLog+1 - maxW; - U32 consumed; - for (consumed = minBits; consumed < memLog - minBits + 1; consumed++) { - U32* const rankValPtr = rankVal[consumed]; - U32 w; - for (w = 1; w < maxW+1; w++) { - rankValPtr[w] = rankVal0[w] >> consumed; - } } } } - - HUF_fillDTableX4(dt, memLog, - sortedSymbol, sizeOfSort, - rankStart0, rankVal, maxW, - tableLog+1); - - return iSize; -} - - -static U32 HUF_decodeSymbolX4(void* op, BIT_DStream_t* DStream, const HUF_DEltX4* dt, const U32 dtLog) -{ - const size_t val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ - memcpy(op, dt+val, 2); - BIT_skipBits(DStream, dt[val].nbBits); - return dt[val].length; -} - -static U32 HUF_decodeLastSymbolX4(void* op, BIT_DStream_t* DStream, const HUF_DEltX4* dt, const U32 dtLog) -{ - const size_t val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ - memcpy(op, dt+val, 1); - if (dt[val].length==1) BIT_skipBits(DStream, dt[val].nbBits); - else { - if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8)) { - BIT_skipBits(DStream, dt[val].nbBits); - if (DStream->bitsConsumed > (sizeof(DStream->bitContainer)*8)) - DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8); /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */ - } } - return 1; -} - - -#define HUF_DECODE_SYMBOLX4_0(ptr, DStreamPtr) \ - ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog) - -#define HUF_DECODE_SYMBOLX4_1(ptr, DStreamPtr) \ - if (MEM_64bits() || (HUF_MAX_TABLELOG<=12)) \ - ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog) - -#define HUF_DECODE_SYMBOLX4_2(ptr, DStreamPtr) \ - if (MEM_64bits()) \ - ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog) - -static inline size_t HUF_decodeStreamX4(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, const HUF_DEltX4* const dt, const U32 dtLog) -{ - BYTE* const pStart = p; - - /* up to 8 symbols at a time */ - while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p < pEnd-7)) { - HUF_DECODE_SYMBOLX4_2(p, bitDPtr); - HUF_DECODE_SYMBOLX4_1(p, bitDPtr); - HUF_DECODE_SYMBOLX4_2(p, bitDPtr); - HUF_DECODE_SYMBOLX4_0(p, bitDPtr); - } - - /* closer to the end */ - while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p <= pEnd-2)) - HUF_DECODE_SYMBOLX4_0(p, bitDPtr); - - while (p <= pEnd-2) - HUF_DECODE_SYMBOLX4_0(p, bitDPtr); /* no need to reload : reached the end of DStream */ - - if (p < pEnd) - p += HUF_decodeLastSymbolX4(p, bitDPtr, dt, dtLog); - - return p-pStart; -} - - -size_t HUF_decompress1X4_usingDTable( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const U32* DTable) -{ - const BYTE* const istart = (const BYTE*) cSrc; - BYTE* const ostart = (BYTE*) dst; - BYTE* const oend = ostart + dstSize; - - const U32 dtLog = DTable[0]; - const void* const dtPtr = DTable; - const HUF_DEltX4* const dt = ((const HUF_DEltX4*)dtPtr) +1; - - /* Init */ - BIT_DStream_t bitD; - { size_t const errorCode = BIT_initDStream(&bitD, istart, cSrcSize); - if (HUF_isError(errorCode)) return errorCode; } - - /* decode */ - HUF_decodeStreamX4(ostart, &bitD, oend, dt, dtLog); - - /* check */ - if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); - - /* decoded size */ - return dstSize; -} - -size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - HUF_CREATE_STATIC_DTABLEX4(DTable, HUF_MAX_TABLELOG); - const BYTE* ip = (const BYTE*) cSrc; - - size_t const hSize = HUF_readDTableX4 (DTable, cSrc, cSrcSize); - if (HUF_isError(hSize)) return hSize; - if (hSize >= cSrcSize) return ERROR(srcSize_wrong); - ip += hSize; - cSrcSize -= hSize; - - return HUF_decompress1X4_usingDTable (dst, dstSize, ip, cSrcSize, DTable); -} - -size_t HUF_decompress4X4_usingDTable( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const U32* DTable) -{ - if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ - - { const BYTE* const istart = (const BYTE*) cSrc; - BYTE* const ostart = (BYTE*) dst; - BYTE* const oend = ostart + dstSize; - const void* const dtPtr = DTable; - const HUF_DEltX4* const dt = ((const HUF_DEltX4*)dtPtr) +1; - const U32 dtLog = DTable[0]; - size_t errorCode; - - /* Init */ - BIT_DStream_t bitD1; - BIT_DStream_t bitD2; - BIT_DStream_t bitD3; - BIT_DStream_t bitD4; - const size_t length1 = MEM_readLE16(istart); - const size_t length2 = MEM_readLE16(istart+2); - const size_t length3 = MEM_readLE16(istart+4); - size_t length4; - const BYTE* const istart1 = istart + 6; /* jumpTable */ - const BYTE* const istart2 = istart1 + length1; - const BYTE* const istart3 = istart2 + length2; - const BYTE* const istart4 = istart3 + length3; - const size_t segmentSize = (dstSize+3) / 4; - BYTE* const opStart2 = ostart + segmentSize; - BYTE* const opStart3 = opStart2 + segmentSize; - BYTE* const opStart4 = opStart3 + segmentSize; - BYTE* op1 = ostart; - BYTE* op2 = opStart2; - BYTE* op3 = opStart3; - BYTE* op4 = opStart4; - U32 endSignal; - - length4 = cSrcSize - (length1 + length2 + length3 + 6); - if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ - errorCode = BIT_initDStream(&bitD1, istart1, length1); - if (HUF_isError(errorCode)) return errorCode; - errorCode = BIT_initDStream(&bitD2, istart2, length2); - if (HUF_isError(errorCode)) return errorCode; - errorCode = BIT_initDStream(&bitD3, istart3, length3); - if (HUF_isError(errorCode)) return errorCode; - errorCode = BIT_initDStream(&bitD4, istart4, length4); - if (HUF_isError(errorCode)) return errorCode; - - /* 16-32 symbols per loop (4-8 symbols per stream) */ - endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); - for ( ; (endSignal==BIT_DStream_unfinished) && (op4<(oend-7)) ; ) { - HUF_DECODE_SYMBOLX4_2(op1, &bitD1); - HUF_DECODE_SYMBOLX4_2(op2, &bitD2); - HUF_DECODE_SYMBOLX4_2(op3, &bitD3); - HUF_DECODE_SYMBOLX4_2(op4, &bitD4); - HUF_DECODE_SYMBOLX4_1(op1, &bitD1); - HUF_DECODE_SYMBOLX4_1(op2, &bitD2); - HUF_DECODE_SYMBOLX4_1(op3, &bitD3); - HUF_DECODE_SYMBOLX4_1(op4, &bitD4); - HUF_DECODE_SYMBOLX4_2(op1, &bitD1); - HUF_DECODE_SYMBOLX4_2(op2, &bitD2); - HUF_DECODE_SYMBOLX4_2(op3, &bitD3); - HUF_DECODE_SYMBOLX4_2(op4, &bitD4); - HUF_DECODE_SYMBOLX4_0(op1, &bitD1); - HUF_DECODE_SYMBOLX4_0(op2, &bitD2); - HUF_DECODE_SYMBOLX4_0(op3, &bitD3); - HUF_DECODE_SYMBOLX4_0(op4, &bitD4); - - endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); - } - - /* check corruption */ - if (op1 > opStart2) return ERROR(corruption_detected); - if (op2 > opStart3) return ERROR(corruption_detected); - if (op3 > opStart4) return ERROR(corruption_detected); - /* note : op4 supposed already verified within main loop */ - - /* finish bitStreams one by one */ - HUF_decodeStreamX4(op1, &bitD1, opStart2, dt, dtLog); - HUF_decodeStreamX4(op2, &bitD2, opStart3, dt, dtLog); - HUF_decodeStreamX4(op3, &bitD3, opStart4, dt, dtLog); - HUF_decodeStreamX4(op4, &bitD4, oend, dt, dtLog); - - /* check */ - endSignal = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); - if (!endSignal) return ERROR(corruption_detected); - - /* decoded size */ - return dstSize; - } -} - - -size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - HUF_CREATE_STATIC_DTABLEX4(DTable, HUF_MAX_TABLELOG); - const BYTE* ip = (const BYTE*) cSrc; - - size_t hSize = HUF_readDTableX4 (DTable, cSrc, cSrcSize); - if (HUF_isError(hSize)) return hSize; - if (hSize >= cSrcSize) return ERROR(srcSize_wrong); - ip += hSize; - cSrcSize -= hSize; - - return HUF_decompress4X4_usingDTable (dst, dstSize, ip, cSrcSize, DTable); -} - - - - -/* ********************************/ -/* Generic decompression selector */ -/* ********************************/ - -typedef struct { U32 tableTime; U32 decode256Time; } algo_time_t; -static const algo_time_t algoTime[16 /* Quantization */][3 /* single, double, quad */] = -{ - /* single, double, quad */ - {{0,0}, {1,1}, {2,2}}, /* Q==0 : impossible */ - {{0,0}, {1,1}, {2,2}}, /* Q==1 : impossible */ - {{ 38,130}, {1313, 74}, {2151, 38}}, /* Q == 2 : 12-18% */ - {{ 448,128}, {1353, 74}, {2238, 41}}, /* Q == 3 : 18-25% */ - {{ 556,128}, {1353, 74}, {2238, 47}}, /* Q == 4 : 25-32% */ - {{ 714,128}, {1418, 74}, {2436, 53}}, /* Q == 5 : 32-38% */ - {{ 883,128}, {1437, 74}, {2464, 61}}, /* Q == 6 : 38-44% */ - {{ 897,128}, {1515, 75}, {2622, 68}}, /* Q == 7 : 44-50% */ - {{ 926,128}, {1613, 75}, {2730, 75}}, /* Q == 8 : 50-56% */ - {{ 947,128}, {1729, 77}, {3359, 77}}, /* Q == 9 : 56-62% */ - {{1107,128}, {2083, 81}, {4006, 84}}, /* Q ==10 : 62-69% */ - {{1177,128}, {2379, 87}, {4785, 88}}, /* Q ==11 : 69-75% */ - {{1242,128}, {2415, 93}, {5155, 84}}, /* Q ==12 : 75-81% */ - {{1349,128}, {2644,106}, {5260,106}}, /* Q ==13 : 81-87% */ - {{1455,128}, {2422,124}, {4174,124}}, /* Q ==14 : 87-93% */ - {{ 722,128}, {1891,145}, {1936,146}}, /* Q ==15 : 93-99% */ -}; - -typedef size_t (*decompressionAlgo)(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); - -size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - static const decompressionAlgo decompress[2] = { HUF_decompress4X2, HUF_decompress4X4 }; - U32 Dtime[3]; /* decompression time estimation */ - - /* validation checks */ - if (dstSize == 0) return ERROR(dstSize_tooSmall); - if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ - if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ - if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ - - /* decoder timing evaluation */ - { U32 const Q = (U32)(cSrcSize * 16 / dstSize); /* Q < 16 since dstSize > cSrcSize */ - U32 const D256 = (U32)(dstSize >> 8); - U32 n; for (n=0; n<3; n++) - Dtime[n] = algoTime[Q][n].tableTime + (algoTime[Q][n].decode256Time * D256); - } - - Dtime[1] += Dtime[1] >> 4; Dtime[2] += Dtime[2] >> 3; /* advantage to algorithms using less memory, for cache eviction */ - - { U32 algoNb = 0; - if (Dtime[1] < Dtime[0]) algoNb = 1; - return decompress[algoNb](dst, dstSize, cSrc, cSrcSize); - } -} +/* ****************************************************************** + Huffman decoder, part of New Generation Entropy library + Copyright (C) 2013-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy + - Public forum : https://groups.google.com/forum/#!forum/lz4c +****************************************************************** */ + +/* ************************************************************** +* Compiler specifics +****************************************************************/ +#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +/* inline is defined */ +#elif defined(_MSC_VER) +# define inline __inline +#else +# define inline /* disable inline */ +#endif + + +#ifdef _MSC_VER /* Visual Studio */ +# define FORCE_INLINE static __forceinline +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#else +# ifdef __GNUC__ +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif +#endif + + +/* ************************************************************** +* Includes +****************************************************************/ +#include <stdlib.h> /* malloc, free, qsort */ +#include <string.h> /* memcpy, memset */ +#include <stdio.h> /* printf (debug) */ +#include "huf_static.h" +#include "bitstream.h" +#include "fse.h" /* header compression */ + + + +/* ************************************************************** +* Error Management +****************************************************************/ +#define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ + + + +/* ******************************************************* +* HUF : Huffman block decompression +*********************************************************/ +typedef struct { BYTE byte; BYTE nbBits; } HUF_DEltX2; /* single-symbol decoding */ + +typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUF_DEltX4; /* double-symbols decoding */ + +typedef struct { BYTE symbol; BYTE weight; } sortedSymbol_t; + + + +/*-***************************/ +/* single-symbol decoding */ +/*-***************************/ + +size_t HUF_readDTableX2 (U16* DTable, const void* src, size_t srcSize) +{ + BYTE huffWeight[HUF_MAX_SYMBOL_VALUE + 1]; + U32 rankVal[HUF_ABSOLUTEMAX_TABLELOG + 1]; /* large enough for values from 0 to 16 */ + U32 tableLog = 0; + size_t iSize; + U32 nbSymbols = 0; + U32 n; + U32 nextRankStart; + void* const dtPtr = DTable + 1; + HUF_DEltX2* const dt = (HUF_DEltX2*)dtPtr; + + HUF_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(U16)); /* if compilation fails here, assertion is false */ + //memset(huffWeight, 0, sizeof(huffWeight)); /* is not necessary, even though some analyzer complain ... */ + + iSize = HUF_readStats(huffWeight, HUF_MAX_SYMBOL_VALUE + 1, rankVal, &nbSymbols, &tableLog, src, srcSize); + if (HUF_isError(iSize)) return iSize; + + /* check result */ + if (tableLog > DTable[0]) return ERROR(tableLog_tooLarge); /* DTable is too small */ + DTable[0] = (U16)tableLog; /* maybe should separate sizeof allocated DTable, from used size of DTable, in case of re-use */ + + /* Prepare ranks */ + nextRankStart = 0; + for (n=1; n<tableLog+1; n++) { + U32 current = nextRankStart; + nextRankStart += (rankVal[n] << (n-1)); + rankVal[n] = current; + } + + /* fill DTable */ + for (n=0; n<nbSymbols; n++) { + const U32 w = huffWeight[n]; + const U32 length = (1 << w) >> 1; + U32 i; + HUF_DEltX2 D; + D.byte = (BYTE)n; D.nbBits = (BYTE)(tableLog + 1 - w); + for (i = rankVal[w]; i < rankVal[w] + length; i++) + dt[i] = D; + rankVal[w] += length; + } + + return iSize; +} + + +static BYTE HUF_decodeSymbolX2(BIT_DStream_t* Dstream, const HUF_DEltX2* dt, const U32 dtLog) +{ + const size_t val = BIT_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */ + const BYTE c = dt[val].byte; + BIT_skipBits(Dstream, dt[val].nbBits); + return c; +} + +#define HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) \ + *ptr++ = HUF_decodeSymbolX2(DStreamPtr, dt, dtLog) + +#define HUF_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \ + if (MEM_64bits() || (HUF_MAX_TABLELOG<=12)) \ + HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) + +#define HUF_DECODE_SYMBOLX2_2(ptr, DStreamPtr) \ + if (MEM_64bits()) \ + HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) + +static inline size_t HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, const HUF_DEltX2* const dt, const U32 dtLog) +{ + BYTE* const pStart = p; + + /* up to 4 symbols at a time */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p <= pEnd-4)) { + HUF_DECODE_SYMBOLX2_2(p, bitDPtr); + HUF_DECODE_SYMBOLX2_1(p, bitDPtr); + HUF_DECODE_SYMBOLX2_2(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + } + + /* closer to the end */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p < pEnd)) + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + + /* no more data to retrieve from bitstream, hence no need to reload */ + while (p < pEnd) + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + + return pEnd-pStart; +} + +size_t HUF_decompress1X2_usingDTable( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const U16* DTable) +{ + BYTE* op = (BYTE*)dst; + BYTE* const oend = op + dstSize; + const U32 dtLog = DTable[0]; + const void* dtPtr = DTable; + const HUF_DEltX2* const dt = ((const HUF_DEltX2*)dtPtr)+1; + BIT_DStream_t bitD; + + { size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize); + if (HUF_isError(errorCode)) return errorCode; } + + HUF_decodeStreamX2(op, &bitD, oend, dt, dtLog); + + /* check */ + if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); + + return dstSize; +} + +size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_MAX_TABLELOG); + const BYTE* ip = (const BYTE*) cSrc; + + size_t const errorCode = HUF_readDTableX2 (DTable, cSrc, cSrcSize); + if (HUF_isError(errorCode)) return errorCode; + if (errorCode >= cSrcSize) return ERROR(srcSize_wrong); + ip += errorCode; + cSrcSize -= errorCode; + + return HUF_decompress1X2_usingDTable (dst, dstSize, ip, cSrcSize, DTable); +} + + +size_t HUF_decompress4X2_usingDTable( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const U16* DTable) +{ + /* Check */ + if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ + + { const BYTE* const istart = (const BYTE*) cSrc; + BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + const void* const dtPtr = DTable; + const HUF_DEltX2* const dt = ((const HUF_DEltX2*)dtPtr) +1; + const U32 dtLog = DTable[0]; + size_t errorCode; + + /* Init */ + BIT_DStream_t bitD1; + BIT_DStream_t bitD2; + BIT_DStream_t bitD3; + BIT_DStream_t bitD4; + const size_t length1 = MEM_readLE16(istart); + const size_t length2 = MEM_readLE16(istart+2); + const size_t length3 = MEM_readLE16(istart+4); + size_t length4; + const BYTE* const istart1 = istart + 6; /* jumpTable */ + const BYTE* const istart2 = istart1 + length1; + const BYTE* const istart3 = istart2 + length2; + const BYTE* const istart4 = istart3 + length3; + const size_t segmentSize = (dstSize+3) / 4; + BYTE* const opStart2 = ostart + segmentSize; + BYTE* const opStart3 = opStart2 + segmentSize; + BYTE* const opStart4 = opStart3 + segmentSize; + BYTE* op1 = ostart; + BYTE* op2 = opStart2; + BYTE* op3 = opStart3; + BYTE* op4 = opStart4; + U32 endSignal; + + length4 = cSrcSize - (length1 + length2 + length3 + 6); + if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ + errorCode = BIT_initDStream(&bitD1, istart1, length1); + if (HUF_isError(errorCode)) return errorCode; + errorCode = BIT_initDStream(&bitD2, istart2, length2); + if (HUF_isError(errorCode)) return errorCode; + errorCode = BIT_initDStream(&bitD3, istart3, length3); + if (HUF_isError(errorCode)) return errorCode; + errorCode = BIT_initDStream(&bitD4, istart4, length4); + if (HUF_isError(errorCode)) return errorCode; + + /* 16-32 symbols per loop (4-8 symbols per stream) */ + endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); + for ( ; (endSignal==BIT_DStream_unfinished) && (op4<(oend-7)) ; ) { + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_1(op1, &bitD1); + HUF_DECODE_SYMBOLX2_1(op2, &bitD2); + HUF_DECODE_SYMBOLX2_1(op3, &bitD3); + HUF_DECODE_SYMBOLX2_1(op4, &bitD4); + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_0(op1, &bitD1); + HUF_DECODE_SYMBOLX2_0(op2, &bitD2); + HUF_DECODE_SYMBOLX2_0(op3, &bitD3); + HUF_DECODE_SYMBOLX2_0(op4, &bitD4); + endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); + } + + /* check corruption */ + if (op1 > opStart2) return ERROR(corruption_detected); + if (op2 > opStart3) return ERROR(corruption_detected); + if (op3 > opStart4) return ERROR(corruption_detected); + /* note : op4 supposed already verified within main loop */ + + /* finish bitStreams one by one */ + HUF_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog); + HUF_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog); + HUF_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog); + HUF_decodeStreamX2(op4, &bitD4, oend, dt, dtLog); + + /* check */ + endSignal = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); + if (!endSignal) return ERROR(corruption_detected); + + /* decoded size */ + return dstSize; + } +} + + +size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_MAX_TABLELOG); + const BYTE* ip = (const BYTE*) cSrc; + + size_t const errorCode = HUF_readDTableX2 (DTable, cSrc, cSrcSize); + if (HUF_isError(errorCode)) return errorCode; + if (errorCode >= cSrcSize) return ERROR(srcSize_wrong); + ip += errorCode; + cSrcSize -= errorCode; + + return HUF_decompress4X2_usingDTable (dst, dstSize, ip, cSrcSize, DTable); +} + + +/* *************************/ +/* double-symbols decoding */ +/* *************************/ + +static void HUF_fillDTableX4Level2(HUF_DEltX4* DTable, U32 sizeLog, const U32 consumed, + const U32* rankValOrigin, const int minWeight, + const sortedSymbol_t* sortedSymbols, const U32 sortedListSize, + U32 nbBitsBaseline, U16 baseSeq) +{ + HUF_DEltX4 DElt; + U32 rankVal[HUF_ABSOLUTEMAX_TABLELOG + 1]; + + /* get pre-calculated rankVal */ + memcpy(rankVal, rankValOrigin, sizeof(rankVal)); + + /* fill skipped values */ + if (minWeight>1) { + U32 i, skipSize = rankVal[minWeight]; + MEM_writeLE16(&(DElt.sequence), baseSeq); + DElt.nbBits = (BYTE)(consumed); + DElt.length = 1; + for (i = 0; i < skipSize; i++) + DTable[i] = DElt; + } + + /* fill DTable */ + { U32 s; for (s=0; s<sortedListSize; s++) { /* note : sortedSymbols already skipped */ + const U32 symbol = sortedSymbols[s].symbol; + const U32 weight = sortedSymbols[s].weight; + const U32 nbBits = nbBitsBaseline - weight; + const U32 length = 1 << (sizeLog-nbBits); + const U32 start = rankVal[weight]; + U32 i = start; + const U32 end = start + length; + + MEM_writeLE16(&(DElt.sequence), (U16)(baseSeq + (symbol << 8))); + DElt.nbBits = (BYTE)(nbBits + consumed); + DElt.length = 2; + do { DTable[i++] = DElt; } while (i<end); /* since length >= 1 */ + + rankVal[weight] += length; + }} +} + +typedef U32 rankVal_t[HUF_ABSOLUTEMAX_TABLELOG][HUF_ABSOLUTEMAX_TABLELOG + 1]; + +static void HUF_fillDTableX4(HUF_DEltX4* DTable, const U32 targetLog, + const sortedSymbol_t* sortedList, const U32 sortedListSize, + const U32* rankStart, rankVal_t rankValOrigin, const U32 maxWeight, + const U32 nbBitsBaseline) +{ + U32 rankVal[HUF_ABSOLUTEMAX_TABLELOG + 1]; + const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */ + const U32 minBits = nbBitsBaseline - maxWeight; + U32 s; + + memcpy(rankVal, rankValOrigin, sizeof(rankVal)); + + /* fill DTable */ + for (s=0; s<sortedListSize; s++) { + const U16 symbol = sortedList[s].symbol; + const U32 weight = sortedList[s].weight; + const U32 nbBits = nbBitsBaseline - weight; + const U32 start = rankVal[weight]; + const U32 length = 1 << (targetLog-nbBits); + + if (targetLog-nbBits >= minBits) { /* enough room for a second symbol */ + U32 sortedRank; + int minWeight = nbBits + scaleLog; + if (minWeight < 1) minWeight = 1; + sortedRank = rankStart[minWeight]; + HUF_fillDTableX4Level2(DTable+start, targetLog-nbBits, nbBits, + rankValOrigin[nbBits], minWeight, + sortedList+sortedRank, sortedListSize-sortedRank, + nbBitsBaseline, symbol); + } else { + HUF_DEltX4 DElt; + MEM_writeLE16(&(DElt.sequence), symbol); + DElt.nbBits = (BYTE)(nbBits); + DElt.length = 1; + { U32 u; + const U32 end = start + length; + for (u = start; u < end; u++) DTable[u] = DElt; + } } + rankVal[weight] += length; + } +} + +size_t HUF_readDTableX4 (U32* DTable, const void* src, size_t srcSize) +{ + BYTE weightList[HUF_MAX_SYMBOL_VALUE + 1]; + sortedSymbol_t sortedSymbol[HUF_MAX_SYMBOL_VALUE + 1]; + U32 rankStats[HUF_ABSOLUTEMAX_TABLELOG + 1] = { 0 }; + U32 rankStart0[HUF_ABSOLUTEMAX_TABLELOG + 2] = { 0 }; + U32* const rankStart = rankStart0+1; + rankVal_t rankVal; + U32 tableLog, maxW, sizeOfSort, nbSymbols; + const U32 memLog = DTable[0]; + size_t iSize; + void* dtPtr = DTable; + HUF_DEltX4* const dt = ((HUF_DEltX4*)dtPtr) + 1; + + HUF_STATIC_ASSERT(sizeof(HUF_DEltX4) == sizeof(U32)); /* if compilation fails here, assertion is false */ + if (memLog > HUF_ABSOLUTEMAX_TABLELOG) return ERROR(tableLog_tooLarge); + //memset(weightList, 0, sizeof(weightList)); /* is not necessary, even though some analyzer complain ... */ + + iSize = HUF_readStats(weightList, HUF_MAX_SYMBOL_VALUE + 1, rankStats, &nbSymbols, &tableLog, src, srcSize); + if (HUF_isError(iSize)) return iSize; + + /* check result */ + if (tableLog > memLog) return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */ + + /* find maxWeight */ + for (maxW = tableLog; rankStats[maxW]==0; maxW--) {} /* necessarily finds a solution before 0 */ + + /* Get start index of each weight */ + { U32 w, nextRankStart = 0; + for (w=1; w<maxW+1; w++) { + U32 current = nextRankStart; + nextRankStart += rankStats[w]; + rankStart[w] = current; + } + rankStart[0] = nextRankStart; /* put all 0w symbols at the end of sorted list*/ + sizeOfSort = nextRankStart; + } + + /* sort symbols by weight */ + { U32 s; + for (s=0; s<nbSymbols; s++) { + U32 const w = weightList[s]; + U32 const r = rankStart[w]++; + sortedSymbol[r].symbol = (BYTE)s; + sortedSymbol[r].weight = (BYTE)w; + } + rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */ + } + + /* Build rankVal */ + { U32* const rankVal0 = rankVal[0]; + { int const rescale = (memLog-tableLog) - 1; /* tableLog <= memLog */ + U32 nextRankVal = 0; + U32 w; + for (w=1; w<maxW+1; w++) { + U32 current = nextRankVal; + nextRankVal += rankStats[w] << (w+rescale); + rankVal0[w] = current; + } } + { U32 const minBits = tableLog+1 - maxW; + U32 consumed; + for (consumed = minBits; consumed < memLog - minBits + 1; consumed++) { + U32* const rankValPtr = rankVal[consumed]; + U32 w; + for (w = 1; w < maxW+1; w++) { + rankValPtr[w] = rankVal0[w] >> consumed; + } } } } + + HUF_fillDTableX4(dt, memLog, + sortedSymbol, sizeOfSort, + rankStart0, rankVal, maxW, + tableLog+1); + + return iSize; +} + + +static U32 HUF_decodeSymbolX4(void* op, BIT_DStream_t* DStream, const HUF_DEltX4* dt, const U32 dtLog) +{ + const size_t val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ + memcpy(op, dt+val, 2); + BIT_skipBits(DStream, dt[val].nbBits); + return dt[val].length; +} + +static U32 HUF_decodeLastSymbolX4(void* op, BIT_DStream_t* DStream, const HUF_DEltX4* dt, const U32 dtLog) +{ + const size_t val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ + memcpy(op, dt+val, 1); + if (dt[val].length==1) BIT_skipBits(DStream, dt[val].nbBits); + else { + if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8)) { + BIT_skipBits(DStream, dt[val].nbBits); + if (DStream->bitsConsumed > (sizeof(DStream->bitContainer)*8)) + DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8); /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */ + } } + return 1; +} + + +#define HUF_DECODE_SYMBOLX4_0(ptr, DStreamPtr) \ + ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog) + +#define HUF_DECODE_SYMBOLX4_1(ptr, DStreamPtr) \ + if (MEM_64bits() || (HUF_MAX_TABLELOG<=12)) \ + ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog) + +#define HUF_DECODE_SYMBOLX4_2(ptr, DStreamPtr) \ + if (MEM_64bits()) \ + ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog) + +static inline size_t HUF_decodeStreamX4(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, const HUF_DEltX4* const dt, const U32 dtLog) +{ + BYTE* const pStart = p; + + /* up to 8 symbols at a time */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p < pEnd-7)) { + HUF_DECODE_SYMBOLX4_2(p, bitDPtr); + HUF_DECODE_SYMBOLX4_1(p, bitDPtr); + HUF_DECODE_SYMBOLX4_2(p, bitDPtr); + HUF_DECODE_SYMBOLX4_0(p, bitDPtr); + } + + /* closer to the end */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p <= pEnd-2)) + HUF_DECODE_SYMBOLX4_0(p, bitDPtr); + + while (p <= pEnd-2) + HUF_DECODE_SYMBOLX4_0(p, bitDPtr); /* no need to reload : reached the end of DStream */ + + if (p < pEnd) + p += HUF_decodeLastSymbolX4(p, bitDPtr, dt, dtLog); + + return p-pStart; +} + + +size_t HUF_decompress1X4_usingDTable( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const U32* DTable) +{ + const BYTE* const istart = (const BYTE*) cSrc; + BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + + const U32 dtLog = DTable[0]; + const void* const dtPtr = DTable; + const HUF_DEltX4* const dt = ((const HUF_DEltX4*)dtPtr) +1; + + /* Init */ + BIT_DStream_t bitD; + { size_t const errorCode = BIT_initDStream(&bitD, istart, cSrcSize); + if (HUF_isError(errorCode)) return errorCode; } + + /* decode */ + HUF_decodeStreamX4(ostart, &bitD, oend, dt, dtLog); + + /* check */ + if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); + + /* decoded size */ + return dstSize; +} + +size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + HUF_CREATE_STATIC_DTABLEX4(DTable, HUF_MAX_TABLELOG); + const BYTE* ip = (const BYTE*) cSrc; + + size_t const hSize = HUF_readDTableX4 (DTable, cSrc, cSrcSize); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; + cSrcSize -= hSize; + + return HUF_decompress1X4_usingDTable (dst, dstSize, ip, cSrcSize, DTable); +} + +size_t HUF_decompress4X4_usingDTable( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const U32* DTable) +{ + if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ + + { const BYTE* const istart = (const BYTE*) cSrc; + BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + const void* const dtPtr = DTable; + const HUF_DEltX4* const dt = ((const HUF_DEltX4*)dtPtr) +1; + const U32 dtLog = DTable[0]; + size_t errorCode; + + /* Init */ + BIT_DStream_t bitD1; + BIT_DStream_t bitD2; + BIT_DStream_t bitD3; + BIT_DStream_t bitD4; + const size_t length1 = MEM_readLE16(istart); + const size_t length2 = MEM_readLE16(istart+2); + const size_t length3 = MEM_readLE16(istart+4); + size_t length4; + const BYTE* const istart1 = istart + 6; /* jumpTable */ + const BYTE* const istart2 = istart1 + length1; + const BYTE* const istart3 = istart2 + length2; + const BYTE* const istart4 = istart3 + length3; + const size_t segmentSize = (dstSize+3) / 4; + BYTE* const opStart2 = ostart + segmentSize; + BYTE* const opStart3 = opStart2 + segmentSize; + BYTE* const opStart4 = opStart3 + segmentSize; + BYTE* op1 = ostart; + BYTE* op2 = opStart2; + BYTE* op3 = opStart3; + BYTE* op4 = opStart4; + U32 endSignal; + + length4 = cSrcSize - (length1 + length2 + length3 + 6); + if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ + errorCode = BIT_initDStream(&bitD1, istart1, length1); + if (HUF_isError(errorCode)) return errorCode; + errorCode = BIT_initDStream(&bitD2, istart2, length2); + if (HUF_isError(errorCode)) return errorCode; + errorCode = BIT_initDStream(&bitD3, istart3, length3); + if (HUF_isError(errorCode)) return errorCode; + errorCode = BIT_initDStream(&bitD4, istart4, length4); + if (HUF_isError(errorCode)) return errorCode; + + /* 16-32 symbols per loop (4-8 symbols per stream) */ + endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); + for ( ; (endSignal==BIT_DStream_unfinished) && (op4<(oend-7)) ; ) { + HUF_DECODE_SYMBOLX4_2(op1, &bitD1); + HUF_DECODE_SYMBOLX4_2(op2, &bitD2); + HUF_DECODE_SYMBOLX4_2(op3, &bitD3); + HUF_DECODE_SYMBOLX4_2(op4, &bitD4); + HUF_DECODE_SYMBOLX4_1(op1, &bitD1); + HUF_DECODE_SYMBOLX4_1(op2, &bitD2); + HUF_DECODE_SYMBOLX4_1(op3, &bitD3); + HUF_DECODE_SYMBOLX4_1(op4, &bitD4); + HUF_DECODE_SYMBOLX4_2(op1, &bitD1); + HUF_DECODE_SYMBOLX4_2(op2, &bitD2); + HUF_DECODE_SYMBOLX4_2(op3, &bitD3); + HUF_DECODE_SYMBOLX4_2(op4, &bitD4); + HUF_DECODE_SYMBOLX4_0(op1, &bitD1); + HUF_DECODE_SYMBOLX4_0(op2, &bitD2); + HUF_DECODE_SYMBOLX4_0(op3, &bitD3); + HUF_DECODE_SYMBOLX4_0(op4, &bitD4); + + endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); + } + + /* check corruption */ + if (op1 > opStart2) return ERROR(corruption_detected); + if (op2 > opStart3) return ERROR(corruption_detected); + if (op3 > opStart4) return ERROR(corruption_detected); + /* note : op4 supposed already verified within main loop */ + + /* finish bitStreams one by one */ + HUF_decodeStreamX4(op1, &bitD1, opStart2, dt, dtLog); + HUF_decodeStreamX4(op2, &bitD2, opStart3, dt, dtLog); + HUF_decodeStreamX4(op3, &bitD3, opStart4, dt, dtLog); + HUF_decodeStreamX4(op4, &bitD4, oend, dt, dtLog); + + /* check */ + endSignal = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); + if (!endSignal) return ERROR(corruption_detected); + + /* decoded size */ + return dstSize; + } +} + + +size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + HUF_CREATE_STATIC_DTABLEX4(DTable, HUF_MAX_TABLELOG); + const BYTE* ip = (const BYTE*) cSrc; + + size_t hSize = HUF_readDTableX4 (DTable, cSrc, cSrcSize); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; + cSrcSize -= hSize; + + return HUF_decompress4X4_usingDTable (dst, dstSize, ip, cSrcSize, DTable); +} + + + + +/* ********************************/ +/* Generic decompression selector */ +/* ********************************/ + +typedef struct { U32 tableTime; U32 decode256Time; } algo_time_t; +static const algo_time_t algoTime[16 /* Quantization */][3 /* single, double, quad */] = +{ + /* single, double, quad */ + {{0,0}, {1,1}, {2,2}}, /* Q==0 : impossible */ + {{0,0}, {1,1}, {2,2}}, /* Q==1 : impossible */ + {{ 38,130}, {1313, 74}, {2151, 38}}, /* Q == 2 : 12-18% */ + {{ 448,128}, {1353, 74}, {2238, 41}}, /* Q == 3 : 18-25% */ + {{ 556,128}, {1353, 74}, {2238, 47}}, /* Q == 4 : 25-32% */ + {{ 714,128}, {1418, 74}, {2436, 53}}, /* Q == 5 : 32-38% */ + {{ 883,128}, {1437, 74}, {2464, 61}}, /* Q == 6 : 38-44% */ + {{ 897,128}, {1515, 75}, {2622, 68}}, /* Q == 7 : 44-50% */ + {{ 926,128}, {1613, 75}, {2730, 75}}, /* Q == 8 : 50-56% */ + {{ 947,128}, {1729, 77}, {3359, 77}}, /* Q == 9 : 56-62% */ + {{1107,128}, {2083, 81}, {4006, 84}}, /* Q ==10 : 62-69% */ + {{1177,128}, {2379, 87}, {4785, 88}}, /* Q ==11 : 69-75% */ + {{1242,128}, {2415, 93}, {5155, 84}}, /* Q ==12 : 75-81% */ + {{1349,128}, {2644,106}, {5260,106}}, /* Q ==13 : 81-87% */ + {{1455,128}, {2422,124}, {4174,124}}, /* Q ==14 : 87-93% */ + {{ 722,128}, {1891,145}, {1936,146}}, /* Q ==15 : 93-99% */ +}; + +typedef size_t (*decompressionAlgo)(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); + +size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + static const decompressionAlgo decompress[2] = { HUF_decompress4X2, HUF_decompress4X4 }; + U32 Dtime[3]; /* decompression time estimation */ + + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ + if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ + if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ + + /* decoder timing evaluation */ + { U32 const Q = (U32)(cSrcSize * 16 / dstSize); /* Q < 16 since dstSize > cSrcSize */ + U32 const D256 = (U32)(dstSize >> 8); + U32 n; for (n=0; n<3; n++) + Dtime[n] = algoTime[Q][n].tableTime + (algoTime[Q][n].decode256Time * D256); + } + + Dtime[1] += Dtime[1] >> 4; Dtime[2] += Dtime[2] >> 3; /* advantage to algorithms using less memory, for cache eviction */ + + { U32 algoNb = 0; + if (Dtime[1] < Dtime[0]) algoNb = 1; + return decompress[algoNb](dst, dstSize, cSrc, cSrcSize); + } +} diff --git a/contrib/libs/zstd06/decompress/zbuff_decompress.c b/contrib/libs/zstd06/decompress/zbuff_decompress.c index 9778b0f915..858b442f72 100644 --- a/contrib/libs/zstd06/decompress/zbuff_decompress.c +++ b/contrib/libs/zstd06/decompress/zbuff_decompress.c @@ -1,265 +1,265 @@ -/* - Buffered version of Zstd compression library - Copyright (C) 2015-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - zstd homepage : http://www.zstd.net/ -*/ - - -/* ************************************* -* Dependencies -***************************************/ -#include <stdlib.h> -#include "error_private.h" -#include "zstd_internal.h" /* MIN, ZSTD_blockHeaderSize */ -#include "zstd_static.h" /* ZSTD_BLOCKSIZE_MAX */ -#include "zbuff_static.h" - - -/*-*************************************************************************** -* Streaming decompression howto -* -* A ZBUFF_DCtx object is required to track streaming operations. -* Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources. -* Use ZBUFF_decompressInit() to start a new decompression operation, -* or ZBUFF_decompressInitDictionary() if decompression requires a dictionary. -* Note that ZBUFF_DCtx objects can be re-init multiple times. -* -* Use ZBUFF_decompressContinue() repetitively to consume your input. -* *srcSizePtr and *dstCapacityPtr can be any size. -* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr. -* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again. -* The content of @dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change @dst. -* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency), -* or 0 when a frame is completely decoded, -* or an error code, which can be tested using ZBUFF_isError(). -* -* Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize() and ZBUFF_recommendedDOutSize() -* output : ZBUFF_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded. -* input : ZBUFF_recommendedDInSize == 128KB + 3; -* just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . -* *******************************************************************************/ - -typedef enum { ZBUFFds_init, ZBUFFds_loadHeader, - ZBUFFds_read, ZBUFFds_load, ZBUFFds_flush } ZBUFF_dStage; - -/* *** Resource management *** */ -struct ZBUFF_DCtx_s { - ZSTD_DCtx* zd; - ZSTD_frameParams fParams; - ZBUFF_dStage stage; - char* inBuff; - size_t inBuffSize; - size_t inPos; - char* outBuff; - size_t outBuffSize; - size_t outStart; - size_t outEnd; - size_t blockSize; - BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; - size_t lhSize; -}; /* typedef'd to ZBUFF_DCtx within "zstd_buffered.h" */ - - -ZBUFF_DCtx* ZBUFF_createDCtx(void) -{ - ZBUFF_DCtx* zbd = (ZBUFF_DCtx*)malloc(sizeof(ZBUFF_DCtx)); - if (zbd==NULL) return NULL; - memset(zbd, 0, sizeof(*zbd)); - zbd->zd = ZSTD_createDCtx(); - zbd->stage = ZBUFFds_init; - return zbd; -} - -size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbd) -{ - if (zbd==NULL) return 0; /* support free on null */ - ZSTD_freeDCtx(zbd->zd); - free(zbd->inBuff); - free(zbd->outBuff); - free(zbd); - return 0; -} - - -/* *** Initialization *** */ - -size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* zbd, const void* dict, size_t dictSize) -{ - zbd->stage = ZBUFFds_loadHeader; - zbd->lhSize = zbd->inPos = zbd->outStart = zbd->outEnd = 0; - return ZSTD_decompressBegin_usingDict(zbd->zd, dict, dictSize); -} - -size_t ZBUFF_decompressInit(ZBUFF_DCtx* zbd) -{ - return ZBUFF_decompressInitDictionary(zbd, NULL, 0); -} - - -/* *** Decompression *** */ - -size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd, - void* dst, size_t* dstCapacityPtr, - const void* src, size_t* srcSizePtr) -{ - const char* const istart = (const char*)src; - const char* const iend = istart + *srcSizePtr; - const char* ip = istart; - char* const ostart = (char*)dst; - char* const oend = ostart + *dstCapacityPtr; - char* op = ostart; - U32 notDone = 1; - - while (notDone) { - switch(zbd->stage) - { - case ZBUFFds_init : - return ERROR(init_missing); - - case ZBUFFds_loadHeader : - { size_t const hSize = ZSTD_getFrameParams(&(zbd->fParams), zbd->headerBuffer, zbd->lhSize); - if (hSize != 0) { - size_t const toLoad = hSize - zbd->lhSize; /* if hSize!=0, hSize > zbd->lhSize */ - if (ZSTD_isError(hSize)) return hSize; - if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */ - memcpy(zbd->headerBuffer + zbd->lhSize, ip, iend-ip); - zbd->lhSize += iend-ip; ip = iend; notDone = 0; - *dstCapacityPtr = 0; - return (hSize - zbd->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */ - } - memcpy(zbd->headerBuffer + zbd->lhSize, ip, toLoad); zbd->lhSize = hSize; ip += toLoad; - break; - } } - - /* Consume header */ - { size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zbd->zd); /* == ZSTD_frameHeaderSize_min */ - size_t const h1Result = ZSTD_decompressContinue(zbd->zd, NULL, 0, zbd->headerBuffer, h1Size); - if (ZSTD_isError(h1Result)) return h1Result; - if (h1Size < zbd->lhSize) { /* long header */ - size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zbd->zd); - size_t const h2Result = ZSTD_decompressContinue(zbd->zd, NULL, 0, zbd->headerBuffer+h1Size, h2Size); - if (ZSTD_isError(h2Result)) return h2Result; - } } - - /* Frame header instruct buffer sizes */ - { size_t const blockSize = MIN(1 << zbd->fParams.windowLog, ZSTD_BLOCKSIZE_MAX); - zbd->blockSize = blockSize; - if (zbd->inBuffSize < blockSize) { - free(zbd->inBuff); - zbd->inBuffSize = blockSize; - zbd->inBuff = (char*)malloc(blockSize); - if (zbd->inBuff == NULL) return ERROR(memory_allocation); - } - { size_t const neededOutSize = ((size_t)1 << zbd->fParams.windowLog) + blockSize; - if (zbd->outBuffSize < neededOutSize) { - free(zbd->outBuff); - zbd->outBuffSize = neededOutSize; - zbd->outBuff = (char*)malloc(neededOutSize); - if (zbd->outBuff == NULL) return ERROR(memory_allocation); - } } } - zbd->stage = ZBUFFds_read; - - case ZBUFFds_read: - { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zbd->zd); - if (neededInSize==0) { /* end of frame */ - zbd->stage = ZBUFFds_init; - notDone = 0; - break; - } - if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */ - size_t const decodedSize = ZSTD_decompressContinue(zbd->zd, - zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart, - ip, neededInSize); - if (ZSTD_isError(decodedSize)) return decodedSize; - ip += neededInSize; - if (!decodedSize) break; /* this was just a header */ - zbd->outEnd = zbd->outStart + decodedSize; - zbd->stage = ZBUFFds_flush; - break; - } - if (ip==iend) { notDone = 0; break; } /* no more input */ - zbd->stage = ZBUFFds_load; - } - - case ZBUFFds_load: - { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zbd->zd); - size_t const toLoad = neededInSize - zbd->inPos; /* should always be <= remaining space within inBuff */ - size_t loadedSize; - if (toLoad > zbd->inBuffSize - zbd->inPos) return ERROR(corruption_detected); /* should never happen */ - loadedSize = ZBUFF_limitCopy(zbd->inBuff + zbd->inPos, toLoad, ip, iend-ip); - ip += loadedSize; - zbd->inPos += loadedSize; - if (loadedSize < toLoad) { notDone = 0; break; } /* not enough input, wait for more */ - - /* decode loaded input */ - { size_t const decodedSize = ZSTD_decompressContinue(zbd->zd, - zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart, - zbd->inBuff, neededInSize); - if (ZSTD_isError(decodedSize)) return decodedSize; - zbd->inPos = 0; /* input is consumed */ - if (!decodedSize) { zbd->stage = ZBUFFds_read; break; } /* this was just a header */ - zbd->outEnd = zbd->outStart + decodedSize; - zbd->stage = ZBUFFds_flush; - // break; /* ZBUFFds_flush follows */ - } } - - case ZBUFFds_flush: - { size_t const toFlushSize = zbd->outEnd - zbd->outStart; - size_t const flushedSize = ZBUFF_limitCopy(op, oend-op, zbd->outBuff + zbd->outStart, toFlushSize); - op += flushedSize; - zbd->outStart += flushedSize; - if (flushedSize == toFlushSize) { - zbd->stage = ZBUFFds_read; - if (zbd->outStart + zbd->blockSize > zbd->outBuffSize) - zbd->outStart = zbd->outEnd = 0; - break; - } - /* cannot flush everything */ - notDone = 0; - break; - } - default: return ERROR(GENERIC); /* impossible */ - } } - - /* result */ - *srcSizePtr = ip-istart; - *dstCapacityPtr = op-ostart; - { size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zbd->zd); - if (nextSrcSizeHint > ZSTD_blockHeaderSize) nextSrcSizeHint+= ZSTD_blockHeaderSize; /* get following block header too */ - nextSrcSizeHint -= zbd->inPos; /* already loaded*/ - return nextSrcSizeHint; - } -} - - - -/* ************************************* -* Tool functions -***************************************/ -size_t ZBUFF_recommendedDInSize(void) { return ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize /* block header size*/ ; } -size_t ZBUFF_recommendedDOutSize(void) { return ZSTD_BLOCKSIZE_MAX; } +/* + Buffered version of Zstd compression library + Copyright (C) 2015-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - zstd homepage : http://www.zstd.net/ +*/ + + +/* ************************************* +* Dependencies +***************************************/ +#include <stdlib.h> +#include "error_private.h" +#include "zstd_internal.h" /* MIN, ZSTD_blockHeaderSize */ +#include "zstd_static.h" /* ZSTD_BLOCKSIZE_MAX */ +#include "zbuff_static.h" + + +/*-*************************************************************************** +* Streaming decompression howto +* +* A ZBUFF_DCtx object is required to track streaming operations. +* Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources. +* Use ZBUFF_decompressInit() to start a new decompression operation, +* or ZBUFF_decompressInitDictionary() if decompression requires a dictionary. +* Note that ZBUFF_DCtx objects can be re-init multiple times. +* +* Use ZBUFF_decompressContinue() repetitively to consume your input. +* *srcSizePtr and *dstCapacityPtr can be any size. +* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr. +* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again. +* The content of @dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change @dst. +* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency), +* or 0 when a frame is completely decoded, +* or an error code, which can be tested using ZBUFF_isError(). +* +* Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize() and ZBUFF_recommendedDOutSize() +* output : ZBUFF_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded. +* input : ZBUFF_recommendedDInSize == 128KB + 3; +* just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . +* *******************************************************************************/ + +typedef enum { ZBUFFds_init, ZBUFFds_loadHeader, + ZBUFFds_read, ZBUFFds_load, ZBUFFds_flush } ZBUFF_dStage; + +/* *** Resource management *** */ +struct ZBUFF_DCtx_s { + ZSTD_DCtx* zd; + ZSTD_frameParams fParams; + ZBUFF_dStage stage; + char* inBuff; + size_t inBuffSize; + size_t inPos; + char* outBuff; + size_t outBuffSize; + size_t outStart; + size_t outEnd; + size_t blockSize; + BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; + size_t lhSize; +}; /* typedef'd to ZBUFF_DCtx within "zstd_buffered.h" */ + + +ZBUFF_DCtx* ZBUFF_createDCtx(void) +{ + ZBUFF_DCtx* zbd = (ZBUFF_DCtx*)malloc(sizeof(ZBUFF_DCtx)); + if (zbd==NULL) return NULL; + memset(zbd, 0, sizeof(*zbd)); + zbd->zd = ZSTD_createDCtx(); + zbd->stage = ZBUFFds_init; + return zbd; +} + +size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbd) +{ + if (zbd==NULL) return 0; /* support free on null */ + ZSTD_freeDCtx(zbd->zd); + free(zbd->inBuff); + free(zbd->outBuff); + free(zbd); + return 0; +} + + +/* *** Initialization *** */ + +size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* zbd, const void* dict, size_t dictSize) +{ + zbd->stage = ZBUFFds_loadHeader; + zbd->lhSize = zbd->inPos = zbd->outStart = zbd->outEnd = 0; + return ZSTD_decompressBegin_usingDict(zbd->zd, dict, dictSize); +} + +size_t ZBUFF_decompressInit(ZBUFF_DCtx* zbd) +{ + return ZBUFF_decompressInitDictionary(zbd, NULL, 0); +} + + +/* *** Decompression *** */ + +size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd, + void* dst, size_t* dstCapacityPtr, + const void* src, size_t* srcSizePtr) +{ + const char* const istart = (const char*)src; + const char* const iend = istart + *srcSizePtr; + const char* ip = istart; + char* const ostart = (char*)dst; + char* const oend = ostart + *dstCapacityPtr; + char* op = ostart; + U32 notDone = 1; + + while (notDone) { + switch(zbd->stage) + { + case ZBUFFds_init : + return ERROR(init_missing); + + case ZBUFFds_loadHeader : + { size_t const hSize = ZSTD_getFrameParams(&(zbd->fParams), zbd->headerBuffer, zbd->lhSize); + if (hSize != 0) { + size_t const toLoad = hSize - zbd->lhSize; /* if hSize!=0, hSize > zbd->lhSize */ + if (ZSTD_isError(hSize)) return hSize; + if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */ + memcpy(zbd->headerBuffer + zbd->lhSize, ip, iend-ip); + zbd->lhSize += iend-ip; ip = iend; notDone = 0; + *dstCapacityPtr = 0; + return (hSize - zbd->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */ + } + memcpy(zbd->headerBuffer + zbd->lhSize, ip, toLoad); zbd->lhSize = hSize; ip += toLoad; + break; + } } + + /* Consume header */ + { size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zbd->zd); /* == ZSTD_frameHeaderSize_min */ + size_t const h1Result = ZSTD_decompressContinue(zbd->zd, NULL, 0, zbd->headerBuffer, h1Size); + if (ZSTD_isError(h1Result)) return h1Result; + if (h1Size < zbd->lhSize) { /* long header */ + size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zbd->zd); + size_t const h2Result = ZSTD_decompressContinue(zbd->zd, NULL, 0, zbd->headerBuffer+h1Size, h2Size); + if (ZSTD_isError(h2Result)) return h2Result; + } } + + /* Frame header instruct buffer sizes */ + { size_t const blockSize = MIN(1 << zbd->fParams.windowLog, ZSTD_BLOCKSIZE_MAX); + zbd->blockSize = blockSize; + if (zbd->inBuffSize < blockSize) { + free(zbd->inBuff); + zbd->inBuffSize = blockSize; + zbd->inBuff = (char*)malloc(blockSize); + if (zbd->inBuff == NULL) return ERROR(memory_allocation); + } + { size_t const neededOutSize = ((size_t)1 << zbd->fParams.windowLog) + blockSize; + if (zbd->outBuffSize < neededOutSize) { + free(zbd->outBuff); + zbd->outBuffSize = neededOutSize; + zbd->outBuff = (char*)malloc(neededOutSize); + if (zbd->outBuff == NULL) return ERROR(memory_allocation); + } } } + zbd->stage = ZBUFFds_read; + + case ZBUFFds_read: + { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zbd->zd); + if (neededInSize==0) { /* end of frame */ + zbd->stage = ZBUFFds_init; + notDone = 0; + break; + } + if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */ + size_t const decodedSize = ZSTD_decompressContinue(zbd->zd, + zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart, + ip, neededInSize); + if (ZSTD_isError(decodedSize)) return decodedSize; + ip += neededInSize; + if (!decodedSize) break; /* this was just a header */ + zbd->outEnd = zbd->outStart + decodedSize; + zbd->stage = ZBUFFds_flush; + break; + } + if (ip==iend) { notDone = 0; break; } /* no more input */ + zbd->stage = ZBUFFds_load; + } + + case ZBUFFds_load: + { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zbd->zd); + size_t const toLoad = neededInSize - zbd->inPos; /* should always be <= remaining space within inBuff */ + size_t loadedSize; + if (toLoad > zbd->inBuffSize - zbd->inPos) return ERROR(corruption_detected); /* should never happen */ + loadedSize = ZBUFF_limitCopy(zbd->inBuff + zbd->inPos, toLoad, ip, iend-ip); + ip += loadedSize; + zbd->inPos += loadedSize; + if (loadedSize < toLoad) { notDone = 0; break; } /* not enough input, wait for more */ + + /* decode loaded input */ + { size_t const decodedSize = ZSTD_decompressContinue(zbd->zd, + zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart, + zbd->inBuff, neededInSize); + if (ZSTD_isError(decodedSize)) return decodedSize; + zbd->inPos = 0; /* input is consumed */ + if (!decodedSize) { zbd->stage = ZBUFFds_read; break; } /* this was just a header */ + zbd->outEnd = zbd->outStart + decodedSize; + zbd->stage = ZBUFFds_flush; + // break; /* ZBUFFds_flush follows */ + } } + + case ZBUFFds_flush: + { size_t const toFlushSize = zbd->outEnd - zbd->outStart; + size_t const flushedSize = ZBUFF_limitCopy(op, oend-op, zbd->outBuff + zbd->outStart, toFlushSize); + op += flushedSize; + zbd->outStart += flushedSize; + if (flushedSize == toFlushSize) { + zbd->stage = ZBUFFds_read; + if (zbd->outStart + zbd->blockSize > zbd->outBuffSize) + zbd->outStart = zbd->outEnd = 0; + break; + } + /* cannot flush everything */ + notDone = 0; + break; + } + default: return ERROR(GENERIC); /* impossible */ + } } + + /* result */ + *srcSizePtr = ip-istart; + *dstCapacityPtr = op-ostart; + { size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zbd->zd); + if (nextSrcSizeHint > ZSTD_blockHeaderSize) nextSrcSizeHint+= ZSTD_blockHeaderSize; /* get following block header too */ + nextSrcSizeHint -= zbd->inPos; /* already loaded*/ + return nextSrcSizeHint; + } +} + + + +/* ************************************* +* Tool functions +***************************************/ +size_t ZBUFF_recommendedDInSize(void) { return ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize /* block header size*/ ; } +size_t ZBUFF_recommendedDOutSize(void) { return ZSTD_BLOCKSIZE_MAX; } diff --git a/contrib/libs/zstd06/decompress/zstd_decompress.c b/contrib/libs/zstd06/decompress/zstd_decompress.c index e65405bc37..177e9c8a21 100644 --- a/contrib/libs/zstd06/decompress/zstd_decompress.c +++ b/contrib/libs/zstd06/decompress/zstd_decompress.c @@ -59,7 +59,7 @@ #include "mem.h" /* low level memory routines */ #include "zstd_internal.h" #include "fse_static.h" -#include "huf_static.h" +#include "huf_static.h" #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) # include "zstd_legacy.h" @@ -83,14 +83,14 @@ #endif -/*-************************************* -* Macros -***************************************/ -#define ZSTD_isError ERR_isError /* for inlining */ -#define FSE_isError ERR_isError -#define HUF_isError ERR_isError - - +/*-************************************* +* Macros +***************************************/ +#define ZSTD_isError ERR_isError /* for inlining */ +#define FSE_isError ERR_isError +#define HUF_isError ERR_isError + + /*_******************************************************* * Memory operations **********************************************************/ @@ -269,8 +269,8 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) static size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize) { if (srcSize < ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); - { U32 const fcsId = (((const BYTE*)src)[4]) >> 6; - return ZSTD_frameHeaderSize_min + ZSTD_fcs_fieldSize[fcsId]; } + { U32 const fcsId = (((const BYTE*)src)[4]) >> 6; + return ZSTD_frameHeaderSize_min + ZSTD_fcs_fieldSize[fcsId]; } } @@ -659,8 +659,8 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState) } -FORCE_INLINE -size_t ZSTD_execSequence(BYTE* op, +FORCE_INLINE +size_t ZSTD_execSequence(BYTE* op, BYTE* const oend, seq_t sequence, const BYTE** litPtr, const BYTE* const litLimit_8, const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd) @@ -845,19 +845,19 @@ size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, } -/*! ZSTD_decompressFrame() : -* `dctx` must be properly initialized */ +/*! ZSTD_decompressFrame() : +* `dctx` must be properly initialized */ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) { const BYTE* ip = (const BYTE*)src; - const BYTE* const iend = ip + srcSize; + const BYTE* const iend = ip + srcSize; BYTE* const ostart = (BYTE* const)dst; BYTE* op = ostart; BYTE* const oend = ostart + dstCapacity; size_t remainingSize = srcSize; - blockProperties_t blockProperties = { bt_compressed, 0 }; + blockProperties_t blockProperties = { bt_compressed, 0 }; /* check */ if (srcSize < ZSTD_frameHeaderSize_min+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong); @@ -925,12 +925,12 @@ size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, const void* src, size_t srcSize, const void* dict, size_t dictSize) { -#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) - { const U32 magicNumber = MEM_readLE32(src); - if (ZSTD_isLegacy(magicNumber)) - return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, dict, dictSize, magicNumber); - } -#endif +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) + { const U32 magicNumber = MEM_readLE32(src); + if (ZSTD_isLegacy(magicNumber)) + return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, dict, dictSize, magicNumber); + } +#endif ZSTD_decompressBegin_usingDict(dctx, dict, dictSize); ZSTD_checkContinuity(dctx, dst); return ZSTD_decompressFrame(dctx, dst, dstCapacity, src, srcSize); @@ -952,7 +952,7 @@ size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t sr regenSize = ZSTD_decompressDCtx(dctx, dst, dstCapacity, src, srcSize); ZSTD_freeDCtx(dctx); return regenSize; -#else /* stack mode */ +#else /* stack mode */ ZSTD_DCtx dctx; return ZSTD_decompressDCtx(&dctx, dst, dstCapacity, src, srcSize); #endif @@ -967,29 +967,29 @@ size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx) return dctx->expected; } -size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) { /* Sanity check */ if (srcSize != dctx->expected) return ERROR(srcSize_wrong); - if (dstCapacity) ZSTD_checkContinuity(dctx, dst); + if (dstCapacity) ZSTD_checkContinuity(dctx, dst); /* Decompress : frame header; part 1 */ switch (dctx->stage) { case ZSTDds_getFrameHeaderSize : - if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */ - dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_min); - if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize; - memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_min); - if (dctx->headerSize > ZSTD_frameHeaderSize_min) { - dctx->expected = dctx->headerSize - ZSTD_frameHeaderSize_min; - dctx->stage = ZSTDds_decodeFrameHeader; - return 0; + if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */ + dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_min); + if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize; + memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_min); + if (dctx->headerSize > ZSTD_frameHeaderSize_min) { + dctx->expected = dctx->headerSize - ZSTD_frameHeaderSize_min; + dctx->stage = ZSTDds_decodeFrameHeader; + return 0; } - dctx->expected = 0; /* not necessary to copy more */ - + dctx->expected = 0; /* not necessary to copy more */ + case ZSTDds_decodeFrameHeader: - { size_t result; + { size_t result; memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_min, src, dctx->expected); result = ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize); if (ZSTD_isError(result)) return result; @@ -998,7 +998,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c return 0; } case ZSTDds_decodeBlockHeader: - { blockProperties_t bp; + { blockProperties_t bp; size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp); if (ZSTD_isError(cBlockSize)) return cBlockSize; if (bp.blockType == bt_end) { @@ -1012,14 +1012,14 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c return 0; } case ZSTDds_decompressBlock: - { size_t rSize; + { size_t rSize; switch(dctx->bType) { case bt_compressed: - rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize); + rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize); break; case bt_raw : - rSize = ZSTD_copyRawBlock(dst, dstCapacity, src, srcSize); + rSize = ZSTD_copyRawBlock(dst, dstCapacity, src, srcSize); break; case bt_rle : return ERROR(GENERIC); /* not yet handled */ @@ -1051,40 +1051,40 @@ static void ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dictSi static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) { - size_t hSize, offcodeHeaderSize, matchlengthHeaderSize, litlengthHeaderSize; + size_t hSize, offcodeHeaderSize, matchlengthHeaderSize, litlengthHeaderSize; hSize = HUF_readDTableX4(dctx->hufTableX4, dict, dictSize); if (HUF_isError(hSize)) return ERROR(dictionary_corrupted); dict = (const char*)dict + hSize; dictSize -= hSize; - { short offcodeNCount[MaxOff+1]; - U32 offcodeMaxValue=MaxOff, offcodeLog=OffFSELog; - offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dict, dictSize); - if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); - { size_t const errorCode = FSE_buildDTable(dctx->OffTable, offcodeNCount, offcodeMaxValue, offcodeLog); - if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); } - dict = (const char*)dict + offcodeHeaderSize; - dictSize -= offcodeHeaderSize; - } - - { short matchlengthNCount[MaxML+1]; - unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog; - matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dict, dictSize); - if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted); - { size_t const errorCode = FSE_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog); - if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); } - dict = (const char*)dict + matchlengthHeaderSize; - dictSize -= matchlengthHeaderSize; - } - - { short litlengthNCount[MaxLL+1]; - unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog; - litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dict, dictSize); - if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted); - { size_t const errorCode = FSE_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog); - if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); } - } + { short offcodeNCount[MaxOff+1]; + U32 offcodeMaxValue=MaxOff, offcodeLog=OffFSELog; + offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dict, dictSize); + if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); + { size_t const errorCode = FSE_buildDTable(dctx->OffTable, offcodeNCount, offcodeMaxValue, offcodeLog); + if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); } + dict = (const char*)dict + offcodeHeaderSize; + dictSize -= offcodeHeaderSize; + } + + { short matchlengthNCount[MaxML+1]; + unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog; + matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dict, dictSize); + if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted); + { size_t const errorCode = FSE_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog); + if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); } + dict = (const char*)dict + matchlengthHeaderSize; + dictSize -= matchlengthHeaderSize; + } + + { short litlengthNCount[MaxLL+1]; + unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog; + litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dict, dictSize); + if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted); + { size_t const errorCode = FSE_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog); + if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); } + } dctx->flagRepeatTable = 1; return hSize + offcodeHeaderSize + matchlengthHeaderSize + litlengthHeaderSize; diff --git a/contrib/libs/zstd06/dictBuilder/divsufsort.h b/contrib/libs/zstd06/dictBuilder/divsufsort.h index 86fe818a9c..84686157ab 100644 --- a/contrib/libs/zstd06/dictBuilder/divsufsort.h +++ b/contrib/libs/zstd06/dictBuilder/divsufsort.h @@ -1,4 +1,4 @@ -#include <contrib/libs/zstd06/renames.h> +#include <contrib/libs/zstd06/renames.h> /* * divsufsort.h for libdivsufsort-lite * Copyright (c) 2003-2008 Yuta Mori All Rights Reserved. diff --git a/contrib/libs/zstd06/dictBuilder/zdict.c b/contrib/libs/zstd06/dictBuilder/zdict.c index 30e8edf9a2..95d291f409 100644 --- a/contrib/libs/zstd06/dictBuilder/zdict.c +++ b/contrib/libs/zstd06/dictBuilder/zdict.c @@ -59,7 +59,7 @@ #include "mem.h" /* read */ #include "error_private.h" #include "fse.h" -#include "huf_static.h" +#include "huf_static.h" #include "zstd_internal.h" #include "divsufsort.h" #include "zdict_static.h" @@ -830,12 +830,12 @@ size_t ZDICT_trainFromBuffer_unsafe( size_t dictSize = 0; /* checks */ - if (maxDictSize <= g_provision_entropySize + g_min_fast_dictContent) return ERROR(dstSize_tooSmall); + if (maxDictSize <= g_provision_entropySize + g_min_fast_dictContent) return ERROR(dstSize_tooSmall); if (!dictList) return ERROR(memory_allocation); /* init */ { unsigned u; for (u=0, sBuffSize=0; u<nbSamples; u++) sBuffSize += sampleSizes[u]; } - if (sBuffSize < DIB_MINSAMPLESSIZE) return 0; /* not enough source to create dictionary */ + if (sBuffSize < DIB_MINSAMPLESSIZE) return 0; /* not enough source to create dictionary */ ZDICT_initDictItem(dictList); g_displayLevel = params.notificationLevel; if (selectivity==0) selectivity = g_selectivity_default; @@ -876,7 +876,7 @@ size_t ZDICT_trainFromBuffer_unsafe( for (u=1; u<dictList->pos; u++) { U32 l = dictList[u].length; ptr -= l; - if (ptr<(BYTE*)dictBuffer) return ERROR(GENERIC); /* should not happen */ + if (ptr<(BYTE*)dictBuffer) return ERROR(GENERIC); /* should not happen */ memcpy(ptr, (const char*)samplesBuffer+dictList[u].pos, l); } diff --git a/contrib/libs/zstd06/dictBuilder/zdict.h b/contrib/libs/zstd06/dictBuilder/zdict.h index 0641f8dbd1..d9e6d3262d 100644 --- a/contrib/libs/zstd06/dictBuilder/zdict.h +++ b/contrib/libs/zstd06/dictBuilder/zdict.h @@ -1,4 +1,4 @@ -#include <contrib/libs/zstd06/renames.h> +#include <contrib/libs/zstd06/renames.h> /* dictBuilder header file Copyright (C) Yann Collet 2016 diff --git a/contrib/libs/zstd06/dictBuilder/zdict_static.h b/contrib/libs/zstd06/dictBuilder/zdict_static.h index c1388fa9e6..f83f917008 100644 --- a/contrib/libs/zstd06/dictBuilder/zdict_static.h +++ b/contrib/libs/zstd06/dictBuilder/zdict_static.h @@ -1,4 +1,4 @@ -#include <contrib/libs/zstd06/renames.h> +#include <contrib/libs/zstd06/renames.h> /* dictBuilder header file for static linking only diff --git a/contrib/libs/zstd06/gen_renames.sh b/contrib/libs/zstd06/gen_renames.sh index 0001bed99e..837166a1a7 100755 --- a/contrib/libs/zstd06/gen_renames.sh +++ b/contrib/libs/zstd06/gen_renames.sh @@ -1,8 +1,8 @@ -#!/bin/bash - -ZSTD_ARCHIVE=${1-./libcontrib-libs-zstd06.a} -nm $ZSTD_ARCHIVE --defined-only -g | egrep '^00' | sed 's/Legacy06_//' | cut -d ' ' -f 3 | awk 'BEGIN{print "#pragma once"}{printf("#define %s Legacy06_%s\n", $1, $1)}' > renames.h - -for f in $(find . -name '*.h' | grep -v renames); do - cat "$f" | grep -v '#include "renames' | awk 'BEGIN{print "#include <contrib/libs/zstd06/renames.h>"}{print}' > "$f.tmp" && grep -q '#include' "$f.tmp" && mv "$f.tmp" "$f" || rm "$f.tmp" -done +#!/bin/bash + +ZSTD_ARCHIVE=${1-./libcontrib-libs-zstd06.a} +nm $ZSTD_ARCHIVE --defined-only -g | egrep '^00' | sed 's/Legacy06_//' | cut -d ' ' -f 3 | awk 'BEGIN{print "#pragma once"}{printf("#define %s Legacy06_%s\n", $1, $1)}' > renames.h + +for f in $(find . -name '*.h' | grep -v renames); do + cat "$f" | grep -v '#include "renames' | awk 'BEGIN{print "#include <contrib/libs/zstd06/renames.h>"}{print}' > "$f.tmp" && grep -q '#include' "$f.tmp" && mv "$f.tmp" "$f" || rm "$f.tmp" +done diff --git a/contrib/libs/zstd06/legacy/zstd_legacy.h b/contrib/libs/zstd06/legacy/zstd_legacy.h index 37092f88af..80fc364cac 100644 --- a/contrib/libs/zstd06/legacy/zstd_legacy.h +++ b/contrib/libs/zstd06/legacy/zstd_legacy.h @@ -1,8 +1,8 @@ -#include <contrib/libs/zstd06/renames.h> +#include <contrib/libs/zstd06/renames.h> /* zstd_legacy - decoder for legacy format Header File - Copyright (C) 2015-2016, Yann Collet. + Copyright (C) 2015-2016, Yann Collet. BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) @@ -48,25 +48,25 @@ extern "C" { #include "zstd_v03.h" #include "zstd_v04.h" #include "zstd_v05.h" -#include "zstd_v07.h" -#include "zstd_v08.h" +#include "zstd_v07.h" +#include "zstd_v08.h" - -/** ZSTD_isLegacy() : - @return : > 0 if supported by legacy decoder. 0 otherwise. - return value is the version. -*/ + +/** ZSTD_isLegacy() : + @return : > 0 if supported by legacy decoder. 0 otherwise. + return value is the version. +*/ MEM_STATIC unsigned ZSTD_isLegacy (U32 magicNumberLE) { switch(magicNumberLE) { - case ZSTDv01_magicNumberLE:return 1; - case ZSTDv02_magicNumber : return 2; - case ZSTDv03_magicNumber : return 3; - case ZSTDv04_magicNumber : return 4; - case ZSTDv05_MAGICNUMBER : return 5; - case ZSTDv07_MAGICNUMBER : return 7; - case ZSTDv08_MAGICNUMBER : return 8; + case ZSTDv01_magicNumberLE:return 1; + case ZSTDv02_magicNumber : return 2; + case ZSTDv03_magicNumber : return 3; + case ZSTDv04_magicNumber : return 4; + case ZSTDv05_MAGICNUMBER : return 5; + case ZSTDv07_MAGICNUMBER : return 7; + case ZSTDv08_MAGICNUMBER : return 8; default : return 0; } } @@ -75,47 +75,47 @@ MEM_STATIC unsigned ZSTD_isLegacy (U32 magicNumberLE) MEM_STATIC size_t ZSTD_decompressLegacy( void* dst, size_t dstCapacity, const void* src, size_t compressedSize, - const void* dict,size_t dictSize, + const void* dict,size_t dictSize, U32 magicNumberLE) { - switch(magicNumberLE) - { - case ZSTDv01_magicNumberLE : - return ZSTDv01_decompress(dst, dstCapacity, src, compressedSize); - case ZSTDv02_magicNumber : - return ZSTDv02_decompress(dst, dstCapacity, src, compressedSize); - case ZSTDv03_magicNumber : - return ZSTDv03_decompress(dst, dstCapacity, src, compressedSize); - case ZSTDv04_magicNumber : - return ZSTDv04_decompress(dst, dstCapacity, src, compressedSize); - case ZSTDv05_MAGICNUMBER : - { - size_t result; - ZSTDv05_DCtx* zd = ZSTDv05_createDCtx(); - if (zd==NULL) return ERROR(memory_allocation); - result = ZSTDv05_decompress_usingDict(zd, dst, dstCapacity, src, compressedSize, dict, dictSize); - ZSTDv05_freeDCtx(zd); - return result; - } - case ZSTDv07_MAGICNUMBER : - { size_t result; - ZSTDv07_DCtx* const zd = ZSTDv07_createDCtx(); - if (zd==NULL) return ERROR(memory_allocation); - result = ZSTDv07_decompress_usingDict(zd, dst, dstCapacity, src, compressedSize, dict, dictSize); - ZSTDv07_freeDCtx(zd); - return result; - } - case ZSTDv08_MAGICNUMBER : - { size_t result; - ZSTDv08_DCtx* const zd = ZSTDv08_createDCtx(); - if (zd==NULL) return ERROR(memory_allocation); - result = ZSTDv08_decompress_usingDict(zd, dst, dstCapacity, src, compressedSize, dict, dictSize); - ZSTDv08_freeDCtx(zd); - return result; - } - default : - return ERROR(prefix_unknown); - } + switch(magicNumberLE) + { + case ZSTDv01_magicNumberLE : + return ZSTDv01_decompress(dst, dstCapacity, src, compressedSize); + case ZSTDv02_magicNumber : + return ZSTDv02_decompress(dst, dstCapacity, src, compressedSize); + case ZSTDv03_magicNumber : + return ZSTDv03_decompress(dst, dstCapacity, src, compressedSize); + case ZSTDv04_magicNumber : + return ZSTDv04_decompress(dst, dstCapacity, src, compressedSize); + case ZSTDv05_MAGICNUMBER : + { + size_t result; + ZSTDv05_DCtx* zd = ZSTDv05_createDCtx(); + if (zd==NULL) return ERROR(memory_allocation); + result = ZSTDv05_decompress_usingDict(zd, dst, dstCapacity, src, compressedSize, dict, dictSize); + ZSTDv05_freeDCtx(zd); + return result; + } + case ZSTDv07_MAGICNUMBER : + { size_t result; + ZSTDv07_DCtx* const zd = ZSTDv07_createDCtx(); + if (zd==NULL) return ERROR(memory_allocation); + result = ZSTDv07_decompress_usingDict(zd, dst, dstCapacity, src, compressedSize, dict, dictSize); + ZSTDv07_freeDCtx(zd); + return result; + } + case ZSTDv08_MAGICNUMBER : + { size_t result; + ZSTDv08_DCtx* const zd = ZSTDv08_createDCtx(); + if (zd==NULL) return ERROR(memory_allocation); + result = ZSTDv08_decompress_usingDict(zd, dst, dstCapacity, src, compressedSize, dict, dictSize); + ZSTDv08_freeDCtx(zd); + return result; + } + default : + return ERROR(prefix_unknown); + } } diff --git a/contrib/libs/zstd06/legacy/zstd_v01.c b/contrib/libs/zstd06/legacy/zstd_v01.c index 3e2bf1c167..d62367df29 100644 --- a/contrib/libs/zstd06/legacy/zstd_v01.c +++ b/contrib/libs/zstd06/legacy/zstd_v01.c @@ -407,7 +407,7 @@ FORCE_INLINE unsigned FSE_highbit32 (register U32 val) static U32 FSE_tableStep(U32 tableSize) { return (tableSize>>1) + (tableSize>>3) + 3; } -#define FSE_DECODE_TYPE FSE_decode_t +#define FSE_DECODE_TYPE FSE_decode_t typedef struct { @@ -415,7 +415,7 @@ typedef struct { U16 fastMode; } FSE_DTableHeader; /* sizeof U32 */ -static size_t FSE_buildDTable +static size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) { void* ptr = dt; diff --git a/contrib/libs/zstd06/legacy/zstd_v01.h b/contrib/libs/zstd06/legacy/zstd_v01.h index cb77e4336b..2b0bea2857 100644 --- a/contrib/libs/zstd06/legacy/zstd_v01.h +++ b/contrib/libs/zstd06/legacy/zstd_v01.h @@ -1,4 +1,4 @@ -#include <contrib/libs/zstd06/renames.h> +#include <contrib/libs/zstd06/renames.h> /* zstd - standard compression library Header File diff --git a/contrib/libs/zstd06/legacy/zstd_v02.c b/contrib/libs/zstd06/legacy/zstd_v02.c index 724ec44f57..2d4cfa59c4 100644 --- a/contrib/libs/zstd06/legacy/zstd_v02.c +++ b/contrib/libs/zstd06/legacy/zstd_v02.c @@ -1266,11 +1266,11 @@ typedef U32 DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)]; /* Function templates */ -#define FSE_DECODE_TYPE FSE_decode_t +#define FSE_DECODE_TYPE FSE_decode_t static U32 FSE_tableStep(U32 tableSize) { return (tableSize>>1) + (tableSize>>3) + 3; } -static size_t FSE_buildDTable +static size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) { void* ptr = dt+1; @@ -1350,7 +1350,7 @@ static unsigned FSE_isError(size_t code) { return ERR_isError(code); } ****************************************************************/ static short FSE_abs(short a) { - return (short)(a<0 ? -a : a); + return (short)(a<0 ? -a : a); } static size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, diff --git a/contrib/libs/zstd06/legacy/zstd_v02.h b/contrib/libs/zstd06/legacy/zstd_v02.h index b6ec9a0e4f..462ea1e303 100644 --- a/contrib/libs/zstd06/legacy/zstd_v02.h +++ b/contrib/libs/zstd06/legacy/zstd_v02.h @@ -1,4 +1,4 @@ -#include <contrib/libs/zstd06/renames.h> +#include <contrib/libs/zstd06/renames.h> /* zstd_v02 - decoder for 0.2 format Header File diff --git a/contrib/libs/zstd06/legacy/zstd_v03.c b/contrib/libs/zstd06/legacy/zstd_v03.c index 90de2ef853..f06b47fdc1 100644 --- a/contrib/libs/zstd06/legacy/zstd_v03.c +++ b/contrib/libs/zstd06/legacy/zstd_v03.c @@ -1265,11 +1265,11 @@ typedef U32 DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)]; /* Function templates */ -#define FSE_DECODE_TYPE FSE_decode_t +#define FSE_DECODE_TYPE FSE_decode_t static U32 FSE_tableStep(U32 tableSize) { return (tableSize>>1) + (tableSize>>3) + 3; } -static size_t FSE_buildDTable +static size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) { void* ptr = dt+1; @@ -2400,7 +2400,7 @@ typedef size_t (*decompressionAlgo)(void* dst, size_t dstSize, const void* cSrc, static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) { - static const decompressionAlgo decompress[3] = { HUF_decompress4X2, HUF_decompress4X4, NULL }; + static const decompressionAlgo decompress[3] = { HUF_decompress4X2, HUF_decompress4X4, NULL }; /* estimate decompression time */ U32 Q; const U32 D256 = (U32)(dstSize >> 8); diff --git a/contrib/libs/zstd06/legacy/zstd_v03.h b/contrib/libs/zstd06/legacy/zstd_v03.h index d599abc22f..38b1f6eb98 100644 --- a/contrib/libs/zstd06/legacy/zstd_v03.h +++ b/contrib/libs/zstd06/legacy/zstd_v03.h @@ -1,4 +1,4 @@ -#include <contrib/libs/zstd06/renames.h> +#include <contrib/libs/zstd06/renames.h> /* zstd_v03 - decoder for 0.3 format Header File diff --git a/contrib/libs/zstd06/legacy/zstd_v04.c b/contrib/libs/zstd06/legacy/zstd_v04.c index 9c111029af..66a47e7a12 100644 --- a/contrib/libs/zstd06/legacy/zstd_v04.c +++ b/contrib/libs/zstd06/legacy/zstd_v04.c @@ -2820,7 +2820,7 @@ typedef size_t (*decompressionAlgo)(void* dst, size_t dstSize, const void* cSrc, static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) { - static const decompressionAlgo decompress[3] = { HUF_decompress4X2, HUF_decompress4X4, NULL }; + static const decompressionAlgo decompress[3] = { HUF_decompress4X2, HUF_decompress4X4, NULL }; /* estimate decompression time */ U32 Q; const U32 D256 = (U32)(dstSize >> 8); @@ -3620,26 +3620,26 @@ static size_t ZSTD_decompressContinue(ZSTD_DCtx* ctx, void* dst, size_t maxDstSi switch (ctx->stage) { case ZSTDds_getFrameHeaderSize : - /* get frame header size */ - if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */ - ctx->headerSize = ZSTD_decodeFrameHeader_Part1(ctx, src, ZSTD_frameHeaderSize_min); - if (ZSTD_isError(ctx->headerSize)) return ctx->headerSize; - memcpy(ctx->headerBuffer, src, ZSTD_frameHeaderSize_min); - if (ctx->headerSize > ZSTD_frameHeaderSize_min) return ERROR(GENERIC); /* impossible */ - ctx->expected = 0; /* not necessary to copy more */ - /* fallthrough */ + /* get frame header size */ + if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */ + ctx->headerSize = ZSTD_decodeFrameHeader_Part1(ctx, src, ZSTD_frameHeaderSize_min); + if (ZSTD_isError(ctx->headerSize)) return ctx->headerSize; + memcpy(ctx->headerBuffer, src, ZSTD_frameHeaderSize_min); + if (ctx->headerSize > ZSTD_frameHeaderSize_min) return ERROR(GENERIC); /* impossible */ + ctx->expected = 0; /* not necessary to copy more */ + /* fallthrough */ case ZSTDds_decodeFrameHeader: - /* get frame header */ - { size_t const result = ZSTD_decodeFrameHeader_Part2(ctx, ctx->headerBuffer, ctx->headerSize); + /* get frame header */ + { size_t const result = ZSTD_decodeFrameHeader_Part2(ctx, ctx->headerBuffer, ctx->headerSize); if (ZSTD_isError(result)) return result; ctx->expected = ZSTD_blockHeaderSize; ctx->stage = ZSTDds_decodeBlockHeader; return 0; } case ZSTDds_decodeBlockHeader: - /* Decode block header */ - { blockProperties_t bp; - size_t const blockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp); + /* Decode block header */ + { blockProperties_t bp; + size_t const blockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp); if (ZSTD_isError(blockSize)) return blockSize; if (bp.blockType == bt_end) { @@ -3854,9 +3854,9 @@ static size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDs case ZBUFFds_readHeader : /* read header from src */ - { size_t const headerSize = ZSTD_getFrameParams(&(zbc->params), src, *srcSizePtr); + { size_t const headerSize = ZSTD_getFrameParams(&(zbc->params), src, *srcSizePtr); if (ZSTD_isError(headerSize)) return headerSize; - if (headerSize) { + if (headerSize) { /* not enough input to decode header : tell how many bytes would be necessary */ memcpy(zbc->headerBuffer+zbc->hPos, src, *srcSizePtr); zbc->hPos += *srcSizePtr; @@ -3870,7 +3870,7 @@ static size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDs case ZBUFFds_loadHeader: /* complete header from src */ - { size_t headerSize = ZBUFF_limitCopy( + { size_t headerSize = ZBUFF_limitCopy( zbc->headerBuffer + zbc->hPos, ZSTD_frameHeaderSize_max - zbc->hPos, src, *srcSizePtr); zbc->hPos += headerSize; @@ -3882,12 +3882,12 @@ static size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDs *maxDstSizePtr = 0; return headerSize - zbc->hPos; } } - /* intentional fallthrough */ + /* intentional fallthrough */ case ZBUFFds_decodeHeader: /* apply header to create / resize buffers */ - { size_t const neededOutSize = (size_t)1 << zbc->params.windowLog; - size_t const neededInSize = BLOCKSIZE; /* a block is never > BLOCKSIZE */ + { size_t const neededOutSize = (size_t)1 << zbc->params.windowLog; + size_t const neededInSize = BLOCKSIZE; /* a block is never > BLOCKSIZE */ if (zbc->inBuffSize < neededInSize) { free(zbc->inBuff); zbc->inBuffSize = neededInSize; @@ -4024,7 +4024,7 @@ size_t ZSTDv04_decompress(void* dst, size_t maxDstSize, const void* src, size_t return regenSize; #else ZSTD_DCtx dctx; - return ZSTDv04_decompressDCtx(&dctx, dst, maxDstSize, src, srcSize); + return ZSTDv04_decompressDCtx(&dctx, dst, maxDstSize, src, srcSize); #endif } @@ -4054,11 +4054,11 @@ size_t ZBUFFv04_decompressContinue(ZBUFFv04_DCtx* dctx, void* dst, size_t* maxDs { return ZBUFF_decompressContinue(dctx, dst, maxDstSizePtr, src, srcSizePtr); } - -ZSTD_DCtx* ZSTDv04_createDCtx(void) { return ZSTD_createDCtx(); } -size_t ZSTDv04_freeDCtx(ZSTD_DCtx* dctx) { return ZSTD_freeDCtx(dctx); } - -size_t ZSTDv04_getFrameParams(ZSTD_parameters* params, const void* src, size_t srcSize) -{ - return ZSTD_getFrameParams(params, src, srcSize); -} + +ZSTD_DCtx* ZSTDv04_createDCtx(void) { return ZSTD_createDCtx(); } +size_t ZSTDv04_freeDCtx(ZSTD_DCtx* dctx) { return ZSTD_freeDCtx(dctx); } + +size_t ZSTDv04_getFrameParams(ZSTD_parameters* params, const void* src, size_t srcSize) +{ + return ZSTD_getFrameParams(params, src, srcSize); +} diff --git a/contrib/libs/zstd06/legacy/zstd_v04.h b/contrib/libs/zstd06/legacy/zstd_v04.h index 07c6a7c1f3..c539000a32 100644 --- a/contrib/libs/zstd06/legacy/zstd_v04.h +++ b/contrib/libs/zstd06/legacy/zstd_v04.h @@ -1,4 +1,4 @@ -#include <contrib/libs/zstd06/renames.h> +#include <contrib/libs/zstd06/renames.h> /* zstd_v04 - decoder for 0.4 format Header File diff --git a/contrib/libs/zstd06/legacy/zstd_v05.c b/contrib/libs/zstd06/legacy/zstd_v05.c index 4bd6796fd5..f3c720fd26 100644 --- a/contrib/libs/zstd06/legacy/zstd_v05.c +++ b/contrib/libs/zstd06/legacy/zstd_v05.c @@ -2375,9 +2375,9 @@ size_t HUFv05_decompress4X2_usingDTable( BYTE* op4 = opStart4; U32 endSignal; - /* Check */ - if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ - + /* Check */ + if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ + length4 = cSrcSize - (length1 + length2 + length3 + 6); if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ errorCode = BITv05_initDStream(&bitD1, istart1, length1); @@ -2860,7 +2860,7 @@ typedef size_t (*decompressionAlgo)(void* dst, size_t dstSize, const void* cSrc, size_t HUFv05_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) { - static const decompressionAlgo decompress[3] = { HUFv05_decompress4X2, HUFv05_decompress4X4, NULL }; + static const decompressionAlgo decompress[3] = { HUFv05_decompress4X2, HUFv05_decompress4X4, NULL }; /* estimate decompression time */ U32 Q; const U32 D256 = (U32)(dstSize >> 8); @@ -2870,7 +2870,7 @@ size_t HUFv05_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cS /* validation checks */ if (dstSize == 0) return ERROR(dstSize_tooSmall); - if (cSrcSize >= dstSize) return ERROR(corruption_detected); /* invalid, or not compressed, but not compressed already dealt with */ + if (cSrcSize >= dstSize) return ERROR(corruption_detected); /* invalid, or not compressed, but not compressed already dealt with */ if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ /* decoder timing evaluation */ @@ -3872,17 +3872,17 @@ size_t ZSTDv05_decompressContinue(ZSTDv05_DCtx* dctx, void* dst, size_t maxDstSi switch (dctx->stage) { case ZSTDv05ds_getFrameHeaderSize : - /* get frame header size */ - if (srcSize != ZSTDv05_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */ - dctx->headerSize = ZSTDv05_decodeFrameHeader_Part1(dctx, src, ZSTDv05_frameHeaderSize_min); - if (ZSTDv05_isError(dctx->headerSize)) return dctx->headerSize; - memcpy(dctx->headerBuffer, src, ZSTDv05_frameHeaderSize_min); - if (dctx->headerSize > ZSTDv05_frameHeaderSize_min) return ERROR(GENERIC); /* should never happen */ - dctx->expected = 0; /* not necessary to copy more */ - /* fallthrough */ + /* get frame header size */ + if (srcSize != ZSTDv05_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */ + dctx->headerSize = ZSTDv05_decodeFrameHeader_Part1(dctx, src, ZSTDv05_frameHeaderSize_min); + if (ZSTDv05_isError(dctx->headerSize)) return dctx->headerSize; + memcpy(dctx->headerBuffer, src, ZSTDv05_frameHeaderSize_min); + if (dctx->headerSize > ZSTDv05_frameHeaderSize_min) return ERROR(GENERIC); /* should never happen */ + dctx->expected = 0; /* not necessary to copy more */ + /* fallthrough */ case ZSTDv05ds_decodeFrameHeader: - /* get frame header */ - { size_t const result = ZSTDv05_decodeFrameHeader_Part2(dctx, dctx->headerBuffer, dctx->headerSize); + /* get frame header */ + { size_t const result = ZSTDv05_decodeFrameHeader_Part2(dctx, dctx->headerBuffer, dctx->headerSize); if (ZSTDv05_isError(result)) return result; dctx->expected = ZSTDv05_blockHeaderSize; dctx->stage = ZSTDv05ds_decodeBlockHeader; @@ -4306,7 +4306,7 @@ size_t ZBUFFv05_decompressContinue(ZBUFFv05_DCtx* zbc, void* dst, size_t* maxDst *srcSizePtr = ip-istart; *maxDstSizePtr = op-ostart; - { size_t nextSrcSizeHint = ZSTDv05_nextSrcSizeToDecompress(zbc->zc); + { size_t nextSrcSizeHint = ZSTDv05_nextSrcSizeToDecompress(zbc->zc); if (nextSrcSizeHint > ZBUFFv05_blockHeaderSize) nextSrcSizeHint+= ZBUFFv05_blockHeaderSize; /* get next block header too */ nextSrcSizeHint -= zbc->inPos; /* already loaded*/ return nextSrcSizeHint; diff --git a/contrib/libs/zstd06/legacy/zstd_v05.h b/contrib/libs/zstd06/legacy/zstd_v05.h index 370fc12619..a20e6c7068 100644 --- a/contrib/libs/zstd06/legacy/zstd_v05.h +++ b/contrib/libs/zstd06/legacy/zstd_v05.h @@ -1,4 +1,4 @@ -#include <contrib/libs/zstd06/renames.h> +#include <contrib/libs/zstd06/renames.h> /* zstd_v05 - decoder for 0.5 format Header File @@ -41,7 +41,7 @@ extern "C" { * Dependencies ***************************************/ #include <stddef.h> /* size_t */ -#include "mem.h" /* U64, U32 */ +#include "mem.h" /* U64, U32 */ /* ************************************* @@ -70,7 +70,7 @@ const char* ZSTDv05_getErrorName(size_t code); /*!< provides readable string /** Decompression context */ typedef struct ZSTDv05_DCtx_s ZSTDv05_DCtx; ZSTDv05_DCtx* ZSTDv05_createDCtx(void); -size_t ZSTDv05_freeDCtx(ZSTDv05_DCtx* dctx); /*!< @return : errorCode */ +size_t ZSTDv05_freeDCtx(ZSTDv05_DCtx* dctx); /*!< @return : errorCode */ /** ZSTDv05_decompressDCtx() : * Same as ZSTDv05_decompress(), but requires an already allocated ZSTDv05_DCtx (see ZSTDv05_createDCtx()) */ @@ -78,40 +78,40 @@ size_t ZSTDv05_decompressDCtx(ZSTDv05_DCtx* ctx, void* dst, size_t dstCapacity, /*-*********************** -* Simple Dictionary API +* Simple Dictionary API *************************/ /*! ZSTDv05_decompress_usingDict() : * Decompression using a pre-defined Dictionary content (see dictBuilder). * Dictionary must be identical to the one used during compression, otherwise regenerated data will be corrupted. * Note : dict can be NULL, in which case, it's equivalent to ZSTDv05_decompressDCtx() */ size_t ZSTDv05_decompress_usingDict(ZSTDv05_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const void* dict,size_t dictSize); - -/*-************************ -* Advanced Streaming API -***************************/ -typedef enum { ZSTDv05_fast, ZSTDv05_greedy, ZSTDv05_lazy, ZSTDv05_lazy2, ZSTDv05_btlazy2, ZSTDv05_opt, ZSTDv05_btopt } ZSTDv05_strategy; -typedef struct { - U64 srcSize; - U32 windowLog; /* the only useful information to retrieve */ - U32 contentLog; U32 hashLog; U32 searchLog; U32 searchLength; U32 targetLength; ZSTDv05_strategy strategy; -} ZSTDv05_parameters; -size_t ZSTDv05_getFrameParams(ZSTDv05_parameters* params, const void* src, size_t srcSize); - -size_t ZSTDv05_decompressBegin_usingDict(ZSTDv05_DCtx* dctx, const void* dict, size_t dictSize); -void ZSTDv05_copyDCtx(ZSTDv05_DCtx* dstDCtx, const ZSTDv05_DCtx* srcDCtx); -size_t ZSTDv05_nextSrcSizeToDecompress(ZSTDv05_DCtx* dctx); -size_t ZSTDv05_decompressContinue(ZSTDv05_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); - - -/*-*********************** -* ZBUFF API -*************************/ + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize); + +/*-************************ +* Advanced Streaming API +***************************/ +typedef enum { ZSTDv05_fast, ZSTDv05_greedy, ZSTDv05_lazy, ZSTDv05_lazy2, ZSTDv05_btlazy2, ZSTDv05_opt, ZSTDv05_btopt } ZSTDv05_strategy; +typedef struct { + U64 srcSize; + U32 windowLog; /* the only useful information to retrieve */ + U32 contentLog; U32 hashLog; U32 searchLog; U32 searchLength; U32 targetLength; ZSTDv05_strategy strategy; +} ZSTDv05_parameters; +size_t ZSTDv05_getFrameParams(ZSTDv05_parameters* params, const void* src, size_t srcSize); + +size_t ZSTDv05_decompressBegin_usingDict(ZSTDv05_DCtx* dctx, const void* dict, size_t dictSize); +void ZSTDv05_copyDCtx(ZSTDv05_DCtx* dstDCtx, const ZSTDv05_DCtx* srcDCtx); +size_t ZSTDv05_nextSrcSizeToDecompress(ZSTDv05_DCtx* dctx); +size_t ZSTDv05_decompressContinue(ZSTDv05_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); + + +/*-*********************** +* ZBUFF API +*************************/ typedef struct ZBUFFv05_DCtx_s ZBUFFv05_DCtx; ZBUFFv05_DCtx* ZBUFFv05_createDCtx(void); -size_t ZBUFFv05_freeDCtx(ZBUFFv05_DCtx* dctx); +size_t ZBUFFv05_freeDCtx(ZBUFFv05_DCtx* dctx); size_t ZBUFFv05_decompressInit(ZBUFFv05_DCtx* dctx); size_t ZBUFFv05_decompressInitDictionary(ZBUFFv05_DCtx* dctx, const void* dict, size_t dictSize); diff --git a/contrib/libs/zstd06/legacy/zstd_v07.c b/contrib/libs/zstd06/legacy/zstd_v07.c index 4941d58b48..b6f604d23a 100644 --- a/contrib/libs/zstd06/legacy/zstd_v07.c +++ b/contrib/libs/zstd06/legacy/zstd_v07.c @@ -1,4936 +1,4936 @@ -/* ****************************************************************** - zstd_v07.c - Decompression module for ZSTD v0.7 legacy format - Copyright (C) 2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Homepage : http://www.zstd.net/ -****************************************************************** */ - -/*- Dependencies -*/ -#include <stddef.h> /* size_t, ptrdiff_t */ -#include <string.h> /* memcpy */ -#include <stdlib.h> /* malloc, free, qsort */ - -#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ -#include "xxhash.h" /* XXH64_* */ -#include "zstd_v07.h" - -#define FSEv07_STATIC_LINKING_ONLY /* FSEv07_MIN_TABLELOG */ -#define HUFv07_STATIC_LINKING_ONLY /* HUFv07_TABLELOG_ABSOLUTEMAX */ -#define ZSTDv07_STATIC_LINKING_ONLY - - -#ifdef ZSTDv07_STATIC_LINKING_ONLY - -/* ==================================================================================== - * The definitions in this section are considered experimental. - * They should never be used with a dynamic library, as they may change in the future. - * They are provided for advanced usages. - * Use them only in association with static linking. - * ==================================================================================== */ - -/*--- Constants ---*/ -#define ZSTDv07_MAGIC_SKIPPABLE_START 0x184D2A50U - -#define ZSTDv07_WINDOWLOG_MAX_32 25 -#define ZSTDv07_WINDOWLOG_MAX_64 27 -#define ZSTDv07_WINDOWLOG_MAX ((U32)(MEM_32bits() ? ZSTDv07_WINDOWLOG_MAX_32 : ZSTDv07_WINDOWLOG_MAX_64)) -#define ZSTDv07_WINDOWLOG_MIN 18 -#define ZSTDv07_CHAINLOG_MAX (ZSTDv07_WINDOWLOG_MAX+1) -#define ZSTDv07_CHAINLOG_MIN 4 -#define ZSTDv07_HASHLOG_MAX ZSTDv07_WINDOWLOG_MAX -#define ZSTDv07_HASHLOG_MIN 12 -#define ZSTDv07_HASHLOG3_MAX 17 -#define ZSTDv07_SEARCHLOG_MAX (ZSTDv07_WINDOWLOG_MAX-1) -#define ZSTDv07_SEARCHLOG_MIN 1 -#define ZSTDv07_SEARCHLENGTH_MAX 7 -#define ZSTDv07_SEARCHLENGTH_MIN 3 -#define ZSTDv07_TARGETLENGTH_MIN 4 -#define ZSTDv07_TARGETLENGTH_MAX 999 - -#define ZSTDv07_FRAMEHEADERSIZE_MAX 18 /* for static allocation */ -static const size_t ZSTDv07_frameHeaderSize_min = 5; -static const size_t ZSTDv07_frameHeaderSize_max = ZSTDv07_FRAMEHEADERSIZE_MAX; -static const size_t ZSTDv07_skippableHeaderSize = 8; /* magic number + skippable frame length */ - - -/* custom memory allocation functions */ -typedef void* (*ZSTDv07_allocFunction) (void* opaque, size_t size); -typedef void (*ZSTDv07_freeFunction) (void* opaque, void* address); -typedef struct { ZSTDv07_allocFunction customAlloc; ZSTDv07_freeFunction customFree; void* opaque; } ZSTDv07_customMem; - - -/*--- Advanced Decompression functions ---*/ - -/*! ZSTDv07_estimateDCtxSize() : - * Gives the potential amount of memory allocated to create a ZSTDv07_DCtx */ -ZSTDLIB_API size_t ZSTDv07_estimateDCtxSize(void); - -/*! ZSTDv07_createDCtx_advanced() : - * Create a ZSTD decompression context using external alloc and free functions */ -ZSTDLIB_API ZSTDv07_DCtx* ZSTDv07_createDCtx_advanced(ZSTDv07_customMem customMem); - -/*! ZSTDv07_sizeofDCtx() : - * Gives the amount of memory used by a given ZSTDv07_DCtx */ -ZSTDLIB_API size_t ZSTDv07_sizeofDCtx(const ZSTDv07_DCtx* dctx); - - -/* ****************************************************************** -* Buffer-less streaming functions (synchronous mode) -********************************************************************/ - -ZSTDLIB_API size_t ZSTDv07_decompressBegin(ZSTDv07_DCtx* dctx); -ZSTDLIB_API size_t ZSTDv07_decompressBegin_usingDict(ZSTDv07_DCtx* dctx, const void* dict, size_t dictSize); -ZSTDLIB_API void ZSTDv07_copyDCtx(ZSTDv07_DCtx* dctx, const ZSTDv07_DCtx* preparedDCtx); - -ZSTDLIB_API size_t ZSTDv07_nextSrcSizeToDecompress(ZSTDv07_DCtx* dctx); -ZSTDLIB_API size_t ZSTDv07_decompressContinue(ZSTDv07_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); - -/* - Buffer-less streaming decompression (synchronous mode) - - A ZSTDv07_DCtx object is required to track streaming operations. - Use ZSTDv07_createDCtx() / ZSTDv07_freeDCtx() to manage it. - A ZSTDv07_DCtx object can be re-used multiple times. - - First optional operation is to retrieve frame parameters, using ZSTDv07_getFrameParams(), which doesn't consume the input. - It can provide the minimum size of rolling buffer required to properly decompress data (`windowSize`), - and optionally the final size of uncompressed content. - (Note : content size is an optional info that may not be present. 0 means : content size unknown) - Frame parameters are extracted from the beginning of compressed frame. - The amount of data to read is variable, from ZSTDv07_frameHeaderSize_min to ZSTDv07_frameHeaderSize_max (so if `srcSize` >= ZSTDv07_frameHeaderSize_max, it will always work) - If `srcSize` is too small for operation to succeed, function will return the minimum size it requires to produce a result. - Result : 0 when successful, it means the ZSTDv07_frameParams structure has been filled. - >0 : means there is not enough data into `src`. Provides the expected size to successfully decode header. - errorCode, which can be tested using ZSTDv07_isError() - - Start decompression, with ZSTDv07_decompressBegin() or ZSTDv07_decompressBegin_usingDict(). - Alternatively, you can copy a prepared context, using ZSTDv07_copyDCtx(). - - Then use ZSTDv07_nextSrcSizeToDecompress() and ZSTDv07_decompressContinue() alternatively. - ZSTDv07_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTDv07_decompressContinue(). - ZSTDv07_decompressContinue() requires this exact amount of bytes, or it will fail. - - @result of ZSTDv07_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity). - It can be zero, which is not an error; it just means ZSTDv07_decompressContinue() has decoded some header. - - ZSTDv07_decompressContinue() needs previous data blocks during decompression, up to `windowSize`. - They should preferably be located contiguously, prior to current block. - Alternatively, a round buffer of sufficient size is also possible. Sufficient size is determined by frame parameters. - ZSTDv07_decompressContinue() is very sensitive to contiguity, - if 2 blocks don't follow each other, make sure that either the compressor breaks contiguity at the same place, - or that previous contiguous segment is large enough to properly handle maximum back-reference. - - A frame is fully decoded when ZSTDv07_nextSrcSizeToDecompress() returns zero. - Context can then be reset to start a new decompression. - - - == Special case : skippable frames == - - Skippable frames allow the integration of user-defined data into a flow of concatenated frames. - Skippable frames will be ignored (skipped) by a decompressor. The format of skippable frame is following: - a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F - b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits - c) Frame Content - any content (User Data) of length equal to Frame Size - For skippable frames ZSTDv07_decompressContinue() always returns 0. - For skippable frames ZSTDv07_getFrameParams() returns fparamsPtr->windowLog==0 what means that a frame is skippable. - It also returns Frame Size as fparamsPtr->frameContentSize. -*/ - - -/* ************************************** -* Block functions -****************************************/ -/*! Block functions produce and decode raw zstd blocks, without frame metadata. - Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes). - User will have to take in charge required information to regenerate data, such as compressed and content sizes. - - A few rules to respect : - - Compressing and decompressing require a context structure - + Use ZSTDv07_createCCtx() and ZSTDv07_createDCtx() - - It is necessary to init context before starting - + compression : ZSTDv07_compressBegin() - + decompression : ZSTDv07_decompressBegin() - + variants _usingDict() are also allowed - + copyCCtx() and copyDCtx() work too - - Block size is limited, it must be <= ZSTDv07_getBlockSizeMax() - + If you need to compress more, cut data into multiple blocks - + Consider using the regular ZSTDv07_compress() instead, as frame metadata costs become negligible when source size is large. - - When a block is considered not compressible enough, ZSTDv07_compressBlock() result will be zero. - In which case, nothing is produced into `dst`. - + User must test for such outcome and deal directly with uncompressed data - + ZSTDv07_decompressBlock() doesn't accept uncompressed data as input !!! - + In case of multiple successive blocks, decoder must be informed of uncompressed block existence to follow proper history. - Use ZSTDv07_insertBlock() in such a case. -*/ - -#define ZSTDv07_BLOCKSIZE_ABSOLUTEMAX (128 * 1024) /* define, for static allocation */ -ZSTDLIB_API size_t ZSTDv07_decompressBlock(ZSTDv07_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); -ZSTDLIB_API size_t ZSTDv07_insertBlock(ZSTDv07_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert block into `dctx` history. Useful for uncompressed blocks */ - - -#endif /* ZSTDv07_STATIC_LINKING_ONLY */ - - -/* ****************************************************************** - mem.h - low-level memory access routines - Copyright (C) 2013-2015, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy - - Public forum : https://groups.google.com/forum/#!forum/lz4c -****************************************************************** */ -#ifndef MEM_H_MODULE -#define MEM_H_MODULE - -#if defined (__cplusplus) -extern "C" { -#endif - -/*-**************************************** -* Compiler specifics -******************************************/ -#if defined(_MSC_VER) /* Visual Studio */ -# include <stdlib.h> /* _byteswap_ulong */ -# include <intrin.h> /* _byteswap_* */ -#endif -#if defined(__GNUC__) -# define MEM_STATIC static __attribute__((unused)) -#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -# define MEM_STATIC static inline -#elif defined(_MSC_VER) -# define MEM_STATIC static __inline -#else -# define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ -#endif - -/* code only tested on 32 and 64 bits systems */ -#define MEM_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; } -MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); } - - -/*-************************************************************** -* Basic Types -*****************************************************************/ -#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) -# include <stdint.h> - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef int16_t S16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; - typedef int64_t S64; -#else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef signed short S16; - typedef unsigned int U32; - typedef signed int S32; - typedef unsigned long long U64; - typedef signed long long S64; -#endif - - -/*-************************************************************** -* Memory I/O -*****************************************************************/ -/* MEM_FORCE_MEMORY_ACCESS : - * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. - * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. - * The below switch allow to select different access method for improved performance. - * Method 0 (default) : use `memcpy()`. Safe and portable. - * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). - * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. - * Method 2 : direct access. This method is portable but violate C standard. - * It can generate buggy code on targets depending on alignment. - * In some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) - * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. - * Prefer these methods in priority order (0 > 1 > 2) - */ -#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ -# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) -# define MEM_FORCE_MEMORY_ACCESS 2 -# elif defined(__INTEL_COMPILER) || \ - (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) -# define MEM_FORCE_MEMORY_ACCESS 1 -# endif -#endif - -MEM_STATIC unsigned MEM_32bits(void) { return sizeof(size_t)==4; } -MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; } - -MEM_STATIC unsigned MEM_isLittleEndian(void) -{ - const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ - return one.c[0]; -} - -#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2) - -/* violates C standard, by lying on structure alignment. -Only use if no other choice to achieve best performance on target platform */ -MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; } -MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; } -MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; } -MEM_STATIC U64 MEM_readST(const void* memPtr) { return *(const size_t*) memPtr; } - -MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; } -MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; } -MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; } - -#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1) - -/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ -/* currently only defined for gcc and icc */ -typedef union { U16 u16; U32 u32; U64 u64; size_t st; } __attribute__((packed)) unalign; - -MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; } -MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } -MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; } -MEM_STATIC U64 MEM_readST(const void* ptr) { return ((const unalign*)ptr)->st; } - -MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; } -MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; } -MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign*)memPtr)->u64 = value; } - -#else - -/* default method, safe and standard. - can sometimes prove slower */ - -MEM_STATIC U16 MEM_read16(const void* memPtr) -{ - U16 val; memcpy(&val, memPtr, sizeof(val)); return val; -} - -MEM_STATIC U32 MEM_read32(const void* memPtr) -{ - U32 val; memcpy(&val, memPtr, sizeof(val)); return val; -} - -MEM_STATIC U64 MEM_read64(const void* memPtr) -{ - U64 val; memcpy(&val, memPtr, sizeof(val)); return val; -} - -MEM_STATIC size_t MEM_readST(const void* memPtr) -{ - size_t val; memcpy(&val, memPtr, sizeof(val)); return val; -} - -MEM_STATIC void MEM_write16(void* memPtr, U16 value) -{ - memcpy(memPtr, &value, sizeof(value)); -} - -MEM_STATIC void MEM_write32(void* memPtr, U32 value) -{ - memcpy(memPtr, &value, sizeof(value)); -} - -MEM_STATIC void MEM_write64(void* memPtr, U64 value) -{ - memcpy(memPtr, &value, sizeof(value)); -} - -#endif /* MEM_FORCE_MEMORY_ACCESS */ - -MEM_STATIC U32 MEM_swap32(U32 in) -{ -#if defined(_MSC_VER) /* Visual Studio */ - return _byteswap_ulong(in); -#elif defined (__GNUC__) - return __builtin_bswap32(in); -#else - return ((in << 24) & 0xff000000 ) | - ((in << 8) & 0x00ff0000 ) | - ((in >> 8) & 0x0000ff00 ) | - ((in >> 24) & 0x000000ff ); -#endif -} - -MEM_STATIC U64 MEM_swap64(U64 in) -{ -#if defined(_MSC_VER) /* Visual Studio */ - return _byteswap_uint64(in); -#elif defined (__GNUC__) - return __builtin_bswap64(in); -#else - return ((in << 56) & 0xff00000000000000ULL) | - ((in << 40) & 0x00ff000000000000ULL) | - ((in << 24) & 0x0000ff0000000000ULL) | - ((in << 8) & 0x000000ff00000000ULL) | - ((in >> 8) & 0x00000000ff000000ULL) | - ((in >> 24) & 0x0000000000ff0000ULL) | - ((in >> 40) & 0x000000000000ff00ULL) | - ((in >> 56) & 0x00000000000000ffULL); -#endif -} - -MEM_STATIC size_t MEM_swapST(size_t in) -{ - if (MEM_32bits()) - return (size_t)MEM_swap32((U32)in); - else - return (size_t)MEM_swap64((U64)in); -} - -/*=== Little endian r/w ===*/ - -MEM_STATIC U16 MEM_readLE16(const void* memPtr) -{ - if (MEM_isLittleEndian()) - return MEM_read16(memPtr); - else { - const BYTE* p = (const BYTE*)memPtr; - return (U16)(p[0] + (p[1]<<8)); - } -} - -MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val) -{ - if (MEM_isLittleEndian()) { - MEM_write16(memPtr, val); - } else { - BYTE* p = (BYTE*)memPtr; - p[0] = (BYTE)val; - p[1] = (BYTE)(val>>8); - } -} - -MEM_STATIC U32 MEM_readLE32(const void* memPtr) -{ - if (MEM_isLittleEndian()) - return MEM_read32(memPtr); - else - return MEM_swap32(MEM_read32(memPtr)); -} - -MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32) -{ - if (MEM_isLittleEndian()) - MEM_write32(memPtr, val32); - else - MEM_write32(memPtr, MEM_swap32(val32)); -} - -MEM_STATIC U64 MEM_readLE64(const void* memPtr) -{ - if (MEM_isLittleEndian()) - return MEM_read64(memPtr); - else - return MEM_swap64(MEM_read64(memPtr)); -} - -MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64) -{ - if (MEM_isLittleEndian()) - MEM_write64(memPtr, val64); - else - MEM_write64(memPtr, MEM_swap64(val64)); -} - -MEM_STATIC size_t MEM_readLEST(const void* memPtr) -{ - if (MEM_32bits()) - return (size_t)MEM_readLE32(memPtr); - else - return (size_t)MEM_readLE64(memPtr); -} - -MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val) -{ - if (MEM_32bits()) - MEM_writeLE32(memPtr, (U32)val); - else - MEM_writeLE64(memPtr, (U64)val); -} - -/*=== Big endian r/w ===*/ - -MEM_STATIC U32 MEM_readBE32(const void* memPtr) -{ - if (MEM_isLittleEndian()) - return MEM_swap32(MEM_read32(memPtr)); - else - return MEM_read32(memPtr); -} - -MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32) -{ - if (MEM_isLittleEndian()) - MEM_write32(memPtr, MEM_swap32(val32)); - else - MEM_write32(memPtr, val32); -} - -MEM_STATIC U64 MEM_readBE64(const void* memPtr) -{ - if (MEM_isLittleEndian()) - return MEM_swap64(MEM_read64(memPtr)); - else - return MEM_read64(memPtr); -} - -MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64) -{ - if (MEM_isLittleEndian()) - MEM_write64(memPtr, MEM_swap64(val64)); - else - MEM_write64(memPtr, val64); -} - -MEM_STATIC size_t MEM_readBEST(const void* memPtr) -{ - if (MEM_32bits()) - return (size_t)MEM_readBE32(memPtr); - else - return (size_t)MEM_readBE64(memPtr); -} - -MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val) -{ - if (MEM_32bits()) - MEM_writeBE32(memPtr, (U32)val); - else - MEM_writeBE64(memPtr, (U64)val); -} - - -/* function safe only for comparisons */ -MEM_STATIC U32 MEM_readMINMATCH(const void* memPtr, U32 length) -{ - switch (length) - { - default : - case 4 : return MEM_read32(memPtr); - case 3 : if (MEM_isLittleEndian()) - return MEM_read32(memPtr)<<8; - else - return MEM_read32(memPtr)>>8; - } -} - -#if defined (__cplusplus) -} -#endif - -#endif /* MEM_H_MODULE */ - -/* ****************************************************************** - Error codes list - Copyright (C) 2016, Yann Collet - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Homepage : http://www.zstd.net -****************************************************************** */ -#ifndef ERROR_PUBLIC_H_MODULE -#define ERROR_PUBLIC_H_MODULE - -#if defined (__cplusplus) -extern "C" { -#endif - - -/* **************************************** -* error codes list -******************************************/ -typedef enum { - ZSTDv07_error_no_error, - ZSTDv07_error_GENERIC, - ZSTDv07_error_prefix_unknown, - ZSTDv07_error_frameParameter_unsupported, - ZSTDv07_error_frameParameter_unsupportedBy32bits, - ZSTDv07_error_compressionParameter_unsupported, - ZSTDv07_error_init_missing, - ZSTDv07_error_memory_allocation, - ZSTDv07_error_stage_wrong, - ZSTDv07_error_dstSize_tooSmall, - ZSTDv07_error_srcSize_wrong, - ZSTDv07_error_corruption_detected, - ZSTDv07_error_checksum_wrong, - ZSTDv07_error_tableLog_tooLarge, - ZSTDv07_error_maxSymbolValue_tooLarge, - ZSTDv07_error_maxSymbolValue_tooSmall, - ZSTDv07_error_dictionary_corrupted, - ZSTDv07_error_dictionary_wrong, - ZSTDv07_error_maxCode -} ZSTDv07_ErrorCode; - -/*! ZSTDv07_getErrorCode() : - convert a `size_t` function result into a `ZSTDv07_ErrorCode` enum type, - which can be used to compare directly with enum list published into "error_public.h" */ -ZSTDv07_ErrorCode ZSTDv07_getErrorCode(size_t functionResult); -const char* ZSTDv07_getErrorString(ZSTDv07_ErrorCode code); - - -#if defined (__cplusplus) -} -#endif - -#endif /* ERROR_PUBLIC_H_MODULE */ -/* ****************************************************************** - Error codes and messages - Copyright (C) 2013-2016, Yann Collet - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Homepage : http://www.zstd.net -****************************************************************** */ -/* Note : this module is expected to remain private, do not expose it */ - -#ifndef ERROR_H_MODULE -#define ERROR_H_MODULE - -#if defined (__cplusplus) -extern "C" { -#endif - - - -/* **************************************** -* Compiler-specific -******************************************/ -#if defined(__GNUC__) -# define ERR_STATIC static __attribute__((unused)) -#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -# define ERR_STATIC static inline -#elif defined(_MSC_VER) -# define ERR_STATIC static __inline -#else -# define ERR_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ -#endif - - -/*-**************************************** -* Customization (error_public.h) -******************************************/ -typedef ZSTDv07_ErrorCode ERR_enum; -#define PREFIX(name) ZSTDv07_error_##name - - -/*-**************************************** -* Error codes handling -******************************************/ -#ifdef ERROR -# undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */ -#endif -#define ERROR(name) ((size_t)-PREFIX(name)) - -ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); } - -ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); } - - -/*-**************************************** -* Error Strings -******************************************/ - -ERR_STATIC const char* ERR_getErrorString(ERR_enum code) -{ - static const char* notErrorCode = "Unspecified error code"; - switch( code ) - { - case PREFIX(no_error): return "No error detected"; - case PREFIX(GENERIC): return "Error (generic)"; - case PREFIX(prefix_unknown): return "Unknown frame descriptor"; - case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter"; - case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode"; - case PREFIX(compressionParameter_unsupported): return "Compression parameter is out of bound"; - case PREFIX(init_missing): return "Context should be init first"; - case PREFIX(memory_allocation): return "Allocation error : not enough memory"; - case PREFIX(stage_wrong): return "Operation not authorized at current processing stage"; - case PREFIX(dstSize_tooSmall): return "Destination buffer is too small"; - case PREFIX(srcSize_wrong): return "Src size incorrect"; - case PREFIX(corruption_detected): return "Corrupted block detected"; - case PREFIX(checksum_wrong): return "Restored data doesn't match checksum"; - case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported"; - case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large"; - case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small"; - case PREFIX(dictionary_corrupted): return "Dictionary is corrupted"; - case PREFIX(dictionary_wrong): return "Dictionary mismatch"; - case PREFIX(maxCode): - default: return notErrorCode; - } -} - -ERR_STATIC const char* ERR_getErrorName(size_t code) -{ - return ERR_getErrorString(ERR_getErrorCode(code)); -} - -#if defined (__cplusplus) -} -#endif - -#endif /* ERROR_H_MODULE */ -/* ****************************************************************** - bitstream - Part of FSE library - header file (to include) - Copyright (C) 2013-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Source repository : https://github.com/Cyan4973/FiniteStateEntropy -****************************************************************** */ -#ifndef BITSTREAM_H_MODULE -#define BITSTREAM_H_MODULE - -#if defined (__cplusplus) -extern "C" { -#endif - - -/* -* This API consists of small unitary functions, which must be inlined for best performance. -* Since link-time-optimization is not available for all compilers, -* these functions are defined into a .h to be included. -*/ - - -/*========================================= -* Target specific -=========================================*/ -#if defined(__BMI__) && defined(__GNUC__) -# include <immintrin.h> /* support for bextr (experimental) */ -#endif - -/*-******************************************** -* bitStream decoding API (read backward) -**********************************************/ -typedef struct -{ - size_t bitContainer; - unsigned bitsConsumed; - const char* ptr; - const char* start; -} BITv07_DStream_t; - -typedef enum { BITv07_DStream_unfinished = 0, - BITv07_DStream_endOfBuffer = 1, - BITv07_DStream_completed = 2, - BITv07_DStream_overflow = 3 } BITv07_DStream_status; /* result of BITv07_reloadDStream() */ - /* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */ - -MEM_STATIC size_t BITv07_initDStream(BITv07_DStream_t* bitD, const void* srcBuffer, size_t srcSize); -MEM_STATIC size_t BITv07_readBits(BITv07_DStream_t* bitD, unsigned nbBits); -MEM_STATIC BITv07_DStream_status BITv07_reloadDStream(BITv07_DStream_t* bitD); -MEM_STATIC unsigned BITv07_endOfDStream(const BITv07_DStream_t* bitD); - - -/* Start by invoking BITv07_initDStream(). -* A chunk of the bitStream is then stored into a local register. -* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t). -* You can then retrieve bitFields stored into the local register, **in reverse order**. -* Local register is explicitly reloaded from memory by the BITv07_reloadDStream() method. -* A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BITv07_DStream_unfinished. -* Otherwise, it can be less than that, so proceed accordingly. -* Checking if DStream has reached its end can be performed with BITv07_endOfDStream(). -*/ - - -/*-**************************************** -* unsafe API -******************************************/ -MEM_STATIC size_t BITv07_readBitsFast(BITv07_DStream_t* bitD, unsigned nbBits); -/* faster, but works only if nbBits >= 1 */ - - - -/*-************************************************************** -* Internal functions -****************************************************************/ -MEM_STATIC unsigned BITv07_highbit32 (register U32 val) -{ -# if defined(_MSC_VER) /* Visual */ - unsigned long r=0; - _BitScanReverse ( &r, val ); - return (unsigned) r; -# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */ - return 31 - __builtin_clz (val); -# else /* Software version */ - static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 }; - U32 v = val; - v |= v >> 1; - v |= v >> 2; - v |= v >> 4; - v |= v >> 8; - v |= v >> 16; - return DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27]; -# endif -} - -/*===== Local Constants =====*/ -static const unsigned BITv07_mask[] = { 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF, 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF }; /* up to 26 bits */ - - -/*-******************************************************** -* bitStream decoding -**********************************************************/ -/*! BITv07_initDStream() : -* Initialize a BITv07_DStream_t. -* `bitD` : a pointer to an already allocated BITv07_DStream_t structure. -* `srcSize` must be the *exact* size of the bitStream, in bytes. -* @return : size of stream (== srcSize) or an errorCode if a problem is detected -*/ -MEM_STATIC size_t BITv07_initDStream(BITv07_DStream_t* bitD, const void* srcBuffer, size_t srcSize) -{ - if (srcSize < 1) { memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); } - - if (srcSize >= sizeof(bitD->bitContainer)) { /* normal case */ - bitD->start = (const char*)srcBuffer; - bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer); - bitD->bitContainer = MEM_readLEST(bitD->ptr); - { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; - bitD->bitsConsumed = lastByte ? 8 - BITv07_highbit32(lastByte) : 0; - if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ } - } else { - bitD->start = (const char*)srcBuffer; - bitD->ptr = bitD->start; - bitD->bitContainer = *(const BYTE*)(bitD->start); - switch(srcSize) - { - case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16); - case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24); - case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32); - case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24; - case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16; - case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8; - default:; - } - { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; - bitD->bitsConsumed = lastByte ? 8 - BITv07_highbit32(lastByte) : 0; - if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ } - bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize)*8; - } - - return srcSize; -} - -MEM_STATIC size_t BITv07_getUpperBits(size_t bitContainer, U32 const start) -{ - return bitContainer >> start; -} - -MEM_STATIC size_t BITv07_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits) -{ -#if defined(__BMI__) && defined(__GNUC__) /* experimental */ -# if defined(__x86_64__) - if (sizeof(bitContainer)==8) - return _bextr_u64(bitContainer, start, nbBits); - else -# endif - return _bextr_u32(bitContainer, start, nbBits); -#else - return (bitContainer >> start) & BITv07_mask[nbBits]; -#endif -} - -MEM_STATIC size_t BITv07_getLowerBits(size_t bitContainer, U32 const nbBits) -{ - return bitContainer & BITv07_mask[nbBits]; -} - -/*! BITv07_lookBits() : - * Provides next n bits from local register. - * local register is not modified. - * On 32-bits, maxNbBits==24. - * On 64-bits, maxNbBits==56. - * @return : value extracted - */ - MEM_STATIC size_t BITv07_lookBits(const BITv07_DStream_t* bitD, U32 nbBits) -{ -#if defined(__BMI__) && defined(__GNUC__) /* experimental; fails if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8 */ - return BITv07_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits); -#else - U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1; - return ((bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> 1) >> ((bitMask-nbBits) & bitMask); -#endif -} - -/*! BITv07_lookBitsFast() : -* unsafe version; only works only if nbBits >= 1 */ -MEM_STATIC size_t BITv07_lookBitsFast(const BITv07_DStream_t* bitD, U32 nbBits) -{ - U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1; - return (bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> (((bitMask+1)-nbBits) & bitMask); -} - -MEM_STATIC void BITv07_skipBits(BITv07_DStream_t* bitD, U32 nbBits) -{ - bitD->bitsConsumed += nbBits; -} - -/*! BITv07_readBits() : - * Read (consume) next n bits from local register and update. - * Pay attention to not read more than nbBits contained into local register. - * @return : extracted value. - */ -MEM_STATIC size_t BITv07_readBits(BITv07_DStream_t* bitD, U32 nbBits) -{ - size_t const value = BITv07_lookBits(bitD, nbBits); - BITv07_skipBits(bitD, nbBits); - return value; -} - -/*! BITv07_readBitsFast() : -* unsafe version; only works only if nbBits >= 1 */ -MEM_STATIC size_t BITv07_readBitsFast(BITv07_DStream_t* bitD, U32 nbBits) -{ - size_t const value = BITv07_lookBitsFast(bitD, nbBits); - BITv07_skipBits(bitD, nbBits); - return value; -} - -/*! BITv07_reloadDStream() : -* Refill `BITv07_DStream_t` from src buffer previously defined (see BITv07_initDStream() ). -* This function is safe, it guarantees it will not read beyond src buffer. -* @return : status of `BITv07_DStream_t` internal register. - if status == unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */ -MEM_STATIC BITv07_DStream_status BITv07_reloadDStream(BITv07_DStream_t* bitD) -{ - if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should not happen => corruption detected */ - return BITv07_DStream_overflow; - - if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer)) { - bitD->ptr -= bitD->bitsConsumed >> 3; - bitD->bitsConsumed &= 7; - bitD->bitContainer = MEM_readLEST(bitD->ptr); - return BITv07_DStream_unfinished; - } - if (bitD->ptr == bitD->start) { - if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return BITv07_DStream_endOfBuffer; - return BITv07_DStream_completed; - } - { U32 nbBytes = bitD->bitsConsumed >> 3; - BITv07_DStream_status result = BITv07_DStream_unfinished; - if (bitD->ptr - nbBytes < bitD->start) { - nbBytes = (U32)(bitD->ptr - bitD->start); /* ptr > start */ - result = BITv07_DStream_endOfBuffer; - } - bitD->ptr -= nbBytes; - bitD->bitsConsumed -= nbBytes*8; - bitD->bitContainer = MEM_readLEST(bitD->ptr); /* reminder : srcSize > sizeof(bitD) */ - return result; - } -} - -/*! BITv07_endOfDStream() : -* @return Tells if DStream has exactly reached its end (all bits consumed). -*/ -MEM_STATIC unsigned BITv07_endOfDStream(const BITv07_DStream_t* DStream) -{ - return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer)*8)); -} - -#if defined (__cplusplus) -} -#endif - -#endif /* BITSTREAM_H_MODULE */ -/* ****************************************************************** - FSE : Finite State Entropy codec - Public Prototypes declaration - Copyright (C) 2013-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Source repository : https://github.com/Cyan4973/FiniteStateEntropy -****************************************************************** */ -#ifndef FSEv07_H -#define FSEv07_H - -#if defined (__cplusplus) -extern "C" { -#endif - - - -/*-**************************************** -* FSE simple functions -******************************************/ - -/*! FSEv07_decompress(): - Decompress FSE data from buffer 'cSrc', of size 'cSrcSize', - into already allocated destination buffer 'dst', of size 'dstCapacity'. - @return : size of regenerated data (<= maxDstSize), - or an error code, which can be tested using FSEv07_isError() . - - ** Important ** : FSEv07_decompress() does not decompress non-compressible nor RLE data !!! - Why ? : making this distinction requires a header. - Header management is intentionally delegated to the user layer, which can better manage special cases. -*/ -size_t FSEv07_decompress(void* dst, size_t dstCapacity, - const void* cSrc, size_t cSrcSize); - - -/* Error Management */ -unsigned FSEv07_isError(size_t code); /* tells if a return value is an error code */ -const char* FSEv07_getErrorName(size_t code); /* provides error code string (useful for debugging) */ - - -/*-***************************************** -* FSE detailed API -******************************************/ -/*! -FSEv07_decompress() does the following: -1. read normalized counters with readNCount() -2. build decoding table 'DTable' from normalized counters -3. decode the data stream using decoding table 'DTable' - -The following API allows targeting specific sub-functions for advanced tasks. -For example, it's possible to compress several blocks using the same 'CTable', -or to save and provide normalized distribution using external method. -*/ - - -/* *** DECOMPRESSION *** */ - -/*! FSEv07_readNCount(): - Read compactly saved 'normalizedCounter' from 'rBuffer'. - @return : size read from 'rBuffer', - or an errorCode, which can be tested using FSEv07_isError(). - maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */ -size_t FSEv07_readNCount (short* normalizedCounter, unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, const void* rBuffer, size_t rBuffSize); - -/*! Constructor and Destructor of FSEv07_DTable. - Note that its size depends on 'tableLog' */ -typedef unsigned FSEv07_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */ -FSEv07_DTable* FSEv07_createDTable(unsigned tableLog); -void FSEv07_freeDTable(FSEv07_DTable* dt); - -/*! FSEv07_buildDTable(): - Builds 'dt', which must be already allocated, using FSEv07_createDTable(). - return : 0, or an errorCode, which can be tested using FSEv07_isError() */ -size_t FSEv07_buildDTable (FSEv07_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); - -/*! FSEv07_decompress_usingDTable(): - Decompress compressed source `cSrc` of size `cSrcSize` using `dt` - into `dst` which must be already allocated. - @return : size of regenerated data (necessarily <= `dstCapacity`), - or an errorCode, which can be tested using FSEv07_isError() */ -size_t FSEv07_decompress_usingDTable(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, const FSEv07_DTable* dt); - -/*! -Tutorial : ----------- -(Note : these functions only decompress FSE-compressed blocks. - If block is uncompressed, use memcpy() instead - If block is a single repeated byte, use memset() instead ) - -The first step is to obtain the normalized frequencies of symbols. -This can be performed by FSEv07_readNCount() if it was saved using FSEv07_writeNCount(). -'normalizedCounter' must be already allocated, and have at least 'maxSymbolValuePtr[0]+1' cells of signed short. -In practice, that means it's necessary to know 'maxSymbolValue' beforehand, -or size the table to handle worst case situations (typically 256). -FSEv07_readNCount() will provide 'tableLog' and 'maxSymbolValue'. -The result of FSEv07_readNCount() is the number of bytes read from 'rBuffer'. -Note that 'rBufferSize' must be at least 4 bytes, even if useful information is less than that. -If there is an error, the function will return an error code, which can be tested using FSEv07_isError(). - -The next step is to build the decompression tables 'FSEv07_DTable' from 'normalizedCounter'. -This is performed by the function FSEv07_buildDTable(). -The space required by 'FSEv07_DTable' must be already allocated using FSEv07_createDTable(). -If there is an error, the function will return an error code, which can be tested using FSEv07_isError(). - -`FSEv07_DTable` can then be used to decompress `cSrc`, with FSEv07_decompress_usingDTable(). -`cSrcSize` must be strictly correct, otherwise decompression will fail. -FSEv07_decompress_usingDTable() result will tell how many bytes were regenerated (<=`dstCapacity`). -If there is an error, the function will return an error code, which can be tested using FSEv07_isError(). (ex: dst buffer too small) -*/ - - -#ifdef FSEv07_STATIC_LINKING_ONLY - - -/* ***************************************** -* Static allocation -*******************************************/ -/* FSE buffer bounds */ -#define FSEv07_NCOUNTBOUND 512 -#define FSEv07_BLOCKBOUND(size) (size + (size>>7)) - -/* It is possible to statically allocate FSE CTable/DTable as a table of unsigned using below macros */ -#define FSEv07_DTABLE_SIZE_U32(maxTableLog) (1 + (1<<maxTableLog)) - - -/* ***************************************** -* FSE advanced API -*******************************************/ -size_t FSEv07_countFast(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize); -/**< same as FSEv07_count(), but blindly trusts that all byte values within src are <= *maxSymbolValuePtr */ - -unsigned FSEv07_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus); -/**< same as FSEv07_optimalTableLog(), which used `minus==2` */ - -size_t FSEv07_buildDTable_raw (FSEv07_DTable* dt, unsigned nbBits); -/**< build a fake FSEv07_DTable, designed to read an uncompressed bitstream where each symbol uses nbBits */ - -size_t FSEv07_buildDTable_rle (FSEv07_DTable* dt, unsigned char symbolValue); -/**< build a fake FSEv07_DTable, designed to always generate the same symbolValue */ - - - -/* ***************************************** -* FSE symbol decompression API -*******************************************/ -typedef struct -{ - size_t state; - const void* table; /* precise table may vary, depending on U16 */ -} FSEv07_DState_t; - - -static void FSEv07_initDState(FSEv07_DState_t* DStatePtr, BITv07_DStream_t* bitD, const FSEv07_DTable* dt); - -static unsigned char FSEv07_decodeSymbol(FSEv07_DState_t* DStatePtr, BITv07_DStream_t* bitD); - -static unsigned FSEv07_endOfDState(const FSEv07_DState_t* DStatePtr); - -/**< -Let's now decompose FSEv07_decompress_usingDTable() into its unitary components. -You will decode FSE-encoded symbols from the bitStream, -and also any other bitFields you put in, **in reverse order**. - -You will need a few variables to track your bitStream. They are : - -BITv07_DStream_t DStream; // Stream context -FSEv07_DState_t DState; // State context. Multiple ones are possible -FSEv07_DTable* DTablePtr; // Decoding table, provided by FSEv07_buildDTable() - -The first thing to do is to init the bitStream. - errorCode = BITv07_initDStream(&DStream, srcBuffer, srcSize); - -You should then retrieve your initial state(s) -(in reverse flushing order if you have several ones) : - errorCode = FSEv07_initDState(&DState, &DStream, DTablePtr); - -You can then decode your data, symbol after symbol. -For information the maximum number of bits read by FSEv07_decodeSymbol() is 'tableLog'. -Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out). - unsigned char symbol = FSEv07_decodeSymbol(&DState, &DStream); - -You can retrieve any bitfield you eventually stored into the bitStream (in reverse order) -Note : maximum allowed nbBits is 25, for 32-bits compatibility - size_t bitField = BITv07_readBits(&DStream, nbBits); - -All above operations only read from local register (which size depends on size_t). -Refueling the register from memory is manually performed by the reload method. - endSignal = FSEv07_reloadDStream(&DStream); - -BITv07_reloadDStream() result tells if there is still some more data to read from DStream. -BITv07_DStream_unfinished : there is still some data left into the DStream. -BITv07_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled. -BITv07_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed. -BITv07_DStream_tooFar : Dstream went too far. Decompression result is corrupted. - -When reaching end of buffer (BITv07_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop, -to properly detect the exact end of stream. -After each decoded symbol, check if DStream is fully consumed using this simple test : - BITv07_reloadDStream(&DStream) >= BITv07_DStream_completed - -When it's done, verify decompression is fully completed, by checking both DStream and the relevant states. -Checking if DStream has reached its end is performed by : - BITv07_endOfDStream(&DStream); -Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible. - FSEv07_endOfDState(&DState); -*/ - - -/* ***************************************** -* FSE unsafe API -*******************************************/ -static unsigned char FSEv07_decodeSymbolFast(FSEv07_DState_t* DStatePtr, BITv07_DStream_t* bitD); -/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */ - - -/*<===== Decompression =====>*/ - -typedef struct { - U16 tableLog; - U16 fastMode; -} FSEv07_DTableHeader; /* sizeof U32 */ - -typedef struct -{ - unsigned short newState; - unsigned char symbol; - unsigned char nbBits; -} FSEv07_decode_t; /* size == U32 */ - -MEM_STATIC void FSEv07_initDState(FSEv07_DState_t* DStatePtr, BITv07_DStream_t* bitD, const FSEv07_DTable* dt) -{ - const void* ptr = dt; - const FSEv07_DTableHeader* const DTableH = (const FSEv07_DTableHeader*)ptr; - DStatePtr->state = BITv07_readBits(bitD, DTableH->tableLog); - BITv07_reloadDStream(bitD); - DStatePtr->table = dt + 1; -} - -MEM_STATIC BYTE FSEv07_peekSymbol(const FSEv07_DState_t* DStatePtr) -{ - FSEv07_decode_t const DInfo = ((const FSEv07_decode_t*)(DStatePtr->table))[DStatePtr->state]; - return DInfo.symbol; -} - -MEM_STATIC void FSEv07_updateState(FSEv07_DState_t* DStatePtr, BITv07_DStream_t* bitD) -{ - FSEv07_decode_t const DInfo = ((const FSEv07_decode_t*)(DStatePtr->table))[DStatePtr->state]; - U32 const nbBits = DInfo.nbBits; - size_t const lowBits = BITv07_readBits(bitD, nbBits); - DStatePtr->state = DInfo.newState + lowBits; -} - -MEM_STATIC BYTE FSEv07_decodeSymbol(FSEv07_DState_t* DStatePtr, BITv07_DStream_t* bitD) -{ - FSEv07_decode_t const DInfo = ((const FSEv07_decode_t*)(DStatePtr->table))[DStatePtr->state]; - U32 const nbBits = DInfo.nbBits; - BYTE const symbol = DInfo.symbol; - size_t const lowBits = BITv07_readBits(bitD, nbBits); - - DStatePtr->state = DInfo.newState + lowBits; - return symbol; -} - -/*! FSEv07_decodeSymbolFast() : - unsafe, only works if no symbol has a probability > 50% */ -MEM_STATIC BYTE FSEv07_decodeSymbolFast(FSEv07_DState_t* DStatePtr, BITv07_DStream_t* bitD) -{ - FSEv07_decode_t const DInfo = ((const FSEv07_decode_t*)(DStatePtr->table))[DStatePtr->state]; - U32 const nbBits = DInfo.nbBits; - BYTE const symbol = DInfo.symbol; - size_t const lowBits = BITv07_readBitsFast(bitD, nbBits); - - DStatePtr->state = DInfo.newState + lowBits; - return symbol; -} - -MEM_STATIC unsigned FSEv07_endOfDState(const FSEv07_DState_t* DStatePtr) -{ - return DStatePtr->state == 0; -} - - - -#ifndef FSEv07_COMMONDEFS_ONLY - -/* ************************************************************** -* Tuning parameters -****************************************************************/ -/*!MEMORY_USAGE : -* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) -* Increasing memory usage improves compression ratio -* Reduced memory usage can improve speed, due to cache effect -* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ -#define FSEv07_MAX_MEMORY_USAGE 14 -#define FSEv07_DEFAULT_MEMORY_USAGE 13 - -/*!FSEv07_MAX_SYMBOL_VALUE : -* Maximum symbol value authorized. -* Required for proper stack allocation */ -#define FSEv07_MAX_SYMBOL_VALUE 255 - - -/* ************************************************************** -* template functions type & suffix -****************************************************************/ -#define FSEv07_FUNCTION_TYPE BYTE -#define FSEv07_FUNCTION_EXTENSION -#define FSEv07_DECODE_TYPE FSEv07_decode_t - - -#endif /* !FSEv07_COMMONDEFS_ONLY */ - - -/* *************************************************************** -* Constants -*****************************************************************/ -#define FSEv07_MAX_TABLELOG (FSEv07_MAX_MEMORY_USAGE-2) -#define FSEv07_MAX_TABLESIZE (1U<<FSEv07_MAX_TABLELOG) -#define FSEv07_MAXTABLESIZE_MASK (FSEv07_MAX_TABLESIZE-1) -#define FSEv07_DEFAULT_TABLELOG (FSEv07_DEFAULT_MEMORY_USAGE-2) -#define FSEv07_MIN_TABLELOG 5 - -#define FSEv07_TABLELOG_ABSOLUTE_MAX 15 -#if FSEv07_MAX_TABLELOG > FSEv07_TABLELOG_ABSOLUTE_MAX -# error "FSEv07_MAX_TABLELOG > FSEv07_TABLELOG_ABSOLUTE_MAX is not supported" -#endif - -#define FSEv07_TABLESTEP(tableSize) ((tableSize>>1) + (tableSize>>3) + 3) - - -#endif /* FSEv07_STATIC_LINKING_ONLY */ - - -#if defined (__cplusplus) -} -#endif - -#endif /* FSEv07_H */ -/* ****************************************************************** - Huffman coder, part of New Generation Entropy library - header file - Copyright (C) 2013-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Source repository : https://github.com/Cyan4973/FiniteStateEntropy -****************************************************************** */ -#ifndef HUFv07_H_298734234 -#define HUFv07_H_298734234 - -#if defined (__cplusplus) -extern "C" { -#endif - - - -/* *** simple functions *** */ -/** -HUFv07_decompress() : - Decompress HUF data from buffer 'cSrc', of size 'cSrcSize', - into already allocated buffer 'dst', of minimum size 'dstSize'. - `dstSize` : **must** be the ***exact*** size of original (uncompressed) data. - Note : in contrast with FSE, HUFv07_decompress can regenerate - RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data, - because it knows size to regenerate. - @return : size of regenerated data (== dstSize), - or an error code, which can be tested using HUFv07_isError() -*/ -size_t HUFv07_decompress(void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize); - - -/* **************************************** -* Tool functions -******************************************/ -#define HUFv07_BLOCKSIZE_MAX (128 * 1024) - -/* Error Management */ -unsigned HUFv07_isError(size_t code); /**< tells if a return value is an error code */ -const char* HUFv07_getErrorName(size_t code); /**< provides error code string (useful for debugging) */ - - -/* *** Advanced function *** */ - - -#ifdef HUFv07_STATIC_LINKING_ONLY - - -/* *** Constants *** */ -#define HUFv07_TABLELOG_ABSOLUTEMAX 16 /* absolute limit of HUFv07_MAX_TABLELOG. Beyond that value, code does not work */ -#define HUFv07_TABLELOG_MAX 12 /* max configured tableLog (for static allocation); can be modified up to HUFv07_ABSOLUTEMAX_TABLELOG */ -#define HUFv07_TABLELOG_DEFAULT 11 /* tableLog by default, when not specified */ -#define HUFv07_SYMBOLVALUE_MAX 255 -#if (HUFv07_TABLELOG_MAX > HUFv07_TABLELOG_ABSOLUTEMAX) -# error "HUFv07_TABLELOG_MAX is too large !" -#endif - - -/* **************************************** -* Static allocation -******************************************/ -/* HUF buffer bounds */ -#define HUFv07_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true if incompressible pre-filtered with fast heuristic */ - -/* static allocation of HUF's DTable */ -typedef U32 HUFv07_DTable; -#define HUFv07_DTABLE_SIZE(maxTableLog) (1 + (1<<(maxTableLog))) -#define HUFv07_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \ - HUFv07_DTable DTable[HUFv07_DTABLE_SIZE((maxTableLog)-1)] = { ((U32)((maxTableLog)-1)*0x1000001) } -#define HUFv07_CREATE_STATIC_DTABLEX4(DTable, maxTableLog) \ - HUFv07_DTable DTable[HUFv07_DTABLE_SIZE(maxTableLog)] = { ((U32)(maxTableLog)*0x1000001) } - - -/* **************************************** -* Advanced decompression functions -******************************************/ -size_t HUFv07_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ -size_t HUFv07_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ - -size_t HUFv07_decompress4X_DCtx (HUFv07_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< decodes RLE and uncompressed */ -size_t HUFv07_decompress4X_hufOnly(HUFv07_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */ -size_t HUFv07_decompress4X2_DCtx(HUFv07_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ -size_t HUFv07_decompress4X4_DCtx(HUFv07_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ - -size_t HUFv07_decompress1X_DCtx (HUFv07_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); -size_t HUFv07_decompress1X2_DCtx(HUFv07_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ -size_t HUFv07_decompress1X4_DCtx(HUFv07_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ - - -/* **************************************** -* HUF detailed API -******************************************/ -/*! -The following API allows targeting specific sub-functions for advanced tasks. -For example, it's possible to compress several blocks using the same 'CTable', -or to save and regenerate 'CTable' using external methods. -*/ -/* FSEv07_count() : find it within "fse.h" */ - -/*! HUFv07_readStats() : - Read compact Huffman tree, saved by HUFv07_writeCTable(). - `huffWeight` is destination buffer. - @return : size read from `src` , or an error Code . - Note : Needed by HUFv07_readCTable() and HUFv07_readDTableXn() . */ -size_t HUFv07_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, - U32* nbSymbolsPtr, U32* tableLogPtr, - const void* src, size_t srcSize); - - -/* -HUFv07_decompress() does the following: -1. select the decompression algorithm (X2, X4) based on pre-computed heuristics -2. build Huffman table from save, using HUFv07_readDTableXn() -3. decode 1 or 4 segments in parallel using HUFv07_decompressSXn_usingDTable -*/ - -/** HUFv07_selectDecoder() : -* Tells which decoder is likely to decode faster, -* based on a set of pre-determined metrics. -* @return : 0==HUFv07_decompress4X2, 1==HUFv07_decompress4X4 . -* Assumption : 0 < cSrcSize < dstSize <= 128 KB */ -U32 HUFv07_selectDecoder (size_t dstSize, size_t cSrcSize); - -size_t HUFv07_readDTableX2 (HUFv07_DTable* DTable, const void* src, size_t srcSize); -size_t HUFv07_readDTableX4 (HUFv07_DTable* DTable, const void* src, size_t srcSize); - -size_t HUFv07_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUFv07_DTable* DTable); -size_t HUFv07_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUFv07_DTable* DTable); -size_t HUFv07_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUFv07_DTable* DTable); - - -/* single stream variants */ -size_t HUFv07_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */ -size_t HUFv07_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */ - -size_t HUFv07_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUFv07_DTable* DTable); -size_t HUFv07_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUFv07_DTable* DTable); -size_t HUFv07_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUFv07_DTable* DTable); - - -#endif /* HUFv07_STATIC_LINKING_ONLY */ - - -#if defined (__cplusplus) -} -#endif - -#endif /* HUFv07_H_298734234 */ -/* - Common functions of New Generation Entropy library - Copyright (C) 2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy - - Public forum : https://groups.google.com/forum/#!forum/lz4c -*************************************************************************** */ - - - -/*-**************************************** -* FSE Error Management -******************************************/ -unsigned FSEv07_isError(size_t code) { return ERR_isError(code); } - -const char* FSEv07_getErrorName(size_t code) { return ERR_getErrorName(code); } - - -/* ************************************************************** -* HUF Error Management -****************************************************************/ -unsigned HUFv07_isError(size_t code) { return ERR_isError(code); } - -const char* HUFv07_getErrorName(size_t code) { return ERR_getErrorName(code); } - - -/*-************************************************************** -* FSE NCount encoding-decoding -****************************************************************/ -static short FSEv07_abs(short a) { return (short)(a<0 ? -a : a); } - -size_t FSEv07_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, - const void* headerBuffer, size_t hbSize) -{ - const BYTE* const istart = (const BYTE*) headerBuffer; - const BYTE* const iend = istart + hbSize; - const BYTE* ip = istart; - int nbBits; - int remaining; - int threshold; - U32 bitStream; - int bitCount; - unsigned charnum = 0; - int previous0 = 0; - - if (hbSize < 4) return ERROR(srcSize_wrong); - bitStream = MEM_readLE32(ip); - nbBits = (bitStream & 0xF) + FSEv07_MIN_TABLELOG; /* extract tableLog */ - if (nbBits > FSEv07_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge); - bitStream >>= 4; - bitCount = 4; - *tableLogPtr = nbBits; - remaining = (1<<nbBits)+1; - threshold = 1<<nbBits; - nbBits++; - - while ((remaining>1) && (charnum<=*maxSVPtr)) { - if (previous0) { - unsigned n0 = charnum; - while ((bitStream & 0xFFFF) == 0xFFFF) { - n0+=24; - if (ip < iend-5) { - ip+=2; - bitStream = MEM_readLE32(ip) >> bitCount; - } else { - bitStream >>= 16; - bitCount+=16; - } } - while ((bitStream & 3) == 3) { - n0+=3; - bitStream>>=2; - bitCount+=2; - } - n0 += bitStream & 3; - bitCount += 2; - if (n0 > *maxSVPtr) return ERROR(maxSymbolValue_tooSmall); - while (charnum < n0) normalizedCounter[charnum++] = 0; - if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { - ip += bitCount>>3; - bitCount &= 7; - bitStream = MEM_readLE32(ip) >> bitCount; - } - else - bitStream >>= 2; - } - { short const max = (short)((2*threshold-1)-remaining); - short count; - - if ((bitStream & (threshold-1)) < (U32)max) { - count = (short)(bitStream & (threshold-1)); - bitCount += nbBits-1; - } else { - count = (short)(bitStream & (2*threshold-1)); - if (count >= threshold) count -= max; - bitCount += nbBits; - } - - count--; /* extra accuracy */ - remaining -= FSEv07_abs(count); - normalizedCounter[charnum++] = count; - previous0 = !count; - while (remaining < threshold) { - nbBits--; - threshold >>= 1; - } - - if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { - ip += bitCount>>3; - bitCount &= 7; - } else { - bitCount -= (int)(8 * (iend - 4 - ip)); - ip = iend - 4; - } - bitStream = MEM_readLE32(ip) >> (bitCount & 31); - } } /* while ((remaining>1) && (charnum<=*maxSVPtr)) */ - if (remaining != 1) return ERROR(GENERIC); - *maxSVPtr = charnum-1; - - ip += (bitCount+7)>>3; - if ((size_t)(ip-istart) > hbSize) return ERROR(srcSize_wrong); - return ip-istart; -} - - -/*! HUFv07_readStats() : - Read compact Huffman tree, saved by HUFv07_writeCTable(). - `huffWeight` is destination buffer. - @return : size read from `src` , or an error Code . - Note : Needed by HUFv07_readCTable() and HUFv07_readDTableXn() . -*/ -size_t HUFv07_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, - U32* nbSymbolsPtr, U32* tableLogPtr, - const void* src, size_t srcSize) -{ - U32 weightTotal; - const BYTE* ip = (const BYTE*) src; - size_t iSize = ip[0]; - size_t oSize; - - //memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */ - - if (iSize >= 128) { /* special header */ - if (iSize >= (242)) { /* RLE */ - static U32 l[14] = { 1, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128 }; - oSize = l[iSize-242]; - memset(huffWeight, 1, hwSize); - iSize = 0; - } - else { /* Incompressible */ - oSize = iSize - 127; - iSize = ((oSize+1)/2); - if (iSize+1 > srcSize) return ERROR(srcSize_wrong); - if (oSize >= hwSize) return ERROR(corruption_detected); - ip += 1; - { U32 n; - for (n=0; n<oSize; n+=2) { - huffWeight[n] = ip[n/2] >> 4; - huffWeight[n+1] = ip[n/2] & 15; - } } } } - else { /* header compressed with FSE (normal case) */ - if (iSize+1 > srcSize) return ERROR(srcSize_wrong); - oSize = FSEv07_decompress(huffWeight, hwSize-1, ip+1, iSize); /* max (hwSize-1) values decoded, as last one is implied */ - if (FSEv07_isError(oSize)) return oSize; - } - - /* collect weight stats */ - memset(rankStats, 0, (HUFv07_TABLELOG_ABSOLUTEMAX + 1) * sizeof(U32)); - weightTotal = 0; - { U32 n; for (n=0; n<oSize; n++) { - if (huffWeight[n] >= HUFv07_TABLELOG_ABSOLUTEMAX) return ERROR(corruption_detected); - rankStats[huffWeight[n]]++; - weightTotal += (1 << huffWeight[n]) >> 1; - } } - - /* get last non-null symbol weight (implied, total must be 2^n) */ - { U32 const tableLog = BITv07_highbit32(weightTotal) + 1; - if (tableLog > HUFv07_TABLELOG_ABSOLUTEMAX) return ERROR(corruption_detected); - *tableLogPtr = tableLog; - /* determine last weight */ - { U32 const total = 1 << tableLog; - U32 const rest = total - weightTotal; - U32 const verif = 1 << BITv07_highbit32(rest); - U32 const lastWeight = BITv07_highbit32(rest) + 1; - if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */ - huffWeight[oSize] = (BYTE)lastWeight; - rankStats[lastWeight]++; - } } - - /* check tree construction validity */ - if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */ - - /* results */ - *nbSymbolsPtr = (U32)(oSize+1); - return iSize+1; -} -/* ****************************************************************** - FSE : Finite State Entropy decoder - Copyright (C) 2013-2015, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy - - Public forum : https://groups.google.com/forum/#!forum/lz4c -****************************************************************** */ - - -/* ************************************************************** -* Compiler specifics -****************************************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# define FORCE_INLINE static __forceinline -# include <intrin.h> /* For Visual 2005 */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */ -#else -# ifdef __GNUC__ -# define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -# define FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define FORCE_INLINE static inline -# endif -#endif - - - - -/* ************************************************************** -* Error Management -****************************************************************/ -#define FSEv07_isError ERR_isError -#define FSEv07_STATIC_ASSERT(c) { enum { FSEv07_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ - - -/* ************************************************************** -* Complex types -****************************************************************/ -typedef U32 DTable_max_t[FSEv07_DTABLE_SIZE_U32(FSEv07_MAX_TABLELOG)]; - - -/* ************************************************************** -* Templates -****************************************************************/ -/* - designed to be included - for type-specific functions (template emulation in C) - Objective is to write these functions only once, for improved maintenance -*/ - -/* safety checks */ -#ifndef FSEv07_FUNCTION_EXTENSION -# error "FSEv07_FUNCTION_EXTENSION must be defined" -#endif -#ifndef FSEv07_FUNCTION_TYPE -# error "FSEv07_FUNCTION_TYPE must be defined" -#endif - -/* Function names */ -#define FSEv07_CAT(X,Y) X##Y -#define FSEv07_FUNCTION_NAME(X,Y) FSEv07_CAT(X,Y) -#define FSEv07_TYPE_NAME(X,Y) FSEv07_CAT(X,Y) - - -/* Function templates */ -FSEv07_DTable* FSEv07_createDTable (unsigned tableLog) -{ - if (tableLog > FSEv07_TABLELOG_ABSOLUTE_MAX) tableLog = FSEv07_TABLELOG_ABSOLUTE_MAX; - return (FSEv07_DTable*)malloc( FSEv07_DTABLE_SIZE_U32(tableLog) * sizeof (U32) ); -} - -void FSEv07_freeDTable (FSEv07_DTable* dt) -{ - free(dt); -} - -size_t FSEv07_buildDTable(FSEv07_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) -{ - void* const tdPtr = dt+1; /* because *dt is unsigned, 32-bits aligned on 32-bits */ - FSEv07_DECODE_TYPE* const tableDecode = (FSEv07_DECODE_TYPE*) (tdPtr); - U16 symbolNext[FSEv07_MAX_SYMBOL_VALUE+1]; - - U32 const maxSV1 = maxSymbolValue + 1; - U32 const tableSize = 1 << tableLog; - U32 highThreshold = tableSize-1; - - /* Sanity Checks */ - if (maxSymbolValue > FSEv07_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge); - if (tableLog > FSEv07_MAX_TABLELOG) return ERROR(tableLog_tooLarge); - - /* Init, lay down lowprob symbols */ - { FSEv07_DTableHeader DTableH; - DTableH.tableLog = (U16)tableLog; - DTableH.fastMode = 1; - { S16 const largeLimit= (S16)(1 << (tableLog-1)); - U32 s; - for (s=0; s<maxSV1; s++) { - if (normalizedCounter[s]==-1) { - tableDecode[highThreshold--].symbol = (FSEv07_FUNCTION_TYPE)s; - symbolNext[s] = 1; - } else { - if (normalizedCounter[s] >= largeLimit) DTableH.fastMode=0; - symbolNext[s] = normalizedCounter[s]; - } } } - memcpy(dt, &DTableH, sizeof(DTableH)); - } - - /* Spread symbols */ - { U32 const tableMask = tableSize-1; - U32 const step = FSEv07_TABLESTEP(tableSize); - U32 s, position = 0; - for (s=0; s<maxSV1; s++) { - int i; - for (i=0; i<normalizedCounter[s]; i++) { - tableDecode[position].symbol = (FSEv07_FUNCTION_TYPE)s; - position = (position + step) & tableMask; - while (position > highThreshold) position = (position + step) & tableMask; /* lowprob area */ - } } - - if (position!=0) return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ - } - - /* Build Decoding table */ - { U32 u; - for (u=0; u<tableSize; u++) { - FSEv07_FUNCTION_TYPE const symbol = (FSEv07_FUNCTION_TYPE)(tableDecode[u].symbol); - U16 nextState = symbolNext[symbol]++; - tableDecode[u].nbBits = (BYTE) (tableLog - BITv07_highbit32 ((U32)nextState) ); - tableDecode[u].newState = (U16) ( (nextState << tableDecode[u].nbBits) - tableSize); - } } - - return 0; -} - - - -#ifndef FSEv07_COMMONDEFS_ONLY - -/*-******************************************************* -* Decompression (Byte symbols) -*********************************************************/ -size_t FSEv07_buildDTable_rle (FSEv07_DTable* dt, BYTE symbolValue) -{ - void* ptr = dt; - FSEv07_DTableHeader* const DTableH = (FSEv07_DTableHeader*)ptr; - void* dPtr = dt + 1; - FSEv07_decode_t* const cell = (FSEv07_decode_t*)dPtr; - - DTableH->tableLog = 0; - DTableH->fastMode = 0; - - cell->newState = 0; - cell->symbol = symbolValue; - cell->nbBits = 0; - - return 0; -} - - -size_t FSEv07_buildDTable_raw (FSEv07_DTable* dt, unsigned nbBits) -{ - void* ptr = dt; - FSEv07_DTableHeader* const DTableH = (FSEv07_DTableHeader*)ptr; - void* dPtr = dt + 1; - FSEv07_decode_t* const dinfo = (FSEv07_decode_t*)dPtr; - const unsigned tableSize = 1 << nbBits; - const unsigned tableMask = tableSize - 1; - const unsigned maxSV1 = tableMask+1; - unsigned s; - - /* Sanity checks */ - if (nbBits < 1) return ERROR(GENERIC); /* min size */ - - /* Build Decoding Table */ - DTableH->tableLog = (U16)nbBits; - DTableH->fastMode = 1; - for (s=0; s<maxSV1; s++) { - dinfo[s].newState = 0; - dinfo[s].symbol = (BYTE)s; - dinfo[s].nbBits = (BYTE)nbBits; - } - - return 0; -} - -FORCE_INLINE size_t FSEv07_decompress_usingDTable_generic( - void* dst, size_t maxDstSize, - const void* cSrc, size_t cSrcSize, - const FSEv07_DTable* dt, const unsigned fast) -{ - BYTE* const ostart = (BYTE*) dst; - BYTE* op = ostart; - BYTE* const omax = op + maxDstSize; - BYTE* const olimit = omax-3; - - BITv07_DStream_t bitD; - FSEv07_DState_t state1; - FSEv07_DState_t state2; - - /* Init */ - { size_t const errorCode = BITv07_initDStream(&bitD, cSrc, cSrcSize); /* replaced last arg by maxCompressed Size */ - if (FSEv07_isError(errorCode)) return errorCode; } - - FSEv07_initDState(&state1, &bitD, dt); - FSEv07_initDState(&state2, &bitD, dt); - -#define FSEv07_GETSYMBOL(statePtr) fast ? FSEv07_decodeSymbolFast(statePtr, &bitD) : FSEv07_decodeSymbol(statePtr, &bitD) - - /* 4 symbols per loop */ - for ( ; (BITv07_reloadDStream(&bitD)==BITv07_DStream_unfinished) && (op<olimit) ; op+=4) { - op[0] = FSEv07_GETSYMBOL(&state1); - - if (FSEv07_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ - BITv07_reloadDStream(&bitD); - - op[1] = FSEv07_GETSYMBOL(&state2); - - if (FSEv07_MAX_TABLELOG*4+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ - { if (BITv07_reloadDStream(&bitD) > BITv07_DStream_unfinished) { op+=2; break; } } - - op[2] = FSEv07_GETSYMBOL(&state1); - - if (FSEv07_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ - BITv07_reloadDStream(&bitD); - - op[3] = FSEv07_GETSYMBOL(&state2); - } - - /* tail */ - /* note : BITv07_reloadDStream(&bitD) >= FSEv07_DStream_partiallyFilled; Ends at exactly BITv07_DStream_completed */ - while (1) { - if (op>(omax-2)) return ERROR(dstSize_tooSmall); - - *op++ = FSEv07_GETSYMBOL(&state1); - - if (BITv07_reloadDStream(&bitD)==BITv07_DStream_overflow) { - *op++ = FSEv07_GETSYMBOL(&state2); - break; - } - - if (op>(omax-2)) return ERROR(dstSize_tooSmall); - - *op++ = FSEv07_GETSYMBOL(&state2); - - if (BITv07_reloadDStream(&bitD)==BITv07_DStream_overflow) { - *op++ = FSEv07_GETSYMBOL(&state1); - break; - } } - - return op-ostart; -} - - -size_t FSEv07_decompress_usingDTable(void* dst, size_t originalSize, - const void* cSrc, size_t cSrcSize, - const FSEv07_DTable* dt) -{ - const void* ptr = dt; - const FSEv07_DTableHeader* DTableH = (const FSEv07_DTableHeader*)ptr; - const U32 fastMode = DTableH->fastMode; - - /* select fast mode (static) */ - if (fastMode) return FSEv07_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1); - return FSEv07_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0); -} - - -size_t FSEv07_decompress(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize) -{ - const BYTE* const istart = (const BYTE*)cSrc; - const BYTE* ip = istart; - short counting[FSEv07_MAX_SYMBOL_VALUE+1]; - DTable_max_t dt; /* Static analyzer seems unable to understand this table will be properly initialized later */ - unsigned tableLog; - unsigned maxSymbolValue = FSEv07_MAX_SYMBOL_VALUE; - - if (cSrcSize<2) return ERROR(srcSize_wrong); /* too small input size */ - - /* normal FSE decoding mode */ - { size_t const NCountLength = FSEv07_readNCount (counting, &maxSymbolValue, &tableLog, istart, cSrcSize); - if (FSEv07_isError(NCountLength)) return NCountLength; - if (NCountLength >= cSrcSize) return ERROR(srcSize_wrong); /* too small input size */ - ip += NCountLength; - cSrcSize -= NCountLength; - } - - { size_t const errorCode = FSEv07_buildDTable (dt, counting, maxSymbolValue, tableLog); - if (FSEv07_isError(errorCode)) return errorCode; } - - return FSEv07_decompress_usingDTable (dst, maxDstSize, ip, cSrcSize, dt); /* always return, even if it is an error code */ -} - - - -#endif /* FSEv07_COMMONDEFS_ONLY */ - -/* ****************************************************************** - Huffman decoder, part of New Generation Entropy library - Copyright (C) 2013-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy - - Public forum : https://groups.google.com/forum/#!forum/lz4c -****************************************************************** */ - -/* ************************************************************** -* Compiler specifics -****************************************************************/ -#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -/* inline is defined */ -#elif defined(_MSC_VER) -# define inline __inline -#else -# define inline /* disable inline */ -#endif - - -#ifdef _MSC_VER /* Visual Studio */ -# define FORCE_INLINE static __forceinline -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -#else -# ifdef __GNUC__ -# define FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define FORCE_INLINE static inline -# endif -#endif - - - -/* ************************************************************** -* Error Management -****************************************************************/ -#define HUFv07_STATIC_ASSERT(c) { enum { HUFv07_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ - - -/*-***************************/ -/* generic DTableDesc */ -/*-***************************/ - -typedef struct { BYTE maxTableLog; BYTE tableType; BYTE tableLog; BYTE reserved; } DTableDesc; - -static DTableDesc HUFv07_getDTableDesc(const HUFv07_DTable* table) -{ - DTableDesc dtd; - memcpy(&dtd, table, sizeof(dtd)); - return dtd; -} - - -/*-***************************/ -/* single-symbol decoding */ -/*-***************************/ - -typedef struct { BYTE byte; BYTE nbBits; } HUFv07_DEltX2; /* single-symbol decoding */ - -size_t HUFv07_readDTableX2 (HUFv07_DTable* DTable, const void* src, size_t srcSize) -{ - BYTE huffWeight[HUFv07_SYMBOLVALUE_MAX + 1]; - U32 rankVal[HUFv07_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */ - U32 tableLog = 0; - U32 nbSymbols = 0; - size_t iSize; - void* const dtPtr = DTable + 1; - HUFv07_DEltX2* const dt = (HUFv07_DEltX2*)dtPtr; - - HUFv07_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUFv07_DTable)); - //memset(huffWeight, 0, sizeof(huffWeight)); /* is not necessary, even though some analyzer complain ... */ - - iSize = HUFv07_readStats(huffWeight, HUFv07_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize); - if (HUFv07_isError(iSize)) return iSize; - - /* Table header */ - { DTableDesc dtd = HUFv07_getDTableDesc(DTable); - if (tableLog > (U32)(dtd.maxTableLog+1)) return ERROR(tableLog_tooLarge); /* DTable too small, huffman tree cannot fit in */ - dtd.tableType = 0; - dtd.tableLog = (BYTE)tableLog; - memcpy(DTable, &dtd, sizeof(dtd)); - } - - /* Prepare ranks */ - { U32 n, nextRankStart = 0; - for (n=1; n<tableLog+1; n++) { - U32 current = nextRankStart; - nextRankStart += (rankVal[n] << (n-1)); - rankVal[n] = current; - } } - - /* fill DTable */ - { U32 n; - for (n=0; n<nbSymbols; n++) { - U32 const w = huffWeight[n]; - U32 const length = (1 << w) >> 1; - U32 i; - HUFv07_DEltX2 D; - D.byte = (BYTE)n; D.nbBits = (BYTE)(tableLog + 1 - w); - for (i = rankVal[w]; i < rankVal[w] + length; i++) - dt[i] = D; - rankVal[w] += length; - } } - - return iSize; -} - - -static BYTE HUFv07_decodeSymbolX2(BITv07_DStream_t* Dstream, const HUFv07_DEltX2* dt, const U32 dtLog) -{ - size_t const val = BITv07_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */ - BYTE const c = dt[val].byte; - BITv07_skipBits(Dstream, dt[val].nbBits); - return c; -} - -#define HUFv07_DECODE_SYMBOLX2_0(ptr, DStreamPtr) \ - *ptr++ = HUFv07_decodeSymbolX2(DStreamPtr, dt, dtLog) - -#define HUFv07_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \ - if (MEM_64bits() || (HUFv07_TABLELOG_MAX<=12)) \ - HUFv07_DECODE_SYMBOLX2_0(ptr, DStreamPtr) - -#define HUFv07_DECODE_SYMBOLX2_2(ptr, DStreamPtr) \ - if (MEM_64bits()) \ - HUFv07_DECODE_SYMBOLX2_0(ptr, DStreamPtr) - -static inline size_t HUFv07_decodeStreamX2(BYTE* p, BITv07_DStream_t* const bitDPtr, BYTE* const pEnd, const HUFv07_DEltX2* const dt, const U32 dtLog) -{ - BYTE* const pStart = p; - - /* up to 4 symbols at a time */ - while ((BITv07_reloadDStream(bitDPtr) == BITv07_DStream_unfinished) && (p <= pEnd-4)) { - HUFv07_DECODE_SYMBOLX2_2(p, bitDPtr); - HUFv07_DECODE_SYMBOLX2_1(p, bitDPtr); - HUFv07_DECODE_SYMBOLX2_2(p, bitDPtr); - HUFv07_DECODE_SYMBOLX2_0(p, bitDPtr); - } - - /* closer to the end */ - while ((BITv07_reloadDStream(bitDPtr) == BITv07_DStream_unfinished) && (p < pEnd)) - HUFv07_DECODE_SYMBOLX2_0(p, bitDPtr); - - /* no more data to retrieve from bitstream, hence no need to reload */ - while (p < pEnd) - HUFv07_DECODE_SYMBOLX2_0(p, bitDPtr); - - return pEnd-pStart; -} - -static size_t HUFv07_decompress1X2_usingDTable_internal( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUFv07_DTable* DTable) -{ - BYTE* op = (BYTE*)dst; - BYTE* const oend = op + dstSize; - const void* dtPtr = DTable + 1; - const HUFv07_DEltX2* const dt = (const HUFv07_DEltX2*)dtPtr; - BITv07_DStream_t bitD; - DTableDesc const dtd = HUFv07_getDTableDesc(DTable); - U32 const dtLog = dtd.tableLog; - - { size_t const errorCode = BITv07_initDStream(&bitD, cSrc, cSrcSize); - if (HUFv07_isError(errorCode)) return errorCode; } - - HUFv07_decodeStreamX2(op, &bitD, oend, dt, dtLog); - - /* check */ - if (!BITv07_endOfDStream(&bitD)) return ERROR(corruption_detected); - - return dstSize; -} - -size_t HUFv07_decompress1X2_usingDTable( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUFv07_DTable* DTable) -{ - DTableDesc dtd = HUFv07_getDTableDesc(DTable); - if (dtd.tableType != 0) return ERROR(GENERIC); - return HUFv07_decompress1X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); -} - -size_t HUFv07_decompress1X2_DCtx (HUFv07_DTable* DCtx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - const BYTE* ip = (const BYTE*) cSrc; - - size_t const hSize = HUFv07_readDTableX2 (DCtx, cSrc, cSrcSize); - if (HUFv07_isError(hSize)) return hSize; - if (hSize >= cSrcSize) return ERROR(srcSize_wrong); - ip += hSize; cSrcSize -= hSize; - - return HUFv07_decompress1X2_usingDTable_internal (dst, dstSize, ip, cSrcSize, DCtx); -} - -size_t HUFv07_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - HUFv07_CREATE_STATIC_DTABLEX2(DTable, HUFv07_TABLELOG_MAX); - return HUFv07_decompress1X2_DCtx (DTable, dst, dstSize, cSrc, cSrcSize); -} - - -static size_t HUFv07_decompress4X2_usingDTable_internal( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUFv07_DTable* DTable) -{ - /* Check */ - if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ - - { const BYTE* const istart = (const BYTE*) cSrc; - BYTE* const ostart = (BYTE*) dst; - BYTE* const oend = ostart + dstSize; - const void* const dtPtr = DTable + 1; - const HUFv07_DEltX2* const dt = (const HUFv07_DEltX2*)dtPtr; - - /* Init */ - BITv07_DStream_t bitD1; - BITv07_DStream_t bitD2; - BITv07_DStream_t bitD3; - BITv07_DStream_t bitD4; - size_t const length1 = MEM_readLE16(istart); - size_t const length2 = MEM_readLE16(istart+2); - size_t const length3 = MEM_readLE16(istart+4); - size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); - const BYTE* const istart1 = istart + 6; /* jumpTable */ - const BYTE* const istart2 = istart1 + length1; - const BYTE* const istart3 = istart2 + length2; - const BYTE* const istart4 = istart3 + length3; - const size_t segmentSize = (dstSize+3) / 4; - BYTE* const opStart2 = ostart + segmentSize; - BYTE* const opStart3 = opStart2 + segmentSize; - BYTE* const opStart4 = opStart3 + segmentSize; - BYTE* op1 = ostart; - BYTE* op2 = opStart2; - BYTE* op3 = opStart3; - BYTE* op4 = opStart4; - U32 endSignal; - DTableDesc const dtd = HUFv07_getDTableDesc(DTable); - U32 const dtLog = dtd.tableLog; - - if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ - { size_t const errorCode = BITv07_initDStream(&bitD1, istart1, length1); - if (HUFv07_isError(errorCode)) return errorCode; } - { size_t const errorCode = BITv07_initDStream(&bitD2, istart2, length2); - if (HUFv07_isError(errorCode)) return errorCode; } - { size_t const errorCode = BITv07_initDStream(&bitD3, istart3, length3); - if (HUFv07_isError(errorCode)) return errorCode; } - { size_t const errorCode = BITv07_initDStream(&bitD4, istart4, length4); - if (HUFv07_isError(errorCode)) return errorCode; } - - /* 16-32 symbols per loop (4-8 symbols per stream) */ - endSignal = BITv07_reloadDStream(&bitD1) | BITv07_reloadDStream(&bitD2) | BITv07_reloadDStream(&bitD3) | BITv07_reloadDStream(&bitD4); - for ( ; (endSignal==BITv07_DStream_unfinished) && (op4<(oend-7)) ; ) { - HUFv07_DECODE_SYMBOLX2_2(op1, &bitD1); - HUFv07_DECODE_SYMBOLX2_2(op2, &bitD2); - HUFv07_DECODE_SYMBOLX2_2(op3, &bitD3); - HUFv07_DECODE_SYMBOLX2_2(op4, &bitD4); - HUFv07_DECODE_SYMBOLX2_1(op1, &bitD1); - HUFv07_DECODE_SYMBOLX2_1(op2, &bitD2); - HUFv07_DECODE_SYMBOLX2_1(op3, &bitD3); - HUFv07_DECODE_SYMBOLX2_1(op4, &bitD4); - HUFv07_DECODE_SYMBOLX2_2(op1, &bitD1); - HUFv07_DECODE_SYMBOLX2_2(op2, &bitD2); - HUFv07_DECODE_SYMBOLX2_2(op3, &bitD3); - HUFv07_DECODE_SYMBOLX2_2(op4, &bitD4); - HUFv07_DECODE_SYMBOLX2_0(op1, &bitD1); - HUFv07_DECODE_SYMBOLX2_0(op2, &bitD2); - HUFv07_DECODE_SYMBOLX2_0(op3, &bitD3); - HUFv07_DECODE_SYMBOLX2_0(op4, &bitD4); - endSignal = BITv07_reloadDStream(&bitD1) | BITv07_reloadDStream(&bitD2) | BITv07_reloadDStream(&bitD3) | BITv07_reloadDStream(&bitD4); - } - - /* check corruption */ - if (op1 > opStart2) return ERROR(corruption_detected); - if (op2 > opStart3) return ERROR(corruption_detected); - if (op3 > opStart4) return ERROR(corruption_detected); - /* note : op4 supposed already verified within main loop */ - - /* finish bitStreams one by one */ - HUFv07_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog); - HUFv07_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog); - HUFv07_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog); - HUFv07_decodeStreamX2(op4, &bitD4, oend, dt, dtLog); - - /* check */ - endSignal = BITv07_endOfDStream(&bitD1) & BITv07_endOfDStream(&bitD2) & BITv07_endOfDStream(&bitD3) & BITv07_endOfDStream(&bitD4); - if (!endSignal) return ERROR(corruption_detected); - - /* decoded size */ - return dstSize; - } -} - - -size_t HUFv07_decompress4X2_usingDTable( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUFv07_DTable* DTable) -{ - DTableDesc dtd = HUFv07_getDTableDesc(DTable); - if (dtd.tableType != 0) return ERROR(GENERIC); - return HUFv07_decompress4X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); -} - - -size_t HUFv07_decompress4X2_DCtx (HUFv07_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - const BYTE* ip = (const BYTE*) cSrc; - - size_t const hSize = HUFv07_readDTableX2 (dctx, cSrc, cSrcSize); - if (HUFv07_isError(hSize)) return hSize; - if (hSize >= cSrcSize) return ERROR(srcSize_wrong); - ip += hSize; cSrcSize -= hSize; - - return HUFv07_decompress4X2_usingDTable_internal (dst, dstSize, ip, cSrcSize, dctx); -} - -size_t HUFv07_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - HUFv07_CREATE_STATIC_DTABLEX2(DTable, HUFv07_TABLELOG_MAX); - return HUFv07_decompress4X2_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); -} - - -/* *************************/ -/* double-symbols decoding */ -/* *************************/ -typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUFv07_DEltX4; /* double-symbols decoding */ - -typedef struct { BYTE symbol; BYTE weight; } sortedSymbol_t; - -static void HUFv07_fillDTableX4Level2(HUFv07_DEltX4* DTable, U32 sizeLog, const U32 consumed, - const U32* rankValOrigin, const int minWeight, - const sortedSymbol_t* sortedSymbols, const U32 sortedListSize, - U32 nbBitsBaseline, U16 baseSeq) -{ - HUFv07_DEltX4 DElt; - U32 rankVal[HUFv07_TABLELOG_ABSOLUTEMAX + 1]; - - /* get pre-calculated rankVal */ - memcpy(rankVal, rankValOrigin, sizeof(rankVal)); - - /* fill skipped values */ - if (minWeight>1) { - U32 i, skipSize = rankVal[minWeight]; - MEM_writeLE16(&(DElt.sequence), baseSeq); - DElt.nbBits = (BYTE)(consumed); - DElt.length = 1; - for (i = 0; i < skipSize; i++) - DTable[i] = DElt; - } - - /* fill DTable */ - { U32 s; for (s=0; s<sortedListSize; s++) { /* note : sortedSymbols already skipped */ - const U32 symbol = sortedSymbols[s].symbol; - const U32 weight = sortedSymbols[s].weight; - const U32 nbBits = nbBitsBaseline - weight; - const U32 length = 1 << (sizeLog-nbBits); - const U32 start = rankVal[weight]; - U32 i = start; - const U32 end = start + length; - - MEM_writeLE16(&(DElt.sequence), (U16)(baseSeq + (symbol << 8))); - DElt.nbBits = (BYTE)(nbBits + consumed); - DElt.length = 2; - do { DTable[i++] = DElt; } while (i<end); /* since length >= 1 */ - - rankVal[weight] += length; - }} -} - -typedef U32 rankVal_t[HUFv07_TABLELOG_ABSOLUTEMAX][HUFv07_TABLELOG_ABSOLUTEMAX + 1]; - -static void HUFv07_fillDTableX4(HUFv07_DEltX4* DTable, const U32 targetLog, - const sortedSymbol_t* sortedList, const U32 sortedListSize, - const U32* rankStart, rankVal_t rankValOrigin, const U32 maxWeight, - const U32 nbBitsBaseline) -{ - U32 rankVal[HUFv07_TABLELOG_ABSOLUTEMAX + 1]; - const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */ - const U32 minBits = nbBitsBaseline - maxWeight; - U32 s; - - memcpy(rankVal, rankValOrigin, sizeof(rankVal)); - - /* fill DTable */ - for (s=0; s<sortedListSize; s++) { - const U16 symbol = sortedList[s].symbol; - const U32 weight = sortedList[s].weight; - const U32 nbBits = nbBitsBaseline - weight; - const U32 start = rankVal[weight]; - const U32 length = 1 << (targetLog-nbBits); - - if (targetLog-nbBits >= minBits) { /* enough room for a second symbol */ - U32 sortedRank; - int minWeight = nbBits + scaleLog; - if (minWeight < 1) minWeight = 1; - sortedRank = rankStart[minWeight]; - HUFv07_fillDTableX4Level2(DTable+start, targetLog-nbBits, nbBits, - rankValOrigin[nbBits], minWeight, - sortedList+sortedRank, sortedListSize-sortedRank, - nbBitsBaseline, symbol); - } else { - HUFv07_DEltX4 DElt; - MEM_writeLE16(&(DElt.sequence), symbol); - DElt.nbBits = (BYTE)(nbBits); - DElt.length = 1; - { U32 u; - const U32 end = start + length; - for (u = start; u < end; u++) DTable[u] = DElt; - } } - rankVal[weight] += length; - } -} - -size_t HUFv07_readDTableX4 (HUFv07_DTable* DTable, const void* src, size_t srcSize) -{ - BYTE weightList[HUFv07_SYMBOLVALUE_MAX + 1]; - sortedSymbol_t sortedSymbol[HUFv07_SYMBOLVALUE_MAX + 1]; - U32 rankStats[HUFv07_TABLELOG_ABSOLUTEMAX + 1] = { 0 }; - U32 rankStart0[HUFv07_TABLELOG_ABSOLUTEMAX + 2] = { 0 }; - U32* const rankStart = rankStart0+1; - rankVal_t rankVal; - U32 tableLog, maxW, sizeOfSort, nbSymbols; - DTableDesc dtd = HUFv07_getDTableDesc(DTable); - U32 const maxTableLog = dtd.maxTableLog; - size_t iSize; - void* dtPtr = DTable+1; /* force compiler to avoid strict-aliasing */ - HUFv07_DEltX4* const dt = (HUFv07_DEltX4*)dtPtr; - - HUFv07_STATIC_ASSERT(sizeof(HUFv07_DEltX4) == sizeof(HUFv07_DTable)); /* if compilation fails here, assertion is false */ - if (maxTableLog > HUFv07_TABLELOG_ABSOLUTEMAX) return ERROR(tableLog_tooLarge); - //memset(weightList, 0, sizeof(weightList)); /* is not necessary, even though some analyzer complain ... */ - - iSize = HUFv07_readStats(weightList, HUFv07_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize); - if (HUFv07_isError(iSize)) return iSize; - - /* check result */ - if (tableLog > maxTableLog) return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */ - - /* find maxWeight */ - for (maxW = tableLog; rankStats[maxW]==0; maxW--) {} /* necessarily finds a solution before 0 */ - - /* Get start index of each weight */ - { U32 w, nextRankStart = 0; - for (w=1; w<maxW+1; w++) { - U32 current = nextRankStart; - nextRankStart += rankStats[w]; - rankStart[w] = current; - } - rankStart[0] = nextRankStart; /* put all 0w symbols at the end of sorted list*/ - sizeOfSort = nextRankStart; - } - - /* sort symbols by weight */ - { U32 s; - for (s=0; s<nbSymbols; s++) { - U32 const w = weightList[s]; - U32 const r = rankStart[w]++; - sortedSymbol[r].symbol = (BYTE)s; - sortedSymbol[r].weight = (BYTE)w; - } - rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */ - } - - /* Build rankVal */ - { U32* const rankVal0 = rankVal[0]; - { int const rescale = (maxTableLog-tableLog) - 1; /* tableLog <= maxTableLog */ - U32 nextRankVal = 0; - U32 w; - for (w=1; w<maxW+1; w++) { - U32 current = nextRankVal; - nextRankVal += rankStats[w] << (w+rescale); - rankVal0[w] = current; - } } - { U32 const minBits = tableLog+1 - maxW; - U32 consumed; - for (consumed = minBits; consumed < maxTableLog - minBits + 1; consumed++) { - U32* const rankValPtr = rankVal[consumed]; - U32 w; - for (w = 1; w < maxW+1; w++) { - rankValPtr[w] = rankVal0[w] >> consumed; - } } } } - - HUFv07_fillDTableX4(dt, maxTableLog, - sortedSymbol, sizeOfSort, - rankStart0, rankVal, maxW, - tableLog+1); - - dtd.tableLog = (BYTE)maxTableLog; - dtd.tableType = 1; - memcpy(DTable, &dtd, sizeof(dtd)); - return iSize; -} - - -static U32 HUFv07_decodeSymbolX4(void* op, BITv07_DStream_t* DStream, const HUFv07_DEltX4* dt, const U32 dtLog) -{ - const size_t val = BITv07_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ - memcpy(op, dt+val, 2); - BITv07_skipBits(DStream, dt[val].nbBits); - return dt[val].length; -} - -static U32 HUFv07_decodeLastSymbolX4(void* op, BITv07_DStream_t* DStream, const HUFv07_DEltX4* dt, const U32 dtLog) -{ - const size_t val = BITv07_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ - memcpy(op, dt+val, 1); - if (dt[val].length==1) BITv07_skipBits(DStream, dt[val].nbBits); - else { - if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8)) { - BITv07_skipBits(DStream, dt[val].nbBits); - if (DStream->bitsConsumed > (sizeof(DStream->bitContainer)*8)) - DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8); /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */ - } } - return 1; -} - - -#define HUFv07_DECODE_SYMBOLX4_0(ptr, DStreamPtr) \ - ptr += HUFv07_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog) - -#define HUFv07_DECODE_SYMBOLX4_1(ptr, DStreamPtr) \ - if (MEM_64bits() || (HUFv07_TABLELOG_MAX<=12)) \ - ptr += HUFv07_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog) - -#define HUFv07_DECODE_SYMBOLX4_2(ptr, DStreamPtr) \ - if (MEM_64bits()) \ - ptr += HUFv07_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog) - -static inline size_t HUFv07_decodeStreamX4(BYTE* p, BITv07_DStream_t* bitDPtr, BYTE* const pEnd, const HUFv07_DEltX4* const dt, const U32 dtLog) -{ - BYTE* const pStart = p; - - /* up to 8 symbols at a time */ - while ((BITv07_reloadDStream(bitDPtr) == BITv07_DStream_unfinished) && (p < pEnd-7)) { - HUFv07_DECODE_SYMBOLX4_2(p, bitDPtr); - HUFv07_DECODE_SYMBOLX4_1(p, bitDPtr); - HUFv07_DECODE_SYMBOLX4_2(p, bitDPtr); - HUFv07_DECODE_SYMBOLX4_0(p, bitDPtr); - } - - /* closer to end : up to 2 symbols at a time */ - while ((BITv07_reloadDStream(bitDPtr) == BITv07_DStream_unfinished) && (p <= pEnd-2)) - HUFv07_DECODE_SYMBOLX4_0(p, bitDPtr); - - while (p <= pEnd-2) - HUFv07_DECODE_SYMBOLX4_0(p, bitDPtr); /* no need to reload : reached the end of DStream */ - - if (p < pEnd) - p += HUFv07_decodeLastSymbolX4(p, bitDPtr, dt, dtLog); - - return p-pStart; -} - - -static size_t HUFv07_decompress1X4_usingDTable_internal( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUFv07_DTable* DTable) -{ - BITv07_DStream_t bitD; - - /* Init */ - { size_t const errorCode = BITv07_initDStream(&bitD, cSrc, cSrcSize); - if (HUFv07_isError(errorCode)) return errorCode; - } - - /* decode */ - { BYTE* const ostart = (BYTE*) dst; - BYTE* const oend = ostart + dstSize; - const void* const dtPtr = DTable+1; /* force compiler to not use strict-aliasing */ - const HUFv07_DEltX4* const dt = (const HUFv07_DEltX4*)dtPtr; - DTableDesc const dtd = HUFv07_getDTableDesc(DTable); - HUFv07_decodeStreamX4(ostart, &bitD, oend, dt, dtd.tableLog); - } - - /* check */ - if (!BITv07_endOfDStream(&bitD)) return ERROR(corruption_detected); - - /* decoded size */ - return dstSize; -} - -size_t HUFv07_decompress1X4_usingDTable( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUFv07_DTable* DTable) -{ - DTableDesc dtd = HUFv07_getDTableDesc(DTable); - if (dtd.tableType != 1) return ERROR(GENERIC); - return HUFv07_decompress1X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); -} - -size_t HUFv07_decompress1X4_DCtx (HUFv07_DTable* DCtx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - const BYTE* ip = (const BYTE*) cSrc; - - size_t const hSize = HUFv07_readDTableX4 (DCtx, cSrc, cSrcSize); - if (HUFv07_isError(hSize)) return hSize; - if (hSize >= cSrcSize) return ERROR(srcSize_wrong); - ip += hSize; cSrcSize -= hSize; - - return HUFv07_decompress1X4_usingDTable_internal (dst, dstSize, ip, cSrcSize, DCtx); -} - -size_t HUFv07_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - HUFv07_CREATE_STATIC_DTABLEX4(DTable, HUFv07_TABLELOG_MAX); - return HUFv07_decompress1X4_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); -} - -static size_t HUFv07_decompress4X4_usingDTable_internal( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUFv07_DTable* DTable) -{ - if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ - - { const BYTE* const istart = (const BYTE*) cSrc; - BYTE* const ostart = (BYTE*) dst; - BYTE* const oend = ostart + dstSize; - const void* const dtPtr = DTable+1; - const HUFv07_DEltX4* const dt = (const HUFv07_DEltX4*)dtPtr; - - /* Init */ - BITv07_DStream_t bitD1; - BITv07_DStream_t bitD2; - BITv07_DStream_t bitD3; - BITv07_DStream_t bitD4; - size_t const length1 = MEM_readLE16(istart); - size_t const length2 = MEM_readLE16(istart+2); - size_t const length3 = MEM_readLE16(istart+4); - size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); - const BYTE* const istart1 = istart + 6; /* jumpTable */ - const BYTE* const istart2 = istart1 + length1; - const BYTE* const istart3 = istart2 + length2; - const BYTE* const istart4 = istart3 + length3; - size_t const segmentSize = (dstSize+3) / 4; - BYTE* const opStart2 = ostart + segmentSize; - BYTE* const opStart3 = opStart2 + segmentSize; - BYTE* const opStart4 = opStart3 + segmentSize; - BYTE* op1 = ostart; - BYTE* op2 = opStart2; - BYTE* op3 = opStart3; - BYTE* op4 = opStart4; - U32 endSignal; - DTableDesc const dtd = HUFv07_getDTableDesc(DTable); - U32 const dtLog = dtd.tableLog; - - if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ - { size_t const errorCode = BITv07_initDStream(&bitD1, istart1, length1); - if (HUFv07_isError(errorCode)) return errorCode; } - { size_t const errorCode = BITv07_initDStream(&bitD2, istart2, length2); - if (HUFv07_isError(errorCode)) return errorCode; } - { size_t const errorCode = BITv07_initDStream(&bitD3, istart3, length3); - if (HUFv07_isError(errorCode)) return errorCode; } - { size_t const errorCode = BITv07_initDStream(&bitD4, istart4, length4); - if (HUFv07_isError(errorCode)) return errorCode; } - - /* 16-32 symbols per loop (4-8 symbols per stream) */ - endSignal = BITv07_reloadDStream(&bitD1) | BITv07_reloadDStream(&bitD2) | BITv07_reloadDStream(&bitD3) | BITv07_reloadDStream(&bitD4); - for ( ; (endSignal==BITv07_DStream_unfinished) && (op4<(oend-7)) ; ) { - HUFv07_DECODE_SYMBOLX4_2(op1, &bitD1); - HUFv07_DECODE_SYMBOLX4_2(op2, &bitD2); - HUFv07_DECODE_SYMBOLX4_2(op3, &bitD3); - HUFv07_DECODE_SYMBOLX4_2(op4, &bitD4); - HUFv07_DECODE_SYMBOLX4_1(op1, &bitD1); - HUFv07_DECODE_SYMBOLX4_1(op2, &bitD2); - HUFv07_DECODE_SYMBOLX4_1(op3, &bitD3); - HUFv07_DECODE_SYMBOLX4_1(op4, &bitD4); - HUFv07_DECODE_SYMBOLX4_2(op1, &bitD1); - HUFv07_DECODE_SYMBOLX4_2(op2, &bitD2); - HUFv07_DECODE_SYMBOLX4_2(op3, &bitD3); - HUFv07_DECODE_SYMBOLX4_2(op4, &bitD4); - HUFv07_DECODE_SYMBOLX4_0(op1, &bitD1); - HUFv07_DECODE_SYMBOLX4_0(op2, &bitD2); - HUFv07_DECODE_SYMBOLX4_0(op3, &bitD3); - HUFv07_DECODE_SYMBOLX4_0(op4, &bitD4); - - endSignal = BITv07_reloadDStream(&bitD1) | BITv07_reloadDStream(&bitD2) | BITv07_reloadDStream(&bitD3) | BITv07_reloadDStream(&bitD4); - } - - /* check corruption */ - if (op1 > opStart2) return ERROR(corruption_detected); - if (op2 > opStart3) return ERROR(corruption_detected); - if (op3 > opStart4) return ERROR(corruption_detected); - /* note : op4 supposed already verified within main loop */ - - /* finish bitStreams one by one */ - HUFv07_decodeStreamX4(op1, &bitD1, opStart2, dt, dtLog); - HUFv07_decodeStreamX4(op2, &bitD2, opStart3, dt, dtLog); - HUFv07_decodeStreamX4(op3, &bitD3, opStart4, dt, dtLog); - HUFv07_decodeStreamX4(op4, &bitD4, oend, dt, dtLog); - - /* check */ - { U32 const endCheck = BITv07_endOfDStream(&bitD1) & BITv07_endOfDStream(&bitD2) & BITv07_endOfDStream(&bitD3) & BITv07_endOfDStream(&bitD4); - if (!endCheck) return ERROR(corruption_detected); } - - /* decoded size */ - return dstSize; - } -} - - -size_t HUFv07_decompress4X4_usingDTable( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUFv07_DTable* DTable) -{ - DTableDesc dtd = HUFv07_getDTableDesc(DTable); - if (dtd.tableType != 1) return ERROR(GENERIC); - return HUFv07_decompress4X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); -} - - -size_t HUFv07_decompress4X4_DCtx (HUFv07_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - const BYTE* ip = (const BYTE*) cSrc; - - size_t hSize = HUFv07_readDTableX4 (dctx, cSrc, cSrcSize); - if (HUFv07_isError(hSize)) return hSize; - if (hSize >= cSrcSize) return ERROR(srcSize_wrong); - ip += hSize; cSrcSize -= hSize; - - return HUFv07_decompress4X4_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx); -} - -size_t HUFv07_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - HUFv07_CREATE_STATIC_DTABLEX4(DTable, HUFv07_TABLELOG_MAX); - return HUFv07_decompress4X4_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); -} - - -/* ********************************/ -/* Generic decompression selector */ -/* ********************************/ - -size_t HUFv07_decompress1X_usingDTable(void* dst, size_t maxDstSize, - const void* cSrc, size_t cSrcSize, - const HUFv07_DTable* DTable) -{ - DTableDesc const dtd = HUFv07_getDTableDesc(DTable); - return dtd.tableType ? HUFv07_decompress1X4_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable) : - HUFv07_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable); -} - -size_t HUFv07_decompress4X_usingDTable(void* dst, size_t maxDstSize, - const void* cSrc, size_t cSrcSize, - const HUFv07_DTable* DTable) -{ - DTableDesc const dtd = HUFv07_getDTableDesc(DTable); - return dtd.tableType ? HUFv07_decompress4X4_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable) : - HUFv07_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable); -} - - -typedef struct { U32 tableTime; U32 decode256Time; } algo_time_t; -static const algo_time_t algoTime[16 /* Quantization */][3 /* single, double, quad */] = -{ - /* single, double, quad */ - {{0,0}, {1,1}, {2,2}}, /* Q==0 : impossible */ - {{0,0}, {1,1}, {2,2}}, /* Q==1 : impossible */ - {{ 38,130}, {1313, 74}, {2151, 38}}, /* Q == 2 : 12-18% */ - {{ 448,128}, {1353, 74}, {2238, 41}}, /* Q == 3 : 18-25% */ - {{ 556,128}, {1353, 74}, {2238, 47}}, /* Q == 4 : 25-32% */ - {{ 714,128}, {1418, 74}, {2436, 53}}, /* Q == 5 : 32-38% */ - {{ 883,128}, {1437, 74}, {2464, 61}}, /* Q == 6 : 38-44% */ - {{ 897,128}, {1515, 75}, {2622, 68}}, /* Q == 7 : 44-50% */ - {{ 926,128}, {1613, 75}, {2730, 75}}, /* Q == 8 : 50-56% */ - {{ 947,128}, {1729, 77}, {3359, 77}}, /* Q == 9 : 56-62% */ - {{1107,128}, {2083, 81}, {4006, 84}}, /* Q ==10 : 62-69% */ - {{1177,128}, {2379, 87}, {4785, 88}}, /* Q ==11 : 69-75% */ - {{1242,128}, {2415, 93}, {5155, 84}}, /* Q ==12 : 75-81% */ - {{1349,128}, {2644,106}, {5260,106}}, /* Q ==13 : 81-87% */ - {{1455,128}, {2422,124}, {4174,124}}, /* Q ==14 : 87-93% */ - {{ 722,128}, {1891,145}, {1936,146}}, /* Q ==15 : 93-99% */ -}; - -/** HUFv07_selectDecoder() : -* Tells which decoder is likely to decode faster, -* based on a set of pre-determined metrics. -* @return : 0==HUFv07_decompress4X2, 1==HUFv07_decompress4X4 . -* Assumption : 0 < cSrcSize < dstSize <= 128 KB */ -U32 HUFv07_selectDecoder (size_t dstSize, size_t cSrcSize) -{ - /* decoder timing evaluation */ - U32 const Q = (U32)(cSrcSize * 16 / dstSize); /* Q < 16 since dstSize > cSrcSize */ - U32 const D256 = (U32)(dstSize >> 8); - U32 const DTime0 = algoTime[Q][0].tableTime + (algoTime[Q][0].decode256Time * D256); - U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256); - DTime1 += DTime1 >> 3; /* advantage to algorithm using less memory, for cache eviction */ - - return DTime1 < DTime0; -} - - -typedef size_t (*decompressionAlgo)(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); - -size_t HUFv07_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - static const decompressionAlgo decompress[2] = { HUFv07_decompress4X2, HUFv07_decompress4X4 }; - - /* validation checks */ - if (dstSize == 0) return ERROR(dstSize_tooSmall); - if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ - if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ - if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ - - { U32 const algoNb = HUFv07_selectDecoder(dstSize, cSrcSize); - return decompress[algoNb](dst, dstSize, cSrc, cSrcSize); - } - - //return HUFv07_decompress4X2(dst, dstSize, cSrc, cSrcSize); /* multi-streams single-symbol decoding */ - //return HUFv07_decompress4X4(dst, dstSize, cSrc, cSrcSize); /* multi-streams double-symbols decoding */ -} - -size_t HUFv07_decompress4X_DCtx (HUFv07_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - /* validation checks */ - if (dstSize == 0) return ERROR(dstSize_tooSmall); - if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ - if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ - if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ - - { U32 const algoNb = HUFv07_selectDecoder(dstSize, cSrcSize); - return algoNb ? HUFv07_decompress4X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) : - HUFv07_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ; - } -} - -size_t HUFv07_decompress4X_hufOnly (HUFv07_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - /* validation checks */ - if (dstSize == 0) return ERROR(dstSize_tooSmall); - if ((cSrcSize >= dstSize) || (cSrcSize <= 1)) return ERROR(corruption_detected); /* invalid */ - - { U32 const algoNb = HUFv07_selectDecoder(dstSize, cSrcSize); - return algoNb ? HUFv07_decompress4X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) : - HUFv07_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ; - } -} - -size_t HUFv07_decompress1X_DCtx (HUFv07_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - /* validation checks */ - if (dstSize == 0) return ERROR(dstSize_tooSmall); - if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ - if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ - if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ - - { U32 const algoNb = HUFv07_selectDecoder(dstSize, cSrcSize); - return algoNb ? HUFv07_decompress1X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) : - HUFv07_decompress1X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ; - } -} -/* - Common functions of Zstd compression library - Copyright (C) 2015-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - zstd homepage : http://www.zstd.net/ -*/ - - - -/*-**************************************** -* ZSTD Error Management -******************************************/ -/*! ZSTDv07_isError() : -* tells if a return value is an error code */ -unsigned ZSTDv07_isError(size_t code) { return ERR_isError(code); } - -/*! ZSTDv07_getErrorName() : -* provides error code string from function result (useful for debugging) */ -const char* ZSTDv07_getErrorName(size_t code) { return ERR_getErrorName(code); } - -/*! ZSTDv07_getError() : -* convert a `size_t` function result into a proper ZSTDv07_errorCode enum */ -ZSTDv07_ErrorCode ZSTDv07_getErrorCode(size_t code) { return ERR_getErrorCode(code); } - -/*! ZSTDv07_getErrorString() : -* provides error code string from enum */ -const char* ZSTDv07_getErrorString(ZSTDv07_ErrorCode code) { return ERR_getErrorName(code); } - - -/* ************************************************************** -* ZBUFF Error Management -****************************************************************/ -unsigned ZBUFFv07_isError(size_t errorCode) { return ERR_isError(errorCode); } - -const char* ZBUFFv07_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); } - - - -void* ZSTDv07_defaultAllocFunction(void* opaque, size_t size) -{ - void* address = malloc(size); - (void)opaque; - /* printf("alloc %p, %d opaque=%p \n", address, (int)size, opaque); */ - return address; -} - -void ZSTDv07_defaultFreeFunction(void* opaque, void* address) -{ - (void)opaque; - /* if (address) printf("free %p opaque=%p \n", address, opaque); */ - free(address); -} -/* - zstd_internal - common functions to include - Header File for include - Copyright (C) 2014-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - zstd homepage : https://www.zstd.net -*/ -#ifndef ZSTDv07_CCOMMON_H_MODULE -#define ZSTDv07_CCOMMON_H_MODULE - - -/*-************************************* -* Common macros -***************************************/ -#define MIN(a,b) ((a)<(b) ? (a) : (b)) -#define MAX(a,b) ((a)>(b) ? (a) : (b)) - - -/*-************************************* -* Common constants -***************************************/ -#define ZSTDv07_OPT_DEBUG 0 /* 3 = compression stats; 5 = check encoded sequences; 9 = full logs */ -#include <stdio.h> -#if defined(ZSTDv07_OPT_DEBUG) && ZSTDv07_OPT_DEBUG>=9 - #define ZSTDv07_LOG_PARSER(...) printf(__VA_ARGS__) - #define ZSTDv07_LOG_ENCODE(...) printf(__VA_ARGS__) - #define ZSTDv07_LOG_BLOCK(...) printf(__VA_ARGS__) -#else - #define ZSTDv07_LOG_PARSER(...) - #define ZSTDv07_LOG_ENCODE(...) - #define ZSTDv07_LOG_BLOCK(...) -#endif - -#define ZSTDv07_OPT_NUM (1<<12) -#define ZSTDv07_DICT_MAGIC 0xEC30A437 /* v0.7 */ - -#define ZSTDv07_REP_NUM 3 -#define ZSTDv07_REP_INIT ZSTDv07_REP_NUM -#define ZSTDv07_REP_MOVE (ZSTDv07_REP_NUM-1) -static const U32 repStartValue[ZSTDv07_REP_NUM] = { 1, 4, 8 }; - -#define KB *(1 <<10) -#define MB *(1 <<20) -#define GB *(1U<<30) - -#define BIT7 128 -#define BIT6 64 -#define BIT5 32 -#define BIT4 16 -#define BIT1 2 -#define BIT0 1 - -#define ZSTDv07_WINDOWLOG_ABSOLUTEMIN 10 -static const size_t ZSTDv07_fcs_fieldSize[4] = { 0, 2, 4, 8 }; -static const size_t ZSTDv07_did_fieldSize[4] = { 0, 1, 2, 4 }; - -#define ZSTDv07_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */ -static const size_t ZSTDv07_blockHeaderSize = ZSTDv07_BLOCKHEADERSIZE; -typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t; - -#define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */ -#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */ + MIN_SEQUENCES_SIZE /* nbSeq==0 */) /* for a non-null block */ - -#define HufLog 12 -typedef enum { lbt_huffman, lbt_repeat, lbt_raw, lbt_rle } litBlockType_t; - -#define LONGNBSEQ 0x7F00 - -#define MINMATCH 3 -#define EQUAL_READ32 4 - -#define Litbits 8 -#define MaxLit ((1<<Litbits) - 1) -#define MaxML 52 -#define MaxLL 35 -#define MaxOff 28 -#define MaxSeq MAX(MaxLL, MaxML) /* Assumption : MaxOff < MaxLL,MaxML */ -#define MLFSELog 9 -#define LLFSELog 9 -#define OffFSELog 8 - -#define FSEv07_ENCODING_RAW 0 -#define FSEv07_ENCODING_RLE 1 -#define FSEv07_ENCODING_STATIC 2 -#define FSEv07_ENCODING_DYNAMIC 3 - -static const U32 LL_bits[MaxLL+1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 1, 1, 1, 2, 2, 3, 3, 4, 6, 7, 8, 9,10,11,12, - 13,14,15,16 }; -static const S16 LL_defaultNorm[MaxLL+1] = { 4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1, - -1,-1,-1,-1 }; -static const U32 LL_defaultNormLog = 6; - -static const U32 ML_bits[MaxML+1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 7, 8, 9,10,11, - 12,13,14,15,16 }; -static const S16 ML_defaultNorm[MaxML+1] = { 1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,-1,-1, - -1,-1,-1,-1,-1 }; -static const U32 ML_defaultNormLog = 6; - -static const S16 OF_defaultNorm[MaxOff+1] = { 1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1,-1,-1,-1,-1,-1 }; -static const U32 OF_defaultNormLog = 5; - - -/*-******************************************* -* Shared functions to include for inlining -*********************************************/ -static void ZSTDv07_copy8(void* dst, const void* src) { memcpy(dst, src, 8); } -#define COPY8(d,s) { ZSTDv07_copy8(d,s); d+=8; s+=8; } - -/*! ZSTDv07_wildcopy() : -* custom version of memcpy(), can copy up to 7 bytes too many (8 bytes if length==0) */ -#define WILDCOPY_OVERLENGTH 8 -MEM_STATIC void ZSTDv07_wildcopy(void* dst, const void* src, size_t length) -{ - const BYTE* ip = (const BYTE*)src; - BYTE* op = (BYTE*)dst; - BYTE* const oend = op + length; - do - COPY8(op, ip) - while (op < oend); -} - - -/*-******************************************* -* Private interfaces -*********************************************/ -typedef struct ZSTDv07_stats_s ZSTDv07_stats_t; - -typedef struct { - U32 off; - U32 len; -} ZSTDv07_match_t; - -typedef struct { - U32 price; - U32 off; - U32 mlen; - U32 litlen; - U32 rep[ZSTDv07_REP_INIT]; -} ZSTDv07_optimal_t; - -struct ZSTDv07_stats_s { U32 unused; }; -MEM_STATIC void ZSTDv07_statsPrint(ZSTDv07_stats_t* stats, U32 searchLength) { (void)stats; (void)searchLength; } -MEM_STATIC void ZSTDv07_statsInit(ZSTDv07_stats_t* stats) { (void)stats; } -MEM_STATIC void ZSTDv07_statsResetFreqs(ZSTDv07_stats_t* stats) { (void)stats; } -MEM_STATIC void ZSTDv07_statsUpdatePrices(ZSTDv07_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) { (void)stats; (void)litLength; (void)literals; (void)offset; (void)matchLength; } - -typedef struct { - void* buffer; - U32* offsetStart; - U32* offset; - BYTE* offCodeStart; - BYTE* litStart; - BYTE* lit; - U16* litLengthStart; - U16* litLength; - BYTE* llCodeStart; - U16* matchLengthStart; - U16* matchLength; - BYTE* mlCodeStart; - U32 longLengthID; /* 0 == no longLength; 1 == Lit.longLength; 2 == Match.longLength; */ - U32 longLengthPos; - /* opt */ - ZSTDv07_optimal_t* priceTable; - ZSTDv07_match_t* matchTable; - U32* matchLengthFreq; - U32* litLengthFreq; - U32* litFreq; - U32* offCodeFreq; - U32 matchLengthSum; - U32 matchSum; - U32 litLengthSum; - U32 litSum; - U32 offCodeSum; - U32 log2matchLengthSum; - U32 log2matchSum; - U32 log2litLengthSum; - U32 log2litSum; - U32 log2offCodeSum; - U32 factor; - U32 cachedPrice; - U32 cachedLitLength; - const BYTE* cachedLiterals; - ZSTDv07_stats_t stats; -} seqStore_t; - -void ZSTDv07_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq); - -/* custom memory allocation functions */ -void* ZSTDv07_defaultAllocFunction(void* opaque, size_t size); -void ZSTDv07_defaultFreeFunction(void* opaque, void* address); -static const ZSTDv07_customMem defaultCustomMem = { ZSTDv07_defaultAllocFunction, ZSTDv07_defaultFreeFunction, NULL }; - -#endif /* ZSTDv07_CCOMMON_H_MODULE */ -/* - zstd - standard compression library - Copyright (C) 2014-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - zstd homepage : http://www.zstd.net -*/ - -/* *************************************************************** -* Tuning parameters -*****************************************************************/ -/*! - * HEAPMODE : - * Select how default decompression function ZSTDv07_decompress() will allocate memory, - * in memory stack (0), or in memory heap (1, requires malloc()) - */ -#ifndef ZSTDv07_HEAPMODE -# define ZSTDv07_HEAPMODE 1 -#endif - - -/*-******************************************************* -* Compiler specifics -*********************************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# define FORCE_INLINE static __forceinline -# include <intrin.h> /* For Visual 2005 */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -# pragma warning(disable : 4324) /* disable: C4324: padded structure */ -#else -# ifdef __GNUC__ -# define FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define FORCE_INLINE static inline -# endif -#endif - - -/*-************************************* -* Macros -***************************************/ -#define ZSTDv07_isError ERR_isError /* for inlining */ -#define FSEv07_isError ERR_isError -#define HUFv07_isError ERR_isError - - -/*_******************************************************* -* Memory operations -**********************************************************/ -static void ZSTDv07_copy4(void* dst, const void* src) { memcpy(dst, src, 4); } - - -/*-************************************************************* -* Context management -***************************************************************/ -typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader, - ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock, - ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTDv07_dStage; - -struct ZSTDv07_DCtx_s -{ - FSEv07_DTable LLTable[FSEv07_DTABLE_SIZE_U32(LLFSELog)]; - FSEv07_DTable OffTable[FSEv07_DTABLE_SIZE_U32(OffFSELog)]; - FSEv07_DTable MLTable[FSEv07_DTABLE_SIZE_U32(MLFSELog)]; - HUFv07_DTable hufTable[HUFv07_DTABLE_SIZE(HufLog)]; /* can accommodate HUFv07_decompress4X */ - const void* previousDstEnd; - const void* base; - const void* vBase; - const void* dictEnd; - size_t expected; - U32 rep[3]; - ZSTDv07_frameParams fParams; - blockType_t bType; /* used in ZSTDv07_decompressContinue(), to transfer blockType between header decoding and block decoding stages */ - ZSTDv07_dStage stage; - U32 litEntropy; - U32 fseEntropy; - XXH64_state_t xxhState; - size_t headerSize; - U32 dictID; - const BYTE* litPtr; - ZSTDv07_customMem customMem; - size_t litBufSize; - size_t litSize; - BYTE litBuffer[ZSTDv07_BLOCKSIZE_ABSOLUTEMAX + WILDCOPY_OVERLENGTH]; - BYTE headerBuffer[ZSTDv07_FRAMEHEADERSIZE_MAX]; -}; /* typedef'd to ZSTDv07_DCtx within "zstd_static.h" */ - -int ZSTDv07_isSkipFrame(ZSTDv07_DCtx* dctx); - -size_t ZSTDv07_sizeofDCtx (const ZSTDv07_DCtx* dctx) { return sizeof(*dctx); } - -size_t ZSTDv07_estimateDCtxSize(void) { return sizeof(ZSTDv07_DCtx); } - -size_t ZSTDv07_decompressBegin(ZSTDv07_DCtx* dctx) -{ - dctx->expected = ZSTDv07_frameHeaderSize_min; - dctx->stage = ZSTDds_getFrameHeaderSize; - dctx->previousDstEnd = NULL; - dctx->base = NULL; - dctx->vBase = NULL; - dctx->dictEnd = NULL; - dctx->hufTable[0] = (HUFv07_DTable)((HufLog)*0x1000001); - dctx->litEntropy = dctx->fseEntropy = 0; - dctx->dictID = 0; - { int i; for (i=0; i<ZSTDv07_REP_NUM; i++) dctx->rep[i] = repStartValue[i]; } - return 0; -} - -ZSTDv07_DCtx* ZSTDv07_createDCtx_advanced(ZSTDv07_customMem customMem) -{ - ZSTDv07_DCtx* dctx; - - if (!customMem.customAlloc && !customMem.customFree) - customMem = defaultCustomMem; - - if (!customMem.customAlloc || !customMem.customFree) - return NULL; - - dctx = (ZSTDv07_DCtx*) customMem.customAlloc(customMem.opaque, sizeof(ZSTDv07_DCtx)); - if (!dctx) return NULL; - memcpy(&dctx->customMem, &customMem, sizeof(ZSTDv07_customMem)); - ZSTDv07_decompressBegin(dctx); - return dctx; -} - -ZSTDv07_DCtx* ZSTDv07_createDCtx(void) -{ - return ZSTDv07_createDCtx_advanced(defaultCustomMem); -} - -size_t ZSTDv07_freeDCtx(ZSTDv07_DCtx* dctx) -{ - if (dctx==NULL) return 0; /* support free on NULL */ - dctx->customMem.customFree(dctx->customMem.opaque, dctx); - return 0; /* reserved as a potential error code in the future */ -} - -void ZSTDv07_copyDCtx(ZSTDv07_DCtx* dstDCtx, const ZSTDv07_DCtx* srcDCtx) -{ - memcpy(dstDCtx, srcDCtx, - sizeof(ZSTDv07_DCtx) - (ZSTDv07_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH + ZSTDv07_frameHeaderSize_max)); /* no need to copy workspace */ -} - - -/*-************************************************************* -* Decompression section -***************************************************************/ - -/* Frame format description - Frame Header - [ Block Header - Block ] - Frame End - 1) Frame Header - - 4 bytes - Magic Number : ZSTDv07_MAGICNUMBER (defined within zstd.h) - - 1 byte - Frame Descriptor - 2) Block Header - - 3 bytes, starting with a 2-bits descriptor - Uncompressed, Compressed, Frame End, unused - 3) Block - See Block Format Description - 4) Frame End - - 3 bytes, compatible with Block Header -*/ - - -/* Frame Header : - - 1 byte - FrameHeaderDescription : - bit 0-1 : dictID (0, 1, 2 or 4 bytes) - bit 2 : checksumFlag - bit 3 : reserved (must be zero) - bit 4 : reserved (unused, can be any value) - bit 5 : Single Segment (if 1, WindowLog byte is not present) - bit 6-7 : FrameContentFieldSize (0, 2, 4, or 8) - if (SkippedWindowLog && !FrameContentFieldsize) FrameContentFieldsize=1; - - Optional : WindowLog (0 or 1 byte) - bit 0-2 : octal Fractional (1/8th) - bit 3-7 : Power of 2, with 0 = 1 KB (up to 2 TB) - - Optional : dictID (0, 1, 2 or 4 bytes) - Automatic adaptation - 0 : no dictID - 1 : 1 - 255 - 2 : 256 - 65535 - 4 : all other values - - Optional : content size (0, 1, 2, 4 or 8 bytes) - 0 : unknown (fcfs==0 and swl==0) - 1 : 0-255 bytes (fcfs==0 and swl==1) - 2 : 256 - 65535+256 (fcfs==1) - 4 : 0 - 4GB-1 (fcfs==2) - 8 : 0 - 16EB-1 (fcfs==3) -*/ - - -/* Compressed Block, format description - - Block = Literal Section - Sequences Section - Prerequisite : size of (compressed) block, maximum size of regenerated data - - 1) Literal Section - - 1.1) Header : 1-5 bytes - flags: 2 bits - 00 compressed by Huff0 - 01 unused - 10 is Raw (uncompressed) - 11 is Rle - Note : using 01 => Huff0 with precomputed table ? - Note : delta map ? => compressed ? - - 1.1.1) Huff0-compressed literal block : 3-5 bytes - srcSize < 1 KB => 3 bytes (2-2-10-10) => single stream - srcSize < 1 KB => 3 bytes (2-2-10-10) - srcSize < 16KB => 4 bytes (2-2-14-14) - else => 5 bytes (2-2-18-18) - big endian convention - - 1.1.2) Raw (uncompressed) literal block header : 1-3 bytes - size : 5 bits: (IS_RAW<<6) + (0<<4) + size - 12 bits: (IS_RAW<<6) + (2<<4) + (size>>8) - size&255 - 20 bits: (IS_RAW<<6) + (3<<4) + (size>>16) - size>>8&255 - size&255 - - 1.1.3) Rle (repeated single byte) literal block header : 1-3 bytes - size : 5 bits: (IS_RLE<<6) + (0<<4) + size - 12 bits: (IS_RLE<<6) + (2<<4) + (size>>8) - size&255 - 20 bits: (IS_RLE<<6) + (3<<4) + (size>>16) - size>>8&255 - size&255 - - 1.1.4) Huff0-compressed literal block, using precomputed CTables : 3-5 bytes - srcSize < 1 KB => 3 bytes (2-2-10-10) => single stream - srcSize < 1 KB => 3 bytes (2-2-10-10) - srcSize < 16KB => 4 bytes (2-2-14-14) - else => 5 bytes (2-2-18-18) - big endian convention - - 1- CTable available (stored into workspace ?) - 2- Small input (fast heuristic ? Full comparison ? depend on clevel ?) - - - 1.2) Literal block content - - 1.2.1) Huff0 block, using sizes from header - See Huff0 format - - 1.2.2) Huff0 block, using prepared table - - 1.2.3) Raw content - - 1.2.4) single byte - - - 2) Sequences section - TO DO -*/ - -/** ZSTDv07_frameHeaderSize() : -* srcSize must be >= ZSTDv07_frameHeaderSize_min. -* @return : size of the Frame Header */ -static size_t ZSTDv07_frameHeaderSize(const void* src, size_t srcSize) -{ - if (srcSize < ZSTDv07_frameHeaderSize_min) return ERROR(srcSize_wrong); - { BYTE const fhd = ((const BYTE*)src)[4]; - U32 const dictID= fhd & 3; - U32 const directMode = (fhd >> 5) & 1; - U32 const fcsId = fhd >> 6; - return ZSTDv07_frameHeaderSize_min + !directMode + ZSTDv07_did_fieldSize[dictID] + ZSTDv07_fcs_fieldSize[fcsId] - + (directMode && !ZSTDv07_fcs_fieldSize[fcsId]); - } -} - - -/** ZSTDv07_getFrameParams() : -* decode Frame Header, or require larger `srcSize`. -* @return : 0, `fparamsPtr` is correctly filled, -* >0, `srcSize` is too small, result is expected `srcSize`, -* or an error code, which can be tested using ZSTDv07_isError() */ -size_t ZSTDv07_getFrameParams(ZSTDv07_frameParams* fparamsPtr, const void* src, size_t srcSize) -{ - const BYTE* ip = (const BYTE*)src; - - if (srcSize < ZSTDv07_frameHeaderSize_min) return ZSTDv07_frameHeaderSize_min; - if (MEM_readLE32(src) != ZSTDv07_MAGICNUMBER) { - if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTDv07_MAGIC_SKIPPABLE_START) { - if (srcSize < ZSTDv07_skippableHeaderSize) return ZSTDv07_skippableHeaderSize; /* magic number + skippable frame length */ - memset(fparamsPtr, 0, sizeof(*fparamsPtr)); - fparamsPtr->frameContentSize = MEM_readLE32((const char *)src + 4); - fparamsPtr->windowSize = 0; /* windowSize==0 means a frame is skippable */ - return 0; - } - return ERROR(prefix_unknown); - } - - /* ensure there is enough `srcSize` to fully read/decode frame header */ - { size_t const fhsize = ZSTDv07_frameHeaderSize(src, srcSize); - if (srcSize < fhsize) return fhsize; } - - { BYTE const fhdByte = ip[4]; - size_t pos = 5; - U32 const dictIDSizeCode = fhdByte&3; - U32 const checksumFlag = (fhdByte>>2)&1; - U32 const directMode = (fhdByte>>5)&1; - U32 const fcsID = fhdByte>>6; - U32 const windowSizeMax = 1U << ZSTDv07_WINDOWLOG_MAX; - U32 windowSize = 0; - U32 dictID = 0; - U64 frameContentSize = 0; - if ((fhdByte & 0x08) != 0) return ERROR(frameParameter_unsupported); /* reserved bits, which must be zero */ - if (!directMode) { - BYTE const wlByte = ip[pos++]; - U32 const windowLog = (wlByte >> 3) + ZSTDv07_WINDOWLOG_ABSOLUTEMIN; - if (windowLog > ZSTDv07_WINDOWLOG_MAX) return ERROR(frameParameter_unsupported); - windowSize = (1U << windowLog); - windowSize += (windowSize >> 3) * (wlByte&7); - } - - switch(dictIDSizeCode) - { - default: /* impossible */ - case 0 : break; - case 1 : dictID = ip[pos]; pos++; break; - case 2 : dictID = MEM_readLE16(ip+pos); pos+=2; break; - case 3 : dictID = MEM_readLE32(ip+pos); pos+=4; break; - } - switch(fcsID) - { - default: /* impossible */ - case 0 : if (directMode) frameContentSize = ip[pos]; break; - case 1 : frameContentSize = MEM_readLE16(ip+pos)+256; break; - case 2 : frameContentSize = MEM_readLE32(ip+pos); break; - case 3 : frameContentSize = MEM_readLE64(ip+pos); break; - } - if (!windowSize) windowSize = (U32)frameContentSize; - if (windowSize > windowSizeMax) return ERROR(frameParameter_unsupported); - fparamsPtr->frameContentSize = frameContentSize; - fparamsPtr->windowSize = windowSize; - fparamsPtr->dictID = dictID; - fparamsPtr->checksumFlag = checksumFlag; - } - return 0; -} - - -/** ZSTDv07_getDecompressedSize() : -* compatible with legacy mode -* @return : decompressed size if known, 0 otherwise - note : 0 can mean any of the following : - - decompressed size is not provided within frame header - - frame header unknown / not supported - - frame header not completely provided (`srcSize` too small) */ -unsigned long long ZSTDv07_getDecompressedSize(const void* src, size_t srcSize) -{ - { ZSTDv07_frameParams fparams; - size_t const frResult = ZSTDv07_getFrameParams(&fparams, src, srcSize); - if (frResult!=0) return 0; - return fparams.frameContentSize; - } -} - - -/** ZSTDv07_decodeFrameHeader() : -* `srcSize` must be the size provided by ZSTDv07_frameHeaderSize(). -* @return : 0 if success, or an error code, which can be tested using ZSTDv07_isError() */ -static size_t ZSTDv07_decodeFrameHeader(ZSTDv07_DCtx* dctx, const void* src, size_t srcSize) -{ - size_t const result = ZSTDv07_getFrameParams(&(dctx->fParams), src, srcSize); - if (dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID)) return ERROR(dictionary_wrong); - if (dctx->fParams.checksumFlag) XXH64_reset(&dctx->xxhState, 0); - return result; -} - - -typedef struct -{ - blockType_t blockType; - U32 origSize; -} blockProperties_t; - -/*! ZSTDv07_getcBlockSize() : -* Provides the size of compressed block from block header `src` */ -size_t ZSTDv07_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr) -{ - const BYTE* const in = (const BYTE* const)src; - U32 cSize; - - if (srcSize < ZSTDv07_blockHeaderSize) return ERROR(srcSize_wrong); - - bpPtr->blockType = (blockType_t)((*in) >> 6); - cSize = in[2] + (in[1]<<8) + ((in[0] & 7)<<16); - bpPtr->origSize = (bpPtr->blockType == bt_rle) ? cSize : 0; - - if (bpPtr->blockType == bt_end) return 0; - if (bpPtr->blockType == bt_rle) return 1; - return cSize; -} - - -static size_t ZSTDv07_copyRawBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize) -{ - if (srcSize > dstCapacity) return ERROR(dstSize_tooSmall); - memcpy(dst, src, srcSize); - return srcSize; -} - - -/*! ZSTDv07_decodeLiteralsBlock() : - @return : nb of bytes read from src (< srcSize ) */ -size_t ZSTDv07_decodeLiteralsBlock(ZSTDv07_DCtx* dctx, - const void* src, size_t srcSize) /* note : srcSize < BLOCKSIZE */ -{ - const BYTE* const istart = (const BYTE*) src; - - if (srcSize < MIN_CBLOCK_SIZE) return ERROR(corruption_detected); - - switch((litBlockType_t)(istart[0]>> 6)) - { - case lbt_huffman: - { size_t litSize, litCSize, singleStream=0; - U32 lhSize = (istart[0] >> 4) & 3; - if (srcSize < 5) return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for lhSize, + cSize (+nbSeq) */ - switch(lhSize) - { - case 0: case 1: default: /* note : default is impossible, since lhSize into [0..3] */ - /* 2 - 2 - 10 - 10 */ - lhSize=3; - singleStream = istart[0] & 16; - litSize = ((istart[0] & 15) << 6) + (istart[1] >> 2); - litCSize = ((istart[1] & 3) << 8) + istart[2]; - break; - case 2: - /* 2 - 2 - 14 - 14 */ - lhSize=4; - litSize = ((istart[0] & 15) << 10) + (istart[1] << 2) + (istart[2] >> 6); - litCSize = ((istart[2] & 63) << 8) + istart[3]; - break; - case 3: - /* 2 - 2 - 18 - 18 */ - lhSize=5; - litSize = ((istart[0] & 15) << 14) + (istart[1] << 6) + (istart[2] >> 2); - litCSize = ((istart[2] & 3) << 16) + (istart[3] << 8) + istart[4]; - break; - } - if (litSize > ZSTDv07_BLOCKSIZE_ABSOLUTEMAX) return ERROR(corruption_detected); - if (litCSize + lhSize > srcSize) return ERROR(corruption_detected); - - if (HUFv07_isError(singleStream ? - HUFv07_decompress1X2_DCtx(dctx->hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize) : - HUFv07_decompress4X_hufOnly (dctx->hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize) )) - return ERROR(corruption_detected); - - dctx->litPtr = dctx->litBuffer; - dctx->litBufSize = ZSTDv07_BLOCKSIZE_ABSOLUTEMAX+8; - dctx->litSize = litSize; - dctx->litEntropy = 1; - return litCSize + lhSize; - } - case lbt_repeat: - { size_t litSize, litCSize; - U32 lhSize = ((istart[0]) >> 4) & 3; - if (lhSize != 1) /* only case supported for now : small litSize, single stream */ - return ERROR(corruption_detected); - if (dctx->litEntropy==0) - return ERROR(dictionary_corrupted); - - /* 2 - 2 - 10 - 10 */ - lhSize=3; - litSize = ((istart[0] & 15) << 6) + (istart[1] >> 2); - litCSize = ((istart[1] & 3) << 8) + istart[2]; - if (litCSize + lhSize > srcSize) return ERROR(corruption_detected); - - { size_t const errorCode = HUFv07_decompress1X4_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->hufTable); - if (HUFv07_isError(errorCode)) return ERROR(corruption_detected); - } - dctx->litPtr = dctx->litBuffer; - dctx->litBufSize = ZSTDv07_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH; - dctx->litSize = litSize; - return litCSize + lhSize; - } - case lbt_raw: - { size_t litSize; - U32 lhSize = ((istart[0]) >> 4) & 3; - switch(lhSize) - { - case 0: case 1: default: /* note : default is impossible, since lhSize into [0..3] */ - lhSize=1; - litSize = istart[0] & 31; - break; - case 2: - litSize = ((istart[0] & 15) << 8) + istart[1]; - break; - case 3: - litSize = ((istart[0] & 15) << 16) + (istart[1] << 8) + istart[2]; - break; - } - - if (lhSize+litSize+WILDCOPY_OVERLENGTH > srcSize) { /* risk reading beyond src buffer with wildcopy */ - if (litSize+lhSize > srcSize) return ERROR(corruption_detected); - memcpy(dctx->litBuffer, istart+lhSize, litSize); - dctx->litPtr = dctx->litBuffer; - dctx->litBufSize = ZSTDv07_BLOCKSIZE_ABSOLUTEMAX+8; - dctx->litSize = litSize; - return lhSize+litSize; - } - /* direct reference into compressed stream */ - dctx->litPtr = istart+lhSize; - dctx->litBufSize = srcSize-lhSize; - dctx->litSize = litSize; - return lhSize+litSize; - } - case lbt_rle: - { size_t litSize; - U32 lhSize = ((istart[0]) >> 4) & 3; - switch(lhSize) - { - case 0: case 1: default: /* note : default is impossible, since lhSize into [0..3] */ - lhSize = 1; - litSize = istart[0] & 31; - break; - case 2: - litSize = ((istart[0] & 15) << 8) + istart[1]; - break; - case 3: - litSize = ((istart[0] & 15) << 16) + (istart[1] << 8) + istart[2]; - if (srcSize<4) return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4 */ - break; - } - if (litSize > ZSTDv07_BLOCKSIZE_ABSOLUTEMAX) return ERROR(corruption_detected); - memset(dctx->litBuffer, istart[lhSize], litSize); - dctx->litPtr = dctx->litBuffer; - dctx->litBufSize = ZSTDv07_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH; - dctx->litSize = litSize; - return lhSize+1; - } - default: - return ERROR(corruption_detected); /* impossible */ - } -} - - -/*! ZSTDv07_buildSeqTable() : - @return : nb bytes read from src, - or an error code if it fails, testable with ZSTDv07_isError() -*/ -FORCE_INLINE size_t ZSTDv07_buildSeqTable(FSEv07_DTable* DTable, U32 type, U32 max, U32 maxLog, - const void* src, size_t srcSize, - const S16* defaultNorm, U32 defaultLog, U32 flagRepeatTable) -{ - switch(type) - { - case FSEv07_ENCODING_RLE : - if (!srcSize) return ERROR(srcSize_wrong); - if ( (*(const BYTE*)src) > max) return ERROR(corruption_detected); - FSEv07_buildDTable_rle(DTable, *(const BYTE*)src); /* if *src > max, data is corrupted */ - return 1; - case FSEv07_ENCODING_RAW : - FSEv07_buildDTable(DTable, defaultNorm, max, defaultLog); - return 0; - case FSEv07_ENCODING_STATIC: - if (!flagRepeatTable) return ERROR(corruption_detected); - return 0; - default : /* impossible */ - case FSEv07_ENCODING_DYNAMIC : - { U32 tableLog; - S16 norm[MaxSeq+1]; - size_t const headerSize = FSEv07_readNCount(norm, &max, &tableLog, src, srcSize); - if (FSEv07_isError(headerSize)) return ERROR(corruption_detected); - if (tableLog > maxLog) return ERROR(corruption_detected); - FSEv07_buildDTable(DTable, norm, max, tableLog); - return headerSize; - } } -} - - -size_t ZSTDv07_decodeSeqHeaders(int* nbSeqPtr, - FSEv07_DTable* DTableLL, FSEv07_DTable* DTableML, FSEv07_DTable* DTableOffb, U32 flagRepeatTable, - const void* src, size_t srcSize) -{ - const BYTE* const istart = (const BYTE* const)src; - const BYTE* const iend = istart + srcSize; - const BYTE* ip = istart; - - /* check */ - if (srcSize < MIN_SEQUENCES_SIZE) return ERROR(srcSize_wrong); - - /* SeqHead */ - { int nbSeq = *ip++; - if (!nbSeq) { *nbSeqPtr=0; return 1; } - if (nbSeq > 0x7F) { - if (nbSeq == 0xFF) - nbSeq = MEM_readLE16(ip) + LONGNBSEQ, ip+=2; - else - nbSeq = ((nbSeq-0x80)<<8) + *ip++; - } - *nbSeqPtr = nbSeq; - } - - /* FSE table descriptors */ - { U32 const LLtype = *ip >> 6; - U32 const OFtype = (*ip >> 4) & 3; - U32 const MLtype = (*ip >> 2) & 3; - ip++; - - /* check */ - if (ip > iend-3) return ERROR(srcSize_wrong); /* min : all 3 are "raw", hence no header, but at least xxLog bits per type */ - - /* Build DTables */ - { size_t const llhSize = ZSTDv07_buildSeqTable(DTableLL, LLtype, MaxLL, LLFSELog, ip, iend-ip, LL_defaultNorm, LL_defaultNormLog, flagRepeatTable); - if (ZSTDv07_isError(llhSize)) return ERROR(corruption_detected); - ip += llhSize; - } - { size_t const ofhSize = ZSTDv07_buildSeqTable(DTableOffb, OFtype, MaxOff, OffFSELog, ip, iend-ip, OF_defaultNorm, OF_defaultNormLog, flagRepeatTable); - if (ZSTDv07_isError(ofhSize)) return ERROR(corruption_detected); - ip += ofhSize; - } - { size_t const mlhSize = ZSTDv07_buildSeqTable(DTableML, MLtype, MaxML, MLFSELog, ip, iend-ip, ML_defaultNorm, ML_defaultNormLog, flagRepeatTable); - if (ZSTDv07_isError(mlhSize)) return ERROR(corruption_detected); - ip += mlhSize; - } } - - return ip-istart; -} - - -typedef struct { - size_t litLength; - size_t matchLength; - size_t offset; -} seq_t; - -typedef struct { - BITv07_DStream_t DStream; - FSEv07_DState_t stateLL; - FSEv07_DState_t stateOffb; - FSEv07_DState_t stateML; - size_t prevOffset[ZSTDv07_REP_INIT]; -} seqState_t; - - -static seq_t ZSTDv07_decodeSequence(seqState_t* seqState) -{ - seq_t seq; - - U32 const llCode = FSEv07_peekSymbol(&(seqState->stateLL)); - U32 const mlCode = FSEv07_peekSymbol(&(seqState->stateML)); - U32 const ofCode = FSEv07_peekSymbol(&(seqState->stateOffb)); /* <= maxOff, by table construction */ - - U32 const llBits = LL_bits[llCode]; - U32 const mlBits = ML_bits[mlCode]; - U32 const ofBits = ofCode; - U32 const totalBits = llBits+mlBits+ofBits; - - static const U32 LL_base[MaxLL+1] = { - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 16, 18, 20, 22, 24, 28, 32, 40, 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, - 0x2000, 0x4000, 0x8000, 0x10000 }; - - static const U32 ML_base[MaxML+1] = { - 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, - 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, - 35, 37, 39, 41, 43, 47, 51, 59, 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, - 0x1003, 0x2003, 0x4003, 0x8003, 0x10003 }; - - static const U32 OF_base[MaxOff+1] = { - 0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, - 0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, - 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, - 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD }; - - /* sequence */ - { size_t offset; - if (!ofCode) - offset = 0; - else { - offset = OF_base[ofCode] + BITv07_readBits(&(seqState->DStream), ofBits); /* <= (ZSTDv07_WINDOWLOG_MAX-1) bits */ - if (MEM_32bits()) BITv07_reloadDStream(&(seqState->DStream)); - } - - if (ofCode <= 1) { - if ((llCode == 0) & (offset <= 1)) offset = 1-offset; - if (offset) { - size_t const temp = seqState->prevOffset[offset]; - if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1]; - seqState->prevOffset[1] = seqState->prevOffset[0]; - seqState->prevOffset[0] = offset = temp; - } else { - offset = seqState->prevOffset[0]; - } - } else { - seqState->prevOffset[2] = seqState->prevOffset[1]; - seqState->prevOffset[1] = seqState->prevOffset[0]; - seqState->prevOffset[0] = offset; - } - seq.offset = offset; - } - - seq.matchLength = ML_base[mlCode] + ((mlCode>31) ? BITv07_readBits(&(seqState->DStream), mlBits) : 0); /* <= 16 bits */ - if (MEM_32bits() && (mlBits+llBits>24)) BITv07_reloadDStream(&(seqState->DStream)); - - seq.litLength = LL_base[llCode] + ((llCode>15) ? BITv07_readBits(&(seqState->DStream), llBits) : 0); /* <= 16 bits */ - if (MEM_32bits() || - (totalBits > 64 - 7 - (LLFSELog+MLFSELog+OffFSELog)) ) BITv07_reloadDStream(&(seqState->DStream)); - - /* ANS state update */ - FSEv07_updateState(&(seqState->stateLL), &(seqState->DStream)); /* <= 9 bits */ - FSEv07_updateState(&(seqState->stateML), &(seqState->DStream)); /* <= 9 bits */ - if (MEM_32bits()) BITv07_reloadDStream(&(seqState->DStream)); /* <= 18 bits */ - FSEv07_updateState(&(seqState->stateOffb), &(seqState->DStream)); /* <= 8 bits */ - - return seq; -} - - -FORCE_INLINE -size_t ZSTDv07_execSequence(BYTE* op, - BYTE* const oend, seq_t sequence, - const BYTE** litPtr, const BYTE* const litLimit_w, - const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd) -{ - BYTE* const oLitEnd = op + sequence.litLength; - size_t const sequenceLength = sequence.litLength + sequence.matchLength; - BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ - BYTE* const oend_w = oend-WILDCOPY_OVERLENGTH; - const BYTE* const iLitEnd = *litPtr + sequence.litLength; - const BYTE* match = oLitEnd - sequence.offset; - - /* check */ - if ((oLitEnd>oend_w) | (oMatchEnd>oend)) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */ - if (iLitEnd > litLimit_w) return ERROR(corruption_detected); /* over-read beyond lit buffer */ - - /* copy Literals */ - ZSTDv07_wildcopy(op, *litPtr, sequence.litLength); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ - op = oLitEnd; - *litPtr = iLitEnd; /* update for next sequence */ - - /* copy Match */ - if (sequence.offset > (size_t)(oLitEnd - base)) { - /* offset beyond prefix */ - if (sequence.offset > (size_t)(oLitEnd - vBase)) return ERROR(corruption_detected); - match = dictEnd - (base-match); - if (match + sequence.matchLength <= dictEnd) { - memmove(oLitEnd, match, sequence.matchLength); - return sequenceLength; - } - /* span extDict & currentPrefixSegment */ - { size_t const length1 = dictEnd - match; - memmove(oLitEnd, match, length1); - op = oLitEnd + length1; - sequence.matchLength -= length1; - match = base; - } } - - /* match within prefix */ - if (sequence.offset < 8) { - /* close range match, overlap */ - static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */ - static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* substracted */ - int const sub2 = dec64table[sequence.offset]; - op[0] = match[0]; - op[1] = match[1]; - op[2] = match[2]; - op[3] = match[3]; - match += dec32table[sequence.offset]; - ZSTDv07_copy4(op+4, match); - match -= sub2; - } else { - ZSTDv07_copy8(op, match); - } - op += 8; match += 8; - - if (oMatchEnd > oend-(16-MINMATCH)) { - if (op < oend_w) { - ZSTDv07_wildcopy(op, match, oend_w - op); - match += oend_w - op; - op = oend_w; - } - while (op < oMatchEnd) *op++ = *match++; - } else { - ZSTDv07_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */ - } - return sequenceLength; -} - - -static size_t ZSTDv07_decompressSequences( - ZSTDv07_DCtx* dctx, - void* dst, size_t maxDstSize, - const void* seqStart, size_t seqSize) -{ - const BYTE* ip = (const BYTE*)seqStart; - const BYTE* const iend = ip + seqSize; - BYTE* const ostart = (BYTE* const)dst; - BYTE* const oend = ostart + maxDstSize; - BYTE* op = ostart; - const BYTE* litPtr = dctx->litPtr; - const BYTE* const litLimit_w = litPtr + dctx->litBufSize - WILDCOPY_OVERLENGTH; - const BYTE* const litEnd = litPtr + dctx->litSize; - FSEv07_DTable* DTableLL = dctx->LLTable; - FSEv07_DTable* DTableML = dctx->MLTable; - FSEv07_DTable* DTableOffb = dctx->OffTable; - const BYTE* const base = (const BYTE*) (dctx->base); - const BYTE* const vBase = (const BYTE*) (dctx->vBase); - const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); - int nbSeq; - - /* Build Decoding Tables */ - { size_t const seqHSize = ZSTDv07_decodeSeqHeaders(&nbSeq, DTableLL, DTableML, DTableOffb, dctx->fseEntropy, ip, seqSize); - if (ZSTDv07_isError(seqHSize)) return seqHSize; - ip += seqHSize; - } - - /* Regen sequences */ - if (nbSeq) { - seqState_t seqState; - dctx->fseEntropy = 1; - { U32 i; for (i=0; i<ZSTDv07_REP_INIT; i++) seqState.prevOffset[i] = dctx->rep[i]; } - { size_t const errorCode = BITv07_initDStream(&(seqState.DStream), ip, iend-ip); - if (ERR_isError(errorCode)) return ERROR(corruption_detected); } - FSEv07_initDState(&(seqState.stateLL), &(seqState.DStream), DTableLL); - FSEv07_initDState(&(seqState.stateOffb), &(seqState.DStream), DTableOffb); - FSEv07_initDState(&(seqState.stateML), &(seqState.DStream), DTableML); - - for ( ; (BITv07_reloadDStream(&(seqState.DStream)) <= BITv07_DStream_completed) && nbSeq ; ) { - nbSeq--; - { seq_t const sequence = ZSTDv07_decodeSequence(&seqState); - size_t const oneSeqSize = ZSTDv07_execSequence(op, oend, sequence, &litPtr, litLimit_w, base, vBase, dictEnd); - if (ZSTDv07_isError(oneSeqSize)) return oneSeqSize; - op += oneSeqSize; - } } - - /* check if reached exact end */ - if (nbSeq) return ERROR(corruption_detected); - /* save reps for next block */ - { U32 i; for (i=0; i<ZSTDv07_REP_INIT; i++) dctx->rep[i] = (U32)(seqState.prevOffset[i]); } - } - - /* last literal segment */ - { size_t const lastLLSize = litEnd - litPtr; - //if (litPtr > litEnd) return ERROR(corruption_detected); /* too many literals already used */ - if (lastLLSize > (size_t)(oend-op)) return ERROR(dstSize_tooSmall); - memcpy(op, litPtr, lastLLSize); - op += lastLLSize; - } - - return op-ostart; -} - - -static void ZSTDv07_checkContinuity(ZSTDv07_DCtx* dctx, const void* dst) -{ - if (dst != dctx->previousDstEnd) { /* not contiguous */ - dctx->dictEnd = dctx->previousDstEnd; - dctx->vBase = (const char*)dst - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->base)); - dctx->base = dst; - dctx->previousDstEnd = dst; - } -} - - -static size_t ZSTDv07_decompressBlock_internal(ZSTDv07_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize) -{ /* blockType == blockCompressed */ - const BYTE* ip = (const BYTE*)src; - - if (srcSize >= ZSTDv07_BLOCKSIZE_ABSOLUTEMAX) return ERROR(srcSize_wrong); - - /* Decode literals sub-block */ - { size_t const litCSize = ZSTDv07_decodeLiteralsBlock(dctx, src, srcSize); - if (ZSTDv07_isError(litCSize)) return litCSize; - ip += litCSize; - srcSize -= litCSize; - } - return ZSTDv07_decompressSequences(dctx, dst, dstCapacity, ip, srcSize); -} - - -size_t ZSTDv07_decompressBlock(ZSTDv07_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize) -{ - size_t dSize; - ZSTDv07_checkContinuity(dctx, dst); - dSize = ZSTDv07_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize); - dctx->previousDstEnd = (char*)dst + dSize; - return dSize; -} - - -/** ZSTDv07_insertBlock() : - insert `src` block into `dctx` history. Useful to track uncompressed blocks. */ -ZSTDLIB_API size_t ZSTDv07_insertBlock(ZSTDv07_DCtx* dctx, const void* blockStart, size_t blockSize) -{ - ZSTDv07_checkContinuity(dctx, blockStart); - dctx->previousDstEnd = (const char*)blockStart + blockSize; - return blockSize; -} - - -size_t ZSTDv07_generateNxBytes(void* dst, size_t dstCapacity, BYTE byte, size_t length) -{ - if (length > dstCapacity) return ERROR(dstSize_tooSmall); - memset(dst, byte, length); - return length; -} - - -/*! ZSTDv07_decompressFrame() : -* `dctx` must be properly initialized */ -static size_t ZSTDv07_decompressFrame(ZSTDv07_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize) -{ - const BYTE* ip = (const BYTE*)src; - const BYTE* const iend = ip + srcSize; - BYTE* const ostart = (BYTE* const)dst; - BYTE* const oend = ostart + dstCapacity; - BYTE* op = ostart; - size_t remainingSize = srcSize; - - /* check */ - if (srcSize < ZSTDv07_frameHeaderSize_min+ZSTDv07_blockHeaderSize) return ERROR(srcSize_wrong); - - /* Frame Header */ - { size_t const frameHeaderSize = ZSTDv07_frameHeaderSize(src, ZSTDv07_frameHeaderSize_min); - if (ZSTDv07_isError(frameHeaderSize)) return frameHeaderSize; - if (srcSize < frameHeaderSize+ZSTDv07_blockHeaderSize) return ERROR(srcSize_wrong); - if (ZSTDv07_decodeFrameHeader(dctx, src, frameHeaderSize)) return ERROR(corruption_detected); - ip += frameHeaderSize; remainingSize -= frameHeaderSize; - } - - /* Loop on each block */ - while (1) { - size_t decodedSize; - blockProperties_t blockProperties; - size_t const cBlockSize = ZSTDv07_getcBlockSize(ip, iend-ip, &blockProperties); - if (ZSTDv07_isError(cBlockSize)) return cBlockSize; - - ip += ZSTDv07_blockHeaderSize; - remainingSize -= ZSTDv07_blockHeaderSize; - if (cBlockSize > remainingSize) return ERROR(srcSize_wrong); - - switch(blockProperties.blockType) - { - case bt_compressed: - decodedSize = ZSTDv07_decompressBlock_internal(dctx, op, oend-op, ip, cBlockSize); - break; - case bt_raw : - decodedSize = ZSTDv07_copyRawBlock(op, oend-op, ip, cBlockSize); - break; - case bt_rle : - decodedSize = ZSTDv07_generateNxBytes(op, oend-op, *ip, blockProperties.origSize); - break; - case bt_end : - /* end of frame */ - if (remainingSize) return ERROR(srcSize_wrong); - decodedSize = 0; - break; - default: - return ERROR(GENERIC); /* impossible */ - } - if (blockProperties.blockType == bt_end) break; /* bt_end */ - - if (ZSTDv07_isError(decodedSize)) return decodedSize; - if (dctx->fParams.checksumFlag) XXH64_update(&dctx->xxhState, op, decodedSize); - op += decodedSize; - ip += cBlockSize; - remainingSize -= cBlockSize; - } - - return op-ostart; -} - - -/*! ZSTDv07_decompress_usingPreparedDCtx() : -* Same as ZSTDv07_decompress_usingDict, but using a reference context `preparedDCtx`, where dictionary has been loaded. -* It avoids reloading the dictionary each time. -* `preparedDCtx` must have been properly initialized using ZSTDv07_decompressBegin_usingDict(). -* Requires 2 contexts : 1 for reference (preparedDCtx), which will not be modified, and 1 to run the decompression operation (dctx) */ -size_t ZSTDv07_decompress_usingPreparedDCtx(ZSTDv07_DCtx* dctx, const ZSTDv07_DCtx* refDCtx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize) -{ - ZSTDv07_copyDCtx(dctx, refDCtx); - ZSTDv07_checkContinuity(dctx, dst); - return ZSTDv07_decompressFrame(dctx, dst, dstCapacity, src, srcSize); -} - - -size_t ZSTDv07_decompress_usingDict(ZSTDv07_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const void* dict, size_t dictSize) -{ - ZSTDv07_decompressBegin_usingDict(dctx, dict, dictSize); - ZSTDv07_checkContinuity(dctx, dst); - return ZSTDv07_decompressFrame(dctx, dst, dstCapacity, src, srcSize); -} - - -size_t ZSTDv07_decompressDCtx(ZSTDv07_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) -{ - return ZSTDv07_decompress_usingDict(dctx, dst, dstCapacity, src, srcSize, NULL, 0); -} - - -size_t ZSTDv07_decompress(void* dst, size_t dstCapacity, const void* src, size_t srcSize) -{ -#if defined(ZSTDv07_HEAPMODE) && (ZSTDv07_HEAPMODE==1) - size_t regenSize; - ZSTDv07_DCtx* const dctx = ZSTDv07_createDCtx(); - if (dctx==NULL) return ERROR(memory_allocation); - regenSize = ZSTDv07_decompressDCtx(dctx, dst, dstCapacity, src, srcSize); - ZSTDv07_freeDCtx(dctx); - return regenSize; -#else /* stack mode */ - ZSTDv07_DCtx dctx; - return ZSTDv07_decompressDCtx(&dctx, dst, dstCapacity, src, srcSize); -#endif -} - - -/*_****************************** -* Streaming Decompression API -********************************/ -size_t ZSTDv07_nextSrcSizeToDecompress(ZSTDv07_DCtx* dctx) -{ - return dctx->expected; -} - -int ZSTDv07_isSkipFrame(ZSTDv07_DCtx* dctx) -{ - return dctx->stage == ZSTDds_skipFrame; -} - -/** ZSTDv07_decompressContinue() : -* @return : nb of bytes generated into `dst` (necessarily <= `dstCapacity) -* or an error code, which can be tested using ZSTDv07_isError() */ -size_t ZSTDv07_decompressContinue(ZSTDv07_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) -{ - /* Sanity check */ - if (srcSize != dctx->expected) return ERROR(srcSize_wrong); - if (dstCapacity) ZSTDv07_checkContinuity(dctx, dst); - - switch (dctx->stage) - { - case ZSTDds_getFrameHeaderSize : - if (srcSize != ZSTDv07_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */ - if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTDv07_MAGIC_SKIPPABLE_START) { - memcpy(dctx->headerBuffer, src, ZSTDv07_frameHeaderSize_min); - dctx->expected = ZSTDv07_skippableHeaderSize - ZSTDv07_frameHeaderSize_min; /* magic number + skippable frame length */ - dctx->stage = ZSTDds_decodeSkippableHeader; - return 0; - } - dctx->headerSize = ZSTDv07_frameHeaderSize(src, ZSTDv07_frameHeaderSize_min); - if (ZSTDv07_isError(dctx->headerSize)) return dctx->headerSize; - memcpy(dctx->headerBuffer, src, ZSTDv07_frameHeaderSize_min); - if (dctx->headerSize > ZSTDv07_frameHeaderSize_min) { - dctx->expected = dctx->headerSize - ZSTDv07_frameHeaderSize_min; - dctx->stage = ZSTDds_decodeFrameHeader; - return 0; - } - dctx->expected = 0; /* not necessary to copy more */ - - case ZSTDds_decodeFrameHeader: - { size_t result; - memcpy(dctx->headerBuffer + ZSTDv07_frameHeaderSize_min, src, dctx->expected); - result = ZSTDv07_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize); - if (ZSTDv07_isError(result)) return result; - dctx->expected = ZSTDv07_blockHeaderSize; - dctx->stage = ZSTDds_decodeBlockHeader; - return 0; - } - case ZSTDds_decodeBlockHeader: - { blockProperties_t bp; - size_t const cBlockSize = ZSTDv07_getcBlockSize(src, ZSTDv07_blockHeaderSize, &bp); - if (ZSTDv07_isError(cBlockSize)) return cBlockSize; - if (bp.blockType == bt_end) { - if (dctx->fParams.checksumFlag) { - U64 const h64 = XXH64_digest(&dctx->xxhState); - U32 const h32 = (U32)(h64>>11) & ((1<<22)-1); - const BYTE* const ip = (const BYTE*)src; - U32 const check32 = ip[2] + (ip[1] << 8) + ((ip[0] & 0x3F) << 16); - if (check32 != h32) return ERROR(checksum_wrong); - } - dctx->expected = 0; - dctx->stage = ZSTDds_getFrameHeaderSize; - } else { - dctx->expected = cBlockSize; - dctx->bType = bp.blockType; - dctx->stage = ZSTDds_decompressBlock; - } - return 0; - } - case ZSTDds_decompressBlock: - { size_t rSize; - switch(dctx->bType) - { - case bt_compressed: - rSize = ZSTDv07_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize); - break; - case bt_raw : - rSize = ZSTDv07_copyRawBlock(dst, dstCapacity, src, srcSize); - break; - case bt_rle : - return ERROR(GENERIC); /* not yet handled */ - break; - case bt_end : /* should never happen (filtered at phase 1) */ - rSize = 0; - break; - default: - return ERROR(GENERIC); /* impossible */ - } - dctx->stage = ZSTDds_decodeBlockHeader; - dctx->expected = ZSTDv07_blockHeaderSize; - dctx->previousDstEnd = (char*)dst + rSize; - if (ZSTDv07_isError(rSize)) return rSize; - if (dctx->fParams.checksumFlag) XXH64_update(&dctx->xxhState, dst, rSize); - return rSize; - } - case ZSTDds_decodeSkippableHeader: - { memcpy(dctx->headerBuffer + ZSTDv07_frameHeaderSize_min, src, dctx->expected); - dctx->expected = MEM_readLE32(dctx->headerBuffer + 4); - dctx->stage = ZSTDds_skipFrame; - return 0; - } - case ZSTDds_skipFrame: - { dctx->expected = 0; - dctx->stage = ZSTDds_getFrameHeaderSize; - return 0; - } - default: - return ERROR(GENERIC); /* impossible */ - } -} - - -static size_t ZSTDv07_refDictContent(ZSTDv07_DCtx* dctx, const void* dict, size_t dictSize) -{ - dctx->dictEnd = dctx->previousDstEnd; - dctx->vBase = (const char*)dict - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->base)); - dctx->base = dict; - dctx->previousDstEnd = (const char*)dict + dictSize; - return 0; -} - -static size_t ZSTDv07_loadEntropy(ZSTDv07_DCtx* dctx, const void* const dict, size_t const dictSize) -{ - const BYTE* dictPtr = (const BYTE*)dict; - const BYTE* const dictEnd = dictPtr + dictSize; - - { size_t const hSize = HUFv07_readDTableX4(dctx->hufTable, dict, dictSize); - if (HUFv07_isError(hSize)) return ERROR(dictionary_corrupted); - dictPtr += hSize; - } - - { short offcodeNCount[MaxOff+1]; - U32 offcodeMaxValue=MaxOff, offcodeLog=OffFSELog; - size_t const offcodeHeaderSize = FSEv07_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr); - if (FSEv07_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); - { size_t const errorCode = FSEv07_buildDTable(dctx->OffTable, offcodeNCount, offcodeMaxValue, offcodeLog); - if (FSEv07_isError(errorCode)) return ERROR(dictionary_corrupted); } - dictPtr += offcodeHeaderSize; - } - - { short matchlengthNCount[MaxML+1]; - unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog; - size_t const matchlengthHeaderSize = FSEv07_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr); - if (FSEv07_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted); - { size_t const errorCode = FSEv07_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog); - if (FSEv07_isError(errorCode)) return ERROR(dictionary_corrupted); } - dictPtr += matchlengthHeaderSize; - } - - { short litlengthNCount[MaxLL+1]; - unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog; - size_t const litlengthHeaderSize = FSEv07_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr); - if (FSEv07_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted); - { size_t const errorCode = FSEv07_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog); - if (FSEv07_isError(errorCode)) return ERROR(dictionary_corrupted); } - dictPtr += litlengthHeaderSize; - } - - if (dictPtr+12 > dictEnd) return ERROR(dictionary_corrupted); - dctx->rep[0] = MEM_readLE32(dictPtr+0); if (dctx->rep[0] >= dictSize) return ERROR(dictionary_corrupted); - dctx->rep[1] = MEM_readLE32(dictPtr+4); if (dctx->rep[1] >= dictSize) return ERROR(dictionary_corrupted); - dctx->rep[2] = MEM_readLE32(dictPtr+8); if (dctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted); - dictPtr += 12; - - dctx->litEntropy = dctx->fseEntropy = 1; - return dictPtr - (const BYTE*)dict; -} - -static size_t ZSTDv07_decompress_insertDictionary(ZSTDv07_DCtx* dctx, const void* dict, size_t dictSize) -{ - if (dictSize < 8) return ZSTDv07_refDictContent(dctx, dict, dictSize); - { U32 const magic = MEM_readLE32(dict); - if (magic != ZSTDv07_DICT_MAGIC) { - return ZSTDv07_refDictContent(dctx, dict, dictSize); /* pure content mode */ - } } - dctx->dictID = MEM_readLE32((const char*)dict + 4); - - /* load entropy tables */ - dict = (const char*)dict + 8; - dictSize -= 8; - { size_t const eSize = ZSTDv07_loadEntropy(dctx, dict, dictSize); - if (ZSTDv07_isError(eSize)) return ERROR(dictionary_corrupted); - dict = (const char*)dict + eSize; - dictSize -= eSize; - } - - /* reference dictionary content */ - return ZSTDv07_refDictContent(dctx, dict, dictSize); -} - - -size_t ZSTDv07_decompressBegin_usingDict(ZSTDv07_DCtx* dctx, const void* dict, size_t dictSize) -{ - { size_t const errorCode = ZSTDv07_decompressBegin(dctx); - if (ZSTDv07_isError(errorCode)) return errorCode; } - - if (dict && dictSize) { - size_t const errorCode = ZSTDv07_decompress_insertDictionary(dctx, dict, dictSize); - if (ZSTDv07_isError(errorCode)) return ERROR(dictionary_corrupted); - } - - return 0; -} - - -struct ZSTDv07_DDict_s { - void* dict; - size_t dictSize; - ZSTDv07_DCtx* refContext; -}; /* typedef'd tp ZSTDv07_CDict within zstd.h */ - -ZSTDv07_DDict* ZSTDv07_createDDict_advanced(const void* dict, size_t dictSize, ZSTDv07_customMem customMem) -{ - if (!customMem.customAlloc && !customMem.customFree) - customMem = defaultCustomMem; - - if (!customMem.customAlloc || !customMem.customFree) - return NULL; - - { ZSTDv07_DDict* const ddict = (ZSTDv07_DDict*) customMem.customAlloc(customMem.opaque, sizeof(*ddict)); - void* const dictContent = customMem.customAlloc(customMem.opaque, dictSize); - ZSTDv07_DCtx* const dctx = ZSTDv07_createDCtx_advanced(customMem); - - if (!dictContent || !ddict || !dctx) { - customMem.customFree(customMem.opaque, dictContent); - customMem.customFree(customMem.opaque, ddict); - customMem.customFree(customMem.opaque, dctx); - return NULL; - } - - memcpy(dictContent, dict, dictSize); - { size_t const errorCode = ZSTDv07_decompressBegin_usingDict(dctx, dictContent, dictSize); - if (ZSTDv07_isError(errorCode)) { - customMem.customFree(customMem.opaque, dictContent); - customMem.customFree(customMem.opaque, ddict); - customMem.customFree(customMem.opaque, dctx); - return NULL; - } } - - ddict->dict = dictContent; - ddict->dictSize = dictSize; - ddict->refContext = dctx; - return ddict; - } -} - -/*! ZSTDv07_createDDict() : -* Create a digested dictionary, ready to start decompression without startup delay. -* `dict` can be released after `ZSTDv07_DDict` creation */ -ZSTDv07_DDict* ZSTDv07_createDDict(const void* dict, size_t dictSize) -{ - ZSTDv07_customMem const allocator = { NULL, NULL, NULL }; - return ZSTDv07_createDDict_advanced(dict, dictSize, allocator); -} - -size_t ZSTDv07_freeDDict(ZSTDv07_DDict* ddict) -{ - ZSTDv07_freeFunction const cFree = ddict->refContext->customMem.customFree; - void* const opaque = ddict->refContext->customMem.opaque; - ZSTDv07_freeDCtx(ddict->refContext); - cFree(opaque, ddict->dict); - cFree(opaque, ddict); - return 0; -} - -/*! ZSTDv07_decompress_usingDDict() : -* Decompression using a pre-digested Dictionary -* Use dictionary without significant overhead. */ -ZSTDLIB_API size_t ZSTDv07_decompress_usingDDict(ZSTDv07_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const ZSTDv07_DDict* ddict) -{ - return ZSTDv07_decompress_usingPreparedDCtx(dctx, ddict->refContext, - dst, dstCapacity, - src, srcSize); -} -/* - Buffered version of Zstd compression library - Copyright (C) 2015-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - zstd homepage : http://www.zstd.net/ -*/ - - - -/*-*************************************************************************** -* Streaming decompression howto -* -* A ZBUFFv07_DCtx object is required to track streaming operations. -* Use ZBUFFv07_createDCtx() and ZBUFFv07_freeDCtx() to create/release resources. -* Use ZBUFFv07_decompressInit() to start a new decompression operation, -* or ZBUFFv07_decompressInitDictionary() if decompression requires a dictionary. -* Note that ZBUFFv07_DCtx objects can be re-init multiple times. -* -* Use ZBUFFv07_decompressContinue() repetitively to consume your input. -* *srcSizePtr and *dstCapacityPtr can be any size. -* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr. -* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again. -* The content of @dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change @dst. -* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency), -* or 0 when a frame is completely decoded, -* or an error code, which can be tested using ZBUFFv07_isError(). -* -* Hint : recommended buffer sizes (not compulsory) : ZBUFFv07_recommendedDInSize() and ZBUFFv07_recommendedDOutSize() -* output : ZBUFFv07_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded. -* input : ZBUFFv07_recommendedDInSize == 128KB + 3; -* just follow indications from ZBUFFv07_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . -* *******************************************************************************/ - -typedef enum { ZBUFFds_init, ZBUFFds_loadHeader, - ZBUFFds_read, ZBUFFds_load, ZBUFFds_flush } ZBUFFv07_dStage; - -/* *** Resource management *** */ -struct ZBUFFv07_DCtx_s { - ZSTDv07_DCtx* zd; - ZSTDv07_frameParams fParams; - ZBUFFv07_dStage stage; - char* inBuff; - size_t inBuffSize; - size_t inPos; - char* outBuff; - size_t outBuffSize; - size_t outStart; - size_t outEnd; - size_t blockSize; - BYTE headerBuffer[ZSTDv07_FRAMEHEADERSIZE_MAX]; - size_t lhSize; - ZSTDv07_customMem customMem; -}; /* typedef'd to ZBUFFv07_DCtx within "zstd_buffered.h" */ - -ZSTDLIB_API ZBUFFv07_DCtx* ZBUFFv07_createDCtx_advanced(ZSTDv07_customMem customMem); - -ZBUFFv07_DCtx* ZBUFFv07_createDCtx(void) -{ - return ZBUFFv07_createDCtx_advanced(defaultCustomMem); -} - -ZBUFFv07_DCtx* ZBUFFv07_createDCtx_advanced(ZSTDv07_customMem customMem) -{ - ZBUFFv07_DCtx* zbd; - - if (!customMem.customAlloc && !customMem.customFree) - customMem = defaultCustomMem; - - if (!customMem.customAlloc || !customMem.customFree) - return NULL; - - zbd = (ZBUFFv07_DCtx*)customMem.customAlloc(customMem.opaque, sizeof(ZBUFFv07_DCtx)); - if (zbd==NULL) return NULL; - memset(zbd, 0, sizeof(ZBUFFv07_DCtx)); - memcpy(&zbd->customMem, &customMem, sizeof(ZSTDv07_customMem)); - zbd->zd = ZSTDv07_createDCtx_advanced(customMem); - if (zbd->zd == NULL) { ZBUFFv07_freeDCtx(zbd); return NULL; } - zbd->stage = ZBUFFds_init; - return zbd; -} - -size_t ZBUFFv07_freeDCtx(ZBUFFv07_DCtx* zbd) -{ - if (zbd==NULL) return 0; /* support free on null */ - ZSTDv07_freeDCtx(zbd->zd); - if (zbd->inBuff) zbd->customMem.customFree(zbd->customMem.opaque, zbd->inBuff); - if (zbd->outBuff) zbd->customMem.customFree(zbd->customMem.opaque, zbd->outBuff); - zbd->customMem.customFree(zbd->customMem.opaque, zbd); - return 0; -} - - -/* *** Initialization *** */ - -size_t ZBUFFv07_decompressInitDictionary(ZBUFFv07_DCtx* zbd, const void* dict, size_t dictSize) -{ - zbd->stage = ZBUFFds_loadHeader; - zbd->lhSize = zbd->inPos = zbd->outStart = zbd->outEnd = 0; - return ZSTDv07_decompressBegin_usingDict(zbd->zd, dict, dictSize); -} - -size_t ZBUFFv07_decompressInit(ZBUFFv07_DCtx* zbd) -{ - return ZBUFFv07_decompressInitDictionary(zbd, NULL, 0); -} - - -/* internal util function */ -MEM_STATIC size_t ZBUFFv07_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) -{ - size_t const length = MIN(dstCapacity, srcSize); - memcpy(dst, src, length); - return length; -} - - -/* *** Decompression *** */ - -size_t ZBUFFv07_decompressContinue(ZBUFFv07_DCtx* zbd, - void* dst, size_t* dstCapacityPtr, - const void* src, size_t* srcSizePtr) -{ - const char* const istart = (const char*)src; - const char* const iend = istart + *srcSizePtr; - const char* ip = istart; - char* const ostart = (char*)dst; - char* const oend = ostart + *dstCapacityPtr; - char* op = ostart; - U32 notDone = 1; - - while (notDone) { - switch(zbd->stage) - { - case ZBUFFds_init : - return ERROR(init_missing); - - case ZBUFFds_loadHeader : - { size_t const hSize = ZSTDv07_getFrameParams(&(zbd->fParams), zbd->headerBuffer, zbd->lhSize); - if (hSize != 0) { - size_t const toLoad = hSize - zbd->lhSize; /* if hSize!=0, hSize > zbd->lhSize */ - if (ZSTDv07_isError(hSize)) return hSize; - if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */ - memcpy(zbd->headerBuffer + zbd->lhSize, ip, iend-ip); - zbd->lhSize += iend-ip; - *dstCapacityPtr = 0; - return (hSize - zbd->lhSize) + ZSTDv07_blockHeaderSize; /* remaining header bytes + next block header */ - } - memcpy(zbd->headerBuffer + zbd->lhSize, ip, toLoad); zbd->lhSize = hSize; ip += toLoad; - break; - } } - - /* Consume header */ - { size_t const h1Size = ZSTDv07_nextSrcSizeToDecompress(zbd->zd); /* == ZSTDv07_frameHeaderSize_min */ - size_t const h1Result = ZSTDv07_decompressContinue(zbd->zd, NULL, 0, zbd->headerBuffer, h1Size); - if (ZSTDv07_isError(h1Result)) return h1Result; - if (h1Size < zbd->lhSize) { /* long header */ - size_t const h2Size = ZSTDv07_nextSrcSizeToDecompress(zbd->zd); - size_t const h2Result = ZSTDv07_decompressContinue(zbd->zd, NULL, 0, zbd->headerBuffer+h1Size, h2Size); - if (ZSTDv07_isError(h2Result)) return h2Result; - } } - - zbd->fParams.windowSize = MAX(zbd->fParams.windowSize, 1U << ZSTDv07_WINDOWLOG_ABSOLUTEMIN); - - /* Frame header instruct buffer sizes */ - { size_t const blockSize = MIN(zbd->fParams.windowSize, ZSTDv07_BLOCKSIZE_ABSOLUTEMAX); - zbd->blockSize = blockSize; - if (zbd->inBuffSize < blockSize) { - zbd->customMem.customFree(zbd->customMem.opaque, zbd->inBuff); - zbd->inBuffSize = blockSize; - zbd->inBuff = (char*)zbd->customMem.customAlloc(zbd->customMem.opaque, blockSize); - if (zbd->inBuff == NULL) return ERROR(memory_allocation); - } - { size_t const neededOutSize = zbd->fParams.windowSize + blockSize; - if (zbd->outBuffSize < neededOutSize) { - zbd->customMem.customFree(zbd->customMem.opaque, zbd->outBuff); - zbd->outBuffSize = neededOutSize; - zbd->outBuff = (char*)zbd->customMem.customAlloc(zbd->customMem.opaque, neededOutSize); - if (zbd->outBuff == NULL) return ERROR(memory_allocation); - } } } - zbd->stage = ZBUFFds_read; - - case ZBUFFds_read: - { size_t const neededInSize = ZSTDv07_nextSrcSizeToDecompress(zbd->zd); - if (neededInSize==0) { /* end of frame */ - zbd->stage = ZBUFFds_init; - notDone = 0; - break; - } - if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */ - const int isSkipFrame = ZSTDv07_isSkipFrame(zbd->zd); - size_t const decodedSize = ZSTDv07_decompressContinue(zbd->zd, - zbd->outBuff + zbd->outStart, (isSkipFrame ? 0 : zbd->outBuffSize - zbd->outStart), - ip, neededInSize); - if (ZSTDv07_isError(decodedSize)) return decodedSize; - ip += neededInSize; - if (!decodedSize && !isSkipFrame) break; /* this was just a header */ - zbd->outEnd = zbd->outStart + decodedSize; - zbd->stage = ZBUFFds_flush; - break; - } - if (ip==iend) { notDone = 0; break; } /* no more input */ - zbd->stage = ZBUFFds_load; - } - - case ZBUFFds_load: - { size_t const neededInSize = ZSTDv07_nextSrcSizeToDecompress(zbd->zd); - size_t const toLoad = neededInSize - zbd->inPos; /* should always be <= remaining space within inBuff */ - size_t loadedSize; - if (toLoad > zbd->inBuffSize - zbd->inPos) return ERROR(corruption_detected); /* should never happen */ - loadedSize = ZBUFFv07_limitCopy(zbd->inBuff + zbd->inPos, toLoad, ip, iend-ip); - ip += loadedSize; - zbd->inPos += loadedSize; - if (loadedSize < toLoad) { notDone = 0; break; } /* not enough input, wait for more */ - - /* decode loaded input */ - { const int isSkipFrame = ZSTDv07_isSkipFrame(zbd->zd); - size_t const decodedSize = ZSTDv07_decompressContinue(zbd->zd, - zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart, - zbd->inBuff, neededInSize); - if (ZSTDv07_isError(decodedSize)) return decodedSize; - zbd->inPos = 0; /* input is consumed */ - if (!decodedSize && !isSkipFrame) { zbd->stage = ZBUFFds_read; break; } /* this was just a header */ - zbd->outEnd = zbd->outStart + decodedSize; - zbd->stage = ZBUFFds_flush; - // break; /* ZBUFFds_flush follows */ - } } - - case ZBUFFds_flush: - { size_t const toFlushSize = zbd->outEnd - zbd->outStart; - size_t const flushedSize = ZBUFFv07_limitCopy(op, oend-op, zbd->outBuff + zbd->outStart, toFlushSize); - op += flushedSize; - zbd->outStart += flushedSize; - if (flushedSize == toFlushSize) { - zbd->stage = ZBUFFds_read; - if (zbd->outStart + zbd->blockSize > zbd->outBuffSize) - zbd->outStart = zbd->outEnd = 0; - break; - } - /* cannot flush everything */ - notDone = 0; - break; - } - default: return ERROR(GENERIC); /* impossible */ - } } - - /* result */ - *srcSizePtr = ip-istart; - *dstCapacityPtr = op-ostart; - { size_t nextSrcSizeHint = ZSTDv07_nextSrcSizeToDecompress(zbd->zd); - nextSrcSizeHint -= zbd->inPos; /* already loaded*/ - return nextSrcSizeHint; - } -} - - - -/* ************************************* -* Tool functions -***************************************/ -size_t ZBUFFv07_recommendedDInSize(void) { return ZSTDv07_BLOCKSIZE_ABSOLUTEMAX + ZSTDv07_blockHeaderSize /* block header size*/ ; } -size_t ZBUFFv07_recommendedDOutSize(void) { return ZSTDv07_BLOCKSIZE_ABSOLUTEMAX; } +/* ****************************************************************** + zstd_v07.c + Decompression module for ZSTD v0.7 legacy format + Copyright (C) 2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - Homepage : http://www.zstd.net/ +****************************************************************** */ + +/*- Dependencies -*/ +#include <stddef.h> /* size_t, ptrdiff_t */ +#include <string.h> /* memcpy */ +#include <stdlib.h> /* malloc, free, qsort */ + +#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ +#include "xxhash.h" /* XXH64_* */ +#include "zstd_v07.h" + +#define FSEv07_STATIC_LINKING_ONLY /* FSEv07_MIN_TABLELOG */ +#define HUFv07_STATIC_LINKING_ONLY /* HUFv07_TABLELOG_ABSOLUTEMAX */ +#define ZSTDv07_STATIC_LINKING_ONLY + + +#ifdef ZSTDv07_STATIC_LINKING_ONLY + +/* ==================================================================================== + * The definitions in this section are considered experimental. + * They should never be used with a dynamic library, as they may change in the future. + * They are provided for advanced usages. + * Use them only in association with static linking. + * ==================================================================================== */ + +/*--- Constants ---*/ +#define ZSTDv07_MAGIC_SKIPPABLE_START 0x184D2A50U + +#define ZSTDv07_WINDOWLOG_MAX_32 25 +#define ZSTDv07_WINDOWLOG_MAX_64 27 +#define ZSTDv07_WINDOWLOG_MAX ((U32)(MEM_32bits() ? ZSTDv07_WINDOWLOG_MAX_32 : ZSTDv07_WINDOWLOG_MAX_64)) +#define ZSTDv07_WINDOWLOG_MIN 18 +#define ZSTDv07_CHAINLOG_MAX (ZSTDv07_WINDOWLOG_MAX+1) +#define ZSTDv07_CHAINLOG_MIN 4 +#define ZSTDv07_HASHLOG_MAX ZSTDv07_WINDOWLOG_MAX +#define ZSTDv07_HASHLOG_MIN 12 +#define ZSTDv07_HASHLOG3_MAX 17 +#define ZSTDv07_SEARCHLOG_MAX (ZSTDv07_WINDOWLOG_MAX-1) +#define ZSTDv07_SEARCHLOG_MIN 1 +#define ZSTDv07_SEARCHLENGTH_MAX 7 +#define ZSTDv07_SEARCHLENGTH_MIN 3 +#define ZSTDv07_TARGETLENGTH_MIN 4 +#define ZSTDv07_TARGETLENGTH_MAX 999 + +#define ZSTDv07_FRAMEHEADERSIZE_MAX 18 /* for static allocation */ +static const size_t ZSTDv07_frameHeaderSize_min = 5; +static const size_t ZSTDv07_frameHeaderSize_max = ZSTDv07_FRAMEHEADERSIZE_MAX; +static const size_t ZSTDv07_skippableHeaderSize = 8; /* magic number + skippable frame length */ + + +/* custom memory allocation functions */ +typedef void* (*ZSTDv07_allocFunction) (void* opaque, size_t size); +typedef void (*ZSTDv07_freeFunction) (void* opaque, void* address); +typedef struct { ZSTDv07_allocFunction customAlloc; ZSTDv07_freeFunction customFree; void* opaque; } ZSTDv07_customMem; + + +/*--- Advanced Decompression functions ---*/ + +/*! ZSTDv07_estimateDCtxSize() : + * Gives the potential amount of memory allocated to create a ZSTDv07_DCtx */ +ZSTDLIB_API size_t ZSTDv07_estimateDCtxSize(void); + +/*! ZSTDv07_createDCtx_advanced() : + * Create a ZSTD decompression context using external alloc and free functions */ +ZSTDLIB_API ZSTDv07_DCtx* ZSTDv07_createDCtx_advanced(ZSTDv07_customMem customMem); + +/*! ZSTDv07_sizeofDCtx() : + * Gives the amount of memory used by a given ZSTDv07_DCtx */ +ZSTDLIB_API size_t ZSTDv07_sizeofDCtx(const ZSTDv07_DCtx* dctx); + + +/* ****************************************************************** +* Buffer-less streaming functions (synchronous mode) +********************************************************************/ + +ZSTDLIB_API size_t ZSTDv07_decompressBegin(ZSTDv07_DCtx* dctx); +ZSTDLIB_API size_t ZSTDv07_decompressBegin_usingDict(ZSTDv07_DCtx* dctx, const void* dict, size_t dictSize); +ZSTDLIB_API void ZSTDv07_copyDCtx(ZSTDv07_DCtx* dctx, const ZSTDv07_DCtx* preparedDCtx); + +ZSTDLIB_API size_t ZSTDv07_nextSrcSizeToDecompress(ZSTDv07_DCtx* dctx); +ZSTDLIB_API size_t ZSTDv07_decompressContinue(ZSTDv07_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); + +/* + Buffer-less streaming decompression (synchronous mode) + + A ZSTDv07_DCtx object is required to track streaming operations. + Use ZSTDv07_createDCtx() / ZSTDv07_freeDCtx() to manage it. + A ZSTDv07_DCtx object can be re-used multiple times. + + First optional operation is to retrieve frame parameters, using ZSTDv07_getFrameParams(), which doesn't consume the input. + It can provide the minimum size of rolling buffer required to properly decompress data (`windowSize`), + and optionally the final size of uncompressed content. + (Note : content size is an optional info that may not be present. 0 means : content size unknown) + Frame parameters are extracted from the beginning of compressed frame. + The amount of data to read is variable, from ZSTDv07_frameHeaderSize_min to ZSTDv07_frameHeaderSize_max (so if `srcSize` >= ZSTDv07_frameHeaderSize_max, it will always work) + If `srcSize` is too small for operation to succeed, function will return the minimum size it requires to produce a result. + Result : 0 when successful, it means the ZSTDv07_frameParams structure has been filled. + >0 : means there is not enough data into `src`. Provides the expected size to successfully decode header. + errorCode, which can be tested using ZSTDv07_isError() + + Start decompression, with ZSTDv07_decompressBegin() or ZSTDv07_decompressBegin_usingDict(). + Alternatively, you can copy a prepared context, using ZSTDv07_copyDCtx(). + + Then use ZSTDv07_nextSrcSizeToDecompress() and ZSTDv07_decompressContinue() alternatively. + ZSTDv07_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTDv07_decompressContinue(). + ZSTDv07_decompressContinue() requires this exact amount of bytes, or it will fail. + + @result of ZSTDv07_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity). + It can be zero, which is not an error; it just means ZSTDv07_decompressContinue() has decoded some header. + + ZSTDv07_decompressContinue() needs previous data blocks during decompression, up to `windowSize`. + They should preferably be located contiguously, prior to current block. + Alternatively, a round buffer of sufficient size is also possible. Sufficient size is determined by frame parameters. + ZSTDv07_decompressContinue() is very sensitive to contiguity, + if 2 blocks don't follow each other, make sure that either the compressor breaks contiguity at the same place, + or that previous contiguous segment is large enough to properly handle maximum back-reference. + + A frame is fully decoded when ZSTDv07_nextSrcSizeToDecompress() returns zero. + Context can then be reset to start a new decompression. + + + == Special case : skippable frames == + + Skippable frames allow the integration of user-defined data into a flow of concatenated frames. + Skippable frames will be ignored (skipped) by a decompressor. The format of skippable frame is following: + a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F + b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits + c) Frame Content - any content (User Data) of length equal to Frame Size + For skippable frames ZSTDv07_decompressContinue() always returns 0. + For skippable frames ZSTDv07_getFrameParams() returns fparamsPtr->windowLog==0 what means that a frame is skippable. + It also returns Frame Size as fparamsPtr->frameContentSize. +*/ + + +/* ************************************** +* Block functions +****************************************/ +/*! Block functions produce and decode raw zstd blocks, without frame metadata. + Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes). + User will have to take in charge required information to regenerate data, such as compressed and content sizes. + + A few rules to respect : + - Compressing and decompressing require a context structure + + Use ZSTDv07_createCCtx() and ZSTDv07_createDCtx() + - It is necessary to init context before starting + + compression : ZSTDv07_compressBegin() + + decompression : ZSTDv07_decompressBegin() + + variants _usingDict() are also allowed + + copyCCtx() and copyDCtx() work too + - Block size is limited, it must be <= ZSTDv07_getBlockSizeMax() + + If you need to compress more, cut data into multiple blocks + + Consider using the regular ZSTDv07_compress() instead, as frame metadata costs become negligible when source size is large. + - When a block is considered not compressible enough, ZSTDv07_compressBlock() result will be zero. + In which case, nothing is produced into `dst`. + + User must test for such outcome and deal directly with uncompressed data + + ZSTDv07_decompressBlock() doesn't accept uncompressed data as input !!! + + In case of multiple successive blocks, decoder must be informed of uncompressed block existence to follow proper history. + Use ZSTDv07_insertBlock() in such a case. +*/ + +#define ZSTDv07_BLOCKSIZE_ABSOLUTEMAX (128 * 1024) /* define, for static allocation */ +ZSTDLIB_API size_t ZSTDv07_decompressBlock(ZSTDv07_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTDLIB_API size_t ZSTDv07_insertBlock(ZSTDv07_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert block into `dctx` history. Useful for uncompressed blocks */ + + +#endif /* ZSTDv07_STATIC_LINKING_ONLY */ + + +/* ****************************************************************** + mem.h + low-level memory access routines + Copyright (C) 2013-2015, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + - Public forum : https://groups.google.com/forum/#!forum/lz4c +****************************************************************** */ +#ifndef MEM_H_MODULE +#define MEM_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif + +/*-**************************************** +* Compiler specifics +******************************************/ +#if defined(_MSC_VER) /* Visual Studio */ +# include <stdlib.h> /* _byteswap_ulong */ +# include <intrin.h> /* _byteswap_* */ +#endif +#if defined(__GNUC__) +# define MEM_STATIC static __attribute__((unused)) +#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define MEM_STATIC static inline +#elif defined(_MSC_VER) +# define MEM_STATIC static __inline +#else +# define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ +#endif + +/* code only tested on 32 and 64 bits systems */ +#define MEM_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; } +MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); } + + +/*-************************************************************** +* Basic Types +*****************************************************************/ +#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include <stdint.h> + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef int16_t S16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; + typedef int64_t S64; +#else + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef signed short S16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; + typedef signed long long S64; +#endif + + +/*-************************************************************** +* Memory I/O +*****************************************************************/ +/* MEM_FORCE_MEMORY_ACCESS : + * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. + * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. + * The below switch allow to select different access method for improved performance. + * Method 0 (default) : use `memcpy()`. Safe and portable. + * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). + * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. + * Method 2 : direct access. This method is portable but violate C standard. + * It can generate buggy code on targets depending on alignment. + * In some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) + * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. + * Prefer these methods in priority order (0 > 1 > 2) + */ +#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ +# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) +# define MEM_FORCE_MEMORY_ACCESS 2 +# elif defined(__INTEL_COMPILER) || \ + (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) +# define MEM_FORCE_MEMORY_ACCESS 1 +# endif +#endif + +MEM_STATIC unsigned MEM_32bits(void) { return sizeof(size_t)==4; } +MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; } + +MEM_STATIC unsigned MEM_isLittleEndian(void) +{ + const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ + return one.c[0]; +} + +#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2) + +/* violates C standard, by lying on structure alignment. +Only use if no other choice to achieve best performance on target platform */ +MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; } +MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; } +MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; } +MEM_STATIC U64 MEM_readST(const void* memPtr) { return *(const size_t*) memPtr; } + +MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; } +MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; } +MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; } + +#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1) + +/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ +/* currently only defined for gcc and icc */ +typedef union { U16 u16; U32 u32; U64 u64; size_t st; } __attribute__((packed)) unalign; + +MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; } +MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } +MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; } +MEM_STATIC U64 MEM_readST(const void* ptr) { return ((const unalign*)ptr)->st; } + +MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; } +MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; } +MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign*)memPtr)->u64 = value; } + +#else + +/* default method, safe and standard. + can sometimes prove slower */ + +MEM_STATIC U16 MEM_read16(const void* memPtr) +{ + U16 val; memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC U32 MEM_read32(const void* memPtr) +{ + U32 val; memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC U64 MEM_read64(const void* memPtr) +{ + U64 val; memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC size_t MEM_readST(const void* memPtr) +{ + size_t val; memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC void MEM_write16(void* memPtr, U16 value) +{ + memcpy(memPtr, &value, sizeof(value)); +} + +MEM_STATIC void MEM_write32(void* memPtr, U32 value) +{ + memcpy(memPtr, &value, sizeof(value)); +} + +MEM_STATIC void MEM_write64(void* memPtr, U64 value) +{ + memcpy(memPtr, &value, sizeof(value)); +} + +#endif /* MEM_FORCE_MEMORY_ACCESS */ + +MEM_STATIC U32 MEM_swap32(U32 in) +{ +#if defined(_MSC_VER) /* Visual Studio */ + return _byteswap_ulong(in); +#elif defined (__GNUC__) + return __builtin_bswap32(in); +#else + return ((in << 24) & 0xff000000 ) | + ((in << 8) & 0x00ff0000 ) | + ((in >> 8) & 0x0000ff00 ) | + ((in >> 24) & 0x000000ff ); +#endif +} + +MEM_STATIC U64 MEM_swap64(U64 in) +{ +#if defined(_MSC_VER) /* Visual Studio */ + return _byteswap_uint64(in); +#elif defined (__GNUC__) + return __builtin_bswap64(in); +#else + return ((in << 56) & 0xff00000000000000ULL) | + ((in << 40) & 0x00ff000000000000ULL) | + ((in << 24) & 0x0000ff0000000000ULL) | + ((in << 8) & 0x000000ff00000000ULL) | + ((in >> 8) & 0x00000000ff000000ULL) | + ((in >> 24) & 0x0000000000ff0000ULL) | + ((in >> 40) & 0x000000000000ff00ULL) | + ((in >> 56) & 0x00000000000000ffULL); +#endif +} + +MEM_STATIC size_t MEM_swapST(size_t in) +{ + if (MEM_32bits()) + return (size_t)MEM_swap32((U32)in); + else + return (size_t)MEM_swap64((U64)in); +} + +/*=== Little endian r/w ===*/ + +MEM_STATIC U16 MEM_readLE16(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_read16(memPtr); + else { + const BYTE* p = (const BYTE*)memPtr; + return (U16)(p[0] + (p[1]<<8)); + } +} + +MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val) +{ + if (MEM_isLittleEndian()) { + MEM_write16(memPtr, val); + } else { + BYTE* p = (BYTE*)memPtr; + p[0] = (BYTE)val; + p[1] = (BYTE)(val>>8); + } +} + +MEM_STATIC U32 MEM_readLE32(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_read32(memPtr); + else + return MEM_swap32(MEM_read32(memPtr)); +} + +MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32) +{ + if (MEM_isLittleEndian()) + MEM_write32(memPtr, val32); + else + MEM_write32(memPtr, MEM_swap32(val32)); +} + +MEM_STATIC U64 MEM_readLE64(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_read64(memPtr); + else + return MEM_swap64(MEM_read64(memPtr)); +} + +MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64) +{ + if (MEM_isLittleEndian()) + MEM_write64(memPtr, val64); + else + MEM_write64(memPtr, MEM_swap64(val64)); +} + +MEM_STATIC size_t MEM_readLEST(const void* memPtr) +{ + if (MEM_32bits()) + return (size_t)MEM_readLE32(memPtr); + else + return (size_t)MEM_readLE64(memPtr); +} + +MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val) +{ + if (MEM_32bits()) + MEM_writeLE32(memPtr, (U32)val); + else + MEM_writeLE64(memPtr, (U64)val); +} + +/*=== Big endian r/w ===*/ + +MEM_STATIC U32 MEM_readBE32(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_swap32(MEM_read32(memPtr)); + else + return MEM_read32(memPtr); +} + +MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32) +{ + if (MEM_isLittleEndian()) + MEM_write32(memPtr, MEM_swap32(val32)); + else + MEM_write32(memPtr, val32); +} + +MEM_STATIC U64 MEM_readBE64(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_swap64(MEM_read64(memPtr)); + else + return MEM_read64(memPtr); +} + +MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64) +{ + if (MEM_isLittleEndian()) + MEM_write64(memPtr, MEM_swap64(val64)); + else + MEM_write64(memPtr, val64); +} + +MEM_STATIC size_t MEM_readBEST(const void* memPtr) +{ + if (MEM_32bits()) + return (size_t)MEM_readBE32(memPtr); + else + return (size_t)MEM_readBE64(memPtr); +} + +MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val) +{ + if (MEM_32bits()) + MEM_writeBE32(memPtr, (U32)val); + else + MEM_writeBE64(memPtr, (U64)val); +} + + +/* function safe only for comparisons */ +MEM_STATIC U32 MEM_readMINMATCH(const void* memPtr, U32 length) +{ + switch (length) + { + default : + case 4 : return MEM_read32(memPtr); + case 3 : if (MEM_isLittleEndian()) + return MEM_read32(memPtr)<<8; + else + return MEM_read32(memPtr)>>8; + } +} + +#if defined (__cplusplus) +} +#endif + +#endif /* MEM_H_MODULE */ + +/* ****************************************************************** + Error codes list + Copyright (C) 2016, Yann Collet + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - Homepage : http://www.zstd.net +****************************************************************** */ +#ifndef ERROR_PUBLIC_H_MODULE +#define ERROR_PUBLIC_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif + + +/* **************************************** +* error codes list +******************************************/ +typedef enum { + ZSTDv07_error_no_error, + ZSTDv07_error_GENERIC, + ZSTDv07_error_prefix_unknown, + ZSTDv07_error_frameParameter_unsupported, + ZSTDv07_error_frameParameter_unsupportedBy32bits, + ZSTDv07_error_compressionParameter_unsupported, + ZSTDv07_error_init_missing, + ZSTDv07_error_memory_allocation, + ZSTDv07_error_stage_wrong, + ZSTDv07_error_dstSize_tooSmall, + ZSTDv07_error_srcSize_wrong, + ZSTDv07_error_corruption_detected, + ZSTDv07_error_checksum_wrong, + ZSTDv07_error_tableLog_tooLarge, + ZSTDv07_error_maxSymbolValue_tooLarge, + ZSTDv07_error_maxSymbolValue_tooSmall, + ZSTDv07_error_dictionary_corrupted, + ZSTDv07_error_dictionary_wrong, + ZSTDv07_error_maxCode +} ZSTDv07_ErrorCode; + +/*! ZSTDv07_getErrorCode() : + convert a `size_t` function result into a `ZSTDv07_ErrorCode` enum type, + which can be used to compare directly with enum list published into "error_public.h" */ +ZSTDv07_ErrorCode ZSTDv07_getErrorCode(size_t functionResult); +const char* ZSTDv07_getErrorString(ZSTDv07_ErrorCode code); + + +#if defined (__cplusplus) +} +#endif + +#endif /* ERROR_PUBLIC_H_MODULE */ +/* ****************************************************************** + Error codes and messages + Copyright (C) 2013-2016, Yann Collet + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - Homepage : http://www.zstd.net +****************************************************************** */ +/* Note : this module is expected to remain private, do not expose it */ + +#ifndef ERROR_H_MODULE +#define ERROR_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif + + + +/* **************************************** +* Compiler-specific +******************************************/ +#if defined(__GNUC__) +# define ERR_STATIC static __attribute__((unused)) +#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define ERR_STATIC static inline +#elif defined(_MSC_VER) +# define ERR_STATIC static __inline +#else +# define ERR_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ +#endif + + +/*-**************************************** +* Customization (error_public.h) +******************************************/ +typedef ZSTDv07_ErrorCode ERR_enum; +#define PREFIX(name) ZSTDv07_error_##name + + +/*-**************************************** +* Error codes handling +******************************************/ +#ifdef ERROR +# undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */ +#endif +#define ERROR(name) ((size_t)-PREFIX(name)) + +ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); } + +ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); } + + +/*-**************************************** +* Error Strings +******************************************/ + +ERR_STATIC const char* ERR_getErrorString(ERR_enum code) +{ + static const char* notErrorCode = "Unspecified error code"; + switch( code ) + { + case PREFIX(no_error): return "No error detected"; + case PREFIX(GENERIC): return "Error (generic)"; + case PREFIX(prefix_unknown): return "Unknown frame descriptor"; + case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter"; + case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode"; + case PREFIX(compressionParameter_unsupported): return "Compression parameter is out of bound"; + case PREFIX(init_missing): return "Context should be init first"; + case PREFIX(memory_allocation): return "Allocation error : not enough memory"; + case PREFIX(stage_wrong): return "Operation not authorized at current processing stage"; + case PREFIX(dstSize_tooSmall): return "Destination buffer is too small"; + case PREFIX(srcSize_wrong): return "Src size incorrect"; + case PREFIX(corruption_detected): return "Corrupted block detected"; + case PREFIX(checksum_wrong): return "Restored data doesn't match checksum"; + case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported"; + case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large"; + case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small"; + case PREFIX(dictionary_corrupted): return "Dictionary is corrupted"; + case PREFIX(dictionary_wrong): return "Dictionary mismatch"; + case PREFIX(maxCode): + default: return notErrorCode; + } +} + +ERR_STATIC const char* ERR_getErrorName(size_t code) +{ + return ERR_getErrorString(ERR_getErrorCode(code)); +} + +#if defined (__cplusplus) +} +#endif + +#endif /* ERROR_H_MODULE */ +/* ****************************************************************** + bitstream + Part of FSE library + header file (to include) + Copyright (C) 2013-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - Source repository : https://github.com/Cyan4973/FiniteStateEntropy +****************************************************************** */ +#ifndef BITSTREAM_H_MODULE +#define BITSTREAM_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif + + +/* +* This API consists of small unitary functions, which must be inlined for best performance. +* Since link-time-optimization is not available for all compilers, +* these functions are defined into a .h to be included. +*/ + + +/*========================================= +* Target specific +=========================================*/ +#if defined(__BMI__) && defined(__GNUC__) +# include <immintrin.h> /* support for bextr (experimental) */ +#endif + +/*-******************************************** +* bitStream decoding API (read backward) +**********************************************/ +typedef struct +{ + size_t bitContainer; + unsigned bitsConsumed; + const char* ptr; + const char* start; +} BITv07_DStream_t; + +typedef enum { BITv07_DStream_unfinished = 0, + BITv07_DStream_endOfBuffer = 1, + BITv07_DStream_completed = 2, + BITv07_DStream_overflow = 3 } BITv07_DStream_status; /* result of BITv07_reloadDStream() */ + /* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */ + +MEM_STATIC size_t BITv07_initDStream(BITv07_DStream_t* bitD, const void* srcBuffer, size_t srcSize); +MEM_STATIC size_t BITv07_readBits(BITv07_DStream_t* bitD, unsigned nbBits); +MEM_STATIC BITv07_DStream_status BITv07_reloadDStream(BITv07_DStream_t* bitD); +MEM_STATIC unsigned BITv07_endOfDStream(const BITv07_DStream_t* bitD); + + +/* Start by invoking BITv07_initDStream(). +* A chunk of the bitStream is then stored into a local register. +* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t). +* You can then retrieve bitFields stored into the local register, **in reverse order**. +* Local register is explicitly reloaded from memory by the BITv07_reloadDStream() method. +* A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BITv07_DStream_unfinished. +* Otherwise, it can be less than that, so proceed accordingly. +* Checking if DStream has reached its end can be performed with BITv07_endOfDStream(). +*/ + + +/*-**************************************** +* unsafe API +******************************************/ +MEM_STATIC size_t BITv07_readBitsFast(BITv07_DStream_t* bitD, unsigned nbBits); +/* faster, but works only if nbBits >= 1 */ + + + +/*-************************************************************** +* Internal functions +****************************************************************/ +MEM_STATIC unsigned BITv07_highbit32 (register U32 val) +{ +# if defined(_MSC_VER) /* Visual */ + unsigned long r=0; + _BitScanReverse ( &r, val ); + return (unsigned) r; +# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */ + return 31 - __builtin_clz (val); +# else /* Software version */ + static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 }; + U32 v = val; + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + return DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27]; +# endif +} + +/*===== Local Constants =====*/ +static const unsigned BITv07_mask[] = { 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF, 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF }; /* up to 26 bits */ + + +/*-******************************************************** +* bitStream decoding +**********************************************************/ +/*! BITv07_initDStream() : +* Initialize a BITv07_DStream_t. +* `bitD` : a pointer to an already allocated BITv07_DStream_t structure. +* `srcSize` must be the *exact* size of the bitStream, in bytes. +* @return : size of stream (== srcSize) or an errorCode if a problem is detected +*/ +MEM_STATIC size_t BITv07_initDStream(BITv07_DStream_t* bitD, const void* srcBuffer, size_t srcSize) +{ + if (srcSize < 1) { memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); } + + if (srcSize >= sizeof(bitD->bitContainer)) { /* normal case */ + bitD->start = (const char*)srcBuffer; + bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer); + bitD->bitContainer = MEM_readLEST(bitD->ptr); + { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; + bitD->bitsConsumed = lastByte ? 8 - BITv07_highbit32(lastByte) : 0; + if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ } + } else { + bitD->start = (const char*)srcBuffer; + bitD->ptr = bitD->start; + bitD->bitContainer = *(const BYTE*)(bitD->start); + switch(srcSize) + { + case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16); + case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24); + case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32); + case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24; + case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16; + case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8; + default:; + } + { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; + bitD->bitsConsumed = lastByte ? 8 - BITv07_highbit32(lastByte) : 0; + if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ } + bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize)*8; + } + + return srcSize; +} + +MEM_STATIC size_t BITv07_getUpperBits(size_t bitContainer, U32 const start) +{ + return bitContainer >> start; +} + +MEM_STATIC size_t BITv07_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits) +{ +#if defined(__BMI__) && defined(__GNUC__) /* experimental */ +# if defined(__x86_64__) + if (sizeof(bitContainer)==8) + return _bextr_u64(bitContainer, start, nbBits); + else +# endif + return _bextr_u32(bitContainer, start, nbBits); +#else + return (bitContainer >> start) & BITv07_mask[nbBits]; +#endif +} + +MEM_STATIC size_t BITv07_getLowerBits(size_t bitContainer, U32 const nbBits) +{ + return bitContainer & BITv07_mask[nbBits]; +} + +/*! BITv07_lookBits() : + * Provides next n bits from local register. + * local register is not modified. + * On 32-bits, maxNbBits==24. + * On 64-bits, maxNbBits==56. + * @return : value extracted + */ + MEM_STATIC size_t BITv07_lookBits(const BITv07_DStream_t* bitD, U32 nbBits) +{ +#if defined(__BMI__) && defined(__GNUC__) /* experimental; fails if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8 */ + return BITv07_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits); +#else + U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1; + return ((bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> 1) >> ((bitMask-nbBits) & bitMask); +#endif +} + +/*! BITv07_lookBitsFast() : +* unsafe version; only works only if nbBits >= 1 */ +MEM_STATIC size_t BITv07_lookBitsFast(const BITv07_DStream_t* bitD, U32 nbBits) +{ + U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1; + return (bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> (((bitMask+1)-nbBits) & bitMask); +} + +MEM_STATIC void BITv07_skipBits(BITv07_DStream_t* bitD, U32 nbBits) +{ + bitD->bitsConsumed += nbBits; +} + +/*! BITv07_readBits() : + * Read (consume) next n bits from local register and update. + * Pay attention to not read more than nbBits contained into local register. + * @return : extracted value. + */ +MEM_STATIC size_t BITv07_readBits(BITv07_DStream_t* bitD, U32 nbBits) +{ + size_t const value = BITv07_lookBits(bitD, nbBits); + BITv07_skipBits(bitD, nbBits); + return value; +} + +/*! BITv07_readBitsFast() : +* unsafe version; only works only if nbBits >= 1 */ +MEM_STATIC size_t BITv07_readBitsFast(BITv07_DStream_t* bitD, U32 nbBits) +{ + size_t const value = BITv07_lookBitsFast(bitD, nbBits); + BITv07_skipBits(bitD, nbBits); + return value; +} + +/*! BITv07_reloadDStream() : +* Refill `BITv07_DStream_t` from src buffer previously defined (see BITv07_initDStream() ). +* This function is safe, it guarantees it will not read beyond src buffer. +* @return : status of `BITv07_DStream_t` internal register. + if status == unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */ +MEM_STATIC BITv07_DStream_status BITv07_reloadDStream(BITv07_DStream_t* bitD) +{ + if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should not happen => corruption detected */ + return BITv07_DStream_overflow; + + if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer)) { + bitD->ptr -= bitD->bitsConsumed >> 3; + bitD->bitsConsumed &= 7; + bitD->bitContainer = MEM_readLEST(bitD->ptr); + return BITv07_DStream_unfinished; + } + if (bitD->ptr == bitD->start) { + if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return BITv07_DStream_endOfBuffer; + return BITv07_DStream_completed; + } + { U32 nbBytes = bitD->bitsConsumed >> 3; + BITv07_DStream_status result = BITv07_DStream_unfinished; + if (bitD->ptr - nbBytes < bitD->start) { + nbBytes = (U32)(bitD->ptr - bitD->start); /* ptr > start */ + result = BITv07_DStream_endOfBuffer; + } + bitD->ptr -= nbBytes; + bitD->bitsConsumed -= nbBytes*8; + bitD->bitContainer = MEM_readLEST(bitD->ptr); /* reminder : srcSize > sizeof(bitD) */ + return result; + } +} + +/*! BITv07_endOfDStream() : +* @return Tells if DStream has exactly reached its end (all bits consumed). +*/ +MEM_STATIC unsigned BITv07_endOfDStream(const BITv07_DStream_t* DStream) +{ + return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer)*8)); +} + +#if defined (__cplusplus) +} +#endif + +#endif /* BITSTREAM_H_MODULE */ +/* ****************************************************************** + FSE : Finite State Entropy codec + Public Prototypes declaration + Copyright (C) 2013-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - Source repository : https://github.com/Cyan4973/FiniteStateEntropy +****************************************************************** */ +#ifndef FSEv07_H +#define FSEv07_H + +#if defined (__cplusplus) +extern "C" { +#endif + + + +/*-**************************************** +* FSE simple functions +******************************************/ + +/*! FSEv07_decompress(): + Decompress FSE data from buffer 'cSrc', of size 'cSrcSize', + into already allocated destination buffer 'dst', of size 'dstCapacity'. + @return : size of regenerated data (<= maxDstSize), + or an error code, which can be tested using FSEv07_isError() . + + ** Important ** : FSEv07_decompress() does not decompress non-compressible nor RLE data !!! + Why ? : making this distinction requires a header. + Header management is intentionally delegated to the user layer, which can better manage special cases. +*/ +size_t FSEv07_decompress(void* dst, size_t dstCapacity, + const void* cSrc, size_t cSrcSize); + + +/* Error Management */ +unsigned FSEv07_isError(size_t code); /* tells if a return value is an error code */ +const char* FSEv07_getErrorName(size_t code); /* provides error code string (useful for debugging) */ + + +/*-***************************************** +* FSE detailed API +******************************************/ +/*! +FSEv07_decompress() does the following: +1. read normalized counters with readNCount() +2. build decoding table 'DTable' from normalized counters +3. decode the data stream using decoding table 'DTable' + +The following API allows targeting specific sub-functions for advanced tasks. +For example, it's possible to compress several blocks using the same 'CTable', +or to save and provide normalized distribution using external method. +*/ + + +/* *** DECOMPRESSION *** */ + +/*! FSEv07_readNCount(): + Read compactly saved 'normalizedCounter' from 'rBuffer'. + @return : size read from 'rBuffer', + or an errorCode, which can be tested using FSEv07_isError(). + maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */ +size_t FSEv07_readNCount (short* normalizedCounter, unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, const void* rBuffer, size_t rBuffSize); + +/*! Constructor and Destructor of FSEv07_DTable. + Note that its size depends on 'tableLog' */ +typedef unsigned FSEv07_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */ +FSEv07_DTable* FSEv07_createDTable(unsigned tableLog); +void FSEv07_freeDTable(FSEv07_DTable* dt); + +/*! FSEv07_buildDTable(): + Builds 'dt', which must be already allocated, using FSEv07_createDTable(). + return : 0, or an errorCode, which can be tested using FSEv07_isError() */ +size_t FSEv07_buildDTable (FSEv07_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); + +/*! FSEv07_decompress_usingDTable(): + Decompress compressed source `cSrc` of size `cSrcSize` using `dt` + into `dst` which must be already allocated. + @return : size of regenerated data (necessarily <= `dstCapacity`), + or an errorCode, which can be tested using FSEv07_isError() */ +size_t FSEv07_decompress_usingDTable(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, const FSEv07_DTable* dt); + +/*! +Tutorial : +---------- +(Note : these functions only decompress FSE-compressed blocks. + If block is uncompressed, use memcpy() instead + If block is a single repeated byte, use memset() instead ) + +The first step is to obtain the normalized frequencies of symbols. +This can be performed by FSEv07_readNCount() if it was saved using FSEv07_writeNCount(). +'normalizedCounter' must be already allocated, and have at least 'maxSymbolValuePtr[0]+1' cells of signed short. +In practice, that means it's necessary to know 'maxSymbolValue' beforehand, +or size the table to handle worst case situations (typically 256). +FSEv07_readNCount() will provide 'tableLog' and 'maxSymbolValue'. +The result of FSEv07_readNCount() is the number of bytes read from 'rBuffer'. +Note that 'rBufferSize' must be at least 4 bytes, even if useful information is less than that. +If there is an error, the function will return an error code, which can be tested using FSEv07_isError(). + +The next step is to build the decompression tables 'FSEv07_DTable' from 'normalizedCounter'. +This is performed by the function FSEv07_buildDTable(). +The space required by 'FSEv07_DTable' must be already allocated using FSEv07_createDTable(). +If there is an error, the function will return an error code, which can be tested using FSEv07_isError(). + +`FSEv07_DTable` can then be used to decompress `cSrc`, with FSEv07_decompress_usingDTable(). +`cSrcSize` must be strictly correct, otherwise decompression will fail. +FSEv07_decompress_usingDTable() result will tell how many bytes were regenerated (<=`dstCapacity`). +If there is an error, the function will return an error code, which can be tested using FSEv07_isError(). (ex: dst buffer too small) +*/ + + +#ifdef FSEv07_STATIC_LINKING_ONLY + + +/* ***************************************** +* Static allocation +*******************************************/ +/* FSE buffer bounds */ +#define FSEv07_NCOUNTBOUND 512 +#define FSEv07_BLOCKBOUND(size) (size + (size>>7)) + +/* It is possible to statically allocate FSE CTable/DTable as a table of unsigned using below macros */ +#define FSEv07_DTABLE_SIZE_U32(maxTableLog) (1 + (1<<maxTableLog)) + + +/* ***************************************** +* FSE advanced API +*******************************************/ +size_t FSEv07_countFast(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize); +/**< same as FSEv07_count(), but blindly trusts that all byte values within src are <= *maxSymbolValuePtr */ + +unsigned FSEv07_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus); +/**< same as FSEv07_optimalTableLog(), which used `minus==2` */ + +size_t FSEv07_buildDTable_raw (FSEv07_DTable* dt, unsigned nbBits); +/**< build a fake FSEv07_DTable, designed to read an uncompressed bitstream where each symbol uses nbBits */ + +size_t FSEv07_buildDTable_rle (FSEv07_DTable* dt, unsigned char symbolValue); +/**< build a fake FSEv07_DTable, designed to always generate the same symbolValue */ + + + +/* ***************************************** +* FSE symbol decompression API +*******************************************/ +typedef struct +{ + size_t state; + const void* table; /* precise table may vary, depending on U16 */ +} FSEv07_DState_t; + + +static void FSEv07_initDState(FSEv07_DState_t* DStatePtr, BITv07_DStream_t* bitD, const FSEv07_DTable* dt); + +static unsigned char FSEv07_decodeSymbol(FSEv07_DState_t* DStatePtr, BITv07_DStream_t* bitD); + +static unsigned FSEv07_endOfDState(const FSEv07_DState_t* DStatePtr); + +/**< +Let's now decompose FSEv07_decompress_usingDTable() into its unitary components. +You will decode FSE-encoded symbols from the bitStream, +and also any other bitFields you put in, **in reverse order**. + +You will need a few variables to track your bitStream. They are : + +BITv07_DStream_t DStream; // Stream context +FSEv07_DState_t DState; // State context. Multiple ones are possible +FSEv07_DTable* DTablePtr; // Decoding table, provided by FSEv07_buildDTable() + +The first thing to do is to init the bitStream. + errorCode = BITv07_initDStream(&DStream, srcBuffer, srcSize); + +You should then retrieve your initial state(s) +(in reverse flushing order if you have several ones) : + errorCode = FSEv07_initDState(&DState, &DStream, DTablePtr); + +You can then decode your data, symbol after symbol. +For information the maximum number of bits read by FSEv07_decodeSymbol() is 'tableLog'. +Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out). + unsigned char symbol = FSEv07_decodeSymbol(&DState, &DStream); + +You can retrieve any bitfield you eventually stored into the bitStream (in reverse order) +Note : maximum allowed nbBits is 25, for 32-bits compatibility + size_t bitField = BITv07_readBits(&DStream, nbBits); + +All above operations only read from local register (which size depends on size_t). +Refueling the register from memory is manually performed by the reload method. + endSignal = FSEv07_reloadDStream(&DStream); + +BITv07_reloadDStream() result tells if there is still some more data to read from DStream. +BITv07_DStream_unfinished : there is still some data left into the DStream. +BITv07_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled. +BITv07_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed. +BITv07_DStream_tooFar : Dstream went too far. Decompression result is corrupted. + +When reaching end of buffer (BITv07_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop, +to properly detect the exact end of stream. +After each decoded symbol, check if DStream is fully consumed using this simple test : + BITv07_reloadDStream(&DStream) >= BITv07_DStream_completed + +When it's done, verify decompression is fully completed, by checking both DStream and the relevant states. +Checking if DStream has reached its end is performed by : + BITv07_endOfDStream(&DStream); +Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible. + FSEv07_endOfDState(&DState); +*/ + + +/* ***************************************** +* FSE unsafe API +*******************************************/ +static unsigned char FSEv07_decodeSymbolFast(FSEv07_DState_t* DStatePtr, BITv07_DStream_t* bitD); +/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */ + + +/*<===== Decompression =====>*/ + +typedef struct { + U16 tableLog; + U16 fastMode; +} FSEv07_DTableHeader; /* sizeof U32 */ + +typedef struct +{ + unsigned short newState; + unsigned char symbol; + unsigned char nbBits; +} FSEv07_decode_t; /* size == U32 */ + +MEM_STATIC void FSEv07_initDState(FSEv07_DState_t* DStatePtr, BITv07_DStream_t* bitD, const FSEv07_DTable* dt) +{ + const void* ptr = dt; + const FSEv07_DTableHeader* const DTableH = (const FSEv07_DTableHeader*)ptr; + DStatePtr->state = BITv07_readBits(bitD, DTableH->tableLog); + BITv07_reloadDStream(bitD); + DStatePtr->table = dt + 1; +} + +MEM_STATIC BYTE FSEv07_peekSymbol(const FSEv07_DState_t* DStatePtr) +{ + FSEv07_decode_t const DInfo = ((const FSEv07_decode_t*)(DStatePtr->table))[DStatePtr->state]; + return DInfo.symbol; +} + +MEM_STATIC void FSEv07_updateState(FSEv07_DState_t* DStatePtr, BITv07_DStream_t* bitD) +{ + FSEv07_decode_t const DInfo = ((const FSEv07_decode_t*)(DStatePtr->table))[DStatePtr->state]; + U32 const nbBits = DInfo.nbBits; + size_t const lowBits = BITv07_readBits(bitD, nbBits); + DStatePtr->state = DInfo.newState + lowBits; +} + +MEM_STATIC BYTE FSEv07_decodeSymbol(FSEv07_DState_t* DStatePtr, BITv07_DStream_t* bitD) +{ + FSEv07_decode_t const DInfo = ((const FSEv07_decode_t*)(DStatePtr->table))[DStatePtr->state]; + U32 const nbBits = DInfo.nbBits; + BYTE const symbol = DInfo.symbol; + size_t const lowBits = BITv07_readBits(bitD, nbBits); + + DStatePtr->state = DInfo.newState + lowBits; + return symbol; +} + +/*! FSEv07_decodeSymbolFast() : + unsafe, only works if no symbol has a probability > 50% */ +MEM_STATIC BYTE FSEv07_decodeSymbolFast(FSEv07_DState_t* DStatePtr, BITv07_DStream_t* bitD) +{ + FSEv07_decode_t const DInfo = ((const FSEv07_decode_t*)(DStatePtr->table))[DStatePtr->state]; + U32 const nbBits = DInfo.nbBits; + BYTE const symbol = DInfo.symbol; + size_t const lowBits = BITv07_readBitsFast(bitD, nbBits); + + DStatePtr->state = DInfo.newState + lowBits; + return symbol; +} + +MEM_STATIC unsigned FSEv07_endOfDState(const FSEv07_DState_t* DStatePtr) +{ + return DStatePtr->state == 0; +} + + + +#ifndef FSEv07_COMMONDEFS_ONLY + +/* ************************************************************** +* Tuning parameters +****************************************************************/ +/*!MEMORY_USAGE : +* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) +* Increasing memory usage improves compression ratio +* Reduced memory usage can improve speed, due to cache effect +* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ +#define FSEv07_MAX_MEMORY_USAGE 14 +#define FSEv07_DEFAULT_MEMORY_USAGE 13 + +/*!FSEv07_MAX_SYMBOL_VALUE : +* Maximum symbol value authorized. +* Required for proper stack allocation */ +#define FSEv07_MAX_SYMBOL_VALUE 255 + + +/* ************************************************************** +* template functions type & suffix +****************************************************************/ +#define FSEv07_FUNCTION_TYPE BYTE +#define FSEv07_FUNCTION_EXTENSION +#define FSEv07_DECODE_TYPE FSEv07_decode_t + + +#endif /* !FSEv07_COMMONDEFS_ONLY */ + + +/* *************************************************************** +* Constants +*****************************************************************/ +#define FSEv07_MAX_TABLELOG (FSEv07_MAX_MEMORY_USAGE-2) +#define FSEv07_MAX_TABLESIZE (1U<<FSEv07_MAX_TABLELOG) +#define FSEv07_MAXTABLESIZE_MASK (FSEv07_MAX_TABLESIZE-1) +#define FSEv07_DEFAULT_TABLELOG (FSEv07_DEFAULT_MEMORY_USAGE-2) +#define FSEv07_MIN_TABLELOG 5 + +#define FSEv07_TABLELOG_ABSOLUTE_MAX 15 +#if FSEv07_MAX_TABLELOG > FSEv07_TABLELOG_ABSOLUTE_MAX +# error "FSEv07_MAX_TABLELOG > FSEv07_TABLELOG_ABSOLUTE_MAX is not supported" +#endif + +#define FSEv07_TABLESTEP(tableSize) ((tableSize>>1) + (tableSize>>3) + 3) + + +#endif /* FSEv07_STATIC_LINKING_ONLY */ + + +#if defined (__cplusplus) +} +#endif + +#endif /* FSEv07_H */ +/* ****************************************************************** + Huffman coder, part of New Generation Entropy library + header file + Copyright (C) 2013-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - Source repository : https://github.com/Cyan4973/FiniteStateEntropy +****************************************************************** */ +#ifndef HUFv07_H_298734234 +#define HUFv07_H_298734234 + +#if defined (__cplusplus) +extern "C" { +#endif + + + +/* *** simple functions *** */ +/** +HUFv07_decompress() : + Decompress HUF data from buffer 'cSrc', of size 'cSrcSize', + into already allocated buffer 'dst', of minimum size 'dstSize'. + `dstSize` : **must** be the ***exact*** size of original (uncompressed) data. + Note : in contrast with FSE, HUFv07_decompress can regenerate + RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data, + because it knows size to regenerate. + @return : size of regenerated data (== dstSize), + or an error code, which can be tested using HUFv07_isError() +*/ +size_t HUFv07_decompress(void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize); + + +/* **************************************** +* Tool functions +******************************************/ +#define HUFv07_BLOCKSIZE_MAX (128 * 1024) + +/* Error Management */ +unsigned HUFv07_isError(size_t code); /**< tells if a return value is an error code */ +const char* HUFv07_getErrorName(size_t code); /**< provides error code string (useful for debugging) */ + + +/* *** Advanced function *** */ + + +#ifdef HUFv07_STATIC_LINKING_ONLY + + +/* *** Constants *** */ +#define HUFv07_TABLELOG_ABSOLUTEMAX 16 /* absolute limit of HUFv07_MAX_TABLELOG. Beyond that value, code does not work */ +#define HUFv07_TABLELOG_MAX 12 /* max configured tableLog (for static allocation); can be modified up to HUFv07_ABSOLUTEMAX_TABLELOG */ +#define HUFv07_TABLELOG_DEFAULT 11 /* tableLog by default, when not specified */ +#define HUFv07_SYMBOLVALUE_MAX 255 +#if (HUFv07_TABLELOG_MAX > HUFv07_TABLELOG_ABSOLUTEMAX) +# error "HUFv07_TABLELOG_MAX is too large !" +#endif + + +/* **************************************** +* Static allocation +******************************************/ +/* HUF buffer bounds */ +#define HUFv07_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true if incompressible pre-filtered with fast heuristic */ + +/* static allocation of HUF's DTable */ +typedef U32 HUFv07_DTable; +#define HUFv07_DTABLE_SIZE(maxTableLog) (1 + (1<<(maxTableLog))) +#define HUFv07_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \ + HUFv07_DTable DTable[HUFv07_DTABLE_SIZE((maxTableLog)-1)] = { ((U32)((maxTableLog)-1)*0x1000001) } +#define HUFv07_CREATE_STATIC_DTABLEX4(DTable, maxTableLog) \ + HUFv07_DTable DTable[HUFv07_DTABLE_SIZE(maxTableLog)] = { ((U32)(maxTableLog)*0x1000001) } + + +/* **************************************** +* Advanced decompression functions +******************************************/ +size_t HUFv07_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ +size_t HUFv07_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ + +size_t HUFv07_decompress4X_DCtx (HUFv07_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< decodes RLE and uncompressed */ +size_t HUFv07_decompress4X_hufOnly(HUFv07_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */ +size_t HUFv07_decompress4X2_DCtx(HUFv07_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ +size_t HUFv07_decompress4X4_DCtx(HUFv07_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ + +size_t HUFv07_decompress1X_DCtx (HUFv07_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); +size_t HUFv07_decompress1X2_DCtx(HUFv07_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ +size_t HUFv07_decompress1X4_DCtx(HUFv07_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ + + +/* **************************************** +* HUF detailed API +******************************************/ +/*! +The following API allows targeting specific sub-functions for advanced tasks. +For example, it's possible to compress several blocks using the same 'CTable', +or to save and regenerate 'CTable' using external methods. +*/ +/* FSEv07_count() : find it within "fse.h" */ + +/*! HUFv07_readStats() : + Read compact Huffman tree, saved by HUFv07_writeCTable(). + `huffWeight` is destination buffer. + @return : size read from `src` , or an error Code . + Note : Needed by HUFv07_readCTable() and HUFv07_readDTableXn() . */ +size_t HUFv07_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize); + + +/* +HUFv07_decompress() does the following: +1. select the decompression algorithm (X2, X4) based on pre-computed heuristics +2. build Huffman table from save, using HUFv07_readDTableXn() +3. decode 1 or 4 segments in parallel using HUFv07_decompressSXn_usingDTable +*/ + +/** HUFv07_selectDecoder() : +* Tells which decoder is likely to decode faster, +* based on a set of pre-determined metrics. +* @return : 0==HUFv07_decompress4X2, 1==HUFv07_decompress4X4 . +* Assumption : 0 < cSrcSize < dstSize <= 128 KB */ +U32 HUFv07_selectDecoder (size_t dstSize, size_t cSrcSize); + +size_t HUFv07_readDTableX2 (HUFv07_DTable* DTable, const void* src, size_t srcSize); +size_t HUFv07_readDTableX4 (HUFv07_DTable* DTable, const void* src, size_t srcSize); + +size_t HUFv07_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUFv07_DTable* DTable); +size_t HUFv07_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUFv07_DTable* DTable); +size_t HUFv07_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUFv07_DTable* DTable); + + +/* single stream variants */ +size_t HUFv07_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */ +size_t HUFv07_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */ + +size_t HUFv07_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUFv07_DTable* DTable); +size_t HUFv07_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUFv07_DTable* DTable); +size_t HUFv07_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUFv07_DTable* DTable); + + +#endif /* HUFv07_STATIC_LINKING_ONLY */ + + +#if defined (__cplusplus) +} +#endif + +#endif /* HUFv07_H_298734234 */ +/* + Common functions of New Generation Entropy library + Copyright (C) 2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy + - Public forum : https://groups.google.com/forum/#!forum/lz4c +*************************************************************************** */ + + + +/*-**************************************** +* FSE Error Management +******************************************/ +unsigned FSEv07_isError(size_t code) { return ERR_isError(code); } + +const char* FSEv07_getErrorName(size_t code) { return ERR_getErrorName(code); } + + +/* ************************************************************** +* HUF Error Management +****************************************************************/ +unsigned HUFv07_isError(size_t code) { return ERR_isError(code); } + +const char* HUFv07_getErrorName(size_t code) { return ERR_getErrorName(code); } + + +/*-************************************************************** +* FSE NCount encoding-decoding +****************************************************************/ +static short FSEv07_abs(short a) { return (short)(a<0 ? -a : a); } + +size_t FSEv07_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize) +{ + const BYTE* const istart = (const BYTE*) headerBuffer; + const BYTE* const iend = istart + hbSize; + const BYTE* ip = istart; + int nbBits; + int remaining; + int threshold; + U32 bitStream; + int bitCount; + unsigned charnum = 0; + int previous0 = 0; + + if (hbSize < 4) return ERROR(srcSize_wrong); + bitStream = MEM_readLE32(ip); + nbBits = (bitStream & 0xF) + FSEv07_MIN_TABLELOG; /* extract tableLog */ + if (nbBits > FSEv07_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge); + bitStream >>= 4; + bitCount = 4; + *tableLogPtr = nbBits; + remaining = (1<<nbBits)+1; + threshold = 1<<nbBits; + nbBits++; + + while ((remaining>1) && (charnum<=*maxSVPtr)) { + if (previous0) { + unsigned n0 = charnum; + while ((bitStream & 0xFFFF) == 0xFFFF) { + n0+=24; + if (ip < iend-5) { + ip+=2; + bitStream = MEM_readLE32(ip) >> bitCount; + } else { + bitStream >>= 16; + bitCount+=16; + } } + while ((bitStream & 3) == 3) { + n0+=3; + bitStream>>=2; + bitCount+=2; + } + n0 += bitStream & 3; + bitCount += 2; + if (n0 > *maxSVPtr) return ERROR(maxSymbolValue_tooSmall); + while (charnum < n0) normalizedCounter[charnum++] = 0; + if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { + ip += bitCount>>3; + bitCount &= 7; + bitStream = MEM_readLE32(ip) >> bitCount; + } + else + bitStream >>= 2; + } + { short const max = (short)((2*threshold-1)-remaining); + short count; + + if ((bitStream & (threshold-1)) < (U32)max) { + count = (short)(bitStream & (threshold-1)); + bitCount += nbBits-1; + } else { + count = (short)(bitStream & (2*threshold-1)); + if (count >= threshold) count -= max; + bitCount += nbBits; + } + + count--; /* extra accuracy */ + remaining -= FSEv07_abs(count); + normalizedCounter[charnum++] = count; + previous0 = !count; + while (remaining < threshold) { + nbBits--; + threshold >>= 1; + } + + if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { + ip += bitCount>>3; + bitCount &= 7; + } else { + bitCount -= (int)(8 * (iend - 4 - ip)); + ip = iend - 4; + } + bitStream = MEM_readLE32(ip) >> (bitCount & 31); + } } /* while ((remaining>1) && (charnum<=*maxSVPtr)) */ + if (remaining != 1) return ERROR(GENERIC); + *maxSVPtr = charnum-1; + + ip += (bitCount+7)>>3; + if ((size_t)(ip-istart) > hbSize) return ERROR(srcSize_wrong); + return ip-istart; +} + + +/*! HUFv07_readStats() : + Read compact Huffman tree, saved by HUFv07_writeCTable(). + `huffWeight` is destination buffer. + @return : size read from `src` , or an error Code . + Note : Needed by HUFv07_readCTable() and HUFv07_readDTableXn() . +*/ +size_t HUFv07_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize) +{ + U32 weightTotal; + const BYTE* ip = (const BYTE*) src; + size_t iSize = ip[0]; + size_t oSize; + + //memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */ + + if (iSize >= 128) { /* special header */ + if (iSize >= (242)) { /* RLE */ + static U32 l[14] = { 1, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128 }; + oSize = l[iSize-242]; + memset(huffWeight, 1, hwSize); + iSize = 0; + } + else { /* Incompressible */ + oSize = iSize - 127; + iSize = ((oSize+1)/2); + if (iSize+1 > srcSize) return ERROR(srcSize_wrong); + if (oSize >= hwSize) return ERROR(corruption_detected); + ip += 1; + { U32 n; + for (n=0; n<oSize; n+=2) { + huffWeight[n] = ip[n/2] >> 4; + huffWeight[n+1] = ip[n/2] & 15; + } } } } + else { /* header compressed with FSE (normal case) */ + if (iSize+1 > srcSize) return ERROR(srcSize_wrong); + oSize = FSEv07_decompress(huffWeight, hwSize-1, ip+1, iSize); /* max (hwSize-1) values decoded, as last one is implied */ + if (FSEv07_isError(oSize)) return oSize; + } + + /* collect weight stats */ + memset(rankStats, 0, (HUFv07_TABLELOG_ABSOLUTEMAX + 1) * sizeof(U32)); + weightTotal = 0; + { U32 n; for (n=0; n<oSize; n++) { + if (huffWeight[n] >= HUFv07_TABLELOG_ABSOLUTEMAX) return ERROR(corruption_detected); + rankStats[huffWeight[n]]++; + weightTotal += (1 << huffWeight[n]) >> 1; + } } + + /* get last non-null symbol weight (implied, total must be 2^n) */ + { U32 const tableLog = BITv07_highbit32(weightTotal) + 1; + if (tableLog > HUFv07_TABLELOG_ABSOLUTEMAX) return ERROR(corruption_detected); + *tableLogPtr = tableLog; + /* determine last weight */ + { U32 const total = 1 << tableLog; + U32 const rest = total - weightTotal; + U32 const verif = 1 << BITv07_highbit32(rest); + U32 const lastWeight = BITv07_highbit32(rest) + 1; + if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */ + huffWeight[oSize] = (BYTE)lastWeight; + rankStats[lastWeight]++; + } } + + /* check tree construction validity */ + if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */ + + /* results */ + *nbSymbolsPtr = (U32)(oSize+1); + return iSize+1; +} +/* ****************************************************************** + FSE : Finite State Entropy decoder + Copyright (C) 2013-2015, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + - Public forum : https://groups.google.com/forum/#!forum/lz4c +****************************************************************** */ + + +/* ************************************************************** +* Compiler specifics +****************************************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# define FORCE_INLINE static __forceinline +# include <intrin.h> /* For Visual 2005 */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */ +#else +# ifdef __GNUC__ +# define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif +#endif + + + + +/* ************************************************************** +* Error Management +****************************************************************/ +#define FSEv07_isError ERR_isError +#define FSEv07_STATIC_ASSERT(c) { enum { FSEv07_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ + + +/* ************************************************************** +* Complex types +****************************************************************/ +typedef U32 DTable_max_t[FSEv07_DTABLE_SIZE_U32(FSEv07_MAX_TABLELOG)]; + + +/* ************************************************************** +* Templates +****************************************************************/ +/* + designed to be included + for type-specific functions (template emulation in C) + Objective is to write these functions only once, for improved maintenance +*/ + +/* safety checks */ +#ifndef FSEv07_FUNCTION_EXTENSION +# error "FSEv07_FUNCTION_EXTENSION must be defined" +#endif +#ifndef FSEv07_FUNCTION_TYPE +# error "FSEv07_FUNCTION_TYPE must be defined" +#endif + +/* Function names */ +#define FSEv07_CAT(X,Y) X##Y +#define FSEv07_FUNCTION_NAME(X,Y) FSEv07_CAT(X,Y) +#define FSEv07_TYPE_NAME(X,Y) FSEv07_CAT(X,Y) + + +/* Function templates */ +FSEv07_DTable* FSEv07_createDTable (unsigned tableLog) +{ + if (tableLog > FSEv07_TABLELOG_ABSOLUTE_MAX) tableLog = FSEv07_TABLELOG_ABSOLUTE_MAX; + return (FSEv07_DTable*)malloc( FSEv07_DTABLE_SIZE_U32(tableLog) * sizeof (U32) ); +} + +void FSEv07_freeDTable (FSEv07_DTable* dt) +{ + free(dt); +} + +size_t FSEv07_buildDTable(FSEv07_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) +{ + void* const tdPtr = dt+1; /* because *dt is unsigned, 32-bits aligned on 32-bits */ + FSEv07_DECODE_TYPE* const tableDecode = (FSEv07_DECODE_TYPE*) (tdPtr); + U16 symbolNext[FSEv07_MAX_SYMBOL_VALUE+1]; + + U32 const maxSV1 = maxSymbolValue + 1; + U32 const tableSize = 1 << tableLog; + U32 highThreshold = tableSize-1; + + /* Sanity Checks */ + if (maxSymbolValue > FSEv07_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge); + if (tableLog > FSEv07_MAX_TABLELOG) return ERROR(tableLog_tooLarge); + + /* Init, lay down lowprob symbols */ + { FSEv07_DTableHeader DTableH; + DTableH.tableLog = (U16)tableLog; + DTableH.fastMode = 1; + { S16 const largeLimit= (S16)(1 << (tableLog-1)); + U32 s; + for (s=0; s<maxSV1; s++) { + if (normalizedCounter[s]==-1) { + tableDecode[highThreshold--].symbol = (FSEv07_FUNCTION_TYPE)s; + symbolNext[s] = 1; + } else { + if (normalizedCounter[s] >= largeLimit) DTableH.fastMode=0; + symbolNext[s] = normalizedCounter[s]; + } } } + memcpy(dt, &DTableH, sizeof(DTableH)); + } + + /* Spread symbols */ + { U32 const tableMask = tableSize-1; + U32 const step = FSEv07_TABLESTEP(tableSize); + U32 s, position = 0; + for (s=0; s<maxSV1; s++) { + int i; + for (i=0; i<normalizedCounter[s]; i++) { + tableDecode[position].symbol = (FSEv07_FUNCTION_TYPE)s; + position = (position + step) & tableMask; + while (position > highThreshold) position = (position + step) & tableMask; /* lowprob area */ + } } + + if (position!=0) return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ + } + + /* Build Decoding table */ + { U32 u; + for (u=0; u<tableSize; u++) { + FSEv07_FUNCTION_TYPE const symbol = (FSEv07_FUNCTION_TYPE)(tableDecode[u].symbol); + U16 nextState = symbolNext[symbol]++; + tableDecode[u].nbBits = (BYTE) (tableLog - BITv07_highbit32 ((U32)nextState) ); + tableDecode[u].newState = (U16) ( (nextState << tableDecode[u].nbBits) - tableSize); + } } + + return 0; +} + + + +#ifndef FSEv07_COMMONDEFS_ONLY + +/*-******************************************************* +* Decompression (Byte symbols) +*********************************************************/ +size_t FSEv07_buildDTable_rle (FSEv07_DTable* dt, BYTE symbolValue) +{ + void* ptr = dt; + FSEv07_DTableHeader* const DTableH = (FSEv07_DTableHeader*)ptr; + void* dPtr = dt + 1; + FSEv07_decode_t* const cell = (FSEv07_decode_t*)dPtr; + + DTableH->tableLog = 0; + DTableH->fastMode = 0; + + cell->newState = 0; + cell->symbol = symbolValue; + cell->nbBits = 0; + + return 0; +} + + +size_t FSEv07_buildDTable_raw (FSEv07_DTable* dt, unsigned nbBits) +{ + void* ptr = dt; + FSEv07_DTableHeader* const DTableH = (FSEv07_DTableHeader*)ptr; + void* dPtr = dt + 1; + FSEv07_decode_t* const dinfo = (FSEv07_decode_t*)dPtr; + const unsigned tableSize = 1 << nbBits; + const unsigned tableMask = tableSize - 1; + const unsigned maxSV1 = tableMask+1; + unsigned s; + + /* Sanity checks */ + if (nbBits < 1) return ERROR(GENERIC); /* min size */ + + /* Build Decoding Table */ + DTableH->tableLog = (U16)nbBits; + DTableH->fastMode = 1; + for (s=0; s<maxSV1; s++) { + dinfo[s].newState = 0; + dinfo[s].symbol = (BYTE)s; + dinfo[s].nbBits = (BYTE)nbBits; + } + + return 0; +} + +FORCE_INLINE size_t FSEv07_decompress_usingDTable_generic( + void* dst, size_t maxDstSize, + const void* cSrc, size_t cSrcSize, + const FSEv07_DTable* dt, const unsigned fast) +{ + BYTE* const ostart = (BYTE*) dst; + BYTE* op = ostart; + BYTE* const omax = op + maxDstSize; + BYTE* const olimit = omax-3; + + BITv07_DStream_t bitD; + FSEv07_DState_t state1; + FSEv07_DState_t state2; + + /* Init */ + { size_t const errorCode = BITv07_initDStream(&bitD, cSrc, cSrcSize); /* replaced last arg by maxCompressed Size */ + if (FSEv07_isError(errorCode)) return errorCode; } + + FSEv07_initDState(&state1, &bitD, dt); + FSEv07_initDState(&state2, &bitD, dt); + +#define FSEv07_GETSYMBOL(statePtr) fast ? FSEv07_decodeSymbolFast(statePtr, &bitD) : FSEv07_decodeSymbol(statePtr, &bitD) + + /* 4 symbols per loop */ + for ( ; (BITv07_reloadDStream(&bitD)==BITv07_DStream_unfinished) && (op<olimit) ; op+=4) { + op[0] = FSEv07_GETSYMBOL(&state1); + + if (FSEv07_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ + BITv07_reloadDStream(&bitD); + + op[1] = FSEv07_GETSYMBOL(&state2); + + if (FSEv07_MAX_TABLELOG*4+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ + { if (BITv07_reloadDStream(&bitD) > BITv07_DStream_unfinished) { op+=2; break; } } + + op[2] = FSEv07_GETSYMBOL(&state1); + + if (FSEv07_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ + BITv07_reloadDStream(&bitD); + + op[3] = FSEv07_GETSYMBOL(&state2); + } + + /* tail */ + /* note : BITv07_reloadDStream(&bitD) >= FSEv07_DStream_partiallyFilled; Ends at exactly BITv07_DStream_completed */ + while (1) { + if (op>(omax-2)) return ERROR(dstSize_tooSmall); + + *op++ = FSEv07_GETSYMBOL(&state1); + + if (BITv07_reloadDStream(&bitD)==BITv07_DStream_overflow) { + *op++ = FSEv07_GETSYMBOL(&state2); + break; + } + + if (op>(omax-2)) return ERROR(dstSize_tooSmall); + + *op++ = FSEv07_GETSYMBOL(&state2); + + if (BITv07_reloadDStream(&bitD)==BITv07_DStream_overflow) { + *op++ = FSEv07_GETSYMBOL(&state1); + break; + } } + + return op-ostart; +} + + +size_t FSEv07_decompress_usingDTable(void* dst, size_t originalSize, + const void* cSrc, size_t cSrcSize, + const FSEv07_DTable* dt) +{ + const void* ptr = dt; + const FSEv07_DTableHeader* DTableH = (const FSEv07_DTableHeader*)ptr; + const U32 fastMode = DTableH->fastMode; + + /* select fast mode (static) */ + if (fastMode) return FSEv07_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1); + return FSEv07_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0); +} + + +size_t FSEv07_decompress(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize) +{ + const BYTE* const istart = (const BYTE*)cSrc; + const BYTE* ip = istart; + short counting[FSEv07_MAX_SYMBOL_VALUE+1]; + DTable_max_t dt; /* Static analyzer seems unable to understand this table will be properly initialized later */ + unsigned tableLog; + unsigned maxSymbolValue = FSEv07_MAX_SYMBOL_VALUE; + + if (cSrcSize<2) return ERROR(srcSize_wrong); /* too small input size */ + + /* normal FSE decoding mode */ + { size_t const NCountLength = FSEv07_readNCount (counting, &maxSymbolValue, &tableLog, istart, cSrcSize); + if (FSEv07_isError(NCountLength)) return NCountLength; + if (NCountLength >= cSrcSize) return ERROR(srcSize_wrong); /* too small input size */ + ip += NCountLength; + cSrcSize -= NCountLength; + } + + { size_t const errorCode = FSEv07_buildDTable (dt, counting, maxSymbolValue, tableLog); + if (FSEv07_isError(errorCode)) return errorCode; } + + return FSEv07_decompress_usingDTable (dst, maxDstSize, ip, cSrcSize, dt); /* always return, even if it is an error code */ +} + + + +#endif /* FSEv07_COMMONDEFS_ONLY */ + +/* ****************************************************************** + Huffman decoder, part of New Generation Entropy library + Copyright (C) 2013-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy + - Public forum : https://groups.google.com/forum/#!forum/lz4c +****************************************************************** */ + +/* ************************************************************** +* Compiler specifics +****************************************************************/ +#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +/* inline is defined */ +#elif defined(_MSC_VER) +# define inline __inline +#else +# define inline /* disable inline */ +#endif + + +#ifdef _MSC_VER /* Visual Studio */ +# define FORCE_INLINE static __forceinline +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#else +# ifdef __GNUC__ +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif +#endif + + + +/* ************************************************************** +* Error Management +****************************************************************/ +#define HUFv07_STATIC_ASSERT(c) { enum { HUFv07_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ + + +/*-***************************/ +/* generic DTableDesc */ +/*-***************************/ + +typedef struct { BYTE maxTableLog; BYTE tableType; BYTE tableLog; BYTE reserved; } DTableDesc; + +static DTableDesc HUFv07_getDTableDesc(const HUFv07_DTable* table) +{ + DTableDesc dtd; + memcpy(&dtd, table, sizeof(dtd)); + return dtd; +} + + +/*-***************************/ +/* single-symbol decoding */ +/*-***************************/ + +typedef struct { BYTE byte; BYTE nbBits; } HUFv07_DEltX2; /* single-symbol decoding */ + +size_t HUFv07_readDTableX2 (HUFv07_DTable* DTable, const void* src, size_t srcSize) +{ + BYTE huffWeight[HUFv07_SYMBOLVALUE_MAX + 1]; + U32 rankVal[HUFv07_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */ + U32 tableLog = 0; + U32 nbSymbols = 0; + size_t iSize; + void* const dtPtr = DTable + 1; + HUFv07_DEltX2* const dt = (HUFv07_DEltX2*)dtPtr; + + HUFv07_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUFv07_DTable)); + //memset(huffWeight, 0, sizeof(huffWeight)); /* is not necessary, even though some analyzer complain ... */ + + iSize = HUFv07_readStats(huffWeight, HUFv07_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize); + if (HUFv07_isError(iSize)) return iSize; + + /* Table header */ + { DTableDesc dtd = HUFv07_getDTableDesc(DTable); + if (tableLog > (U32)(dtd.maxTableLog+1)) return ERROR(tableLog_tooLarge); /* DTable too small, huffman tree cannot fit in */ + dtd.tableType = 0; + dtd.tableLog = (BYTE)tableLog; + memcpy(DTable, &dtd, sizeof(dtd)); + } + + /* Prepare ranks */ + { U32 n, nextRankStart = 0; + for (n=1; n<tableLog+1; n++) { + U32 current = nextRankStart; + nextRankStart += (rankVal[n] << (n-1)); + rankVal[n] = current; + } } + + /* fill DTable */ + { U32 n; + for (n=0; n<nbSymbols; n++) { + U32 const w = huffWeight[n]; + U32 const length = (1 << w) >> 1; + U32 i; + HUFv07_DEltX2 D; + D.byte = (BYTE)n; D.nbBits = (BYTE)(tableLog + 1 - w); + for (i = rankVal[w]; i < rankVal[w] + length; i++) + dt[i] = D; + rankVal[w] += length; + } } + + return iSize; +} + + +static BYTE HUFv07_decodeSymbolX2(BITv07_DStream_t* Dstream, const HUFv07_DEltX2* dt, const U32 dtLog) +{ + size_t const val = BITv07_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */ + BYTE const c = dt[val].byte; + BITv07_skipBits(Dstream, dt[val].nbBits); + return c; +} + +#define HUFv07_DECODE_SYMBOLX2_0(ptr, DStreamPtr) \ + *ptr++ = HUFv07_decodeSymbolX2(DStreamPtr, dt, dtLog) + +#define HUFv07_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \ + if (MEM_64bits() || (HUFv07_TABLELOG_MAX<=12)) \ + HUFv07_DECODE_SYMBOLX2_0(ptr, DStreamPtr) + +#define HUFv07_DECODE_SYMBOLX2_2(ptr, DStreamPtr) \ + if (MEM_64bits()) \ + HUFv07_DECODE_SYMBOLX2_0(ptr, DStreamPtr) + +static inline size_t HUFv07_decodeStreamX2(BYTE* p, BITv07_DStream_t* const bitDPtr, BYTE* const pEnd, const HUFv07_DEltX2* const dt, const U32 dtLog) +{ + BYTE* const pStart = p; + + /* up to 4 symbols at a time */ + while ((BITv07_reloadDStream(bitDPtr) == BITv07_DStream_unfinished) && (p <= pEnd-4)) { + HUFv07_DECODE_SYMBOLX2_2(p, bitDPtr); + HUFv07_DECODE_SYMBOLX2_1(p, bitDPtr); + HUFv07_DECODE_SYMBOLX2_2(p, bitDPtr); + HUFv07_DECODE_SYMBOLX2_0(p, bitDPtr); + } + + /* closer to the end */ + while ((BITv07_reloadDStream(bitDPtr) == BITv07_DStream_unfinished) && (p < pEnd)) + HUFv07_DECODE_SYMBOLX2_0(p, bitDPtr); + + /* no more data to retrieve from bitstream, hence no need to reload */ + while (p < pEnd) + HUFv07_DECODE_SYMBOLX2_0(p, bitDPtr); + + return pEnd-pStart; +} + +static size_t HUFv07_decompress1X2_usingDTable_internal( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUFv07_DTable* DTable) +{ + BYTE* op = (BYTE*)dst; + BYTE* const oend = op + dstSize; + const void* dtPtr = DTable + 1; + const HUFv07_DEltX2* const dt = (const HUFv07_DEltX2*)dtPtr; + BITv07_DStream_t bitD; + DTableDesc const dtd = HUFv07_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; + + { size_t const errorCode = BITv07_initDStream(&bitD, cSrc, cSrcSize); + if (HUFv07_isError(errorCode)) return errorCode; } + + HUFv07_decodeStreamX2(op, &bitD, oend, dt, dtLog); + + /* check */ + if (!BITv07_endOfDStream(&bitD)) return ERROR(corruption_detected); + + return dstSize; +} + +size_t HUFv07_decompress1X2_usingDTable( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUFv07_DTable* DTable) +{ + DTableDesc dtd = HUFv07_getDTableDesc(DTable); + if (dtd.tableType != 0) return ERROR(GENERIC); + return HUFv07_decompress1X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); +} + +size_t HUFv07_decompress1X2_DCtx (HUFv07_DTable* DCtx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t const hSize = HUFv07_readDTableX2 (DCtx, cSrc, cSrcSize); + if (HUFv07_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUFv07_decompress1X2_usingDTable_internal (dst, dstSize, ip, cSrcSize, DCtx); +} + +size_t HUFv07_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + HUFv07_CREATE_STATIC_DTABLEX2(DTable, HUFv07_TABLELOG_MAX); + return HUFv07_decompress1X2_DCtx (DTable, dst, dstSize, cSrc, cSrcSize); +} + + +static size_t HUFv07_decompress4X2_usingDTable_internal( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUFv07_DTable* DTable) +{ + /* Check */ + if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ + + { const BYTE* const istart = (const BYTE*) cSrc; + BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + const void* const dtPtr = DTable + 1; + const HUFv07_DEltX2* const dt = (const HUFv07_DEltX2*)dtPtr; + + /* Init */ + BITv07_DStream_t bitD1; + BITv07_DStream_t bitD2; + BITv07_DStream_t bitD3; + BITv07_DStream_t bitD4; + size_t const length1 = MEM_readLE16(istart); + size_t const length2 = MEM_readLE16(istart+2); + size_t const length3 = MEM_readLE16(istart+4); + size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); + const BYTE* const istart1 = istart + 6; /* jumpTable */ + const BYTE* const istart2 = istart1 + length1; + const BYTE* const istart3 = istart2 + length2; + const BYTE* const istart4 = istart3 + length3; + const size_t segmentSize = (dstSize+3) / 4; + BYTE* const opStart2 = ostart + segmentSize; + BYTE* const opStart3 = opStart2 + segmentSize; + BYTE* const opStart4 = opStart3 + segmentSize; + BYTE* op1 = ostart; + BYTE* op2 = opStart2; + BYTE* op3 = opStart3; + BYTE* op4 = opStart4; + U32 endSignal; + DTableDesc const dtd = HUFv07_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; + + if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ + { size_t const errorCode = BITv07_initDStream(&bitD1, istart1, length1); + if (HUFv07_isError(errorCode)) return errorCode; } + { size_t const errorCode = BITv07_initDStream(&bitD2, istart2, length2); + if (HUFv07_isError(errorCode)) return errorCode; } + { size_t const errorCode = BITv07_initDStream(&bitD3, istart3, length3); + if (HUFv07_isError(errorCode)) return errorCode; } + { size_t const errorCode = BITv07_initDStream(&bitD4, istart4, length4); + if (HUFv07_isError(errorCode)) return errorCode; } + + /* 16-32 symbols per loop (4-8 symbols per stream) */ + endSignal = BITv07_reloadDStream(&bitD1) | BITv07_reloadDStream(&bitD2) | BITv07_reloadDStream(&bitD3) | BITv07_reloadDStream(&bitD4); + for ( ; (endSignal==BITv07_DStream_unfinished) && (op4<(oend-7)) ; ) { + HUFv07_DECODE_SYMBOLX2_2(op1, &bitD1); + HUFv07_DECODE_SYMBOLX2_2(op2, &bitD2); + HUFv07_DECODE_SYMBOLX2_2(op3, &bitD3); + HUFv07_DECODE_SYMBOLX2_2(op4, &bitD4); + HUFv07_DECODE_SYMBOLX2_1(op1, &bitD1); + HUFv07_DECODE_SYMBOLX2_1(op2, &bitD2); + HUFv07_DECODE_SYMBOLX2_1(op3, &bitD3); + HUFv07_DECODE_SYMBOLX2_1(op4, &bitD4); + HUFv07_DECODE_SYMBOLX2_2(op1, &bitD1); + HUFv07_DECODE_SYMBOLX2_2(op2, &bitD2); + HUFv07_DECODE_SYMBOLX2_2(op3, &bitD3); + HUFv07_DECODE_SYMBOLX2_2(op4, &bitD4); + HUFv07_DECODE_SYMBOLX2_0(op1, &bitD1); + HUFv07_DECODE_SYMBOLX2_0(op2, &bitD2); + HUFv07_DECODE_SYMBOLX2_0(op3, &bitD3); + HUFv07_DECODE_SYMBOLX2_0(op4, &bitD4); + endSignal = BITv07_reloadDStream(&bitD1) | BITv07_reloadDStream(&bitD2) | BITv07_reloadDStream(&bitD3) | BITv07_reloadDStream(&bitD4); + } + + /* check corruption */ + if (op1 > opStart2) return ERROR(corruption_detected); + if (op2 > opStart3) return ERROR(corruption_detected); + if (op3 > opStart4) return ERROR(corruption_detected); + /* note : op4 supposed already verified within main loop */ + + /* finish bitStreams one by one */ + HUFv07_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog); + HUFv07_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog); + HUFv07_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog); + HUFv07_decodeStreamX2(op4, &bitD4, oend, dt, dtLog); + + /* check */ + endSignal = BITv07_endOfDStream(&bitD1) & BITv07_endOfDStream(&bitD2) & BITv07_endOfDStream(&bitD3) & BITv07_endOfDStream(&bitD4); + if (!endSignal) return ERROR(corruption_detected); + + /* decoded size */ + return dstSize; + } +} + + +size_t HUFv07_decompress4X2_usingDTable( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUFv07_DTable* DTable) +{ + DTableDesc dtd = HUFv07_getDTableDesc(DTable); + if (dtd.tableType != 0) return ERROR(GENERIC); + return HUFv07_decompress4X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); +} + + +size_t HUFv07_decompress4X2_DCtx (HUFv07_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t const hSize = HUFv07_readDTableX2 (dctx, cSrc, cSrcSize); + if (HUFv07_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUFv07_decompress4X2_usingDTable_internal (dst, dstSize, ip, cSrcSize, dctx); +} + +size_t HUFv07_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + HUFv07_CREATE_STATIC_DTABLEX2(DTable, HUFv07_TABLELOG_MAX); + return HUFv07_decompress4X2_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); +} + + +/* *************************/ +/* double-symbols decoding */ +/* *************************/ +typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUFv07_DEltX4; /* double-symbols decoding */ + +typedef struct { BYTE symbol; BYTE weight; } sortedSymbol_t; + +static void HUFv07_fillDTableX4Level2(HUFv07_DEltX4* DTable, U32 sizeLog, const U32 consumed, + const U32* rankValOrigin, const int minWeight, + const sortedSymbol_t* sortedSymbols, const U32 sortedListSize, + U32 nbBitsBaseline, U16 baseSeq) +{ + HUFv07_DEltX4 DElt; + U32 rankVal[HUFv07_TABLELOG_ABSOLUTEMAX + 1]; + + /* get pre-calculated rankVal */ + memcpy(rankVal, rankValOrigin, sizeof(rankVal)); + + /* fill skipped values */ + if (minWeight>1) { + U32 i, skipSize = rankVal[minWeight]; + MEM_writeLE16(&(DElt.sequence), baseSeq); + DElt.nbBits = (BYTE)(consumed); + DElt.length = 1; + for (i = 0; i < skipSize; i++) + DTable[i] = DElt; + } + + /* fill DTable */ + { U32 s; for (s=0; s<sortedListSize; s++) { /* note : sortedSymbols already skipped */ + const U32 symbol = sortedSymbols[s].symbol; + const U32 weight = sortedSymbols[s].weight; + const U32 nbBits = nbBitsBaseline - weight; + const U32 length = 1 << (sizeLog-nbBits); + const U32 start = rankVal[weight]; + U32 i = start; + const U32 end = start + length; + + MEM_writeLE16(&(DElt.sequence), (U16)(baseSeq + (symbol << 8))); + DElt.nbBits = (BYTE)(nbBits + consumed); + DElt.length = 2; + do { DTable[i++] = DElt; } while (i<end); /* since length >= 1 */ + + rankVal[weight] += length; + }} +} + +typedef U32 rankVal_t[HUFv07_TABLELOG_ABSOLUTEMAX][HUFv07_TABLELOG_ABSOLUTEMAX + 1]; + +static void HUFv07_fillDTableX4(HUFv07_DEltX4* DTable, const U32 targetLog, + const sortedSymbol_t* sortedList, const U32 sortedListSize, + const U32* rankStart, rankVal_t rankValOrigin, const U32 maxWeight, + const U32 nbBitsBaseline) +{ + U32 rankVal[HUFv07_TABLELOG_ABSOLUTEMAX + 1]; + const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */ + const U32 minBits = nbBitsBaseline - maxWeight; + U32 s; + + memcpy(rankVal, rankValOrigin, sizeof(rankVal)); + + /* fill DTable */ + for (s=0; s<sortedListSize; s++) { + const U16 symbol = sortedList[s].symbol; + const U32 weight = sortedList[s].weight; + const U32 nbBits = nbBitsBaseline - weight; + const U32 start = rankVal[weight]; + const U32 length = 1 << (targetLog-nbBits); + + if (targetLog-nbBits >= minBits) { /* enough room for a second symbol */ + U32 sortedRank; + int minWeight = nbBits + scaleLog; + if (minWeight < 1) minWeight = 1; + sortedRank = rankStart[minWeight]; + HUFv07_fillDTableX4Level2(DTable+start, targetLog-nbBits, nbBits, + rankValOrigin[nbBits], minWeight, + sortedList+sortedRank, sortedListSize-sortedRank, + nbBitsBaseline, symbol); + } else { + HUFv07_DEltX4 DElt; + MEM_writeLE16(&(DElt.sequence), symbol); + DElt.nbBits = (BYTE)(nbBits); + DElt.length = 1; + { U32 u; + const U32 end = start + length; + for (u = start; u < end; u++) DTable[u] = DElt; + } } + rankVal[weight] += length; + } +} + +size_t HUFv07_readDTableX4 (HUFv07_DTable* DTable, const void* src, size_t srcSize) +{ + BYTE weightList[HUFv07_SYMBOLVALUE_MAX + 1]; + sortedSymbol_t sortedSymbol[HUFv07_SYMBOLVALUE_MAX + 1]; + U32 rankStats[HUFv07_TABLELOG_ABSOLUTEMAX + 1] = { 0 }; + U32 rankStart0[HUFv07_TABLELOG_ABSOLUTEMAX + 2] = { 0 }; + U32* const rankStart = rankStart0+1; + rankVal_t rankVal; + U32 tableLog, maxW, sizeOfSort, nbSymbols; + DTableDesc dtd = HUFv07_getDTableDesc(DTable); + U32 const maxTableLog = dtd.maxTableLog; + size_t iSize; + void* dtPtr = DTable+1; /* force compiler to avoid strict-aliasing */ + HUFv07_DEltX4* const dt = (HUFv07_DEltX4*)dtPtr; + + HUFv07_STATIC_ASSERT(sizeof(HUFv07_DEltX4) == sizeof(HUFv07_DTable)); /* if compilation fails here, assertion is false */ + if (maxTableLog > HUFv07_TABLELOG_ABSOLUTEMAX) return ERROR(tableLog_tooLarge); + //memset(weightList, 0, sizeof(weightList)); /* is not necessary, even though some analyzer complain ... */ + + iSize = HUFv07_readStats(weightList, HUFv07_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize); + if (HUFv07_isError(iSize)) return iSize; + + /* check result */ + if (tableLog > maxTableLog) return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */ + + /* find maxWeight */ + for (maxW = tableLog; rankStats[maxW]==0; maxW--) {} /* necessarily finds a solution before 0 */ + + /* Get start index of each weight */ + { U32 w, nextRankStart = 0; + for (w=1; w<maxW+1; w++) { + U32 current = nextRankStart; + nextRankStart += rankStats[w]; + rankStart[w] = current; + } + rankStart[0] = nextRankStart; /* put all 0w symbols at the end of sorted list*/ + sizeOfSort = nextRankStart; + } + + /* sort symbols by weight */ + { U32 s; + for (s=0; s<nbSymbols; s++) { + U32 const w = weightList[s]; + U32 const r = rankStart[w]++; + sortedSymbol[r].symbol = (BYTE)s; + sortedSymbol[r].weight = (BYTE)w; + } + rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */ + } + + /* Build rankVal */ + { U32* const rankVal0 = rankVal[0]; + { int const rescale = (maxTableLog-tableLog) - 1; /* tableLog <= maxTableLog */ + U32 nextRankVal = 0; + U32 w; + for (w=1; w<maxW+1; w++) { + U32 current = nextRankVal; + nextRankVal += rankStats[w] << (w+rescale); + rankVal0[w] = current; + } } + { U32 const minBits = tableLog+1 - maxW; + U32 consumed; + for (consumed = minBits; consumed < maxTableLog - minBits + 1; consumed++) { + U32* const rankValPtr = rankVal[consumed]; + U32 w; + for (w = 1; w < maxW+1; w++) { + rankValPtr[w] = rankVal0[w] >> consumed; + } } } } + + HUFv07_fillDTableX4(dt, maxTableLog, + sortedSymbol, sizeOfSort, + rankStart0, rankVal, maxW, + tableLog+1); + + dtd.tableLog = (BYTE)maxTableLog; + dtd.tableType = 1; + memcpy(DTable, &dtd, sizeof(dtd)); + return iSize; +} + + +static U32 HUFv07_decodeSymbolX4(void* op, BITv07_DStream_t* DStream, const HUFv07_DEltX4* dt, const U32 dtLog) +{ + const size_t val = BITv07_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ + memcpy(op, dt+val, 2); + BITv07_skipBits(DStream, dt[val].nbBits); + return dt[val].length; +} + +static U32 HUFv07_decodeLastSymbolX4(void* op, BITv07_DStream_t* DStream, const HUFv07_DEltX4* dt, const U32 dtLog) +{ + const size_t val = BITv07_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ + memcpy(op, dt+val, 1); + if (dt[val].length==1) BITv07_skipBits(DStream, dt[val].nbBits); + else { + if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8)) { + BITv07_skipBits(DStream, dt[val].nbBits); + if (DStream->bitsConsumed > (sizeof(DStream->bitContainer)*8)) + DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8); /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */ + } } + return 1; +} + + +#define HUFv07_DECODE_SYMBOLX4_0(ptr, DStreamPtr) \ + ptr += HUFv07_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog) + +#define HUFv07_DECODE_SYMBOLX4_1(ptr, DStreamPtr) \ + if (MEM_64bits() || (HUFv07_TABLELOG_MAX<=12)) \ + ptr += HUFv07_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog) + +#define HUFv07_DECODE_SYMBOLX4_2(ptr, DStreamPtr) \ + if (MEM_64bits()) \ + ptr += HUFv07_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog) + +static inline size_t HUFv07_decodeStreamX4(BYTE* p, BITv07_DStream_t* bitDPtr, BYTE* const pEnd, const HUFv07_DEltX4* const dt, const U32 dtLog) +{ + BYTE* const pStart = p; + + /* up to 8 symbols at a time */ + while ((BITv07_reloadDStream(bitDPtr) == BITv07_DStream_unfinished) && (p < pEnd-7)) { + HUFv07_DECODE_SYMBOLX4_2(p, bitDPtr); + HUFv07_DECODE_SYMBOLX4_1(p, bitDPtr); + HUFv07_DECODE_SYMBOLX4_2(p, bitDPtr); + HUFv07_DECODE_SYMBOLX4_0(p, bitDPtr); + } + + /* closer to end : up to 2 symbols at a time */ + while ((BITv07_reloadDStream(bitDPtr) == BITv07_DStream_unfinished) && (p <= pEnd-2)) + HUFv07_DECODE_SYMBOLX4_0(p, bitDPtr); + + while (p <= pEnd-2) + HUFv07_DECODE_SYMBOLX4_0(p, bitDPtr); /* no need to reload : reached the end of DStream */ + + if (p < pEnd) + p += HUFv07_decodeLastSymbolX4(p, bitDPtr, dt, dtLog); + + return p-pStart; +} + + +static size_t HUFv07_decompress1X4_usingDTable_internal( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUFv07_DTable* DTable) +{ + BITv07_DStream_t bitD; + + /* Init */ + { size_t const errorCode = BITv07_initDStream(&bitD, cSrc, cSrcSize); + if (HUFv07_isError(errorCode)) return errorCode; + } + + /* decode */ + { BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + const void* const dtPtr = DTable+1; /* force compiler to not use strict-aliasing */ + const HUFv07_DEltX4* const dt = (const HUFv07_DEltX4*)dtPtr; + DTableDesc const dtd = HUFv07_getDTableDesc(DTable); + HUFv07_decodeStreamX4(ostart, &bitD, oend, dt, dtd.tableLog); + } + + /* check */ + if (!BITv07_endOfDStream(&bitD)) return ERROR(corruption_detected); + + /* decoded size */ + return dstSize; +} + +size_t HUFv07_decompress1X4_usingDTable( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUFv07_DTable* DTable) +{ + DTableDesc dtd = HUFv07_getDTableDesc(DTable); + if (dtd.tableType != 1) return ERROR(GENERIC); + return HUFv07_decompress1X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); +} + +size_t HUFv07_decompress1X4_DCtx (HUFv07_DTable* DCtx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t const hSize = HUFv07_readDTableX4 (DCtx, cSrc, cSrcSize); + if (HUFv07_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUFv07_decompress1X4_usingDTable_internal (dst, dstSize, ip, cSrcSize, DCtx); +} + +size_t HUFv07_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + HUFv07_CREATE_STATIC_DTABLEX4(DTable, HUFv07_TABLELOG_MAX); + return HUFv07_decompress1X4_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); +} + +static size_t HUFv07_decompress4X4_usingDTable_internal( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUFv07_DTable* DTable) +{ + if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ + + { const BYTE* const istart = (const BYTE*) cSrc; + BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + const void* const dtPtr = DTable+1; + const HUFv07_DEltX4* const dt = (const HUFv07_DEltX4*)dtPtr; + + /* Init */ + BITv07_DStream_t bitD1; + BITv07_DStream_t bitD2; + BITv07_DStream_t bitD3; + BITv07_DStream_t bitD4; + size_t const length1 = MEM_readLE16(istart); + size_t const length2 = MEM_readLE16(istart+2); + size_t const length3 = MEM_readLE16(istart+4); + size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); + const BYTE* const istart1 = istart + 6; /* jumpTable */ + const BYTE* const istart2 = istart1 + length1; + const BYTE* const istart3 = istart2 + length2; + const BYTE* const istart4 = istart3 + length3; + size_t const segmentSize = (dstSize+3) / 4; + BYTE* const opStart2 = ostart + segmentSize; + BYTE* const opStart3 = opStart2 + segmentSize; + BYTE* const opStart4 = opStart3 + segmentSize; + BYTE* op1 = ostart; + BYTE* op2 = opStart2; + BYTE* op3 = opStart3; + BYTE* op4 = opStart4; + U32 endSignal; + DTableDesc const dtd = HUFv07_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; + + if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ + { size_t const errorCode = BITv07_initDStream(&bitD1, istart1, length1); + if (HUFv07_isError(errorCode)) return errorCode; } + { size_t const errorCode = BITv07_initDStream(&bitD2, istart2, length2); + if (HUFv07_isError(errorCode)) return errorCode; } + { size_t const errorCode = BITv07_initDStream(&bitD3, istart3, length3); + if (HUFv07_isError(errorCode)) return errorCode; } + { size_t const errorCode = BITv07_initDStream(&bitD4, istart4, length4); + if (HUFv07_isError(errorCode)) return errorCode; } + + /* 16-32 symbols per loop (4-8 symbols per stream) */ + endSignal = BITv07_reloadDStream(&bitD1) | BITv07_reloadDStream(&bitD2) | BITv07_reloadDStream(&bitD3) | BITv07_reloadDStream(&bitD4); + for ( ; (endSignal==BITv07_DStream_unfinished) && (op4<(oend-7)) ; ) { + HUFv07_DECODE_SYMBOLX4_2(op1, &bitD1); + HUFv07_DECODE_SYMBOLX4_2(op2, &bitD2); + HUFv07_DECODE_SYMBOLX4_2(op3, &bitD3); + HUFv07_DECODE_SYMBOLX4_2(op4, &bitD4); + HUFv07_DECODE_SYMBOLX4_1(op1, &bitD1); + HUFv07_DECODE_SYMBOLX4_1(op2, &bitD2); + HUFv07_DECODE_SYMBOLX4_1(op3, &bitD3); + HUFv07_DECODE_SYMBOLX4_1(op4, &bitD4); + HUFv07_DECODE_SYMBOLX4_2(op1, &bitD1); + HUFv07_DECODE_SYMBOLX4_2(op2, &bitD2); + HUFv07_DECODE_SYMBOLX4_2(op3, &bitD3); + HUFv07_DECODE_SYMBOLX4_2(op4, &bitD4); + HUFv07_DECODE_SYMBOLX4_0(op1, &bitD1); + HUFv07_DECODE_SYMBOLX4_0(op2, &bitD2); + HUFv07_DECODE_SYMBOLX4_0(op3, &bitD3); + HUFv07_DECODE_SYMBOLX4_0(op4, &bitD4); + + endSignal = BITv07_reloadDStream(&bitD1) | BITv07_reloadDStream(&bitD2) | BITv07_reloadDStream(&bitD3) | BITv07_reloadDStream(&bitD4); + } + + /* check corruption */ + if (op1 > opStart2) return ERROR(corruption_detected); + if (op2 > opStart3) return ERROR(corruption_detected); + if (op3 > opStart4) return ERROR(corruption_detected); + /* note : op4 supposed already verified within main loop */ + + /* finish bitStreams one by one */ + HUFv07_decodeStreamX4(op1, &bitD1, opStart2, dt, dtLog); + HUFv07_decodeStreamX4(op2, &bitD2, opStart3, dt, dtLog); + HUFv07_decodeStreamX4(op3, &bitD3, opStart4, dt, dtLog); + HUFv07_decodeStreamX4(op4, &bitD4, oend, dt, dtLog); + + /* check */ + { U32 const endCheck = BITv07_endOfDStream(&bitD1) & BITv07_endOfDStream(&bitD2) & BITv07_endOfDStream(&bitD3) & BITv07_endOfDStream(&bitD4); + if (!endCheck) return ERROR(corruption_detected); } + + /* decoded size */ + return dstSize; + } +} + + +size_t HUFv07_decompress4X4_usingDTable( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUFv07_DTable* DTable) +{ + DTableDesc dtd = HUFv07_getDTableDesc(DTable); + if (dtd.tableType != 1) return ERROR(GENERIC); + return HUFv07_decompress4X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); +} + + +size_t HUFv07_decompress4X4_DCtx (HUFv07_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t hSize = HUFv07_readDTableX4 (dctx, cSrc, cSrcSize); + if (HUFv07_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUFv07_decompress4X4_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx); +} + +size_t HUFv07_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + HUFv07_CREATE_STATIC_DTABLEX4(DTable, HUFv07_TABLELOG_MAX); + return HUFv07_decompress4X4_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); +} + + +/* ********************************/ +/* Generic decompression selector */ +/* ********************************/ + +size_t HUFv07_decompress1X_usingDTable(void* dst, size_t maxDstSize, + const void* cSrc, size_t cSrcSize, + const HUFv07_DTable* DTable) +{ + DTableDesc const dtd = HUFv07_getDTableDesc(DTable); + return dtd.tableType ? HUFv07_decompress1X4_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable) : + HUFv07_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable); +} + +size_t HUFv07_decompress4X_usingDTable(void* dst, size_t maxDstSize, + const void* cSrc, size_t cSrcSize, + const HUFv07_DTable* DTable) +{ + DTableDesc const dtd = HUFv07_getDTableDesc(DTable); + return dtd.tableType ? HUFv07_decompress4X4_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable) : + HUFv07_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable); +} + + +typedef struct { U32 tableTime; U32 decode256Time; } algo_time_t; +static const algo_time_t algoTime[16 /* Quantization */][3 /* single, double, quad */] = +{ + /* single, double, quad */ + {{0,0}, {1,1}, {2,2}}, /* Q==0 : impossible */ + {{0,0}, {1,1}, {2,2}}, /* Q==1 : impossible */ + {{ 38,130}, {1313, 74}, {2151, 38}}, /* Q == 2 : 12-18% */ + {{ 448,128}, {1353, 74}, {2238, 41}}, /* Q == 3 : 18-25% */ + {{ 556,128}, {1353, 74}, {2238, 47}}, /* Q == 4 : 25-32% */ + {{ 714,128}, {1418, 74}, {2436, 53}}, /* Q == 5 : 32-38% */ + {{ 883,128}, {1437, 74}, {2464, 61}}, /* Q == 6 : 38-44% */ + {{ 897,128}, {1515, 75}, {2622, 68}}, /* Q == 7 : 44-50% */ + {{ 926,128}, {1613, 75}, {2730, 75}}, /* Q == 8 : 50-56% */ + {{ 947,128}, {1729, 77}, {3359, 77}}, /* Q == 9 : 56-62% */ + {{1107,128}, {2083, 81}, {4006, 84}}, /* Q ==10 : 62-69% */ + {{1177,128}, {2379, 87}, {4785, 88}}, /* Q ==11 : 69-75% */ + {{1242,128}, {2415, 93}, {5155, 84}}, /* Q ==12 : 75-81% */ + {{1349,128}, {2644,106}, {5260,106}}, /* Q ==13 : 81-87% */ + {{1455,128}, {2422,124}, {4174,124}}, /* Q ==14 : 87-93% */ + {{ 722,128}, {1891,145}, {1936,146}}, /* Q ==15 : 93-99% */ +}; + +/** HUFv07_selectDecoder() : +* Tells which decoder is likely to decode faster, +* based on a set of pre-determined metrics. +* @return : 0==HUFv07_decompress4X2, 1==HUFv07_decompress4X4 . +* Assumption : 0 < cSrcSize < dstSize <= 128 KB */ +U32 HUFv07_selectDecoder (size_t dstSize, size_t cSrcSize) +{ + /* decoder timing evaluation */ + U32 const Q = (U32)(cSrcSize * 16 / dstSize); /* Q < 16 since dstSize > cSrcSize */ + U32 const D256 = (U32)(dstSize >> 8); + U32 const DTime0 = algoTime[Q][0].tableTime + (algoTime[Q][0].decode256Time * D256); + U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256); + DTime1 += DTime1 >> 3; /* advantage to algorithm using less memory, for cache eviction */ + + return DTime1 < DTime0; +} + + +typedef size_t (*decompressionAlgo)(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); + +size_t HUFv07_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + static const decompressionAlgo decompress[2] = { HUFv07_decompress4X2, HUFv07_decompress4X4 }; + + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ + if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ + if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ + + { U32 const algoNb = HUFv07_selectDecoder(dstSize, cSrcSize); + return decompress[algoNb](dst, dstSize, cSrc, cSrcSize); + } + + //return HUFv07_decompress4X2(dst, dstSize, cSrc, cSrcSize); /* multi-streams single-symbol decoding */ + //return HUFv07_decompress4X4(dst, dstSize, cSrc, cSrcSize); /* multi-streams double-symbols decoding */ +} + +size_t HUFv07_decompress4X_DCtx (HUFv07_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ + if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ + if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ + + { U32 const algoNb = HUFv07_selectDecoder(dstSize, cSrcSize); + return algoNb ? HUFv07_decompress4X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) : + HUFv07_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ; + } +} + +size_t HUFv07_decompress4X_hufOnly (HUFv07_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if ((cSrcSize >= dstSize) || (cSrcSize <= 1)) return ERROR(corruption_detected); /* invalid */ + + { U32 const algoNb = HUFv07_selectDecoder(dstSize, cSrcSize); + return algoNb ? HUFv07_decompress4X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) : + HUFv07_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ; + } +} + +size_t HUFv07_decompress1X_DCtx (HUFv07_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ + if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ + if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ + + { U32 const algoNb = HUFv07_selectDecoder(dstSize, cSrcSize); + return algoNb ? HUFv07_decompress1X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) : + HUFv07_decompress1X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ; + } +} +/* + Common functions of Zstd compression library + Copyright (C) 2015-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - zstd homepage : http://www.zstd.net/ +*/ + + + +/*-**************************************** +* ZSTD Error Management +******************************************/ +/*! ZSTDv07_isError() : +* tells if a return value is an error code */ +unsigned ZSTDv07_isError(size_t code) { return ERR_isError(code); } + +/*! ZSTDv07_getErrorName() : +* provides error code string from function result (useful for debugging) */ +const char* ZSTDv07_getErrorName(size_t code) { return ERR_getErrorName(code); } + +/*! ZSTDv07_getError() : +* convert a `size_t` function result into a proper ZSTDv07_errorCode enum */ +ZSTDv07_ErrorCode ZSTDv07_getErrorCode(size_t code) { return ERR_getErrorCode(code); } + +/*! ZSTDv07_getErrorString() : +* provides error code string from enum */ +const char* ZSTDv07_getErrorString(ZSTDv07_ErrorCode code) { return ERR_getErrorName(code); } + + +/* ************************************************************** +* ZBUFF Error Management +****************************************************************/ +unsigned ZBUFFv07_isError(size_t errorCode) { return ERR_isError(errorCode); } + +const char* ZBUFFv07_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); } + + + +void* ZSTDv07_defaultAllocFunction(void* opaque, size_t size) +{ + void* address = malloc(size); + (void)opaque; + /* printf("alloc %p, %d opaque=%p \n", address, (int)size, opaque); */ + return address; +} + +void ZSTDv07_defaultFreeFunction(void* opaque, void* address) +{ + (void)opaque; + /* if (address) printf("free %p opaque=%p \n", address, opaque); */ + free(address); +} +/* + zstd_internal - common functions to include + Header File for include + Copyright (C) 2014-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - zstd homepage : https://www.zstd.net +*/ +#ifndef ZSTDv07_CCOMMON_H_MODULE +#define ZSTDv07_CCOMMON_H_MODULE + + +/*-************************************* +* Common macros +***************************************/ +#define MIN(a,b) ((a)<(b) ? (a) : (b)) +#define MAX(a,b) ((a)>(b) ? (a) : (b)) + + +/*-************************************* +* Common constants +***************************************/ +#define ZSTDv07_OPT_DEBUG 0 /* 3 = compression stats; 5 = check encoded sequences; 9 = full logs */ +#include <stdio.h> +#if defined(ZSTDv07_OPT_DEBUG) && ZSTDv07_OPT_DEBUG>=9 + #define ZSTDv07_LOG_PARSER(...) printf(__VA_ARGS__) + #define ZSTDv07_LOG_ENCODE(...) printf(__VA_ARGS__) + #define ZSTDv07_LOG_BLOCK(...) printf(__VA_ARGS__) +#else + #define ZSTDv07_LOG_PARSER(...) + #define ZSTDv07_LOG_ENCODE(...) + #define ZSTDv07_LOG_BLOCK(...) +#endif + +#define ZSTDv07_OPT_NUM (1<<12) +#define ZSTDv07_DICT_MAGIC 0xEC30A437 /* v0.7 */ + +#define ZSTDv07_REP_NUM 3 +#define ZSTDv07_REP_INIT ZSTDv07_REP_NUM +#define ZSTDv07_REP_MOVE (ZSTDv07_REP_NUM-1) +static const U32 repStartValue[ZSTDv07_REP_NUM] = { 1, 4, 8 }; + +#define KB *(1 <<10) +#define MB *(1 <<20) +#define GB *(1U<<30) + +#define BIT7 128 +#define BIT6 64 +#define BIT5 32 +#define BIT4 16 +#define BIT1 2 +#define BIT0 1 + +#define ZSTDv07_WINDOWLOG_ABSOLUTEMIN 10 +static const size_t ZSTDv07_fcs_fieldSize[4] = { 0, 2, 4, 8 }; +static const size_t ZSTDv07_did_fieldSize[4] = { 0, 1, 2, 4 }; + +#define ZSTDv07_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */ +static const size_t ZSTDv07_blockHeaderSize = ZSTDv07_BLOCKHEADERSIZE; +typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t; + +#define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */ +#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */ + MIN_SEQUENCES_SIZE /* nbSeq==0 */) /* for a non-null block */ + +#define HufLog 12 +typedef enum { lbt_huffman, lbt_repeat, lbt_raw, lbt_rle } litBlockType_t; + +#define LONGNBSEQ 0x7F00 + +#define MINMATCH 3 +#define EQUAL_READ32 4 + +#define Litbits 8 +#define MaxLit ((1<<Litbits) - 1) +#define MaxML 52 +#define MaxLL 35 +#define MaxOff 28 +#define MaxSeq MAX(MaxLL, MaxML) /* Assumption : MaxOff < MaxLL,MaxML */ +#define MLFSELog 9 +#define LLFSELog 9 +#define OffFSELog 8 + +#define FSEv07_ENCODING_RAW 0 +#define FSEv07_ENCODING_RLE 1 +#define FSEv07_ENCODING_STATIC 2 +#define FSEv07_ENCODING_DYNAMIC 3 + +static const U32 LL_bits[MaxLL+1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 2, 2, 3, 3, 4, 6, 7, 8, 9,10,11,12, + 13,14,15,16 }; +static const S16 LL_defaultNorm[MaxLL+1] = { 4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1, + -1,-1,-1,-1 }; +static const U32 LL_defaultNormLog = 6; + +static const U32 ML_bits[MaxML+1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 7, 8, 9,10,11, + 12,13,14,15,16 }; +static const S16 ML_defaultNorm[MaxML+1] = { 1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,-1,-1, + -1,-1,-1,-1,-1 }; +static const U32 ML_defaultNormLog = 6; + +static const S16 OF_defaultNorm[MaxOff+1] = { 1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1,-1,-1,-1,-1,-1 }; +static const U32 OF_defaultNormLog = 5; + + +/*-******************************************* +* Shared functions to include for inlining +*********************************************/ +static void ZSTDv07_copy8(void* dst, const void* src) { memcpy(dst, src, 8); } +#define COPY8(d,s) { ZSTDv07_copy8(d,s); d+=8; s+=8; } + +/*! ZSTDv07_wildcopy() : +* custom version of memcpy(), can copy up to 7 bytes too many (8 bytes if length==0) */ +#define WILDCOPY_OVERLENGTH 8 +MEM_STATIC void ZSTDv07_wildcopy(void* dst, const void* src, size_t length) +{ + const BYTE* ip = (const BYTE*)src; + BYTE* op = (BYTE*)dst; + BYTE* const oend = op + length; + do + COPY8(op, ip) + while (op < oend); +} + + +/*-******************************************* +* Private interfaces +*********************************************/ +typedef struct ZSTDv07_stats_s ZSTDv07_stats_t; + +typedef struct { + U32 off; + U32 len; +} ZSTDv07_match_t; + +typedef struct { + U32 price; + U32 off; + U32 mlen; + U32 litlen; + U32 rep[ZSTDv07_REP_INIT]; +} ZSTDv07_optimal_t; + +struct ZSTDv07_stats_s { U32 unused; }; +MEM_STATIC void ZSTDv07_statsPrint(ZSTDv07_stats_t* stats, U32 searchLength) { (void)stats; (void)searchLength; } +MEM_STATIC void ZSTDv07_statsInit(ZSTDv07_stats_t* stats) { (void)stats; } +MEM_STATIC void ZSTDv07_statsResetFreqs(ZSTDv07_stats_t* stats) { (void)stats; } +MEM_STATIC void ZSTDv07_statsUpdatePrices(ZSTDv07_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) { (void)stats; (void)litLength; (void)literals; (void)offset; (void)matchLength; } + +typedef struct { + void* buffer; + U32* offsetStart; + U32* offset; + BYTE* offCodeStart; + BYTE* litStart; + BYTE* lit; + U16* litLengthStart; + U16* litLength; + BYTE* llCodeStart; + U16* matchLengthStart; + U16* matchLength; + BYTE* mlCodeStart; + U32 longLengthID; /* 0 == no longLength; 1 == Lit.longLength; 2 == Match.longLength; */ + U32 longLengthPos; + /* opt */ + ZSTDv07_optimal_t* priceTable; + ZSTDv07_match_t* matchTable; + U32* matchLengthFreq; + U32* litLengthFreq; + U32* litFreq; + U32* offCodeFreq; + U32 matchLengthSum; + U32 matchSum; + U32 litLengthSum; + U32 litSum; + U32 offCodeSum; + U32 log2matchLengthSum; + U32 log2matchSum; + U32 log2litLengthSum; + U32 log2litSum; + U32 log2offCodeSum; + U32 factor; + U32 cachedPrice; + U32 cachedLitLength; + const BYTE* cachedLiterals; + ZSTDv07_stats_t stats; +} seqStore_t; + +void ZSTDv07_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq); + +/* custom memory allocation functions */ +void* ZSTDv07_defaultAllocFunction(void* opaque, size_t size); +void ZSTDv07_defaultFreeFunction(void* opaque, void* address); +static const ZSTDv07_customMem defaultCustomMem = { ZSTDv07_defaultAllocFunction, ZSTDv07_defaultFreeFunction, NULL }; + +#endif /* ZSTDv07_CCOMMON_H_MODULE */ +/* + zstd - standard compression library + Copyright (C) 2014-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - zstd homepage : http://www.zstd.net +*/ + +/* *************************************************************** +* Tuning parameters +*****************************************************************/ +/*! + * HEAPMODE : + * Select how default decompression function ZSTDv07_decompress() will allocate memory, + * in memory stack (0), or in memory heap (1, requires malloc()) + */ +#ifndef ZSTDv07_HEAPMODE +# define ZSTDv07_HEAPMODE 1 +#endif + + +/*-******************************************************* +* Compiler specifics +*********************************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# define FORCE_INLINE static __forceinline +# include <intrin.h> /* For Visual 2005 */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4324) /* disable: C4324: padded structure */ +#else +# ifdef __GNUC__ +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif +#endif + + +/*-************************************* +* Macros +***************************************/ +#define ZSTDv07_isError ERR_isError /* for inlining */ +#define FSEv07_isError ERR_isError +#define HUFv07_isError ERR_isError + + +/*_******************************************************* +* Memory operations +**********************************************************/ +static void ZSTDv07_copy4(void* dst, const void* src) { memcpy(dst, src, 4); } + + +/*-************************************************************* +* Context management +***************************************************************/ +typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader, + ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock, + ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTDv07_dStage; + +struct ZSTDv07_DCtx_s +{ + FSEv07_DTable LLTable[FSEv07_DTABLE_SIZE_U32(LLFSELog)]; + FSEv07_DTable OffTable[FSEv07_DTABLE_SIZE_U32(OffFSELog)]; + FSEv07_DTable MLTable[FSEv07_DTABLE_SIZE_U32(MLFSELog)]; + HUFv07_DTable hufTable[HUFv07_DTABLE_SIZE(HufLog)]; /* can accommodate HUFv07_decompress4X */ + const void* previousDstEnd; + const void* base; + const void* vBase; + const void* dictEnd; + size_t expected; + U32 rep[3]; + ZSTDv07_frameParams fParams; + blockType_t bType; /* used in ZSTDv07_decompressContinue(), to transfer blockType between header decoding and block decoding stages */ + ZSTDv07_dStage stage; + U32 litEntropy; + U32 fseEntropy; + XXH64_state_t xxhState; + size_t headerSize; + U32 dictID; + const BYTE* litPtr; + ZSTDv07_customMem customMem; + size_t litBufSize; + size_t litSize; + BYTE litBuffer[ZSTDv07_BLOCKSIZE_ABSOLUTEMAX + WILDCOPY_OVERLENGTH]; + BYTE headerBuffer[ZSTDv07_FRAMEHEADERSIZE_MAX]; +}; /* typedef'd to ZSTDv07_DCtx within "zstd_static.h" */ + +int ZSTDv07_isSkipFrame(ZSTDv07_DCtx* dctx); + +size_t ZSTDv07_sizeofDCtx (const ZSTDv07_DCtx* dctx) { return sizeof(*dctx); } + +size_t ZSTDv07_estimateDCtxSize(void) { return sizeof(ZSTDv07_DCtx); } + +size_t ZSTDv07_decompressBegin(ZSTDv07_DCtx* dctx) +{ + dctx->expected = ZSTDv07_frameHeaderSize_min; + dctx->stage = ZSTDds_getFrameHeaderSize; + dctx->previousDstEnd = NULL; + dctx->base = NULL; + dctx->vBase = NULL; + dctx->dictEnd = NULL; + dctx->hufTable[0] = (HUFv07_DTable)((HufLog)*0x1000001); + dctx->litEntropy = dctx->fseEntropy = 0; + dctx->dictID = 0; + { int i; for (i=0; i<ZSTDv07_REP_NUM; i++) dctx->rep[i] = repStartValue[i]; } + return 0; +} + +ZSTDv07_DCtx* ZSTDv07_createDCtx_advanced(ZSTDv07_customMem customMem) +{ + ZSTDv07_DCtx* dctx; + + if (!customMem.customAlloc && !customMem.customFree) + customMem = defaultCustomMem; + + if (!customMem.customAlloc || !customMem.customFree) + return NULL; + + dctx = (ZSTDv07_DCtx*) customMem.customAlloc(customMem.opaque, sizeof(ZSTDv07_DCtx)); + if (!dctx) return NULL; + memcpy(&dctx->customMem, &customMem, sizeof(ZSTDv07_customMem)); + ZSTDv07_decompressBegin(dctx); + return dctx; +} + +ZSTDv07_DCtx* ZSTDv07_createDCtx(void) +{ + return ZSTDv07_createDCtx_advanced(defaultCustomMem); +} + +size_t ZSTDv07_freeDCtx(ZSTDv07_DCtx* dctx) +{ + if (dctx==NULL) return 0; /* support free on NULL */ + dctx->customMem.customFree(dctx->customMem.opaque, dctx); + return 0; /* reserved as a potential error code in the future */ +} + +void ZSTDv07_copyDCtx(ZSTDv07_DCtx* dstDCtx, const ZSTDv07_DCtx* srcDCtx) +{ + memcpy(dstDCtx, srcDCtx, + sizeof(ZSTDv07_DCtx) - (ZSTDv07_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH + ZSTDv07_frameHeaderSize_max)); /* no need to copy workspace */ +} + + +/*-************************************************************* +* Decompression section +***************************************************************/ + +/* Frame format description + Frame Header - [ Block Header - Block ] - Frame End + 1) Frame Header + - 4 bytes - Magic Number : ZSTDv07_MAGICNUMBER (defined within zstd.h) + - 1 byte - Frame Descriptor + 2) Block Header + - 3 bytes, starting with a 2-bits descriptor + Uncompressed, Compressed, Frame End, unused + 3) Block + See Block Format Description + 4) Frame End + - 3 bytes, compatible with Block Header +*/ + + +/* Frame Header : + + 1 byte - FrameHeaderDescription : + bit 0-1 : dictID (0, 1, 2 or 4 bytes) + bit 2 : checksumFlag + bit 3 : reserved (must be zero) + bit 4 : reserved (unused, can be any value) + bit 5 : Single Segment (if 1, WindowLog byte is not present) + bit 6-7 : FrameContentFieldSize (0, 2, 4, or 8) + if (SkippedWindowLog && !FrameContentFieldsize) FrameContentFieldsize=1; + + Optional : WindowLog (0 or 1 byte) + bit 0-2 : octal Fractional (1/8th) + bit 3-7 : Power of 2, with 0 = 1 KB (up to 2 TB) + + Optional : dictID (0, 1, 2 or 4 bytes) + Automatic adaptation + 0 : no dictID + 1 : 1 - 255 + 2 : 256 - 65535 + 4 : all other values + + Optional : content size (0, 1, 2, 4 or 8 bytes) + 0 : unknown (fcfs==0 and swl==0) + 1 : 0-255 bytes (fcfs==0 and swl==1) + 2 : 256 - 65535+256 (fcfs==1) + 4 : 0 - 4GB-1 (fcfs==2) + 8 : 0 - 16EB-1 (fcfs==3) +*/ + + +/* Compressed Block, format description + + Block = Literal Section - Sequences Section + Prerequisite : size of (compressed) block, maximum size of regenerated data + + 1) Literal Section + + 1.1) Header : 1-5 bytes + flags: 2 bits + 00 compressed by Huff0 + 01 unused + 10 is Raw (uncompressed) + 11 is Rle + Note : using 01 => Huff0 with precomputed table ? + Note : delta map ? => compressed ? + + 1.1.1) Huff0-compressed literal block : 3-5 bytes + srcSize < 1 KB => 3 bytes (2-2-10-10) => single stream + srcSize < 1 KB => 3 bytes (2-2-10-10) + srcSize < 16KB => 4 bytes (2-2-14-14) + else => 5 bytes (2-2-18-18) + big endian convention + + 1.1.2) Raw (uncompressed) literal block header : 1-3 bytes + size : 5 bits: (IS_RAW<<6) + (0<<4) + size + 12 bits: (IS_RAW<<6) + (2<<4) + (size>>8) + size&255 + 20 bits: (IS_RAW<<6) + (3<<4) + (size>>16) + size>>8&255 + size&255 + + 1.1.3) Rle (repeated single byte) literal block header : 1-3 bytes + size : 5 bits: (IS_RLE<<6) + (0<<4) + size + 12 bits: (IS_RLE<<6) + (2<<4) + (size>>8) + size&255 + 20 bits: (IS_RLE<<6) + (3<<4) + (size>>16) + size>>8&255 + size&255 + + 1.1.4) Huff0-compressed literal block, using precomputed CTables : 3-5 bytes + srcSize < 1 KB => 3 bytes (2-2-10-10) => single stream + srcSize < 1 KB => 3 bytes (2-2-10-10) + srcSize < 16KB => 4 bytes (2-2-14-14) + else => 5 bytes (2-2-18-18) + big endian convention + + 1- CTable available (stored into workspace ?) + 2- Small input (fast heuristic ? Full comparison ? depend on clevel ?) + + + 1.2) Literal block content + + 1.2.1) Huff0 block, using sizes from header + See Huff0 format + + 1.2.2) Huff0 block, using prepared table + + 1.2.3) Raw content + + 1.2.4) single byte + + + 2) Sequences section + TO DO +*/ + +/** ZSTDv07_frameHeaderSize() : +* srcSize must be >= ZSTDv07_frameHeaderSize_min. +* @return : size of the Frame Header */ +static size_t ZSTDv07_frameHeaderSize(const void* src, size_t srcSize) +{ + if (srcSize < ZSTDv07_frameHeaderSize_min) return ERROR(srcSize_wrong); + { BYTE const fhd = ((const BYTE*)src)[4]; + U32 const dictID= fhd & 3; + U32 const directMode = (fhd >> 5) & 1; + U32 const fcsId = fhd >> 6; + return ZSTDv07_frameHeaderSize_min + !directMode + ZSTDv07_did_fieldSize[dictID] + ZSTDv07_fcs_fieldSize[fcsId] + + (directMode && !ZSTDv07_fcs_fieldSize[fcsId]); + } +} + + +/** ZSTDv07_getFrameParams() : +* decode Frame Header, or require larger `srcSize`. +* @return : 0, `fparamsPtr` is correctly filled, +* >0, `srcSize` is too small, result is expected `srcSize`, +* or an error code, which can be tested using ZSTDv07_isError() */ +size_t ZSTDv07_getFrameParams(ZSTDv07_frameParams* fparamsPtr, const void* src, size_t srcSize) +{ + const BYTE* ip = (const BYTE*)src; + + if (srcSize < ZSTDv07_frameHeaderSize_min) return ZSTDv07_frameHeaderSize_min; + if (MEM_readLE32(src) != ZSTDv07_MAGICNUMBER) { + if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTDv07_MAGIC_SKIPPABLE_START) { + if (srcSize < ZSTDv07_skippableHeaderSize) return ZSTDv07_skippableHeaderSize; /* magic number + skippable frame length */ + memset(fparamsPtr, 0, sizeof(*fparamsPtr)); + fparamsPtr->frameContentSize = MEM_readLE32((const char *)src + 4); + fparamsPtr->windowSize = 0; /* windowSize==0 means a frame is skippable */ + return 0; + } + return ERROR(prefix_unknown); + } + + /* ensure there is enough `srcSize` to fully read/decode frame header */ + { size_t const fhsize = ZSTDv07_frameHeaderSize(src, srcSize); + if (srcSize < fhsize) return fhsize; } + + { BYTE const fhdByte = ip[4]; + size_t pos = 5; + U32 const dictIDSizeCode = fhdByte&3; + U32 const checksumFlag = (fhdByte>>2)&1; + U32 const directMode = (fhdByte>>5)&1; + U32 const fcsID = fhdByte>>6; + U32 const windowSizeMax = 1U << ZSTDv07_WINDOWLOG_MAX; + U32 windowSize = 0; + U32 dictID = 0; + U64 frameContentSize = 0; + if ((fhdByte & 0x08) != 0) return ERROR(frameParameter_unsupported); /* reserved bits, which must be zero */ + if (!directMode) { + BYTE const wlByte = ip[pos++]; + U32 const windowLog = (wlByte >> 3) + ZSTDv07_WINDOWLOG_ABSOLUTEMIN; + if (windowLog > ZSTDv07_WINDOWLOG_MAX) return ERROR(frameParameter_unsupported); + windowSize = (1U << windowLog); + windowSize += (windowSize >> 3) * (wlByte&7); + } + + switch(dictIDSizeCode) + { + default: /* impossible */ + case 0 : break; + case 1 : dictID = ip[pos]; pos++; break; + case 2 : dictID = MEM_readLE16(ip+pos); pos+=2; break; + case 3 : dictID = MEM_readLE32(ip+pos); pos+=4; break; + } + switch(fcsID) + { + default: /* impossible */ + case 0 : if (directMode) frameContentSize = ip[pos]; break; + case 1 : frameContentSize = MEM_readLE16(ip+pos)+256; break; + case 2 : frameContentSize = MEM_readLE32(ip+pos); break; + case 3 : frameContentSize = MEM_readLE64(ip+pos); break; + } + if (!windowSize) windowSize = (U32)frameContentSize; + if (windowSize > windowSizeMax) return ERROR(frameParameter_unsupported); + fparamsPtr->frameContentSize = frameContentSize; + fparamsPtr->windowSize = windowSize; + fparamsPtr->dictID = dictID; + fparamsPtr->checksumFlag = checksumFlag; + } + return 0; +} + + +/** ZSTDv07_getDecompressedSize() : +* compatible with legacy mode +* @return : decompressed size if known, 0 otherwise + note : 0 can mean any of the following : + - decompressed size is not provided within frame header + - frame header unknown / not supported + - frame header not completely provided (`srcSize` too small) */ +unsigned long long ZSTDv07_getDecompressedSize(const void* src, size_t srcSize) +{ + { ZSTDv07_frameParams fparams; + size_t const frResult = ZSTDv07_getFrameParams(&fparams, src, srcSize); + if (frResult!=0) return 0; + return fparams.frameContentSize; + } +} + + +/** ZSTDv07_decodeFrameHeader() : +* `srcSize` must be the size provided by ZSTDv07_frameHeaderSize(). +* @return : 0 if success, or an error code, which can be tested using ZSTDv07_isError() */ +static size_t ZSTDv07_decodeFrameHeader(ZSTDv07_DCtx* dctx, const void* src, size_t srcSize) +{ + size_t const result = ZSTDv07_getFrameParams(&(dctx->fParams), src, srcSize); + if (dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID)) return ERROR(dictionary_wrong); + if (dctx->fParams.checksumFlag) XXH64_reset(&dctx->xxhState, 0); + return result; +} + + +typedef struct +{ + blockType_t blockType; + U32 origSize; +} blockProperties_t; + +/*! ZSTDv07_getcBlockSize() : +* Provides the size of compressed block from block header `src` */ +size_t ZSTDv07_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr) +{ + const BYTE* const in = (const BYTE* const)src; + U32 cSize; + + if (srcSize < ZSTDv07_blockHeaderSize) return ERROR(srcSize_wrong); + + bpPtr->blockType = (blockType_t)((*in) >> 6); + cSize = in[2] + (in[1]<<8) + ((in[0] & 7)<<16); + bpPtr->origSize = (bpPtr->blockType == bt_rle) ? cSize : 0; + + if (bpPtr->blockType == bt_end) return 0; + if (bpPtr->blockType == bt_rle) return 1; + return cSize; +} + + +static size_t ZSTDv07_copyRawBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + if (srcSize > dstCapacity) return ERROR(dstSize_tooSmall); + memcpy(dst, src, srcSize); + return srcSize; +} + + +/*! ZSTDv07_decodeLiteralsBlock() : + @return : nb of bytes read from src (< srcSize ) */ +size_t ZSTDv07_decodeLiteralsBlock(ZSTDv07_DCtx* dctx, + const void* src, size_t srcSize) /* note : srcSize < BLOCKSIZE */ +{ + const BYTE* const istart = (const BYTE*) src; + + if (srcSize < MIN_CBLOCK_SIZE) return ERROR(corruption_detected); + + switch((litBlockType_t)(istart[0]>> 6)) + { + case lbt_huffman: + { size_t litSize, litCSize, singleStream=0; + U32 lhSize = (istart[0] >> 4) & 3; + if (srcSize < 5) return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for lhSize, + cSize (+nbSeq) */ + switch(lhSize) + { + case 0: case 1: default: /* note : default is impossible, since lhSize into [0..3] */ + /* 2 - 2 - 10 - 10 */ + lhSize=3; + singleStream = istart[0] & 16; + litSize = ((istart[0] & 15) << 6) + (istart[1] >> 2); + litCSize = ((istart[1] & 3) << 8) + istart[2]; + break; + case 2: + /* 2 - 2 - 14 - 14 */ + lhSize=4; + litSize = ((istart[0] & 15) << 10) + (istart[1] << 2) + (istart[2] >> 6); + litCSize = ((istart[2] & 63) << 8) + istart[3]; + break; + case 3: + /* 2 - 2 - 18 - 18 */ + lhSize=5; + litSize = ((istart[0] & 15) << 14) + (istart[1] << 6) + (istart[2] >> 2); + litCSize = ((istart[2] & 3) << 16) + (istart[3] << 8) + istart[4]; + break; + } + if (litSize > ZSTDv07_BLOCKSIZE_ABSOLUTEMAX) return ERROR(corruption_detected); + if (litCSize + lhSize > srcSize) return ERROR(corruption_detected); + + if (HUFv07_isError(singleStream ? + HUFv07_decompress1X2_DCtx(dctx->hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize) : + HUFv07_decompress4X_hufOnly (dctx->hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize) )) + return ERROR(corruption_detected); + + dctx->litPtr = dctx->litBuffer; + dctx->litBufSize = ZSTDv07_BLOCKSIZE_ABSOLUTEMAX+8; + dctx->litSize = litSize; + dctx->litEntropy = 1; + return litCSize + lhSize; + } + case lbt_repeat: + { size_t litSize, litCSize; + U32 lhSize = ((istart[0]) >> 4) & 3; + if (lhSize != 1) /* only case supported for now : small litSize, single stream */ + return ERROR(corruption_detected); + if (dctx->litEntropy==0) + return ERROR(dictionary_corrupted); + + /* 2 - 2 - 10 - 10 */ + lhSize=3; + litSize = ((istart[0] & 15) << 6) + (istart[1] >> 2); + litCSize = ((istart[1] & 3) << 8) + istart[2]; + if (litCSize + lhSize > srcSize) return ERROR(corruption_detected); + + { size_t const errorCode = HUFv07_decompress1X4_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->hufTable); + if (HUFv07_isError(errorCode)) return ERROR(corruption_detected); + } + dctx->litPtr = dctx->litBuffer; + dctx->litBufSize = ZSTDv07_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH; + dctx->litSize = litSize; + return litCSize + lhSize; + } + case lbt_raw: + { size_t litSize; + U32 lhSize = ((istart[0]) >> 4) & 3; + switch(lhSize) + { + case 0: case 1: default: /* note : default is impossible, since lhSize into [0..3] */ + lhSize=1; + litSize = istart[0] & 31; + break; + case 2: + litSize = ((istart[0] & 15) << 8) + istart[1]; + break; + case 3: + litSize = ((istart[0] & 15) << 16) + (istart[1] << 8) + istart[2]; + break; + } + + if (lhSize+litSize+WILDCOPY_OVERLENGTH > srcSize) { /* risk reading beyond src buffer with wildcopy */ + if (litSize+lhSize > srcSize) return ERROR(corruption_detected); + memcpy(dctx->litBuffer, istart+lhSize, litSize); + dctx->litPtr = dctx->litBuffer; + dctx->litBufSize = ZSTDv07_BLOCKSIZE_ABSOLUTEMAX+8; + dctx->litSize = litSize; + return lhSize+litSize; + } + /* direct reference into compressed stream */ + dctx->litPtr = istart+lhSize; + dctx->litBufSize = srcSize-lhSize; + dctx->litSize = litSize; + return lhSize+litSize; + } + case lbt_rle: + { size_t litSize; + U32 lhSize = ((istart[0]) >> 4) & 3; + switch(lhSize) + { + case 0: case 1: default: /* note : default is impossible, since lhSize into [0..3] */ + lhSize = 1; + litSize = istart[0] & 31; + break; + case 2: + litSize = ((istart[0] & 15) << 8) + istart[1]; + break; + case 3: + litSize = ((istart[0] & 15) << 16) + (istart[1] << 8) + istart[2]; + if (srcSize<4) return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4 */ + break; + } + if (litSize > ZSTDv07_BLOCKSIZE_ABSOLUTEMAX) return ERROR(corruption_detected); + memset(dctx->litBuffer, istart[lhSize], litSize); + dctx->litPtr = dctx->litBuffer; + dctx->litBufSize = ZSTDv07_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH; + dctx->litSize = litSize; + return lhSize+1; + } + default: + return ERROR(corruption_detected); /* impossible */ + } +} + + +/*! ZSTDv07_buildSeqTable() : + @return : nb bytes read from src, + or an error code if it fails, testable with ZSTDv07_isError() +*/ +FORCE_INLINE size_t ZSTDv07_buildSeqTable(FSEv07_DTable* DTable, U32 type, U32 max, U32 maxLog, + const void* src, size_t srcSize, + const S16* defaultNorm, U32 defaultLog, U32 flagRepeatTable) +{ + switch(type) + { + case FSEv07_ENCODING_RLE : + if (!srcSize) return ERROR(srcSize_wrong); + if ( (*(const BYTE*)src) > max) return ERROR(corruption_detected); + FSEv07_buildDTable_rle(DTable, *(const BYTE*)src); /* if *src > max, data is corrupted */ + return 1; + case FSEv07_ENCODING_RAW : + FSEv07_buildDTable(DTable, defaultNorm, max, defaultLog); + return 0; + case FSEv07_ENCODING_STATIC: + if (!flagRepeatTable) return ERROR(corruption_detected); + return 0; + default : /* impossible */ + case FSEv07_ENCODING_DYNAMIC : + { U32 tableLog; + S16 norm[MaxSeq+1]; + size_t const headerSize = FSEv07_readNCount(norm, &max, &tableLog, src, srcSize); + if (FSEv07_isError(headerSize)) return ERROR(corruption_detected); + if (tableLog > maxLog) return ERROR(corruption_detected); + FSEv07_buildDTable(DTable, norm, max, tableLog); + return headerSize; + } } +} + + +size_t ZSTDv07_decodeSeqHeaders(int* nbSeqPtr, + FSEv07_DTable* DTableLL, FSEv07_DTable* DTableML, FSEv07_DTable* DTableOffb, U32 flagRepeatTable, + const void* src, size_t srcSize) +{ + const BYTE* const istart = (const BYTE* const)src; + const BYTE* const iend = istart + srcSize; + const BYTE* ip = istart; + + /* check */ + if (srcSize < MIN_SEQUENCES_SIZE) return ERROR(srcSize_wrong); + + /* SeqHead */ + { int nbSeq = *ip++; + if (!nbSeq) { *nbSeqPtr=0; return 1; } + if (nbSeq > 0x7F) { + if (nbSeq == 0xFF) + nbSeq = MEM_readLE16(ip) + LONGNBSEQ, ip+=2; + else + nbSeq = ((nbSeq-0x80)<<8) + *ip++; + } + *nbSeqPtr = nbSeq; + } + + /* FSE table descriptors */ + { U32 const LLtype = *ip >> 6; + U32 const OFtype = (*ip >> 4) & 3; + U32 const MLtype = (*ip >> 2) & 3; + ip++; + + /* check */ + if (ip > iend-3) return ERROR(srcSize_wrong); /* min : all 3 are "raw", hence no header, but at least xxLog bits per type */ + + /* Build DTables */ + { size_t const llhSize = ZSTDv07_buildSeqTable(DTableLL, LLtype, MaxLL, LLFSELog, ip, iend-ip, LL_defaultNorm, LL_defaultNormLog, flagRepeatTable); + if (ZSTDv07_isError(llhSize)) return ERROR(corruption_detected); + ip += llhSize; + } + { size_t const ofhSize = ZSTDv07_buildSeqTable(DTableOffb, OFtype, MaxOff, OffFSELog, ip, iend-ip, OF_defaultNorm, OF_defaultNormLog, flagRepeatTable); + if (ZSTDv07_isError(ofhSize)) return ERROR(corruption_detected); + ip += ofhSize; + } + { size_t const mlhSize = ZSTDv07_buildSeqTable(DTableML, MLtype, MaxML, MLFSELog, ip, iend-ip, ML_defaultNorm, ML_defaultNormLog, flagRepeatTable); + if (ZSTDv07_isError(mlhSize)) return ERROR(corruption_detected); + ip += mlhSize; + } } + + return ip-istart; +} + + +typedef struct { + size_t litLength; + size_t matchLength; + size_t offset; +} seq_t; + +typedef struct { + BITv07_DStream_t DStream; + FSEv07_DState_t stateLL; + FSEv07_DState_t stateOffb; + FSEv07_DState_t stateML; + size_t prevOffset[ZSTDv07_REP_INIT]; +} seqState_t; + + +static seq_t ZSTDv07_decodeSequence(seqState_t* seqState) +{ + seq_t seq; + + U32 const llCode = FSEv07_peekSymbol(&(seqState->stateLL)); + U32 const mlCode = FSEv07_peekSymbol(&(seqState->stateML)); + U32 const ofCode = FSEv07_peekSymbol(&(seqState->stateOffb)); /* <= maxOff, by table construction */ + + U32 const llBits = LL_bits[llCode]; + U32 const mlBits = ML_bits[mlCode]; + U32 const ofBits = ofCode; + U32 const totalBits = llBits+mlBits+ofBits; + + static const U32 LL_base[MaxLL+1] = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 18, 20, 22, 24, 28, 32, 40, 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, + 0x2000, 0x4000, 0x8000, 0x10000 }; + + static const U32 ML_base[MaxML+1] = { + 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, + 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, + 35, 37, 39, 41, 43, 47, 51, 59, 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, + 0x1003, 0x2003, 0x4003, 0x8003, 0x10003 }; + + static const U32 OF_base[MaxOff+1] = { + 0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, + 0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, + 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, + 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD }; + + /* sequence */ + { size_t offset; + if (!ofCode) + offset = 0; + else { + offset = OF_base[ofCode] + BITv07_readBits(&(seqState->DStream), ofBits); /* <= (ZSTDv07_WINDOWLOG_MAX-1) bits */ + if (MEM_32bits()) BITv07_reloadDStream(&(seqState->DStream)); + } + + if (ofCode <= 1) { + if ((llCode == 0) & (offset <= 1)) offset = 1-offset; + if (offset) { + size_t const temp = seqState->prevOffset[offset]; + if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1]; + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset = temp; + } else { + offset = seqState->prevOffset[0]; + } + } else { + seqState->prevOffset[2] = seqState->prevOffset[1]; + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset; + } + seq.offset = offset; + } + + seq.matchLength = ML_base[mlCode] + ((mlCode>31) ? BITv07_readBits(&(seqState->DStream), mlBits) : 0); /* <= 16 bits */ + if (MEM_32bits() && (mlBits+llBits>24)) BITv07_reloadDStream(&(seqState->DStream)); + + seq.litLength = LL_base[llCode] + ((llCode>15) ? BITv07_readBits(&(seqState->DStream), llBits) : 0); /* <= 16 bits */ + if (MEM_32bits() || + (totalBits > 64 - 7 - (LLFSELog+MLFSELog+OffFSELog)) ) BITv07_reloadDStream(&(seqState->DStream)); + + /* ANS state update */ + FSEv07_updateState(&(seqState->stateLL), &(seqState->DStream)); /* <= 9 bits */ + FSEv07_updateState(&(seqState->stateML), &(seqState->DStream)); /* <= 9 bits */ + if (MEM_32bits()) BITv07_reloadDStream(&(seqState->DStream)); /* <= 18 bits */ + FSEv07_updateState(&(seqState->stateOffb), &(seqState->DStream)); /* <= 8 bits */ + + return seq; +} + + +FORCE_INLINE +size_t ZSTDv07_execSequence(BYTE* op, + BYTE* const oend, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit_w, + const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ + BYTE* const oend_w = oend-WILDCOPY_OVERLENGTH; + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = oLitEnd - sequence.offset; + + /* check */ + if ((oLitEnd>oend_w) | (oMatchEnd>oend)) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */ + if (iLitEnd > litLimit_w) return ERROR(corruption_detected); /* over-read beyond lit buffer */ + + /* copy Literals */ + ZSTDv07_wildcopy(op, *litPtr, sequence.litLength); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ + op = oLitEnd; + *litPtr = iLitEnd; /* update for next sequence */ + + /* copy Match */ + if (sequence.offset > (size_t)(oLitEnd - base)) { + /* offset beyond prefix */ + if (sequence.offset > (size_t)(oLitEnd - vBase)) return ERROR(corruption_detected); + match = dictEnd - (base-match); + if (match + sequence.matchLength <= dictEnd) { + memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { size_t const length1 = dictEnd - match; + memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = base; + } } + + /* match within prefix */ + if (sequence.offset < 8) { + /* close range match, overlap */ + static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */ + static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* substracted */ + int const sub2 = dec64table[sequence.offset]; + op[0] = match[0]; + op[1] = match[1]; + op[2] = match[2]; + op[3] = match[3]; + match += dec32table[sequence.offset]; + ZSTDv07_copy4(op+4, match); + match -= sub2; + } else { + ZSTDv07_copy8(op, match); + } + op += 8; match += 8; + + if (oMatchEnd > oend-(16-MINMATCH)) { + if (op < oend_w) { + ZSTDv07_wildcopy(op, match, oend_w - op); + match += oend_w - op; + op = oend_w; + } + while (op < oMatchEnd) *op++ = *match++; + } else { + ZSTDv07_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */ + } + return sequenceLength; +} + + +static size_t ZSTDv07_decompressSequences( + ZSTDv07_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize) +{ + const BYTE* ip = (const BYTE*)seqStart; + const BYTE* const iend = ip + seqSize; + BYTE* const ostart = (BYTE* const)dst; + BYTE* const oend = ostart + maxDstSize; + BYTE* op = ostart; + const BYTE* litPtr = dctx->litPtr; + const BYTE* const litLimit_w = litPtr + dctx->litBufSize - WILDCOPY_OVERLENGTH; + const BYTE* const litEnd = litPtr + dctx->litSize; + FSEv07_DTable* DTableLL = dctx->LLTable; + FSEv07_DTable* DTableML = dctx->MLTable; + FSEv07_DTable* DTableOffb = dctx->OffTable; + const BYTE* const base = (const BYTE*) (dctx->base); + const BYTE* const vBase = (const BYTE*) (dctx->vBase); + const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); + int nbSeq; + + /* Build Decoding Tables */ + { size_t const seqHSize = ZSTDv07_decodeSeqHeaders(&nbSeq, DTableLL, DTableML, DTableOffb, dctx->fseEntropy, ip, seqSize); + if (ZSTDv07_isError(seqHSize)) return seqHSize; + ip += seqHSize; + } + + /* Regen sequences */ + if (nbSeq) { + seqState_t seqState; + dctx->fseEntropy = 1; + { U32 i; for (i=0; i<ZSTDv07_REP_INIT; i++) seqState.prevOffset[i] = dctx->rep[i]; } + { size_t const errorCode = BITv07_initDStream(&(seqState.DStream), ip, iend-ip); + if (ERR_isError(errorCode)) return ERROR(corruption_detected); } + FSEv07_initDState(&(seqState.stateLL), &(seqState.DStream), DTableLL); + FSEv07_initDState(&(seqState.stateOffb), &(seqState.DStream), DTableOffb); + FSEv07_initDState(&(seqState.stateML), &(seqState.DStream), DTableML); + + for ( ; (BITv07_reloadDStream(&(seqState.DStream)) <= BITv07_DStream_completed) && nbSeq ; ) { + nbSeq--; + { seq_t const sequence = ZSTDv07_decodeSequence(&seqState); + size_t const oneSeqSize = ZSTDv07_execSequence(op, oend, sequence, &litPtr, litLimit_w, base, vBase, dictEnd); + if (ZSTDv07_isError(oneSeqSize)) return oneSeqSize; + op += oneSeqSize; + } } + + /* check if reached exact end */ + if (nbSeq) return ERROR(corruption_detected); + /* save reps for next block */ + { U32 i; for (i=0; i<ZSTDv07_REP_INIT; i++) dctx->rep[i] = (U32)(seqState.prevOffset[i]); } + } + + /* last literal segment */ + { size_t const lastLLSize = litEnd - litPtr; + //if (litPtr > litEnd) return ERROR(corruption_detected); /* too many literals already used */ + if (lastLLSize > (size_t)(oend-op)) return ERROR(dstSize_tooSmall); + memcpy(op, litPtr, lastLLSize); + op += lastLLSize; + } + + return op-ostart; +} + + +static void ZSTDv07_checkContinuity(ZSTDv07_DCtx* dctx, const void* dst) +{ + if (dst != dctx->previousDstEnd) { /* not contiguous */ + dctx->dictEnd = dctx->previousDstEnd; + dctx->vBase = (const char*)dst - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->base)); + dctx->base = dst; + dctx->previousDstEnd = dst; + } +} + + +static size_t ZSTDv07_decompressBlock_internal(ZSTDv07_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ /* blockType == blockCompressed */ + const BYTE* ip = (const BYTE*)src; + + if (srcSize >= ZSTDv07_BLOCKSIZE_ABSOLUTEMAX) return ERROR(srcSize_wrong); + + /* Decode literals sub-block */ + { size_t const litCSize = ZSTDv07_decodeLiteralsBlock(dctx, src, srcSize); + if (ZSTDv07_isError(litCSize)) return litCSize; + ip += litCSize; + srcSize -= litCSize; + } + return ZSTDv07_decompressSequences(dctx, dst, dstCapacity, ip, srcSize); +} + + +size_t ZSTDv07_decompressBlock(ZSTDv07_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + size_t dSize; + ZSTDv07_checkContinuity(dctx, dst); + dSize = ZSTDv07_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize); + dctx->previousDstEnd = (char*)dst + dSize; + return dSize; +} + + +/** ZSTDv07_insertBlock() : + insert `src` block into `dctx` history. Useful to track uncompressed blocks. */ +ZSTDLIB_API size_t ZSTDv07_insertBlock(ZSTDv07_DCtx* dctx, const void* blockStart, size_t blockSize) +{ + ZSTDv07_checkContinuity(dctx, blockStart); + dctx->previousDstEnd = (const char*)blockStart + blockSize; + return blockSize; +} + + +size_t ZSTDv07_generateNxBytes(void* dst, size_t dstCapacity, BYTE byte, size_t length) +{ + if (length > dstCapacity) return ERROR(dstSize_tooSmall); + memset(dst, byte, length); + return length; +} + + +/*! ZSTDv07_decompressFrame() : +* `dctx` must be properly initialized */ +static size_t ZSTDv07_decompressFrame(ZSTDv07_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + const BYTE* ip = (const BYTE*)src; + const BYTE* const iend = ip + srcSize; + BYTE* const ostart = (BYTE* const)dst; + BYTE* const oend = ostart + dstCapacity; + BYTE* op = ostart; + size_t remainingSize = srcSize; + + /* check */ + if (srcSize < ZSTDv07_frameHeaderSize_min+ZSTDv07_blockHeaderSize) return ERROR(srcSize_wrong); + + /* Frame Header */ + { size_t const frameHeaderSize = ZSTDv07_frameHeaderSize(src, ZSTDv07_frameHeaderSize_min); + if (ZSTDv07_isError(frameHeaderSize)) return frameHeaderSize; + if (srcSize < frameHeaderSize+ZSTDv07_blockHeaderSize) return ERROR(srcSize_wrong); + if (ZSTDv07_decodeFrameHeader(dctx, src, frameHeaderSize)) return ERROR(corruption_detected); + ip += frameHeaderSize; remainingSize -= frameHeaderSize; + } + + /* Loop on each block */ + while (1) { + size_t decodedSize; + blockProperties_t blockProperties; + size_t const cBlockSize = ZSTDv07_getcBlockSize(ip, iend-ip, &blockProperties); + if (ZSTDv07_isError(cBlockSize)) return cBlockSize; + + ip += ZSTDv07_blockHeaderSize; + remainingSize -= ZSTDv07_blockHeaderSize; + if (cBlockSize > remainingSize) return ERROR(srcSize_wrong); + + switch(blockProperties.blockType) + { + case bt_compressed: + decodedSize = ZSTDv07_decompressBlock_internal(dctx, op, oend-op, ip, cBlockSize); + break; + case bt_raw : + decodedSize = ZSTDv07_copyRawBlock(op, oend-op, ip, cBlockSize); + break; + case bt_rle : + decodedSize = ZSTDv07_generateNxBytes(op, oend-op, *ip, blockProperties.origSize); + break; + case bt_end : + /* end of frame */ + if (remainingSize) return ERROR(srcSize_wrong); + decodedSize = 0; + break; + default: + return ERROR(GENERIC); /* impossible */ + } + if (blockProperties.blockType == bt_end) break; /* bt_end */ + + if (ZSTDv07_isError(decodedSize)) return decodedSize; + if (dctx->fParams.checksumFlag) XXH64_update(&dctx->xxhState, op, decodedSize); + op += decodedSize; + ip += cBlockSize; + remainingSize -= cBlockSize; + } + + return op-ostart; +} + + +/*! ZSTDv07_decompress_usingPreparedDCtx() : +* Same as ZSTDv07_decompress_usingDict, but using a reference context `preparedDCtx`, where dictionary has been loaded. +* It avoids reloading the dictionary each time. +* `preparedDCtx` must have been properly initialized using ZSTDv07_decompressBegin_usingDict(). +* Requires 2 contexts : 1 for reference (preparedDCtx), which will not be modified, and 1 to run the decompression operation (dctx) */ +size_t ZSTDv07_decompress_usingPreparedDCtx(ZSTDv07_DCtx* dctx, const ZSTDv07_DCtx* refDCtx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + ZSTDv07_copyDCtx(dctx, refDCtx); + ZSTDv07_checkContinuity(dctx, dst); + return ZSTDv07_decompressFrame(dctx, dst, dstCapacity, src, srcSize); +} + + +size_t ZSTDv07_decompress_usingDict(ZSTDv07_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict, size_t dictSize) +{ + ZSTDv07_decompressBegin_usingDict(dctx, dict, dictSize); + ZSTDv07_checkContinuity(dctx, dst); + return ZSTDv07_decompressFrame(dctx, dst, dstCapacity, src, srcSize); +} + + +size_t ZSTDv07_decompressDCtx(ZSTDv07_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + return ZSTDv07_decompress_usingDict(dctx, dst, dstCapacity, src, srcSize, NULL, 0); +} + + +size_t ZSTDv07_decompress(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ +#if defined(ZSTDv07_HEAPMODE) && (ZSTDv07_HEAPMODE==1) + size_t regenSize; + ZSTDv07_DCtx* const dctx = ZSTDv07_createDCtx(); + if (dctx==NULL) return ERROR(memory_allocation); + regenSize = ZSTDv07_decompressDCtx(dctx, dst, dstCapacity, src, srcSize); + ZSTDv07_freeDCtx(dctx); + return regenSize; +#else /* stack mode */ + ZSTDv07_DCtx dctx; + return ZSTDv07_decompressDCtx(&dctx, dst, dstCapacity, src, srcSize); +#endif +} + + +/*_****************************** +* Streaming Decompression API +********************************/ +size_t ZSTDv07_nextSrcSizeToDecompress(ZSTDv07_DCtx* dctx) +{ + return dctx->expected; +} + +int ZSTDv07_isSkipFrame(ZSTDv07_DCtx* dctx) +{ + return dctx->stage == ZSTDds_skipFrame; +} + +/** ZSTDv07_decompressContinue() : +* @return : nb of bytes generated into `dst` (necessarily <= `dstCapacity) +* or an error code, which can be tested using ZSTDv07_isError() */ +size_t ZSTDv07_decompressContinue(ZSTDv07_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + /* Sanity check */ + if (srcSize != dctx->expected) return ERROR(srcSize_wrong); + if (dstCapacity) ZSTDv07_checkContinuity(dctx, dst); + + switch (dctx->stage) + { + case ZSTDds_getFrameHeaderSize : + if (srcSize != ZSTDv07_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */ + if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTDv07_MAGIC_SKIPPABLE_START) { + memcpy(dctx->headerBuffer, src, ZSTDv07_frameHeaderSize_min); + dctx->expected = ZSTDv07_skippableHeaderSize - ZSTDv07_frameHeaderSize_min; /* magic number + skippable frame length */ + dctx->stage = ZSTDds_decodeSkippableHeader; + return 0; + } + dctx->headerSize = ZSTDv07_frameHeaderSize(src, ZSTDv07_frameHeaderSize_min); + if (ZSTDv07_isError(dctx->headerSize)) return dctx->headerSize; + memcpy(dctx->headerBuffer, src, ZSTDv07_frameHeaderSize_min); + if (dctx->headerSize > ZSTDv07_frameHeaderSize_min) { + dctx->expected = dctx->headerSize - ZSTDv07_frameHeaderSize_min; + dctx->stage = ZSTDds_decodeFrameHeader; + return 0; + } + dctx->expected = 0; /* not necessary to copy more */ + + case ZSTDds_decodeFrameHeader: + { size_t result; + memcpy(dctx->headerBuffer + ZSTDv07_frameHeaderSize_min, src, dctx->expected); + result = ZSTDv07_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize); + if (ZSTDv07_isError(result)) return result; + dctx->expected = ZSTDv07_blockHeaderSize; + dctx->stage = ZSTDds_decodeBlockHeader; + return 0; + } + case ZSTDds_decodeBlockHeader: + { blockProperties_t bp; + size_t const cBlockSize = ZSTDv07_getcBlockSize(src, ZSTDv07_blockHeaderSize, &bp); + if (ZSTDv07_isError(cBlockSize)) return cBlockSize; + if (bp.blockType == bt_end) { + if (dctx->fParams.checksumFlag) { + U64 const h64 = XXH64_digest(&dctx->xxhState); + U32 const h32 = (U32)(h64>>11) & ((1<<22)-1); + const BYTE* const ip = (const BYTE*)src; + U32 const check32 = ip[2] + (ip[1] << 8) + ((ip[0] & 0x3F) << 16); + if (check32 != h32) return ERROR(checksum_wrong); + } + dctx->expected = 0; + dctx->stage = ZSTDds_getFrameHeaderSize; + } else { + dctx->expected = cBlockSize; + dctx->bType = bp.blockType; + dctx->stage = ZSTDds_decompressBlock; + } + return 0; + } + case ZSTDds_decompressBlock: + { size_t rSize; + switch(dctx->bType) + { + case bt_compressed: + rSize = ZSTDv07_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize); + break; + case bt_raw : + rSize = ZSTDv07_copyRawBlock(dst, dstCapacity, src, srcSize); + break; + case bt_rle : + return ERROR(GENERIC); /* not yet handled */ + break; + case bt_end : /* should never happen (filtered at phase 1) */ + rSize = 0; + break; + default: + return ERROR(GENERIC); /* impossible */ + } + dctx->stage = ZSTDds_decodeBlockHeader; + dctx->expected = ZSTDv07_blockHeaderSize; + dctx->previousDstEnd = (char*)dst + rSize; + if (ZSTDv07_isError(rSize)) return rSize; + if (dctx->fParams.checksumFlag) XXH64_update(&dctx->xxhState, dst, rSize); + return rSize; + } + case ZSTDds_decodeSkippableHeader: + { memcpy(dctx->headerBuffer + ZSTDv07_frameHeaderSize_min, src, dctx->expected); + dctx->expected = MEM_readLE32(dctx->headerBuffer + 4); + dctx->stage = ZSTDds_skipFrame; + return 0; + } + case ZSTDds_skipFrame: + { dctx->expected = 0; + dctx->stage = ZSTDds_getFrameHeaderSize; + return 0; + } + default: + return ERROR(GENERIC); /* impossible */ + } +} + + +static size_t ZSTDv07_refDictContent(ZSTDv07_DCtx* dctx, const void* dict, size_t dictSize) +{ + dctx->dictEnd = dctx->previousDstEnd; + dctx->vBase = (const char*)dict - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->base)); + dctx->base = dict; + dctx->previousDstEnd = (const char*)dict + dictSize; + return 0; +} + +static size_t ZSTDv07_loadEntropy(ZSTDv07_DCtx* dctx, const void* const dict, size_t const dictSize) +{ + const BYTE* dictPtr = (const BYTE*)dict; + const BYTE* const dictEnd = dictPtr + dictSize; + + { size_t const hSize = HUFv07_readDTableX4(dctx->hufTable, dict, dictSize); + if (HUFv07_isError(hSize)) return ERROR(dictionary_corrupted); + dictPtr += hSize; + } + + { short offcodeNCount[MaxOff+1]; + U32 offcodeMaxValue=MaxOff, offcodeLog=OffFSELog; + size_t const offcodeHeaderSize = FSEv07_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr); + if (FSEv07_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); + { size_t const errorCode = FSEv07_buildDTable(dctx->OffTable, offcodeNCount, offcodeMaxValue, offcodeLog); + if (FSEv07_isError(errorCode)) return ERROR(dictionary_corrupted); } + dictPtr += offcodeHeaderSize; + } + + { short matchlengthNCount[MaxML+1]; + unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog; + size_t const matchlengthHeaderSize = FSEv07_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr); + if (FSEv07_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted); + { size_t const errorCode = FSEv07_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog); + if (FSEv07_isError(errorCode)) return ERROR(dictionary_corrupted); } + dictPtr += matchlengthHeaderSize; + } + + { short litlengthNCount[MaxLL+1]; + unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog; + size_t const litlengthHeaderSize = FSEv07_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr); + if (FSEv07_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted); + { size_t const errorCode = FSEv07_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog); + if (FSEv07_isError(errorCode)) return ERROR(dictionary_corrupted); } + dictPtr += litlengthHeaderSize; + } + + if (dictPtr+12 > dictEnd) return ERROR(dictionary_corrupted); + dctx->rep[0] = MEM_readLE32(dictPtr+0); if (dctx->rep[0] >= dictSize) return ERROR(dictionary_corrupted); + dctx->rep[1] = MEM_readLE32(dictPtr+4); if (dctx->rep[1] >= dictSize) return ERROR(dictionary_corrupted); + dctx->rep[2] = MEM_readLE32(dictPtr+8); if (dctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted); + dictPtr += 12; + + dctx->litEntropy = dctx->fseEntropy = 1; + return dictPtr - (const BYTE*)dict; +} + +static size_t ZSTDv07_decompress_insertDictionary(ZSTDv07_DCtx* dctx, const void* dict, size_t dictSize) +{ + if (dictSize < 8) return ZSTDv07_refDictContent(dctx, dict, dictSize); + { U32 const magic = MEM_readLE32(dict); + if (magic != ZSTDv07_DICT_MAGIC) { + return ZSTDv07_refDictContent(dctx, dict, dictSize); /* pure content mode */ + } } + dctx->dictID = MEM_readLE32((const char*)dict + 4); + + /* load entropy tables */ + dict = (const char*)dict + 8; + dictSize -= 8; + { size_t const eSize = ZSTDv07_loadEntropy(dctx, dict, dictSize); + if (ZSTDv07_isError(eSize)) return ERROR(dictionary_corrupted); + dict = (const char*)dict + eSize; + dictSize -= eSize; + } + + /* reference dictionary content */ + return ZSTDv07_refDictContent(dctx, dict, dictSize); +} + + +size_t ZSTDv07_decompressBegin_usingDict(ZSTDv07_DCtx* dctx, const void* dict, size_t dictSize) +{ + { size_t const errorCode = ZSTDv07_decompressBegin(dctx); + if (ZSTDv07_isError(errorCode)) return errorCode; } + + if (dict && dictSize) { + size_t const errorCode = ZSTDv07_decompress_insertDictionary(dctx, dict, dictSize); + if (ZSTDv07_isError(errorCode)) return ERROR(dictionary_corrupted); + } + + return 0; +} + + +struct ZSTDv07_DDict_s { + void* dict; + size_t dictSize; + ZSTDv07_DCtx* refContext; +}; /* typedef'd tp ZSTDv07_CDict within zstd.h */ + +ZSTDv07_DDict* ZSTDv07_createDDict_advanced(const void* dict, size_t dictSize, ZSTDv07_customMem customMem) +{ + if (!customMem.customAlloc && !customMem.customFree) + customMem = defaultCustomMem; + + if (!customMem.customAlloc || !customMem.customFree) + return NULL; + + { ZSTDv07_DDict* const ddict = (ZSTDv07_DDict*) customMem.customAlloc(customMem.opaque, sizeof(*ddict)); + void* const dictContent = customMem.customAlloc(customMem.opaque, dictSize); + ZSTDv07_DCtx* const dctx = ZSTDv07_createDCtx_advanced(customMem); + + if (!dictContent || !ddict || !dctx) { + customMem.customFree(customMem.opaque, dictContent); + customMem.customFree(customMem.opaque, ddict); + customMem.customFree(customMem.opaque, dctx); + return NULL; + } + + memcpy(dictContent, dict, dictSize); + { size_t const errorCode = ZSTDv07_decompressBegin_usingDict(dctx, dictContent, dictSize); + if (ZSTDv07_isError(errorCode)) { + customMem.customFree(customMem.opaque, dictContent); + customMem.customFree(customMem.opaque, ddict); + customMem.customFree(customMem.opaque, dctx); + return NULL; + } } + + ddict->dict = dictContent; + ddict->dictSize = dictSize; + ddict->refContext = dctx; + return ddict; + } +} + +/*! ZSTDv07_createDDict() : +* Create a digested dictionary, ready to start decompression without startup delay. +* `dict` can be released after `ZSTDv07_DDict` creation */ +ZSTDv07_DDict* ZSTDv07_createDDict(const void* dict, size_t dictSize) +{ + ZSTDv07_customMem const allocator = { NULL, NULL, NULL }; + return ZSTDv07_createDDict_advanced(dict, dictSize, allocator); +} + +size_t ZSTDv07_freeDDict(ZSTDv07_DDict* ddict) +{ + ZSTDv07_freeFunction const cFree = ddict->refContext->customMem.customFree; + void* const opaque = ddict->refContext->customMem.opaque; + ZSTDv07_freeDCtx(ddict->refContext); + cFree(opaque, ddict->dict); + cFree(opaque, ddict); + return 0; +} + +/*! ZSTDv07_decompress_usingDDict() : +* Decompression using a pre-digested Dictionary +* Use dictionary without significant overhead. */ +ZSTDLIB_API size_t ZSTDv07_decompress_usingDDict(ZSTDv07_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTDv07_DDict* ddict) +{ + return ZSTDv07_decompress_usingPreparedDCtx(dctx, ddict->refContext, + dst, dstCapacity, + src, srcSize); +} +/* + Buffered version of Zstd compression library + Copyright (C) 2015-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - zstd homepage : http://www.zstd.net/ +*/ + + + +/*-*************************************************************************** +* Streaming decompression howto +* +* A ZBUFFv07_DCtx object is required to track streaming operations. +* Use ZBUFFv07_createDCtx() and ZBUFFv07_freeDCtx() to create/release resources. +* Use ZBUFFv07_decompressInit() to start a new decompression operation, +* or ZBUFFv07_decompressInitDictionary() if decompression requires a dictionary. +* Note that ZBUFFv07_DCtx objects can be re-init multiple times. +* +* Use ZBUFFv07_decompressContinue() repetitively to consume your input. +* *srcSizePtr and *dstCapacityPtr can be any size. +* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr. +* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again. +* The content of @dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change @dst. +* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency), +* or 0 when a frame is completely decoded, +* or an error code, which can be tested using ZBUFFv07_isError(). +* +* Hint : recommended buffer sizes (not compulsory) : ZBUFFv07_recommendedDInSize() and ZBUFFv07_recommendedDOutSize() +* output : ZBUFFv07_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded. +* input : ZBUFFv07_recommendedDInSize == 128KB + 3; +* just follow indications from ZBUFFv07_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . +* *******************************************************************************/ + +typedef enum { ZBUFFds_init, ZBUFFds_loadHeader, + ZBUFFds_read, ZBUFFds_load, ZBUFFds_flush } ZBUFFv07_dStage; + +/* *** Resource management *** */ +struct ZBUFFv07_DCtx_s { + ZSTDv07_DCtx* zd; + ZSTDv07_frameParams fParams; + ZBUFFv07_dStage stage; + char* inBuff; + size_t inBuffSize; + size_t inPos; + char* outBuff; + size_t outBuffSize; + size_t outStart; + size_t outEnd; + size_t blockSize; + BYTE headerBuffer[ZSTDv07_FRAMEHEADERSIZE_MAX]; + size_t lhSize; + ZSTDv07_customMem customMem; +}; /* typedef'd to ZBUFFv07_DCtx within "zstd_buffered.h" */ + +ZSTDLIB_API ZBUFFv07_DCtx* ZBUFFv07_createDCtx_advanced(ZSTDv07_customMem customMem); + +ZBUFFv07_DCtx* ZBUFFv07_createDCtx(void) +{ + return ZBUFFv07_createDCtx_advanced(defaultCustomMem); +} + +ZBUFFv07_DCtx* ZBUFFv07_createDCtx_advanced(ZSTDv07_customMem customMem) +{ + ZBUFFv07_DCtx* zbd; + + if (!customMem.customAlloc && !customMem.customFree) + customMem = defaultCustomMem; + + if (!customMem.customAlloc || !customMem.customFree) + return NULL; + + zbd = (ZBUFFv07_DCtx*)customMem.customAlloc(customMem.opaque, sizeof(ZBUFFv07_DCtx)); + if (zbd==NULL) return NULL; + memset(zbd, 0, sizeof(ZBUFFv07_DCtx)); + memcpy(&zbd->customMem, &customMem, sizeof(ZSTDv07_customMem)); + zbd->zd = ZSTDv07_createDCtx_advanced(customMem); + if (zbd->zd == NULL) { ZBUFFv07_freeDCtx(zbd); return NULL; } + zbd->stage = ZBUFFds_init; + return zbd; +} + +size_t ZBUFFv07_freeDCtx(ZBUFFv07_DCtx* zbd) +{ + if (zbd==NULL) return 0; /* support free on null */ + ZSTDv07_freeDCtx(zbd->zd); + if (zbd->inBuff) zbd->customMem.customFree(zbd->customMem.opaque, zbd->inBuff); + if (zbd->outBuff) zbd->customMem.customFree(zbd->customMem.opaque, zbd->outBuff); + zbd->customMem.customFree(zbd->customMem.opaque, zbd); + return 0; +} + + +/* *** Initialization *** */ + +size_t ZBUFFv07_decompressInitDictionary(ZBUFFv07_DCtx* zbd, const void* dict, size_t dictSize) +{ + zbd->stage = ZBUFFds_loadHeader; + zbd->lhSize = zbd->inPos = zbd->outStart = zbd->outEnd = 0; + return ZSTDv07_decompressBegin_usingDict(zbd->zd, dict, dictSize); +} + +size_t ZBUFFv07_decompressInit(ZBUFFv07_DCtx* zbd) +{ + return ZBUFFv07_decompressInitDictionary(zbd, NULL, 0); +} + + +/* internal util function */ +MEM_STATIC size_t ZBUFFv07_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + size_t const length = MIN(dstCapacity, srcSize); + memcpy(dst, src, length); + return length; +} + + +/* *** Decompression *** */ + +size_t ZBUFFv07_decompressContinue(ZBUFFv07_DCtx* zbd, + void* dst, size_t* dstCapacityPtr, + const void* src, size_t* srcSizePtr) +{ + const char* const istart = (const char*)src; + const char* const iend = istart + *srcSizePtr; + const char* ip = istart; + char* const ostart = (char*)dst; + char* const oend = ostart + *dstCapacityPtr; + char* op = ostart; + U32 notDone = 1; + + while (notDone) { + switch(zbd->stage) + { + case ZBUFFds_init : + return ERROR(init_missing); + + case ZBUFFds_loadHeader : + { size_t const hSize = ZSTDv07_getFrameParams(&(zbd->fParams), zbd->headerBuffer, zbd->lhSize); + if (hSize != 0) { + size_t const toLoad = hSize - zbd->lhSize; /* if hSize!=0, hSize > zbd->lhSize */ + if (ZSTDv07_isError(hSize)) return hSize; + if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */ + memcpy(zbd->headerBuffer + zbd->lhSize, ip, iend-ip); + zbd->lhSize += iend-ip; + *dstCapacityPtr = 0; + return (hSize - zbd->lhSize) + ZSTDv07_blockHeaderSize; /* remaining header bytes + next block header */ + } + memcpy(zbd->headerBuffer + zbd->lhSize, ip, toLoad); zbd->lhSize = hSize; ip += toLoad; + break; + } } + + /* Consume header */ + { size_t const h1Size = ZSTDv07_nextSrcSizeToDecompress(zbd->zd); /* == ZSTDv07_frameHeaderSize_min */ + size_t const h1Result = ZSTDv07_decompressContinue(zbd->zd, NULL, 0, zbd->headerBuffer, h1Size); + if (ZSTDv07_isError(h1Result)) return h1Result; + if (h1Size < zbd->lhSize) { /* long header */ + size_t const h2Size = ZSTDv07_nextSrcSizeToDecompress(zbd->zd); + size_t const h2Result = ZSTDv07_decompressContinue(zbd->zd, NULL, 0, zbd->headerBuffer+h1Size, h2Size); + if (ZSTDv07_isError(h2Result)) return h2Result; + } } + + zbd->fParams.windowSize = MAX(zbd->fParams.windowSize, 1U << ZSTDv07_WINDOWLOG_ABSOLUTEMIN); + + /* Frame header instruct buffer sizes */ + { size_t const blockSize = MIN(zbd->fParams.windowSize, ZSTDv07_BLOCKSIZE_ABSOLUTEMAX); + zbd->blockSize = blockSize; + if (zbd->inBuffSize < blockSize) { + zbd->customMem.customFree(zbd->customMem.opaque, zbd->inBuff); + zbd->inBuffSize = blockSize; + zbd->inBuff = (char*)zbd->customMem.customAlloc(zbd->customMem.opaque, blockSize); + if (zbd->inBuff == NULL) return ERROR(memory_allocation); + } + { size_t const neededOutSize = zbd->fParams.windowSize + blockSize; + if (zbd->outBuffSize < neededOutSize) { + zbd->customMem.customFree(zbd->customMem.opaque, zbd->outBuff); + zbd->outBuffSize = neededOutSize; + zbd->outBuff = (char*)zbd->customMem.customAlloc(zbd->customMem.opaque, neededOutSize); + if (zbd->outBuff == NULL) return ERROR(memory_allocation); + } } } + zbd->stage = ZBUFFds_read; + + case ZBUFFds_read: + { size_t const neededInSize = ZSTDv07_nextSrcSizeToDecompress(zbd->zd); + if (neededInSize==0) { /* end of frame */ + zbd->stage = ZBUFFds_init; + notDone = 0; + break; + } + if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */ + const int isSkipFrame = ZSTDv07_isSkipFrame(zbd->zd); + size_t const decodedSize = ZSTDv07_decompressContinue(zbd->zd, + zbd->outBuff + zbd->outStart, (isSkipFrame ? 0 : zbd->outBuffSize - zbd->outStart), + ip, neededInSize); + if (ZSTDv07_isError(decodedSize)) return decodedSize; + ip += neededInSize; + if (!decodedSize && !isSkipFrame) break; /* this was just a header */ + zbd->outEnd = zbd->outStart + decodedSize; + zbd->stage = ZBUFFds_flush; + break; + } + if (ip==iend) { notDone = 0; break; } /* no more input */ + zbd->stage = ZBUFFds_load; + } + + case ZBUFFds_load: + { size_t const neededInSize = ZSTDv07_nextSrcSizeToDecompress(zbd->zd); + size_t const toLoad = neededInSize - zbd->inPos; /* should always be <= remaining space within inBuff */ + size_t loadedSize; + if (toLoad > zbd->inBuffSize - zbd->inPos) return ERROR(corruption_detected); /* should never happen */ + loadedSize = ZBUFFv07_limitCopy(zbd->inBuff + zbd->inPos, toLoad, ip, iend-ip); + ip += loadedSize; + zbd->inPos += loadedSize; + if (loadedSize < toLoad) { notDone = 0; break; } /* not enough input, wait for more */ + + /* decode loaded input */ + { const int isSkipFrame = ZSTDv07_isSkipFrame(zbd->zd); + size_t const decodedSize = ZSTDv07_decompressContinue(zbd->zd, + zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart, + zbd->inBuff, neededInSize); + if (ZSTDv07_isError(decodedSize)) return decodedSize; + zbd->inPos = 0; /* input is consumed */ + if (!decodedSize && !isSkipFrame) { zbd->stage = ZBUFFds_read; break; } /* this was just a header */ + zbd->outEnd = zbd->outStart + decodedSize; + zbd->stage = ZBUFFds_flush; + // break; /* ZBUFFds_flush follows */ + } } + + case ZBUFFds_flush: + { size_t const toFlushSize = zbd->outEnd - zbd->outStart; + size_t const flushedSize = ZBUFFv07_limitCopy(op, oend-op, zbd->outBuff + zbd->outStart, toFlushSize); + op += flushedSize; + zbd->outStart += flushedSize; + if (flushedSize == toFlushSize) { + zbd->stage = ZBUFFds_read; + if (zbd->outStart + zbd->blockSize > zbd->outBuffSize) + zbd->outStart = zbd->outEnd = 0; + break; + } + /* cannot flush everything */ + notDone = 0; + break; + } + default: return ERROR(GENERIC); /* impossible */ + } } + + /* result */ + *srcSizePtr = ip-istart; + *dstCapacityPtr = op-ostart; + { size_t nextSrcSizeHint = ZSTDv07_nextSrcSizeToDecompress(zbd->zd); + nextSrcSizeHint -= zbd->inPos; /* already loaded*/ + return nextSrcSizeHint; + } +} + + + +/* ************************************* +* Tool functions +***************************************/ +size_t ZBUFFv07_recommendedDInSize(void) { return ZSTDv07_BLOCKSIZE_ABSOLUTEMAX + ZSTDv07_blockHeaderSize /* block header size*/ ; } +size_t ZBUFFv07_recommendedDOutSize(void) { return ZSTDv07_BLOCKSIZE_ABSOLUTEMAX; } diff --git a/contrib/libs/zstd06/legacy/zstd_v07.h b/contrib/libs/zstd06/legacy/zstd_v07.h index 40f70047b4..c1a1f9798f 100644 --- a/contrib/libs/zstd06/legacy/zstd_v07.h +++ b/contrib/libs/zstd06/legacy/zstd_v07.h @@ -1,208 +1,208 @@ -#include <contrib/libs/zstd06/renames.h> -/* - zstd - standard compression library - Header File - Copyright (C) 2014-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - zstd source repository : https://github.com/Cyan4973/zstd -*/ -#ifndef ZSTDv07_H_235446 -#define ZSTDv07_H_235446 - -#if defined (__cplusplus) -extern "C" { -#endif - -/*====== Dependency ======*/ -#include <stddef.h> /* size_t */ - - -/*====== Export for Windows ======*/ -/*! -* ZSTDv07_DLL_EXPORT : -* Enable exporting of functions when building a Windows DLL -*/ -#if defined(_WIN32) && defined(ZSTDv07_DLL_EXPORT) && (ZSTDv07_DLL_EXPORT==1) -# define ZSTDLIB_API __declspec(dllexport) -#else -# define ZSTDLIB_API -#endif - - - -/* ************************************* -* Simple API -***************************************/ -/*! ZSTDv07_getDecompressedSize() : -* @return : decompressed size if known, 0 otherwise. - note 1 : if `0`, follow up with ZSTDv07_getFrameParams() to know precise failure cause. - note 2 : decompressed size could be wrong or intentionally modified ! - always ensure results fit within application's authorized limits */ -unsigned long long ZSTDv07_getDecompressedSize(const void* src, size_t srcSize); - -/*! ZSTDv07_decompress() : - `compressedSize` : must be _exact_ size of compressed input, otherwise decompression will fail. - `dstCapacity` must be equal or larger than originalSize. - @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), - or an errorCode if it fails (which can be tested using ZSTDv07_isError()) */ -ZSTDLIB_API size_t ZSTDv07_decompress( void* dst, size_t dstCapacity, - const void* src, size_t compressedSize); - -/*====== Helper functions ======*/ -ZSTDLIB_API unsigned ZSTDv07_isError(size_t code); /*!< tells if a `size_t` function result is an error code */ -ZSTDLIB_API const char* ZSTDv07_getErrorName(size_t code); /*!< provides readable string from an error code */ - - -/*-************************************* -* Explicit memory management -***************************************/ -/** Decompression context */ -typedef struct ZSTDv07_DCtx_s ZSTDv07_DCtx; -ZSTDLIB_API ZSTDv07_DCtx* ZSTDv07_createDCtx(void); -ZSTDLIB_API size_t ZSTDv07_freeDCtx(ZSTDv07_DCtx* dctx); /*!< @return : errorCode */ - -/** ZSTDv07_decompressDCtx() : -* Same as ZSTDv07_decompress(), requires an allocated ZSTDv07_DCtx (see ZSTDv07_createDCtx()) */ -ZSTDLIB_API size_t ZSTDv07_decompressDCtx(ZSTDv07_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); - - -/*-************************ -* Simple dictionary API -***************************/ -/*! ZSTDv07_decompress_usingDict() : -* Decompression using a pre-defined Dictionary content (see dictBuilder). -* Dictionary must be identical to the one used during compression. -* Note : This function load the dictionary, resulting in a significant startup time */ -ZSTDLIB_API size_t ZSTDv07_decompress_usingDict(ZSTDv07_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const void* dict,size_t dictSize); - - -/*-************************** -* Advanced Dictionary API -****************************/ -/*! ZSTDv07_createDDict() : -* Create a digested dictionary, ready to start decompression operation without startup delay. -* `dict` can be released after creation */ -typedef struct ZSTDv07_DDict_s ZSTDv07_DDict; -ZSTDLIB_API ZSTDv07_DDict* ZSTDv07_createDDict(const void* dict, size_t dictSize); -ZSTDLIB_API size_t ZSTDv07_freeDDict(ZSTDv07_DDict* ddict); - -/*! ZSTDv07_decompress_usingDDict() : -* Decompression using a pre-digested Dictionary -* Faster startup than ZSTDv07_decompress_usingDict(), recommended when same dictionary is used multiple times. */ -ZSTDLIB_API size_t ZSTDv07_decompress_usingDDict(ZSTDv07_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const ZSTDv07_DDict* ddict); - -typedef struct { - unsigned long long frameContentSize; - unsigned windowSize; - unsigned dictID; - unsigned checksumFlag; -} ZSTDv07_frameParams; - -ZSTDLIB_API size_t ZSTDv07_getFrameParams(ZSTDv07_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */ - - - -/* *************************************************************** -* Compiler specifics -*****************************************************************/ -/* ZSTDv07_DLL_EXPORT : -* Enable exporting of functions when building a Windows DLL */ -#if defined(_WIN32) && defined(ZSTDv07_DLL_EXPORT) && (ZSTDv07_DLL_EXPORT==1) -# define ZSTDLIB_API __declspec(dllexport) -#else -# define ZSTDLIB_API -#endif - - -/* ************************************* -* Streaming functions -***************************************/ -typedef struct ZBUFFv07_DCtx_s ZBUFFv07_DCtx; -ZSTDLIB_API ZBUFFv07_DCtx* ZBUFFv07_createDCtx(void); -ZSTDLIB_API size_t ZBUFFv07_freeDCtx(ZBUFFv07_DCtx* dctx); - -ZSTDLIB_API size_t ZBUFFv07_decompressInit(ZBUFFv07_DCtx* dctx); -ZSTDLIB_API size_t ZBUFFv07_decompressInitDictionary(ZBUFFv07_DCtx* dctx, const void* dict, size_t dictSize); - -ZSTDLIB_API size_t ZBUFFv07_decompressContinue(ZBUFFv07_DCtx* dctx, - void* dst, size_t* dstCapacityPtr, - const void* src, size_t* srcSizePtr); - -/*-*************************************************************************** -* Streaming decompression howto -* -* A ZBUFFv07_DCtx object is required to track streaming operations. -* Use ZBUFFv07_createDCtx() and ZBUFFv07_freeDCtx() to create/release resources. -* Use ZBUFFv07_decompressInit() to start a new decompression operation, -* or ZBUFFv07_decompressInitDictionary() if decompression requires a dictionary. -* Note that ZBUFFv07_DCtx objects can be re-init multiple times. -* -* Use ZBUFFv07_decompressContinue() repetitively to consume your input. -* *srcSizePtr and *dstCapacityPtr can be any size. -* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr. -* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again. -* The content of `dst` will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change `dst`. -* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency), -* or 0 when a frame is completely decoded, -* or an error code, which can be tested using ZBUFFv07_isError(). -* -* Hint : recommended buffer sizes (not compulsory) : ZBUFFv07_recommendedDInSize() and ZBUFFv07_recommendedDOutSize() -* output : ZBUFFv07_recommendedDOutSize== 128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded. -* input : ZBUFFv07_recommendedDInSize == 128KB + 3; -* just follow indications from ZBUFFv07_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . -* *******************************************************************************/ - - -/* ************************************* -* Tool functions -***************************************/ -ZSTDLIB_API unsigned ZBUFFv07_isError(size_t errorCode); -ZSTDLIB_API const char* ZBUFFv07_getErrorName(size_t errorCode); - -/** Functions below provide recommended buffer sizes for Compression or Decompression operations. -* These sizes are just hints, they tend to offer better latency */ -ZSTDLIB_API size_t ZBUFFv07_recommendedDInSize(void); -ZSTDLIB_API size_t ZBUFFv07_recommendedDOutSize(void); - - -/*-************************************* -* Constants -***************************************/ -#define ZSTDv07_MAGICNUMBER 0xFD2FB527 /* v0.7 */ - - -#if defined (__cplusplus) -} -#endif - -#endif /* ZSTDv07_H_235446 */ +#include <contrib/libs/zstd06/renames.h> +/* + zstd - standard compression library + Header File + Copyright (C) 2014-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - zstd source repository : https://github.com/Cyan4973/zstd +*/ +#ifndef ZSTDv07_H_235446 +#define ZSTDv07_H_235446 + +#if defined (__cplusplus) +extern "C" { +#endif + +/*====== Dependency ======*/ +#include <stddef.h> /* size_t */ + + +/*====== Export for Windows ======*/ +/*! +* ZSTDv07_DLL_EXPORT : +* Enable exporting of functions when building a Windows DLL +*/ +#if defined(_WIN32) && defined(ZSTDv07_DLL_EXPORT) && (ZSTDv07_DLL_EXPORT==1) +# define ZSTDLIB_API __declspec(dllexport) +#else +# define ZSTDLIB_API +#endif + + + +/* ************************************* +* Simple API +***************************************/ +/*! ZSTDv07_getDecompressedSize() : +* @return : decompressed size if known, 0 otherwise. + note 1 : if `0`, follow up with ZSTDv07_getFrameParams() to know precise failure cause. + note 2 : decompressed size could be wrong or intentionally modified ! + always ensure results fit within application's authorized limits */ +unsigned long long ZSTDv07_getDecompressedSize(const void* src, size_t srcSize); + +/*! ZSTDv07_decompress() : + `compressedSize` : must be _exact_ size of compressed input, otherwise decompression will fail. + `dstCapacity` must be equal or larger than originalSize. + @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), + or an errorCode if it fails (which can be tested using ZSTDv07_isError()) */ +ZSTDLIB_API size_t ZSTDv07_decompress( void* dst, size_t dstCapacity, + const void* src, size_t compressedSize); + +/*====== Helper functions ======*/ +ZSTDLIB_API unsigned ZSTDv07_isError(size_t code); /*!< tells if a `size_t` function result is an error code */ +ZSTDLIB_API const char* ZSTDv07_getErrorName(size_t code); /*!< provides readable string from an error code */ + + +/*-************************************* +* Explicit memory management +***************************************/ +/** Decompression context */ +typedef struct ZSTDv07_DCtx_s ZSTDv07_DCtx; +ZSTDLIB_API ZSTDv07_DCtx* ZSTDv07_createDCtx(void); +ZSTDLIB_API size_t ZSTDv07_freeDCtx(ZSTDv07_DCtx* dctx); /*!< @return : errorCode */ + +/** ZSTDv07_decompressDCtx() : +* Same as ZSTDv07_decompress(), requires an allocated ZSTDv07_DCtx (see ZSTDv07_createDCtx()) */ +ZSTDLIB_API size_t ZSTDv07_decompressDCtx(ZSTDv07_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); + + +/*-************************ +* Simple dictionary API +***************************/ +/*! ZSTDv07_decompress_usingDict() : +* Decompression using a pre-defined Dictionary content (see dictBuilder). +* Dictionary must be identical to the one used during compression. +* Note : This function load the dictionary, resulting in a significant startup time */ +ZSTDLIB_API size_t ZSTDv07_decompress_usingDict(ZSTDv07_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize); + + +/*-************************** +* Advanced Dictionary API +****************************/ +/*! ZSTDv07_createDDict() : +* Create a digested dictionary, ready to start decompression operation without startup delay. +* `dict` can be released after creation */ +typedef struct ZSTDv07_DDict_s ZSTDv07_DDict; +ZSTDLIB_API ZSTDv07_DDict* ZSTDv07_createDDict(const void* dict, size_t dictSize); +ZSTDLIB_API size_t ZSTDv07_freeDDict(ZSTDv07_DDict* ddict); + +/*! ZSTDv07_decompress_usingDDict() : +* Decompression using a pre-digested Dictionary +* Faster startup than ZSTDv07_decompress_usingDict(), recommended when same dictionary is used multiple times. */ +ZSTDLIB_API size_t ZSTDv07_decompress_usingDDict(ZSTDv07_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTDv07_DDict* ddict); + +typedef struct { + unsigned long long frameContentSize; + unsigned windowSize; + unsigned dictID; + unsigned checksumFlag; +} ZSTDv07_frameParams; + +ZSTDLIB_API size_t ZSTDv07_getFrameParams(ZSTDv07_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */ + + + +/* *************************************************************** +* Compiler specifics +*****************************************************************/ +/* ZSTDv07_DLL_EXPORT : +* Enable exporting of functions when building a Windows DLL */ +#if defined(_WIN32) && defined(ZSTDv07_DLL_EXPORT) && (ZSTDv07_DLL_EXPORT==1) +# define ZSTDLIB_API __declspec(dllexport) +#else +# define ZSTDLIB_API +#endif + + +/* ************************************* +* Streaming functions +***************************************/ +typedef struct ZBUFFv07_DCtx_s ZBUFFv07_DCtx; +ZSTDLIB_API ZBUFFv07_DCtx* ZBUFFv07_createDCtx(void); +ZSTDLIB_API size_t ZBUFFv07_freeDCtx(ZBUFFv07_DCtx* dctx); + +ZSTDLIB_API size_t ZBUFFv07_decompressInit(ZBUFFv07_DCtx* dctx); +ZSTDLIB_API size_t ZBUFFv07_decompressInitDictionary(ZBUFFv07_DCtx* dctx, const void* dict, size_t dictSize); + +ZSTDLIB_API size_t ZBUFFv07_decompressContinue(ZBUFFv07_DCtx* dctx, + void* dst, size_t* dstCapacityPtr, + const void* src, size_t* srcSizePtr); + +/*-*************************************************************************** +* Streaming decompression howto +* +* A ZBUFFv07_DCtx object is required to track streaming operations. +* Use ZBUFFv07_createDCtx() and ZBUFFv07_freeDCtx() to create/release resources. +* Use ZBUFFv07_decompressInit() to start a new decompression operation, +* or ZBUFFv07_decompressInitDictionary() if decompression requires a dictionary. +* Note that ZBUFFv07_DCtx objects can be re-init multiple times. +* +* Use ZBUFFv07_decompressContinue() repetitively to consume your input. +* *srcSizePtr and *dstCapacityPtr can be any size. +* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr. +* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again. +* The content of `dst` will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change `dst`. +* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency), +* or 0 when a frame is completely decoded, +* or an error code, which can be tested using ZBUFFv07_isError(). +* +* Hint : recommended buffer sizes (not compulsory) : ZBUFFv07_recommendedDInSize() and ZBUFFv07_recommendedDOutSize() +* output : ZBUFFv07_recommendedDOutSize== 128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded. +* input : ZBUFFv07_recommendedDInSize == 128KB + 3; +* just follow indications from ZBUFFv07_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . +* *******************************************************************************/ + + +/* ************************************* +* Tool functions +***************************************/ +ZSTDLIB_API unsigned ZBUFFv07_isError(size_t errorCode); +ZSTDLIB_API const char* ZBUFFv07_getErrorName(size_t errorCode); + +/** Functions below provide recommended buffer sizes for Compression or Decompression operations. +* These sizes are just hints, they tend to offer better latency */ +ZSTDLIB_API size_t ZBUFFv07_recommendedDInSize(void); +ZSTDLIB_API size_t ZBUFFv07_recommendedDOutSize(void); + + +/*-************************************* +* Constants +***************************************/ +#define ZSTDv07_MAGICNUMBER 0xFD2FB527 /* v0.7 */ + + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTDv07_H_235446 */ diff --git a/contrib/libs/zstd06/legacy/zstd_v08.c b/contrib/libs/zstd06/legacy/zstd_v08.c index 10fafbb100..6814910eb0 100644 --- a/contrib/libs/zstd06/legacy/zstd_v08.c +++ b/contrib/libs/zstd06/legacy/zstd_v08.c @@ -1,4921 +1,4921 @@ -/* ****************************************************************** - zstd_v08.c - Decompression module for ZSTD v0.8 legacy format - Copyright (C) 2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Homepage : http://www.zstd.net/ -****************************************************************** */ - -/*- Dependencies -*/ -#include <stddef.h> /* size_t, ptrdiff_t */ -#include <string.h> /* memcpy */ -#include <stdlib.h> /* malloc, free, qsort */ - -#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ -#include "xxhash.h" /* XXH64_* */ -#include "zstd_v08.h" - -#define FSEv08_STATIC_LINKING_ONLY /* FSEv08_MIN_TABLELOG */ -#define HUFv08_STATIC_LINKING_ONLY /* HUFv08_TABLELOG_ABSOLUTEMAX */ -#define ZSTDv08_STATIC_LINKING_ONLY - - - -#ifdef ZSTDv08_STATIC_LINKING_ONLY - -/* ==================================================================================== - * The definitions in this section are considered experimental. - * They should never be used with a dynamic library, as they may change in the future. - * They are provided for advanced usages. - * Use them only in association with static linking. - * ==================================================================================== */ - -/*--- Constants ---*/ -#define ZSTDv08_MAGIC_SKIPPABLE_START 0x184D2A50U - -#define ZSTDv08_WINDOWLOG_MAX_32 25 -#define ZSTDv08_WINDOWLOG_MAX_64 27 -#define ZSTDv08_WINDOWLOG_MAX ((U32)(MEM_32bits() ? ZSTDv08_WINDOWLOG_MAX_32 : ZSTDv08_WINDOWLOG_MAX_64)) -#define ZSTDv08_WINDOWLOG_MIN 18 -#define ZSTDv08_CHAINLOG_MAX (ZSTDv08_WINDOWLOG_MAX+1) -#define ZSTDv08_CHAINLOG_MIN 4 -#define ZSTDv08_HASHLOG_MAX ZSTDv08_WINDOWLOG_MAX -#define ZSTDv08_HASHLOG_MIN 12 -#define ZSTDv08_HASHLOG3_MAX 17 -#define ZSTDv08_SEARCHLOG_MAX (ZSTDv08_WINDOWLOG_MAX-1) -#define ZSTDv08_SEARCHLOG_MIN 1 -#define ZSTDv08_SEARCHLENGTH_MAX 7 -#define ZSTDv08_SEARCHLENGTH_MIN 3 -#define ZSTDv08_TARGETLENGTH_MIN 4 -#define ZSTDv08_TARGETLENGTH_MAX 999 - -#define ZSTDv08_FRAMEHEADERSIZE_MAX 18 /* for static allocation */ -static const size_t ZSTDv08_frameHeaderSize_min = 5; -static const size_t ZSTDv08_frameHeaderSize_max = ZSTDv08_FRAMEHEADERSIZE_MAX; -static const size_t ZSTDv08_skippableHeaderSize = 8; /* magic number + skippable frame length */ - - -/* custom memory allocation functions */ -typedef void* (*ZSTDv08_allocFunction) (void* opaque, size_t size); -typedef void (*ZSTDv08_freeFunction) (void* opaque, void* address); -typedef struct { ZSTDv08_allocFunction customAlloc; ZSTDv08_freeFunction customFree; void* opaque; } ZSTDv08_customMem; - - -/*--- Advanced Decompression functions ---*/ - -/*! ZSTDv08_estimateDCtxSize() : - * Gives the potential amount of memory allocated to create a ZSTDv08_DCtx */ -ZSTDLIB_API size_t ZSTDv08_estimateDCtxSize(void); - -/*! ZSTDv08_createDCtx_advanced() : - * Create a ZSTD decompression context using external alloc and free functions */ -ZSTDLIB_API ZSTDv08_DCtx* ZSTDv08_createDCtx_advanced(ZSTDv08_customMem customMem); - -/*! ZSTDv08_sizeofDCtx() : - * Gives the amount of memory used by a given ZSTDv08_DCtx */ -ZSTDLIB_API size_t ZSTDv08_sizeofDCtx(const ZSTDv08_DCtx* dctx); - - -/* ****************************************************************** -* Buffer-less streaming functions (synchronous mode) -********************************************************************/ -/* This is an advanced API, giving full control over buffer management, for users which need direct control over memory. -* But it's also a complex one, with a lot of restrictions (documented below). -* For an easier streaming API, look into common/zbuff.h -* which removes all restrictions by allocating and managing its own internal buffer */ - -ZSTDLIB_API size_t ZSTDv08_decompressBegin(ZSTDv08_DCtx* dctx); -ZSTDLIB_API size_t ZSTDv08_decompressBegin_usingDict(ZSTDv08_DCtx* dctx, const void* dict, size_t dictSize); -ZSTDLIB_API void ZSTDv08_copyDCtx(ZSTDv08_DCtx* dctx, const ZSTDv08_DCtx* preparedDCtx); - -ZSTDLIB_API size_t ZSTDv08_nextSrcSizeToDecompress(ZSTDv08_DCtx* dctx); -ZSTDLIB_API size_t ZSTDv08_decompressContinue(ZSTDv08_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); - -typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTDv08_nextInputType_e; -ZSTDLIB_API ZSTDv08_nextInputType_e ZSTDv08_nextInputType(ZSTDv08_DCtx* dctx); - -/* - Buffer-less streaming decompression (synchronous mode) - - A ZSTDv08_DCtx object is required to track streaming operations. - Use ZSTDv08_createDCtx() / ZSTDv08_freeDCtx() to manage it. - A ZSTDv08_DCtx object can be re-used multiple times. - - First typical operation is to retrieve frame parameters, using ZSTDv08_getFrameParams(). - It fills a ZSTDv08_frameParams structure which provide important information to correctly decode the frame, - such as the minimum rolling buffer size to allocate to decompress data (`windowSize`), - and the dictionary ID used. - (Note : content size is optional, it may not be present. 0 means : content size unknown). - Note that these values could be wrong, either because of data malformation, or because an attacker is spoofing deliberate false information. - As a consequence, check that values remain within valid application range, especially `windowSize`, before allocation. - Each application can set its own limit, depending on local restrictions. For extended interoperability, it is recommended to support at least 8 MB. - Frame parameters are extracted from the beginning of the compressed frame. - Data fragment must be large enough to ensure successful decoding, typically `ZSTDv08_frameHeaderSize_max` bytes. - @result : 0 : successful decoding, the `ZSTDv08_frameParams` structure is correctly filled. - >0 : `srcSize` is too small, please provide at least @result bytes on next attempt. - errorCode, which can be tested using ZSTDv08_isError(). - - Start decompression, with ZSTDv08_decompressBegin() or ZSTDv08_decompressBegin_usingDict(). - Alternatively, you can copy a prepared context, using ZSTDv08_copyDCtx(). - - Then use ZSTDv08_nextSrcSizeToDecompress() and ZSTDv08_decompressContinue() alternatively. - ZSTDv08_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTDv08_decompressContinue(). - ZSTDv08_decompressContinue() requires this _exact_ amount of bytes, or it will fail. - - @result of ZSTDv08_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity). - It can be zero, which is not an error; it just means ZSTDv08_decompressContinue() has decoded some metadata item. - It can also be an error code, which can be tested with ZSTDv08_isError(). - - ZSTDv08_decompressContinue() needs previous data blocks during decompression, up to `windowSize`. - They should preferably be located contiguously, prior to current block. - Alternatively, a round buffer of sufficient size is also possible. Sufficient size is determined by frame parameters. - ZSTDv08_decompressContinue() is very sensitive to contiguity, - if 2 blocks don't follow each other, make sure that either the compressor breaks contiguity at the same place, - or that previous contiguous segment is large enough to properly handle maximum back-reference. - - A frame is fully decoded when ZSTDv08_nextSrcSizeToDecompress() returns zero. - Context can then be reset to start a new decompression. - - Note : it's possible to know if next input to present is a header or a block, using ZSTDv08_nextInputType(). - This information is not required to properly decode a frame. - - == Special case : skippable frames == - - Skippable frames allow integration of user-defined data into a flow of concatenated frames. - Skippable frames will be ignored (skipped) by a decompressor. The format of skippable frames is as follows : - a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F - b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits - c) Frame Content - any content (User Data) of length equal to Frame Size - For skippable frames ZSTDv08_decompressContinue() always returns 0. - For skippable frames ZSTDv08_getFrameParams() returns fparamsPtr->windowLog==0 what means that a frame is skippable. - It also returns Frame Size as fparamsPtr->frameContentSize. -*/ - - -/* ************************************** -* Block functions -****************************************/ -/*! Block functions produce and decode raw zstd blocks, without frame metadata. - Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes). - User will have to take in charge required information to regenerate data, such as compressed and content sizes. - - A few rules to respect : - - Compressing and decompressing require a context structure - + Use ZSTDv08_createCCtx() and ZSTDv08_createDCtx() - - It is necessary to init context before starting - + compression : ZSTDv08_compressBegin() - + decompression : ZSTDv08_decompressBegin() - + variants _usingDict() are also allowed - + copyCCtx() and copyDCtx() work too - - Block size is limited, it must be <= ZSTDv08_getBlockSizeMax() - + If you need to compress more, cut data into multiple blocks - + Consider using the regular ZSTDv08_compress() instead, as frame metadata costs become negligible when source size is large. - - When a block is considered not compressible enough, ZSTDv08_compressBlock() result will be zero. - In which case, nothing is produced into `dst`. - + User must test for such outcome and deal directly with uncompressed data - + ZSTDv08_decompressBlock() doesn't accept uncompressed data as input !!! - + In case of multiple successive blocks, decoder must be informed of uncompressed block existence to follow proper history. - Use ZSTDv08_insertBlock() in such a case. -*/ - -#define ZSTDv08_BLOCKSIZE_ABSOLUTEMAX (128 * 1024) /* define, for static allocation */ -ZSTDLIB_API size_t ZSTDv08_decompressBlock(ZSTDv08_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); -ZSTDLIB_API size_t ZSTDv08_insertBlock(ZSTDv08_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert block into `dctx` history. Useful for uncompressed blocks */ - - -#endif /* ZSTDv08_STATIC_LINKING_ONLY */ - - -/* ==================================================================================== - * The definitions in this section are considered experimental. - * They should never be used in association with a dynamic library, as they may change in the future. - * They are provided for advanced usages. - * Use them only in association with static linking. - * ==================================================================================== */ - -/*! ZBUFFv08_createDCtx_advanced() : - * Create a ZBUFF decompression context using external alloc and free functions */ -ZSTDLIB_API ZBUFFv08_DCtx* ZBUFFv08_createDCtx_advanced(ZSTDv08_customMem customMem); - - - -/* ****************************************************************** - mem.h - low-level memory access routines - Copyright (C) 2013-2015, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy - - Public forum : https://groups.google.com/forum/#!forum/lz4c -****************************************************************** */ -#ifndef MEM_H_MODULE -#define MEM_H_MODULE - -#if defined (__cplusplus) -extern "C" { -#endif - - - -/*-**************************************** -* Compiler specifics -******************************************/ -#if defined(_MSC_VER) /* Visual Studio */ -# include <stdlib.h> /* _byteswap_ulong */ -# include <intrin.h> /* _byteswap_* */ -#endif -#if defined(__GNUC__) -# define MEM_STATIC static __inline __attribute__((unused)) -#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -# define MEM_STATIC static inline -#elif defined(_MSC_VER) -# define MEM_STATIC static __inline -#else -# define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ -#endif - -/* code only tested on 32 and 64 bits systems */ -#define MEM_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; } -MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); } - - -/*-************************************************************** -* Basic Types -*****************************************************************/ -#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) -# include <stdint.h> - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef int16_t S16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; - typedef int64_t S64; -#else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef signed short S16; - typedef unsigned int U32; - typedef signed int S32; - typedef unsigned long long U64; - typedef signed long long S64; -#endif - - -/*-************************************************************** -* Memory I/O -*****************************************************************/ -/* MEM_FORCE_MEMORY_ACCESS : - * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. - * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. - * The below switch allow to select different access method for improved performance. - * Method 0 (default) : use `memcpy()`. Safe and portable. - * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). - * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. - * Method 2 : direct access. This method is portable but violate C standard. - * It can generate buggy code on targets depending on alignment. - * In some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) - * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. - * Prefer these methods in priority order (0 > 1 > 2) - */ -#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ -# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) -# define MEM_FORCE_MEMORY_ACCESS 2 -# elif defined(__INTEL_COMPILER) || \ - (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) -# define MEM_FORCE_MEMORY_ACCESS 1 -# endif -#endif - -MEM_STATIC unsigned MEM_32bits(void) { return sizeof(size_t)==4; } -MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; } - -MEM_STATIC unsigned MEM_isLittleEndian(void) -{ - const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ - return one.c[0]; -} - -#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2) - -/* violates C standard, by lying on structure alignment. -Only use if no other choice to achieve best performance on target platform */ -MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; } -MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; } -MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; } -MEM_STATIC U64 MEM_readST(const void* memPtr) { return *(const size_t*) memPtr; } - -MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; } -MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; } -MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; } - -#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1) - -/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ -/* currently only defined for gcc and icc */ -typedef union { U16 u16; U32 u32; U64 u64; size_t st; } __attribute__((packed)) unalign; - -MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; } -MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } -MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; } -MEM_STATIC U64 MEM_readST(const void* ptr) { return ((const unalign*)ptr)->st; } - -MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; } -MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; } -MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign*)memPtr)->u64 = value; } - -#else - -/* default method, safe and standard. - can sometimes prove slower */ - -MEM_STATIC U16 MEM_read16(const void* memPtr) -{ - U16 val; memcpy(&val, memPtr, sizeof(val)); return val; -} - -MEM_STATIC U32 MEM_read32(const void* memPtr) -{ - U32 val; memcpy(&val, memPtr, sizeof(val)); return val; -} - -MEM_STATIC U64 MEM_read64(const void* memPtr) -{ - U64 val; memcpy(&val, memPtr, sizeof(val)); return val; -} - -MEM_STATIC size_t MEM_readST(const void* memPtr) -{ - size_t val; memcpy(&val, memPtr, sizeof(val)); return val; -} - -MEM_STATIC void MEM_write16(void* memPtr, U16 value) -{ - memcpy(memPtr, &value, sizeof(value)); -} - -MEM_STATIC void MEM_write32(void* memPtr, U32 value) -{ - memcpy(memPtr, &value, sizeof(value)); -} - -MEM_STATIC void MEM_write64(void* memPtr, U64 value) -{ - memcpy(memPtr, &value, sizeof(value)); -} - -#endif /* MEM_FORCE_MEMORY_ACCESS */ - -MEM_STATIC U32 MEM_swap32(U32 in) -{ -#if defined(_MSC_VER) /* Visual Studio */ - return _byteswap_ulong(in); -#elif defined (__GNUC__) - return __builtin_bswap32(in); -#else - return ((in << 24) & 0xff000000 ) | - ((in << 8) & 0x00ff0000 ) | - ((in >> 8) & 0x0000ff00 ) | - ((in >> 24) & 0x000000ff ); -#endif -} - -MEM_STATIC U64 MEM_swap64(U64 in) -{ -#if defined(_MSC_VER) /* Visual Studio */ - return _byteswap_uint64(in); -#elif defined (__GNUC__) - return __builtin_bswap64(in); -#else - return ((in << 56) & 0xff00000000000000ULL) | - ((in << 40) & 0x00ff000000000000ULL) | - ((in << 24) & 0x0000ff0000000000ULL) | - ((in << 8) & 0x000000ff00000000ULL) | - ((in >> 8) & 0x00000000ff000000ULL) | - ((in >> 24) & 0x0000000000ff0000ULL) | - ((in >> 40) & 0x000000000000ff00ULL) | - ((in >> 56) & 0x00000000000000ffULL); -#endif -} - -MEM_STATIC size_t MEM_swapST(size_t in) -{ - if (MEM_32bits()) - return (size_t)MEM_swap32((U32)in); - else - return (size_t)MEM_swap64((U64)in); -} - -/*=== Little endian r/w ===*/ - -MEM_STATIC U16 MEM_readLE16(const void* memPtr) -{ - if (MEM_isLittleEndian()) - return MEM_read16(memPtr); - else { - const BYTE* p = (const BYTE*)memPtr; - return (U16)(p[0] + (p[1]<<8)); - } -} - -MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val) -{ - if (MEM_isLittleEndian()) { - MEM_write16(memPtr, val); - } else { - BYTE* p = (BYTE*)memPtr; - p[0] = (BYTE)val; - p[1] = (BYTE)(val>>8); - } -} - -MEM_STATIC U32 MEM_readLE24(const void* memPtr) -{ - return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16); -} - -MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val) -{ - MEM_writeLE16(memPtr, (U16)val); - ((BYTE*)memPtr)[2] = (BYTE)(val>>16); -} - -MEM_STATIC U32 MEM_readLE32(const void* memPtr) -{ - if (MEM_isLittleEndian()) - return MEM_read32(memPtr); - else - return MEM_swap32(MEM_read32(memPtr)); -} - -MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32) -{ - if (MEM_isLittleEndian()) - MEM_write32(memPtr, val32); - else - MEM_write32(memPtr, MEM_swap32(val32)); -} - -MEM_STATIC U64 MEM_readLE64(const void* memPtr) -{ - if (MEM_isLittleEndian()) - return MEM_read64(memPtr); - else - return MEM_swap64(MEM_read64(memPtr)); -} - -MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64) -{ - if (MEM_isLittleEndian()) - MEM_write64(memPtr, val64); - else - MEM_write64(memPtr, MEM_swap64(val64)); -} - -MEM_STATIC size_t MEM_readLEST(const void* memPtr) -{ - if (MEM_32bits()) - return (size_t)MEM_readLE32(memPtr); - else - return (size_t)MEM_readLE64(memPtr); -} - -MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val) -{ - if (MEM_32bits()) - MEM_writeLE32(memPtr, (U32)val); - else - MEM_writeLE64(memPtr, (U64)val); -} - -/*=== Big endian r/w ===*/ - -MEM_STATIC U32 MEM_readBE32(const void* memPtr) -{ - if (MEM_isLittleEndian()) - return MEM_swap32(MEM_read32(memPtr)); - else - return MEM_read32(memPtr); -} - -MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32) -{ - if (MEM_isLittleEndian()) - MEM_write32(memPtr, MEM_swap32(val32)); - else - MEM_write32(memPtr, val32); -} - -MEM_STATIC U64 MEM_readBE64(const void* memPtr) -{ - if (MEM_isLittleEndian()) - return MEM_swap64(MEM_read64(memPtr)); - else - return MEM_read64(memPtr); -} - -MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64) -{ - if (MEM_isLittleEndian()) - MEM_write64(memPtr, MEM_swap64(val64)); - else - MEM_write64(memPtr, val64); -} - -MEM_STATIC size_t MEM_readBEST(const void* memPtr) -{ - if (MEM_32bits()) - return (size_t)MEM_readBE32(memPtr); - else - return (size_t)MEM_readBE64(memPtr); -} - -MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val) -{ - if (MEM_32bits()) - MEM_writeBE32(memPtr, (U32)val); - else - MEM_writeBE64(memPtr, (U64)val); -} - - -/* function safe only for comparisons */ -MEM_STATIC U32 MEM_readMINMATCH(const void* memPtr, U32 length) -{ - switch (length) - { - default : - case 4 : return MEM_read32(memPtr); - case 3 : if (MEM_isLittleEndian()) - return MEM_read32(memPtr)<<8; - else - return MEM_read32(memPtr)>>8; - } -} - -#if defined (__cplusplus) -} -#endif - -#endif /* MEM_H_MODULE */ -/* ****************************************************************** - Error codes list - Copyright (C) 2016, Yann Collet - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Homepage : http://www.zstd.net -****************************************************************** */ -#ifndef ERROR_PUBLIC_H_MODULE -#define ERROR_PUBLIC_H_MODULE - -#if defined (__cplusplus) -extern "C" { -#endif - - -/* **************************************** -* error codes list -******************************************/ -typedef enum { - ZSTDv08_error_no_error, - ZSTDv08_error_GENERIC, - ZSTDv08_error_prefix_unknown, - ZSTDv08_error_frameParameter_unsupported, - ZSTDv08_error_frameParameter_unsupportedBy32bits, - ZSTDv08_error_compressionParameter_unsupported, - ZSTDv08_error_init_missing, - ZSTDv08_error_memory_allocation, - ZSTDv08_error_stage_wrong, - ZSTDv08_error_dstSize_tooSmall, - ZSTDv08_error_srcSize_wrong, - ZSTDv08_error_corruption_detected, - ZSTDv08_error_checksum_wrong, - ZSTDv08_error_tableLog_tooLarge, - ZSTDv08_error_maxSymbolValue_tooLarge, - ZSTDv08_error_maxSymbolValue_tooSmall, - ZSTDv08_error_dictionary_corrupted, - ZSTDv08_error_dictionary_wrong, - ZSTDv08_error_maxCode -} ZSTDv08_ErrorCode; - -/*! ZSTDv08_getErrorCode() : - convert a `size_t` function result into a `ZSTDv08_ErrorCode` enum type, - which can be used to compare directly with enum list published into "error_public.h" */ -ZSTDv08_ErrorCode ZSTDv08_getErrorCode(size_t functionResult); -const char* ZSTDv08_getErrorString(ZSTDv08_ErrorCode code); - - -#if defined (__cplusplus) -} -#endif - -#endif /* ERROR_PUBLIC_H_MODULE */ -/* ****************************************************************** - Error codes and messages - Copyright (C) 2013-2016, Yann Collet - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Homepage : http://www.zstd.net -****************************************************************** */ -/* Note : this module is expected to remain private, do not expose it */ - -#ifndef ERROR_H_MODULE -#define ERROR_H_MODULE - -#if defined (__cplusplus) -extern "C" { -#endif - - - -/* **************************************** -* Compiler-specific -******************************************/ -#if defined(__GNUC__) -# define ERR_STATIC static __attribute__((unused)) -#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -# define ERR_STATIC static inline -#elif defined(_MSC_VER) -# define ERR_STATIC static __inline -#else -# define ERR_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ -#endif - - -/*-**************************************** -* Customization (error_public.h) -******************************************/ -typedef ZSTDv08_ErrorCode ERR_enum; -#define PREFIX(name) ZSTDv08_error_##name - - -/*-**************************************** -* Error codes handling -******************************************/ -#ifdef ERROR -# undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */ -#endif -#define ERROR(name) ((size_t)-PREFIX(name)) - -ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); } - -ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); } - - -/*-**************************************** -* Error Strings -******************************************/ - -ERR_STATIC const char* ERR_getErrorString(ERR_enum code) -{ - static const char* notErrorCode = "Unspecified error code"; - switch( code ) - { - case PREFIX(no_error): return "No error detected"; - case PREFIX(GENERIC): return "Error (generic)"; - case PREFIX(prefix_unknown): return "Unknown frame descriptor"; - case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter"; - case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode"; - case PREFIX(compressionParameter_unsupported): return "Compression parameter is out of bound"; - case PREFIX(init_missing): return "Context should be init first"; - case PREFIX(memory_allocation): return "Allocation error : not enough memory"; - case PREFIX(stage_wrong): return "Operation not authorized at current processing stage"; - case PREFIX(dstSize_tooSmall): return "Destination buffer is too small"; - case PREFIX(srcSize_wrong): return "Src size incorrect"; - case PREFIX(corruption_detected): return "Corrupted block detected"; - case PREFIX(checksum_wrong): return "Restored data doesn't match checksum"; - case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported"; - case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large"; - case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small"; - case PREFIX(dictionary_corrupted): return "Dictionary is corrupted"; - case PREFIX(dictionary_wrong): return "Dictionary mismatch"; - case PREFIX(maxCode): - default: return notErrorCode; - } -} - -ERR_STATIC const char* ERR_getErrorName(size_t code) -{ - return ERR_getErrorString(ERR_getErrorCode(code)); -} - -#if defined (__cplusplus) -} -#endif - -#endif /* ERROR_H_MODULE */ -/* ****************************************************************** - bitstream - Part of FSE library - header file (to include) - Copyright (C) 2013-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Source repository : https://github.com/Cyan4973/FiniteStateEntropy -****************************************************************** */ -#ifndef BITSTREAM_H_MODULE -#define BITSTREAM_H_MODULE - -#if defined (__cplusplus) -extern "C" { -#endif - - -/* -* This API consists of small unitary functions, which must be inlined for best performance. -* Since link-time-optimization is not available for all compilers, -* these functions are defined into a .h to be included. -*/ - - -/*========================================= -* Target specific -=========================================*/ -#if defined(__BMI__) && defined(__GNUC__) -# include <immintrin.h> /* support for bextr (experimental) */ -#endif - - -/*-******************************************** -* bitStream decoding API (read backward) -**********************************************/ -typedef struct -{ - size_t bitContainer; - unsigned bitsConsumed; - const char* ptr; - const char* start; -} BIT_DStream_t; - -typedef enum { BIT_DStream_unfinished = 0, - BIT_DStream_endOfBuffer = 1, - BIT_DStream_completed = 2, - BIT_DStream_overflow = 3 } BIT_DStream_status; /* result of BIT_reloadDStream() */ - /* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */ - -MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize); -MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits); -MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD); -MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD); - - -/* Start by invoking BIT_initDStream(). -* A chunk of the bitStream is then stored into a local register. -* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t). -* You can then retrieve bitFields stored into the local register, **in reverse order**. -* Local register is explicitly reloaded from memory by the BIT_reloadDStream() method. -* A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished. -* Otherwise, it can be less than that, so proceed accordingly. -* Checking if DStream has reached its end can be performed with BIT_endOfDStream(). -*/ - - -/*-**************************************** -* unsafe API -******************************************/ -MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits); -/* faster, but works only if nbBits >= 1 */ - - - -/*-************************************************************** -* Internal functions -****************************************************************/ -MEM_STATIC unsigned BIT_highbit32 (register U32 val) -{ -# if defined(_MSC_VER) /* Visual */ - unsigned long r=0; - _BitScanReverse ( &r, val ); - return (unsigned) r; -# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */ - return 31 - __builtin_clz (val); -# else /* Software version */ - static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 }; - U32 v = val; - v |= v >> 1; - v |= v >> 2; - v |= v >> 4; - v |= v >> 8; - v |= v >> 16; - return DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27]; -# endif -} - -/*===== Local Constants =====*/ -static const unsigned BIT_mask[] = { 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF, 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF }; /* up to 26 bits */ - - -/*-******************************************************** -* bitStream decoding -**********************************************************/ -/*! BIT_initDStream() : -* Initialize a BIT_DStream_t. -* `bitD` : a pointer to an already allocated BIT_DStream_t structure. -* `srcSize` must be the *exact* size of the bitStream, in bytes. -* @return : size of stream (== srcSize) or an errorCode if a problem is detected -*/ -MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize) -{ - if (srcSize < 1) { memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); } - - if (srcSize >= sizeof(bitD->bitContainer)) { /* normal case */ - bitD->start = (const char*)srcBuffer; - bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer); - bitD->bitContainer = MEM_readLEST(bitD->ptr); - { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; - bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; - if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ } - } else { - bitD->start = (const char*)srcBuffer; - bitD->ptr = bitD->start; - bitD->bitContainer = *(const BYTE*)(bitD->start); - switch(srcSize) - { - case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16); - case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24); - case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32); - case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24; - case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16; - case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8; - default:; - } - { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; - bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; - if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ } - bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize)*8; - } - - return srcSize; -} - -MEM_STATIC size_t BIT_getUpperBits(size_t bitContainer, U32 const start) -{ - return bitContainer >> start; -} - -MEM_STATIC size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits) -{ -#if defined(__BMI__) && defined(__GNUC__) /* experimental */ -# if defined(__x86_64__) - if (sizeof(bitContainer)==8) - return _bextr_u64(bitContainer, start, nbBits); - else -# endif - return _bextr_u32(bitContainer, start, nbBits); -#else - return (bitContainer >> start) & BIT_mask[nbBits]; -#endif -} - -MEM_STATIC size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits) -{ - return bitContainer & BIT_mask[nbBits]; -} - -/*! BIT_lookBits() : - * Provides next n bits from local register. - * local register is not modified. - * On 32-bits, maxNbBits==24. - * On 64-bits, maxNbBits==56. - * @return : value extracted - */ - MEM_STATIC size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits) -{ -#if defined(__BMI__) && defined(__GNUC__) /* experimental; fails if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8 */ - return BIT_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits); -#else - U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1; - return ((bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> 1) >> ((bitMask-nbBits) & bitMask); -#endif -} - -/*! BIT_lookBitsFast() : -* unsafe version; only works only if nbBits >= 1 */ -MEM_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits) -{ - U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1; - return (bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> (((bitMask+1)-nbBits) & bitMask); -} - -MEM_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits) -{ - bitD->bitsConsumed += nbBits; -} - -/*! BIT_readBits() : - * Read (consume) next n bits from local register and update. - * Pay attention to not read more than nbBits contained into local register. - * @return : extracted value. - */ -MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits) -{ - size_t const value = BIT_lookBits(bitD, nbBits); - BIT_skipBits(bitD, nbBits); - return value; -} - -/*! BIT_readBitsFast() : -* unsafe version; only works only if nbBits >= 1 */ -MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits) -{ - size_t const value = BIT_lookBitsFast(bitD, nbBits); - BIT_skipBits(bitD, nbBits); - return value; -} - -/*! BIT_reloadDStream() : -* Refill `BIT_DStream_t` from src buffer previously defined (see BIT_initDStream() ). -* This function is safe, it guarantees it will not read beyond src buffer. -* @return : status of `BIT_DStream_t` internal register. - if status == unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */ -MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD) -{ - if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should not happen => corruption detected */ - return BIT_DStream_overflow; - - if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer)) { - bitD->ptr -= bitD->bitsConsumed >> 3; - bitD->bitsConsumed &= 7; - bitD->bitContainer = MEM_readLEST(bitD->ptr); - return BIT_DStream_unfinished; - } - if (bitD->ptr == bitD->start) { - if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return BIT_DStream_endOfBuffer; - return BIT_DStream_completed; - } - { U32 nbBytes = bitD->bitsConsumed >> 3; - BIT_DStream_status result = BIT_DStream_unfinished; - if (bitD->ptr - nbBytes < bitD->start) { - nbBytes = (U32)(bitD->ptr - bitD->start); /* ptr > start */ - result = BIT_DStream_endOfBuffer; - } - bitD->ptr -= nbBytes; - bitD->bitsConsumed -= nbBytes*8; - bitD->bitContainer = MEM_readLEST(bitD->ptr); /* reminder : srcSize > sizeof(bitD) */ - return result; - } -} - -/*! BIT_endOfDStream() : -* @return Tells if DStream has exactly reached its end (all bits consumed). -*/ -MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream) -{ - return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer)*8)); -} - -#if defined (__cplusplus) -} -#endif - -#endif /* BITSTREAM_H_MODULE */ -/* ****************************************************************** - FSE : Finite State Entropy codec - Public Prototypes declaration - Copyright (C) 2013-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Source repository : https://github.com/Cyan4973/FiniteStateEntropy -****************************************************************** */ -#ifndef FSEv08_H -#define FSEv08_H - -#if defined (__cplusplus) -extern "C" { -#endif - - - -/*-**************************************** -* FSE simple functions -******************************************/ -/*! FSEv08_decompress(): - Decompress FSE data from buffer 'cSrc', of size 'cSrcSize', - into already allocated destination buffer 'dst', of size 'dstCapacity'. - @return : size of regenerated data (<= maxDstSize), - or an error code, which can be tested using FSEv08_isError() . - - ** Important ** : FSEv08_decompress() does not decompress non-compressible nor RLE data !!! - Why ? : making this distinction requires a header. - Header management is intentionally delegated to the user layer, which can better manage special cases. -*/ -size_t FSEv08_decompress(void* dst, size_t dstCapacity, - const void* cSrc, size_t cSrcSize); - - -/*-***************************************** -* Tool functions -******************************************/ -/* Error Management */ -unsigned FSEv08_isError(size_t code); /* tells if a return value is an error code */ -const char* FSEv08_getErrorName(size_t code); /* provides error code string (useful for debugging) */ - - -/*-***************************************** -* FSE detailed API -******************************************/ -/*! -FSEv08_decompress() does the following: -1. read normalized counters with readNCount() -2. build decoding table 'DTable' from normalized counters -3. decode the data stream using decoding table 'DTable' - -The following API allows targeting specific sub-functions for advanced tasks. -For example, it's possible to compress several blocks using the same 'CTable', -or to save and provide normalized distribution using external method. -*/ - - -/* *** DECOMPRESSION *** */ - -/*! FSEv08_readNCount(): - Read compactly saved 'normalizedCounter' from 'rBuffer'. - @return : size read from 'rBuffer', - or an errorCode, which can be tested using FSEv08_isError(). - maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */ -size_t FSEv08_readNCount (short* normalizedCounter, unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, const void* rBuffer, size_t rBuffSize); - -/*! Constructor and Destructor of FSEv08_DTable. - Note that its size depends on 'tableLog' */ -typedef unsigned FSEv08_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */ -FSEv08_DTable* FSEv08_createDTable(unsigned tableLog); -void FSEv08_freeDTable(FSEv08_DTable* dt); - -/*! FSEv08_buildDTable(): - Builds 'dt', which must be already allocated, using FSEv08_createDTable(). - return : 0, or an errorCode, which can be tested using FSEv08_isError() */ -size_t FSEv08_buildDTable (FSEv08_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); - -/*! FSEv08_decompress_usingDTable(): - Decompress compressed source `cSrc` of size `cSrcSize` using `dt` - into `dst` which must be already allocated. - @return : size of regenerated data (necessarily <= `dstCapacity`), - or an errorCode, which can be tested using FSEv08_isError() */ -size_t FSEv08_decompress_usingDTable(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, const FSEv08_DTable* dt); - -/*! -Tutorial : ----------- -(Note : these functions only decompress FSE-compressed blocks. - If block is uncompressed, use memcpy() instead - If block is a single repeated byte, use memset() instead ) - -The first step is to obtain the normalized frequencies of symbols. -This can be performed by FSEv08_readNCount() if it was saved using FSEv08_writeNCount(). -'normalizedCounter' must be already allocated, and have at least 'maxSymbolValuePtr[0]+1' cells of signed short. -In practice, that means it's necessary to know 'maxSymbolValue' beforehand, -or size the table to handle worst case situations (typically 256). -FSEv08_readNCount() will provide 'tableLog' and 'maxSymbolValue'. -The result of FSEv08_readNCount() is the number of bytes read from 'rBuffer'. -Note that 'rBufferSize' must be at least 4 bytes, even if useful information is less than that. -If there is an error, the function will return an error code, which can be tested using FSEv08_isError(). - -The next step is to build the decompression tables 'FSEv08_DTable' from 'normalizedCounter'. -This is performed by the function FSEv08_buildDTable(). -The space required by 'FSEv08_DTable' must be already allocated using FSEv08_createDTable(). -If there is an error, the function will return an error code, which can be tested using FSEv08_isError(). - -`FSEv08_DTable` can then be used to decompress `cSrc`, with FSEv08_decompress_usingDTable(). -`cSrcSize` must be strictly correct, otherwise decompression will fail. -FSEv08_decompress_usingDTable() result will tell how many bytes were regenerated (<=`dstCapacity`). -If there is an error, the function will return an error code, which can be tested using FSEv08_isError(). (ex: dst buffer too small) -*/ - - -#ifdef FSEv08_STATIC_LINKING_ONLY - - -/* ***************************************** -* Static allocation -*******************************************/ -/* FSE buffer bounds */ -#define FSEv08_NCOUNTBOUND 512 -#define FSEv08_BLOCKBOUND(size) (size + (size>>7)) - -/* It is possible to statically allocate FSE CTable/DTable as a table of unsigned using below macros */ -#define FSEv08_DTABLE_SIZE_U32(maxTableLog) (1 + (1<<maxTableLog)) - - -/* ***************************************** -* FSE advanced API -*******************************************/ -size_t FSEv08_countFast(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize); -/**< same as FSEv08_count(), but blindly trusts that all byte values within src are <= *maxSymbolValuePtr */ - -unsigned FSEv08_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus); -/**< same as FSEv08_optimalTableLog(), which used `minus==2` */ - -size_t FSEv08_buildDTable_raw (FSEv08_DTable* dt, unsigned nbBits); -/**< build a fake FSEv08_DTable, designed to read an uncompressed bitstream where each symbol uses nbBits */ - -size_t FSEv08_buildDTable_rle (FSEv08_DTable* dt, unsigned char symbolValue); -/**< build a fake FSEv08_DTable, designed to always generate the same symbolValue */ - - - -/* ***************************************** -* FSE symbol decompression API -*******************************************/ -typedef struct -{ - size_t state; - const void* table; /* precise table may vary, depending on U16 */ -} FSEv08_DState_t; - - -static void FSEv08_initDState(FSEv08_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSEv08_DTable* dt); - -static unsigned char FSEv08_decodeSymbol(FSEv08_DState_t* DStatePtr, BIT_DStream_t* bitD); - -static unsigned FSEv08_endOfDState(const FSEv08_DState_t* DStatePtr); - -/**< -Let's now decompose FSEv08_decompress_usingDTable() into its unitary components. -You will decode FSE-encoded symbols from the bitStream, -and also any other bitFields you put in, **in reverse order**. - -You will need a few variables to track your bitStream. They are : - -BIT_DStream_t DStream; // Stream context -FSEv08_DState_t DState; // State context. Multiple ones are possible -FSEv08_DTable* DTablePtr; // Decoding table, provided by FSEv08_buildDTable() - -The first thing to do is to init the bitStream. - errorCode = BIT_initDStream(&DStream, srcBuffer, srcSize); - -You should then retrieve your initial state(s) -(in reverse flushing order if you have several ones) : - errorCode = FSEv08_initDState(&DState, &DStream, DTablePtr); - -You can then decode your data, symbol after symbol. -For information the maximum number of bits read by FSEv08_decodeSymbol() is 'tableLog'. -Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out). - unsigned char symbol = FSEv08_decodeSymbol(&DState, &DStream); - -You can retrieve any bitfield you eventually stored into the bitStream (in reverse order) -Note : maximum allowed nbBits is 25, for 32-bits compatibility - size_t bitField = BIT_readBits(&DStream, nbBits); - -All above operations only read from local register (which size depends on size_t). -Refueling the register from memory is manually performed by the reload method. - endSignal = FSEv08_reloadDStream(&DStream); - -BIT_reloadDStream() result tells if there is still some more data to read from DStream. -BIT_DStream_unfinished : there is still some data left into the DStream. -BIT_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled. -BIT_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed. -BIT_DStream_tooFar : Dstream went too far. Decompression result is corrupted. - -When reaching end of buffer (BIT_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop, -to properly detect the exact end of stream. -After each decoded symbol, check if DStream is fully consumed using this simple test : - BIT_reloadDStream(&DStream) >= BIT_DStream_completed - -When it's done, verify decompression is fully completed, by checking both DStream and the relevant states. -Checking if DStream has reached its end is performed by : - BIT_endOfDStream(&DStream); -Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible. - FSEv08_endOfDState(&DState); -*/ - - -/* ***************************************** -* FSE unsafe API -*******************************************/ -static unsigned char FSEv08_decodeSymbolFast(FSEv08_DState_t* DStatePtr, BIT_DStream_t* bitD); -/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */ - - -/* ***************************************** -* Implementation of inlined functions -*******************************************/ -/*<===== Decompression =====>*/ - -typedef struct { - U16 tableLog; - U16 fastMode; -} FSEv08_DTableHeader; /* sizeof U32 */ - -typedef struct -{ - unsigned short newState; - unsigned char symbol; - unsigned char nbBits; -} FSEv08_decode_t; /* size == U32 */ - -MEM_STATIC void FSEv08_initDState(FSEv08_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSEv08_DTable* dt) -{ - const void* ptr = dt; - const FSEv08_DTableHeader* const DTableH = (const FSEv08_DTableHeader*)ptr; - DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog); - BIT_reloadDStream(bitD); - DStatePtr->table = dt + 1; -} - -MEM_STATIC BYTE FSEv08_peekSymbol(const FSEv08_DState_t* DStatePtr) -{ - FSEv08_decode_t const DInfo = ((const FSEv08_decode_t*)(DStatePtr->table))[DStatePtr->state]; - return DInfo.symbol; -} - -MEM_STATIC void FSEv08_updateState(FSEv08_DState_t* DStatePtr, BIT_DStream_t* bitD) -{ - FSEv08_decode_t const DInfo = ((const FSEv08_decode_t*)(DStatePtr->table))[DStatePtr->state]; - U32 const nbBits = DInfo.nbBits; - size_t const lowBits = BIT_readBits(bitD, nbBits); - DStatePtr->state = DInfo.newState + lowBits; -} - -MEM_STATIC BYTE FSEv08_decodeSymbol(FSEv08_DState_t* DStatePtr, BIT_DStream_t* bitD) -{ - FSEv08_decode_t const DInfo = ((const FSEv08_decode_t*)(DStatePtr->table))[DStatePtr->state]; - U32 const nbBits = DInfo.nbBits; - BYTE const symbol = DInfo.symbol; - size_t const lowBits = BIT_readBits(bitD, nbBits); - - DStatePtr->state = DInfo.newState + lowBits; - return symbol; -} - -/*! FSEv08_decodeSymbolFast() : - unsafe, only works if no symbol has a probability > 50% */ -MEM_STATIC BYTE FSEv08_decodeSymbolFast(FSEv08_DState_t* DStatePtr, BIT_DStream_t* bitD) -{ - FSEv08_decode_t const DInfo = ((const FSEv08_decode_t*)(DStatePtr->table))[DStatePtr->state]; - U32 const nbBits = DInfo.nbBits; - BYTE const symbol = DInfo.symbol; - size_t const lowBits = BIT_readBitsFast(bitD, nbBits); - - DStatePtr->state = DInfo.newState + lowBits; - return symbol; -} - -MEM_STATIC unsigned FSEv08_endOfDState(const FSEv08_DState_t* DStatePtr) -{ - return DStatePtr->state == 0; -} - - - -#ifndef FSEv08_COMMONDEFS_ONLY - -/* ************************************************************** -* Tuning parameters -****************************************************************/ -/*!MEMORY_USAGE : -* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) -* Increasing memory usage improves compression ratio -* Reduced memory usage can improve speed, due to cache effect -* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ -#define FSEv08_MAX_MEMORY_USAGE 14 -#define FSEv08_DEFAULT_MEMORY_USAGE 13 - -/*!FSEv08_MAX_SYMBOL_VALUE : -* Maximum symbol value authorized. -* Required for proper stack allocation */ -#define FSEv08_MAX_SYMBOL_VALUE 255 - - -/* ************************************************************** -* template functions type & suffix -****************************************************************/ -#define FSEv08_FUNCTION_TYPE BYTE -#define FSEv08_FUNCTION_EXTENSION -#define FSEv08_DECODE_TYPE FSEv08_decode_t - - -#endif /* !FSEv08_COMMONDEFS_ONLY */ - - -/* *************************************************************** -* Constants -*****************************************************************/ -#define FSEv08_MAX_TABLELOG (FSEv08_MAX_MEMORY_USAGE-2) -#define FSEv08_MAX_TABLESIZE (1U<<FSEv08_MAX_TABLELOG) -#define FSEv08_MAXTABLESIZE_MASK (FSEv08_MAX_TABLESIZE-1) -#define FSEv08_DEFAULT_TABLELOG (FSEv08_DEFAULT_MEMORY_USAGE-2) -#define FSEv08_MIN_TABLELOG 5 - -#define FSEv08_TABLELOG_ABSOLUTE_MAX 15 -#if FSEv08_MAX_TABLELOG > FSEv08_TABLELOG_ABSOLUTE_MAX -# error "FSEv08_MAX_TABLELOG > FSEv08_TABLELOG_ABSOLUTE_MAX is not supported" -#endif - -#define FSEv08_TABLESTEP(tableSize) ((tableSize>>1) + (tableSize>>3) + 3) - - -#endif /* FSEv08_STATIC_LINKING_ONLY */ - - -#if defined (__cplusplus) -} -#endif - -#endif /* FSEv08_H */ -/* ****************************************************************** - Huffman coder, part of New Generation Entropy library - header file - Copyright (C) 2013-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Source repository : https://github.com/Cyan4973/FiniteStateEntropy -****************************************************************** */ -#ifndef HUFv08_H_298734234 -#define HUFv08_H_298734234 - -#if defined (__cplusplus) -extern "C" { -#endif - - -/* *** simple functions *** */ -/** -HUFv08_decompress() : - Decompress HUF data from buffer 'cSrc', of size 'cSrcSize', - into already allocated buffer 'dst', of minimum size 'dstSize'. - `dstSize` : **must** be the ***exact*** size of original (uncompressed) data. - Note : in contrast with FSE, HUFv08_decompress can regenerate - RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data, - because it knows size to regenerate. - @return : size of regenerated data (== dstSize), - or an error code, which can be tested using HUFv08_isError() -*/ -size_t HUFv08_decompress(void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize); - - -/* **************************************** -* Tool functions -******************************************/ -#define HUFv08_BLOCKSIZE_MAX (128 * 1024) - -/* Error Management */ -unsigned HUFv08_isError(size_t code); /**< tells if a return value is an error code */ -const char* HUFv08_getErrorName(size_t code); /**< provides error code string (useful for debugging) */ - - -/* *** Advanced function *** */ - - -#ifdef HUFv08_STATIC_LINKING_ONLY - - -/* *** Constants *** */ -#define HUFv08_TABLELOG_ABSOLUTEMAX 16 /* absolute limit of HUFv08_MAX_TABLELOG. Beyond that value, code does not work */ -#define HUFv08_TABLELOG_MAX 12 /* max configured tableLog (for static allocation); can be modified up to HUFv08_ABSOLUTEMAX_TABLELOG */ -#define HUFv08_TABLELOG_DEFAULT 11 /* tableLog by default, when not specified */ -#define HUFv08_SYMBOLVALUE_MAX 255 -#if (HUFv08_TABLELOG_MAX > HUFv08_TABLELOG_ABSOLUTEMAX) -# error "HUFv08_TABLELOG_MAX is too large !" -#endif - - -/* **************************************** -* Static allocation -******************************************/ -/* HUF buffer bounds */ -#define HUFv08_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true if incompressible pre-filtered with fast heuristic */ - -/* static allocation of HUF's DTable */ -typedef U32 HUFv08_DTable; -#define HUFv08_DTABLE_SIZE(maxTableLog) (1 + (1<<(maxTableLog))) -#define HUFv08_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \ - HUFv08_DTable DTable[HUFv08_DTABLE_SIZE((maxTableLog)-1)] = { ((U32)((maxTableLog)-1)*0x1000001) } -#define HUFv08_CREATE_STATIC_DTABLEX4(DTable, maxTableLog) \ - HUFv08_DTable DTable[HUFv08_DTABLE_SIZE(maxTableLog)] = { ((U32)(maxTableLog)*0x1000001) } - - -/* **************************************** -* Advanced decompression functions -******************************************/ -size_t HUFv08_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ -size_t HUFv08_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ - -size_t HUFv08_decompress4X_DCtx (HUFv08_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< decodes RLE and uncompressed */ -size_t HUFv08_decompress4X_hufOnly(HUFv08_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */ -size_t HUFv08_decompress4X2_DCtx(HUFv08_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ -size_t HUFv08_decompress4X4_DCtx(HUFv08_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ - -size_t HUFv08_decompress1X_DCtx (HUFv08_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); -size_t HUFv08_decompress1X2_DCtx(HUFv08_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ -size_t HUFv08_decompress1X4_DCtx(HUFv08_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ - - -/* **************************************** -* HUF detailed API -******************************************/ -/* -HUFv08_decompress() does the following: -1. select the decompression algorithm (X2, X4) based on pre-computed heuristics -2. build Huffman table from save, using HUFv08_readDTableXn() -3. decode 1 or 4 segments in parallel using HUFv08_decompressSXn_usingDTable -*/ - -/** HUFv08_selectDecoder() : -* Tells which decoder is likely to decode faster, -* based on a set of pre-determined metrics. -* @return : 0==HUFv08_decompress4X2, 1==HUFv08_decompress4X4 . -* Assumption : 0 < cSrcSize < dstSize <= 128 KB */ -U32 HUFv08_selectDecoder (size_t dstSize, size_t cSrcSize); - -size_t HUFv08_readDTableX2 (HUFv08_DTable* DTable, const void* src, size_t srcSize); -size_t HUFv08_readDTableX4 (HUFv08_DTable* DTable, const void* src, size_t srcSize); - -size_t HUFv08_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUFv08_DTable* DTable); -size_t HUFv08_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUFv08_DTable* DTable); -size_t HUFv08_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUFv08_DTable* DTable); - - -/* single stream variants */ - -size_t HUFv08_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */ -size_t HUFv08_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */ - -size_t HUFv08_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUFv08_DTable* DTable); -size_t HUFv08_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUFv08_DTable* DTable); -size_t HUFv08_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUFv08_DTable* DTable); - - -#endif /* HUFv08_STATIC_LINKING_ONLY */ - - -#if defined (__cplusplus) -} -#endif - -#endif /* HUFv08_H_298734234 */ -/* - Common functions of New Generation Entropy library - Copyright (C) 2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy - - Public forum : https://groups.google.com/forum/#!forum/lz4c -*************************************************************************** */ - - -/*-**************************************** -* FSE Error Management -******************************************/ -unsigned FSEv08_isError(size_t code) { return ERR_isError(code); } - -const char* FSEv08_getErrorName(size_t code) { return ERR_getErrorName(code); } - - -/* ************************************************************** -* HUF Error Management -****************************************************************/ -unsigned HUFv08_isError(size_t code) { return ERR_isError(code); } - -const char* HUFv08_getErrorName(size_t code) { return ERR_getErrorName(code); } - - -/*-************************************************************** -* FSE NCount encoding-decoding -****************************************************************/ -static short FSEv08_abs(short a) { return (short)(a<0 ? -a : a); } - -size_t FSEv08_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, - const void* headerBuffer, size_t hbSize) -{ - const BYTE* const istart = (const BYTE*) headerBuffer; - const BYTE* const iend = istart + hbSize; - const BYTE* ip = istart; - int nbBits; - int remaining; - int threshold; - U32 bitStream; - int bitCount; - unsigned charnum = 0; - int previous0 = 0; - - if (hbSize < 4) return ERROR(srcSize_wrong); - bitStream = MEM_readLE32(ip); - nbBits = (bitStream & 0xF) + FSEv08_MIN_TABLELOG; /* extract tableLog */ - if (nbBits > FSEv08_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge); - bitStream >>= 4; - bitCount = 4; - *tableLogPtr = nbBits; - remaining = (1<<nbBits)+1; - threshold = 1<<nbBits; - nbBits++; - - while ((remaining>1) & (charnum<=*maxSVPtr)) { - if (previous0) { - unsigned n0 = charnum; - while ((bitStream & 0xFFFF) == 0xFFFF) { - n0 += 24; - if (ip < iend-5) { - ip += 2; - bitStream = MEM_readLE32(ip) >> bitCount; - } else { - bitStream >>= 16; - bitCount += 16; - } } - while ((bitStream & 3) == 3) { - n0 += 3; - bitStream >>= 2; - bitCount += 2; - } - n0 += bitStream & 3; - bitCount += 2; - if (n0 > *maxSVPtr) return ERROR(maxSymbolValue_tooSmall); - while (charnum < n0) normalizedCounter[charnum++] = 0; - if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { - ip += bitCount>>3; - bitCount &= 7; - bitStream = MEM_readLE32(ip) >> bitCount; - } else { - bitStream >>= 2; - } } - { short const max = (short)((2*threshold-1)-remaining); - short count; - - if ((bitStream & (threshold-1)) < (U32)max) { - count = (short)(bitStream & (threshold-1)); - bitCount += nbBits-1; - } else { - count = (short)(bitStream & (2*threshold-1)); - if (count >= threshold) count -= max; - bitCount += nbBits; - } - - count--; /* extra accuracy */ - remaining -= FSEv08_abs(count); - normalizedCounter[charnum++] = count; - previous0 = !count; - while (remaining < threshold) { - nbBits--; - threshold >>= 1; - } - - if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { - ip += bitCount>>3; - bitCount &= 7; - } else { - bitCount -= (int)(8 * (iend - 4 - ip)); - ip = iend - 4; - } - bitStream = MEM_readLE32(ip) >> (bitCount & 31); - } } /* while ((remaining>1) & (charnum<=*maxSVPtr)) */ - if (remaining != 1) return ERROR(corruption_detected); - if (bitCount > 32) return ERROR(corruption_detected); - *maxSVPtr = charnum-1; - - ip += (bitCount+7)>>3; - return ip-istart; -} - - -/*! HUFv08_readStats() : - Read compact Huffman tree, saved by HUFv08_writeCTable(). - `huffWeight` is destination buffer. - @return : size read from `src` , or an error Code . - Note : Needed by HUFv08_readCTable() and HUFv08_readDTableX?() . -*/ -size_t HUFv08_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, - U32* nbSymbolsPtr, U32* tableLogPtr, - const void* src, size_t srcSize) -{ - U32 weightTotal; - const BYTE* ip = (const BYTE*) src; - size_t iSize = ip[0]; - size_t oSize; - - /* memset(huffWeight, 0, hwSize); *//* is not necessary, even though some analyzer complain ... */ - - if (iSize >= 128) { /* special header */ - oSize = iSize - 127; - iSize = ((oSize+1)/2); - if (iSize+1 > srcSize) return ERROR(srcSize_wrong); - if (oSize >= hwSize) return ERROR(corruption_detected); - ip += 1; - { U32 n; - for (n=0; n<oSize; n+=2) { - huffWeight[n] = ip[n/2] >> 4; - huffWeight[n+1] = ip[n/2] & 15; - } } } - else { /* header compressed with FSE (normal case) */ - if (iSize+1 > srcSize) return ERROR(srcSize_wrong); - oSize = FSEv08_decompress(huffWeight, hwSize-1, ip+1, iSize); /* max (hwSize-1) values decoded, as last one is implied */ - if (FSEv08_isError(oSize)) return oSize; - } - - /* collect weight stats */ - memset(rankStats, 0, (HUFv08_TABLELOG_ABSOLUTEMAX + 1) * sizeof(U32)); - weightTotal = 0; - { U32 n; for (n=0; n<oSize; n++) { - if (huffWeight[n] >= HUFv08_TABLELOG_ABSOLUTEMAX) return ERROR(corruption_detected); - rankStats[huffWeight[n]]++; - weightTotal += (1 << huffWeight[n]) >> 1; - } } - - /* get last non-null symbol weight (implied, total must be 2^n) */ - { U32 const tableLog = BIT_highbit32(weightTotal) + 1; - if (tableLog > HUFv08_TABLELOG_ABSOLUTEMAX) return ERROR(corruption_detected); - *tableLogPtr = tableLog; - /* determine last weight */ - { U32 const total = 1 << tableLog; - U32 const rest = total - weightTotal; - U32 const verif = 1 << BIT_highbit32(rest); - U32 const lastWeight = BIT_highbit32(rest) + 1; - if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */ - huffWeight[oSize] = (BYTE)lastWeight; - rankStats[lastWeight]++; - } } - - /* check tree construction validity */ - if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */ - - /* results */ - *nbSymbolsPtr = (U32)(oSize+1); - return iSize+1; -} -/* ****************************************************************** - FSE : Finite State Entropy decoder - Copyright (C) 2013-2015, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy - - Public forum : https://groups.google.com/forum/#!forum/lz4c -****************************************************************** */ - - -/* ************************************************************** -* Compiler specifics -****************************************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# define FORCE_INLINE static __forceinline -# include <intrin.h> /* For Visual 2005 */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */ -#else -# ifdef __GNUC__ -# define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -# define FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define FORCE_INLINE static inline -# endif -#endif - - -/* ************************************************************** -* Error Management -****************************************************************/ -#define FSEv08_isError ERR_isError -#define FSEv08_STATIC_ASSERT(c) { enum { FSEv08_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ - - -/* ************************************************************** -* Complex types -****************************************************************/ -typedef U32 DTable_max_t[FSEv08_DTABLE_SIZE_U32(FSEv08_MAX_TABLELOG)]; - - -/* ************************************************************** -* Templates -****************************************************************/ -/* - designed to be included - for type-specific functions (template emulation in C) - Objective is to write these functions only once, for improved maintenance -*/ - -/* safety checks */ -#ifndef FSEv08_FUNCTION_EXTENSION -# error "FSEv08_FUNCTION_EXTENSION must be defined" -#endif -#ifndef FSEv08_FUNCTION_TYPE -# error "FSEv08_FUNCTION_TYPE must be defined" -#endif - -/* Function names */ -#define FSEv08_CAT(X,Y) X##Y -#define FSEv08_FUNCTION_NAME(X,Y) FSEv08_CAT(X,Y) -#define FSEv08_TYPE_NAME(X,Y) FSEv08_CAT(X,Y) - - -/* Function templates */ -FSEv08_DTable* FSEv08_createDTable (unsigned tableLog) -{ - if (tableLog > FSEv08_TABLELOG_ABSOLUTE_MAX) tableLog = FSEv08_TABLELOG_ABSOLUTE_MAX; - return (FSEv08_DTable*)malloc( FSEv08_DTABLE_SIZE_U32(tableLog) * sizeof (U32) ); -} - -void FSEv08_freeDTable (FSEv08_DTable* dt) -{ - free(dt); -} - -size_t FSEv08_buildDTable(FSEv08_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) -{ - void* const tdPtr = dt+1; /* because *dt is unsigned, 32-bits aligned on 32-bits */ - FSEv08_DECODE_TYPE* const tableDecode = (FSEv08_DECODE_TYPE*) (tdPtr); - U16 symbolNext[FSEv08_MAX_SYMBOL_VALUE+1]; - - U32 const maxSV1 = maxSymbolValue + 1; - U32 const tableSize = 1 << tableLog; - U32 highThreshold = tableSize-1; - - /* Sanity Checks */ - if (maxSymbolValue > FSEv08_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge); - if (tableLog > FSEv08_MAX_TABLELOG) return ERROR(tableLog_tooLarge); - - /* Init, lay down lowprob symbols */ - { FSEv08_DTableHeader DTableH; - DTableH.tableLog = (U16)tableLog; - DTableH.fastMode = 1; - { S16 const largeLimit= (S16)(1 << (tableLog-1)); - U32 s; - for (s=0; s<maxSV1; s++) { - if (normalizedCounter[s]==-1) { - tableDecode[highThreshold--].symbol = (FSEv08_FUNCTION_TYPE)s; - symbolNext[s] = 1; - } else { - if (normalizedCounter[s] >= largeLimit) DTableH.fastMode=0; - symbolNext[s] = normalizedCounter[s]; - } } } - memcpy(dt, &DTableH, sizeof(DTableH)); - } - - /* Spread symbols */ - { U32 const tableMask = tableSize-1; - U32 const step = FSEv08_TABLESTEP(tableSize); - U32 s, position = 0; - for (s=0; s<maxSV1; s++) { - int i; - for (i=0; i<normalizedCounter[s]; i++) { - tableDecode[position].symbol = (FSEv08_FUNCTION_TYPE)s; - position = (position + step) & tableMask; - while (position > highThreshold) position = (position + step) & tableMask; /* lowprob area */ - } } - - if (position!=0) return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ - } - - /* Build Decoding table */ - { U32 u; - for (u=0; u<tableSize; u++) { - FSEv08_FUNCTION_TYPE const symbol = (FSEv08_FUNCTION_TYPE)(tableDecode[u].symbol); - U16 nextState = symbolNext[symbol]++; - tableDecode[u].nbBits = (BYTE) (tableLog - BIT_highbit32 ((U32)nextState) ); - tableDecode[u].newState = (U16) ( (nextState << tableDecode[u].nbBits) - tableSize); - } } - - return 0; -} - - - -#ifndef FSEv08_COMMONDEFS_ONLY - -/*-******************************************************* -* Decompression (Byte symbols) -*********************************************************/ -size_t FSEv08_buildDTable_rle (FSEv08_DTable* dt, BYTE symbolValue) -{ - void* ptr = dt; - FSEv08_DTableHeader* const DTableH = (FSEv08_DTableHeader*)ptr; - void* dPtr = dt + 1; - FSEv08_decode_t* const cell = (FSEv08_decode_t*)dPtr; - - DTableH->tableLog = 0; - DTableH->fastMode = 0; - - cell->newState = 0; - cell->symbol = symbolValue; - cell->nbBits = 0; - - return 0; -} - - -size_t FSEv08_buildDTable_raw (FSEv08_DTable* dt, unsigned nbBits) -{ - void* ptr = dt; - FSEv08_DTableHeader* const DTableH = (FSEv08_DTableHeader*)ptr; - void* dPtr = dt + 1; - FSEv08_decode_t* const dinfo = (FSEv08_decode_t*)dPtr; - const unsigned tableSize = 1 << nbBits; - const unsigned tableMask = tableSize - 1; - const unsigned maxSV1 = tableMask+1; - unsigned s; - - /* Sanity checks */ - if (nbBits < 1) return ERROR(GENERIC); /* min size */ - - /* Build Decoding Table */ - DTableH->tableLog = (U16)nbBits; - DTableH->fastMode = 1; - for (s=0; s<maxSV1; s++) { - dinfo[s].newState = 0; - dinfo[s].symbol = (BYTE)s; - dinfo[s].nbBits = (BYTE)nbBits; - } - - return 0; -} - -FORCE_INLINE size_t FSEv08_decompress_usingDTable_generic( - void* dst, size_t maxDstSize, - const void* cSrc, size_t cSrcSize, - const FSEv08_DTable* dt, const unsigned fast) -{ - BYTE* const ostart = (BYTE*) dst; - BYTE* op = ostart; - BYTE* const omax = op + maxDstSize; - BYTE* const olimit = omax-3; - - BIT_DStream_t bitD; - FSEv08_DState_t state1; - FSEv08_DState_t state2; - - /* Init */ - { size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize); /* replaced last arg by maxCompressed Size */ - if (FSEv08_isError(errorCode)) return errorCode; } - - FSEv08_initDState(&state1, &bitD, dt); - FSEv08_initDState(&state2, &bitD, dt); - -#define FSEv08_GETSYMBOL(statePtr) fast ? FSEv08_decodeSymbolFast(statePtr, &bitD) : FSEv08_decodeSymbol(statePtr, &bitD) - - /* 4 symbols per loop */ - for ( ; (BIT_reloadDStream(&bitD)==BIT_DStream_unfinished) && (op<olimit) ; op+=4) { - op[0] = FSEv08_GETSYMBOL(&state1); - - if (FSEv08_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ - BIT_reloadDStream(&bitD); - - op[1] = FSEv08_GETSYMBOL(&state2); - - if (FSEv08_MAX_TABLELOG*4+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ - { if (BIT_reloadDStream(&bitD) > BIT_DStream_unfinished) { op+=2; break; } } - - op[2] = FSEv08_GETSYMBOL(&state1); - - if (FSEv08_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ - BIT_reloadDStream(&bitD); - - op[3] = FSEv08_GETSYMBOL(&state2); - } - - /* tail */ - /* note : BIT_reloadDStream(&bitD) >= FSEv08_DStream_partiallyFilled; Ends at exactly BIT_DStream_completed */ - while (1) { - if (op>(omax-2)) return ERROR(dstSize_tooSmall); - - *op++ = FSEv08_GETSYMBOL(&state1); - - if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) { - *op++ = FSEv08_GETSYMBOL(&state2); - break; - } - - if (op>(omax-2)) return ERROR(dstSize_tooSmall); - - *op++ = FSEv08_GETSYMBOL(&state2); - - if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) { - *op++ = FSEv08_GETSYMBOL(&state1); - break; - } } - - return op-ostart; -} - - -size_t FSEv08_decompress_usingDTable(void* dst, size_t originalSize, - const void* cSrc, size_t cSrcSize, - const FSEv08_DTable* dt) -{ - const void* ptr = dt; - const FSEv08_DTableHeader* DTableH = (const FSEv08_DTableHeader*)ptr; - const U32 fastMode = DTableH->fastMode; - - /* select fast mode (static) */ - if (fastMode) return FSEv08_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1); - return FSEv08_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0); -} - - -size_t FSEv08_decompress(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize) -{ - const BYTE* const istart = (const BYTE*)cSrc; - const BYTE* ip = istart; - short counting[FSEv08_MAX_SYMBOL_VALUE+1]; - DTable_max_t dt; /* Static analyzer seems unable to understand this table will be properly initialized later */ - unsigned tableLog; - unsigned maxSymbolValue = FSEv08_MAX_SYMBOL_VALUE; - - if (cSrcSize<2) return ERROR(srcSize_wrong); /* too small input size */ - - /* normal FSE decoding mode */ - { size_t const NCountLength = FSEv08_readNCount (counting, &maxSymbolValue, &tableLog, istart, cSrcSize); - if (FSEv08_isError(NCountLength)) return NCountLength; - if (NCountLength >= cSrcSize) return ERROR(srcSize_wrong); /* too small input size */ - ip += NCountLength; - cSrcSize -= NCountLength; - } - - { size_t const errorCode = FSEv08_buildDTable (dt, counting, maxSymbolValue, tableLog); - if (FSEv08_isError(errorCode)) return errorCode; } - - return FSEv08_decompress_usingDTable (dst, maxDstSize, ip, cSrcSize, dt); /* always return, even if it is an error code */ -} - - - -#endif /* FSEv08_COMMONDEFS_ONLY */ -/* - Common functions of Zstd compression library - Copyright (C) 2015-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - zstd homepage : http://www.zstd.net/ -*/ - - - -/*-**************************************** -* ZSTD Error Management -******************************************/ -/*! ZSTDv08_isError() : -* tells if a return value is an error code */ -unsigned ZSTDv08_isError(size_t code) { return ERR_isError(code); } - -/*! ZSTDv08_getErrorName() : -* provides error code string from function result (useful for debugging) */ -const char* ZSTDv08_getErrorName(size_t code) { return ERR_getErrorName(code); } - -/*! ZSTDv08_getError() : -* convert a `size_t` function result into a proper ZSTDv08_errorCode enum */ -ZSTDv08_ErrorCode ZSTDv08_getErrorCode(size_t code) { return ERR_getErrorCode(code); } - -/*! ZSTDv08_getErrorString() : -* provides error code string from enum */ -const char* ZSTDv08_getErrorString(ZSTDv08_ErrorCode code) { return ERR_getErrorName(code); } - - -/* ************************************************************** -* ZBUFF Error Management -****************************************************************/ -unsigned ZBUFFv08_isError(size_t errorCode) { return ERR_isError(errorCode); } - -const char* ZBUFFv08_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); } - - - -void* ZSTDv08_defaultAllocFunction(void* opaque, size_t size) -{ - void* address = malloc(size); - (void)opaque; - /* printf("alloc %p, %d opaque=%p \n", address, (int)size, opaque); */ - return address; -} - -void ZSTDv08_defaultFreeFunction(void* opaque, void* address) -{ - (void)opaque; - /* if (address) printf("free %p opaque=%p \n", address, opaque); */ - free(address); -} -/* ****************************************************************** - Huffman decoder, part of New Generation Entropy library - Copyright (C) 2013-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy - - Public forum : https://groups.google.com/forum/#!forum/lz4c -****************************************************************** */ - -/* ************************************************************** -* Compiler specifics -****************************************************************/ -#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -/* inline is defined */ -#elif defined(_MSC_VER) -# define inline __inline -#else -# define inline /* disable inline */ -#endif - - -#ifdef _MSC_VER /* Visual Studio */ -# define FORCE_INLINE static __forceinline -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -#else -# ifdef __GNUC__ -# define FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define FORCE_INLINE static inline -# endif -#endif - - -/* ************************************************************** -* Error Management -****************************************************************/ -#define HUFv08_STATIC_ASSERT(c) { enum { HUFv08_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ - - -/*-***************************/ -/* generic DTableDesc */ -/*-***************************/ - -typedef struct { BYTE maxTableLog; BYTE tableType; BYTE tableLog; BYTE reserved; } DTableDesc; - -static DTableDesc HUFv08_getDTableDesc(const HUFv08_DTable* table) -{ - DTableDesc dtd; - memcpy(&dtd, table, sizeof(dtd)); - return dtd; -} - - -/*-***************************/ -/* single-symbol decoding */ -/*-***************************/ - -typedef struct { BYTE byte; BYTE nbBits; } HUFv08_DEltX2; /* single-symbol decoding */ - -size_t HUFv08_readDTableX2 (HUFv08_DTable* DTable, const void* src, size_t srcSize) -{ - BYTE huffWeight[HUFv08_SYMBOLVALUE_MAX + 1]; - U32 rankVal[HUFv08_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */ - U32 tableLog = 0; - U32 nbSymbols = 0; - size_t iSize; - void* const dtPtr = DTable + 1; - HUFv08_DEltX2* const dt = (HUFv08_DEltX2*)dtPtr; - - HUFv08_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUFv08_DTable)); - //memset(huffWeight, 0, sizeof(huffWeight)); /* is not necessary, even though some analyzer complain ... */ - - iSize = HUFv08_readStats(huffWeight, HUFv08_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize); - if (HUFv08_isError(iSize)) return iSize; - - /* Table header */ - { DTableDesc dtd = HUFv08_getDTableDesc(DTable); - if (tableLog > (U32)(dtd.maxTableLog+1)) return ERROR(tableLog_tooLarge); /* DTable too small, huffman tree cannot fit in */ - dtd.tableType = 0; - dtd.tableLog = (BYTE)tableLog; - memcpy(DTable, &dtd, sizeof(dtd)); - } - - /* Prepare ranks */ - { U32 n, nextRankStart = 0; - for (n=1; n<tableLog+1; n++) { - U32 current = nextRankStart; - nextRankStart += (rankVal[n] << (n-1)); - rankVal[n] = current; - } } - - /* fill DTable */ - { U32 n; - for (n=0; n<nbSymbols; n++) { - U32 const w = huffWeight[n]; - U32 const length = (1 << w) >> 1; - U32 i; - HUFv08_DEltX2 D; - D.byte = (BYTE)n; D.nbBits = (BYTE)(tableLog + 1 - w); - for (i = rankVal[w]; i < rankVal[w] + length; i++) - dt[i] = D; - rankVal[w] += length; - } } - - return iSize; -} - - -static BYTE HUFv08_decodeSymbolX2(BIT_DStream_t* Dstream, const HUFv08_DEltX2* dt, const U32 dtLog) -{ - size_t const val = BIT_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */ - BYTE const c = dt[val].byte; - BIT_skipBits(Dstream, dt[val].nbBits); - return c; -} - -#define HUFv08_DECODE_SYMBOLX2_0(ptr, DStreamPtr) \ - *ptr++ = HUFv08_decodeSymbolX2(DStreamPtr, dt, dtLog) - -#define HUFv08_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \ - if (MEM_64bits() || (HUFv08_TABLELOG_MAX<=12)) \ - HUFv08_DECODE_SYMBOLX2_0(ptr, DStreamPtr) - -#define HUFv08_DECODE_SYMBOLX2_2(ptr, DStreamPtr) \ - if (MEM_64bits()) \ - HUFv08_DECODE_SYMBOLX2_0(ptr, DStreamPtr) - -static inline size_t HUFv08_decodeStreamX2(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, const HUFv08_DEltX2* const dt, const U32 dtLog) -{ - BYTE* const pStart = p; - - /* up to 4 symbols at a time */ - while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p <= pEnd-4)) { - HUFv08_DECODE_SYMBOLX2_2(p, bitDPtr); - HUFv08_DECODE_SYMBOLX2_1(p, bitDPtr); - HUFv08_DECODE_SYMBOLX2_2(p, bitDPtr); - HUFv08_DECODE_SYMBOLX2_0(p, bitDPtr); - } - - /* closer to the end */ - while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p < pEnd)) - HUFv08_DECODE_SYMBOLX2_0(p, bitDPtr); - - /* no more data to retrieve from bitstream, hence no need to reload */ - while (p < pEnd) - HUFv08_DECODE_SYMBOLX2_0(p, bitDPtr); - - return pEnd-pStart; -} - -static size_t HUFv08_decompress1X2_usingDTable_internal( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUFv08_DTable* DTable) -{ - BYTE* op = (BYTE*)dst; - BYTE* const oend = op + dstSize; - const void* dtPtr = DTable + 1; - const HUFv08_DEltX2* const dt = (const HUFv08_DEltX2*)dtPtr; - BIT_DStream_t bitD; - DTableDesc const dtd = HUFv08_getDTableDesc(DTable); - U32 const dtLog = dtd.tableLog; - - { size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize); - if (HUFv08_isError(errorCode)) return errorCode; } - - HUFv08_decodeStreamX2(op, &bitD, oend, dt, dtLog); - - /* check */ - if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); - - return dstSize; -} - -size_t HUFv08_decompress1X2_usingDTable( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUFv08_DTable* DTable) -{ - DTableDesc dtd = HUFv08_getDTableDesc(DTable); - if (dtd.tableType != 0) return ERROR(GENERIC); - return HUFv08_decompress1X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); -} - -size_t HUFv08_decompress1X2_DCtx (HUFv08_DTable* DCtx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - const BYTE* ip = (const BYTE*) cSrc; - - size_t const hSize = HUFv08_readDTableX2 (DCtx, cSrc, cSrcSize); - if (HUFv08_isError(hSize)) return hSize; - if (hSize >= cSrcSize) return ERROR(srcSize_wrong); - ip += hSize; cSrcSize -= hSize; - - return HUFv08_decompress1X2_usingDTable_internal (dst, dstSize, ip, cSrcSize, DCtx); -} - -size_t HUFv08_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - HUFv08_CREATE_STATIC_DTABLEX2(DTable, HUFv08_TABLELOG_MAX); - return HUFv08_decompress1X2_DCtx (DTable, dst, dstSize, cSrc, cSrcSize); -} - - -static size_t HUFv08_decompress4X2_usingDTable_internal( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUFv08_DTable* DTable) -{ - /* Check */ - if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ - - { const BYTE* const istart = (const BYTE*) cSrc; - BYTE* const ostart = (BYTE*) dst; - BYTE* const oend = ostart + dstSize; - const void* const dtPtr = DTable + 1; - const HUFv08_DEltX2* const dt = (const HUFv08_DEltX2*)dtPtr; - - /* Init */ - BIT_DStream_t bitD1; - BIT_DStream_t bitD2; - BIT_DStream_t bitD3; - BIT_DStream_t bitD4; - size_t const length1 = MEM_readLE16(istart); - size_t const length2 = MEM_readLE16(istart+2); - size_t const length3 = MEM_readLE16(istart+4); - size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); - const BYTE* const istart1 = istart + 6; /* jumpTable */ - const BYTE* const istart2 = istart1 + length1; - const BYTE* const istart3 = istart2 + length2; - const BYTE* const istart4 = istart3 + length3; - const size_t segmentSize = (dstSize+3) / 4; - BYTE* const opStart2 = ostart + segmentSize; - BYTE* const opStart3 = opStart2 + segmentSize; - BYTE* const opStart4 = opStart3 + segmentSize; - BYTE* op1 = ostart; - BYTE* op2 = opStart2; - BYTE* op3 = opStart3; - BYTE* op4 = opStart4; - U32 endSignal; - DTableDesc const dtd = HUFv08_getDTableDesc(DTable); - U32 const dtLog = dtd.tableLog; - - if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ - { size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1); - if (HUFv08_isError(errorCode)) return errorCode; } - { size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2); - if (HUFv08_isError(errorCode)) return errorCode; } - { size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3); - if (HUFv08_isError(errorCode)) return errorCode; } - { size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4); - if (HUFv08_isError(errorCode)) return errorCode; } - - /* 16-32 symbols per loop (4-8 symbols per stream) */ - endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); - for ( ; (endSignal==BIT_DStream_unfinished) && (op4<(oend-7)) ; ) { - HUFv08_DECODE_SYMBOLX2_2(op1, &bitD1); - HUFv08_DECODE_SYMBOLX2_2(op2, &bitD2); - HUFv08_DECODE_SYMBOLX2_2(op3, &bitD3); - HUFv08_DECODE_SYMBOLX2_2(op4, &bitD4); - HUFv08_DECODE_SYMBOLX2_1(op1, &bitD1); - HUFv08_DECODE_SYMBOLX2_1(op2, &bitD2); - HUFv08_DECODE_SYMBOLX2_1(op3, &bitD3); - HUFv08_DECODE_SYMBOLX2_1(op4, &bitD4); - HUFv08_DECODE_SYMBOLX2_2(op1, &bitD1); - HUFv08_DECODE_SYMBOLX2_2(op2, &bitD2); - HUFv08_DECODE_SYMBOLX2_2(op3, &bitD3); - HUFv08_DECODE_SYMBOLX2_2(op4, &bitD4); - HUFv08_DECODE_SYMBOLX2_0(op1, &bitD1); - HUFv08_DECODE_SYMBOLX2_0(op2, &bitD2); - HUFv08_DECODE_SYMBOLX2_0(op3, &bitD3); - HUFv08_DECODE_SYMBOLX2_0(op4, &bitD4); - endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); - } - - /* check corruption */ - if (op1 > opStart2) return ERROR(corruption_detected); - if (op2 > opStart3) return ERROR(corruption_detected); - if (op3 > opStart4) return ERROR(corruption_detected); - /* note : op4 supposed already verified within main loop */ - - /* finish bitStreams one by one */ - HUFv08_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog); - HUFv08_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog); - HUFv08_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog); - HUFv08_decodeStreamX2(op4, &bitD4, oend, dt, dtLog); - - /* check */ - endSignal = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); - if (!endSignal) return ERROR(corruption_detected); - - /* decoded size */ - return dstSize; - } -} - - -size_t HUFv08_decompress4X2_usingDTable( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUFv08_DTable* DTable) -{ - DTableDesc dtd = HUFv08_getDTableDesc(DTable); - if (dtd.tableType != 0) return ERROR(GENERIC); - return HUFv08_decompress4X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); -} - - -size_t HUFv08_decompress4X2_DCtx (HUFv08_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - const BYTE* ip = (const BYTE*) cSrc; - - size_t const hSize = HUFv08_readDTableX2 (dctx, cSrc, cSrcSize); - if (HUFv08_isError(hSize)) return hSize; - if (hSize >= cSrcSize) return ERROR(srcSize_wrong); - ip += hSize; cSrcSize -= hSize; - - return HUFv08_decompress4X2_usingDTable_internal (dst, dstSize, ip, cSrcSize, dctx); -} - -size_t HUFv08_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - HUFv08_CREATE_STATIC_DTABLEX2(DTable, HUFv08_TABLELOG_MAX); - return HUFv08_decompress4X2_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); -} - - -/* *************************/ -/* double-symbols decoding */ -/* *************************/ -typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUFv08_DEltX4; /* double-symbols decoding */ - -typedef struct { BYTE symbol; BYTE weight; } sortedSymbol_t; - -static void HUFv08_fillDTableX4Level2(HUFv08_DEltX4* DTable, U32 sizeLog, const U32 consumed, - const U32* rankValOrigin, const int minWeight, - const sortedSymbol_t* sortedSymbols, const U32 sortedListSize, - U32 nbBitsBaseline, U16 baseSeq) -{ - HUFv08_DEltX4 DElt; - U32 rankVal[HUFv08_TABLELOG_ABSOLUTEMAX + 1]; - - /* get pre-calculated rankVal */ - memcpy(rankVal, rankValOrigin, sizeof(rankVal)); - - /* fill skipped values */ - if (minWeight>1) { - U32 i, skipSize = rankVal[minWeight]; - MEM_writeLE16(&(DElt.sequence), baseSeq); - DElt.nbBits = (BYTE)(consumed); - DElt.length = 1; - for (i = 0; i < skipSize; i++) - DTable[i] = DElt; - } - - /* fill DTable */ - { U32 s; for (s=0; s<sortedListSize; s++) { /* note : sortedSymbols already skipped */ - const U32 symbol = sortedSymbols[s].symbol; - const U32 weight = sortedSymbols[s].weight; - const U32 nbBits = nbBitsBaseline - weight; - const U32 length = 1 << (sizeLog-nbBits); - const U32 start = rankVal[weight]; - U32 i = start; - const U32 end = start + length; - - MEM_writeLE16(&(DElt.sequence), (U16)(baseSeq + (symbol << 8))); - DElt.nbBits = (BYTE)(nbBits + consumed); - DElt.length = 2; - do { DTable[i++] = DElt; } while (i<end); /* since length >= 1 */ - - rankVal[weight] += length; - }} -} - -typedef U32 rankVal_t[HUFv08_TABLELOG_ABSOLUTEMAX][HUFv08_TABLELOG_ABSOLUTEMAX + 1]; - -static void HUFv08_fillDTableX4(HUFv08_DEltX4* DTable, const U32 targetLog, - const sortedSymbol_t* sortedList, const U32 sortedListSize, - const U32* rankStart, rankVal_t rankValOrigin, const U32 maxWeight, - const U32 nbBitsBaseline) -{ - U32 rankVal[HUFv08_TABLELOG_ABSOLUTEMAX + 1]; - const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */ - const U32 minBits = nbBitsBaseline - maxWeight; - U32 s; - - memcpy(rankVal, rankValOrigin, sizeof(rankVal)); - - /* fill DTable */ - for (s=0; s<sortedListSize; s++) { - const U16 symbol = sortedList[s].symbol; - const U32 weight = sortedList[s].weight; - const U32 nbBits = nbBitsBaseline - weight; - const U32 start = rankVal[weight]; - const U32 length = 1 << (targetLog-nbBits); - - if (targetLog-nbBits >= minBits) { /* enough room for a second symbol */ - U32 sortedRank; - int minWeight = nbBits + scaleLog; - if (minWeight < 1) minWeight = 1; - sortedRank = rankStart[minWeight]; - HUFv08_fillDTableX4Level2(DTable+start, targetLog-nbBits, nbBits, - rankValOrigin[nbBits], minWeight, - sortedList+sortedRank, sortedListSize-sortedRank, - nbBitsBaseline, symbol); - } else { - HUFv08_DEltX4 DElt; - MEM_writeLE16(&(DElt.sequence), symbol); - DElt.nbBits = (BYTE)(nbBits); - DElt.length = 1; - { U32 u; - const U32 end = start + length; - for (u = start; u < end; u++) DTable[u] = DElt; - } } - rankVal[weight] += length; - } -} - -size_t HUFv08_readDTableX4 (HUFv08_DTable* DTable, const void* src, size_t srcSize) -{ - BYTE weightList[HUFv08_SYMBOLVALUE_MAX + 1]; - sortedSymbol_t sortedSymbol[HUFv08_SYMBOLVALUE_MAX + 1]; - U32 rankStats[HUFv08_TABLELOG_ABSOLUTEMAX + 1] = { 0 }; - U32 rankStart0[HUFv08_TABLELOG_ABSOLUTEMAX + 2] = { 0 }; - U32* const rankStart = rankStart0+1; - rankVal_t rankVal; - U32 tableLog, maxW, sizeOfSort, nbSymbols; - DTableDesc dtd = HUFv08_getDTableDesc(DTable); - U32 const maxTableLog = dtd.maxTableLog; - size_t iSize; - void* dtPtr = DTable+1; /* force compiler to avoid strict-aliasing */ - HUFv08_DEltX4* const dt = (HUFv08_DEltX4*)dtPtr; - - HUFv08_STATIC_ASSERT(sizeof(HUFv08_DEltX4) == sizeof(HUFv08_DTable)); /* if compilation fails here, assertion is false */ - if (maxTableLog > HUFv08_TABLELOG_ABSOLUTEMAX) return ERROR(tableLog_tooLarge); - //memset(weightList, 0, sizeof(weightList)); /* is not necessary, even though some analyzer complain ... */ - - iSize = HUFv08_readStats(weightList, HUFv08_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize); - if (HUFv08_isError(iSize)) return iSize; - - /* check result */ - if (tableLog > maxTableLog) return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */ - - /* find maxWeight */ - for (maxW = tableLog; rankStats[maxW]==0; maxW--) {} /* necessarily finds a solution before 0 */ - - /* Get start index of each weight */ - { U32 w, nextRankStart = 0; - for (w=1; w<maxW+1; w++) { - U32 current = nextRankStart; - nextRankStart += rankStats[w]; - rankStart[w] = current; - } - rankStart[0] = nextRankStart; /* put all 0w symbols at the end of sorted list*/ - sizeOfSort = nextRankStart; - } - - /* sort symbols by weight */ - { U32 s; - for (s=0; s<nbSymbols; s++) { - U32 const w = weightList[s]; - U32 const r = rankStart[w]++; - sortedSymbol[r].symbol = (BYTE)s; - sortedSymbol[r].weight = (BYTE)w; - } - rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */ - } - - /* Build rankVal */ - { U32* const rankVal0 = rankVal[0]; - { int const rescale = (maxTableLog-tableLog) - 1; /* tableLog <= maxTableLog */ - U32 nextRankVal = 0; - U32 w; - for (w=1; w<maxW+1; w++) { - U32 current = nextRankVal; - nextRankVal += rankStats[w] << (w+rescale); - rankVal0[w] = current; - } } - { U32 const minBits = tableLog+1 - maxW; - U32 consumed; - for (consumed = minBits; consumed < maxTableLog - minBits + 1; consumed++) { - U32* const rankValPtr = rankVal[consumed]; - U32 w; - for (w = 1; w < maxW+1; w++) { - rankValPtr[w] = rankVal0[w] >> consumed; - } } } } - - HUFv08_fillDTableX4(dt, maxTableLog, - sortedSymbol, sizeOfSort, - rankStart0, rankVal, maxW, - tableLog+1); - - dtd.tableLog = (BYTE)maxTableLog; - dtd.tableType = 1; - memcpy(DTable, &dtd, sizeof(dtd)); - return iSize; -} - - -static U32 HUFv08_decodeSymbolX4(void* op, BIT_DStream_t* DStream, const HUFv08_DEltX4* dt, const U32 dtLog) -{ - const size_t val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ - memcpy(op, dt+val, 2); - BIT_skipBits(DStream, dt[val].nbBits); - return dt[val].length; -} - -static U32 HUFv08_decodeLastSymbolX4(void* op, BIT_DStream_t* DStream, const HUFv08_DEltX4* dt, const U32 dtLog) -{ - const size_t val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ - memcpy(op, dt+val, 1); - if (dt[val].length==1) BIT_skipBits(DStream, dt[val].nbBits); - else { - if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8)) { - BIT_skipBits(DStream, dt[val].nbBits); - if (DStream->bitsConsumed > (sizeof(DStream->bitContainer)*8)) - DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8); /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */ - } } - return 1; -} - - -#define HUFv08_DECODE_SYMBOLX4_0(ptr, DStreamPtr) \ - ptr += HUFv08_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog) - -#define HUFv08_DECODE_SYMBOLX4_1(ptr, DStreamPtr) \ - if (MEM_64bits() || (HUFv08_TABLELOG_MAX<=12)) \ - ptr += HUFv08_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog) - -#define HUFv08_DECODE_SYMBOLX4_2(ptr, DStreamPtr) \ - if (MEM_64bits()) \ - ptr += HUFv08_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog) - -static inline size_t HUFv08_decodeStreamX4(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, const HUFv08_DEltX4* const dt, const U32 dtLog) -{ - BYTE* const pStart = p; - - /* up to 8 symbols at a time */ - while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p < pEnd-7)) { - HUFv08_DECODE_SYMBOLX4_2(p, bitDPtr); - HUFv08_DECODE_SYMBOLX4_1(p, bitDPtr); - HUFv08_DECODE_SYMBOLX4_2(p, bitDPtr); - HUFv08_DECODE_SYMBOLX4_0(p, bitDPtr); - } - - /* closer to end : up to 2 symbols at a time */ - while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p <= pEnd-2)) - HUFv08_DECODE_SYMBOLX4_0(p, bitDPtr); - - while (p <= pEnd-2) - HUFv08_DECODE_SYMBOLX4_0(p, bitDPtr); /* no need to reload : reached the end of DStream */ - - if (p < pEnd) - p += HUFv08_decodeLastSymbolX4(p, bitDPtr, dt, dtLog); - - return p-pStart; -} - - -static size_t HUFv08_decompress1X4_usingDTable_internal( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUFv08_DTable* DTable) -{ - BIT_DStream_t bitD; - - /* Init */ - { size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize); - if (HUFv08_isError(errorCode)) return errorCode; - } - - /* decode */ - { BYTE* const ostart = (BYTE*) dst; - BYTE* const oend = ostart + dstSize; - const void* const dtPtr = DTable+1; /* force compiler to not use strict-aliasing */ - const HUFv08_DEltX4* const dt = (const HUFv08_DEltX4*)dtPtr; - DTableDesc const dtd = HUFv08_getDTableDesc(DTable); - HUFv08_decodeStreamX4(ostart, &bitD, oend, dt, dtd.tableLog); - } - - /* check */ - if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); - - /* decoded size */ - return dstSize; -} - -size_t HUFv08_decompress1X4_usingDTable( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUFv08_DTable* DTable) -{ - DTableDesc dtd = HUFv08_getDTableDesc(DTable); - if (dtd.tableType != 1) return ERROR(GENERIC); - return HUFv08_decompress1X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); -} - -size_t HUFv08_decompress1X4_DCtx (HUFv08_DTable* DCtx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - const BYTE* ip = (const BYTE*) cSrc; - - size_t const hSize = HUFv08_readDTableX4 (DCtx, cSrc, cSrcSize); - if (HUFv08_isError(hSize)) return hSize; - if (hSize >= cSrcSize) return ERROR(srcSize_wrong); - ip += hSize; cSrcSize -= hSize; - - return HUFv08_decompress1X4_usingDTable_internal (dst, dstSize, ip, cSrcSize, DCtx); -} - -size_t HUFv08_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - HUFv08_CREATE_STATIC_DTABLEX4(DTable, HUFv08_TABLELOG_MAX); - return HUFv08_decompress1X4_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); -} - -static size_t HUFv08_decompress4X4_usingDTable_internal( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUFv08_DTable* DTable) -{ - if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ - - { const BYTE* const istart = (const BYTE*) cSrc; - BYTE* const ostart = (BYTE*) dst; - BYTE* const oend = ostart + dstSize; - const void* const dtPtr = DTable+1; - const HUFv08_DEltX4* const dt = (const HUFv08_DEltX4*)dtPtr; - - /* Init */ - BIT_DStream_t bitD1; - BIT_DStream_t bitD2; - BIT_DStream_t bitD3; - BIT_DStream_t bitD4; - size_t const length1 = MEM_readLE16(istart); - size_t const length2 = MEM_readLE16(istart+2); - size_t const length3 = MEM_readLE16(istart+4); - size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); - const BYTE* const istart1 = istart + 6; /* jumpTable */ - const BYTE* const istart2 = istart1 + length1; - const BYTE* const istart3 = istart2 + length2; - const BYTE* const istart4 = istart3 + length3; - size_t const segmentSize = (dstSize+3) / 4; - BYTE* const opStart2 = ostart + segmentSize; - BYTE* const opStart3 = opStart2 + segmentSize; - BYTE* const opStart4 = opStart3 + segmentSize; - BYTE* op1 = ostart; - BYTE* op2 = opStart2; - BYTE* op3 = opStart3; - BYTE* op4 = opStart4; - U32 endSignal; - DTableDesc const dtd = HUFv08_getDTableDesc(DTable); - U32 const dtLog = dtd.tableLog; - - if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ - { size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1); - if (HUFv08_isError(errorCode)) return errorCode; } - { size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2); - if (HUFv08_isError(errorCode)) return errorCode; } - { size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3); - if (HUFv08_isError(errorCode)) return errorCode; } - { size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4); - if (HUFv08_isError(errorCode)) return errorCode; } - - /* 16-32 symbols per loop (4-8 symbols per stream) */ - endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); - for ( ; (endSignal==BIT_DStream_unfinished) && (op4<(oend-7)) ; ) { - HUFv08_DECODE_SYMBOLX4_2(op1, &bitD1); - HUFv08_DECODE_SYMBOLX4_2(op2, &bitD2); - HUFv08_DECODE_SYMBOLX4_2(op3, &bitD3); - HUFv08_DECODE_SYMBOLX4_2(op4, &bitD4); - HUFv08_DECODE_SYMBOLX4_1(op1, &bitD1); - HUFv08_DECODE_SYMBOLX4_1(op2, &bitD2); - HUFv08_DECODE_SYMBOLX4_1(op3, &bitD3); - HUFv08_DECODE_SYMBOLX4_1(op4, &bitD4); - HUFv08_DECODE_SYMBOLX4_2(op1, &bitD1); - HUFv08_DECODE_SYMBOLX4_2(op2, &bitD2); - HUFv08_DECODE_SYMBOLX4_2(op3, &bitD3); - HUFv08_DECODE_SYMBOLX4_2(op4, &bitD4); - HUFv08_DECODE_SYMBOLX4_0(op1, &bitD1); - HUFv08_DECODE_SYMBOLX4_0(op2, &bitD2); - HUFv08_DECODE_SYMBOLX4_0(op3, &bitD3); - HUFv08_DECODE_SYMBOLX4_0(op4, &bitD4); - - endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); - } - - /* check corruption */ - if (op1 > opStart2) return ERROR(corruption_detected); - if (op2 > opStart3) return ERROR(corruption_detected); - if (op3 > opStart4) return ERROR(corruption_detected); - /* note : op4 supposed already verified within main loop */ - - /* finish bitStreams one by one */ - HUFv08_decodeStreamX4(op1, &bitD1, opStart2, dt, dtLog); - HUFv08_decodeStreamX4(op2, &bitD2, opStart3, dt, dtLog); - HUFv08_decodeStreamX4(op3, &bitD3, opStart4, dt, dtLog); - HUFv08_decodeStreamX4(op4, &bitD4, oend, dt, dtLog); - - /* check */ - { U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); - if (!endCheck) return ERROR(corruption_detected); } - - /* decoded size */ - return dstSize; - } -} - - -size_t HUFv08_decompress4X4_usingDTable( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUFv08_DTable* DTable) -{ - DTableDesc dtd = HUFv08_getDTableDesc(DTable); - if (dtd.tableType != 1) return ERROR(GENERIC); - return HUFv08_decompress4X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); -} - - -size_t HUFv08_decompress4X4_DCtx (HUFv08_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - const BYTE* ip = (const BYTE*) cSrc; - - size_t hSize = HUFv08_readDTableX4 (dctx, cSrc, cSrcSize); - if (HUFv08_isError(hSize)) return hSize; - if (hSize >= cSrcSize) return ERROR(srcSize_wrong); - ip += hSize; cSrcSize -= hSize; - - return HUFv08_decompress4X4_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx); -} - -size_t HUFv08_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - HUFv08_CREATE_STATIC_DTABLEX4(DTable, HUFv08_TABLELOG_MAX); - return HUFv08_decompress4X4_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); -} - - -/* ********************************/ -/* Generic decompression selector */ -/* ********************************/ - -size_t HUFv08_decompress1X_usingDTable(void* dst, size_t maxDstSize, - const void* cSrc, size_t cSrcSize, - const HUFv08_DTable* DTable) -{ - DTableDesc const dtd = HUFv08_getDTableDesc(DTable); - return dtd.tableType ? HUFv08_decompress1X4_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable) : - HUFv08_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable); -} - -size_t HUFv08_decompress4X_usingDTable(void* dst, size_t maxDstSize, - const void* cSrc, size_t cSrcSize, - const HUFv08_DTable* DTable) -{ - DTableDesc const dtd = HUFv08_getDTableDesc(DTable); - return dtd.tableType ? HUFv08_decompress4X4_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable) : - HUFv08_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable); -} - - -typedef struct { U32 tableTime; U32 decode256Time; } algo_time_t; -static const algo_time_t algoTime[16 /* Quantization */][3 /* single, double, quad */] = -{ - /* single, double, quad */ - {{0,0}, {1,1}, {2,2}}, /* Q==0 : impossible */ - {{0,0}, {1,1}, {2,2}}, /* Q==1 : impossible */ - {{ 38,130}, {1313, 74}, {2151, 38}}, /* Q == 2 : 12-18% */ - {{ 448,128}, {1353, 74}, {2238, 41}}, /* Q == 3 : 18-25% */ - {{ 556,128}, {1353, 74}, {2238, 47}}, /* Q == 4 : 25-32% */ - {{ 714,128}, {1418, 74}, {2436, 53}}, /* Q == 5 : 32-38% */ - {{ 883,128}, {1437, 74}, {2464, 61}}, /* Q == 6 : 38-44% */ - {{ 897,128}, {1515, 75}, {2622, 68}}, /* Q == 7 : 44-50% */ - {{ 926,128}, {1613, 75}, {2730, 75}}, /* Q == 8 : 50-56% */ - {{ 947,128}, {1729, 77}, {3359, 77}}, /* Q == 9 : 56-62% */ - {{1107,128}, {2083, 81}, {4006, 84}}, /* Q ==10 : 62-69% */ - {{1177,128}, {2379, 87}, {4785, 88}}, /* Q ==11 : 69-75% */ - {{1242,128}, {2415, 93}, {5155, 84}}, /* Q ==12 : 75-81% */ - {{1349,128}, {2644,106}, {5260,106}}, /* Q ==13 : 81-87% */ - {{1455,128}, {2422,124}, {4174,124}}, /* Q ==14 : 87-93% */ - {{ 722,128}, {1891,145}, {1936,146}}, /* Q ==15 : 93-99% */ -}; - -/** HUFv08_selectDecoder() : -* Tells which decoder is likely to decode faster, -* based on a set of pre-determined metrics. -* @return : 0==HUFv08_decompress4X2, 1==HUFv08_decompress4X4 . -* Assumption : 0 < cSrcSize < dstSize <= 128 KB */ -U32 HUFv08_selectDecoder (size_t dstSize, size_t cSrcSize) -{ - /* decoder timing evaluation */ - U32 const Q = (U32)(cSrcSize * 16 / dstSize); /* Q < 16 since dstSize > cSrcSize */ - U32 const D256 = (U32)(dstSize >> 8); - U32 const DTime0 = algoTime[Q][0].tableTime + (algoTime[Q][0].decode256Time * D256); - U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256); - DTime1 += DTime1 >> 3; /* advantage to algorithm using less memory, for cache eviction */ - - return DTime1 < DTime0; -} - - -typedef size_t (*decompressionAlgo)(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); - -size_t HUFv08_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - static const decompressionAlgo decompress[2] = { HUFv08_decompress4X2, HUFv08_decompress4X4 }; - - /* validation checks */ - if (dstSize == 0) return ERROR(dstSize_tooSmall); - if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ - if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ - if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ - - { U32 const algoNb = HUFv08_selectDecoder(dstSize, cSrcSize); - return decompress[algoNb](dst, dstSize, cSrc, cSrcSize); - } - - //return HUFv08_decompress4X2(dst, dstSize, cSrc, cSrcSize); /* multi-streams single-symbol decoding */ - //return HUFv08_decompress4X4(dst, dstSize, cSrc, cSrcSize); /* multi-streams double-symbols decoding */ -} - -size_t HUFv08_decompress4X_DCtx (HUFv08_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - /* validation checks */ - if (dstSize == 0) return ERROR(dstSize_tooSmall); - if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ - if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ - if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ - - { U32 const algoNb = HUFv08_selectDecoder(dstSize, cSrcSize); - return algoNb ? HUFv08_decompress4X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) : - HUFv08_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ; - } -} - -size_t HUFv08_decompress4X_hufOnly (HUFv08_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - /* validation checks */ - if (dstSize == 0) return ERROR(dstSize_tooSmall); - if ((cSrcSize >= dstSize) || (cSrcSize <= 1)) return ERROR(corruption_detected); /* invalid */ - - { U32 const algoNb = HUFv08_selectDecoder(dstSize, cSrcSize); - return algoNb ? HUFv08_decompress4X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) : - HUFv08_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ; - } -} - -size_t HUFv08_decompress1X_DCtx (HUFv08_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - /* validation checks */ - if (dstSize == 0) return ERROR(dstSize_tooSmall); - if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ - if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ - if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ - - { U32 const algoNb = HUFv08_selectDecoder(dstSize, cSrcSize); - return algoNb ? HUFv08_decompress1X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) : - HUFv08_decompress1X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ; - } -} -/* - zstd_internal - common functions to include - Header File for include - Copyright (C) 2014-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - zstd homepage : https://www.zstd.net -*/ -#ifndef ZSTDv08_CCOMMON_H_MODULE -#define ZSTDv08_CCOMMON_H_MODULE - - -/*-************************************* -* Common macros -***************************************/ -#define MIN(a,b) ((a)<(b) ? (a) : (b)) -#define MAX(a,b) ((a)>(b) ? (a) : (b)) - - -/*-************************************* -* Common constants -***************************************/ -#define ZSTDv08_OPT_DEBUG 0 /* 3 = compression stats; 5 = check encoded sequences; 9 = full logs */ -#if defined(ZSTDv08_OPT_DEBUG) && ZSTDv08_OPT_DEBUG>=9 - #include <stdio.h> - #include <stdlib.h> - #define ZSTDv08_LOG_PARSER(...) printf(__VA_ARGS__) - #define ZSTDv08_LOG_ENCODE(...) printf(__VA_ARGS__) - #define ZSTDv08_LOG_BLOCK(...) printf(__VA_ARGS__) -#else - #define ZSTDv08_LOG_PARSER(...) - #define ZSTDv08_LOG_ENCODE(...) - #define ZSTDv08_LOG_BLOCK(...) -#endif - -#define ZSTDv08_OPT_NUM (1<<12) -#define ZSTDv08_DICT_MAGIC 0xEC30A437 /* v0.7+ */ - -#define ZSTDv08_REP_NUM 3 /* number of repcodes */ -#define ZSTDv08_REP_CHECK (ZSTDv08_REP_NUM-0) /* number of repcodes to check by the optimal parser */ -#define ZSTDv08_REP_MOVE (ZSTDv08_REP_NUM-1) -static const U32 repStartValue[ZSTDv08_REP_NUM] = { 1, 4, 8 }; - -#define KB *(1 <<10) -#define MB *(1 <<20) -#define GB *(1U<<30) - -#define BIT7 128 -#define BIT6 64 -#define BIT5 32 -#define BIT4 16 -#define BIT1 2 -#define BIT0 1 - -#define ZSTDv08_WINDOWLOG_ABSOLUTEMIN 10 -static const size_t ZSTDv08_fcs_fieldSize[4] = { 0, 2, 4, 8 }; -static const size_t ZSTDv08_did_fieldSize[4] = { 0, 1, 2, 4 }; - -#define ZSTDv08_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */ -static const size_t ZSTDv08_blockHeaderSize = ZSTDv08_BLOCKHEADERSIZE; -typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e; - -#define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */ -#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */ + MIN_SEQUENCES_SIZE /* nbSeq==0 */) /* for a non-null block */ - -#define HufLog 12 -typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingType_e; - -#define LONGNBSEQ 0x7F00 - -#define MINMATCH 3 -#define EQUAL_READ32 4 - -#define Litbits 8 -#define MaxLit ((1<<Litbits) - 1) -#define MaxML 52 -#define MaxLL 35 -#define MaxOff 28 -#define MaxSeq MAX(MaxLL, MaxML) /* Assumption : MaxOff < MaxLL,MaxML */ -#define MLFSELog 9 -#define LLFSELog 9 -#define OffFSELog 8 - -static const U32 LL_bits[MaxLL+1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 1, 1, 1, 2, 2, 3, 3, 4, 6, 7, 8, 9,10,11,12, - 13,14,15,16 }; -static const S16 LL_defaultNorm[MaxLL+1] = { 4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1, - -1,-1,-1,-1 }; -static const U32 LL_defaultNormLog = 6; - -static const U32 ML_bits[MaxML+1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 7, 8, 9,10,11, - 12,13,14,15,16 }; -static const S16 ML_defaultNorm[MaxML+1] = { 1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,-1,-1, - -1,-1,-1,-1,-1 }; -static const U32 ML_defaultNormLog = 6; - -static const S16 OF_defaultNorm[MaxOff+1] = { 1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1,-1,-1,-1,-1,-1 }; -static const U32 OF_defaultNormLog = 5; - - -/*-******************************************* -* Shared functions to include for inlining -*********************************************/ -static void ZSTDv08_copy8(void* dst, const void* src) { memcpy(dst, src, 8); } -#define COPY8(d,s) { ZSTDv08_copy8(d,s); d+=8; s+=8; } - -/*! ZSTDv08_wildcopy() : -* custom version of memcpy(), can copy up to 7 bytes too many (8 bytes if length==0) */ -#define WILDCOPY_OVERLENGTH 8 -MEM_STATIC void ZSTDv08_wildcopy(void* dst, const void* src, size_t length) -{ - const BYTE* ip = (const BYTE*)src; - BYTE* op = (BYTE*)dst; - BYTE* const oend = op + length; - do - COPY8(op, ip) - while (op < oend); -} - - -/*-******************************************* -* Private interfaces -*********************************************/ -typedef struct ZSTDv08_stats_s ZSTDv08_stats_t; - -typedef struct { - U32 off; - U32 len; -} ZSTDv08_match_t; - -typedef struct { - U32 price; - U32 off; - U32 mlen; - U32 litlen; - U32 rep[ZSTDv08_REP_NUM]; -} ZSTDv08_optimal_t; - -//#if ZSTDv08_OPT_DEBUG == 3 -// #include ".debug/zstd_stats.h" -//#else - struct ZSTDv08_stats_s { U32 unused; }; - MEM_STATIC void ZSTDv08_statsPrint(ZSTDv08_stats_t* stats, U32 searchLength) { (void)stats; (void)searchLength; } - MEM_STATIC void ZSTDv08_statsInit(ZSTDv08_stats_t* stats) { (void)stats; } - MEM_STATIC void ZSTDv08_statsResetFreqs(ZSTDv08_stats_t* stats) { (void)stats; } - MEM_STATIC void ZSTDv08_statsUpdatePrices(ZSTDv08_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) { (void)stats; (void)litLength; (void)literals; (void)offset; (void)matchLength; } -//#endif /* #if ZSTDv08_OPT_DEBUG == 3 */ - - -typedef struct seqDef_s { - U32 offset; - U16 litLength; - U16 matchLength; -} seqDef; - - -typedef struct { - seqDef* sequencesStart; - seqDef* sequences; - BYTE* litStart; - BYTE* lit; - BYTE* llCode; - BYTE* mlCode; - BYTE* ofCode; - U32 longLengthID; /* 0 == no longLength; 1 == Lit.longLength; 2 == Match.longLength; */ - U32 longLengthPos; - /* opt */ - ZSTDv08_optimal_t* priceTable; - ZSTDv08_match_t* matchTable; - U32* matchLengthFreq; - U32* litLengthFreq; - U32* litFreq; - U32* offCodeFreq; - U32 matchLengthSum; - U32 matchSum; - U32 litLengthSum; - U32 litSum; - U32 offCodeSum; - U32 log2matchLengthSum; - U32 log2matchSum; - U32 log2litLengthSum; - U32 log2litSum; - U32 log2offCodeSum; - U32 factor; - U32 cachedPrice; - U32 cachedLitLength; - const BYTE* cachedLiterals; - ZSTDv08_stats_t stats; -} seqStore_t; - -void ZSTDv08_seqToCodes(const seqStore_t* seqStorePtr); -int ZSTDv08_isSkipFrame(ZSTDv08_DCtx* dctx); - -/* custom memory allocation functions */ -void* ZSTDv08_defaultAllocFunction(void* opaque, size_t size); -void ZSTDv08_defaultFreeFunction(void* opaque, void* address); -static const ZSTDv08_customMem defaultCustomMem = { ZSTDv08_defaultAllocFunction, ZSTDv08_defaultFreeFunction, NULL }; - -/*====== common function ======*/ - -MEM_STATIC U32 ZSTDv08_highbit32(U32 val) -{ -# if defined(_MSC_VER) /* Visual */ - unsigned long r=0; - _BitScanReverse(&r, val); - return (unsigned)r; -# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */ - return 31 - __builtin_clz(val); -# else /* Software version */ - static const int DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 }; - U32 v = val; - int r; - v |= v >> 1; - v |= v >> 2; - v |= v >> 4; - v |= v >> 8; - v |= v >> 16; - r = DeBruijnClz[(U32)(v * 0x07C4ACDDU) >> 27]; - return r; -# endif -} - - -#endif /* ZSTDv08_CCOMMON_H_MODULE */ -/* - zstd - standard compression library - Copyright (C) 2014-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - zstd homepage : http://www.zstd.net -*/ - -/* *************************************************************** -* Tuning parameters -*****************************************************************/ -/*! - * HEAPMODE : - * Select how default decompression function ZSTDv08_decompress() will allocate memory, - * in memory stack (0), or in memory heap (1, requires malloc()) - */ -#ifndef ZSTDv08_HEAPMODE -# define ZSTDv08_HEAPMODE 1 -#endif - - - -/*-******************************************************* -* Compiler specifics -*********************************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# define FORCE_INLINE static __forceinline -# include <intrin.h> /* For Visual 2005 */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -# pragma warning(disable : 4324) /* disable: C4324: padded structure */ -#else -# ifdef __GNUC__ -# define FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define FORCE_INLINE static inline -# endif -#endif - - -/*-************************************* -* Macros -***************************************/ -#define ZSTDv08_isError ERR_isError /* for inlining */ -#define FSEv08_isError ERR_isError -#define HUFv08_isError ERR_isError - - -/*_******************************************************* -* Memory operations -**********************************************************/ -static void ZSTDv08_copy4(void* dst, const void* src) { memcpy(dst, src, 4); } - - -/*-************************************************************* -* Context management -***************************************************************/ -typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader, - ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock, - ZSTDds_decompressLastBlock, ZSTDds_checkChecksum, - ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTDv08_dStage; - -struct ZSTDv08_DCtx_s -{ - FSEv08_DTable LLTable[FSEv08_DTABLE_SIZE_U32(LLFSELog)]; - FSEv08_DTable OffTable[FSEv08_DTABLE_SIZE_U32(OffFSELog)]; - FSEv08_DTable MLTable[FSEv08_DTABLE_SIZE_U32(MLFSELog)]; - HUFv08_DTable hufTable[HUFv08_DTABLE_SIZE(HufLog)]; /* can accommodate HUFv08_decompress4X */ - const void* previousDstEnd; - const void* base; - const void* vBase; - const void* dictEnd; - size_t expected; - U32 rep[ZSTDv08_REP_NUM]; - ZSTDv08_frameParams fParams; - blockType_e bType; /* used in ZSTDv08_decompressContinue(), to transfer blockType between header decoding and block decoding stages */ - ZSTDv08_dStage stage; - U32 litEntropy; - U32 fseEntropy; - XXH64_state_t xxhState; - size_t headerSize; - U32 dictID; - const BYTE* litPtr; - ZSTDv08_customMem customMem; - size_t litBufSize; - size_t litSize; - size_t rleSize; - BYTE litBuffer[ZSTDv08_BLOCKSIZE_ABSOLUTEMAX + WILDCOPY_OVERLENGTH]; - BYTE headerBuffer[ZSTDv08_FRAMEHEADERSIZE_MAX]; -}; /* typedef'd to ZSTDv08_DCtx within "zstd_static.h" */ - -size_t ZSTDv08_sizeofDCtx (const ZSTDv08_DCtx* dctx) { return sizeof(*dctx); } - -size_t ZSTDv08_estimateDCtxSize(void) { return sizeof(ZSTDv08_DCtx); } - -size_t ZSTDv08_decompressBegin(ZSTDv08_DCtx* dctx) -{ - dctx->expected = ZSTDv08_frameHeaderSize_min; - dctx->stage = ZSTDds_getFrameHeaderSize; - dctx->previousDstEnd = NULL; - dctx->base = NULL; - dctx->vBase = NULL; - dctx->dictEnd = NULL; - dctx->hufTable[0] = (HUFv08_DTable)((HufLog)*0x1000001); - dctx->litEntropy = dctx->fseEntropy = 0; - dctx->dictID = 0; - { int i; for (i=0; i<ZSTDv08_REP_NUM; i++) dctx->rep[i] = repStartValue[i]; } - return 0; -} - -ZSTDv08_DCtx* ZSTDv08_createDCtx_advanced(ZSTDv08_customMem customMem) -{ - ZSTDv08_DCtx* dctx; - - if (!customMem.customAlloc && !customMem.customFree) - customMem = defaultCustomMem; - - if (!customMem.customAlloc || !customMem.customFree) - return NULL; - - dctx = (ZSTDv08_DCtx*) customMem.customAlloc(customMem.opaque, sizeof(ZSTDv08_DCtx)); - if (!dctx) return NULL; - memcpy(&dctx->customMem, &customMem, sizeof(ZSTDv08_customMem)); - ZSTDv08_decompressBegin(dctx); - return dctx; -} - -ZSTDv08_DCtx* ZSTDv08_createDCtx(void) -{ - return ZSTDv08_createDCtx_advanced(defaultCustomMem); -} - -size_t ZSTDv08_freeDCtx(ZSTDv08_DCtx* dctx) -{ - if (dctx==NULL) return 0; /* support free on NULL */ - dctx->customMem.customFree(dctx->customMem.opaque, dctx); - return 0; /* reserved as a potential error code in the future */ -} - -void ZSTDv08_copyDCtx(ZSTDv08_DCtx* dstDCtx, const ZSTDv08_DCtx* srcDCtx) -{ - memcpy(dstDCtx, srcDCtx, - sizeof(ZSTDv08_DCtx) - (ZSTDv08_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH + ZSTDv08_frameHeaderSize_max)); /* no need to copy workspace */ -} - - -/*-************************************************************* -* Decompression section -***************************************************************/ - -/* See compression format details in : zstd_compression_format.md */ - -/** ZSTDv08_frameHeaderSize() : -* srcSize must be >= ZSTDv08_frameHeaderSize_min. -* @return : size of the Frame Header */ -static size_t ZSTDv08_frameHeaderSize(const void* src, size_t srcSize) -{ - if (srcSize < ZSTDv08_frameHeaderSize_min) return ERROR(srcSize_wrong); - { BYTE const fhd = ((const BYTE*)src)[4]; - U32 const dictID= fhd & 3; - U32 const singleSegment = (fhd >> 5) & 1; - U32 const fcsId = fhd >> 6; - return ZSTDv08_frameHeaderSize_min + !singleSegment + ZSTDv08_did_fieldSize[dictID] + ZSTDv08_fcs_fieldSize[fcsId] - + (singleSegment && !ZSTDv08_fcs_fieldSize[fcsId]); - } -} - - -/** ZSTDv08_getFrameParams() : -* decode Frame Header, or require larger `srcSize`. -* @return : 0, `fparamsPtr` is correctly filled, -* >0, `srcSize` is too small, result is expected `srcSize`, -* or an error code, which can be tested using ZSTDv08_isError() */ -size_t ZSTDv08_getFrameParams(ZSTDv08_frameParams* fparamsPtr, const void* src, size_t srcSize) -{ - const BYTE* ip = (const BYTE*)src; - - if (srcSize < ZSTDv08_frameHeaderSize_min) return ZSTDv08_frameHeaderSize_min; - if (MEM_readLE32(src) != ZSTDv08_MAGICNUMBER) { - if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTDv08_MAGIC_SKIPPABLE_START) { - if (srcSize < ZSTDv08_skippableHeaderSize) return ZSTDv08_skippableHeaderSize; /* magic number + skippable frame length */ - memset(fparamsPtr, 0, sizeof(*fparamsPtr)); - fparamsPtr->frameContentSize = MEM_readLE32((const char *)src + 4); - fparamsPtr->windowSize = 0; /* windowSize==0 means a frame is skippable */ - return 0; - } - return ERROR(prefix_unknown); - } - - /* ensure there is enough `srcSize` to fully read/decode frame header */ - { size_t const fhsize = ZSTDv08_frameHeaderSize(src, srcSize); - if (srcSize < fhsize) return fhsize; } - - { BYTE const fhdByte = ip[4]; - size_t pos = 5; - U32 const dictIDSizeCode = fhdByte&3; - U32 const checksumFlag = (fhdByte>>2)&1; - U32 const singleSegment = (fhdByte>>5)&1; - U32 const fcsID = fhdByte>>6; - U32 const windowSizeMax = 1U << ZSTDv08_WINDOWLOG_MAX; - U32 windowSize = 0; - U32 dictID = 0; - U64 frameContentSize = 0; - if ((fhdByte & 0x08) != 0) return ERROR(frameParameter_unsupported); /* reserved bits, which must be zero */ - if (!singleSegment) { - BYTE const wlByte = ip[pos++]; - U32 const windowLog = (wlByte >> 3) + ZSTDv08_WINDOWLOG_ABSOLUTEMIN; - if (windowLog > ZSTDv08_WINDOWLOG_MAX) return ERROR(frameParameter_unsupported); - windowSize = (1U << windowLog); - windowSize += (windowSize >> 3) * (wlByte&7); - } - - switch(dictIDSizeCode) - { - default: /* impossible */ - case 0 : break; - case 1 : dictID = ip[pos]; pos++; break; - case 2 : dictID = MEM_readLE16(ip+pos); pos+=2; break; - case 3 : dictID = MEM_readLE32(ip+pos); pos+=4; break; - } - switch(fcsID) - { - default: /* impossible */ - case 0 : if (singleSegment) frameContentSize = ip[pos]; break; - case 1 : frameContentSize = MEM_readLE16(ip+pos)+256; break; - case 2 : frameContentSize = MEM_readLE32(ip+pos); break; - case 3 : frameContentSize = MEM_readLE64(ip+pos); break; - } - if (!windowSize) windowSize = (U32)frameContentSize; - if (windowSize > windowSizeMax) return ERROR(frameParameter_unsupported); - fparamsPtr->frameContentSize = frameContentSize; - fparamsPtr->windowSize = windowSize; - fparamsPtr->dictID = dictID; - fparamsPtr->checksumFlag = checksumFlag; - } - return 0; -} - - -/** ZSTDv08_getDecompressedSize() : -* compatible with legacy mode -* @return : decompressed size if known, 0 otherwise - note : 0 can mean any of the following : - - decompressed size is not present within frame header - - frame header unknown / not supported - - frame header not complete (`srcSize` too small) */ -unsigned long long ZSTDv08_getDecompressedSize(const void* src, size_t srcSize) -{ - { ZSTDv08_frameParams fparams; - size_t const frResult = ZSTDv08_getFrameParams(&fparams, src, srcSize); - if (frResult!=0) return 0; - return fparams.frameContentSize; - } -} - - -/** ZSTDv08_decodeFrameHeader() : -* `srcSize` must be the size provided by ZSTDv08_frameHeaderSize(). -* @return : 0 if success, or an error code, which can be tested using ZSTDv08_isError() */ -static size_t ZSTDv08_decodeFrameHeader(ZSTDv08_DCtx* dctx, const void* src, size_t srcSize) -{ - size_t const result = ZSTDv08_getFrameParams(&(dctx->fParams), src, srcSize); - if (dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID)) return ERROR(dictionary_wrong); - if (dctx->fParams.checksumFlag) XXH64_reset(&dctx->xxhState, 0); - return result; -} - - -typedef struct -{ - blockType_e blockType; - U32 lastBlock; - U32 origSize; -} blockProperties_t; - -/*! ZSTDv08_getcBlockSize() : -* Provides the size of compressed block from block header `src` */ -size_t ZSTDv08_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr) -{ - if (srcSize < ZSTDv08_blockHeaderSize) return ERROR(srcSize_wrong); - { U32 const cBlockHeader = MEM_readLE24(src); - U32 const cSize = cBlockHeader >> 3; - bpPtr->lastBlock = cBlockHeader & 1; - bpPtr->blockType = (blockType_e)((cBlockHeader >> 1) & 3); - bpPtr->origSize = cSize; /* only useful for RLE */ - if (bpPtr->blockType == bt_rle) return 1; - if (bpPtr->blockType == bt_reserved) return ERROR(corruption_detected); - return cSize; - } -} - - -static size_t ZSTDv08_copyRawBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize) -{ - if (srcSize > dstCapacity) return ERROR(dstSize_tooSmall); - memcpy(dst, src, srcSize); - return srcSize; -} - - -static size_t ZSTDv08_setRleBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize, size_t regenSize) -{ - if (srcSize != 1) return ERROR(srcSize_wrong); - if (regenSize > dstCapacity) return ERROR(dstSize_tooSmall); - memset(dst, *(const BYTE*)src, regenSize); - return regenSize; -} - -/*! ZSTDv08_decodeLiteralsBlock() : - @return : nb of bytes read from src (< srcSize ) */ -size_t ZSTDv08_decodeLiteralsBlock(ZSTDv08_DCtx* dctx, - const void* src, size_t srcSize) /* note : srcSize < BLOCKSIZE */ -{ - if (srcSize < MIN_CBLOCK_SIZE) return ERROR(corruption_detected); - - { const BYTE* const istart = (const BYTE*) src; - symbolEncodingType_e const litEncType = (symbolEncodingType_e)(istart[0] & 3); - - switch(litEncType) - { - case set_repeat: - if (dctx->litEntropy==0) return ERROR(dictionary_corrupted); - /* fall-through */ - case set_compressed: - if (srcSize < 5) return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for case 3 */ - { size_t lhSize, litSize, litCSize; - U32 singleStream=0; - U32 const lhlCode = (istart[0] >> 2) & 3; - U32 const lhc = MEM_readLE32(istart); - switch(lhlCode) - { - case 0: case 1: default: /* note : default is impossible, since lhlCode into [0..3] */ - /* 2 - 2 - 10 - 10 */ - { singleStream = !lhlCode; - lhSize = 3; - litSize = (lhc >> 4) & 0x3FF; - litCSize = (lhc >> 14) & 0x3FF; - break; - } - case 2: - /* 2 - 2 - 14 - 14 */ - { lhSize = 4; - litSize = (lhc >> 4) & 0x3FFF; - litCSize = lhc >> 18; - break; - } - case 3: - /* 2 - 2 - 18 - 18 */ - { lhSize = 5; - litSize = (lhc >> 4) & 0x3FFFF; - litCSize = (lhc >> 22) + (istart[4] << 10); - break; - } - } - if (litSize > ZSTDv08_BLOCKSIZE_ABSOLUTEMAX) return ERROR(corruption_detected); - if (litCSize + lhSize > srcSize) return ERROR(corruption_detected); - - if (HUFv08_isError((litEncType==set_repeat) ? - ( singleStream ? - HUFv08_decompress1X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->hufTable) : - HUFv08_decompress4X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->hufTable) ) : - ( singleStream ? - HUFv08_decompress1X2_DCtx(dctx->hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize) : - HUFv08_decompress4X_hufOnly (dctx->hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize)) )) - return ERROR(corruption_detected); - - dctx->litPtr = dctx->litBuffer; - dctx->litBufSize = ZSTDv08_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH; - dctx->litSize = litSize; - dctx->litEntropy = 1; - return litCSize + lhSize; - } - - case set_basic: - { size_t litSize, lhSize; - U32 const lhlCode = ((istart[0]) >> 2) & 3; - switch(lhlCode) - { - case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */ - lhSize = 1; - litSize = istart[0] >> 3; - break; - case 1: - lhSize = 2; - litSize = MEM_readLE16(istart) >> 4; - break; - case 3: - lhSize = 3; - litSize = MEM_readLE24(istart) >> 4; - break; - } - - if (lhSize+litSize+WILDCOPY_OVERLENGTH > srcSize) { /* risk reading beyond src buffer with wildcopy */ - if (litSize+lhSize > srcSize) return ERROR(corruption_detected); - memcpy(dctx->litBuffer, istart+lhSize, litSize); - dctx->litPtr = dctx->litBuffer; - dctx->litBufSize = ZSTDv08_BLOCKSIZE_ABSOLUTEMAX+8; - dctx->litSize = litSize; - return lhSize+litSize; - } - /* direct reference into compressed stream */ - dctx->litPtr = istart+lhSize; - dctx->litBufSize = srcSize-lhSize; - dctx->litSize = litSize; - return lhSize+litSize; - } - - case set_rle: - { U32 const lhlCode = ((istart[0]) >> 2) & 3; - size_t litSize, lhSize; - switch(lhlCode) - { - case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */ - lhSize = 1; - litSize = istart[0] >> 3; - break; - case 1: - lhSize = 2; - litSize = MEM_readLE16(istart) >> 4; - break; - case 3: - lhSize = 3; - litSize = MEM_readLE24(istart) >> 4; - if (srcSize<4) return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4 */ - break; - } - if (litSize > ZSTDv08_BLOCKSIZE_ABSOLUTEMAX) return ERROR(corruption_detected); - memset(dctx->litBuffer, istart[lhSize], litSize); - dctx->litPtr = dctx->litBuffer; - dctx->litBufSize = ZSTDv08_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH; - dctx->litSize = litSize; - return lhSize+1; - } - default: - return ERROR(corruption_detected); /* impossible */ - } - - } -} - - -/*! ZSTDv08_buildSeqTable() : - @return : nb bytes read from src, - or an error code if it fails, testable with ZSTDv08_isError() -*/ -FORCE_INLINE size_t ZSTDv08_buildSeqTable(FSEv08_DTable* DTable, symbolEncodingType_e type, U32 max, U32 maxLog, - const void* src, size_t srcSize, - const S16* defaultNorm, U32 defaultLog, U32 flagRepeatTable) -{ - switch(type) - { - case set_rle : - if (!srcSize) return ERROR(srcSize_wrong); - if ( (*(const BYTE*)src) > max) return ERROR(corruption_detected); - FSEv08_buildDTable_rle(DTable, *(const BYTE*)src); /* if *src > max, data is corrupted */ - return 1; - case set_basic : - FSEv08_buildDTable(DTable, defaultNorm, max, defaultLog); - return 0; - case set_repeat: - if (!flagRepeatTable) return ERROR(corruption_detected); - return 0; - default : /* impossible */ - case set_compressed : - { U32 tableLog; - S16 norm[MaxSeq+1]; - size_t const headerSize = FSEv08_readNCount(norm, &max, &tableLog, src, srcSize); - if (FSEv08_isError(headerSize)) return ERROR(corruption_detected); - if (tableLog > maxLog) return ERROR(corruption_detected); - FSEv08_buildDTable(DTable, norm, max, tableLog); - return headerSize; - } } -} - - -size_t ZSTDv08_decodeSeqHeaders(int* nbSeqPtr, - FSEv08_DTable* DTableLL, FSEv08_DTable* DTableML, FSEv08_DTable* DTableOffb, U32 flagRepeatTable, - const void* src, size_t srcSize) -{ - const BYTE* const istart = (const BYTE* const)src; - const BYTE* const iend = istart + srcSize; - const BYTE* ip = istart; - - /* check */ - if (srcSize < MIN_SEQUENCES_SIZE) return ERROR(srcSize_wrong); - - /* SeqHead */ - { int nbSeq = *ip++; - if (!nbSeq) { *nbSeqPtr=0; return 1; } - if (nbSeq > 0x7F) { - if (nbSeq == 0xFF) - nbSeq = MEM_readLE16(ip) + LONGNBSEQ, ip+=2; - else - nbSeq = ((nbSeq-0x80)<<8) + *ip++; - } - *nbSeqPtr = nbSeq; - } - - /* FSE table descriptors */ - if (ip+4 > iend) return ERROR(srcSize_wrong); /* minimum possible size */ - { symbolEncodingType_e const LLtype = (symbolEncodingType_e)(*ip >> 6); - symbolEncodingType_e const OFtype = (symbolEncodingType_e)((*ip >> 4) & 3); - symbolEncodingType_e const MLtype = (symbolEncodingType_e)((*ip >> 2) & 3); - ip++; - - /* Build DTables */ - { size_t const llhSize = ZSTDv08_buildSeqTable(DTableLL, LLtype, MaxLL, LLFSELog, ip, iend-ip, LL_defaultNorm, LL_defaultNormLog, flagRepeatTable); - if (ZSTDv08_isError(llhSize)) return ERROR(corruption_detected); - ip += llhSize; - } - { size_t const ofhSize = ZSTDv08_buildSeqTable(DTableOffb, OFtype, MaxOff, OffFSELog, ip, iend-ip, OF_defaultNorm, OF_defaultNormLog, flagRepeatTable); - if (ZSTDv08_isError(ofhSize)) return ERROR(corruption_detected); - ip += ofhSize; - } - { size_t const mlhSize = ZSTDv08_buildSeqTable(DTableML, MLtype, MaxML, MLFSELog, ip, iend-ip, ML_defaultNorm, ML_defaultNormLog, flagRepeatTable); - if (ZSTDv08_isError(mlhSize)) return ERROR(corruption_detected); - ip += mlhSize; - } } - - return ip-istart; -} - - -typedef struct { - size_t litLength; - size_t matchLength; - size_t offset; -} seq_t; - -typedef struct { - BIT_DStream_t DStream; - FSEv08_DState_t stateLL; - FSEv08_DState_t stateOffb; - FSEv08_DState_t stateML; - size_t prevOffset[ZSTDv08_REP_NUM]; -} seqState_t; - - -static seq_t ZSTDv08_decodeSequence(seqState_t* seqState) -{ - seq_t seq; - - U32 const llCode = FSEv08_peekSymbol(&(seqState->stateLL)); - U32 const mlCode = FSEv08_peekSymbol(&(seqState->stateML)); - U32 const ofCode = FSEv08_peekSymbol(&(seqState->stateOffb)); /* <= maxOff, by table construction */ - - U32 const llBits = LL_bits[llCode]; - U32 const mlBits = ML_bits[mlCode]; - U32 const ofBits = ofCode; - U32 const totalBits = llBits+mlBits+ofBits; - - static const U32 LL_base[MaxLL+1] = { - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 16, 18, 20, 22, 24, 28, 32, 40, 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, - 0x2000, 0x4000, 0x8000, 0x10000 }; - - static const U32 ML_base[MaxML+1] = { - 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, - 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, - 35, 37, 39, 41, 43, 47, 51, 59, 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, - 0x1003, 0x2003, 0x4003, 0x8003, 0x10003 }; - - static const U32 OF_base[MaxOff+1] = { - 0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, - 0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, - 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, - 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD }; - - /* sequence */ - { size_t offset; - if (!ofCode) - offset = 0; - else { - offset = OF_base[ofCode] + BIT_readBits(&(seqState->DStream), ofBits); /* <= (ZSTDv08_WINDOWLOG_MAX-1) bits */ - if (MEM_32bits()) BIT_reloadDStream(&(seqState->DStream)); - } - - if (ofCode <= 1) { - offset += (llCode==0); - if (offset) { - size_t const temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; - if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1]; - seqState->prevOffset[1] = seqState->prevOffset[0]; - seqState->prevOffset[0] = offset = temp; - } else { - offset = seqState->prevOffset[0]; - } - } else { - seqState->prevOffset[2] = seqState->prevOffset[1]; - seqState->prevOffset[1] = seqState->prevOffset[0]; - seqState->prevOffset[0] = offset; - } - seq.offset = offset; - } - - seq.matchLength = ML_base[mlCode] + ((mlCode>31) ? BIT_readBits(&(seqState->DStream), mlBits) : 0); /* <= 16 bits */ - if (MEM_32bits() && (mlBits+llBits>24)) BIT_reloadDStream(&(seqState->DStream)); - - seq.litLength = LL_base[llCode] + ((llCode>15) ? BIT_readBits(&(seqState->DStream), llBits) : 0); /* <= 16 bits */ - if (MEM_32bits() || - (totalBits > 64 - 7 - (LLFSELog+MLFSELog+OffFSELog)) ) BIT_reloadDStream(&(seqState->DStream)); - - /* ANS state update */ - FSEv08_updateState(&(seqState->stateLL), &(seqState->DStream)); /* <= 9 bits */ - FSEv08_updateState(&(seqState->stateML), &(seqState->DStream)); /* <= 9 bits */ - if (MEM_32bits()) BIT_reloadDStream(&(seqState->DStream)); /* <= 18 bits */ - FSEv08_updateState(&(seqState->stateOffb), &(seqState->DStream)); /* <= 8 bits */ - - return seq; -} - - -FORCE_INLINE -size_t ZSTDv08_execSequence(BYTE* op, - BYTE* const oend, seq_t sequence, - const BYTE** litPtr, const BYTE* const litLimit_w, - const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd) -{ - BYTE* const oLitEnd = op + sequence.litLength; - size_t const sequenceLength = sequence.litLength + sequence.matchLength; - BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ - BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; - const BYTE* const iLitEnd = *litPtr + sequence.litLength; - const BYTE* match = oLitEnd - sequence.offset; - - /* check */ - if ((oLitEnd>oend_w) | (oMatchEnd>oend)) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */ - if (iLitEnd > litLimit_w) return ERROR(corruption_detected); /* over-read beyond lit buffer */ - - /* copy Literals */ - ZSTDv08_wildcopy(op, *litPtr, sequence.litLength); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ - op = oLitEnd; - *litPtr = iLitEnd; /* update for next sequence */ - - /* copy Match */ - if (sequence.offset > (size_t)(oLitEnd - base)) { - /* offset beyond prefix */ - if (sequence.offset > (size_t)(oLitEnd - vBase)) return ERROR(corruption_detected); - match = dictEnd - (base-match); - if (match + sequence.matchLength <= dictEnd) { - memmove(oLitEnd, match, sequence.matchLength); - return sequenceLength; - } - /* span extDict & currentPrefixSegment */ - { size_t const length1 = dictEnd - match; - memmove(oLitEnd, match, length1); - op = oLitEnd + length1; - sequence.matchLength -= length1; - match = base; - } } - - /* match within prefix */ - if (sequence.offset < 8) { - /* close range match, overlap */ - static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */ - static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* substracted */ - int const sub2 = dec64table[sequence.offset]; - op[0] = match[0]; - op[1] = match[1]; - op[2] = match[2]; - op[3] = match[3]; - match += dec32table[sequence.offset]; - ZSTDv08_copy4(op+4, match); - match -= sub2; - } else { - ZSTDv08_copy8(op, match); - } - op += 8; match += 8; - - if (oMatchEnd > oend-(16-MINMATCH)) { - if (op < oend_w) { - ZSTDv08_wildcopy(op, match, oend_w - op); - match += oend_w - op; - op = oend_w; - } - while (op < oMatchEnd) *op++ = *match++; - } else { - ZSTDv08_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */ - } - return sequenceLength; -} - - -static size_t ZSTDv08_decompressSequences( - ZSTDv08_DCtx* dctx, - void* dst, size_t maxDstSize, - const void* seqStart, size_t seqSize) -{ - const BYTE* ip = (const BYTE*)seqStart; - const BYTE* const iend = ip + seqSize; - BYTE* const ostart = (BYTE* const)dst; - BYTE* const oend = ostart + maxDstSize; - BYTE* op = ostart; - const BYTE* litPtr = dctx->litPtr; - const BYTE* const litLimit_w = litPtr + dctx->litBufSize - WILDCOPY_OVERLENGTH; - const BYTE* const litEnd = litPtr + dctx->litSize; - FSEv08_DTable* DTableLL = dctx->LLTable; - FSEv08_DTable* DTableML = dctx->MLTable; - FSEv08_DTable* DTableOffb = dctx->OffTable; - const BYTE* const base = (const BYTE*) (dctx->base); - const BYTE* const vBase = (const BYTE*) (dctx->vBase); - const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); - int nbSeq; - - /* Build Decoding Tables */ - { size_t const seqHSize = ZSTDv08_decodeSeqHeaders(&nbSeq, DTableLL, DTableML, DTableOffb, dctx->fseEntropy, ip, seqSize); - if (ZSTDv08_isError(seqHSize)) return seqHSize; - ip += seqHSize; - } - - /* Regen sequences */ - if (nbSeq) { - seqState_t seqState; - dctx->fseEntropy = 1; - { U32 i; for (i=0; i<ZSTDv08_REP_NUM; i++) seqState.prevOffset[i] = dctx->rep[i]; } - { size_t const errorCode = BIT_initDStream(&(seqState.DStream), ip, iend-ip); - if (ERR_isError(errorCode)) return ERROR(corruption_detected); } - FSEv08_initDState(&(seqState.stateLL), &(seqState.DStream), DTableLL); - FSEv08_initDState(&(seqState.stateOffb), &(seqState.DStream), DTableOffb); - FSEv08_initDState(&(seqState.stateML), &(seqState.DStream), DTableML); - - for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && nbSeq ; ) { - nbSeq--; - { seq_t const sequence = ZSTDv08_decodeSequence(&seqState); - size_t const oneSeqSize = ZSTDv08_execSequence(op, oend, sequence, &litPtr, litLimit_w, base, vBase, dictEnd); - if (ZSTDv08_isError(oneSeqSize)) return oneSeqSize; - op += oneSeqSize; - } } - - /* check if reached exact end */ - if (nbSeq) return ERROR(corruption_detected); - /* save reps for next block */ - { U32 i; for (i=0; i<ZSTDv08_REP_NUM; i++) dctx->rep[i] = (U32)(seqState.prevOffset[i]); } - } - - /* last literal segment */ - { size_t const lastLLSize = litEnd - litPtr; - if (lastLLSize > (size_t)(oend-op)) return ERROR(dstSize_tooSmall); - memcpy(op, litPtr, lastLLSize); - op += lastLLSize; - } - - return op-ostart; -} - - -static void ZSTDv08_checkContinuity(ZSTDv08_DCtx* dctx, const void* dst) -{ - if (dst != dctx->previousDstEnd) { /* not contiguous */ - dctx->dictEnd = dctx->previousDstEnd; - dctx->vBase = (const char*)dst - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->base)); - dctx->base = dst; - dctx->previousDstEnd = dst; - } -} - - -static size_t ZSTDv08_decompressBlock_internal(ZSTDv08_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize) -{ /* blockType == blockCompressed */ - const BYTE* ip = (const BYTE*)src; - - if (srcSize >= ZSTDv08_BLOCKSIZE_ABSOLUTEMAX) return ERROR(srcSize_wrong); - - /* Decode literals sub-block */ - { size_t const litCSize = ZSTDv08_decodeLiteralsBlock(dctx, src, srcSize); - if (ZSTDv08_isError(litCSize)) return litCSize; - ip += litCSize; - srcSize -= litCSize; - } - return ZSTDv08_decompressSequences(dctx, dst, dstCapacity, ip, srcSize); -} - - -size_t ZSTDv08_decompressBlock(ZSTDv08_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize) -{ - size_t dSize; - ZSTDv08_checkContinuity(dctx, dst); - dSize = ZSTDv08_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize); - dctx->previousDstEnd = (char*)dst + dSize; - return dSize; -} - - -/** ZSTDv08_insertBlock() : - insert `src` block into `dctx` history. Useful to track uncompressed blocks. */ -ZSTDLIB_API size_t ZSTDv08_insertBlock(ZSTDv08_DCtx* dctx, const void* blockStart, size_t blockSize) -{ - ZSTDv08_checkContinuity(dctx, blockStart); - dctx->previousDstEnd = (const char*)blockStart + blockSize; - return blockSize; -} - - -size_t ZSTDv08_generateNxBytes(void* dst, size_t dstCapacity, BYTE byte, size_t length) -{ - if (length > dstCapacity) return ERROR(dstSize_tooSmall); - memset(dst, byte, length); - return length; -} - - -/*! ZSTDv08_decompressFrame() : -* `dctx` must be properly initialized */ -static size_t ZSTDv08_decompressFrame(ZSTDv08_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize) -{ - const BYTE* ip = (const BYTE*)src; - BYTE* const ostart = (BYTE* const)dst; - BYTE* const oend = ostart + dstCapacity; - BYTE* op = ostart; - size_t remainingSize = srcSize; - - /* check */ - if (srcSize < ZSTDv08_frameHeaderSize_min+ZSTDv08_blockHeaderSize) return ERROR(srcSize_wrong); - - /* Frame Header */ - { size_t const frameHeaderSize = ZSTDv08_frameHeaderSize(src, ZSTDv08_frameHeaderSize_min); - size_t result; - if (ZSTDv08_isError(frameHeaderSize)) return frameHeaderSize; - if (srcSize < frameHeaderSize+ZSTDv08_blockHeaderSize) return ERROR(srcSize_wrong); - result = ZSTDv08_decodeFrameHeader(dctx, src, frameHeaderSize); - if (ZSTDv08_isError(result)) return result; - ip += frameHeaderSize; remainingSize -= frameHeaderSize; - } - - /* Loop on each block */ - while (1) { - size_t decodedSize; - blockProperties_t blockProperties; - size_t const cBlockSize = ZSTDv08_getcBlockSize(ip, remainingSize, &blockProperties); - if (ZSTDv08_isError(cBlockSize)) return cBlockSize; - - ip += ZSTDv08_blockHeaderSize; - remainingSize -= ZSTDv08_blockHeaderSize; - if (cBlockSize > remainingSize) return ERROR(srcSize_wrong); - - switch(blockProperties.blockType) - { - case bt_compressed: - decodedSize = ZSTDv08_decompressBlock_internal(dctx, op, oend-op, ip, cBlockSize); - break; - case bt_raw : - decodedSize = ZSTDv08_copyRawBlock(op, oend-op, ip, cBlockSize); - break; - case bt_rle : - decodedSize = ZSTDv08_generateNxBytes(op, oend-op, *ip, blockProperties.origSize); - break; - case bt_reserved : - default: - return ERROR(corruption_detected); - } - - if (ZSTDv08_isError(decodedSize)) return decodedSize; - if (dctx->fParams.checksumFlag) XXH64_update(&dctx->xxhState, op, decodedSize); - op += decodedSize; - ip += cBlockSize; - remainingSize -= cBlockSize; - if (blockProperties.lastBlock) break; - } - - if (dctx->fParams.checksumFlag) { /* Frame content checksum verification */ - U32 const checkCalc = (U32)XXH64_digest(&dctx->xxhState); - U32 checkRead; - if (remainingSize<4) return ERROR(checksum_wrong); - checkRead = MEM_readLE32(ip); - if (checkRead != checkCalc) return ERROR(checksum_wrong); - remainingSize -= 4; - } - - if (remainingSize) return ERROR(srcSize_wrong); - return op-ostart; -} - - -/*! ZSTDv08_decompress_usingPreparedDCtx() : -* Same as ZSTDv08_decompress_usingDict, but using a reference context `preparedDCtx`, where dictionary has been loaded. -* It avoids reloading the dictionary each time. -* `preparedDCtx` must have been properly initialized using ZSTDv08_decompressBegin_usingDict(). -* Requires 2 contexts : 1 for reference (preparedDCtx), which will not be modified, and 1 to run the decompression operation (dctx) */ -size_t ZSTDv08_decompress_usingPreparedDCtx(ZSTDv08_DCtx* dctx, const ZSTDv08_DCtx* refDCtx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize) -{ - ZSTDv08_copyDCtx(dctx, refDCtx); - ZSTDv08_checkContinuity(dctx, dst); - return ZSTDv08_decompressFrame(dctx, dst, dstCapacity, src, srcSize); -} - - -size_t ZSTDv08_decompress_usingDict(ZSTDv08_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const void* dict, size_t dictSize) -{ - ZSTDv08_decompressBegin_usingDict(dctx, dict, dictSize); - ZSTDv08_checkContinuity(dctx, dst); - return ZSTDv08_decompressFrame(dctx, dst, dstCapacity, src, srcSize); -} - - -size_t ZSTDv08_decompressDCtx(ZSTDv08_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) -{ - return ZSTDv08_decompress_usingDict(dctx, dst, dstCapacity, src, srcSize, NULL, 0); -} - - -size_t ZSTDv08_decompress(void* dst, size_t dstCapacity, const void* src, size_t srcSize) -{ -#if defined(ZSTDv08_HEAPMODE) && (ZSTDv08_HEAPMODE==1) - size_t regenSize; - ZSTDv08_DCtx* const dctx = ZSTDv08_createDCtx(); - if (dctx==NULL) return ERROR(memory_allocation); - regenSize = ZSTDv08_decompressDCtx(dctx, dst, dstCapacity, src, srcSize); - ZSTDv08_freeDCtx(dctx); - return regenSize; -#else /* stack mode */ - ZSTDv08_DCtx dctx; - return ZSTDv08_decompressDCtx(&dctx, dst, dstCapacity, src, srcSize); -#endif -} - - -/*-********************************** -* Streaming Decompression API -************************************/ -size_t ZSTDv08_nextSrcSizeToDecompress(ZSTDv08_DCtx* dctx) { return dctx->expected; } - -ZSTDv08_nextInputType_e ZSTDv08_nextInputType(ZSTDv08_DCtx* dctx) { - switch(dctx->stage) - { - default: /* should not happen */ - case ZSTDds_getFrameHeaderSize: - case ZSTDds_decodeFrameHeader: - return ZSTDnit_frameHeader; - case ZSTDds_decodeBlockHeader: - return ZSTDnit_blockHeader; - case ZSTDds_decompressBlock: - return ZSTDnit_block; - case ZSTDds_decompressLastBlock: - return ZSTDnit_lastBlock; - case ZSTDds_checkChecksum: - return ZSTDnit_checksum; - case ZSTDds_decodeSkippableHeader: - case ZSTDds_skipFrame: - return ZSTDnit_skippableFrame; - } -} - -int ZSTDv08_isSkipFrame(ZSTDv08_DCtx* dctx) { return dctx->stage == ZSTDds_skipFrame; } /* for zbuff */ - -/** ZSTDv08_decompressContinue() : -* @return : nb of bytes generated into `dst` (necessarily <= `dstCapacity) -* or an error code, which can be tested using ZSTDv08_isError() */ -size_t ZSTDv08_decompressContinue(ZSTDv08_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) -{ - /* Sanity check */ - if (srcSize != dctx->expected) return ERROR(srcSize_wrong); - if (dstCapacity) ZSTDv08_checkContinuity(dctx, dst); - - switch (dctx->stage) - { - case ZSTDds_getFrameHeaderSize : - if (srcSize != ZSTDv08_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */ - if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTDv08_MAGIC_SKIPPABLE_START) { - memcpy(dctx->headerBuffer, src, ZSTDv08_frameHeaderSize_min); - dctx->expected = ZSTDv08_skippableHeaderSize - ZSTDv08_frameHeaderSize_min; /* magic number + skippable frame length */ - dctx->stage = ZSTDds_decodeSkippableHeader; - return 0; - } - dctx->headerSize = ZSTDv08_frameHeaderSize(src, ZSTDv08_frameHeaderSize_min); - if (ZSTDv08_isError(dctx->headerSize)) return dctx->headerSize; - memcpy(dctx->headerBuffer, src, ZSTDv08_frameHeaderSize_min); - if (dctx->headerSize > ZSTDv08_frameHeaderSize_min) { - dctx->expected = dctx->headerSize - ZSTDv08_frameHeaderSize_min; - dctx->stage = ZSTDds_decodeFrameHeader; - return 0; - } - dctx->expected = 0; /* not necessary to copy more */ - - case ZSTDds_decodeFrameHeader: - { size_t result; - memcpy(dctx->headerBuffer + ZSTDv08_frameHeaderSize_min, src, dctx->expected); - result = ZSTDv08_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize); - if (ZSTDv08_isError(result)) return result; - dctx->expected = ZSTDv08_blockHeaderSize; - dctx->stage = ZSTDds_decodeBlockHeader; - return 0; - } - case ZSTDds_decodeBlockHeader: - { blockProperties_t bp; - size_t const cBlockSize = ZSTDv08_getcBlockSize(src, ZSTDv08_blockHeaderSize, &bp); - if (ZSTDv08_isError(cBlockSize)) return cBlockSize; - dctx->expected = cBlockSize; - dctx->bType = bp.blockType; - dctx->rleSize = bp.origSize; - if (cBlockSize) { - dctx->stage = bp.lastBlock ? ZSTDds_decompressLastBlock : ZSTDds_decompressBlock; - return 0; - } - /* empty block */ - if (bp.lastBlock) { - if (dctx->fParams.checksumFlag) { - dctx->expected = 4; - dctx->stage = ZSTDds_checkChecksum; - } else { - dctx->expected = 0; /* end of frame */ - dctx->stage = ZSTDds_getFrameHeaderSize; - } - } else { - dctx->expected = 3; /* go directly to next header */ - dctx->stage = ZSTDds_decodeBlockHeader; - } - return 0; - } - case ZSTDds_decompressLastBlock: - case ZSTDds_decompressBlock: - { size_t rSize; - switch(dctx->bType) - { - case bt_compressed: - rSize = ZSTDv08_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize); - break; - case bt_raw : - rSize = ZSTDv08_copyRawBlock(dst, dstCapacity, src, srcSize); - break; - case bt_rle : - rSize = ZSTDv08_setRleBlock(dst, dstCapacity, src, srcSize, dctx->rleSize); - break; - case bt_reserved : /* should never happen */ - default: - return ERROR(corruption_detected); - } - if (ZSTDv08_isError(rSize)) return rSize; - if (dctx->fParams.checksumFlag) XXH64_update(&dctx->xxhState, dst, rSize); - - if (dctx->stage == ZSTDds_decompressLastBlock) { /* end of frame */ - if (dctx->fParams.checksumFlag) { /* another round for frame checksum */ - dctx->expected = 4; - dctx->stage = ZSTDds_checkChecksum; - } else { - dctx->expected = 0; /* ends here */ - dctx->stage = ZSTDds_getFrameHeaderSize; - } - } else { - dctx->stage = ZSTDds_decodeBlockHeader; - dctx->expected = ZSTDv08_blockHeaderSize; - dctx->previousDstEnd = (char*)dst + rSize; - } - return rSize; - } - case ZSTDds_checkChecksum: - { U32 const h32 = (U32)XXH64_digest(&dctx->xxhState); - U32 const check32 = MEM_readLE32(src); /* srcSize == 4, guaranteed by dctx->expected */ - if (check32 != h32) return ERROR(checksum_wrong); - dctx->expected = 0; - dctx->stage = ZSTDds_getFrameHeaderSize; - return 0; - } - case ZSTDds_decodeSkippableHeader: - { memcpy(dctx->headerBuffer + ZSTDv08_frameHeaderSize_min, src, dctx->expected); - dctx->expected = MEM_readLE32(dctx->headerBuffer + 4); - dctx->stage = ZSTDds_skipFrame; - return 0; - } - case ZSTDds_skipFrame: - { dctx->expected = 0; - dctx->stage = ZSTDds_getFrameHeaderSize; - return 0; - } - default: - return ERROR(GENERIC); /* impossible */ - } -} - - -static size_t ZSTDv08_refDictContent(ZSTDv08_DCtx* dctx, const void* dict, size_t dictSize) -{ - dctx->dictEnd = dctx->previousDstEnd; - dctx->vBase = (const char*)dict - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->base)); - dctx->base = dict; - dctx->previousDstEnd = (const char*)dict + dictSize; - return 0; -} - -static size_t ZSTDv08_loadEntropy(ZSTDv08_DCtx* dctx, const void* const dict, size_t const dictSize) -{ - const BYTE* dictPtr = (const BYTE*)dict; - const BYTE* const dictEnd = dictPtr + dictSize; - - { size_t const hSize = HUFv08_readDTableX4(dctx->hufTable, dict, dictSize); - if (HUFv08_isError(hSize)) return ERROR(dictionary_corrupted); - dictPtr += hSize; - } - - { short offcodeNCount[MaxOff+1]; - U32 offcodeMaxValue=MaxOff, offcodeLog=OffFSELog; - size_t const offcodeHeaderSize = FSEv08_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr); - if (FSEv08_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); - { size_t const errorCode = FSEv08_buildDTable(dctx->OffTable, offcodeNCount, offcodeMaxValue, offcodeLog); - if (FSEv08_isError(errorCode)) return ERROR(dictionary_corrupted); } - dictPtr += offcodeHeaderSize; - } - - { short matchlengthNCount[MaxML+1]; - unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog; - size_t const matchlengthHeaderSize = FSEv08_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr); - if (FSEv08_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted); - { size_t const errorCode = FSEv08_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog); - if (FSEv08_isError(errorCode)) return ERROR(dictionary_corrupted); } - dictPtr += matchlengthHeaderSize; - } - - { short litlengthNCount[MaxLL+1]; - unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog; - size_t const litlengthHeaderSize = FSEv08_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr); - if (FSEv08_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted); - { size_t const errorCode = FSEv08_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog); - if (FSEv08_isError(errorCode)) return ERROR(dictionary_corrupted); } - dictPtr += litlengthHeaderSize; - } - - if (dictPtr+12 > dictEnd) return ERROR(dictionary_corrupted); - dctx->rep[0] = MEM_readLE32(dictPtr+0); if (dctx->rep[0] >= dictSize) return ERROR(dictionary_corrupted); - dctx->rep[1] = MEM_readLE32(dictPtr+4); if (dctx->rep[1] >= dictSize) return ERROR(dictionary_corrupted); - dctx->rep[2] = MEM_readLE32(dictPtr+8); if (dctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted); - dictPtr += 12; - - dctx->litEntropy = dctx->fseEntropy = 1; - return dictPtr - (const BYTE*)dict; -} - -static size_t ZSTDv08_decompress_insertDictionary(ZSTDv08_DCtx* dctx, const void* dict, size_t dictSize) -{ - if (dictSize < 8) return ZSTDv08_refDictContent(dctx, dict, dictSize); - { U32 const magic = MEM_readLE32(dict); - if (magic != ZSTDv08_DICT_MAGIC) { - return ZSTDv08_refDictContent(dctx, dict, dictSize); /* pure content mode */ - } } - dctx->dictID = MEM_readLE32((const char*)dict + 4); - - /* load entropy tables */ - dict = (const char*)dict + 8; - dictSize -= 8; - { size_t const eSize = ZSTDv08_loadEntropy(dctx, dict, dictSize); - if (ZSTDv08_isError(eSize)) return ERROR(dictionary_corrupted); - dict = (const char*)dict + eSize; - dictSize -= eSize; - } - - /* reference dictionary content */ - return ZSTDv08_refDictContent(dctx, dict, dictSize); -} - - -size_t ZSTDv08_decompressBegin_usingDict(ZSTDv08_DCtx* dctx, const void* dict, size_t dictSize) -{ - { size_t const errorCode = ZSTDv08_decompressBegin(dctx); - if (ZSTDv08_isError(errorCode)) return errorCode; } - - if (dict && dictSize) { - size_t const errorCode = ZSTDv08_decompress_insertDictionary(dctx, dict, dictSize); - if (ZSTDv08_isError(errorCode)) return ERROR(dictionary_corrupted); - } - - return 0; -} - - -struct ZSTDv08_DDict_s { - void* dict; - size_t dictSize; - ZSTDv08_DCtx* refContext; -}; /* typedef'd tp ZSTDv08_CDict within zstd.h */ - -ZSTDv08_DDict* ZSTDv08_createDDict_advanced(const void* dict, size_t dictSize, ZSTDv08_customMem customMem) -{ - if (!customMem.customAlloc && !customMem.customFree) - customMem = defaultCustomMem; - - if (!customMem.customAlloc || !customMem.customFree) - return NULL; - - { ZSTDv08_DDict* const ddict = (ZSTDv08_DDict*) customMem.customAlloc(customMem.opaque, sizeof(*ddict)); - void* const dictContent = customMem.customAlloc(customMem.opaque, dictSize); - ZSTDv08_DCtx* const dctx = ZSTDv08_createDCtx_advanced(customMem); - - if (!dictContent || !ddict || !dctx) { - customMem.customFree(customMem.opaque, dictContent); - customMem.customFree(customMem.opaque, ddict); - customMem.customFree(customMem.opaque, dctx); - return NULL; - } - - memcpy(dictContent, dict, dictSize); - { size_t const errorCode = ZSTDv08_decompressBegin_usingDict(dctx, dictContent, dictSize); - if (ZSTDv08_isError(errorCode)) { - customMem.customFree(customMem.opaque, dictContent); - customMem.customFree(customMem.opaque, ddict); - customMem.customFree(customMem.opaque, dctx); - return NULL; - } } - - ddict->dict = dictContent; - ddict->dictSize = dictSize; - ddict->refContext = dctx; - return ddict; - } -} - -/*! ZSTDv08_createDDict() : -* Create a digested dictionary, ready to start decompression without startup delay. -* `dict` can be released after `ZSTDv08_DDict` creation */ -ZSTDv08_DDict* ZSTDv08_createDDict(const void* dict, size_t dictSize) -{ - ZSTDv08_customMem const allocator = { NULL, NULL, NULL }; - return ZSTDv08_createDDict_advanced(dict, dictSize, allocator); -} - -size_t ZSTDv08_freeDDict(ZSTDv08_DDict* ddict) -{ - ZSTDv08_freeFunction const cFree = ddict->refContext->customMem.customFree; - void* const opaque = ddict->refContext->customMem.opaque; - ZSTDv08_freeDCtx(ddict->refContext); - cFree(opaque, ddict->dict); - cFree(opaque, ddict); - return 0; -} - -/*! ZSTDv08_decompress_usingDDict() : -* Decompression using a pre-digested Dictionary -* Use dictionary without significant overhead. */ -ZSTDLIB_API size_t ZSTDv08_decompress_usingDDict(ZSTDv08_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const ZSTDv08_DDict* ddict) -{ - return ZSTDv08_decompress_usingPreparedDCtx(dctx, ddict->refContext, - dst, dstCapacity, - src, srcSize); -} -/* - Buffered version of Zstd compression library - Copyright (C) 2015-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - zstd homepage : http://www.zstd.net/ -*/ - - -/*-*************************************************************************** -* Streaming decompression howto -* -* A ZBUFFv08_DCtx object is required to track streaming operations. -* Use ZBUFFv08_createDCtx() and ZBUFFv08_freeDCtx() to create/release resources. -* Use ZBUFFv08_decompressInit() to start a new decompression operation, -* or ZBUFFv08_decompressInitDictionary() if decompression requires a dictionary. -* Note that ZBUFFv08_DCtx objects can be re-init multiple times. -* -* Use ZBUFFv08_decompressContinue() repetitively to consume your input. -* *srcSizePtr and *dstCapacityPtr can be any size. -* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr. -* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again. -* The content of @dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change @dst. -* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency), -* or 0 when a frame is completely decoded, -* or an error code, which can be tested using ZBUFFv08_isError(). -* -* Hint : recommended buffer sizes (not compulsory) : ZBUFFv08_recommendedDInSize() and ZBUFFv08_recommendedDOutSize() -* output : ZBUFFv08_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded. -* input : ZBUFFv08_recommendedDInSize == 128KB + 3; -* just follow indications from ZBUFFv08_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . -* *******************************************************************************/ - -typedef enum { ZBUFFds_init, ZBUFFds_loadHeader, - ZBUFFds_read, ZBUFFds_load, ZBUFFds_flush } ZBUFFv08_dStage; - -/* *** Resource management *** */ -struct ZBUFFv08_DCtx_s { - ZSTDv08_DCtx* zd; - ZSTDv08_frameParams fParams; - ZBUFFv08_dStage stage; - char* inBuff; - size_t inBuffSize; - size_t inPos; - char* outBuff; - size_t outBuffSize; - size_t outStart; - size_t outEnd; - size_t blockSize; - BYTE headerBuffer[ZSTDv08_FRAMEHEADERSIZE_MAX]; - size_t lhSize; - ZSTDv08_customMem customMem; -}; /* typedef'd to ZBUFFv08_DCtx within "zstd_buffered.h" */ - - -ZBUFFv08_DCtx* ZBUFFv08_createDCtx(void) -{ - return ZBUFFv08_createDCtx_advanced(defaultCustomMem); -} - -ZBUFFv08_DCtx* ZBUFFv08_createDCtx_advanced(ZSTDv08_customMem customMem) -{ - ZBUFFv08_DCtx* zbd; - - if (!customMem.customAlloc && !customMem.customFree) - customMem = defaultCustomMem; - - if (!customMem.customAlloc || !customMem.customFree) - return NULL; - - zbd = (ZBUFFv08_DCtx*)customMem.customAlloc(customMem.opaque, sizeof(ZBUFFv08_DCtx)); - if (zbd==NULL) return NULL; - memset(zbd, 0, sizeof(ZBUFFv08_DCtx)); - memcpy(&zbd->customMem, &customMem, sizeof(ZSTDv08_customMem)); - zbd->zd = ZSTDv08_createDCtx_advanced(customMem); - if (zbd->zd == NULL) { ZBUFFv08_freeDCtx(zbd); return NULL; } - zbd->stage = ZBUFFds_init; - return zbd; -} - -size_t ZBUFFv08_freeDCtx(ZBUFFv08_DCtx* zbd) -{ - if (zbd==NULL) return 0; /* support free on null */ - ZSTDv08_freeDCtx(zbd->zd); - if (zbd->inBuff) zbd->customMem.customFree(zbd->customMem.opaque, zbd->inBuff); - if (zbd->outBuff) zbd->customMem.customFree(zbd->customMem.opaque, zbd->outBuff); - zbd->customMem.customFree(zbd->customMem.opaque, zbd); - return 0; -} - - -/* *** Initialization *** */ - -size_t ZBUFFv08_decompressInitDictionary(ZBUFFv08_DCtx* zbd, const void* dict, size_t dictSize) -{ - zbd->stage = ZBUFFds_loadHeader; - zbd->lhSize = zbd->inPos = zbd->outStart = zbd->outEnd = 0; - return ZSTDv08_decompressBegin_usingDict(zbd->zd, dict, dictSize); -} - -size_t ZBUFFv08_decompressInit(ZBUFFv08_DCtx* zbd) -{ - return ZBUFFv08_decompressInitDictionary(zbd, NULL, 0); -} - - -/* internal util function */ -MEM_STATIC size_t ZBUFFv08_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) -{ - size_t const length = MIN(dstCapacity, srcSize); - memcpy(dst, src, length); - return length; -} - - -/* *** Decompression *** */ - -size_t ZBUFFv08_decompressContinue(ZBUFFv08_DCtx* zbd, - void* dst, size_t* dstCapacityPtr, - const void* src, size_t* srcSizePtr) -{ - const char* const istart = (const char*)src; - const char* const iend = istart + *srcSizePtr; - const char* ip = istart; - char* const ostart = (char*)dst; - char* const oend = ostart + *dstCapacityPtr; - char* op = ostart; - U32 someMoreWork = 1; - - while (someMoreWork) { - switch(zbd->stage) - { - case ZBUFFds_init : - return ERROR(init_missing); - - case ZBUFFds_loadHeader : - { size_t const hSize = ZSTDv08_getFrameParams(&(zbd->fParams), zbd->headerBuffer, zbd->lhSize); - if (ZSTDv08_isError(hSize)) return hSize; - if (hSize != 0) { /* need more input */ - size_t const toLoad = hSize - zbd->lhSize; /* if hSize!=0, hSize > zbd->lhSize */ - if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */ - memcpy(zbd->headerBuffer + zbd->lhSize, ip, iend-ip); - zbd->lhSize += iend-ip; - *dstCapacityPtr = 0; - return (hSize - zbd->lhSize) + ZSTDv08_blockHeaderSize; /* remaining header bytes + next block header */ - } - memcpy(zbd->headerBuffer + zbd->lhSize, ip, toLoad); zbd->lhSize = hSize; ip += toLoad; - break; - } } - - /* Consume header */ - { size_t const h1Size = ZSTDv08_nextSrcSizeToDecompress(zbd->zd); /* == ZSTDv08_frameHeaderSize_min */ - size_t const h1Result = ZSTDv08_decompressContinue(zbd->zd, NULL, 0, zbd->headerBuffer, h1Size); - if (ZSTDv08_isError(h1Result)) return h1Result; /* should not happen : already checked */ - if (h1Size < zbd->lhSize) { /* long header */ - size_t const h2Size = ZSTDv08_nextSrcSizeToDecompress(zbd->zd); - size_t const h2Result = ZSTDv08_decompressContinue(zbd->zd, NULL, 0, zbd->headerBuffer+h1Size, h2Size); - if (ZSTDv08_isError(h2Result)) return h2Result; - } } - - zbd->fParams.windowSize = MAX(zbd->fParams.windowSize, 1U << ZSTDv08_WINDOWLOG_ABSOLUTEMIN); - - /* Frame header instruct buffer sizes */ - { size_t const blockSize = MIN(zbd->fParams.windowSize, ZSTDv08_BLOCKSIZE_ABSOLUTEMAX); - size_t const neededOutSize = zbd->fParams.windowSize + blockSize; - zbd->blockSize = blockSize; - if (zbd->inBuffSize < blockSize) { - zbd->customMem.customFree(zbd->customMem.opaque, zbd->inBuff); - zbd->inBuffSize = blockSize; - zbd->inBuff = (char*)zbd->customMem.customAlloc(zbd->customMem.opaque, blockSize); - if (zbd->inBuff == NULL) return ERROR(memory_allocation); - } - if (zbd->outBuffSize < neededOutSize) { - zbd->customMem.customFree(zbd->customMem.opaque, zbd->outBuff); - zbd->outBuffSize = neededOutSize; - zbd->outBuff = (char*)zbd->customMem.customAlloc(zbd->customMem.opaque, neededOutSize); - if (zbd->outBuff == NULL) return ERROR(memory_allocation); - } } - zbd->stage = ZBUFFds_read; - /* pass-through */ - - case ZBUFFds_read: - { size_t const neededInSize = ZSTDv08_nextSrcSizeToDecompress(zbd->zd); - if (neededInSize==0) { /* end of frame */ - zbd->stage = ZBUFFds_init; - someMoreWork = 0; - break; - } - if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */ - const int isSkipFrame = ZSTDv08_isSkipFrame(zbd->zd); - size_t const decodedSize = ZSTDv08_decompressContinue(zbd->zd, - zbd->outBuff + zbd->outStart, (isSkipFrame ? 0 : zbd->outBuffSize - zbd->outStart), - ip, neededInSize); - if (ZSTDv08_isError(decodedSize)) return decodedSize; - ip += neededInSize; - if (!decodedSize && !isSkipFrame) break; /* this was just a header */ - zbd->outEnd = zbd->outStart + decodedSize; - zbd->stage = ZBUFFds_flush; - break; - } - if (ip==iend) { someMoreWork = 0; break; } /* no more input */ - zbd->stage = ZBUFFds_load; - /* pass-through */ - } - - case ZBUFFds_load: - { size_t const neededInSize = ZSTDv08_nextSrcSizeToDecompress(zbd->zd); - size_t const toLoad = neededInSize - zbd->inPos; /* should always be <= remaining space within inBuff */ - size_t loadedSize; - if (toLoad > zbd->inBuffSize - zbd->inPos) return ERROR(corruption_detected); /* should never happen */ - loadedSize = ZBUFFv08_limitCopy(zbd->inBuff + zbd->inPos, toLoad, ip, iend-ip); - ip += loadedSize; - zbd->inPos += loadedSize; - if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */ - - /* decode loaded input */ - { const int isSkipFrame = ZSTDv08_isSkipFrame(zbd->zd); - size_t const decodedSize = ZSTDv08_decompressContinue(zbd->zd, - zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart, - zbd->inBuff, neededInSize); - if (ZSTDv08_isError(decodedSize)) return decodedSize; - zbd->inPos = 0; /* input is consumed */ - if (!decodedSize && !isSkipFrame) { zbd->stage = ZBUFFds_read; break; } /* this was just a header */ - zbd->outEnd = zbd->outStart + decodedSize; - zbd->stage = ZBUFFds_flush; - /* pass-through */ - } } - - case ZBUFFds_flush: - { size_t const toFlushSize = zbd->outEnd - zbd->outStart; - size_t const flushedSize = ZBUFFv08_limitCopy(op, oend-op, zbd->outBuff + zbd->outStart, toFlushSize); - op += flushedSize; - zbd->outStart += flushedSize; - if (flushedSize == toFlushSize) { /* flush completed */ - zbd->stage = ZBUFFds_read; - if (zbd->outStart + zbd->blockSize > zbd->outBuffSize) - zbd->outStart = zbd->outEnd = 0; - break; - } - /* cannot flush everything */ - someMoreWork = 0; - break; - } - default: return ERROR(GENERIC); /* impossible */ - } } - - /* result */ - *srcSizePtr = ip-istart; - *dstCapacityPtr = op-ostart; - { size_t nextSrcSizeHint = ZSTDv08_nextSrcSizeToDecompress(zbd->zd); - if (!nextSrcSizeHint) return (zbd->outEnd != zbd->outStart); /* return 0 only if fully flushed too */ - nextSrcSizeHint += ZSTDv08_blockHeaderSize * (ZSTDv08_nextInputType(zbd->zd) == ZSTDnit_block); - if (zbd->inPos > nextSrcSizeHint) return ERROR(GENERIC); /* should never happen */ - nextSrcSizeHint -= zbd->inPos; /* already loaded*/ - return nextSrcSizeHint; - } -} - - -/* ************************************* -* Tool functions -***************************************/ -size_t ZBUFFv08_recommendedDInSize(void) { return ZSTDv08_BLOCKSIZE_ABSOLUTEMAX + ZSTDv08_blockHeaderSize /* block header size*/ ; } -size_t ZBUFFv08_recommendedDOutSize(void) { return ZSTDv08_BLOCKSIZE_ABSOLUTEMAX; } +/* ****************************************************************** + zstd_v08.c + Decompression module for ZSTD v0.8 legacy format + Copyright (C) 2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - Homepage : http://www.zstd.net/ +****************************************************************** */ + +/*- Dependencies -*/ +#include <stddef.h> /* size_t, ptrdiff_t */ +#include <string.h> /* memcpy */ +#include <stdlib.h> /* malloc, free, qsort */ + +#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ +#include "xxhash.h" /* XXH64_* */ +#include "zstd_v08.h" + +#define FSEv08_STATIC_LINKING_ONLY /* FSEv08_MIN_TABLELOG */ +#define HUFv08_STATIC_LINKING_ONLY /* HUFv08_TABLELOG_ABSOLUTEMAX */ +#define ZSTDv08_STATIC_LINKING_ONLY + + + +#ifdef ZSTDv08_STATIC_LINKING_ONLY + +/* ==================================================================================== + * The definitions in this section are considered experimental. + * They should never be used with a dynamic library, as they may change in the future. + * They are provided for advanced usages. + * Use them only in association with static linking. + * ==================================================================================== */ + +/*--- Constants ---*/ +#define ZSTDv08_MAGIC_SKIPPABLE_START 0x184D2A50U + +#define ZSTDv08_WINDOWLOG_MAX_32 25 +#define ZSTDv08_WINDOWLOG_MAX_64 27 +#define ZSTDv08_WINDOWLOG_MAX ((U32)(MEM_32bits() ? ZSTDv08_WINDOWLOG_MAX_32 : ZSTDv08_WINDOWLOG_MAX_64)) +#define ZSTDv08_WINDOWLOG_MIN 18 +#define ZSTDv08_CHAINLOG_MAX (ZSTDv08_WINDOWLOG_MAX+1) +#define ZSTDv08_CHAINLOG_MIN 4 +#define ZSTDv08_HASHLOG_MAX ZSTDv08_WINDOWLOG_MAX +#define ZSTDv08_HASHLOG_MIN 12 +#define ZSTDv08_HASHLOG3_MAX 17 +#define ZSTDv08_SEARCHLOG_MAX (ZSTDv08_WINDOWLOG_MAX-1) +#define ZSTDv08_SEARCHLOG_MIN 1 +#define ZSTDv08_SEARCHLENGTH_MAX 7 +#define ZSTDv08_SEARCHLENGTH_MIN 3 +#define ZSTDv08_TARGETLENGTH_MIN 4 +#define ZSTDv08_TARGETLENGTH_MAX 999 + +#define ZSTDv08_FRAMEHEADERSIZE_MAX 18 /* for static allocation */ +static const size_t ZSTDv08_frameHeaderSize_min = 5; +static const size_t ZSTDv08_frameHeaderSize_max = ZSTDv08_FRAMEHEADERSIZE_MAX; +static const size_t ZSTDv08_skippableHeaderSize = 8; /* magic number + skippable frame length */ + + +/* custom memory allocation functions */ +typedef void* (*ZSTDv08_allocFunction) (void* opaque, size_t size); +typedef void (*ZSTDv08_freeFunction) (void* opaque, void* address); +typedef struct { ZSTDv08_allocFunction customAlloc; ZSTDv08_freeFunction customFree; void* opaque; } ZSTDv08_customMem; + + +/*--- Advanced Decompression functions ---*/ + +/*! ZSTDv08_estimateDCtxSize() : + * Gives the potential amount of memory allocated to create a ZSTDv08_DCtx */ +ZSTDLIB_API size_t ZSTDv08_estimateDCtxSize(void); + +/*! ZSTDv08_createDCtx_advanced() : + * Create a ZSTD decompression context using external alloc and free functions */ +ZSTDLIB_API ZSTDv08_DCtx* ZSTDv08_createDCtx_advanced(ZSTDv08_customMem customMem); + +/*! ZSTDv08_sizeofDCtx() : + * Gives the amount of memory used by a given ZSTDv08_DCtx */ +ZSTDLIB_API size_t ZSTDv08_sizeofDCtx(const ZSTDv08_DCtx* dctx); + + +/* ****************************************************************** +* Buffer-less streaming functions (synchronous mode) +********************************************************************/ +/* This is an advanced API, giving full control over buffer management, for users which need direct control over memory. +* But it's also a complex one, with a lot of restrictions (documented below). +* For an easier streaming API, look into common/zbuff.h +* which removes all restrictions by allocating and managing its own internal buffer */ + +ZSTDLIB_API size_t ZSTDv08_decompressBegin(ZSTDv08_DCtx* dctx); +ZSTDLIB_API size_t ZSTDv08_decompressBegin_usingDict(ZSTDv08_DCtx* dctx, const void* dict, size_t dictSize); +ZSTDLIB_API void ZSTDv08_copyDCtx(ZSTDv08_DCtx* dctx, const ZSTDv08_DCtx* preparedDCtx); + +ZSTDLIB_API size_t ZSTDv08_nextSrcSizeToDecompress(ZSTDv08_DCtx* dctx); +ZSTDLIB_API size_t ZSTDv08_decompressContinue(ZSTDv08_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); + +typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTDv08_nextInputType_e; +ZSTDLIB_API ZSTDv08_nextInputType_e ZSTDv08_nextInputType(ZSTDv08_DCtx* dctx); + +/* + Buffer-less streaming decompression (synchronous mode) + + A ZSTDv08_DCtx object is required to track streaming operations. + Use ZSTDv08_createDCtx() / ZSTDv08_freeDCtx() to manage it. + A ZSTDv08_DCtx object can be re-used multiple times. + + First typical operation is to retrieve frame parameters, using ZSTDv08_getFrameParams(). + It fills a ZSTDv08_frameParams structure which provide important information to correctly decode the frame, + such as the minimum rolling buffer size to allocate to decompress data (`windowSize`), + and the dictionary ID used. + (Note : content size is optional, it may not be present. 0 means : content size unknown). + Note that these values could be wrong, either because of data malformation, or because an attacker is spoofing deliberate false information. + As a consequence, check that values remain within valid application range, especially `windowSize`, before allocation. + Each application can set its own limit, depending on local restrictions. For extended interoperability, it is recommended to support at least 8 MB. + Frame parameters are extracted from the beginning of the compressed frame. + Data fragment must be large enough to ensure successful decoding, typically `ZSTDv08_frameHeaderSize_max` bytes. + @result : 0 : successful decoding, the `ZSTDv08_frameParams` structure is correctly filled. + >0 : `srcSize` is too small, please provide at least @result bytes on next attempt. + errorCode, which can be tested using ZSTDv08_isError(). + + Start decompression, with ZSTDv08_decompressBegin() or ZSTDv08_decompressBegin_usingDict(). + Alternatively, you can copy a prepared context, using ZSTDv08_copyDCtx(). + + Then use ZSTDv08_nextSrcSizeToDecompress() and ZSTDv08_decompressContinue() alternatively. + ZSTDv08_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTDv08_decompressContinue(). + ZSTDv08_decompressContinue() requires this _exact_ amount of bytes, or it will fail. + + @result of ZSTDv08_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity). + It can be zero, which is not an error; it just means ZSTDv08_decompressContinue() has decoded some metadata item. + It can also be an error code, which can be tested with ZSTDv08_isError(). + + ZSTDv08_decompressContinue() needs previous data blocks during decompression, up to `windowSize`. + They should preferably be located contiguously, prior to current block. + Alternatively, a round buffer of sufficient size is also possible. Sufficient size is determined by frame parameters. + ZSTDv08_decompressContinue() is very sensitive to contiguity, + if 2 blocks don't follow each other, make sure that either the compressor breaks contiguity at the same place, + or that previous contiguous segment is large enough to properly handle maximum back-reference. + + A frame is fully decoded when ZSTDv08_nextSrcSizeToDecompress() returns zero. + Context can then be reset to start a new decompression. + + Note : it's possible to know if next input to present is a header or a block, using ZSTDv08_nextInputType(). + This information is not required to properly decode a frame. + + == Special case : skippable frames == + + Skippable frames allow integration of user-defined data into a flow of concatenated frames. + Skippable frames will be ignored (skipped) by a decompressor. The format of skippable frames is as follows : + a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F + b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits + c) Frame Content - any content (User Data) of length equal to Frame Size + For skippable frames ZSTDv08_decompressContinue() always returns 0. + For skippable frames ZSTDv08_getFrameParams() returns fparamsPtr->windowLog==0 what means that a frame is skippable. + It also returns Frame Size as fparamsPtr->frameContentSize. +*/ + + +/* ************************************** +* Block functions +****************************************/ +/*! Block functions produce and decode raw zstd blocks, without frame metadata. + Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes). + User will have to take in charge required information to regenerate data, such as compressed and content sizes. + + A few rules to respect : + - Compressing and decompressing require a context structure + + Use ZSTDv08_createCCtx() and ZSTDv08_createDCtx() + - It is necessary to init context before starting + + compression : ZSTDv08_compressBegin() + + decompression : ZSTDv08_decompressBegin() + + variants _usingDict() are also allowed + + copyCCtx() and copyDCtx() work too + - Block size is limited, it must be <= ZSTDv08_getBlockSizeMax() + + If you need to compress more, cut data into multiple blocks + + Consider using the regular ZSTDv08_compress() instead, as frame metadata costs become negligible when source size is large. + - When a block is considered not compressible enough, ZSTDv08_compressBlock() result will be zero. + In which case, nothing is produced into `dst`. + + User must test for such outcome and deal directly with uncompressed data + + ZSTDv08_decompressBlock() doesn't accept uncompressed data as input !!! + + In case of multiple successive blocks, decoder must be informed of uncompressed block existence to follow proper history. + Use ZSTDv08_insertBlock() in such a case. +*/ + +#define ZSTDv08_BLOCKSIZE_ABSOLUTEMAX (128 * 1024) /* define, for static allocation */ +ZSTDLIB_API size_t ZSTDv08_decompressBlock(ZSTDv08_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTDLIB_API size_t ZSTDv08_insertBlock(ZSTDv08_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert block into `dctx` history. Useful for uncompressed blocks */ + + +#endif /* ZSTDv08_STATIC_LINKING_ONLY */ + + +/* ==================================================================================== + * The definitions in this section are considered experimental. + * They should never be used in association with a dynamic library, as they may change in the future. + * They are provided for advanced usages. + * Use them only in association with static linking. + * ==================================================================================== */ + +/*! ZBUFFv08_createDCtx_advanced() : + * Create a ZBUFF decompression context using external alloc and free functions */ +ZSTDLIB_API ZBUFFv08_DCtx* ZBUFFv08_createDCtx_advanced(ZSTDv08_customMem customMem); + + + +/* ****************************************************************** + mem.h + low-level memory access routines + Copyright (C) 2013-2015, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + - Public forum : https://groups.google.com/forum/#!forum/lz4c +****************************************************************** */ +#ifndef MEM_H_MODULE +#define MEM_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif + + + +/*-**************************************** +* Compiler specifics +******************************************/ +#if defined(_MSC_VER) /* Visual Studio */ +# include <stdlib.h> /* _byteswap_ulong */ +# include <intrin.h> /* _byteswap_* */ +#endif +#if defined(__GNUC__) +# define MEM_STATIC static __inline __attribute__((unused)) +#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define MEM_STATIC static inline +#elif defined(_MSC_VER) +# define MEM_STATIC static __inline +#else +# define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ +#endif + +/* code only tested on 32 and 64 bits systems */ +#define MEM_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; } +MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); } + + +/*-************************************************************** +* Basic Types +*****************************************************************/ +#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include <stdint.h> + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef int16_t S16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; + typedef int64_t S64; +#else + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef signed short S16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; + typedef signed long long S64; +#endif + + +/*-************************************************************** +* Memory I/O +*****************************************************************/ +/* MEM_FORCE_MEMORY_ACCESS : + * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. + * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. + * The below switch allow to select different access method for improved performance. + * Method 0 (default) : use `memcpy()`. Safe and portable. + * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). + * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. + * Method 2 : direct access. This method is portable but violate C standard. + * It can generate buggy code on targets depending on alignment. + * In some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) + * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. + * Prefer these methods in priority order (0 > 1 > 2) + */ +#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ +# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) +# define MEM_FORCE_MEMORY_ACCESS 2 +# elif defined(__INTEL_COMPILER) || \ + (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) +# define MEM_FORCE_MEMORY_ACCESS 1 +# endif +#endif + +MEM_STATIC unsigned MEM_32bits(void) { return sizeof(size_t)==4; } +MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; } + +MEM_STATIC unsigned MEM_isLittleEndian(void) +{ + const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ + return one.c[0]; +} + +#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2) + +/* violates C standard, by lying on structure alignment. +Only use if no other choice to achieve best performance on target platform */ +MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; } +MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; } +MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; } +MEM_STATIC U64 MEM_readST(const void* memPtr) { return *(const size_t*) memPtr; } + +MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; } +MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; } +MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; } + +#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1) + +/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ +/* currently only defined for gcc and icc */ +typedef union { U16 u16; U32 u32; U64 u64; size_t st; } __attribute__((packed)) unalign; + +MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; } +MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } +MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; } +MEM_STATIC U64 MEM_readST(const void* ptr) { return ((const unalign*)ptr)->st; } + +MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; } +MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; } +MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign*)memPtr)->u64 = value; } + +#else + +/* default method, safe and standard. + can sometimes prove slower */ + +MEM_STATIC U16 MEM_read16(const void* memPtr) +{ + U16 val; memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC U32 MEM_read32(const void* memPtr) +{ + U32 val; memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC U64 MEM_read64(const void* memPtr) +{ + U64 val; memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC size_t MEM_readST(const void* memPtr) +{ + size_t val; memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC void MEM_write16(void* memPtr, U16 value) +{ + memcpy(memPtr, &value, sizeof(value)); +} + +MEM_STATIC void MEM_write32(void* memPtr, U32 value) +{ + memcpy(memPtr, &value, sizeof(value)); +} + +MEM_STATIC void MEM_write64(void* memPtr, U64 value) +{ + memcpy(memPtr, &value, sizeof(value)); +} + +#endif /* MEM_FORCE_MEMORY_ACCESS */ + +MEM_STATIC U32 MEM_swap32(U32 in) +{ +#if defined(_MSC_VER) /* Visual Studio */ + return _byteswap_ulong(in); +#elif defined (__GNUC__) + return __builtin_bswap32(in); +#else + return ((in << 24) & 0xff000000 ) | + ((in << 8) & 0x00ff0000 ) | + ((in >> 8) & 0x0000ff00 ) | + ((in >> 24) & 0x000000ff ); +#endif +} + +MEM_STATIC U64 MEM_swap64(U64 in) +{ +#if defined(_MSC_VER) /* Visual Studio */ + return _byteswap_uint64(in); +#elif defined (__GNUC__) + return __builtin_bswap64(in); +#else + return ((in << 56) & 0xff00000000000000ULL) | + ((in << 40) & 0x00ff000000000000ULL) | + ((in << 24) & 0x0000ff0000000000ULL) | + ((in << 8) & 0x000000ff00000000ULL) | + ((in >> 8) & 0x00000000ff000000ULL) | + ((in >> 24) & 0x0000000000ff0000ULL) | + ((in >> 40) & 0x000000000000ff00ULL) | + ((in >> 56) & 0x00000000000000ffULL); +#endif +} + +MEM_STATIC size_t MEM_swapST(size_t in) +{ + if (MEM_32bits()) + return (size_t)MEM_swap32((U32)in); + else + return (size_t)MEM_swap64((U64)in); +} + +/*=== Little endian r/w ===*/ + +MEM_STATIC U16 MEM_readLE16(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_read16(memPtr); + else { + const BYTE* p = (const BYTE*)memPtr; + return (U16)(p[0] + (p[1]<<8)); + } +} + +MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val) +{ + if (MEM_isLittleEndian()) { + MEM_write16(memPtr, val); + } else { + BYTE* p = (BYTE*)memPtr; + p[0] = (BYTE)val; + p[1] = (BYTE)(val>>8); + } +} + +MEM_STATIC U32 MEM_readLE24(const void* memPtr) +{ + return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16); +} + +MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val) +{ + MEM_writeLE16(memPtr, (U16)val); + ((BYTE*)memPtr)[2] = (BYTE)(val>>16); +} + +MEM_STATIC U32 MEM_readLE32(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_read32(memPtr); + else + return MEM_swap32(MEM_read32(memPtr)); +} + +MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32) +{ + if (MEM_isLittleEndian()) + MEM_write32(memPtr, val32); + else + MEM_write32(memPtr, MEM_swap32(val32)); +} + +MEM_STATIC U64 MEM_readLE64(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_read64(memPtr); + else + return MEM_swap64(MEM_read64(memPtr)); +} + +MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64) +{ + if (MEM_isLittleEndian()) + MEM_write64(memPtr, val64); + else + MEM_write64(memPtr, MEM_swap64(val64)); +} + +MEM_STATIC size_t MEM_readLEST(const void* memPtr) +{ + if (MEM_32bits()) + return (size_t)MEM_readLE32(memPtr); + else + return (size_t)MEM_readLE64(memPtr); +} + +MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val) +{ + if (MEM_32bits()) + MEM_writeLE32(memPtr, (U32)val); + else + MEM_writeLE64(memPtr, (U64)val); +} + +/*=== Big endian r/w ===*/ + +MEM_STATIC U32 MEM_readBE32(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_swap32(MEM_read32(memPtr)); + else + return MEM_read32(memPtr); +} + +MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32) +{ + if (MEM_isLittleEndian()) + MEM_write32(memPtr, MEM_swap32(val32)); + else + MEM_write32(memPtr, val32); +} + +MEM_STATIC U64 MEM_readBE64(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_swap64(MEM_read64(memPtr)); + else + return MEM_read64(memPtr); +} + +MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64) +{ + if (MEM_isLittleEndian()) + MEM_write64(memPtr, MEM_swap64(val64)); + else + MEM_write64(memPtr, val64); +} + +MEM_STATIC size_t MEM_readBEST(const void* memPtr) +{ + if (MEM_32bits()) + return (size_t)MEM_readBE32(memPtr); + else + return (size_t)MEM_readBE64(memPtr); +} + +MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val) +{ + if (MEM_32bits()) + MEM_writeBE32(memPtr, (U32)val); + else + MEM_writeBE64(memPtr, (U64)val); +} + + +/* function safe only for comparisons */ +MEM_STATIC U32 MEM_readMINMATCH(const void* memPtr, U32 length) +{ + switch (length) + { + default : + case 4 : return MEM_read32(memPtr); + case 3 : if (MEM_isLittleEndian()) + return MEM_read32(memPtr)<<8; + else + return MEM_read32(memPtr)>>8; + } +} + +#if defined (__cplusplus) +} +#endif + +#endif /* MEM_H_MODULE */ +/* ****************************************************************** + Error codes list + Copyright (C) 2016, Yann Collet + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - Homepage : http://www.zstd.net +****************************************************************** */ +#ifndef ERROR_PUBLIC_H_MODULE +#define ERROR_PUBLIC_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif + + +/* **************************************** +* error codes list +******************************************/ +typedef enum { + ZSTDv08_error_no_error, + ZSTDv08_error_GENERIC, + ZSTDv08_error_prefix_unknown, + ZSTDv08_error_frameParameter_unsupported, + ZSTDv08_error_frameParameter_unsupportedBy32bits, + ZSTDv08_error_compressionParameter_unsupported, + ZSTDv08_error_init_missing, + ZSTDv08_error_memory_allocation, + ZSTDv08_error_stage_wrong, + ZSTDv08_error_dstSize_tooSmall, + ZSTDv08_error_srcSize_wrong, + ZSTDv08_error_corruption_detected, + ZSTDv08_error_checksum_wrong, + ZSTDv08_error_tableLog_tooLarge, + ZSTDv08_error_maxSymbolValue_tooLarge, + ZSTDv08_error_maxSymbolValue_tooSmall, + ZSTDv08_error_dictionary_corrupted, + ZSTDv08_error_dictionary_wrong, + ZSTDv08_error_maxCode +} ZSTDv08_ErrorCode; + +/*! ZSTDv08_getErrorCode() : + convert a `size_t` function result into a `ZSTDv08_ErrorCode` enum type, + which can be used to compare directly with enum list published into "error_public.h" */ +ZSTDv08_ErrorCode ZSTDv08_getErrorCode(size_t functionResult); +const char* ZSTDv08_getErrorString(ZSTDv08_ErrorCode code); + + +#if defined (__cplusplus) +} +#endif + +#endif /* ERROR_PUBLIC_H_MODULE */ +/* ****************************************************************** + Error codes and messages + Copyright (C) 2013-2016, Yann Collet + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - Homepage : http://www.zstd.net +****************************************************************** */ +/* Note : this module is expected to remain private, do not expose it */ + +#ifndef ERROR_H_MODULE +#define ERROR_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif + + + +/* **************************************** +* Compiler-specific +******************************************/ +#if defined(__GNUC__) +# define ERR_STATIC static __attribute__((unused)) +#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define ERR_STATIC static inline +#elif defined(_MSC_VER) +# define ERR_STATIC static __inline +#else +# define ERR_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ +#endif + + +/*-**************************************** +* Customization (error_public.h) +******************************************/ +typedef ZSTDv08_ErrorCode ERR_enum; +#define PREFIX(name) ZSTDv08_error_##name + + +/*-**************************************** +* Error codes handling +******************************************/ +#ifdef ERROR +# undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */ +#endif +#define ERROR(name) ((size_t)-PREFIX(name)) + +ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); } + +ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); } + + +/*-**************************************** +* Error Strings +******************************************/ + +ERR_STATIC const char* ERR_getErrorString(ERR_enum code) +{ + static const char* notErrorCode = "Unspecified error code"; + switch( code ) + { + case PREFIX(no_error): return "No error detected"; + case PREFIX(GENERIC): return "Error (generic)"; + case PREFIX(prefix_unknown): return "Unknown frame descriptor"; + case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter"; + case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode"; + case PREFIX(compressionParameter_unsupported): return "Compression parameter is out of bound"; + case PREFIX(init_missing): return "Context should be init first"; + case PREFIX(memory_allocation): return "Allocation error : not enough memory"; + case PREFIX(stage_wrong): return "Operation not authorized at current processing stage"; + case PREFIX(dstSize_tooSmall): return "Destination buffer is too small"; + case PREFIX(srcSize_wrong): return "Src size incorrect"; + case PREFIX(corruption_detected): return "Corrupted block detected"; + case PREFIX(checksum_wrong): return "Restored data doesn't match checksum"; + case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported"; + case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large"; + case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small"; + case PREFIX(dictionary_corrupted): return "Dictionary is corrupted"; + case PREFIX(dictionary_wrong): return "Dictionary mismatch"; + case PREFIX(maxCode): + default: return notErrorCode; + } +} + +ERR_STATIC const char* ERR_getErrorName(size_t code) +{ + return ERR_getErrorString(ERR_getErrorCode(code)); +} + +#if defined (__cplusplus) +} +#endif + +#endif /* ERROR_H_MODULE */ +/* ****************************************************************** + bitstream + Part of FSE library + header file (to include) + Copyright (C) 2013-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - Source repository : https://github.com/Cyan4973/FiniteStateEntropy +****************************************************************** */ +#ifndef BITSTREAM_H_MODULE +#define BITSTREAM_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif + + +/* +* This API consists of small unitary functions, which must be inlined for best performance. +* Since link-time-optimization is not available for all compilers, +* these functions are defined into a .h to be included. +*/ + + +/*========================================= +* Target specific +=========================================*/ +#if defined(__BMI__) && defined(__GNUC__) +# include <immintrin.h> /* support for bextr (experimental) */ +#endif + + +/*-******************************************** +* bitStream decoding API (read backward) +**********************************************/ +typedef struct +{ + size_t bitContainer; + unsigned bitsConsumed; + const char* ptr; + const char* start; +} BIT_DStream_t; + +typedef enum { BIT_DStream_unfinished = 0, + BIT_DStream_endOfBuffer = 1, + BIT_DStream_completed = 2, + BIT_DStream_overflow = 3 } BIT_DStream_status; /* result of BIT_reloadDStream() */ + /* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */ + +MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize); +MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits); +MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD); +MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD); + + +/* Start by invoking BIT_initDStream(). +* A chunk of the bitStream is then stored into a local register. +* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t). +* You can then retrieve bitFields stored into the local register, **in reverse order**. +* Local register is explicitly reloaded from memory by the BIT_reloadDStream() method. +* A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished. +* Otherwise, it can be less than that, so proceed accordingly. +* Checking if DStream has reached its end can be performed with BIT_endOfDStream(). +*/ + + +/*-**************************************** +* unsafe API +******************************************/ +MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits); +/* faster, but works only if nbBits >= 1 */ + + + +/*-************************************************************** +* Internal functions +****************************************************************/ +MEM_STATIC unsigned BIT_highbit32 (register U32 val) +{ +# if defined(_MSC_VER) /* Visual */ + unsigned long r=0; + _BitScanReverse ( &r, val ); + return (unsigned) r; +# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */ + return 31 - __builtin_clz (val); +# else /* Software version */ + static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 }; + U32 v = val; + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + return DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27]; +# endif +} + +/*===== Local Constants =====*/ +static const unsigned BIT_mask[] = { 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF, 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF }; /* up to 26 bits */ + + +/*-******************************************************** +* bitStream decoding +**********************************************************/ +/*! BIT_initDStream() : +* Initialize a BIT_DStream_t. +* `bitD` : a pointer to an already allocated BIT_DStream_t structure. +* `srcSize` must be the *exact* size of the bitStream, in bytes. +* @return : size of stream (== srcSize) or an errorCode if a problem is detected +*/ +MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize) +{ + if (srcSize < 1) { memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); } + + if (srcSize >= sizeof(bitD->bitContainer)) { /* normal case */ + bitD->start = (const char*)srcBuffer; + bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer); + bitD->bitContainer = MEM_readLEST(bitD->ptr); + { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; + bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; + if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ } + } else { + bitD->start = (const char*)srcBuffer; + bitD->ptr = bitD->start; + bitD->bitContainer = *(const BYTE*)(bitD->start); + switch(srcSize) + { + case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16); + case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24); + case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32); + case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24; + case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16; + case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8; + default:; + } + { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; + bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; + if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ } + bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize)*8; + } + + return srcSize; +} + +MEM_STATIC size_t BIT_getUpperBits(size_t bitContainer, U32 const start) +{ + return bitContainer >> start; +} + +MEM_STATIC size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits) +{ +#if defined(__BMI__) && defined(__GNUC__) /* experimental */ +# if defined(__x86_64__) + if (sizeof(bitContainer)==8) + return _bextr_u64(bitContainer, start, nbBits); + else +# endif + return _bextr_u32(bitContainer, start, nbBits); +#else + return (bitContainer >> start) & BIT_mask[nbBits]; +#endif +} + +MEM_STATIC size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits) +{ + return bitContainer & BIT_mask[nbBits]; +} + +/*! BIT_lookBits() : + * Provides next n bits from local register. + * local register is not modified. + * On 32-bits, maxNbBits==24. + * On 64-bits, maxNbBits==56. + * @return : value extracted + */ + MEM_STATIC size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits) +{ +#if defined(__BMI__) && defined(__GNUC__) /* experimental; fails if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8 */ + return BIT_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits); +#else + U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1; + return ((bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> 1) >> ((bitMask-nbBits) & bitMask); +#endif +} + +/*! BIT_lookBitsFast() : +* unsafe version; only works only if nbBits >= 1 */ +MEM_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits) +{ + U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1; + return (bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> (((bitMask+1)-nbBits) & bitMask); +} + +MEM_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits) +{ + bitD->bitsConsumed += nbBits; +} + +/*! BIT_readBits() : + * Read (consume) next n bits from local register and update. + * Pay attention to not read more than nbBits contained into local register. + * @return : extracted value. + */ +MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits) +{ + size_t const value = BIT_lookBits(bitD, nbBits); + BIT_skipBits(bitD, nbBits); + return value; +} + +/*! BIT_readBitsFast() : +* unsafe version; only works only if nbBits >= 1 */ +MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits) +{ + size_t const value = BIT_lookBitsFast(bitD, nbBits); + BIT_skipBits(bitD, nbBits); + return value; +} + +/*! BIT_reloadDStream() : +* Refill `BIT_DStream_t` from src buffer previously defined (see BIT_initDStream() ). +* This function is safe, it guarantees it will not read beyond src buffer. +* @return : status of `BIT_DStream_t` internal register. + if status == unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */ +MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD) +{ + if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should not happen => corruption detected */ + return BIT_DStream_overflow; + + if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer)) { + bitD->ptr -= bitD->bitsConsumed >> 3; + bitD->bitsConsumed &= 7; + bitD->bitContainer = MEM_readLEST(bitD->ptr); + return BIT_DStream_unfinished; + } + if (bitD->ptr == bitD->start) { + if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return BIT_DStream_endOfBuffer; + return BIT_DStream_completed; + } + { U32 nbBytes = bitD->bitsConsumed >> 3; + BIT_DStream_status result = BIT_DStream_unfinished; + if (bitD->ptr - nbBytes < bitD->start) { + nbBytes = (U32)(bitD->ptr - bitD->start); /* ptr > start */ + result = BIT_DStream_endOfBuffer; + } + bitD->ptr -= nbBytes; + bitD->bitsConsumed -= nbBytes*8; + bitD->bitContainer = MEM_readLEST(bitD->ptr); /* reminder : srcSize > sizeof(bitD) */ + return result; + } +} + +/*! BIT_endOfDStream() : +* @return Tells if DStream has exactly reached its end (all bits consumed). +*/ +MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream) +{ + return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer)*8)); +} + +#if defined (__cplusplus) +} +#endif + +#endif /* BITSTREAM_H_MODULE */ +/* ****************************************************************** + FSE : Finite State Entropy codec + Public Prototypes declaration + Copyright (C) 2013-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - Source repository : https://github.com/Cyan4973/FiniteStateEntropy +****************************************************************** */ +#ifndef FSEv08_H +#define FSEv08_H + +#if defined (__cplusplus) +extern "C" { +#endif + + + +/*-**************************************** +* FSE simple functions +******************************************/ +/*! FSEv08_decompress(): + Decompress FSE data from buffer 'cSrc', of size 'cSrcSize', + into already allocated destination buffer 'dst', of size 'dstCapacity'. + @return : size of regenerated data (<= maxDstSize), + or an error code, which can be tested using FSEv08_isError() . + + ** Important ** : FSEv08_decompress() does not decompress non-compressible nor RLE data !!! + Why ? : making this distinction requires a header. + Header management is intentionally delegated to the user layer, which can better manage special cases. +*/ +size_t FSEv08_decompress(void* dst, size_t dstCapacity, + const void* cSrc, size_t cSrcSize); + + +/*-***************************************** +* Tool functions +******************************************/ +/* Error Management */ +unsigned FSEv08_isError(size_t code); /* tells if a return value is an error code */ +const char* FSEv08_getErrorName(size_t code); /* provides error code string (useful for debugging) */ + + +/*-***************************************** +* FSE detailed API +******************************************/ +/*! +FSEv08_decompress() does the following: +1. read normalized counters with readNCount() +2. build decoding table 'DTable' from normalized counters +3. decode the data stream using decoding table 'DTable' + +The following API allows targeting specific sub-functions for advanced tasks. +For example, it's possible to compress several blocks using the same 'CTable', +or to save and provide normalized distribution using external method. +*/ + + +/* *** DECOMPRESSION *** */ + +/*! FSEv08_readNCount(): + Read compactly saved 'normalizedCounter' from 'rBuffer'. + @return : size read from 'rBuffer', + or an errorCode, which can be tested using FSEv08_isError(). + maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */ +size_t FSEv08_readNCount (short* normalizedCounter, unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, const void* rBuffer, size_t rBuffSize); + +/*! Constructor and Destructor of FSEv08_DTable. + Note that its size depends on 'tableLog' */ +typedef unsigned FSEv08_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */ +FSEv08_DTable* FSEv08_createDTable(unsigned tableLog); +void FSEv08_freeDTable(FSEv08_DTable* dt); + +/*! FSEv08_buildDTable(): + Builds 'dt', which must be already allocated, using FSEv08_createDTable(). + return : 0, or an errorCode, which can be tested using FSEv08_isError() */ +size_t FSEv08_buildDTable (FSEv08_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); + +/*! FSEv08_decompress_usingDTable(): + Decompress compressed source `cSrc` of size `cSrcSize` using `dt` + into `dst` which must be already allocated. + @return : size of regenerated data (necessarily <= `dstCapacity`), + or an errorCode, which can be tested using FSEv08_isError() */ +size_t FSEv08_decompress_usingDTable(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, const FSEv08_DTable* dt); + +/*! +Tutorial : +---------- +(Note : these functions only decompress FSE-compressed blocks. + If block is uncompressed, use memcpy() instead + If block is a single repeated byte, use memset() instead ) + +The first step is to obtain the normalized frequencies of symbols. +This can be performed by FSEv08_readNCount() if it was saved using FSEv08_writeNCount(). +'normalizedCounter' must be already allocated, and have at least 'maxSymbolValuePtr[0]+1' cells of signed short. +In practice, that means it's necessary to know 'maxSymbolValue' beforehand, +or size the table to handle worst case situations (typically 256). +FSEv08_readNCount() will provide 'tableLog' and 'maxSymbolValue'. +The result of FSEv08_readNCount() is the number of bytes read from 'rBuffer'. +Note that 'rBufferSize' must be at least 4 bytes, even if useful information is less than that. +If there is an error, the function will return an error code, which can be tested using FSEv08_isError(). + +The next step is to build the decompression tables 'FSEv08_DTable' from 'normalizedCounter'. +This is performed by the function FSEv08_buildDTable(). +The space required by 'FSEv08_DTable' must be already allocated using FSEv08_createDTable(). +If there is an error, the function will return an error code, which can be tested using FSEv08_isError(). + +`FSEv08_DTable` can then be used to decompress `cSrc`, with FSEv08_decompress_usingDTable(). +`cSrcSize` must be strictly correct, otherwise decompression will fail. +FSEv08_decompress_usingDTable() result will tell how many bytes were regenerated (<=`dstCapacity`). +If there is an error, the function will return an error code, which can be tested using FSEv08_isError(). (ex: dst buffer too small) +*/ + + +#ifdef FSEv08_STATIC_LINKING_ONLY + + +/* ***************************************** +* Static allocation +*******************************************/ +/* FSE buffer bounds */ +#define FSEv08_NCOUNTBOUND 512 +#define FSEv08_BLOCKBOUND(size) (size + (size>>7)) + +/* It is possible to statically allocate FSE CTable/DTable as a table of unsigned using below macros */ +#define FSEv08_DTABLE_SIZE_U32(maxTableLog) (1 + (1<<maxTableLog)) + + +/* ***************************************** +* FSE advanced API +*******************************************/ +size_t FSEv08_countFast(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize); +/**< same as FSEv08_count(), but blindly trusts that all byte values within src are <= *maxSymbolValuePtr */ + +unsigned FSEv08_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus); +/**< same as FSEv08_optimalTableLog(), which used `minus==2` */ + +size_t FSEv08_buildDTable_raw (FSEv08_DTable* dt, unsigned nbBits); +/**< build a fake FSEv08_DTable, designed to read an uncompressed bitstream where each symbol uses nbBits */ + +size_t FSEv08_buildDTable_rle (FSEv08_DTable* dt, unsigned char symbolValue); +/**< build a fake FSEv08_DTable, designed to always generate the same symbolValue */ + + + +/* ***************************************** +* FSE symbol decompression API +*******************************************/ +typedef struct +{ + size_t state; + const void* table; /* precise table may vary, depending on U16 */ +} FSEv08_DState_t; + + +static void FSEv08_initDState(FSEv08_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSEv08_DTable* dt); + +static unsigned char FSEv08_decodeSymbol(FSEv08_DState_t* DStatePtr, BIT_DStream_t* bitD); + +static unsigned FSEv08_endOfDState(const FSEv08_DState_t* DStatePtr); + +/**< +Let's now decompose FSEv08_decompress_usingDTable() into its unitary components. +You will decode FSE-encoded symbols from the bitStream, +and also any other bitFields you put in, **in reverse order**. + +You will need a few variables to track your bitStream. They are : + +BIT_DStream_t DStream; // Stream context +FSEv08_DState_t DState; // State context. Multiple ones are possible +FSEv08_DTable* DTablePtr; // Decoding table, provided by FSEv08_buildDTable() + +The first thing to do is to init the bitStream. + errorCode = BIT_initDStream(&DStream, srcBuffer, srcSize); + +You should then retrieve your initial state(s) +(in reverse flushing order if you have several ones) : + errorCode = FSEv08_initDState(&DState, &DStream, DTablePtr); + +You can then decode your data, symbol after symbol. +For information the maximum number of bits read by FSEv08_decodeSymbol() is 'tableLog'. +Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out). + unsigned char symbol = FSEv08_decodeSymbol(&DState, &DStream); + +You can retrieve any bitfield you eventually stored into the bitStream (in reverse order) +Note : maximum allowed nbBits is 25, for 32-bits compatibility + size_t bitField = BIT_readBits(&DStream, nbBits); + +All above operations only read from local register (which size depends on size_t). +Refueling the register from memory is manually performed by the reload method. + endSignal = FSEv08_reloadDStream(&DStream); + +BIT_reloadDStream() result tells if there is still some more data to read from DStream. +BIT_DStream_unfinished : there is still some data left into the DStream. +BIT_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled. +BIT_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed. +BIT_DStream_tooFar : Dstream went too far. Decompression result is corrupted. + +When reaching end of buffer (BIT_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop, +to properly detect the exact end of stream. +After each decoded symbol, check if DStream is fully consumed using this simple test : + BIT_reloadDStream(&DStream) >= BIT_DStream_completed + +When it's done, verify decompression is fully completed, by checking both DStream and the relevant states. +Checking if DStream has reached its end is performed by : + BIT_endOfDStream(&DStream); +Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible. + FSEv08_endOfDState(&DState); +*/ + + +/* ***************************************** +* FSE unsafe API +*******************************************/ +static unsigned char FSEv08_decodeSymbolFast(FSEv08_DState_t* DStatePtr, BIT_DStream_t* bitD); +/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */ + + +/* ***************************************** +* Implementation of inlined functions +*******************************************/ +/*<===== Decompression =====>*/ + +typedef struct { + U16 tableLog; + U16 fastMode; +} FSEv08_DTableHeader; /* sizeof U32 */ + +typedef struct +{ + unsigned short newState; + unsigned char symbol; + unsigned char nbBits; +} FSEv08_decode_t; /* size == U32 */ + +MEM_STATIC void FSEv08_initDState(FSEv08_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSEv08_DTable* dt) +{ + const void* ptr = dt; + const FSEv08_DTableHeader* const DTableH = (const FSEv08_DTableHeader*)ptr; + DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog); + BIT_reloadDStream(bitD); + DStatePtr->table = dt + 1; +} + +MEM_STATIC BYTE FSEv08_peekSymbol(const FSEv08_DState_t* DStatePtr) +{ + FSEv08_decode_t const DInfo = ((const FSEv08_decode_t*)(DStatePtr->table))[DStatePtr->state]; + return DInfo.symbol; +} + +MEM_STATIC void FSEv08_updateState(FSEv08_DState_t* DStatePtr, BIT_DStream_t* bitD) +{ + FSEv08_decode_t const DInfo = ((const FSEv08_decode_t*)(DStatePtr->table))[DStatePtr->state]; + U32 const nbBits = DInfo.nbBits; + size_t const lowBits = BIT_readBits(bitD, nbBits); + DStatePtr->state = DInfo.newState + lowBits; +} + +MEM_STATIC BYTE FSEv08_decodeSymbol(FSEv08_DState_t* DStatePtr, BIT_DStream_t* bitD) +{ + FSEv08_decode_t const DInfo = ((const FSEv08_decode_t*)(DStatePtr->table))[DStatePtr->state]; + U32 const nbBits = DInfo.nbBits; + BYTE const symbol = DInfo.symbol; + size_t const lowBits = BIT_readBits(bitD, nbBits); + + DStatePtr->state = DInfo.newState + lowBits; + return symbol; +} + +/*! FSEv08_decodeSymbolFast() : + unsafe, only works if no symbol has a probability > 50% */ +MEM_STATIC BYTE FSEv08_decodeSymbolFast(FSEv08_DState_t* DStatePtr, BIT_DStream_t* bitD) +{ + FSEv08_decode_t const DInfo = ((const FSEv08_decode_t*)(DStatePtr->table))[DStatePtr->state]; + U32 const nbBits = DInfo.nbBits; + BYTE const symbol = DInfo.symbol; + size_t const lowBits = BIT_readBitsFast(bitD, nbBits); + + DStatePtr->state = DInfo.newState + lowBits; + return symbol; +} + +MEM_STATIC unsigned FSEv08_endOfDState(const FSEv08_DState_t* DStatePtr) +{ + return DStatePtr->state == 0; +} + + + +#ifndef FSEv08_COMMONDEFS_ONLY + +/* ************************************************************** +* Tuning parameters +****************************************************************/ +/*!MEMORY_USAGE : +* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) +* Increasing memory usage improves compression ratio +* Reduced memory usage can improve speed, due to cache effect +* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ +#define FSEv08_MAX_MEMORY_USAGE 14 +#define FSEv08_DEFAULT_MEMORY_USAGE 13 + +/*!FSEv08_MAX_SYMBOL_VALUE : +* Maximum symbol value authorized. +* Required for proper stack allocation */ +#define FSEv08_MAX_SYMBOL_VALUE 255 + + +/* ************************************************************** +* template functions type & suffix +****************************************************************/ +#define FSEv08_FUNCTION_TYPE BYTE +#define FSEv08_FUNCTION_EXTENSION +#define FSEv08_DECODE_TYPE FSEv08_decode_t + + +#endif /* !FSEv08_COMMONDEFS_ONLY */ + + +/* *************************************************************** +* Constants +*****************************************************************/ +#define FSEv08_MAX_TABLELOG (FSEv08_MAX_MEMORY_USAGE-2) +#define FSEv08_MAX_TABLESIZE (1U<<FSEv08_MAX_TABLELOG) +#define FSEv08_MAXTABLESIZE_MASK (FSEv08_MAX_TABLESIZE-1) +#define FSEv08_DEFAULT_TABLELOG (FSEv08_DEFAULT_MEMORY_USAGE-2) +#define FSEv08_MIN_TABLELOG 5 + +#define FSEv08_TABLELOG_ABSOLUTE_MAX 15 +#if FSEv08_MAX_TABLELOG > FSEv08_TABLELOG_ABSOLUTE_MAX +# error "FSEv08_MAX_TABLELOG > FSEv08_TABLELOG_ABSOLUTE_MAX is not supported" +#endif + +#define FSEv08_TABLESTEP(tableSize) ((tableSize>>1) + (tableSize>>3) + 3) + + +#endif /* FSEv08_STATIC_LINKING_ONLY */ + + +#if defined (__cplusplus) +} +#endif + +#endif /* FSEv08_H */ +/* ****************************************************************** + Huffman coder, part of New Generation Entropy library + header file + Copyright (C) 2013-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - Source repository : https://github.com/Cyan4973/FiniteStateEntropy +****************************************************************** */ +#ifndef HUFv08_H_298734234 +#define HUFv08_H_298734234 + +#if defined (__cplusplus) +extern "C" { +#endif + + +/* *** simple functions *** */ +/** +HUFv08_decompress() : + Decompress HUF data from buffer 'cSrc', of size 'cSrcSize', + into already allocated buffer 'dst', of minimum size 'dstSize'. + `dstSize` : **must** be the ***exact*** size of original (uncompressed) data. + Note : in contrast with FSE, HUFv08_decompress can regenerate + RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data, + because it knows size to regenerate. + @return : size of regenerated data (== dstSize), + or an error code, which can be tested using HUFv08_isError() +*/ +size_t HUFv08_decompress(void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize); + + +/* **************************************** +* Tool functions +******************************************/ +#define HUFv08_BLOCKSIZE_MAX (128 * 1024) + +/* Error Management */ +unsigned HUFv08_isError(size_t code); /**< tells if a return value is an error code */ +const char* HUFv08_getErrorName(size_t code); /**< provides error code string (useful for debugging) */ + + +/* *** Advanced function *** */ + + +#ifdef HUFv08_STATIC_LINKING_ONLY + + +/* *** Constants *** */ +#define HUFv08_TABLELOG_ABSOLUTEMAX 16 /* absolute limit of HUFv08_MAX_TABLELOG. Beyond that value, code does not work */ +#define HUFv08_TABLELOG_MAX 12 /* max configured tableLog (for static allocation); can be modified up to HUFv08_ABSOLUTEMAX_TABLELOG */ +#define HUFv08_TABLELOG_DEFAULT 11 /* tableLog by default, when not specified */ +#define HUFv08_SYMBOLVALUE_MAX 255 +#if (HUFv08_TABLELOG_MAX > HUFv08_TABLELOG_ABSOLUTEMAX) +# error "HUFv08_TABLELOG_MAX is too large !" +#endif + + +/* **************************************** +* Static allocation +******************************************/ +/* HUF buffer bounds */ +#define HUFv08_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true if incompressible pre-filtered with fast heuristic */ + +/* static allocation of HUF's DTable */ +typedef U32 HUFv08_DTable; +#define HUFv08_DTABLE_SIZE(maxTableLog) (1 + (1<<(maxTableLog))) +#define HUFv08_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \ + HUFv08_DTable DTable[HUFv08_DTABLE_SIZE((maxTableLog)-1)] = { ((U32)((maxTableLog)-1)*0x1000001) } +#define HUFv08_CREATE_STATIC_DTABLEX4(DTable, maxTableLog) \ + HUFv08_DTable DTable[HUFv08_DTABLE_SIZE(maxTableLog)] = { ((U32)(maxTableLog)*0x1000001) } + + +/* **************************************** +* Advanced decompression functions +******************************************/ +size_t HUFv08_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ +size_t HUFv08_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ + +size_t HUFv08_decompress4X_DCtx (HUFv08_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< decodes RLE and uncompressed */ +size_t HUFv08_decompress4X_hufOnly(HUFv08_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */ +size_t HUFv08_decompress4X2_DCtx(HUFv08_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ +size_t HUFv08_decompress4X4_DCtx(HUFv08_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ + +size_t HUFv08_decompress1X_DCtx (HUFv08_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); +size_t HUFv08_decompress1X2_DCtx(HUFv08_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ +size_t HUFv08_decompress1X4_DCtx(HUFv08_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ + + +/* **************************************** +* HUF detailed API +******************************************/ +/* +HUFv08_decompress() does the following: +1. select the decompression algorithm (X2, X4) based on pre-computed heuristics +2. build Huffman table from save, using HUFv08_readDTableXn() +3. decode 1 or 4 segments in parallel using HUFv08_decompressSXn_usingDTable +*/ + +/** HUFv08_selectDecoder() : +* Tells which decoder is likely to decode faster, +* based on a set of pre-determined metrics. +* @return : 0==HUFv08_decompress4X2, 1==HUFv08_decompress4X4 . +* Assumption : 0 < cSrcSize < dstSize <= 128 KB */ +U32 HUFv08_selectDecoder (size_t dstSize, size_t cSrcSize); + +size_t HUFv08_readDTableX2 (HUFv08_DTable* DTable, const void* src, size_t srcSize); +size_t HUFv08_readDTableX4 (HUFv08_DTable* DTable, const void* src, size_t srcSize); + +size_t HUFv08_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUFv08_DTable* DTable); +size_t HUFv08_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUFv08_DTable* DTable); +size_t HUFv08_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUFv08_DTable* DTable); + + +/* single stream variants */ + +size_t HUFv08_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */ +size_t HUFv08_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */ + +size_t HUFv08_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUFv08_DTable* DTable); +size_t HUFv08_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUFv08_DTable* DTable); +size_t HUFv08_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUFv08_DTable* DTable); + + +#endif /* HUFv08_STATIC_LINKING_ONLY */ + + +#if defined (__cplusplus) +} +#endif + +#endif /* HUFv08_H_298734234 */ +/* + Common functions of New Generation Entropy library + Copyright (C) 2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy + - Public forum : https://groups.google.com/forum/#!forum/lz4c +*************************************************************************** */ + + +/*-**************************************** +* FSE Error Management +******************************************/ +unsigned FSEv08_isError(size_t code) { return ERR_isError(code); } + +const char* FSEv08_getErrorName(size_t code) { return ERR_getErrorName(code); } + + +/* ************************************************************** +* HUF Error Management +****************************************************************/ +unsigned HUFv08_isError(size_t code) { return ERR_isError(code); } + +const char* HUFv08_getErrorName(size_t code) { return ERR_getErrorName(code); } + + +/*-************************************************************** +* FSE NCount encoding-decoding +****************************************************************/ +static short FSEv08_abs(short a) { return (short)(a<0 ? -a : a); } + +size_t FSEv08_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize) +{ + const BYTE* const istart = (const BYTE*) headerBuffer; + const BYTE* const iend = istart + hbSize; + const BYTE* ip = istart; + int nbBits; + int remaining; + int threshold; + U32 bitStream; + int bitCount; + unsigned charnum = 0; + int previous0 = 0; + + if (hbSize < 4) return ERROR(srcSize_wrong); + bitStream = MEM_readLE32(ip); + nbBits = (bitStream & 0xF) + FSEv08_MIN_TABLELOG; /* extract tableLog */ + if (nbBits > FSEv08_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge); + bitStream >>= 4; + bitCount = 4; + *tableLogPtr = nbBits; + remaining = (1<<nbBits)+1; + threshold = 1<<nbBits; + nbBits++; + + while ((remaining>1) & (charnum<=*maxSVPtr)) { + if (previous0) { + unsigned n0 = charnum; + while ((bitStream & 0xFFFF) == 0xFFFF) { + n0 += 24; + if (ip < iend-5) { + ip += 2; + bitStream = MEM_readLE32(ip) >> bitCount; + } else { + bitStream >>= 16; + bitCount += 16; + } } + while ((bitStream & 3) == 3) { + n0 += 3; + bitStream >>= 2; + bitCount += 2; + } + n0 += bitStream & 3; + bitCount += 2; + if (n0 > *maxSVPtr) return ERROR(maxSymbolValue_tooSmall); + while (charnum < n0) normalizedCounter[charnum++] = 0; + if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { + ip += bitCount>>3; + bitCount &= 7; + bitStream = MEM_readLE32(ip) >> bitCount; + } else { + bitStream >>= 2; + } } + { short const max = (short)((2*threshold-1)-remaining); + short count; + + if ((bitStream & (threshold-1)) < (U32)max) { + count = (short)(bitStream & (threshold-1)); + bitCount += nbBits-1; + } else { + count = (short)(bitStream & (2*threshold-1)); + if (count >= threshold) count -= max; + bitCount += nbBits; + } + + count--; /* extra accuracy */ + remaining -= FSEv08_abs(count); + normalizedCounter[charnum++] = count; + previous0 = !count; + while (remaining < threshold) { + nbBits--; + threshold >>= 1; + } + + if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { + ip += bitCount>>3; + bitCount &= 7; + } else { + bitCount -= (int)(8 * (iend - 4 - ip)); + ip = iend - 4; + } + bitStream = MEM_readLE32(ip) >> (bitCount & 31); + } } /* while ((remaining>1) & (charnum<=*maxSVPtr)) */ + if (remaining != 1) return ERROR(corruption_detected); + if (bitCount > 32) return ERROR(corruption_detected); + *maxSVPtr = charnum-1; + + ip += (bitCount+7)>>3; + return ip-istart; +} + + +/*! HUFv08_readStats() : + Read compact Huffman tree, saved by HUFv08_writeCTable(). + `huffWeight` is destination buffer. + @return : size read from `src` , or an error Code . + Note : Needed by HUFv08_readCTable() and HUFv08_readDTableX?() . +*/ +size_t HUFv08_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize) +{ + U32 weightTotal; + const BYTE* ip = (const BYTE*) src; + size_t iSize = ip[0]; + size_t oSize; + + /* memset(huffWeight, 0, hwSize); *//* is not necessary, even though some analyzer complain ... */ + + if (iSize >= 128) { /* special header */ + oSize = iSize - 127; + iSize = ((oSize+1)/2); + if (iSize+1 > srcSize) return ERROR(srcSize_wrong); + if (oSize >= hwSize) return ERROR(corruption_detected); + ip += 1; + { U32 n; + for (n=0; n<oSize; n+=2) { + huffWeight[n] = ip[n/2] >> 4; + huffWeight[n+1] = ip[n/2] & 15; + } } } + else { /* header compressed with FSE (normal case) */ + if (iSize+1 > srcSize) return ERROR(srcSize_wrong); + oSize = FSEv08_decompress(huffWeight, hwSize-1, ip+1, iSize); /* max (hwSize-1) values decoded, as last one is implied */ + if (FSEv08_isError(oSize)) return oSize; + } + + /* collect weight stats */ + memset(rankStats, 0, (HUFv08_TABLELOG_ABSOLUTEMAX + 1) * sizeof(U32)); + weightTotal = 0; + { U32 n; for (n=0; n<oSize; n++) { + if (huffWeight[n] >= HUFv08_TABLELOG_ABSOLUTEMAX) return ERROR(corruption_detected); + rankStats[huffWeight[n]]++; + weightTotal += (1 << huffWeight[n]) >> 1; + } } + + /* get last non-null symbol weight (implied, total must be 2^n) */ + { U32 const tableLog = BIT_highbit32(weightTotal) + 1; + if (tableLog > HUFv08_TABLELOG_ABSOLUTEMAX) return ERROR(corruption_detected); + *tableLogPtr = tableLog; + /* determine last weight */ + { U32 const total = 1 << tableLog; + U32 const rest = total - weightTotal; + U32 const verif = 1 << BIT_highbit32(rest); + U32 const lastWeight = BIT_highbit32(rest) + 1; + if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */ + huffWeight[oSize] = (BYTE)lastWeight; + rankStats[lastWeight]++; + } } + + /* check tree construction validity */ + if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */ + + /* results */ + *nbSymbolsPtr = (U32)(oSize+1); + return iSize+1; +} +/* ****************************************************************** + FSE : Finite State Entropy decoder + Copyright (C) 2013-2015, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + - Public forum : https://groups.google.com/forum/#!forum/lz4c +****************************************************************** */ + + +/* ************************************************************** +* Compiler specifics +****************************************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# define FORCE_INLINE static __forceinline +# include <intrin.h> /* For Visual 2005 */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */ +#else +# ifdef __GNUC__ +# define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif +#endif + + +/* ************************************************************** +* Error Management +****************************************************************/ +#define FSEv08_isError ERR_isError +#define FSEv08_STATIC_ASSERT(c) { enum { FSEv08_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ + + +/* ************************************************************** +* Complex types +****************************************************************/ +typedef U32 DTable_max_t[FSEv08_DTABLE_SIZE_U32(FSEv08_MAX_TABLELOG)]; + + +/* ************************************************************** +* Templates +****************************************************************/ +/* + designed to be included + for type-specific functions (template emulation in C) + Objective is to write these functions only once, for improved maintenance +*/ + +/* safety checks */ +#ifndef FSEv08_FUNCTION_EXTENSION +# error "FSEv08_FUNCTION_EXTENSION must be defined" +#endif +#ifndef FSEv08_FUNCTION_TYPE +# error "FSEv08_FUNCTION_TYPE must be defined" +#endif + +/* Function names */ +#define FSEv08_CAT(X,Y) X##Y +#define FSEv08_FUNCTION_NAME(X,Y) FSEv08_CAT(X,Y) +#define FSEv08_TYPE_NAME(X,Y) FSEv08_CAT(X,Y) + + +/* Function templates */ +FSEv08_DTable* FSEv08_createDTable (unsigned tableLog) +{ + if (tableLog > FSEv08_TABLELOG_ABSOLUTE_MAX) tableLog = FSEv08_TABLELOG_ABSOLUTE_MAX; + return (FSEv08_DTable*)malloc( FSEv08_DTABLE_SIZE_U32(tableLog) * sizeof (U32) ); +} + +void FSEv08_freeDTable (FSEv08_DTable* dt) +{ + free(dt); +} + +size_t FSEv08_buildDTable(FSEv08_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) +{ + void* const tdPtr = dt+1; /* because *dt is unsigned, 32-bits aligned on 32-bits */ + FSEv08_DECODE_TYPE* const tableDecode = (FSEv08_DECODE_TYPE*) (tdPtr); + U16 symbolNext[FSEv08_MAX_SYMBOL_VALUE+1]; + + U32 const maxSV1 = maxSymbolValue + 1; + U32 const tableSize = 1 << tableLog; + U32 highThreshold = tableSize-1; + + /* Sanity Checks */ + if (maxSymbolValue > FSEv08_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge); + if (tableLog > FSEv08_MAX_TABLELOG) return ERROR(tableLog_tooLarge); + + /* Init, lay down lowprob symbols */ + { FSEv08_DTableHeader DTableH; + DTableH.tableLog = (U16)tableLog; + DTableH.fastMode = 1; + { S16 const largeLimit= (S16)(1 << (tableLog-1)); + U32 s; + for (s=0; s<maxSV1; s++) { + if (normalizedCounter[s]==-1) { + tableDecode[highThreshold--].symbol = (FSEv08_FUNCTION_TYPE)s; + symbolNext[s] = 1; + } else { + if (normalizedCounter[s] >= largeLimit) DTableH.fastMode=0; + symbolNext[s] = normalizedCounter[s]; + } } } + memcpy(dt, &DTableH, sizeof(DTableH)); + } + + /* Spread symbols */ + { U32 const tableMask = tableSize-1; + U32 const step = FSEv08_TABLESTEP(tableSize); + U32 s, position = 0; + for (s=0; s<maxSV1; s++) { + int i; + for (i=0; i<normalizedCounter[s]; i++) { + tableDecode[position].symbol = (FSEv08_FUNCTION_TYPE)s; + position = (position + step) & tableMask; + while (position > highThreshold) position = (position + step) & tableMask; /* lowprob area */ + } } + + if (position!=0) return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ + } + + /* Build Decoding table */ + { U32 u; + for (u=0; u<tableSize; u++) { + FSEv08_FUNCTION_TYPE const symbol = (FSEv08_FUNCTION_TYPE)(tableDecode[u].symbol); + U16 nextState = symbolNext[symbol]++; + tableDecode[u].nbBits = (BYTE) (tableLog - BIT_highbit32 ((U32)nextState) ); + tableDecode[u].newState = (U16) ( (nextState << tableDecode[u].nbBits) - tableSize); + } } + + return 0; +} + + + +#ifndef FSEv08_COMMONDEFS_ONLY + +/*-******************************************************* +* Decompression (Byte symbols) +*********************************************************/ +size_t FSEv08_buildDTable_rle (FSEv08_DTable* dt, BYTE symbolValue) +{ + void* ptr = dt; + FSEv08_DTableHeader* const DTableH = (FSEv08_DTableHeader*)ptr; + void* dPtr = dt + 1; + FSEv08_decode_t* const cell = (FSEv08_decode_t*)dPtr; + + DTableH->tableLog = 0; + DTableH->fastMode = 0; + + cell->newState = 0; + cell->symbol = symbolValue; + cell->nbBits = 0; + + return 0; +} + + +size_t FSEv08_buildDTable_raw (FSEv08_DTable* dt, unsigned nbBits) +{ + void* ptr = dt; + FSEv08_DTableHeader* const DTableH = (FSEv08_DTableHeader*)ptr; + void* dPtr = dt + 1; + FSEv08_decode_t* const dinfo = (FSEv08_decode_t*)dPtr; + const unsigned tableSize = 1 << nbBits; + const unsigned tableMask = tableSize - 1; + const unsigned maxSV1 = tableMask+1; + unsigned s; + + /* Sanity checks */ + if (nbBits < 1) return ERROR(GENERIC); /* min size */ + + /* Build Decoding Table */ + DTableH->tableLog = (U16)nbBits; + DTableH->fastMode = 1; + for (s=0; s<maxSV1; s++) { + dinfo[s].newState = 0; + dinfo[s].symbol = (BYTE)s; + dinfo[s].nbBits = (BYTE)nbBits; + } + + return 0; +} + +FORCE_INLINE size_t FSEv08_decompress_usingDTable_generic( + void* dst, size_t maxDstSize, + const void* cSrc, size_t cSrcSize, + const FSEv08_DTable* dt, const unsigned fast) +{ + BYTE* const ostart = (BYTE*) dst; + BYTE* op = ostart; + BYTE* const omax = op + maxDstSize; + BYTE* const olimit = omax-3; + + BIT_DStream_t bitD; + FSEv08_DState_t state1; + FSEv08_DState_t state2; + + /* Init */ + { size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize); /* replaced last arg by maxCompressed Size */ + if (FSEv08_isError(errorCode)) return errorCode; } + + FSEv08_initDState(&state1, &bitD, dt); + FSEv08_initDState(&state2, &bitD, dt); + +#define FSEv08_GETSYMBOL(statePtr) fast ? FSEv08_decodeSymbolFast(statePtr, &bitD) : FSEv08_decodeSymbol(statePtr, &bitD) + + /* 4 symbols per loop */ + for ( ; (BIT_reloadDStream(&bitD)==BIT_DStream_unfinished) && (op<olimit) ; op+=4) { + op[0] = FSEv08_GETSYMBOL(&state1); + + if (FSEv08_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ + BIT_reloadDStream(&bitD); + + op[1] = FSEv08_GETSYMBOL(&state2); + + if (FSEv08_MAX_TABLELOG*4+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ + { if (BIT_reloadDStream(&bitD) > BIT_DStream_unfinished) { op+=2; break; } } + + op[2] = FSEv08_GETSYMBOL(&state1); + + if (FSEv08_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ + BIT_reloadDStream(&bitD); + + op[3] = FSEv08_GETSYMBOL(&state2); + } + + /* tail */ + /* note : BIT_reloadDStream(&bitD) >= FSEv08_DStream_partiallyFilled; Ends at exactly BIT_DStream_completed */ + while (1) { + if (op>(omax-2)) return ERROR(dstSize_tooSmall); + + *op++ = FSEv08_GETSYMBOL(&state1); + + if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) { + *op++ = FSEv08_GETSYMBOL(&state2); + break; + } + + if (op>(omax-2)) return ERROR(dstSize_tooSmall); + + *op++ = FSEv08_GETSYMBOL(&state2); + + if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) { + *op++ = FSEv08_GETSYMBOL(&state1); + break; + } } + + return op-ostart; +} + + +size_t FSEv08_decompress_usingDTable(void* dst, size_t originalSize, + const void* cSrc, size_t cSrcSize, + const FSEv08_DTable* dt) +{ + const void* ptr = dt; + const FSEv08_DTableHeader* DTableH = (const FSEv08_DTableHeader*)ptr; + const U32 fastMode = DTableH->fastMode; + + /* select fast mode (static) */ + if (fastMode) return FSEv08_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1); + return FSEv08_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0); +} + + +size_t FSEv08_decompress(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize) +{ + const BYTE* const istart = (const BYTE*)cSrc; + const BYTE* ip = istart; + short counting[FSEv08_MAX_SYMBOL_VALUE+1]; + DTable_max_t dt; /* Static analyzer seems unable to understand this table will be properly initialized later */ + unsigned tableLog; + unsigned maxSymbolValue = FSEv08_MAX_SYMBOL_VALUE; + + if (cSrcSize<2) return ERROR(srcSize_wrong); /* too small input size */ + + /* normal FSE decoding mode */ + { size_t const NCountLength = FSEv08_readNCount (counting, &maxSymbolValue, &tableLog, istart, cSrcSize); + if (FSEv08_isError(NCountLength)) return NCountLength; + if (NCountLength >= cSrcSize) return ERROR(srcSize_wrong); /* too small input size */ + ip += NCountLength; + cSrcSize -= NCountLength; + } + + { size_t const errorCode = FSEv08_buildDTable (dt, counting, maxSymbolValue, tableLog); + if (FSEv08_isError(errorCode)) return errorCode; } + + return FSEv08_decompress_usingDTable (dst, maxDstSize, ip, cSrcSize, dt); /* always return, even if it is an error code */ +} + + + +#endif /* FSEv08_COMMONDEFS_ONLY */ +/* + Common functions of Zstd compression library + Copyright (C) 2015-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - zstd homepage : http://www.zstd.net/ +*/ + + + +/*-**************************************** +* ZSTD Error Management +******************************************/ +/*! ZSTDv08_isError() : +* tells if a return value is an error code */ +unsigned ZSTDv08_isError(size_t code) { return ERR_isError(code); } + +/*! ZSTDv08_getErrorName() : +* provides error code string from function result (useful for debugging) */ +const char* ZSTDv08_getErrorName(size_t code) { return ERR_getErrorName(code); } + +/*! ZSTDv08_getError() : +* convert a `size_t` function result into a proper ZSTDv08_errorCode enum */ +ZSTDv08_ErrorCode ZSTDv08_getErrorCode(size_t code) { return ERR_getErrorCode(code); } + +/*! ZSTDv08_getErrorString() : +* provides error code string from enum */ +const char* ZSTDv08_getErrorString(ZSTDv08_ErrorCode code) { return ERR_getErrorName(code); } + + +/* ************************************************************** +* ZBUFF Error Management +****************************************************************/ +unsigned ZBUFFv08_isError(size_t errorCode) { return ERR_isError(errorCode); } + +const char* ZBUFFv08_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); } + + + +void* ZSTDv08_defaultAllocFunction(void* opaque, size_t size) +{ + void* address = malloc(size); + (void)opaque; + /* printf("alloc %p, %d opaque=%p \n", address, (int)size, opaque); */ + return address; +} + +void ZSTDv08_defaultFreeFunction(void* opaque, void* address) +{ + (void)opaque; + /* if (address) printf("free %p opaque=%p \n", address, opaque); */ + free(address); +} +/* ****************************************************************** + Huffman decoder, part of New Generation Entropy library + Copyright (C) 2013-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy + - Public forum : https://groups.google.com/forum/#!forum/lz4c +****************************************************************** */ + +/* ************************************************************** +* Compiler specifics +****************************************************************/ +#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +/* inline is defined */ +#elif defined(_MSC_VER) +# define inline __inline +#else +# define inline /* disable inline */ +#endif + + +#ifdef _MSC_VER /* Visual Studio */ +# define FORCE_INLINE static __forceinline +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#else +# ifdef __GNUC__ +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif +#endif + + +/* ************************************************************** +* Error Management +****************************************************************/ +#define HUFv08_STATIC_ASSERT(c) { enum { HUFv08_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ + + +/*-***************************/ +/* generic DTableDesc */ +/*-***************************/ + +typedef struct { BYTE maxTableLog; BYTE tableType; BYTE tableLog; BYTE reserved; } DTableDesc; + +static DTableDesc HUFv08_getDTableDesc(const HUFv08_DTable* table) +{ + DTableDesc dtd; + memcpy(&dtd, table, sizeof(dtd)); + return dtd; +} + + +/*-***************************/ +/* single-symbol decoding */ +/*-***************************/ + +typedef struct { BYTE byte; BYTE nbBits; } HUFv08_DEltX2; /* single-symbol decoding */ + +size_t HUFv08_readDTableX2 (HUFv08_DTable* DTable, const void* src, size_t srcSize) +{ + BYTE huffWeight[HUFv08_SYMBOLVALUE_MAX + 1]; + U32 rankVal[HUFv08_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */ + U32 tableLog = 0; + U32 nbSymbols = 0; + size_t iSize; + void* const dtPtr = DTable + 1; + HUFv08_DEltX2* const dt = (HUFv08_DEltX2*)dtPtr; + + HUFv08_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUFv08_DTable)); + //memset(huffWeight, 0, sizeof(huffWeight)); /* is not necessary, even though some analyzer complain ... */ + + iSize = HUFv08_readStats(huffWeight, HUFv08_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize); + if (HUFv08_isError(iSize)) return iSize; + + /* Table header */ + { DTableDesc dtd = HUFv08_getDTableDesc(DTable); + if (tableLog > (U32)(dtd.maxTableLog+1)) return ERROR(tableLog_tooLarge); /* DTable too small, huffman tree cannot fit in */ + dtd.tableType = 0; + dtd.tableLog = (BYTE)tableLog; + memcpy(DTable, &dtd, sizeof(dtd)); + } + + /* Prepare ranks */ + { U32 n, nextRankStart = 0; + for (n=1; n<tableLog+1; n++) { + U32 current = nextRankStart; + nextRankStart += (rankVal[n] << (n-1)); + rankVal[n] = current; + } } + + /* fill DTable */ + { U32 n; + for (n=0; n<nbSymbols; n++) { + U32 const w = huffWeight[n]; + U32 const length = (1 << w) >> 1; + U32 i; + HUFv08_DEltX2 D; + D.byte = (BYTE)n; D.nbBits = (BYTE)(tableLog + 1 - w); + for (i = rankVal[w]; i < rankVal[w] + length; i++) + dt[i] = D; + rankVal[w] += length; + } } + + return iSize; +} + + +static BYTE HUFv08_decodeSymbolX2(BIT_DStream_t* Dstream, const HUFv08_DEltX2* dt, const U32 dtLog) +{ + size_t const val = BIT_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */ + BYTE const c = dt[val].byte; + BIT_skipBits(Dstream, dt[val].nbBits); + return c; +} + +#define HUFv08_DECODE_SYMBOLX2_0(ptr, DStreamPtr) \ + *ptr++ = HUFv08_decodeSymbolX2(DStreamPtr, dt, dtLog) + +#define HUFv08_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \ + if (MEM_64bits() || (HUFv08_TABLELOG_MAX<=12)) \ + HUFv08_DECODE_SYMBOLX2_0(ptr, DStreamPtr) + +#define HUFv08_DECODE_SYMBOLX2_2(ptr, DStreamPtr) \ + if (MEM_64bits()) \ + HUFv08_DECODE_SYMBOLX2_0(ptr, DStreamPtr) + +static inline size_t HUFv08_decodeStreamX2(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, const HUFv08_DEltX2* const dt, const U32 dtLog) +{ + BYTE* const pStart = p; + + /* up to 4 symbols at a time */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p <= pEnd-4)) { + HUFv08_DECODE_SYMBOLX2_2(p, bitDPtr); + HUFv08_DECODE_SYMBOLX2_1(p, bitDPtr); + HUFv08_DECODE_SYMBOLX2_2(p, bitDPtr); + HUFv08_DECODE_SYMBOLX2_0(p, bitDPtr); + } + + /* closer to the end */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p < pEnd)) + HUFv08_DECODE_SYMBOLX2_0(p, bitDPtr); + + /* no more data to retrieve from bitstream, hence no need to reload */ + while (p < pEnd) + HUFv08_DECODE_SYMBOLX2_0(p, bitDPtr); + + return pEnd-pStart; +} + +static size_t HUFv08_decompress1X2_usingDTable_internal( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUFv08_DTable* DTable) +{ + BYTE* op = (BYTE*)dst; + BYTE* const oend = op + dstSize; + const void* dtPtr = DTable + 1; + const HUFv08_DEltX2* const dt = (const HUFv08_DEltX2*)dtPtr; + BIT_DStream_t bitD; + DTableDesc const dtd = HUFv08_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; + + { size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize); + if (HUFv08_isError(errorCode)) return errorCode; } + + HUFv08_decodeStreamX2(op, &bitD, oend, dt, dtLog); + + /* check */ + if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); + + return dstSize; +} + +size_t HUFv08_decompress1X2_usingDTable( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUFv08_DTable* DTable) +{ + DTableDesc dtd = HUFv08_getDTableDesc(DTable); + if (dtd.tableType != 0) return ERROR(GENERIC); + return HUFv08_decompress1X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); +} + +size_t HUFv08_decompress1X2_DCtx (HUFv08_DTable* DCtx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t const hSize = HUFv08_readDTableX2 (DCtx, cSrc, cSrcSize); + if (HUFv08_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUFv08_decompress1X2_usingDTable_internal (dst, dstSize, ip, cSrcSize, DCtx); +} + +size_t HUFv08_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + HUFv08_CREATE_STATIC_DTABLEX2(DTable, HUFv08_TABLELOG_MAX); + return HUFv08_decompress1X2_DCtx (DTable, dst, dstSize, cSrc, cSrcSize); +} + + +static size_t HUFv08_decompress4X2_usingDTable_internal( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUFv08_DTable* DTable) +{ + /* Check */ + if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ + + { const BYTE* const istart = (const BYTE*) cSrc; + BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + const void* const dtPtr = DTable + 1; + const HUFv08_DEltX2* const dt = (const HUFv08_DEltX2*)dtPtr; + + /* Init */ + BIT_DStream_t bitD1; + BIT_DStream_t bitD2; + BIT_DStream_t bitD3; + BIT_DStream_t bitD4; + size_t const length1 = MEM_readLE16(istart); + size_t const length2 = MEM_readLE16(istart+2); + size_t const length3 = MEM_readLE16(istart+4); + size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); + const BYTE* const istart1 = istart + 6; /* jumpTable */ + const BYTE* const istart2 = istart1 + length1; + const BYTE* const istart3 = istart2 + length2; + const BYTE* const istart4 = istart3 + length3; + const size_t segmentSize = (dstSize+3) / 4; + BYTE* const opStart2 = ostart + segmentSize; + BYTE* const opStart3 = opStart2 + segmentSize; + BYTE* const opStart4 = opStart3 + segmentSize; + BYTE* op1 = ostart; + BYTE* op2 = opStart2; + BYTE* op3 = opStart3; + BYTE* op4 = opStart4; + U32 endSignal; + DTableDesc const dtd = HUFv08_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; + + if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ + { size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1); + if (HUFv08_isError(errorCode)) return errorCode; } + { size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2); + if (HUFv08_isError(errorCode)) return errorCode; } + { size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3); + if (HUFv08_isError(errorCode)) return errorCode; } + { size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4); + if (HUFv08_isError(errorCode)) return errorCode; } + + /* 16-32 symbols per loop (4-8 symbols per stream) */ + endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); + for ( ; (endSignal==BIT_DStream_unfinished) && (op4<(oend-7)) ; ) { + HUFv08_DECODE_SYMBOLX2_2(op1, &bitD1); + HUFv08_DECODE_SYMBOLX2_2(op2, &bitD2); + HUFv08_DECODE_SYMBOLX2_2(op3, &bitD3); + HUFv08_DECODE_SYMBOLX2_2(op4, &bitD4); + HUFv08_DECODE_SYMBOLX2_1(op1, &bitD1); + HUFv08_DECODE_SYMBOLX2_1(op2, &bitD2); + HUFv08_DECODE_SYMBOLX2_1(op3, &bitD3); + HUFv08_DECODE_SYMBOLX2_1(op4, &bitD4); + HUFv08_DECODE_SYMBOLX2_2(op1, &bitD1); + HUFv08_DECODE_SYMBOLX2_2(op2, &bitD2); + HUFv08_DECODE_SYMBOLX2_2(op3, &bitD3); + HUFv08_DECODE_SYMBOLX2_2(op4, &bitD4); + HUFv08_DECODE_SYMBOLX2_0(op1, &bitD1); + HUFv08_DECODE_SYMBOLX2_0(op2, &bitD2); + HUFv08_DECODE_SYMBOLX2_0(op3, &bitD3); + HUFv08_DECODE_SYMBOLX2_0(op4, &bitD4); + endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); + } + + /* check corruption */ + if (op1 > opStart2) return ERROR(corruption_detected); + if (op2 > opStart3) return ERROR(corruption_detected); + if (op3 > opStart4) return ERROR(corruption_detected); + /* note : op4 supposed already verified within main loop */ + + /* finish bitStreams one by one */ + HUFv08_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog); + HUFv08_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog); + HUFv08_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog); + HUFv08_decodeStreamX2(op4, &bitD4, oend, dt, dtLog); + + /* check */ + endSignal = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); + if (!endSignal) return ERROR(corruption_detected); + + /* decoded size */ + return dstSize; + } +} + + +size_t HUFv08_decompress4X2_usingDTable( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUFv08_DTable* DTable) +{ + DTableDesc dtd = HUFv08_getDTableDesc(DTable); + if (dtd.tableType != 0) return ERROR(GENERIC); + return HUFv08_decompress4X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); +} + + +size_t HUFv08_decompress4X2_DCtx (HUFv08_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t const hSize = HUFv08_readDTableX2 (dctx, cSrc, cSrcSize); + if (HUFv08_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUFv08_decompress4X2_usingDTable_internal (dst, dstSize, ip, cSrcSize, dctx); +} + +size_t HUFv08_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + HUFv08_CREATE_STATIC_DTABLEX2(DTable, HUFv08_TABLELOG_MAX); + return HUFv08_decompress4X2_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); +} + + +/* *************************/ +/* double-symbols decoding */ +/* *************************/ +typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUFv08_DEltX4; /* double-symbols decoding */ + +typedef struct { BYTE symbol; BYTE weight; } sortedSymbol_t; + +static void HUFv08_fillDTableX4Level2(HUFv08_DEltX4* DTable, U32 sizeLog, const U32 consumed, + const U32* rankValOrigin, const int minWeight, + const sortedSymbol_t* sortedSymbols, const U32 sortedListSize, + U32 nbBitsBaseline, U16 baseSeq) +{ + HUFv08_DEltX4 DElt; + U32 rankVal[HUFv08_TABLELOG_ABSOLUTEMAX + 1]; + + /* get pre-calculated rankVal */ + memcpy(rankVal, rankValOrigin, sizeof(rankVal)); + + /* fill skipped values */ + if (minWeight>1) { + U32 i, skipSize = rankVal[minWeight]; + MEM_writeLE16(&(DElt.sequence), baseSeq); + DElt.nbBits = (BYTE)(consumed); + DElt.length = 1; + for (i = 0; i < skipSize; i++) + DTable[i] = DElt; + } + + /* fill DTable */ + { U32 s; for (s=0; s<sortedListSize; s++) { /* note : sortedSymbols already skipped */ + const U32 symbol = sortedSymbols[s].symbol; + const U32 weight = sortedSymbols[s].weight; + const U32 nbBits = nbBitsBaseline - weight; + const U32 length = 1 << (sizeLog-nbBits); + const U32 start = rankVal[weight]; + U32 i = start; + const U32 end = start + length; + + MEM_writeLE16(&(DElt.sequence), (U16)(baseSeq + (symbol << 8))); + DElt.nbBits = (BYTE)(nbBits + consumed); + DElt.length = 2; + do { DTable[i++] = DElt; } while (i<end); /* since length >= 1 */ + + rankVal[weight] += length; + }} +} + +typedef U32 rankVal_t[HUFv08_TABLELOG_ABSOLUTEMAX][HUFv08_TABLELOG_ABSOLUTEMAX + 1]; + +static void HUFv08_fillDTableX4(HUFv08_DEltX4* DTable, const U32 targetLog, + const sortedSymbol_t* sortedList, const U32 sortedListSize, + const U32* rankStart, rankVal_t rankValOrigin, const U32 maxWeight, + const U32 nbBitsBaseline) +{ + U32 rankVal[HUFv08_TABLELOG_ABSOLUTEMAX + 1]; + const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */ + const U32 minBits = nbBitsBaseline - maxWeight; + U32 s; + + memcpy(rankVal, rankValOrigin, sizeof(rankVal)); + + /* fill DTable */ + for (s=0; s<sortedListSize; s++) { + const U16 symbol = sortedList[s].symbol; + const U32 weight = sortedList[s].weight; + const U32 nbBits = nbBitsBaseline - weight; + const U32 start = rankVal[weight]; + const U32 length = 1 << (targetLog-nbBits); + + if (targetLog-nbBits >= minBits) { /* enough room for a second symbol */ + U32 sortedRank; + int minWeight = nbBits + scaleLog; + if (minWeight < 1) minWeight = 1; + sortedRank = rankStart[minWeight]; + HUFv08_fillDTableX4Level2(DTable+start, targetLog-nbBits, nbBits, + rankValOrigin[nbBits], minWeight, + sortedList+sortedRank, sortedListSize-sortedRank, + nbBitsBaseline, symbol); + } else { + HUFv08_DEltX4 DElt; + MEM_writeLE16(&(DElt.sequence), symbol); + DElt.nbBits = (BYTE)(nbBits); + DElt.length = 1; + { U32 u; + const U32 end = start + length; + for (u = start; u < end; u++) DTable[u] = DElt; + } } + rankVal[weight] += length; + } +} + +size_t HUFv08_readDTableX4 (HUFv08_DTable* DTable, const void* src, size_t srcSize) +{ + BYTE weightList[HUFv08_SYMBOLVALUE_MAX + 1]; + sortedSymbol_t sortedSymbol[HUFv08_SYMBOLVALUE_MAX + 1]; + U32 rankStats[HUFv08_TABLELOG_ABSOLUTEMAX + 1] = { 0 }; + U32 rankStart0[HUFv08_TABLELOG_ABSOLUTEMAX + 2] = { 0 }; + U32* const rankStart = rankStart0+1; + rankVal_t rankVal; + U32 tableLog, maxW, sizeOfSort, nbSymbols; + DTableDesc dtd = HUFv08_getDTableDesc(DTable); + U32 const maxTableLog = dtd.maxTableLog; + size_t iSize; + void* dtPtr = DTable+1; /* force compiler to avoid strict-aliasing */ + HUFv08_DEltX4* const dt = (HUFv08_DEltX4*)dtPtr; + + HUFv08_STATIC_ASSERT(sizeof(HUFv08_DEltX4) == sizeof(HUFv08_DTable)); /* if compilation fails here, assertion is false */ + if (maxTableLog > HUFv08_TABLELOG_ABSOLUTEMAX) return ERROR(tableLog_tooLarge); + //memset(weightList, 0, sizeof(weightList)); /* is not necessary, even though some analyzer complain ... */ + + iSize = HUFv08_readStats(weightList, HUFv08_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize); + if (HUFv08_isError(iSize)) return iSize; + + /* check result */ + if (tableLog > maxTableLog) return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */ + + /* find maxWeight */ + for (maxW = tableLog; rankStats[maxW]==0; maxW--) {} /* necessarily finds a solution before 0 */ + + /* Get start index of each weight */ + { U32 w, nextRankStart = 0; + for (w=1; w<maxW+1; w++) { + U32 current = nextRankStart; + nextRankStart += rankStats[w]; + rankStart[w] = current; + } + rankStart[0] = nextRankStart; /* put all 0w symbols at the end of sorted list*/ + sizeOfSort = nextRankStart; + } + + /* sort symbols by weight */ + { U32 s; + for (s=0; s<nbSymbols; s++) { + U32 const w = weightList[s]; + U32 const r = rankStart[w]++; + sortedSymbol[r].symbol = (BYTE)s; + sortedSymbol[r].weight = (BYTE)w; + } + rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */ + } + + /* Build rankVal */ + { U32* const rankVal0 = rankVal[0]; + { int const rescale = (maxTableLog-tableLog) - 1; /* tableLog <= maxTableLog */ + U32 nextRankVal = 0; + U32 w; + for (w=1; w<maxW+1; w++) { + U32 current = nextRankVal; + nextRankVal += rankStats[w] << (w+rescale); + rankVal0[w] = current; + } } + { U32 const minBits = tableLog+1 - maxW; + U32 consumed; + for (consumed = minBits; consumed < maxTableLog - minBits + 1; consumed++) { + U32* const rankValPtr = rankVal[consumed]; + U32 w; + for (w = 1; w < maxW+1; w++) { + rankValPtr[w] = rankVal0[w] >> consumed; + } } } } + + HUFv08_fillDTableX4(dt, maxTableLog, + sortedSymbol, sizeOfSort, + rankStart0, rankVal, maxW, + tableLog+1); + + dtd.tableLog = (BYTE)maxTableLog; + dtd.tableType = 1; + memcpy(DTable, &dtd, sizeof(dtd)); + return iSize; +} + + +static U32 HUFv08_decodeSymbolX4(void* op, BIT_DStream_t* DStream, const HUFv08_DEltX4* dt, const U32 dtLog) +{ + const size_t val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ + memcpy(op, dt+val, 2); + BIT_skipBits(DStream, dt[val].nbBits); + return dt[val].length; +} + +static U32 HUFv08_decodeLastSymbolX4(void* op, BIT_DStream_t* DStream, const HUFv08_DEltX4* dt, const U32 dtLog) +{ + const size_t val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ + memcpy(op, dt+val, 1); + if (dt[val].length==1) BIT_skipBits(DStream, dt[val].nbBits); + else { + if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8)) { + BIT_skipBits(DStream, dt[val].nbBits); + if (DStream->bitsConsumed > (sizeof(DStream->bitContainer)*8)) + DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8); /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */ + } } + return 1; +} + + +#define HUFv08_DECODE_SYMBOLX4_0(ptr, DStreamPtr) \ + ptr += HUFv08_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog) + +#define HUFv08_DECODE_SYMBOLX4_1(ptr, DStreamPtr) \ + if (MEM_64bits() || (HUFv08_TABLELOG_MAX<=12)) \ + ptr += HUFv08_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog) + +#define HUFv08_DECODE_SYMBOLX4_2(ptr, DStreamPtr) \ + if (MEM_64bits()) \ + ptr += HUFv08_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog) + +static inline size_t HUFv08_decodeStreamX4(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, const HUFv08_DEltX4* const dt, const U32 dtLog) +{ + BYTE* const pStart = p; + + /* up to 8 symbols at a time */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p < pEnd-7)) { + HUFv08_DECODE_SYMBOLX4_2(p, bitDPtr); + HUFv08_DECODE_SYMBOLX4_1(p, bitDPtr); + HUFv08_DECODE_SYMBOLX4_2(p, bitDPtr); + HUFv08_DECODE_SYMBOLX4_0(p, bitDPtr); + } + + /* closer to end : up to 2 symbols at a time */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p <= pEnd-2)) + HUFv08_DECODE_SYMBOLX4_0(p, bitDPtr); + + while (p <= pEnd-2) + HUFv08_DECODE_SYMBOLX4_0(p, bitDPtr); /* no need to reload : reached the end of DStream */ + + if (p < pEnd) + p += HUFv08_decodeLastSymbolX4(p, bitDPtr, dt, dtLog); + + return p-pStart; +} + + +static size_t HUFv08_decompress1X4_usingDTable_internal( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUFv08_DTable* DTable) +{ + BIT_DStream_t bitD; + + /* Init */ + { size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize); + if (HUFv08_isError(errorCode)) return errorCode; + } + + /* decode */ + { BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + const void* const dtPtr = DTable+1; /* force compiler to not use strict-aliasing */ + const HUFv08_DEltX4* const dt = (const HUFv08_DEltX4*)dtPtr; + DTableDesc const dtd = HUFv08_getDTableDesc(DTable); + HUFv08_decodeStreamX4(ostart, &bitD, oend, dt, dtd.tableLog); + } + + /* check */ + if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); + + /* decoded size */ + return dstSize; +} + +size_t HUFv08_decompress1X4_usingDTable( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUFv08_DTable* DTable) +{ + DTableDesc dtd = HUFv08_getDTableDesc(DTable); + if (dtd.tableType != 1) return ERROR(GENERIC); + return HUFv08_decompress1X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); +} + +size_t HUFv08_decompress1X4_DCtx (HUFv08_DTable* DCtx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t const hSize = HUFv08_readDTableX4 (DCtx, cSrc, cSrcSize); + if (HUFv08_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUFv08_decompress1X4_usingDTable_internal (dst, dstSize, ip, cSrcSize, DCtx); +} + +size_t HUFv08_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + HUFv08_CREATE_STATIC_DTABLEX4(DTable, HUFv08_TABLELOG_MAX); + return HUFv08_decompress1X4_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); +} + +static size_t HUFv08_decompress4X4_usingDTable_internal( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUFv08_DTable* DTable) +{ + if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ + + { const BYTE* const istart = (const BYTE*) cSrc; + BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + const void* const dtPtr = DTable+1; + const HUFv08_DEltX4* const dt = (const HUFv08_DEltX4*)dtPtr; + + /* Init */ + BIT_DStream_t bitD1; + BIT_DStream_t bitD2; + BIT_DStream_t bitD3; + BIT_DStream_t bitD4; + size_t const length1 = MEM_readLE16(istart); + size_t const length2 = MEM_readLE16(istart+2); + size_t const length3 = MEM_readLE16(istart+4); + size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); + const BYTE* const istart1 = istart + 6; /* jumpTable */ + const BYTE* const istart2 = istart1 + length1; + const BYTE* const istart3 = istart2 + length2; + const BYTE* const istart4 = istart3 + length3; + size_t const segmentSize = (dstSize+3) / 4; + BYTE* const opStart2 = ostart + segmentSize; + BYTE* const opStart3 = opStart2 + segmentSize; + BYTE* const opStart4 = opStart3 + segmentSize; + BYTE* op1 = ostart; + BYTE* op2 = opStart2; + BYTE* op3 = opStart3; + BYTE* op4 = opStart4; + U32 endSignal; + DTableDesc const dtd = HUFv08_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; + + if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ + { size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1); + if (HUFv08_isError(errorCode)) return errorCode; } + { size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2); + if (HUFv08_isError(errorCode)) return errorCode; } + { size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3); + if (HUFv08_isError(errorCode)) return errorCode; } + { size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4); + if (HUFv08_isError(errorCode)) return errorCode; } + + /* 16-32 symbols per loop (4-8 symbols per stream) */ + endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); + for ( ; (endSignal==BIT_DStream_unfinished) && (op4<(oend-7)) ; ) { + HUFv08_DECODE_SYMBOLX4_2(op1, &bitD1); + HUFv08_DECODE_SYMBOLX4_2(op2, &bitD2); + HUFv08_DECODE_SYMBOLX4_2(op3, &bitD3); + HUFv08_DECODE_SYMBOLX4_2(op4, &bitD4); + HUFv08_DECODE_SYMBOLX4_1(op1, &bitD1); + HUFv08_DECODE_SYMBOLX4_1(op2, &bitD2); + HUFv08_DECODE_SYMBOLX4_1(op3, &bitD3); + HUFv08_DECODE_SYMBOLX4_1(op4, &bitD4); + HUFv08_DECODE_SYMBOLX4_2(op1, &bitD1); + HUFv08_DECODE_SYMBOLX4_2(op2, &bitD2); + HUFv08_DECODE_SYMBOLX4_2(op3, &bitD3); + HUFv08_DECODE_SYMBOLX4_2(op4, &bitD4); + HUFv08_DECODE_SYMBOLX4_0(op1, &bitD1); + HUFv08_DECODE_SYMBOLX4_0(op2, &bitD2); + HUFv08_DECODE_SYMBOLX4_0(op3, &bitD3); + HUFv08_DECODE_SYMBOLX4_0(op4, &bitD4); + + endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); + } + + /* check corruption */ + if (op1 > opStart2) return ERROR(corruption_detected); + if (op2 > opStart3) return ERROR(corruption_detected); + if (op3 > opStart4) return ERROR(corruption_detected); + /* note : op4 supposed already verified within main loop */ + + /* finish bitStreams one by one */ + HUFv08_decodeStreamX4(op1, &bitD1, opStart2, dt, dtLog); + HUFv08_decodeStreamX4(op2, &bitD2, opStart3, dt, dtLog); + HUFv08_decodeStreamX4(op3, &bitD3, opStart4, dt, dtLog); + HUFv08_decodeStreamX4(op4, &bitD4, oend, dt, dtLog); + + /* check */ + { U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); + if (!endCheck) return ERROR(corruption_detected); } + + /* decoded size */ + return dstSize; + } +} + + +size_t HUFv08_decompress4X4_usingDTable( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUFv08_DTable* DTable) +{ + DTableDesc dtd = HUFv08_getDTableDesc(DTable); + if (dtd.tableType != 1) return ERROR(GENERIC); + return HUFv08_decompress4X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); +} + + +size_t HUFv08_decompress4X4_DCtx (HUFv08_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t hSize = HUFv08_readDTableX4 (dctx, cSrc, cSrcSize); + if (HUFv08_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUFv08_decompress4X4_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx); +} + +size_t HUFv08_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + HUFv08_CREATE_STATIC_DTABLEX4(DTable, HUFv08_TABLELOG_MAX); + return HUFv08_decompress4X4_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); +} + + +/* ********************************/ +/* Generic decompression selector */ +/* ********************************/ + +size_t HUFv08_decompress1X_usingDTable(void* dst, size_t maxDstSize, + const void* cSrc, size_t cSrcSize, + const HUFv08_DTable* DTable) +{ + DTableDesc const dtd = HUFv08_getDTableDesc(DTable); + return dtd.tableType ? HUFv08_decompress1X4_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable) : + HUFv08_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable); +} + +size_t HUFv08_decompress4X_usingDTable(void* dst, size_t maxDstSize, + const void* cSrc, size_t cSrcSize, + const HUFv08_DTable* DTable) +{ + DTableDesc const dtd = HUFv08_getDTableDesc(DTable); + return dtd.tableType ? HUFv08_decompress4X4_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable) : + HUFv08_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable); +} + + +typedef struct { U32 tableTime; U32 decode256Time; } algo_time_t; +static const algo_time_t algoTime[16 /* Quantization */][3 /* single, double, quad */] = +{ + /* single, double, quad */ + {{0,0}, {1,1}, {2,2}}, /* Q==0 : impossible */ + {{0,0}, {1,1}, {2,2}}, /* Q==1 : impossible */ + {{ 38,130}, {1313, 74}, {2151, 38}}, /* Q == 2 : 12-18% */ + {{ 448,128}, {1353, 74}, {2238, 41}}, /* Q == 3 : 18-25% */ + {{ 556,128}, {1353, 74}, {2238, 47}}, /* Q == 4 : 25-32% */ + {{ 714,128}, {1418, 74}, {2436, 53}}, /* Q == 5 : 32-38% */ + {{ 883,128}, {1437, 74}, {2464, 61}}, /* Q == 6 : 38-44% */ + {{ 897,128}, {1515, 75}, {2622, 68}}, /* Q == 7 : 44-50% */ + {{ 926,128}, {1613, 75}, {2730, 75}}, /* Q == 8 : 50-56% */ + {{ 947,128}, {1729, 77}, {3359, 77}}, /* Q == 9 : 56-62% */ + {{1107,128}, {2083, 81}, {4006, 84}}, /* Q ==10 : 62-69% */ + {{1177,128}, {2379, 87}, {4785, 88}}, /* Q ==11 : 69-75% */ + {{1242,128}, {2415, 93}, {5155, 84}}, /* Q ==12 : 75-81% */ + {{1349,128}, {2644,106}, {5260,106}}, /* Q ==13 : 81-87% */ + {{1455,128}, {2422,124}, {4174,124}}, /* Q ==14 : 87-93% */ + {{ 722,128}, {1891,145}, {1936,146}}, /* Q ==15 : 93-99% */ +}; + +/** HUFv08_selectDecoder() : +* Tells which decoder is likely to decode faster, +* based on a set of pre-determined metrics. +* @return : 0==HUFv08_decompress4X2, 1==HUFv08_decompress4X4 . +* Assumption : 0 < cSrcSize < dstSize <= 128 KB */ +U32 HUFv08_selectDecoder (size_t dstSize, size_t cSrcSize) +{ + /* decoder timing evaluation */ + U32 const Q = (U32)(cSrcSize * 16 / dstSize); /* Q < 16 since dstSize > cSrcSize */ + U32 const D256 = (U32)(dstSize >> 8); + U32 const DTime0 = algoTime[Q][0].tableTime + (algoTime[Q][0].decode256Time * D256); + U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256); + DTime1 += DTime1 >> 3; /* advantage to algorithm using less memory, for cache eviction */ + + return DTime1 < DTime0; +} + + +typedef size_t (*decompressionAlgo)(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); + +size_t HUFv08_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + static const decompressionAlgo decompress[2] = { HUFv08_decompress4X2, HUFv08_decompress4X4 }; + + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ + if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ + if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ + + { U32 const algoNb = HUFv08_selectDecoder(dstSize, cSrcSize); + return decompress[algoNb](dst, dstSize, cSrc, cSrcSize); + } + + //return HUFv08_decompress4X2(dst, dstSize, cSrc, cSrcSize); /* multi-streams single-symbol decoding */ + //return HUFv08_decompress4X4(dst, dstSize, cSrc, cSrcSize); /* multi-streams double-symbols decoding */ +} + +size_t HUFv08_decompress4X_DCtx (HUFv08_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ + if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ + if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ + + { U32 const algoNb = HUFv08_selectDecoder(dstSize, cSrcSize); + return algoNb ? HUFv08_decompress4X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) : + HUFv08_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ; + } +} + +size_t HUFv08_decompress4X_hufOnly (HUFv08_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if ((cSrcSize >= dstSize) || (cSrcSize <= 1)) return ERROR(corruption_detected); /* invalid */ + + { U32 const algoNb = HUFv08_selectDecoder(dstSize, cSrcSize); + return algoNb ? HUFv08_decompress4X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) : + HUFv08_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ; + } +} + +size_t HUFv08_decompress1X_DCtx (HUFv08_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ + if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ + if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ + + { U32 const algoNb = HUFv08_selectDecoder(dstSize, cSrcSize); + return algoNb ? HUFv08_decompress1X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) : + HUFv08_decompress1X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ; + } +} +/* + zstd_internal - common functions to include + Header File for include + Copyright (C) 2014-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - zstd homepage : https://www.zstd.net +*/ +#ifndef ZSTDv08_CCOMMON_H_MODULE +#define ZSTDv08_CCOMMON_H_MODULE + + +/*-************************************* +* Common macros +***************************************/ +#define MIN(a,b) ((a)<(b) ? (a) : (b)) +#define MAX(a,b) ((a)>(b) ? (a) : (b)) + + +/*-************************************* +* Common constants +***************************************/ +#define ZSTDv08_OPT_DEBUG 0 /* 3 = compression stats; 5 = check encoded sequences; 9 = full logs */ +#if defined(ZSTDv08_OPT_DEBUG) && ZSTDv08_OPT_DEBUG>=9 + #include <stdio.h> + #include <stdlib.h> + #define ZSTDv08_LOG_PARSER(...) printf(__VA_ARGS__) + #define ZSTDv08_LOG_ENCODE(...) printf(__VA_ARGS__) + #define ZSTDv08_LOG_BLOCK(...) printf(__VA_ARGS__) +#else + #define ZSTDv08_LOG_PARSER(...) + #define ZSTDv08_LOG_ENCODE(...) + #define ZSTDv08_LOG_BLOCK(...) +#endif + +#define ZSTDv08_OPT_NUM (1<<12) +#define ZSTDv08_DICT_MAGIC 0xEC30A437 /* v0.7+ */ + +#define ZSTDv08_REP_NUM 3 /* number of repcodes */ +#define ZSTDv08_REP_CHECK (ZSTDv08_REP_NUM-0) /* number of repcodes to check by the optimal parser */ +#define ZSTDv08_REP_MOVE (ZSTDv08_REP_NUM-1) +static const U32 repStartValue[ZSTDv08_REP_NUM] = { 1, 4, 8 }; + +#define KB *(1 <<10) +#define MB *(1 <<20) +#define GB *(1U<<30) + +#define BIT7 128 +#define BIT6 64 +#define BIT5 32 +#define BIT4 16 +#define BIT1 2 +#define BIT0 1 + +#define ZSTDv08_WINDOWLOG_ABSOLUTEMIN 10 +static const size_t ZSTDv08_fcs_fieldSize[4] = { 0, 2, 4, 8 }; +static const size_t ZSTDv08_did_fieldSize[4] = { 0, 1, 2, 4 }; + +#define ZSTDv08_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */ +static const size_t ZSTDv08_blockHeaderSize = ZSTDv08_BLOCKHEADERSIZE; +typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e; + +#define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */ +#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */ + MIN_SEQUENCES_SIZE /* nbSeq==0 */) /* for a non-null block */ + +#define HufLog 12 +typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingType_e; + +#define LONGNBSEQ 0x7F00 + +#define MINMATCH 3 +#define EQUAL_READ32 4 + +#define Litbits 8 +#define MaxLit ((1<<Litbits) - 1) +#define MaxML 52 +#define MaxLL 35 +#define MaxOff 28 +#define MaxSeq MAX(MaxLL, MaxML) /* Assumption : MaxOff < MaxLL,MaxML */ +#define MLFSELog 9 +#define LLFSELog 9 +#define OffFSELog 8 + +static const U32 LL_bits[MaxLL+1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 2, 2, 3, 3, 4, 6, 7, 8, 9,10,11,12, + 13,14,15,16 }; +static const S16 LL_defaultNorm[MaxLL+1] = { 4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1, + -1,-1,-1,-1 }; +static const U32 LL_defaultNormLog = 6; + +static const U32 ML_bits[MaxML+1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 7, 8, 9,10,11, + 12,13,14,15,16 }; +static const S16 ML_defaultNorm[MaxML+1] = { 1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,-1,-1, + -1,-1,-1,-1,-1 }; +static const U32 ML_defaultNormLog = 6; + +static const S16 OF_defaultNorm[MaxOff+1] = { 1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1,-1,-1,-1,-1,-1 }; +static const U32 OF_defaultNormLog = 5; + + +/*-******************************************* +* Shared functions to include for inlining +*********************************************/ +static void ZSTDv08_copy8(void* dst, const void* src) { memcpy(dst, src, 8); } +#define COPY8(d,s) { ZSTDv08_copy8(d,s); d+=8; s+=8; } + +/*! ZSTDv08_wildcopy() : +* custom version of memcpy(), can copy up to 7 bytes too many (8 bytes if length==0) */ +#define WILDCOPY_OVERLENGTH 8 +MEM_STATIC void ZSTDv08_wildcopy(void* dst, const void* src, size_t length) +{ + const BYTE* ip = (const BYTE*)src; + BYTE* op = (BYTE*)dst; + BYTE* const oend = op + length; + do + COPY8(op, ip) + while (op < oend); +} + + +/*-******************************************* +* Private interfaces +*********************************************/ +typedef struct ZSTDv08_stats_s ZSTDv08_stats_t; + +typedef struct { + U32 off; + U32 len; +} ZSTDv08_match_t; + +typedef struct { + U32 price; + U32 off; + U32 mlen; + U32 litlen; + U32 rep[ZSTDv08_REP_NUM]; +} ZSTDv08_optimal_t; + +//#if ZSTDv08_OPT_DEBUG == 3 +// #include ".debug/zstd_stats.h" +//#else + struct ZSTDv08_stats_s { U32 unused; }; + MEM_STATIC void ZSTDv08_statsPrint(ZSTDv08_stats_t* stats, U32 searchLength) { (void)stats; (void)searchLength; } + MEM_STATIC void ZSTDv08_statsInit(ZSTDv08_stats_t* stats) { (void)stats; } + MEM_STATIC void ZSTDv08_statsResetFreqs(ZSTDv08_stats_t* stats) { (void)stats; } + MEM_STATIC void ZSTDv08_statsUpdatePrices(ZSTDv08_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) { (void)stats; (void)litLength; (void)literals; (void)offset; (void)matchLength; } +//#endif /* #if ZSTDv08_OPT_DEBUG == 3 */ + + +typedef struct seqDef_s { + U32 offset; + U16 litLength; + U16 matchLength; +} seqDef; + + +typedef struct { + seqDef* sequencesStart; + seqDef* sequences; + BYTE* litStart; + BYTE* lit; + BYTE* llCode; + BYTE* mlCode; + BYTE* ofCode; + U32 longLengthID; /* 0 == no longLength; 1 == Lit.longLength; 2 == Match.longLength; */ + U32 longLengthPos; + /* opt */ + ZSTDv08_optimal_t* priceTable; + ZSTDv08_match_t* matchTable; + U32* matchLengthFreq; + U32* litLengthFreq; + U32* litFreq; + U32* offCodeFreq; + U32 matchLengthSum; + U32 matchSum; + U32 litLengthSum; + U32 litSum; + U32 offCodeSum; + U32 log2matchLengthSum; + U32 log2matchSum; + U32 log2litLengthSum; + U32 log2litSum; + U32 log2offCodeSum; + U32 factor; + U32 cachedPrice; + U32 cachedLitLength; + const BYTE* cachedLiterals; + ZSTDv08_stats_t stats; +} seqStore_t; + +void ZSTDv08_seqToCodes(const seqStore_t* seqStorePtr); +int ZSTDv08_isSkipFrame(ZSTDv08_DCtx* dctx); + +/* custom memory allocation functions */ +void* ZSTDv08_defaultAllocFunction(void* opaque, size_t size); +void ZSTDv08_defaultFreeFunction(void* opaque, void* address); +static const ZSTDv08_customMem defaultCustomMem = { ZSTDv08_defaultAllocFunction, ZSTDv08_defaultFreeFunction, NULL }; + +/*====== common function ======*/ + +MEM_STATIC U32 ZSTDv08_highbit32(U32 val) +{ +# if defined(_MSC_VER) /* Visual */ + unsigned long r=0; + _BitScanReverse(&r, val); + return (unsigned)r; +# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */ + return 31 - __builtin_clz(val); +# else /* Software version */ + static const int DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 }; + U32 v = val; + int r; + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + r = DeBruijnClz[(U32)(v * 0x07C4ACDDU) >> 27]; + return r; +# endif +} + + +#endif /* ZSTDv08_CCOMMON_H_MODULE */ +/* + zstd - standard compression library + Copyright (C) 2014-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - zstd homepage : http://www.zstd.net +*/ + +/* *************************************************************** +* Tuning parameters +*****************************************************************/ +/*! + * HEAPMODE : + * Select how default decompression function ZSTDv08_decompress() will allocate memory, + * in memory stack (0), or in memory heap (1, requires malloc()) + */ +#ifndef ZSTDv08_HEAPMODE +# define ZSTDv08_HEAPMODE 1 +#endif + + + +/*-******************************************************* +* Compiler specifics +*********************************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# define FORCE_INLINE static __forceinline +# include <intrin.h> /* For Visual 2005 */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4324) /* disable: C4324: padded structure */ +#else +# ifdef __GNUC__ +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif +#endif + + +/*-************************************* +* Macros +***************************************/ +#define ZSTDv08_isError ERR_isError /* for inlining */ +#define FSEv08_isError ERR_isError +#define HUFv08_isError ERR_isError + + +/*_******************************************************* +* Memory operations +**********************************************************/ +static void ZSTDv08_copy4(void* dst, const void* src) { memcpy(dst, src, 4); } + + +/*-************************************************************* +* Context management +***************************************************************/ +typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader, + ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock, + ZSTDds_decompressLastBlock, ZSTDds_checkChecksum, + ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTDv08_dStage; + +struct ZSTDv08_DCtx_s +{ + FSEv08_DTable LLTable[FSEv08_DTABLE_SIZE_U32(LLFSELog)]; + FSEv08_DTable OffTable[FSEv08_DTABLE_SIZE_U32(OffFSELog)]; + FSEv08_DTable MLTable[FSEv08_DTABLE_SIZE_U32(MLFSELog)]; + HUFv08_DTable hufTable[HUFv08_DTABLE_SIZE(HufLog)]; /* can accommodate HUFv08_decompress4X */ + const void* previousDstEnd; + const void* base; + const void* vBase; + const void* dictEnd; + size_t expected; + U32 rep[ZSTDv08_REP_NUM]; + ZSTDv08_frameParams fParams; + blockType_e bType; /* used in ZSTDv08_decompressContinue(), to transfer blockType between header decoding and block decoding stages */ + ZSTDv08_dStage stage; + U32 litEntropy; + U32 fseEntropy; + XXH64_state_t xxhState; + size_t headerSize; + U32 dictID; + const BYTE* litPtr; + ZSTDv08_customMem customMem; + size_t litBufSize; + size_t litSize; + size_t rleSize; + BYTE litBuffer[ZSTDv08_BLOCKSIZE_ABSOLUTEMAX + WILDCOPY_OVERLENGTH]; + BYTE headerBuffer[ZSTDv08_FRAMEHEADERSIZE_MAX]; +}; /* typedef'd to ZSTDv08_DCtx within "zstd_static.h" */ + +size_t ZSTDv08_sizeofDCtx (const ZSTDv08_DCtx* dctx) { return sizeof(*dctx); } + +size_t ZSTDv08_estimateDCtxSize(void) { return sizeof(ZSTDv08_DCtx); } + +size_t ZSTDv08_decompressBegin(ZSTDv08_DCtx* dctx) +{ + dctx->expected = ZSTDv08_frameHeaderSize_min; + dctx->stage = ZSTDds_getFrameHeaderSize; + dctx->previousDstEnd = NULL; + dctx->base = NULL; + dctx->vBase = NULL; + dctx->dictEnd = NULL; + dctx->hufTable[0] = (HUFv08_DTable)((HufLog)*0x1000001); + dctx->litEntropy = dctx->fseEntropy = 0; + dctx->dictID = 0; + { int i; for (i=0; i<ZSTDv08_REP_NUM; i++) dctx->rep[i] = repStartValue[i]; } + return 0; +} + +ZSTDv08_DCtx* ZSTDv08_createDCtx_advanced(ZSTDv08_customMem customMem) +{ + ZSTDv08_DCtx* dctx; + + if (!customMem.customAlloc && !customMem.customFree) + customMem = defaultCustomMem; + + if (!customMem.customAlloc || !customMem.customFree) + return NULL; + + dctx = (ZSTDv08_DCtx*) customMem.customAlloc(customMem.opaque, sizeof(ZSTDv08_DCtx)); + if (!dctx) return NULL; + memcpy(&dctx->customMem, &customMem, sizeof(ZSTDv08_customMem)); + ZSTDv08_decompressBegin(dctx); + return dctx; +} + +ZSTDv08_DCtx* ZSTDv08_createDCtx(void) +{ + return ZSTDv08_createDCtx_advanced(defaultCustomMem); +} + +size_t ZSTDv08_freeDCtx(ZSTDv08_DCtx* dctx) +{ + if (dctx==NULL) return 0; /* support free on NULL */ + dctx->customMem.customFree(dctx->customMem.opaque, dctx); + return 0; /* reserved as a potential error code in the future */ +} + +void ZSTDv08_copyDCtx(ZSTDv08_DCtx* dstDCtx, const ZSTDv08_DCtx* srcDCtx) +{ + memcpy(dstDCtx, srcDCtx, + sizeof(ZSTDv08_DCtx) - (ZSTDv08_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH + ZSTDv08_frameHeaderSize_max)); /* no need to copy workspace */ +} + + +/*-************************************************************* +* Decompression section +***************************************************************/ + +/* See compression format details in : zstd_compression_format.md */ + +/** ZSTDv08_frameHeaderSize() : +* srcSize must be >= ZSTDv08_frameHeaderSize_min. +* @return : size of the Frame Header */ +static size_t ZSTDv08_frameHeaderSize(const void* src, size_t srcSize) +{ + if (srcSize < ZSTDv08_frameHeaderSize_min) return ERROR(srcSize_wrong); + { BYTE const fhd = ((const BYTE*)src)[4]; + U32 const dictID= fhd & 3; + U32 const singleSegment = (fhd >> 5) & 1; + U32 const fcsId = fhd >> 6; + return ZSTDv08_frameHeaderSize_min + !singleSegment + ZSTDv08_did_fieldSize[dictID] + ZSTDv08_fcs_fieldSize[fcsId] + + (singleSegment && !ZSTDv08_fcs_fieldSize[fcsId]); + } +} + + +/** ZSTDv08_getFrameParams() : +* decode Frame Header, or require larger `srcSize`. +* @return : 0, `fparamsPtr` is correctly filled, +* >0, `srcSize` is too small, result is expected `srcSize`, +* or an error code, which can be tested using ZSTDv08_isError() */ +size_t ZSTDv08_getFrameParams(ZSTDv08_frameParams* fparamsPtr, const void* src, size_t srcSize) +{ + const BYTE* ip = (const BYTE*)src; + + if (srcSize < ZSTDv08_frameHeaderSize_min) return ZSTDv08_frameHeaderSize_min; + if (MEM_readLE32(src) != ZSTDv08_MAGICNUMBER) { + if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTDv08_MAGIC_SKIPPABLE_START) { + if (srcSize < ZSTDv08_skippableHeaderSize) return ZSTDv08_skippableHeaderSize; /* magic number + skippable frame length */ + memset(fparamsPtr, 0, sizeof(*fparamsPtr)); + fparamsPtr->frameContentSize = MEM_readLE32((const char *)src + 4); + fparamsPtr->windowSize = 0; /* windowSize==0 means a frame is skippable */ + return 0; + } + return ERROR(prefix_unknown); + } + + /* ensure there is enough `srcSize` to fully read/decode frame header */ + { size_t const fhsize = ZSTDv08_frameHeaderSize(src, srcSize); + if (srcSize < fhsize) return fhsize; } + + { BYTE const fhdByte = ip[4]; + size_t pos = 5; + U32 const dictIDSizeCode = fhdByte&3; + U32 const checksumFlag = (fhdByte>>2)&1; + U32 const singleSegment = (fhdByte>>5)&1; + U32 const fcsID = fhdByte>>6; + U32 const windowSizeMax = 1U << ZSTDv08_WINDOWLOG_MAX; + U32 windowSize = 0; + U32 dictID = 0; + U64 frameContentSize = 0; + if ((fhdByte & 0x08) != 0) return ERROR(frameParameter_unsupported); /* reserved bits, which must be zero */ + if (!singleSegment) { + BYTE const wlByte = ip[pos++]; + U32 const windowLog = (wlByte >> 3) + ZSTDv08_WINDOWLOG_ABSOLUTEMIN; + if (windowLog > ZSTDv08_WINDOWLOG_MAX) return ERROR(frameParameter_unsupported); + windowSize = (1U << windowLog); + windowSize += (windowSize >> 3) * (wlByte&7); + } + + switch(dictIDSizeCode) + { + default: /* impossible */ + case 0 : break; + case 1 : dictID = ip[pos]; pos++; break; + case 2 : dictID = MEM_readLE16(ip+pos); pos+=2; break; + case 3 : dictID = MEM_readLE32(ip+pos); pos+=4; break; + } + switch(fcsID) + { + default: /* impossible */ + case 0 : if (singleSegment) frameContentSize = ip[pos]; break; + case 1 : frameContentSize = MEM_readLE16(ip+pos)+256; break; + case 2 : frameContentSize = MEM_readLE32(ip+pos); break; + case 3 : frameContentSize = MEM_readLE64(ip+pos); break; + } + if (!windowSize) windowSize = (U32)frameContentSize; + if (windowSize > windowSizeMax) return ERROR(frameParameter_unsupported); + fparamsPtr->frameContentSize = frameContentSize; + fparamsPtr->windowSize = windowSize; + fparamsPtr->dictID = dictID; + fparamsPtr->checksumFlag = checksumFlag; + } + return 0; +} + + +/** ZSTDv08_getDecompressedSize() : +* compatible with legacy mode +* @return : decompressed size if known, 0 otherwise + note : 0 can mean any of the following : + - decompressed size is not present within frame header + - frame header unknown / not supported + - frame header not complete (`srcSize` too small) */ +unsigned long long ZSTDv08_getDecompressedSize(const void* src, size_t srcSize) +{ + { ZSTDv08_frameParams fparams; + size_t const frResult = ZSTDv08_getFrameParams(&fparams, src, srcSize); + if (frResult!=0) return 0; + return fparams.frameContentSize; + } +} + + +/** ZSTDv08_decodeFrameHeader() : +* `srcSize` must be the size provided by ZSTDv08_frameHeaderSize(). +* @return : 0 if success, or an error code, which can be tested using ZSTDv08_isError() */ +static size_t ZSTDv08_decodeFrameHeader(ZSTDv08_DCtx* dctx, const void* src, size_t srcSize) +{ + size_t const result = ZSTDv08_getFrameParams(&(dctx->fParams), src, srcSize); + if (dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID)) return ERROR(dictionary_wrong); + if (dctx->fParams.checksumFlag) XXH64_reset(&dctx->xxhState, 0); + return result; +} + + +typedef struct +{ + blockType_e blockType; + U32 lastBlock; + U32 origSize; +} blockProperties_t; + +/*! ZSTDv08_getcBlockSize() : +* Provides the size of compressed block from block header `src` */ +size_t ZSTDv08_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr) +{ + if (srcSize < ZSTDv08_blockHeaderSize) return ERROR(srcSize_wrong); + { U32 const cBlockHeader = MEM_readLE24(src); + U32 const cSize = cBlockHeader >> 3; + bpPtr->lastBlock = cBlockHeader & 1; + bpPtr->blockType = (blockType_e)((cBlockHeader >> 1) & 3); + bpPtr->origSize = cSize; /* only useful for RLE */ + if (bpPtr->blockType == bt_rle) return 1; + if (bpPtr->blockType == bt_reserved) return ERROR(corruption_detected); + return cSize; + } +} + + +static size_t ZSTDv08_copyRawBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + if (srcSize > dstCapacity) return ERROR(dstSize_tooSmall); + memcpy(dst, src, srcSize); + return srcSize; +} + + +static size_t ZSTDv08_setRleBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize, size_t regenSize) +{ + if (srcSize != 1) return ERROR(srcSize_wrong); + if (regenSize > dstCapacity) return ERROR(dstSize_tooSmall); + memset(dst, *(const BYTE*)src, regenSize); + return regenSize; +} + +/*! ZSTDv08_decodeLiteralsBlock() : + @return : nb of bytes read from src (< srcSize ) */ +size_t ZSTDv08_decodeLiteralsBlock(ZSTDv08_DCtx* dctx, + const void* src, size_t srcSize) /* note : srcSize < BLOCKSIZE */ +{ + if (srcSize < MIN_CBLOCK_SIZE) return ERROR(corruption_detected); + + { const BYTE* const istart = (const BYTE*) src; + symbolEncodingType_e const litEncType = (symbolEncodingType_e)(istart[0] & 3); + + switch(litEncType) + { + case set_repeat: + if (dctx->litEntropy==0) return ERROR(dictionary_corrupted); + /* fall-through */ + case set_compressed: + if (srcSize < 5) return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for case 3 */ + { size_t lhSize, litSize, litCSize; + U32 singleStream=0; + U32 const lhlCode = (istart[0] >> 2) & 3; + U32 const lhc = MEM_readLE32(istart); + switch(lhlCode) + { + case 0: case 1: default: /* note : default is impossible, since lhlCode into [0..3] */ + /* 2 - 2 - 10 - 10 */ + { singleStream = !lhlCode; + lhSize = 3; + litSize = (lhc >> 4) & 0x3FF; + litCSize = (lhc >> 14) & 0x3FF; + break; + } + case 2: + /* 2 - 2 - 14 - 14 */ + { lhSize = 4; + litSize = (lhc >> 4) & 0x3FFF; + litCSize = lhc >> 18; + break; + } + case 3: + /* 2 - 2 - 18 - 18 */ + { lhSize = 5; + litSize = (lhc >> 4) & 0x3FFFF; + litCSize = (lhc >> 22) + (istart[4] << 10); + break; + } + } + if (litSize > ZSTDv08_BLOCKSIZE_ABSOLUTEMAX) return ERROR(corruption_detected); + if (litCSize + lhSize > srcSize) return ERROR(corruption_detected); + + if (HUFv08_isError((litEncType==set_repeat) ? + ( singleStream ? + HUFv08_decompress1X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->hufTable) : + HUFv08_decompress4X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->hufTable) ) : + ( singleStream ? + HUFv08_decompress1X2_DCtx(dctx->hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize) : + HUFv08_decompress4X_hufOnly (dctx->hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize)) )) + return ERROR(corruption_detected); + + dctx->litPtr = dctx->litBuffer; + dctx->litBufSize = ZSTDv08_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH; + dctx->litSize = litSize; + dctx->litEntropy = 1; + return litCSize + lhSize; + } + + case set_basic: + { size_t litSize, lhSize; + U32 const lhlCode = ((istart[0]) >> 2) & 3; + switch(lhlCode) + { + case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */ + lhSize = 1; + litSize = istart[0] >> 3; + break; + case 1: + lhSize = 2; + litSize = MEM_readLE16(istart) >> 4; + break; + case 3: + lhSize = 3; + litSize = MEM_readLE24(istart) >> 4; + break; + } + + if (lhSize+litSize+WILDCOPY_OVERLENGTH > srcSize) { /* risk reading beyond src buffer with wildcopy */ + if (litSize+lhSize > srcSize) return ERROR(corruption_detected); + memcpy(dctx->litBuffer, istart+lhSize, litSize); + dctx->litPtr = dctx->litBuffer; + dctx->litBufSize = ZSTDv08_BLOCKSIZE_ABSOLUTEMAX+8; + dctx->litSize = litSize; + return lhSize+litSize; + } + /* direct reference into compressed stream */ + dctx->litPtr = istart+lhSize; + dctx->litBufSize = srcSize-lhSize; + dctx->litSize = litSize; + return lhSize+litSize; + } + + case set_rle: + { U32 const lhlCode = ((istart[0]) >> 2) & 3; + size_t litSize, lhSize; + switch(lhlCode) + { + case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */ + lhSize = 1; + litSize = istart[0] >> 3; + break; + case 1: + lhSize = 2; + litSize = MEM_readLE16(istart) >> 4; + break; + case 3: + lhSize = 3; + litSize = MEM_readLE24(istart) >> 4; + if (srcSize<4) return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4 */ + break; + } + if (litSize > ZSTDv08_BLOCKSIZE_ABSOLUTEMAX) return ERROR(corruption_detected); + memset(dctx->litBuffer, istart[lhSize], litSize); + dctx->litPtr = dctx->litBuffer; + dctx->litBufSize = ZSTDv08_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH; + dctx->litSize = litSize; + return lhSize+1; + } + default: + return ERROR(corruption_detected); /* impossible */ + } + + } +} + + +/*! ZSTDv08_buildSeqTable() : + @return : nb bytes read from src, + or an error code if it fails, testable with ZSTDv08_isError() +*/ +FORCE_INLINE size_t ZSTDv08_buildSeqTable(FSEv08_DTable* DTable, symbolEncodingType_e type, U32 max, U32 maxLog, + const void* src, size_t srcSize, + const S16* defaultNorm, U32 defaultLog, U32 flagRepeatTable) +{ + switch(type) + { + case set_rle : + if (!srcSize) return ERROR(srcSize_wrong); + if ( (*(const BYTE*)src) > max) return ERROR(corruption_detected); + FSEv08_buildDTable_rle(DTable, *(const BYTE*)src); /* if *src > max, data is corrupted */ + return 1; + case set_basic : + FSEv08_buildDTable(DTable, defaultNorm, max, defaultLog); + return 0; + case set_repeat: + if (!flagRepeatTable) return ERROR(corruption_detected); + return 0; + default : /* impossible */ + case set_compressed : + { U32 tableLog; + S16 norm[MaxSeq+1]; + size_t const headerSize = FSEv08_readNCount(norm, &max, &tableLog, src, srcSize); + if (FSEv08_isError(headerSize)) return ERROR(corruption_detected); + if (tableLog > maxLog) return ERROR(corruption_detected); + FSEv08_buildDTable(DTable, norm, max, tableLog); + return headerSize; + } } +} + + +size_t ZSTDv08_decodeSeqHeaders(int* nbSeqPtr, + FSEv08_DTable* DTableLL, FSEv08_DTable* DTableML, FSEv08_DTable* DTableOffb, U32 flagRepeatTable, + const void* src, size_t srcSize) +{ + const BYTE* const istart = (const BYTE* const)src; + const BYTE* const iend = istart + srcSize; + const BYTE* ip = istart; + + /* check */ + if (srcSize < MIN_SEQUENCES_SIZE) return ERROR(srcSize_wrong); + + /* SeqHead */ + { int nbSeq = *ip++; + if (!nbSeq) { *nbSeqPtr=0; return 1; } + if (nbSeq > 0x7F) { + if (nbSeq == 0xFF) + nbSeq = MEM_readLE16(ip) + LONGNBSEQ, ip+=2; + else + nbSeq = ((nbSeq-0x80)<<8) + *ip++; + } + *nbSeqPtr = nbSeq; + } + + /* FSE table descriptors */ + if (ip+4 > iend) return ERROR(srcSize_wrong); /* minimum possible size */ + { symbolEncodingType_e const LLtype = (symbolEncodingType_e)(*ip >> 6); + symbolEncodingType_e const OFtype = (symbolEncodingType_e)((*ip >> 4) & 3); + symbolEncodingType_e const MLtype = (symbolEncodingType_e)((*ip >> 2) & 3); + ip++; + + /* Build DTables */ + { size_t const llhSize = ZSTDv08_buildSeqTable(DTableLL, LLtype, MaxLL, LLFSELog, ip, iend-ip, LL_defaultNorm, LL_defaultNormLog, flagRepeatTable); + if (ZSTDv08_isError(llhSize)) return ERROR(corruption_detected); + ip += llhSize; + } + { size_t const ofhSize = ZSTDv08_buildSeqTable(DTableOffb, OFtype, MaxOff, OffFSELog, ip, iend-ip, OF_defaultNorm, OF_defaultNormLog, flagRepeatTable); + if (ZSTDv08_isError(ofhSize)) return ERROR(corruption_detected); + ip += ofhSize; + } + { size_t const mlhSize = ZSTDv08_buildSeqTable(DTableML, MLtype, MaxML, MLFSELog, ip, iend-ip, ML_defaultNorm, ML_defaultNormLog, flagRepeatTable); + if (ZSTDv08_isError(mlhSize)) return ERROR(corruption_detected); + ip += mlhSize; + } } + + return ip-istart; +} + + +typedef struct { + size_t litLength; + size_t matchLength; + size_t offset; +} seq_t; + +typedef struct { + BIT_DStream_t DStream; + FSEv08_DState_t stateLL; + FSEv08_DState_t stateOffb; + FSEv08_DState_t stateML; + size_t prevOffset[ZSTDv08_REP_NUM]; +} seqState_t; + + +static seq_t ZSTDv08_decodeSequence(seqState_t* seqState) +{ + seq_t seq; + + U32 const llCode = FSEv08_peekSymbol(&(seqState->stateLL)); + U32 const mlCode = FSEv08_peekSymbol(&(seqState->stateML)); + U32 const ofCode = FSEv08_peekSymbol(&(seqState->stateOffb)); /* <= maxOff, by table construction */ + + U32 const llBits = LL_bits[llCode]; + U32 const mlBits = ML_bits[mlCode]; + U32 const ofBits = ofCode; + U32 const totalBits = llBits+mlBits+ofBits; + + static const U32 LL_base[MaxLL+1] = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 18, 20, 22, 24, 28, 32, 40, 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, + 0x2000, 0x4000, 0x8000, 0x10000 }; + + static const U32 ML_base[MaxML+1] = { + 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, + 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, + 35, 37, 39, 41, 43, 47, 51, 59, 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, + 0x1003, 0x2003, 0x4003, 0x8003, 0x10003 }; + + static const U32 OF_base[MaxOff+1] = { + 0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, + 0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, + 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, + 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD }; + + /* sequence */ + { size_t offset; + if (!ofCode) + offset = 0; + else { + offset = OF_base[ofCode] + BIT_readBits(&(seqState->DStream), ofBits); /* <= (ZSTDv08_WINDOWLOG_MAX-1) bits */ + if (MEM_32bits()) BIT_reloadDStream(&(seqState->DStream)); + } + + if (ofCode <= 1) { + offset += (llCode==0); + if (offset) { + size_t const temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; + if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1]; + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset = temp; + } else { + offset = seqState->prevOffset[0]; + } + } else { + seqState->prevOffset[2] = seqState->prevOffset[1]; + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset; + } + seq.offset = offset; + } + + seq.matchLength = ML_base[mlCode] + ((mlCode>31) ? BIT_readBits(&(seqState->DStream), mlBits) : 0); /* <= 16 bits */ + if (MEM_32bits() && (mlBits+llBits>24)) BIT_reloadDStream(&(seqState->DStream)); + + seq.litLength = LL_base[llCode] + ((llCode>15) ? BIT_readBits(&(seqState->DStream), llBits) : 0); /* <= 16 bits */ + if (MEM_32bits() || + (totalBits > 64 - 7 - (LLFSELog+MLFSELog+OffFSELog)) ) BIT_reloadDStream(&(seqState->DStream)); + + /* ANS state update */ + FSEv08_updateState(&(seqState->stateLL), &(seqState->DStream)); /* <= 9 bits */ + FSEv08_updateState(&(seqState->stateML), &(seqState->DStream)); /* <= 9 bits */ + if (MEM_32bits()) BIT_reloadDStream(&(seqState->DStream)); /* <= 18 bits */ + FSEv08_updateState(&(seqState->stateOffb), &(seqState->DStream)); /* <= 8 bits */ + + return seq; +} + + +FORCE_INLINE +size_t ZSTDv08_execSequence(BYTE* op, + BYTE* const oend, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit_w, + const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ + BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = oLitEnd - sequence.offset; + + /* check */ + if ((oLitEnd>oend_w) | (oMatchEnd>oend)) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */ + if (iLitEnd > litLimit_w) return ERROR(corruption_detected); /* over-read beyond lit buffer */ + + /* copy Literals */ + ZSTDv08_wildcopy(op, *litPtr, sequence.litLength); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ + op = oLitEnd; + *litPtr = iLitEnd; /* update for next sequence */ + + /* copy Match */ + if (sequence.offset > (size_t)(oLitEnd - base)) { + /* offset beyond prefix */ + if (sequence.offset > (size_t)(oLitEnd - vBase)) return ERROR(corruption_detected); + match = dictEnd - (base-match); + if (match + sequence.matchLength <= dictEnd) { + memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { size_t const length1 = dictEnd - match; + memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = base; + } } + + /* match within prefix */ + if (sequence.offset < 8) { + /* close range match, overlap */ + static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */ + static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* substracted */ + int const sub2 = dec64table[sequence.offset]; + op[0] = match[0]; + op[1] = match[1]; + op[2] = match[2]; + op[3] = match[3]; + match += dec32table[sequence.offset]; + ZSTDv08_copy4(op+4, match); + match -= sub2; + } else { + ZSTDv08_copy8(op, match); + } + op += 8; match += 8; + + if (oMatchEnd > oend-(16-MINMATCH)) { + if (op < oend_w) { + ZSTDv08_wildcopy(op, match, oend_w - op); + match += oend_w - op; + op = oend_w; + } + while (op < oMatchEnd) *op++ = *match++; + } else { + ZSTDv08_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */ + } + return sequenceLength; +} + + +static size_t ZSTDv08_decompressSequences( + ZSTDv08_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize) +{ + const BYTE* ip = (const BYTE*)seqStart; + const BYTE* const iend = ip + seqSize; + BYTE* const ostart = (BYTE* const)dst; + BYTE* const oend = ostart + maxDstSize; + BYTE* op = ostart; + const BYTE* litPtr = dctx->litPtr; + const BYTE* const litLimit_w = litPtr + dctx->litBufSize - WILDCOPY_OVERLENGTH; + const BYTE* const litEnd = litPtr + dctx->litSize; + FSEv08_DTable* DTableLL = dctx->LLTable; + FSEv08_DTable* DTableML = dctx->MLTable; + FSEv08_DTable* DTableOffb = dctx->OffTable; + const BYTE* const base = (const BYTE*) (dctx->base); + const BYTE* const vBase = (const BYTE*) (dctx->vBase); + const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); + int nbSeq; + + /* Build Decoding Tables */ + { size_t const seqHSize = ZSTDv08_decodeSeqHeaders(&nbSeq, DTableLL, DTableML, DTableOffb, dctx->fseEntropy, ip, seqSize); + if (ZSTDv08_isError(seqHSize)) return seqHSize; + ip += seqHSize; + } + + /* Regen sequences */ + if (nbSeq) { + seqState_t seqState; + dctx->fseEntropy = 1; + { U32 i; for (i=0; i<ZSTDv08_REP_NUM; i++) seqState.prevOffset[i] = dctx->rep[i]; } + { size_t const errorCode = BIT_initDStream(&(seqState.DStream), ip, iend-ip); + if (ERR_isError(errorCode)) return ERROR(corruption_detected); } + FSEv08_initDState(&(seqState.stateLL), &(seqState.DStream), DTableLL); + FSEv08_initDState(&(seqState.stateOffb), &(seqState.DStream), DTableOffb); + FSEv08_initDState(&(seqState.stateML), &(seqState.DStream), DTableML); + + for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && nbSeq ; ) { + nbSeq--; + { seq_t const sequence = ZSTDv08_decodeSequence(&seqState); + size_t const oneSeqSize = ZSTDv08_execSequence(op, oend, sequence, &litPtr, litLimit_w, base, vBase, dictEnd); + if (ZSTDv08_isError(oneSeqSize)) return oneSeqSize; + op += oneSeqSize; + } } + + /* check if reached exact end */ + if (nbSeq) return ERROR(corruption_detected); + /* save reps for next block */ + { U32 i; for (i=0; i<ZSTDv08_REP_NUM; i++) dctx->rep[i] = (U32)(seqState.prevOffset[i]); } + } + + /* last literal segment */ + { size_t const lastLLSize = litEnd - litPtr; + if (lastLLSize > (size_t)(oend-op)) return ERROR(dstSize_tooSmall); + memcpy(op, litPtr, lastLLSize); + op += lastLLSize; + } + + return op-ostart; +} + + +static void ZSTDv08_checkContinuity(ZSTDv08_DCtx* dctx, const void* dst) +{ + if (dst != dctx->previousDstEnd) { /* not contiguous */ + dctx->dictEnd = dctx->previousDstEnd; + dctx->vBase = (const char*)dst - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->base)); + dctx->base = dst; + dctx->previousDstEnd = dst; + } +} + + +static size_t ZSTDv08_decompressBlock_internal(ZSTDv08_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ /* blockType == blockCompressed */ + const BYTE* ip = (const BYTE*)src; + + if (srcSize >= ZSTDv08_BLOCKSIZE_ABSOLUTEMAX) return ERROR(srcSize_wrong); + + /* Decode literals sub-block */ + { size_t const litCSize = ZSTDv08_decodeLiteralsBlock(dctx, src, srcSize); + if (ZSTDv08_isError(litCSize)) return litCSize; + ip += litCSize; + srcSize -= litCSize; + } + return ZSTDv08_decompressSequences(dctx, dst, dstCapacity, ip, srcSize); +} + + +size_t ZSTDv08_decompressBlock(ZSTDv08_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + size_t dSize; + ZSTDv08_checkContinuity(dctx, dst); + dSize = ZSTDv08_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize); + dctx->previousDstEnd = (char*)dst + dSize; + return dSize; +} + + +/** ZSTDv08_insertBlock() : + insert `src` block into `dctx` history. Useful to track uncompressed blocks. */ +ZSTDLIB_API size_t ZSTDv08_insertBlock(ZSTDv08_DCtx* dctx, const void* blockStart, size_t blockSize) +{ + ZSTDv08_checkContinuity(dctx, blockStart); + dctx->previousDstEnd = (const char*)blockStart + blockSize; + return blockSize; +} + + +size_t ZSTDv08_generateNxBytes(void* dst, size_t dstCapacity, BYTE byte, size_t length) +{ + if (length > dstCapacity) return ERROR(dstSize_tooSmall); + memset(dst, byte, length); + return length; +} + + +/*! ZSTDv08_decompressFrame() : +* `dctx` must be properly initialized */ +static size_t ZSTDv08_decompressFrame(ZSTDv08_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + const BYTE* ip = (const BYTE*)src; + BYTE* const ostart = (BYTE* const)dst; + BYTE* const oend = ostart + dstCapacity; + BYTE* op = ostart; + size_t remainingSize = srcSize; + + /* check */ + if (srcSize < ZSTDv08_frameHeaderSize_min+ZSTDv08_blockHeaderSize) return ERROR(srcSize_wrong); + + /* Frame Header */ + { size_t const frameHeaderSize = ZSTDv08_frameHeaderSize(src, ZSTDv08_frameHeaderSize_min); + size_t result; + if (ZSTDv08_isError(frameHeaderSize)) return frameHeaderSize; + if (srcSize < frameHeaderSize+ZSTDv08_blockHeaderSize) return ERROR(srcSize_wrong); + result = ZSTDv08_decodeFrameHeader(dctx, src, frameHeaderSize); + if (ZSTDv08_isError(result)) return result; + ip += frameHeaderSize; remainingSize -= frameHeaderSize; + } + + /* Loop on each block */ + while (1) { + size_t decodedSize; + blockProperties_t blockProperties; + size_t const cBlockSize = ZSTDv08_getcBlockSize(ip, remainingSize, &blockProperties); + if (ZSTDv08_isError(cBlockSize)) return cBlockSize; + + ip += ZSTDv08_blockHeaderSize; + remainingSize -= ZSTDv08_blockHeaderSize; + if (cBlockSize > remainingSize) return ERROR(srcSize_wrong); + + switch(blockProperties.blockType) + { + case bt_compressed: + decodedSize = ZSTDv08_decompressBlock_internal(dctx, op, oend-op, ip, cBlockSize); + break; + case bt_raw : + decodedSize = ZSTDv08_copyRawBlock(op, oend-op, ip, cBlockSize); + break; + case bt_rle : + decodedSize = ZSTDv08_generateNxBytes(op, oend-op, *ip, blockProperties.origSize); + break; + case bt_reserved : + default: + return ERROR(corruption_detected); + } + + if (ZSTDv08_isError(decodedSize)) return decodedSize; + if (dctx->fParams.checksumFlag) XXH64_update(&dctx->xxhState, op, decodedSize); + op += decodedSize; + ip += cBlockSize; + remainingSize -= cBlockSize; + if (blockProperties.lastBlock) break; + } + + if (dctx->fParams.checksumFlag) { /* Frame content checksum verification */ + U32 const checkCalc = (U32)XXH64_digest(&dctx->xxhState); + U32 checkRead; + if (remainingSize<4) return ERROR(checksum_wrong); + checkRead = MEM_readLE32(ip); + if (checkRead != checkCalc) return ERROR(checksum_wrong); + remainingSize -= 4; + } + + if (remainingSize) return ERROR(srcSize_wrong); + return op-ostart; +} + + +/*! ZSTDv08_decompress_usingPreparedDCtx() : +* Same as ZSTDv08_decompress_usingDict, but using a reference context `preparedDCtx`, where dictionary has been loaded. +* It avoids reloading the dictionary each time. +* `preparedDCtx` must have been properly initialized using ZSTDv08_decompressBegin_usingDict(). +* Requires 2 contexts : 1 for reference (preparedDCtx), which will not be modified, and 1 to run the decompression operation (dctx) */ +size_t ZSTDv08_decompress_usingPreparedDCtx(ZSTDv08_DCtx* dctx, const ZSTDv08_DCtx* refDCtx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + ZSTDv08_copyDCtx(dctx, refDCtx); + ZSTDv08_checkContinuity(dctx, dst); + return ZSTDv08_decompressFrame(dctx, dst, dstCapacity, src, srcSize); +} + + +size_t ZSTDv08_decompress_usingDict(ZSTDv08_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict, size_t dictSize) +{ + ZSTDv08_decompressBegin_usingDict(dctx, dict, dictSize); + ZSTDv08_checkContinuity(dctx, dst); + return ZSTDv08_decompressFrame(dctx, dst, dstCapacity, src, srcSize); +} + + +size_t ZSTDv08_decompressDCtx(ZSTDv08_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + return ZSTDv08_decompress_usingDict(dctx, dst, dstCapacity, src, srcSize, NULL, 0); +} + + +size_t ZSTDv08_decompress(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ +#if defined(ZSTDv08_HEAPMODE) && (ZSTDv08_HEAPMODE==1) + size_t regenSize; + ZSTDv08_DCtx* const dctx = ZSTDv08_createDCtx(); + if (dctx==NULL) return ERROR(memory_allocation); + regenSize = ZSTDv08_decompressDCtx(dctx, dst, dstCapacity, src, srcSize); + ZSTDv08_freeDCtx(dctx); + return regenSize; +#else /* stack mode */ + ZSTDv08_DCtx dctx; + return ZSTDv08_decompressDCtx(&dctx, dst, dstCapacity, src, srcSize); +#endif +} + + +/*-********************************** +* Streaming Decompression API +************************************/ +size_t ZSTDv08_nextSrcSizeToDecompress(ZSTDv08_DCtx* dctx) { return dctx->expected; } + +ZSTDv08_nextInputType_e ZSTDv08_nextInputType(ZSTDv08_DCtx* dctx) { + switch(dctx->stage) + { + default: /* should not happen */ + case ZSTDds_getFrameHeaderSize: + case ZSTDds_decodeFrameHeader: + return ZSTDnit_frameHeader; + case ZSTDds_decodeBlockHeader: + return ZSTDnit_blockHeader; + case ZSTDds_decompressBlock: + return ZSTDnit_block; + case ZSTDds_decompressLastBlock: + return ZSTDnit_lastBlock; + case ZSTDds_checkChecksum: + return ZSTDnit_checksum; + case ZSTDds_decodeSkippableHeader: + case ZSTDds_skipFrame: + return ZSTDnit_skippableFrame; + } +} + +int ZSTDv08_isSkipFrame(ZSTDv08_DCtx* dctx) { return dctx->stage == ZSTDds_skipFrame; } /* for zbuff */ + +/** ZSTDv08_decompressContinue() : +* @return : nb of bytes generated into `dst` (necessarily <= `dstCapacity) +* or an error code, which can be tested using ZSTDv08_isError() */ +size_t ZSTDv08_decompressContinue(ZSTDv08_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + /* Sanity check */ + if (srcSize != dctx->expected) return ERROR(srcSize_wrong); + if (dstCapacity) ZSTDv08_checkContinuity(dctx, dst); + + switch (dctx->stage) + { + case ZSTDds_getFrameHeaderSize : + if (srcSize != ZSTDv08_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */ + if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTDv08_MAGIC_SKIPPABLE_START) { + memcpy(dctx->headerBuffer, src, ZSTDv08_frameHeaderSize_min); + dctx->expected = ZSTDv08_skippableHeaderSize - ZSTDv08_frameHeaderSize_min; /* magic number + skippable frame length */ + dctx->stage = ZSTDds_decodeSkippableHeader; + return 0; + } + dctx->headerSize = ZSTDv08_frameHeaderSize(src, ZSTDv08_frameHeaderSize_min); + if (ZSTDv08_isError(dctx->headerSize)) return dctx->headerSize; + memcpy(dctx->headerBuffer, src, ZSTDv08_frameHeaderSize_min); + if (dctx->headerSize > ZSTDv08_frameHeaderSize_min) { + dctx->expected = dctx->headerSize - ZSTDv08_frameHeaderSize_min; + dctx->stage = ZSTDds_decodeFrameHeader; + return 0; + } + dctx->expected = 0; /* not necessary to copy more */ + + case ZSTDds_decodeFrameHeader: + { size_t result; + memcpy(dctx->headerBuffer + ZSTDv08_frameHeaderSize_min, src, dctx->expected); + result = ZSTDv08_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize); + if (ZSTDv08_isError(result)) return result; + dctx->expected = ZSTDv08_blockHeaderSize; + dctx->stage = ZSTDds_decodeBlockHeader; + return 0; + } + case ZSTDds_decodeBlockHeader: + { blockProperties_t bp; + size_t const cBlockSize = ZSTDv08_getcBlockSize(src, ZSTDv08_blockHeaderSize, &bp); + if (ZSTDv08_isError(cBlockSize)) return cBlockSize; + dctx->expected = cBlockSize; + dctx->bType = bp.blockType; + dctx->rleSize = bp.origSize; + if (cBlockSize) { + dctx->stage = bp.lastBlock ? ZSTDds_decompressLastBlock : ZSTDds_decompressBlock; + return 0; + } + /* empty block */ + if (bp.lastBlock) { + if (dctx->fParams.checksumFlag) { + dctx->expected = 4; + dctx->stage = ZSTDds_checkChecksum; + } else { + dctx->expected = 0; /* end of frame */ + dctx->stage = ZSTDds_getFrameHeaderSize; + } + } else { + dctx->expected = 3; /* go directly to next header */ + dctx->stage = ZSTDds_decodeBlockHeader; + } + return 0; + } + case ZSTDds_decompressLastBlock: + case ZSTDds_decompressBlock: + { size_t rSize; + switch(dctx->bType) + { + case bt_compressed: + rSize = ZSTDv08_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize); + break; + case bt_raw : + rSize = ZSTDv08_copyRawBlock(dst, dstCapacity, src, srcSize); + break; + case bt_rle : + rSize = ZSTDv08_setRleBlock(dst, dstCapacity, src, srcSize, dctx->rleSize); + break; + case bt_reserved : /* should never happen */ + default: + return ERROR(corruption_detected); + } + if (ZSTDv08_isError(rSize)) return rSize; + if (dctx->fParams.checksumFlag) XXH64_update(&dctx->xxhState, dst, rSize); + + if (dctx->stage == ZSTDds_decompressLastBlock) { /* end of frame */ + if (dctx->fParams.checksumFlag) { /* another round for frame checksum */ + dctx->expected = 4; + dctx->stage = ZSTDds_checkChecksum; + } else { + dctx->expected = 0; /* ends here */ + dctx->stage = ZSTDds_getFrameHeaderSize; + } + } else { + dctx->stage = ZSTDds_decodeBlockHeader; + dctx->expected = ZSTDv08_blockHeaderSize; + dctx->previousDstEnd = (char*)dst + rSize; + } + return rSize; + } + case ZSTDds_checkChecksum: + { U32 const h32 = (U32)XXH64_digest(&dctx->xxhState); + U32 const check32 = MEM_readLE32(src); /* srcSize == 4, guaranteed by dctx->expected */ + if (check32 != h32) return ERROR(checksum_wrong); + dctx->expected = 0; + dctx->stage = ZSTDds_getFrameHeaderSize; + return 0; + } + case ZSTDds_decodeSkippableHeader: + { memcpy(dctx->headerBuffer + ZSTDv08_frameHeaderSize_min, src, dctx->expected); + dctx->expected = MEM_readLE32(dctx->headerBuffer + 4); + dctx->stage = ZSTDds_skipFrame; + return 0; + } + case ZSTDds_skipFrame: + { dctx->expected = 0; + dctx->stage = ZSTDds_getFrameHeaderSize; + return 0; + } + default: + return ERROR(GENERIC); /* impossible */ + } +} + + +static size_t ZSTDv08_refDictContent(ZSTDv08_DCtx* dctx, const void* dict, size_t dictSize) +{ + dctx->dictEnd = dctx->previousDstEnd; + dctx->vBase = (const char*)dict - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->base)); + dctx->base = dict; + dctx->previousDstEnd = (const char*)dict + dictSize; + return 0; +} + +static size_t ZSTDv08_loadEntropy(ZSTDv08_DCtx* dctx, const void* const dict, size_t const dictSize) +{ + const BYTE* dictPtr = (const BYTE*)dict; + const BYTE* const dictEnd = dictPtr + dictSize; + + { size_t const hSize = HUFv08_readDTableX4(dctx->hufTable, dict, dictSize); + if (HUFv08_isError(hSize)) return ERROR(dictionary_corrupted); + dictPtr += hSize; + } + + { short offcodeNCount[MaxOff+1]; + U32 offcodeMaxValue=MaxOff, offcodeLog=OffFSELog; + size_t const offcodeHeaderSize = FSEv08_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr); + if (FSEv08_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); + { size_t const errorCode = FSEv08_buildDTable(dctx->OffTable, offcodeNCount, offcodeMaxValue, offcodeLog); + if (FSEv08_isError(errorCode)) return ERROR(dictionary_corrupted); } + dictPtr += offcodeHeaderSize; + } + + { short matchlengthNCount[MaxML+1]; + unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog; + size_t const matchlengthHeaderSize = FSEv08_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr); + if (FSEv08_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted); + { size_t const errorCode = FSEv08_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog); + if (FSEv08_isError(errorCode)) return ERROR(dictionary_corrupted); } + dictPtr += matchlengthHeaderSize; + } + + { short litlengthNCount[MaxLL+1]; + unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog; + size_t const litlengthHeaderSize = FSEv08_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr); + if (FSEv08_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted); + { size_t const errorCode = FSEv08_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog); + if (FSEv08_isError(errorCode)) return ERROR(dictionary_corrupted); } + dictPtr += litlengthHeaderSize; + } + + if (dictPtr+12 > dictEnd) return ERROR(dictionary_corrupted); + dctx->rep[0] = MEM_readLE32(dictPtr+0); if (dctx->rep[0] >= dictSize) return ERROR(dictionary_corrupted); + dctx->rep[1] = MEM_readLE32(dictPtr+4); if (dctx->rep[1] >= dictSize) return ERROR(dictionary_corrupted); + dctx->rep[2] = MEM_readLE32(dictPtr+8); if (dctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted); + dictPtr += 12; + + dctx->litEntropy = dctx->fseEntropy = 1; + return dictPtr - (const BYTE*)dict; +} + +static size_t ZSTDv08_decompress_insertDictionary(ZSTDv08_DCtx* dctx, const void* dict, size_t dictSize) +{ + if (dictSize < 8) return ZSTDv08_refDictContent(dctx, dict, dictSize); + { U32 const magic = MEM_readLE32(dict); + if (magic != ZSTDv08_DICT_MAGIC) { + return ZSTDv08_refDictContent(dctx, dict, dictSize); /* pure content mode */ + } } + dctx->dictID = MEM_readLE32((const char*)dict + 4); + + /* load entropy tables */ + dict = (const char*)dict + 8; + dictSize -= 8; + { size_t const eSize = ZSTDv08_loadEntropy(dctx, dict, dictSize); + if (ZSTDv08_isError(eSize)) return ERROR(dictionary_corrupted); + dict = (const char*)dict + eSize; + dictSize -= eSize; + } + + /* reference dictionary content */ + return ZSTDv08_refDictContent(dctx, dict, dictSize); +} + + +size_t ZSTDv08_decompressBegin_usingDict(ZSTDv08_DCtx* dctx, const void* dict, size_t dictSize) +{ + { size_t const errorCode = ZSTDv08_decompressBegin(dctx); + if (ZSTDv08_isError(errorCode)) return errorCode; } + + if (dict && dictSize) { + size_t const errorCode = ZSTDv08_decompress_insertDictionary(dctx, dict, dictSize); + if (ZSTDv08_isError(errorCode)) return ERROR(dictionary_corrupted); + } + + return 0; +} + + +struct ZSTDv08_DDict_s { + void* dict; + size_t dictSize; + ZSTDv08_DCtx* refContext; +}; /* typedef'd tp ZSTDv08_CDict within zstd.h */ + +ZSTDv08_DDict* ZSTDv08_createDDict_advanced(const void* dict, size_t dictSize, ZSTDv08_customMem customMem) +{ + if (!customMem.customAlloc && !customMem.customFree) + customMem = defaultCustomMem; + + if (!customMem.customAlloc || !customMem.customFree) + return NULL; + + { ZSTDv08_DDict* const ddict = (ZSTDv08_DDict*) customMem.customAlloc(customMem.opaque, sizeof(*ddict)); + void* const dictContent = customMem.customAlloc(customMem.opaque, dictSize); + ZSTDv08_DCtx* const dctx = ZSTDv08_createDCtx_advanced(customMem); + + if (!dictContent || !ddict || !dctx) { + customMem.customFree(customMem.opaque, dictContent); + customMem.customFree(customMem.opaque, ddict); + customMem.customFree(customMem.opaque, dctx); + return NULL; + } + + memcpy(dictContent, dict, dictSize); + { size_t const errorCode = ZSTDv08_decompressBegin_usingDict(dctx, dictContent, dictSize); + if (ZSTDv08_isError(errorCode)) { + customMem.customFree(customMem.opaque, dictContent); + customMem.customFree(customMem.opaque, ddict); + customMem.customFree(customMem.opaque, dctx); + return NULL; + } } + + ddict->dict = dictContent; + ddict->dictSize = dictSize; + ddict->refContext = dctx; + return ddict; + } +} + +/*! ZSTDv08_createDDict() : +* Create a digested dictionary, ready to start decompression without startup delay. +* `dict` can be released after `ZSTDv08_DDict` creation */ +ZSTDv08_DDict* ZSTDv08_createDDict(const void* dict, size_t dictSize) +{ + ZSTDv08_customMem const allocator = { NULL, NULL, NULL }; + return ZSTDv08_createDDict_advanced(dict, dictSize, allocator); +} + +size_t ZSTDv08_freeDDict(ZSTDv08_DDict* ddict) +{ + ZSTDv08_freeFunction const cFree = ddict->refContext->customMem.customFree; + void* const opaque = ddict->refContext->customMem.opaque; + ZSTDv08_freeDCtx(ddict->refContext); + cFree(opaque, ddict->dict); + cFree(opaque, ddict); + return 0; +} + +/*! ZSTDv08_decompress_usingDDict() : +* Decompression using a pre-digested Dictionary +* Use dictionary without significant overhead. */ +ZSTDLIB_API size_t ZSTDv08_decompress_usingDDict(ZSTDv08_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTDv08_DDict* ddict) +{ + return ZSTDv08_decompress_usingPreparedDCtx(dctx, ddict->refContext, + dst, dstCapacity, + src, srcSize); +} +/* + Buffered version of Zstd compression library + Copyright (C) 2015-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - zstd homepage : http://www.zstd.net/ +*/ + + +/*-*************************************************************************** +* Streaming decompression howto +* +* A ZBUFFv08_DCtx object is required to track streaming operations. +* Use ZBUFFv08_createDCtx() and ZBUFFv08_freeDCtx() to create/release resources. +* Use ZBUFFv08_decompressInit() to start a new decompression operation, +* or ZBUFFv08_decompressInitDictionary() if decompression requires a dictionary. +* Note that ZBUFFv08_DCtx objects can be re-init multiple times. +* +* Use ZBUFFv08_decompressContinue() repetitively to consume your input. +* *srcSizePtr and *dstCapacityPtr can be any size. +* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr. +* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again. +* The content of @dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change @dst. +* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency), +* or 0 when a frame is completely decoded, +* or an error code, which can be tested using ZBUFFv08_isError(). +* +* Hint : recommended buffer sizes (not compulsory) : ZBUFFv08_recommendedDInSize() and ZBUFFv08_recommendedDOutSize() +* output : ZBUFFv08_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded. +* input : ZBUFFv08_recommendedDInSize == 128KB + 3; +* just follow indications from ZBUFFv08_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . +* *******************************************************************************/ + +typedef enum { ZBUFFds_init, ZBUFFds_loadHeader, + ZBUFFds_read, ZBUFFds_load, ZBUFFds_flush } ZBUFFv08_dStage; + +/* *** Resource management *** */ +struct ZBUFFv08_DCtx_s { + ZSTDv08_DCtx* zd; + ZSTDv08_frameParams fParams; + ZBUFFv08_dStage stage; + char* inBuff; + size_t inBuffSize; + size_t inPos; + char* outBuff; + size_t outBuffSize; + size_t outStart; + size_t outEnd; + size_t blockSize; + BYTE headerBuffer[ZSTDv08_FRAMEHEADERSIZE_MAX]; + size_t lhSize; + ZSTDv08_customMem customMem; +}; /* typedef'd to ZBUFFv08_DCtx within "zstd_buffered.h" */ + + +ZBUFFv08_DCtx* ZBUFFv08_createDCtx(void) +{ + return ZBUFFv08_createDCtx_advanced(defaultCustomMem); +} + +ZBUFFv08_DCtx* ZBUFFv08_createDCtx_advanced(ZSTDv08_customMem customMem) +{ + ZBUFFv08_DCtx* zbd; + + if (!customMem.customAlloc && !customMem.customFree) + customMem = defaultCustomMem; + + if (!customMem.customAlloc || !customMem.customFree) + return NULL; + + zbd = (ZBUFFv08_DCtx*)customMem.customAlloc(customMem.opaque, sizeof(ZBUFFv08_DCtx)); + if (zbd==NULL) return NULL; + memset(zbd, 0, sizeof(ZBUFFv08_DCtx)); + memcpy(&zbd->customMem, &customMem, sizeof(ZSTDv08_customMem)); + zbd->zd = ZSTDv08_createDCtx_advanced(customMem); + if (zbd->zd == NULL) { ZBUFFv08_freeDCtx(zbd); return NULL; } + zbd->stage = ZBUFFds_init; + return zbd; +} + +size_t ZBUFFv08_freeDCtx(ZBUFFv08_DCtx* zbd) +{ + if (zbd==NULL) return 0; /* support free on null */ + ZSTDv08_freeDCtx(zbd->zd); + if (zbd->inBuff) zbd->customMem.customFree(zbd->customMem.opaque, zbd->inBuff); + if (zbd->outBuff) zbd->customMem.customFree(zbd->customMem.opaque, zbd->outBuff); + zbd->customMem.customFree(zbd->customMem.opaque, zbd); + return 0; +} + + +/* *** Initialization *** */ + +size_t ZBUFFv08_decompressInitDictionary(ZBUFFv08_DCtx* zbd, const void* dict, size_t dictSize) +{ + zbd->stage = ZBUFFds_loadHeader; + zbd->lhSize = zbd->inPos = zbd->outStart = zbd->outEnd = 0; + return ZSTDv08_decompressBegin_usingDict(zbd->zd, dict, dictSize); +} + +size_t ZBUFFv08_decompressInit(ZBUFFv08_DCtx* zbd) +{ + return ZBUFFv08_decompressInitDictionary(zbd, NULL, 0); +} + + +/* internal util function */ +MEM_STATIC size_t ZBUFFv08_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + size_t const length = MIN(dstCapacity, srcSize); + memcpy(dst, src, length); + return length; +} + + +/* *** Decompression *** */ + +size_t ZBUFFv08_decompressContinue(ZBUFFv08_DCtx* zbd, + void* dst, size_t* dstCapacityPtr, + const void* src, size_t* srcSizePtr) +{ + const char* const istart = (const char*)src; + const char* const iend = istart + *srcSizePtr; + const char* ip = istart; + char* const ostart = (char*)dst; + char* const oend = ostart + *dstCapacityPtr; + char* op = ostart; + U32 someMoreWork = 1; + + while (someMoreWork) { + switch(zbd->stage) + { + case ZBUFFds_init : + return ERROR(init_missing); + + case ZBUFFds_loadHeader : + { size_t const hSize = ZSTDv08_getFrameParams(&(zbd->fParams), zbd->headerBuffer, zbd->lhSize); + if (ZSTDv08_isError(hSize)) return hSize; + if (hSize != 0) { /* need more input */ + size_t const toLoad = hSize - zbd->lhSize; /* if hSize!=0, hSize > zbd->lhSize */ + if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */ + memcpy(zbd->headerBuffer + zbd->lhSize, ip, iend-ip); + zbd->lhSize += iend-ip; + *dstCapacityPtr = 0; + return (hSize - zbd->lhSize) + ZSTDv08_blockHeaderSize; /* remaining header bytes + next block header */ + } + memcpy(zbd->headerBuffer + zbd->lhSize, ip, toLoad); zbd->lhSize = hSize; ip += toLoad; + break; + } } + + /* Consume header */ + { size_t const h1Size = ZSTDv08_nextSrcSizeToDecompress(zbd->zd); /* == ZSTDv08_frameHeaderSize_min */ + size_t const h1Result = ZSTDv08_decompressContinue(zbd->zd, NULL, 0, zbd->headerBuffer, h1Size); + if (ZSTDv08_isError(h1Result)) return h1Result; /* should not happen : already checked */ + if (h1Size < zbd->lhSize) { /* long header */ + size_t const h2Size = ZSTDv08_nextSrcSizeToDecompress(zbd->zd); + size_t const h2Result = ZSTDv08_decompressContinue(zbd->zd, NULL, 0, zbd->headerBuffer+h1Size, h2Size); + if (ZSTDv08_isError(h2Result)) return h2Result; + } } + + zbd->fParams.windowSize = MAX(zbd->fParams.windowSize, 1U << ZSTDv08_WINDOWLOG_ABSOLUTEMIN); + + /* Frame header instruct buffer sizes */ + { size_t const blockSize = MIN(zbd->fParams.windowSize, ZSTDv08_BLOCKSIZE_ABSOLUTEMAX); + size_t const neededOutSize = zbd->fParams.windowSize + blockSize; + zbd->blockSize = blockSize; + if (zbd->inBuffSize < blockSize) { + zbd->customMem.customFree(zbd->customMem.opaque, zbd->inBuff); + zbd->inBuffSize = blockSize; + zbd->inBuff = (char*)zbd->customMem.customAlloc(zbd->customMem.opaque, blockSize); + if (zbd->inBuff == NULL) return ERROR(memory_allocation); + } + if (zbd->outBuffSize < neededOutSize) { + zbd->customMem.customFree(zbd->customMem.opaque, zbd->outBuff); + zbd->outBuffSize = neededOutSize; + zbd->outBuff = (char*)zbd->customMem.customAlloc(zbd->customMem.opaque, neededOutSize); + if (zbd->outBuff == NULL) return ERROR(memory_allocation); + } } + zbd->stage = ZBUFFds_read; + /* pass-through */ + + case ZBUFFds_read: + { size_t const neededInSize = ZSTDv08_nextSrcSizeToDecompress(zbd->zd); + if (neededInSize==0) { /* end of frame */ + zbd->stage = ZBUFFds_init; + someMoreWork = 0; + break; + } + if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */ + const int isSkipFrame = ZSTDv08_isSkipFrame(zbd->zd); + size_t const decodedSize = ZSTDv08_decompressContinue(zbd->zd, + zbd->outBuff + zbd->outStart, (isSkipFrame ? 0 : zbd->outBuffSize - zbd->outStart), + ip, neededInSize); + if (ZSTDv08_isError(decodedSize)) return decodedSize; + ip += neededInSize; + if (!decodedSize && !isSkipFrame) break; /* this was just a header */ + zbd->outEnd = zbd->outStart + decodedSize; + zbd->stage = ZBUFFds_flush; + break; + } + if (ip==iend) { someMoreWork = 0; break; } /* no more input */ + zbd->stage = ZBUFFds_load; + /* pass-through */ + } + + case ZBUFFds_load: + { size_t const neededInSize = ZSTDv08_nextSrcSizeToDecompress(zbd->zd); + size_t const toLoad = neededInSize - zbd->inPos; /* should always be <= remaining space within inBuff */ + size_t loadedSize; + if (toLoad > zbd->inBuffSize - zbd->inPos) return ERROR(corruption_detected); /* should never happen */ + loadedSize = ZBUFFv08_limitCopy(zbd->inBuff + zbd->inPos, toLoad, ip, iend-ip); + ip += loadedSize; + zbd->inPos += loadedSize; + if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */ + + /* decode loaded input */ + { const int isSkipFrame = ZSTDv08_isSkipFrame(zbd->zd); + size_t const decodedSize = ZSTDv08_decompressContinue(zbd->zd, + zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart, + zbd->inBuff, neededInSize); + if (ZSTDv08_isError(decodedSize)) return decodedSize; + zbd->inPos = 0; /* input is consumed */ + if (!decodedSize && !isSkipFrame) { zbd->stage = ZBUFFds_read; break; } /* this was just a header */ + zbd->outEnd = zbd->outStart + decodedSize; + zbd->stage = ZBUFFds_flush; + /* pass-through */ + } } + + case ZBUFFds_flush: + { size_t const toFlushSize = zbd->outEnd - zbd->outStart; + size_t const flushedSize = ZBUFFv08_limitCopy(op, oend-op, zbd->outBuff + zbd->outStart, toFlushSize); + op += flushedSize; + zbd->outStart += flushedSize; + if (flushedSize == toFlushSize) { /* flush completed */ + zbd->stage = ZBUFFds_read; + if (zbd->outStart + zbd->blockSize > zbd->outBuffSize) + zbd->outStart = zbd->outEnd = 0; + break; + } + /* cannot flush everything */ + someMoreWork = 0; + break; + } + default: return ERROR(GENERIC); /* impossible */ + } } + + /* result */ + *srcSizePtr = ip-istart; + *dstCapacityPtr = op-ostart; + { size_t nextSrcSizeHint = ZSTDv08_nextSrcSizeToDecompress(zbd->zd); + if (!nextSrcSizeHint) return (zbd->outEnd != zbd->outStart); /* return 0 only if fully flushed too */ + nextSrcSizeHint += ZSTDv08_blockHeaderSize * (ZSTDv08_nextInputType(zbd->zd) == ZSTDnit_block); + if (zbd->inPos > nextSrcSizeHint) return ERROR(GENERIC); /* should never happen */ + nextSrcSizeHint -= zbd->inPos; /* already loaded*/ + return nextSrcSizeHint; + } +} + + +/* ************************************* +* Tool functions +***************************************/ +size_t ZBUFFv08_recommendedDInSize(void) { return ZSTDv08_BLOCKSIZE_ABSOLUTEMAX + ZSTDv08_blockHeaderSize /* block header size*/ ; } +size_t ZBUFFv08_recommendedDOutSize(void) { return ZSTDv08_BLOCKSIZE_ABSOLUTEMAX; } diff --git a/contrib/libs/zstd06/legacy/zstd_v08.h b/contrib/libs/zstd06/legacy/zstd_v08.h index ba2831d55a..df87980ae2 100644 --- a/contrib/libs/zstd06/legacy/zstd_v08.h +++ b/contrib/libs/zstd06/legacy/zstd_v08.h @@ -1,226 +1,226 @@ -#include <contrib/libs/zstd06/renames.h> -/* - zstd_v08 - decoder for 0.8 format - Header File - Copyright (C) 2014-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - zstd source repository : https://github.com/Cyan4973/zstd -*/ -#ifndef ZSTDv08_H_235446 -#define ZSTDv08_H_235446 - -#if defined (__cplusplus) -extern "C" { -#endif - -/*====== Dependency ======*/ -#include <stddef.h> /* size_t */ - - -/*====== Export for Windows ======*/ -/*! -* ZSTDv08_DLL_EXPORT : -* Enable exporting of functions when building a Windows DLL -*/ -#if defined(_WIN32) && defined(ZSTDv08_DLL_EXPORT) && (ZSTDv08_DLL_EXPORT==1) -# define ZSTDLIB_API __declspec(dllexport) -#else -# define ZSTDLIB_API -#endif - - -/* ************************************* -* Simple API -***************************************/ -/*! ZSTDv08_getDecompressedSize() : -* @return : decompressed size as a 64-bits value _if known_, 0 otherwise. -* note 1 : decompressed size can be very large (64-bits value), -* potentially larger than what local system can handle as a single memory segment. -* In which case, it's necessary to use streaming mode to decompress data. -* note 2 : decompressed size is an optional field, that may not be present. -* When `return==0`, consider data to decompress could have any size. -* In which case, it's necessary to use streaming mode to decompress data, -* or rely on application's implied limits. -* (For example, it may know that its own data is necessarily cut into blocks <= 16 KB). -* note 3 : decompressed size could be wrong or intentionally modified ! -* Always ensure result fits within application's authorized limits ! -* Each application can have its own set of conditions. -* If the intention is to decompress public data compressed by zstd command line utility, -* it is recommended to support at least 8 MB for extended compatibility. -* note 4 : when `return==0`, if precise failure cause is needed, use ZSTDv08_getFrameParams() to know more. */ -unsigned long long ZSTDv08_getDecompressedSize(const void* src, size_t srcSize); - -/*! ZSTDv08_decompress() : - `compressedSize` : must be the _exact_ size of compressed input, otherwise decompression will fail. - `dstCapacity` must be equal or larger than originalSize (see ZSTDv08_getDecompressedSize() ). - If originalSize is unknown, and if there is no implied application-specific limitations, - it's necessary to use streaming mode to decompress data. - @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), - or an errorCode if it fails (which can be tested using ZSTDv08_isError()) */ -ZSTDLIB_API size_t ZSTDv08_decompress( void* dst, size_t dstCapacity, - const void* src, size_t compressedSize); - - -/*====== Helper functions ======*/ -ZSTDLIB_API unsigned ZSTDv08_isError(size_t code); /*!< tells if a `size_t` function result is an error code */ -ZSTDLIB_API const char* ZSTDv08_getErrorName(size_t code); /*!< provides readable string from an error code */ - - -/*-************************************* -* Explicit memory management -***************************************/ -/** Decompression context */ -typedef struct ZSTDv08_DCtx_s ZSTDv08_DCtx; /*< incomplete type */ -ZSTDLIB_API ZSTDv08_DCtx* ZSTDv08_createDCtx(void); -ZSTDLIB_API size_t ZSTDv08_freeDCtx(ZSTDv08_DCtx* dctx); - -/** ZSTDv08_decompressDCtx() : -* Same as ZSTDv08_decompress(), requires an allocated ZSTDv08_DCtx (see ZSTDv08_createDCtx()) */ -ZSTDLIB_API size_t ZSTDv08_decompressDCtx(ZSTDv08_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); - - -/*-************************ -* Simple dictionary API -***************************/ -/*! ZSTDv08_decompress_usingDict() : -* Decompression using a predefined Dictionary (see dictBuilder/zdict.h). -* Dictionary must be identical to the one used during compression. -* Note : This function load the dictionary, resulting in a significant startup time */ -ZSTDLIB_API size_t ZSTDv08_decompress_usingDict(ZSTDv08_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const void* dict,size_t dictSize); - - -/*-************************** -* Fast Dictionary API -****************************/ -/*! ZSTDv08_createDDict() : -* Create a digested dictionary, ready to start decompression operation without startup delay. -* `dict` can be released after creation */ -typedef struct ZSTDv08_DDict_s ZSTDv08_DDict; -ZSTDLIB_API ZSTDv08_DDict* ZSTDv08_createDDict(const void* dict, size_t dictSize); -ZSTDLIB_API size_t ZSTDv08_freeDDict(ZSTDv08_DDict* ddict); - -/*! ZSTDv08_decompress_usingDDict() : -* Decompression using a digested Dictionary -* Faster startup than ZSTDv08_decompress_usingDict(), recommended when same dictionary is used multiple times. */ -ZSTDLIB_API size_t ZSTDv08_decompress_usingDDict(ZSTDv08_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const ZSTDv08_DDict* ddict); - -typedef struct { - unsigned long long frameContentSize; - unsigned windowSize; - unsigned dictID; - unsigned checksumFlag; -} ZSTDv08_frameParams; - -ZSTDLIB_API size_t ZSTDv08_getFrameParams(ZSTDv08_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input, see details below */ - - -/* *************************************************************** -* Compiler specifics -*****************************************************************/ -/* ZSTDv08_DLL_EXPORT : -* Enable exporting of functions when building a Windows DLL */ -#if defined(_WIN32) && defined(ZSTDv08_DLL_EXPORT) && (ZSTDv08_DLL_EXPORT==1) -# define ZSTDLIB_API __declspec(dllexport) -#else -# define ZSTDLIB_API -#endif - - -/* ************************************* -* Streaming functions -***************************************/ -/* This is the easier "buffered" streaming API, -* using an internal buffer to lift all restrictions on user-provided buffers -* which can be any size, any place, for both input and output. -* ZBUFF and ZSTD are 100% interoperable, -* frames created by one can be decoded by the other one */ - -typedef struct ZBUFFv08_DCtx_s ZBUFFv08_DCtx; -ZSTDLIB_API ZBUFFv08_DCtx* ZBUFFv08_createDCtx(void); -ZSTDLIB_API size_t ZBUFFv08_freeDCtx(ZBUFFv08_DCtx* dctx); - -ZSTDLIB_API size_t ZBUFFv08_decompressInit(ZBUFFv08_DCtx* dctx); -ZSTDLIB_API size_t ZBUFFv08_decompressInitDictionary(ZBUFFv08_DCtx* dctx, const void* dict, size_t dictSize); - -ZSTDLIB_API size_t ZBUFFv08_decompressContinue(ZBUFFv08_DCtx* dctx, - void* dst, size_t* dstCapacityPtr, - const void* src, size_t* srcSizePtr); - -/*-*************************************************************************** -* Streaming decompression howto -* -* A ZBUFFv08_DCtx object is required to track streaming operations. -* Use ZBUFFv08_createDCtx() and ZBUFFv08_freeDCtx() to create/release resources. -* Use ZBUFFv08_decompressInit() to start a new decompression operation, -* or ZBUFFv08_decompressInitDictionary() if decompression requires a dictionary. -* Note that ZBUFFv08_DCtx objects can be re-init multiple times. -* -* Use ZBUFFv08_decompressContinue() repetitively to consume your input. -* *srcSizePtr and *dstCapacityPtr can be any size. -* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr. -* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again. -* The content of `dst` will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change `dst`. -* @return : 0 when a frame is completely decoded and fully flushed, -* 1 when there is still some data left within internal buffer to flush, -* >1 when more data is expected, with value being a suggested next input size (it's just a hint, which helps latency), -* or an error code, which can be tested using ZBUFFv08_isError(). -* -* Hint : recommended buffer sizes (not compulsory) : ZBUFFv08_recommendedDInSize() and ZBUFFv08_recommendedDOutSize() -* output : ZBUFFv08_recommendedDOutSize== 128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded. -* input : ZBUFFv08_recommendedDInSize == 128KB + 3; -* just follow indications from ZBUFFv08_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . -* *******************************************************************************/ - - -/* ************************************* -* Tool functions -***************************************/ -ZSTDLIB_API unsigned ZBUFFv08_isError(size_t errorCode); -ZSTDLIB_API const char* ZBUFFv08_getErrorName(size_t errorCode); - -/** Functions below provide recommended buffer sizes for Compression or Decompression operations. -* These sizes are just hints, they tend to offer better latency */ -ZSTDLIB_API size_t ZBUFFv08_recommendedDInSize(void); -ZSTDLIB_API size_t ZBUFFv08_recommendedDOutSize(void); - - - -#define ZSTDv08_MAGICNUMBER 0xFD2FB528 /* v0.8 */ - - -#if defined (__cplusplus) -} -#endif - -#endif /* ZSTDv08_H_235446 */ - +#include <contrib/libs/zstd06/renames.h> +/* + zstd_v08 - decoder for 0.8 format + Header File + Copyright (C) 2014-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - zstd source repository : https://github.com/Cyan4973/zstd +*/ +#ifndef ZSTDv08_H_235446 +#define ZSTDv08_H_235446 + +#if defined (__cplusplus) +extern "C" { +#endif + +/*====== Dependency ======*/ +#include <stddef.h> /* size_t */ + + +/*====== Export for Windows ======*/ +/*! +* ZSTDv08_DLL_EXPORT : +* Enable exporting of functions when building a Windows DLL +*/ +#if defined(_WIN32) && defined(ZSTDv08_DLL_EXPORT) && (ZSTDv08_DLL_EXPORT==1) +# define ZSTDLIB_API __declspec(dllexport) +#else +# define ZSTDLIB_API +#endif + + +/* ************************************* +* Simple API +***************************************/ +/*! ZSTDv08_getDecompressedSize() : +* @return : decompressed size as a 64-bits value _if known_, 0 otherwise. +* note 1 : decompressed size can be very large (64-bits value), +* potentially larger than what local system can handle as a single memory segment. +* In which case, it's necessary to use streaming mode to decompress data. +* note 2 : decompressed size is an optional field, that may not be present. +* When `return==0`, consider data to decompress could have any size. +* In which case, it's necessary to use streaming mode to decompress data, +* or rely on application's implied limits. +* (For example, it may know that its own data is necessarily cut into blocks <= 16 KB). +* note 3 : decompressed size could be wrong or intentionally modified ! +* Always ensure result fits within application's authorized limits ! +* Each application can have its own set of conditions. +* If the intention is to decompress public data compressed by zstd command line utility, +* it is recommended to support at least 8 MB for extended compatibility. +* note 4 : when `return==0`, if precise failure cause is needed, use ZSTDv08_getFrameParams() to know more. */ +unsigned long long ZSTDv08_getDecompressedSize(const void* src, size_t srcSize); + +/*! ZSTDv08_decompress() : + `compressedSize` : must be the _exact_ size of compressed input, otherwise decompression will fail. + `dstCapacity` must be equal or larger than originalSize (see ZSTDv08_getDecompressedSize() ). + If originalSize is unknown, and if there is no implied application-specific limitations, + it's necessary to use streaming mode to decompress data. + @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), + or an errorCode if it fails (which can be tested using ZSTDv08_isError()) */ +ZSTDLIB_API size_t ZSTDv08_decompress( void* dst, size_t dstCapacity, + const void* src, size_t compressedSize); + + +/*====== Helper functions ======*/ +ZSTDLIB_API unsigned ZSTDv08_isError(size_t code); /*!< tells if a `size_t` function result is an error code */ +ZSTDLIB_API const char* ZSTDv08_getErrorName(size_t code); /*!< provides readable string from an error code */ + + +/*-************************************* +* Explicit memory management +***************************************/ +/** Decompression context */ +typedef struct ZSTDv08_DCtx_s ZSTDv08_DCtx; /*< incomplete type */ +ZSTDLIB_API ZSTDv08_DCtx* ZSTDv08_createDCtx(void); +ZSTDLIB_API size_t ZSTDv08_freeDCtx(ZSTDv08_DCtx* dctx); + +/** ZSTDv08_decompressDCtx() : +* Same as ZSTDv08_decompress(), requires an allocated ZSTDv08_DCtx (see ZSTDv08_createDCtx()) */ +ZSTDLIB_API size_t ZSTDv08_decompressDCtx(ZSTDv08_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); + + +/*-************************ +* Simple dictionary API +***************************/ +/*! ZSTDv08_decompress_usingDict() : +* Decompression using a predefined Dictionary (see dictBuilder/zdict.h). +* Dictionary must be identical to the one used during compression. +* Note : This function load the dictionary, resulting in a significant startup time */ +ZSTDLIB_API size_t ZSTDv08_decompress_usingDict(ZSTDv08_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize); + + +/*-************************** +* Fast Dictionary API +****************************/ +/*! ZSTDv08_createDDict() : +* Create a digested dictionary, ready to start decompression operation without startup delay. +* `dict` can be released after creation */ +typedef struct ZSTDv08_DDict_s ZSTDv08_DDict; +ZSTDLIB_API ZSTDv08_DDict* ZSTDv08_createDDict(const void* dict, size_t dictSize); +ZSTDLIB_API size_t ZSTDv08_freeDDict(ZSTDv08_DDict* ddict); + +/*! ZSTDv08_decompress_usingDDict() : +* Decompression using a digested Dictionary +* Faster startup than ZSTDv08_decompress_usingDict(), recommended when same dictionary is used multiple times. */ +ZSTDLIB_API size_t ZSTDv08_decompress_usingDDict(ZSTDv08_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTDv08_DDict* ddict); + +typedef struct { + unsigned long long frameContentSize; + unsigned windowSize; + unsigned dictID; + unsigned checksumFlag; +} ZSTDv08_frameParams; + +ZSTDLIB_API size_t ZSTDv08_getFrameParams(ZSTDv08_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input, see details below */ + + +/* *************************************************************** +* Compiler specifics +*****************************************************************/ +/* ZSTDv08_DLL_EXPORT : +* Enable exporting of functions when building a Windows DLL */ +#if defined(_WIN32) && defined(ZSTDv08_DLL_EXPORT) && (ZSTDv08_DLL_EXPORT==1) +# define ZSTDLIB_API __declspec(dllexport) +#else +# define ZSTDLIB_API +#endif + + +/* ************************************* +* Streaming functions +***************************************/ +/* This is the easier "buffered" streaming API, +* using an internal buffer to lift all restrictions on user-provided buffers +* which can be any size, any place, for both input and output. +* ZBUFF and ZSTD are 100% interoperable, +* frames created by one can be decoded by the other one */ + +typedef struct ZBUFFv08_DCtx_s ZBUFFv08_DCtx; +ZSTDLIB_API ZBUFFv08_DCtx* ZBUFFv08_createDCtx(void); +ZSTDLIB_API size_t ZBUFFv08_freeDCtx(ZBUFFv08_DCtx* dctx); + +ZSTDLIB_API size_t ZBUFFv08_decompressInit(ZBUFFv08_DCtx* dctx); +ZSTDLIB_API size_t ZBUFFv08_decompressInitDictionary(ZBUFFv08_DCtx* dctx, const void* dict, size_t dictSize); + +ZSTDLIB_API size_t ZBUFFv08_decompressContinue(ZBUFFv08_DCtx* dctx, + void* dst, size_t* dstCapacityPtr, + const void* src, size_t* srcSizePtr); + +/*-*************************************************************************** +* Streaming decompression howto +* +* A ZBUFFv08_DCtx object is required to track streaming operations. +* Use ZBUFFv08_createDCtx() and ZBUFFv08_freeDCtx() to create/release resources. +* Use ZBUFFv08_decompressInit() to start a new decompression operation, +* or ZBUFFv08_decompressInitDictionary() if decompression requires a dictionary. +* Note that ZBUFFv08_DCtx objects can be re-init multiple times. +* +* Use ZBUFFv08_decompressContinue() repetitively to consume your input. +* *srcSizePtr and *dstCapacityPtr can be any size. +* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr. +* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again. +* The content of `dst` will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change `dst`. +* @return : 0 when a frame is completely decoded and fully flushed, +* 1 when there is still some data left within internal buffer to flush, +* >1 when more data is expected, with value being a suggested next input size (it's just a hint, which helps latency), +* or an error code, which can be tested using ZBUFFv08_isError(). +* +* Hint : recommended buffer sizes (not compulsory) : ZBUFFv08_recommendedDInSize() and ZBUFFv08_recommendedDOutSize() +* output : ZBUFFv08_recommendedDOutSize== 128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded. +* input : ZBUFFv08_recommendedDInSize == 128KB + 3; +* just follow indications from ZBUFFv08_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . +* *******************************************************************************/ + + +/* ************************************* +* Tool functions +***************************************/ +ZSTDLIB_API unsigned ZBUFFv08_isError(size_t errorCode); +ZSTDLIB_API const char* ZBUFFv08_getErrorName(size_t errorCode); + +/** Functions below provide recommended buffer sizes for Compression or Decompression operations. +* These sizes are just hints, they tend to offer better latency */ +ZSTDLIB_API size_t ZBUFFv08_recommendedDInSize(void); +ZSTDLIB_API size_t ZBUFFv08_recommendedDOutSize(void); + + + +#define ZSTDv08_MAGICNUMBER 0xFD2FB528 /* v0.8 */ + + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTDv08_H_235446 */ + diff --git a/contrib/libs/zstd06/renames.h b/contrib/libs/zstd06/renames.h index 661ef5ec2a..f354e16b21 100644 --- a/contrib/libs/zstd06/renames.h +++ b/contrib/libs/zstd06/renames.h @@ -1,386 +1,386 @@ -#pragma once -#define FSEv08_buildDTable Legacy06_FSEv08_buildDTable -#define FSEv08_buildDTable_raw Legacy06_FSEv08_buildDTable_raw -#define FSEv08_buildDTable_rle Legacy06_FSEv08_buildDTable_rle -#define FSEv08_createDTable Legacy06_FSEv08_createDTable -#define FSEv08_decompress Legacy06_FSEv08_decompress -#define FSEv08_decompress_usingDTable Legacy06_FSEv08_decompress_usingDTable -#define FSEv08_freeDTable Legacy06_FSEv08_freeDTable -#define FSEv08_getErrorName Legacy06_FSEv08_getErrorName -#define FSEv08_isError Legacy06_FSEv08_isError -#define FSEv08_readNCount Legacy06_FSEv08_readNCount -#define HUFv08_decompress Legacy06_HUFv08_decompress -#define HUFv08_decompress1X2 Legacy06_HUFv08_decompress1X2 -#define HUFv08_decompress1X2_DCtx Legacy06_HUFv08_decompress1X2_DCtx -#define HUFv08_decompress1X2_usingDTable Legacy06_HUFv08_decompress1X2_usingDTable -#define HUFv08_decompress1X4 Legacy06_HUFv08_decompress1X4 -#define HUFv08_decompress1X4_DCtx Legacy06_HUFv08_decompress1X4_DCtx -#define HUFv08_decompress1X4_usingDTable Legacy06_HUFv08_decompress1X4_usingDTable -#define HUFv08_decompress1X_DCtx Legacy06_HUFv08_decompress1X_DCtx -#define HUFv08_decompress1X_usingDTable Legacy06_HUFv08_decompress1X_usingDTable -#define HUFv08_decompress4X2 Legacy06_HUFv08_decompress4X2 -#define HUFv08_decompress4X2_DCtx Legacy06_HUFv08_decompress4X2_DCtx -#define HUFv08_decompress4X2_usingDTable Legacy06_HUFv08_decompress4X2_usingDTable -#define HUFv08_decompress4X4 Legacy06_HUFv08_decompress4X4 -#define HUFv08_decompress4X4_DCtx Legacy06_HUFv08_decompress4X4_DCtx -#define HUFv08_decompress4X4_usingDTable Legacy06_HUFv08_decompress4X4_usingDTable -#define HUFv08_decompress4X_DCtx Legacy06_HUFv08_decompress4X_DCtx -#define HUFv08_decompress4X_hufOnly Legacy06_HUFv08_decompress4X_hufOnly -#define HUFv08_decompress4X_usingDTable Legacy06_HUFv08_decompress4X_usingDTable -#define HUFv08_getErrorName Legacy06_HUFv08_getErrorName -#define HUFv08_isError Legacy06_HUFv08_isError -#define HUFv08_readDTableX2 Legacy06_HUFv08_readDTableX2 -#define HUFv08_readDTableX4 Legacy06_HUFv08_readDTableX4 -#define HUFv08_readStats Legacy06_HUFv08_readStats -#define HUFv08_selectDecoder Legacy06_HUFv08_selectDecoder -#define ZBUFFv08_createDCtx Legacy06_ZBUFFv08_createDCtx -#define ZBUFFv08_createDCtx_advanced Legacy06_ZBUFFv08_createDCtx_advanced -#define ZBUFFv08_decompressContinue Legacy06_ZBUFFv08_decompressContinue -#define ZBUFFv08_decompressInit Legacy06_ZBUFFv08_decompressInit -#define ZBUFFv08_decompressInitDictionary Legacy06_ZBUFFv08_decompressInitDictionary -#define ZBUFFv08_freeDCtx Legacy06_ZBUFFv08_freeDCtx -#define ZBUFFv08_getErrorName Legacy06_ZBUFFv08_getErrorName -#define ZBUFFv08_isError Legacy06_ZBUFFv08_isError -#define ZBUFFv08_recommendedDInSize Legacy06_ZBUFFv08_recommendedDInSize -#define ZBUFFv08_recommendedDOutSize Legacy06_ZBUFFv08_recommendedDOutSize -#define ZSTDv08_copyDCtx Legacy06_ZSTDv08_copyDCtx -#define ZSTDv08_createDCtx Legacy06_ZSTDv08_createDCtx -#define ZSTDv08_createDCtx_advanced Legacy06_ZSTDv08_createDCtx_advanced -#define ZSTDv08_createDDict Legacy06_ZSTDv08_createDDict -#define ZSTDv08_createDDict_advanced Legacy06_ZSTDv08_createDDict_advanced -#define ZSTDv08_decodeLiteralsBlock Legacy06_ZSTDv08_decodeLiteralsBlock -#define ZSTDv08_decodeSeqHeaders Legacy06_ZSTDv08_decodeSeqHeaders -#define ZSTDv08_decompress Legacy06_ZSTDv08_decompress -#define ZSTDv08_decompressBegin Legacy06_ZSTDv08_decompressBegin -#define ZSTDv08_decompressBegin_usingDict Legacy06_ZSTDv08_decompressBegin_usingDict -#define ZSTDv08_decompressBlock Legacy06_ZSTDv08_decompressBlock -#define ZSTDv08_decompressContinue Legacy06_ZSTDv08_decompressContinue -#define ZSTDv08_decompressDCtx Legacy06_ZSTDv08_decompressDCtx -#define ZSTDv08_decompress_usingDDict Legacy06_ZSTDv08_decompress_usingDDict -#define ZSTDv08_decompress_usingDict Legacy06_ZSTDv08_decompress_usingDict -#define ZSTDv08_decompress_usingPreparedDCtx Legacy06_ZSTDv08_decompress_usingPreparedDCtx -#define ZSTDv08_defaultAllocFunction Legacy06_ZSTDv08_defaultAllocFunction -#define ZSTDv08_defaultFreeFunction Legacy06_ZSTDv08_defaultFreeFunction -#define ZSTDv08_estimateDCtxSize Legacy06_ZSTDv08_estimateDCtxSize -#define ZSTDv08_freeDCtx Legacy06_ZSTDv08_freeDCtx -#define ZSTDv08_freeDDict Legacy06_ZSTDv08_freeDDict -#define ZSTDv08_generateNxBytes Legacy06_ZSTDv08_generateNxBytes -#define ZSTDv08_getcBlockSize Legacy06_ZSTDv08_getcBlockSize -#define ZSTDv08_getDecompressedSize Legacy06_ZSTDv08_getDecompressedSize -#define ZSTDv08_getErrorCode Legacy06_ZSTDv08_getErrorCode -#define ZSTDv08_getErrorName Legacy06_ZSTDv08_getErrorName -#define ZSTDv08_getErrorString Legacy06_ZSTDv08_getErrorString -#define ZSTDv08_getFrameParams Legacy06_ZSTDv08_getFrameParams -#define ZSTDv08_insertBlock Legacy06_ZSTDv08_insertBlock -#define ZSTDv08_isError Legacy06_ZSTDv08_isError -#define ZSTDv08_isSkipFrame Legacy06_ZSTDv08_isSkipFrame -#define ZSTDv08_nextInputType Legacy06_ZSTDv08_nextInputType -#define ZSTDv08_nextSrcSizeToDecompress Legacy06_ZSTDv08_nextSrcSizeToDecompress -#define ZSTDv08_sizeofDCtx Legacy06_ZSTDv08_sizeofDCtx -#define FSEv07_buildDTable Legacy06_FSEv07_buildDTable -#define FSEv07_buildDTable_raw Legacy06_FSEv07_buildDTable_raw -#define FSEv07_buildDTable_rle Legacy06_FSEv07_buildDTable_rle -#define FSEv07_createDTable Legacy06_FSEv07_createDTable -#define FSEv07_decompress Legacy06_FSEv07_decompress -#define FSEv07_decompress_usingDTable Legacy06_FSEv07_decompress_usingDTable -#define FSEv07_freeDTable Legacy06_FSEv07_freeDTable -#define FSEv07_getErrorName Legacy06_FSEv07_getErrorName -#define FSEv07_isError Legacy06_FSEv07_isError -#define FSEv07_readNCount Legacy06_FSEv07_readNCount -#define HUFv07_decompress Legacy06_HUFv07_decompress -#define HUFv07_decompress1X2 Legacy06_HUFv07_decompress1X2 -#define HUFv07_decompress1X2_DCtx Legacy06_HUFv07_decompress1X2_DCtx -#define HUFv07_decompress1X2_usingDTable Legacy06_HUFv07_decompress1X2_usingDTable -#define HUFv07_decompress1X4 Legacy06_HUFv07_decompress1X4 -#define HUFv07_decompress1X4_DCtx Legacy06_HUFv07_decompress1X4_DCtx -#define HUFv07_decompress1X4_usingDTable Legacy06_HUFv07_decompress1X4_usingDTable -#define HUFv07_decompress1X_DCtx Legacy06_HUFv07_decompress1X_DCtx -#define HUFv07_decompress1X_usingDTable Legacy06_HUFv07_decompress1X_usingDTable -#define HUFv07_decompress4X2 Legacy06_HUFv07_decompress4X2 -#define HUFv07_decompress4X2_DCtx Legacy06_HUFv07_decompress4X2_DCtx -#define HUFv07_decompress4X2_usingDTable Legacy06_HUFv07_decompress4X2_usingDTable -#define HUFv07_decompress4X4 Legacy06_HUFv07_decompress4X4 -#define HUFv07_decompress4X4_DCtx Legacy06_HUFv07_decompress4X4_DCtx -#define HUFv07_decompress4X4_usingDTable Legacy06_HUFv07_decompress4X4_usingDTable -#define HUFv07_decompress4X_DCtx Legacy06_HUFv07_decompress4X_DCtx -#define HUFv07_decompress4X_hufOnly Legacy06_HUFv07_decompress4X_hufOnly -#define HUFv07_decompress4X_usingDTable Legacy06_HUFv07_decompress4X_usingDTable -#define HUFv07_getErrorName Legacy06_HUFv07_getErrorName -#define HUFv07_isError Legacy06_HUFv07_isError -#define HUFv07_readDTableX2 Legacy06_HUFv07_readDTableX2 -#define HUFv07_readDTableX4 Legacy06_HUFv07_readDTableX4 -#define HUFv07_readStats Legacy06_HUFv07_readStats -#define HUFv07_selectDecoder Legacy06_HUFv07_selectDecoder -#define ZBUFFv07_createDCtx Legacy06_ZBUFFv07_createDCtx -#define ZBUFFv07_createDCtx_advanced Legacy06_ZBUFFv07_createDCtx_advanced -#define ZBUFFv07_decompressContinue Legacy06_ZBUFFv07_decompressContinue -#define ZBUFFv07_decompressInit Legacy06_ZBUFFv07_decompressInit -#define ZBUFFv07_decompressInitDictionary Legacy06_ZBUFFv07_decompressInitDictionary -#define ZBUFFv07_freeDCtx Legacy06_ZBUFFv07_freeDCtx -#define ZBUFFv07_getErrorName Legacy06_ZBUFFv07_getErrorName -#define ZBUFFv07_isError Legacy06_ZBUFFv07_isError -#define ZBUFFv07_recommendedDInSize Legacy06_ZBUFFv07_recommendedDInSize -#define ZBUFFv07_recommendedDOutSize Legacy06_ZBUFFv07_recommendedDOutSize -#define ZSTDv07_copyDCtx Legacy06_ZSTDv07_copyDCtx -#define ZSTDv07_createDCtx Legacy06_ZSTDv07_createDCtx -#define ZSTDv07_createDCtx_advanced Legacy06_ZSTDv07_createDCtx_advanced -#define ZSTDv07_createDDict Legacy06_ZSTDv07_createDDict -#define ZSTDv07_createDDict_advanced Legacy06_ZSTDv07_createDDict_advanced -#define ZSTDv07_decodeLiteralsBlock Legacy06_ZSTDv07_decodeLiteralsBlock -#define ZSTDv07_decodeSeqHeaders Legacy06_ZSTDv07_decodeSeqHeaders -#define ZSTDv07_decompress Legacy06_ZSTDv07_decompress -#define ZSTDv07_decompressBegin Legacy06_ZSTDv07_decompressBegin -#define ZSTDv07_decompressBegin_usingDict Legacy06_ZSTDv07_decompressBegin_usingDict -#define ZSTDv07_decompressBlock Legacy06_ZSTDv07_decompressBlock -#define ZSTDv07_decompressContinue Legacy06_ZSTDv07_decompressContinue -#define ZSTDv07_decompressDCtx Legacy06_ZSTDv07_decompressDCtx -#define ZSTDv07_decompress_usingDDict Legacy06_ZSTDv07_decompress_usingDDict -#define ZSTDv07_decompress_usingDict Legacy06_ZSTDv07_decompress_usingDict -#define ZSTDv07_decompress_usingPreparedDCtx Legacy06_ZSTDv07_decompress_usingPreparedDCtx -#define ZSTDv07_defaultAllocFunction Legacy06_ZSTDv07_defaultAllocFunction -#define ZSTDv07_defaultFreeFunction Legacy06_ZSTDv07_defaultFreeFunction -#define ZSTDv07_estimateDCtxSize Legacy06_ZSTDv07_estimateDCtxSize -#define ZSTDv07_freeDCtx Legacy06_ZSTDv07_freeDCtx -#define ZSTDv07_freeDDict Legacy06_ZSTDv07_freeDDict -#define ZSTDv07_generateNxBytes Legacy06_ZSTDv07_generateNxBytes -#define ZSTDv07_getcBlockSize Legacy06_ZSTDv07_getcBlockSize -#define ZSTDv07_getDecompressedSize Legacy06_ZSTDv07_getDecompressedSize -#define ZSTDv07_getErrorCode Legacy06_ZSTDv07_getErrorCode -#define ZSTDv07_getErrorName Legacy06_ZSTDv07_getErrorName -#define ZSTDv07_getErrorString Legacy06_ZSTDv07_getErrorString -#define ZSTDv07_getFrameParams Legacy06_ZSTDv07_getFrameParams -#define ZSTDv07_insertBlock Legacy06_ZSTDv07_insertBlock -#define ZSTDv07_isError Legacy06_ZSTDv07_isError -#define ZSTDv07_isSkipFrame Legacy06_ZSTDv07_isSkipFrame -#define ZSTDv07_nextSrcSizeToDecompress Legacy06_ZSTDv07_nextSrcSizeToDecompress -#define ZSTDv07_sizeofDCtx Legacy06_ZSTDv07_sizeofDCtx -#define FSEv05_buildDTable Legacy06_FSEv05_buildDTable -#define FSEv05_buildDTable_raw Legacy06_FSEv05_buildDTable_raw -#define FSEv05_buildDTable_rle Legacy06_FSEv05_buildDTable_rle -#define FSEv05_createDTable Legacy06_FSEv05_createDTable -#define FSEv05_decompress Legacy06_FSEv05_decompress -#define FSEv05_decompress_usingDTable Legacy06_FSEv05_decompress_usingDTable -#define FSEv05_freeDTable Legacy06_FSEv05_freeDTable -#define FSEv05_getErrorName Legacy06_FSEv05_getErrorName -#define FSEv05_isError Legacy06_FSEv05_isError -#define FSEv05_readNCount Legacy06_FSEv05_readNCount -#define HUFv05_decompress Legacy06_HUFv05_decompress -#define HUFv05_decompress1X2 Legacy06_HUFv05_decompress1X2 -#define HUFv05_decompress1X2_usingDTable Legacy06_HUFv05_decompress1X2_usingDTable -#define HUFv05_decompress1X4 Legacy06_HUFv05_decompress1X4 -#define HUFv05_decompress1X4_usingDTable Legacy06_HUFv05_decompress1X4_usingDTable -#define HUFv05_decompress4X2 Legacy06_HUFv05_decompress4X2 -#define HUFv05_decompress4X2_usingDTable Legacy06_HUFv05_decompress4X2_usingDTable -#define HUFv05_decompress4X4 Legacy06_HUFv05_decompress4X4 -#define HUFv05_decompress4X4_usingDTable Legacy06_HUFv05_decompress4X4_usingDTable -#define HUFv05_getErrorName Legacy06_HUFv05_getErrorName -#define HUFv05_isError Legacy06_HUFv05_isError -#define HUFv05_readDTableX2 Legacy06_HUFv05_readDTableX2 -#define HUFv05_readDTableX4 Legacy06_HUFv05_readDTableX4 -#define ZBUFFv05_createDCtx Legacy06_ZBUFFv05_createDCtx -#define ZBUFFv05_decompressContinue Legacy06_ZBUFFv05_decompressContinue -#define ZBUFFv05_decompressInit Legacy06_ZBUFFv05_decompressInit -#define ZBUFFv05_decompressInitDictionary Legacy06_ZBUFFv05_decompressInitDictionary -#define ZBUFFv05_freeDCtx Legacy06_ZBUFFv05_freeDCtx -#define ZBUFFv05_getErrorName Legacy06_ZBUFFv05_getErrorName -#define ZBUFFv05_isError Legacy06_ZBUFFv05_isError -#define ZBUFFv05_recommendedDInSize Legacy06_ZBUFFv05_recommendedDInSize -#define ZBUFFv05_recommendedDOutSize Legacy06_ZBUFFv05_recommendedDOutSize -#define ZSTDv05_copyDCtx Legacy06_ZSTDv05_copyDCtx -#define ZSTDv05_createDCtx Legacy06_ZSTDv05_createDCtx -#define ZSTDv05_decodeLiteralsBlock Legacy06_ZSTDv05_decodeLiteralsBlock -#define ZSTDv05_decodeSeqHeaders Legacy06_ZSTDv05_decodeSeqHeaders -#define ZSTDv05_decompress Legacy06_ZSTDv05_decompress -#define ZSTDv05_decompressBegin Legacy06_ZSTDv05_decompressBegin -#define ZSTDv05_decompressBegin_usingDict Legacy06_ZSTDv05_decompressBegin_usingDict -#define ZSTDv05_decompressBlock Legacy06_ZSTDv05_decompressBlock -#define ZSTDv05_decompressContinue Legacy06_ZSTDv05_decompressContinue -#define ZSTDv05_decompressDCtx Legacy06_ZSTDv05_decompressDCtx -#define ZSTDv05_decompress_usingDict Legacy06_ZSTDv05_decompress_usingDict -#define ZSTDv05_decompress_usingPreparedDCtx Legacy06_ZSTDv05_decompress_usingPreparedDCtx -#define ZSTDv05_freeDCtx Legacy06_ZSTDv05_freeDCtx -#define ZSTDv05_getcBlockSize Legacy06_ZSTDv05_getcBlockSize -#define ZSTDv05_getError Legacy06_ZSTDv05_getError -#define ZSTDv05_getErrorName Legacy06_ZSTDv05_getErrorName -#define ZSTDv05_getFrameParams Legacy06_ZSTDv05_getFrameParams -#define ZSTDv05_isError Legacy06_ZSTDv05_isError -#define ZSTDv05_nextSrcSizeToDecompress Legacy06_ZSTDv05_nextSrcSizeToDecompress -#define ZSTDv05_sizeofDCtx Legacy06_ZSTDv05_sizeofDCtx -#define ZBUFFv04_createDCtx Legacy06_ZBUFFv04_createDCtx -#define ZBUFFv04_decompressContinue Legacy06_ZBUFFv04_decompressContinue -#define ZBUFFv04_decompressInit Legacy06_ZBUFFv04_decompressInit -#define ZBUFFv04_decompressWithDictionary Legacy06_ZBUFFv04_decompressWithDictionary -#define ZBUFFv04_freeDCtx Legacy06_ZBUFFv04_freeDCtx -#define ZBUFFv04_getErrorName Legacy06_ZBUFFv04_getErrorName -#define ZBUFFv04_isError Legacy06_ZBUFFv04_isError -#define ZBUFFv04_recommendedDInSize Legacy06_ZBUFFv04_recommendedDInSize -#define ZBUFFv04_recommendedDOutSize Legacy06_ZBUFFv04_recommendedDOutSize -#define ZSTDv04_createDCtx Legacy06_ZSTDv04_createDCtx -#define ZSTDv04_decompress Legacy06_ZSTDv04_decompress -#define ZSTDv04_decompressContinue Legacy06_ZSTDv04_decompressContinue -#define ZSTDv04_decompressDCtx Legacy06_ZSTDv04_decompressDCtx -#define ZSTDv04_freeDCtx Legacy06_ZSTDv04_freeDCtx -#define ZSTDv04_getFrameParams Legacy06_ZSTDv04_getFrameParams -#define ZSTDv04_nextSrcSizeToDecompress Legacy06_ZSTDv04_nextSrcSizeToDecompress -#define ZSTDv04_resetDCtx Legacy06_ZSTDv04_resetDCtx -#define ZSTDv03_createDCtx Legacy06_ZSTDv03_createDCtx -#define ZSTDv03_decompress Legacy06_ZSTDv03_decompress -#define ZSTDv03_decompressContinue Legacy06_ZSTDv03_decompressContinue -#define ZSTDv03_freeDCtx Legacy06_ZSTDv03_freeDCtx -#define ZSTDv03_isError Legacy06_ZSTDv03_isError -#define ZSTDv03_nextSrcSizeToDecompress Legacy06_ZSTDv03_nextSrcSizeToDecompress -#define ZSTDv03_resetDCtx Legacy06_ZSTDv03_resetDCtx -#define ZSTDv02_createDCtx Legacy06_ZSTDv02_createDCtx -#define ZSTDv02_decompress Legacy06_ZSTDv02_decompress -#define ZSTDv02_decompressContinue Legacy06_ZSTDv02_decompressContinue -#define ZSTDv02_freeDCtx Legacy06_ZSTDv02_freeDCtx -#define ZSTDv02_isError Legacy06_ZSTDv02_isError -#define ZSTDv02_nextSrcSizeToDecompress Legacy06_ZSTDv02_nextSrcSizeToDecompress -#define ZSTDv02_resetDCtx Legacy06_ZSTDv02_resetDCtx -#define ZSTDv01_createDCtx Legacy06_ZSTDv01_createDCtx -#define ZSTDv01_decompress Legacy06_ZSTDv01_decompress -#define ZSTDv01_decompressContinue Legacy06_ZSTDv01_decompressContinue -#define ZSTDv01_decompressDCtx Legacy06_ZSTDv01_decompressDCtx -#define ZSTDv01_freeDCtx Legacy06_ZSTDv01_freeDCtx -#define ZSTDv01_isError Legacy06_ZSTDv01_isError -#define ZSTDv01_nextSrcSizeToDecompress Legacy06_ZSTDv01_nextSrcSizeToDecompress -#define ZSTDv01_resetDCtx Legacy06_ZSTDv01_resetDCtx -#define ZDICT_getErrorName Legacy06_ZDICT_getErrorName -#define ZDICT_isError Legacy06_ZDICT_isError -#define ZDICT_trainFromBuffer Legacy06_ZDICT_trainFromBuffer -#define ZDICT_trainFromBuffer_advanced Legacy06_ZDICT_trainFromBuffer_advanced -#define ZDICT_trainFromBuffer_unsafe Legacy06_ZDICT_trainFromBuffer_unsafe -#define divbwt Legacy06_divbwt -#define divsufsort Legacy06_divsufsort -#define ZSTD_copyDCtx Legacy06_ZSTD_copyDCtx -#define ZSTD_createDCtx Legacy06_ZSTD_createDCtx -#define ZSTD_decodeLiteralsBlock Legacy06_ZSTD_decodeLiteralsBlock -#define ZSTD_decodeSeqHeaders Legacy06_ZSTD_decodeSeqHeaders -#define ZSTD_decompress Legacy06_ZSTD_decompress -#define ZSTD_decompressBegin Legacy06_ZSTD_decompressBegin -#define ZSTD_decompressBegin_usingDict Legacy06_ZSTD_decompressBegin_usingDict -#define ZSTD_decompressBlock Legacy06_ZSTD_decompressBlock -#define ZSTD_decompressContinue Legacy06_ZSTD_decompressContinue -#define ZSTD_decompressDCtx Legacy06_ZSTD_decompressDCtx -#define ZSTD_decompress_usingDict Legacy06_ZSTD_decompress_usingDict -#define ZSTD_decompress_usingPreparedDCtx Legacy06_ZSTD_decompress_usingPreparedDCtx -#define ZSTD_freeDCtx Legacy06_ZSTD_freeDCtx -#define ZSTD_getcBlockSize Legacy06_ZSTD_getcBlockSize -#define ZSTD_getFrameParams Legacy06_ZSTD_getFrameParams -#define ZSTD_nextSrcSizeToDecompress Legacy06_ZSTD_nextSrcSizeToDecompress -#define ZSTD_sizeofDCtx Legacy06_ZSTD_sizeofDCtx -#define ZBUFF_createDCtx Legacy06_ZBUFF_createDCtx -#define ZBUFF_decompressContinue Legacy06_ZBUFF_decompressContinue -#define ZBUFF_decompressInit Legacy06_ZBUFF_decompressInit -#define ZBUFF_decompressInitDictionary Legacy06_ZBUFF_decompressInitDictionary -#define ZBUFF_freeDCtx Legacy06_ZBUFF_freeDCtx -#define ZBUFF_recommendedDInSize Legacy06_ZBUFF_recommendedDInSize -#define ZBUFF_recommendedDOutSize Legacy06_ZBUFF_recommendedDOutSize -#define HUF_decompress Legacy06_HUF_decompress -#define HUF_decompress1X2 Legacy06_HUF_decompress1X2 -#define HUF_decompress1X2_usingDTable Legacy06_HUF_decompress1X2_usingDTable -#define HUF_decompress1X4 Legacy06_HUF_decompress1X4 -#define HUF_decompress1X4_usingDTable Legacy06_HUF_decompress1X4_usingDTable -#define HUF_decompress4X2 Legacy06_HUF_decompress4X2 -#define HUF_decompress4X2_usingDTable Legacy06_HUF_decompress4X2_usingDTable -#define HUF_decompress4X4 Legacy06_HUF_decompress4X4 -#define HUF_decompress4X4_usingDTable Legacy06_HUF_decompress4X4_usingDTable -#define HUF_readDTableX2 Legacy06_HUF_readDTableX2 -#define HUF_readDTableX4 Legacy06_HUF_readDTableX4 -#define ZSTD_adjustCParams Legacy06_ZSTD_adjustCParams -#define ZSTD_checkCParams Legacy06_ZSTD_checkCParams -#define ZSTD_checkCParams_advanced Legacy06_ZSTD_checkCParams_advanced -#define ZSTD_compress Legacy06_ZSTD_compress -#define ZSTD_compress_advanced Legacy06_ZSTD_compress_advanced -#define ZSTD_compressBegin Legacy06_ZSTD_compressBegin -#define ZSTD_compressBegin_advanced Legacy06_ZSTD_compressBegin_advanced -#define ZSTD_compressBegin_usingDict Legacy06_ZSTD_compressBegin_usingDict -#define ZSTD_compressBlock Legacy06_ZSTD_compressBlock -#define ZSTD_compressBlock_greedy_extDict Legacy06_ZSTD_compressBlock_greedy_extDict -#define ZSTD_compressBound Legacy06_ZSTD_compressBound -#define ZSTD_compressCCtx Legacy06_ZSTD_compressCCtx -#define ZSTD_compressContinue Legacy06_ZSTD_compressContinue -#define ZSTD_compressEnd Legacy06_ZSTD_compressEnd -#define ZSTD_compressSequences Legacy06_ZSTD_compressSequences -#define ZSTD_compress_usingDict Legacy06_ZSTD_compress_usingDict -#define ZSTD_compress_usingPreparedCCtx Legacy06_ZSTD_compress_usingPreparedCCtx -#define ZSTD_copyCCtx Legacy06_ZSTD_copyCCtx -#define ZSTD_createCCtx Legacy06_ZSTD_createCCtx -#define ZSTD_freeCCtx Legacy06_ZSTD_freeCCtx -#define ZSTD_getCParams Legacy06_ZSTD_getCParams -#define ZSTD_getSeqStore Legacy06_ZSTD_getSeqStore -#define ZSTD_maxCLevel Legacy06_ZSTD_maxCLevel -#define ZSTD_noCompressBlock Legacy06_ZSTD_noCompressBlock -#define ZSTD_seqToCodes Legacy06_ZSTD_seqToCodes -#define ZSTD_sizeofCCtx Legacy06_ZSTD_sizeofCCtx -#define ZBUFF_compressContinue Legacy06_ZBUFF_compressContinue -#define ZBUFF_compressEnd Legacy06_ZBUFF_compressEnd -#define ZBUFF_compressFlush Legacy06_ZBUFF_compressFlush -#define ZBUFF_compressInit Legacy06_ZBUFF_compressInit -#define ZBUFF_compressInit_advanced Legacy06_ZBUFF_compressInit_advanced -#define ZBUFF_compressInitDictionary Legacy06_ZBUFF_compressInitDictionary -#define ZBUFF_createCCtx Legacy06_ZBUFF_createCCtx -#define ZBUFF_freeCCtx Legacy06_ZBUFF_freeCCtx -#define ZBUFF_recommendedCInSize Legacy06_ZBUFF_recommendedCInSize -#define ZBUFF_recommendedCOutSize Legacy06_ZBUFF_recommendedCOutSize -#define HUF_buildCTable Legacy06_HUF_buildCTable -#define HUF_compress Legacy06_HUF_compress -#define HUF_compress1X Legacy06_HUF_compress1X -#define HUF_compress1X_usingCTable Legacy06_HUF_compress1X_usingCTable -#define HUF_compress2 Legacy06_HUF_compress2 -#define HUF_compress4X_usingCTable Legacy06_HUF_compress4X_usingCTable -#define HUF_compressBound Legacy06_HUF_compressBound -#define HUF_readCTable Legacy06_HUF_readCTable -#define HUF_writeCTable Legacy06_HUF_writeCTable -#define FSE_buildCTable Legacy06_FSE_buildCTable -#define FSE_buildCTable_raw Legacy06_FSE_buildCTable_raw -#define FSE_buildCTable_rle Legacy06_FSE_buildCTable_rle -#define FSE_compress Legacy06_FSE_compress -#define FSE_compress2 Legacy06_FSE_compress2 -#define FSE_compressBound Legacy06_FSE_compressBound -#define FSE_compress_usingCTable Legacy06_FSE_compress_usingCTable -#define FSE_count Legacy06_FSE_count -#define FSE_countFast Legacy06_FSE_countFast -#define FSE_createCTable Legacy06_FSE_createCTable -#define FSE_freeCTable Legacy06_FSE_freeCTable -#define FSE_NCountWriteBound Legacy06_FSE_NCountWriteBound -#define FSE_normalizeCount Legacy06_FSE_normalizeCount -#define FSE_optimalTableLog Legacy06_FSE_optimalTableLog -#define FSE_sizeof_CTable Legacy06_FSE_sizeof_CTable -#define FSE_writeNCount Legacy06_FSE_writeNCount -#define ZBUFF_getErrorName Legacy06_ZBUFF_getErrorName -#define ZBUFF_isError Legacy06_ZBUFF_isError -#define ZSTD_getErrorCode Legacy06_ZSTD_getErrorCode -#define ZSTD_getErrorName Legacy06_ZSTD_getErrorName -#define ZSTD_getErrorString Legacy06_ZSTD_getErrorString -#define ZSTD_isError Legacy06_ZSTD_isError -#define ZSTD_versionNumber Legacy06_ZSTD_versionNumber -#define XXH32 Legacy06_XXH32 -#define XXH32_canonicalFromHash Legacy06_XXH32_canonicalFromHash -#define XXH32_createState Legacy06_XXH32_createState -#define XXH32_digest Legacy06_XXH32_digest -#define XXH32_freeState Legacy06_XXH32_freeState -#define XXH32_hashFromCanonical Legacy06_XXH32_hashFromCanonical -#define XXH32_reset Legacy06_XXH32_reset -#define XXH32_update Legacy06_XXH32_update -#define XXH64 Legacy06_XXH64 -#define XXH64_canonicalFromHash Legacy06_XXH64_canonicalFromHash -#define XXH64_createState Legacy06_XXH64_createState -#define XXH64_digest Legacy06_XXH64_digest -#define XXH64_freeState Legacy06_XXH64_freeState -#define XXH64_hashFromCanonical Legacy06_XXH64_hashFromCanonical -#define XXH64_reset Legacy06_XXH64_reset -#define XXH64_update Legacy06_XXH64_update -#define XXH_versionNumber Legacy06_XXH_versionNumber -#define FSE_buildDTable Legacy06_FSE_buildDTable -#define FSE_buildDTable_raw Legacy06_FSE_buildDTable_raw -#define FSE_buildDTable_rle Legacy06_FSE_buildDTable_rle -#define FSE_createDTable Legacy06_FSE_createDTable -#define FSE_decompress Legacy06_FSE_decompress -#define FSE_decompress_usingDTable Legacy06_FSE_decompress_usingDTable -#define FSE_freeDTable Legacy06_FSE_freeDTable -#define FSE_getErrorName Legacy06_FSE_getErrorName -#define FSE_isError Legacy06_FSE_isError -#define FSE_readNCount Legacy06_FSE_readNCount -#define HUF_getErrorName Legacy06_HUF_getErrorName -#define HUF_isError Legacy06_HUF_isError +#pragma once +#define FSEv08_buildDTable Legacy06_FSEv08_buildDTable +#define FSEv08_buildDTable_raw Legacy06_FSEv08_buildDTable_raw +#define FSEv08_buildDTable_rle Legacy06_FSEv08_buildDTable_rle +#define FSEv08_createDTable Legacy06_FSEv08_createDTable +#define FSEv08_decompress Legacy06_FSEv08_decompress +#define FSEv08_decompress_usingDTable Legacy06_FSEv08_decompress_usingDTable +#define FSEv08_freeDTable Legacy06_FSEv08_freeDTable +#define FSEv08_getErrorName Legacy06_FSEv08_getErrorName +#define FSEv08_isError Legacy06_FSEv08_isError +#define FSEv08_readNCount Legacy06_FSEv08_readNCount +#define HUFv08_decompress Legacy06_HUFv08_decompress +#define HUFv08_decompress1X2 Legacy06_HUFv08_decompress1X2 +#define HUFv08_decompress1X2_DCtx Legacy06_HUFv08_decompress1X2_DCtx +#define HUFv08_decompress1X2_usingDTable Legacy06_HUFv08_decompress1X2_usingDTable +#define HUFv08_decompress1X4 Legacy06_HUFv08_decompress1X4 +#define HUFv08_decompress1X4_DCtx Legacy06_HUFv08_decompress1X4_DCtx +#define HUFv08_decompress1X4_usingDTable Legacy06_HUFv08_decompress1X4_usingDTable +#define HUFv08_decompress1X_DCtx Legacy06_HUFv08_decompress1X_DCtx +#define HUFv08_decompress1X_usingDTable Legacy06_HUFv08_decompress1X_usingDTable +#define HUFv08_decompress4X2 Legacy06_HUFv08_decompress4X2 +#define HUFv08_decompress4X2_DCtx Legacy06_HUFv08_decompress4X2_DCtx +#define HUFv08_decompress4X2_usingDTable Legacy06_HUFv08_decompress4X2_usingDTable +#define HUFv08_decompress4X4 Legacy06_HUFv08_decompress4X4 +#define HUFv08_decompress4X4_DCtx Legacy06_HUFv08_decompress4X4_DCtx +#define HUFv08_decompress4X4_usingDTable Legacy06_HUFv08_decompress4X4_usingDTable +#define HUFv08_decompress4X_DCtx Legacy06_HUFv08_decompress4X_DCtx +#define HUFv08_decompress4X_hufOnly Legacy06_HUFv08_decompress4X_hufOnly +#define HUFv08_decompress4X_usingDTable Legacy06_HUFv08_decompress4X_usingDTable +#define HUFv08_getErrorName Legacy06_HUFv08_getErrorName +#define HUFv08_isError Legacy06_HUFv08_isError +#define HUFv08_readDTableX2 Legacy06_HUFv08_readDTableX2 +#define HUFv08_readDTableX4 Legacy06_HUFv08_readDTableX4 +#define HUFv08_readStats Legacy06_HUFv08_readStats +#define HUFv08_selectDecoder Legacy06_HUFv08_selectDecoder +#define ZBUFFv08_createDCtx Legacy06_ZBUFFv08_createDCtx +#define ZBUFFv08_createDCtx_advanced Legacy06_ZBUFFv08_createDCtx_advanced +#define ZBUFFv08_decompressContinue Legacy06_ZBUFFv08_decompressContinue +#define ZBUFFv08_decompressInit Legacy06_ZBUFFv08_decompressInit +#define ZBUFFv08_decompressInitDictionary Legacy06_ZBUFFv08_decompressInitDictionary +#define ZBUFFv08_freeDCtx Legacy06_ZBUFFv08_freeDCtx +#define ZBUFFv08_getErrorName Legacy06_ZBUFFv08_getErrorName +#define ZBUFFv08_isError Legacy06_ZBUFFv08_isError +#define ZBUFFv08_recommendedDInSize Legacy06_ZBUFFv08_recommendedDInSize +#define ZBUFFv08_recommendedDOutSize Legacy06_ZBUFFv08_recommendedDOutSize +#define ZSTDv08_copyDCtx Legacy06_ZSTDv08_copyDCtx +#define ZSTDv08_createDCtx Legacy06_ZSTDv08_createDCtx +#define ZSTDv08_createDCtx_advanced Legacy06_ZSTDv08_createDCtx_advanced +#define ZSTDv08_createDDict Legacy06_ZSTDv08_createDDict +#define ZSTDv08_createDDict_advanced Legacy06_ZSTDv08_createDDict_advanced +#define ZSTDv08_decodeLiteralsBlock Legacy06_ZSTDv08_decodeLiteralsBlock +#define ZSTDv08_decodeSeqHeaders Legacy06_ZSTDv08_decodeSeqHeaders +#define ZSTDv08_decompress Legacy06_ZSTDv08_decompress +#define ZSTDv08_decompressBegin Legacy06_ZSTDv08_decompressBegin +#define ZSTDv08_decompressBegin_usingDict Legacy06_ZSTDv08_decompressBegin_usingDict +#define ZSTDv08_decompressBlock Legacy06_ZSTDv08_decompressBlock +#define ZSTDv08_decompressContinue Legacy06_ZSTDv08_decompressContinue +#define ZSTDv08_decompressDCtx Legacy06_ZSTDv08_decompressDCtx +#define ZSTDv08_decompress_usingDDict Legacy06_ZSTDv08_decompress_usingDDict +#define ZSTDv08_decompress_usingDict Legacy06_ZSTDv08_decompress_usingDict +#define ZSTDv08_decompress_usingPreparedDCtx Legacy06_ZSTDv08_decompress_usingPreparedDCtx +#define ZSTDv08_defaultAllocFunction Legacy06_ZSTDv08_defaultAllocFunction +#define ZSTDv08_defaultFreeFunction Legacy06_ZSTDv08_defaultFreeFunction +#define ZSTDv08_estimateDCtxSize Legacy06_ZSTDv08_estimateDCtxSize +#define ZSTDv08_freeDCtx Legacy06_ZSTDv08_freeDCtx +#define ZSTDv08_freeDDict Legacy06_ZSTDv08_freeDDict +#define ZSTDv08_generateNxBytes Legacy06_ZSTDv08_generateNxBytes +#define ZSTDv08_getcBlockSize Legacy06_ZSTDv08_getcBlockSize +#define ZSTDv08_getDecompressedSize Legacy06_ZSTDv08_getDecompressedSize +#define ZSTDv08_getErrorCode Legacy06_ZSTDv08_getErrorCode +#define ZSTDv08_getErrorName Legacy06_ZSTDv08_getErrorName +#define ZSTDv08_getErrorString Legacy06_ZSTDv08_getErrorString +#define ZSTDv08_getFrameParams Legacy06_ZSTDv08_getFrameParams +#define ZSTDv08_insertBlock Legacy06_ZSTDv08_insertBlock +#define ZSTDv08_isError Legacy06_ZSTDv08_isError +#define ZSTDv08_isSkipFrame Legacy06_ZSTDv08_isSkipFrame +#define ZSTDv08_nextInputType Legacy06_ZSTDv08_nextInputType +#define ZSTDv08_nextSrcSizeToDecompress Legacy06_ZSTDv08_nextSrcSizeToDecompress +#define ZSTDv08_sizeofDCtx Legacy06_ZSTDv08_sizeofDCtx +#define FSEv07_buildDTable Legacy06_FSEv07_buildDTable +#define FSEv07_buildDTable_raw Legacy06_FSEv07_buildDTable_raw +#define FSEv07_buildDTable_rle Legacy06_FSEv07_buildDTable_rle +#define FSEv07_createDTable Legacy06_FSEv07_createDTable +#define FSEv07_decompress Legacy06_FSEv07_decompress +#define FSEv07_decompress_usingDTable Legacy06_FSEv07_decompress_usingDTable +#define FSEv07_freeDTable Legacy06_FSEv07_freeDTable +#define FSEv07_getErrorName Legacy06_FSEv07_getErrorName +#define FSEv07_isError Legacy06_FSEv07_isError +#define FSEv07_readNCount Legacy06_FSEv07_readNCount +#define HUFv07_decompress Legacy06_HUFv07_decompress +#define HUFv07_decompress1X2 Legacy06_HUFv07_decompress1X2 +#define HUFv07_decompress1X2_DCtx Legacy06_HUFv07_decompress1X2_DCtx +#define HUFv07_decompress1X2_usingDTable Legacy06_HUFv07_decompress1X2_usingDTable +#define HUFv07_decompress1X4 Legacy06_HUFv07_decompress1X4 +#define HUFv07_decompress1X4_DCtx Legacy06_HUFv07_decompress1X4_DCtx +#define HUFv07_decompress1X4_usingDTable Legacy06_HUFv07_decompress1X4_usingDTable +#define HUFv07_decompress1X_DCtx Legacy06_HUFv07_decompress1X_DCtx +#define HUFv07_decompress1X_usingDTable Legacy06_HUFv07_decompress1X_usingDTable +#define HUFv07_decompress4X2 Legacy06_HUFv07_decompress4X2 +#define HUFv07_decompress4X2_DCtx Legacy06_HUFv07_decompress4X2_DCtx +#define HUFv07_decompress4X2_usingDTable Legacy06_HUFv07_decompress4X2_usingDTable +#define HUFv07_decompress4X4 Legacy06_HUFv07_decompress4X4 +#define HUFv07_decompress4X4_DCtx Legacy06_HUFv07_decompress4X4_DCtx +#define HUFv07_decompress4X4_usingDTable Legacy06_HUFv07_decompress4X4_usingDTable +#define HUFv07_decompress4X_DCtx Legacy06_HUFv07_decompress4X_DCtx +#define HUFv07_decompress4X_hufOnly Legacy06_HUFv07_decompress4X_hufOnly +#define HUFv07_decompress4X_usingDTable Legacy06_HUFv07_decompress4X_usingDTable +#define HUFv07_getErrorName Legacy06_HUFv07_getErrorName +#define HUFv07_isError Legacy06_HUFv07_isError +#define HUFv07_readDTableX2 Legacy06_HUFv07_readDTableX2 +#define HUFv07_readDTableX4 Legacy06_HUFv07_readDTableX4 +#define HUFv07_readStats Legacy06_HUFv07_readStats +#define HUFv07_selectDecoder Legacy06_HUFv07_selectDecoder +#define ZBUFFv07_createDCtx Legacy06_ZBUFFv07_createDCtx +#define ZBUFFv07_createDCtx_advanced Legacy06_ZBUFFv07_createDCtx_advanced +#define ZBUFFv07_decompressContinue Legacy06_ZBUFFv07_decompressContinue +#define ZBUFFv07_decompressInit Legacy06_ZBUFFv07_decompressInit +#define ZBUFFv07_decompressInitDictionary Legacy06_ZBUFFv07_decompressInitDictionary +#define ZBUFFv07_freeDCtx Legacy06_ZBUFFv07_freeDCtx +#define ZBUFFv07_getErrorName Legacy06_ZBUFFv07_getErrorName +#define ZBUFFv07_isError Legacy06_ZBUFFv07_isError +#define ZBUFFv07_recommendedDInSize Legacy06_ZBUFFv07_recommendedDInSize +#define ZBUFFv07_recommendedDOutSize Legacy06_ZBUFFv07_recommendedDOutSize +#define ZSTDv07_copyDCtx Legacy06_ZSTDv07_copyDCtx +#define ZSTDv07_createDCtx Legacy06_ZSTDv07_createDCtx +#define ZSTDv07_createDCtx_advanced Legacy06_ZSTDv07_createDCtx_advanced +#define ZSTDv07_createDDict Legacy06_ZSTDv07_createDDict +#define ZSTDv07_createDDict_advanced Legacy06_ZSTDv07_createDDict_advanced +#define ZSTDv07_decodeLiteralsBlock Legacy06_ZSTDv07_decodeLiteralsBlock +#define ZSTDv07_decodeSeqHeaders Legacy06_ZSTDv07_decodeSeqHeaders +#define ZSTDv07_decompress Legacy06_ZSTDv07_decompress +#define ZSTDv07_decompressBegin Legacy06_ZSTDv07_decompressBegin +#define ZSTDv07_decompressBegin_usingDict Legacy06_ZSTDv07_decompressBegin_usingDict +#define ZSTDv07_decompressBlock Legacy06_ZSTDv07_decompressBlock +#define ZSTDv07_decompressContinue Legacy06_ZSTDv07_decompressContinue +#define ZSTDv07_decompressDCtx Legacy06_ZSTDv07_decompressDCtx +#define ZSTDv07_decompress_usingDDict Legacy06_ZSTDv07_decompress_usingDDict +#define ZSTDv07_decompress_usingDict Legacy06_ZSTDv07_decompress_usingDict +#define ZSTDv07_decompress_usingPreparedDCtx Legacy06_ZSTDv07_decompress_usingPreparedDCtx +#define ZSTDv07_defaultAllocFunction Legacy06_ZSTDv07_defaultAllocFunction +#define ZSTDv07_defaultFreeFunction Legacy06_ZSTDv07_defaultFreeFunction +#define ZSTDv07_estimateDCtxSize Legacy06_ZSTDv07_estimateDCtxSize +#define ZSTDv07_freeDCtx Legacy06_ZSTDv07_freeDCtx +#define ZSTDv07_freeDDict Legacy06_ZSTDv07_freeDDict +#define ZSTDv07_generateNxBytes Legacy06_ZSTDv07_generateNxBytes +#define ZSTDv07_getcBlockSize Legacy06_ZSTDv07_getcBlockSize +#define ZSTDv07_getDecompressedSize Legacy06_ZSTDv07_getDecompressedSize +#define ZSTDv07_getErrorCode Legacy06_ZSTDv07_getErrorCode +#define ZSTDv07_getErrorName Legacy06_ZSTDv07_getErrorName +#define ZSTDv07_getErrorString Legacy06_ZSTDv07_getErrorString +#define ZSTDv07_getFrameParams Legacy06_ZSTDv07_getFrameParams +#define ZSTDv07_insertBlock Legacy06_ZSTDv07_insertBlock +#define ZSTDv07_isError Legacy06_ZSTDv07_isError +#define ZSTDv07_isSkipFrame Legacy06_ZSTDv07_isSkipFrame +#define ZSTDv07_nextSrcSizeToDecompress Legacy06_ZSTDv07_nextSrcSizeToDecompress +#define ZSTDv07_sizeofDCtx Legacy06_ZSTDv07_sizeofDCtx +#define FSEv05_buildDTable Legacy06_FSEv05_buildDTable +#define FSEv05_buildDTable_raw Legacy06_FSEv05_buildDTable_raw +#define FSEv05_buildDTable_rle Legacy06_FSEv05_buildDTable_rle +#define FSEv05_createDTable Legacy06_FSEv05_createDTable +#define FSEv05_decompress Legacy06_FSEv05_decompress +#define FSEv05_decompress_usingDTable Legacy06_FSEv05_decompress_usingDTable +#define FSEv05_freeDTable Legacy06_FSEv05_freeDTable +#define FSEv05_getErrorName Legacy06_FSEv05_getErrorName +#define FSEv05_isError Legacy06_FSEv05_isError +#define FSEv05_readNCount Legacy06_FSEv05_readNCount +#define HUFv05_decompress Legacy06_HUFv05_decompress +#define HUFv05_decompress1X2 Legacy06_HUFv05_decompress1X2 +#define HUFv05_decompress1X2_usingDTable Legacy06_HUFv05_decompress1X2_usingDTable +#define HUFv05_decompress1X4 Legacy06_HUFv05_decompress1X4 +#define HUFv05_decompress1X4_usingDTable Legacy06_HUFv05_decompress1X4_usingDTable +#define HUFv05_decompress4X2 Legacy06_HUFv05_decompress4X2 +#define HUFv05_decompress4X2_usingDTable Legacy06_HUFv05_decompress4X2_usingDTable +#define HUFv05_decompress4X4 Legacy06_HUFv05_decompress4X4 +#define HUFv05_decompress4X4_usingDTable Legacy06_HUFv05_decompress4X4_usingDTable +#define HUFv05_getErrorName Legacy06_HUFv05_getErrorName +#define HUFv05_isError Legacy06_HUFv05_isError +#define HUFv05_readDTableX2 Legacy06_HUFv05_readDTableX2 +#define HUFv05_readDTableX4 Legacy06_HUFv05_readDTableX4 +#define ZBUFFv05_createDCtx Legacy06_ZBUFFv05_createDCtx +#define ZBUFFv05_decompressContinue Legacy06_ZBUFFv05_decompressContinue +#define ZBUFFv05_decompressInit Legacy06_ZBUFFv05_decompressInit +#define ZBUFFv05_decompressInitDictionary Legacy06_ZBUFFv05_decompressInitDictionary +#define ZBUFFv05_freeDCtx Legacy06_ZBUFFv05_freeDCtx +#define ZBUFFv05_getErrorName Legacy06_ZBUFFv05_getErrorName +#define ZBUFFv05_isError Legacy06_ZBUFFv05_isError +#define ZBUFFv05_recommendedDInSize Legacy06_ZBUFFv05_recommendedDInSize +#define ZBUFFv05_recommendedDOutSize Legacy06_ZBUFFv05_recommendedDOutSize +#define ZSTDv05_copyDCtx Legacy06_ZSTDv05_copyDCtx +#define ZSTDv05_createDCtx Legacy06_ZSTDv05_createDCtx +#define ZSTDv05_decodeLiteralsBlock Legacy06_ZSTDv05_decodeLiteralsBlock +#define ZSTDv05_decodeSeqHeaders Legacy06_ZSTDv05_decodeSeqHeaders +#define ZSTDv05_decompress Legacy06_ZSTDv05_decompress +#define ZSTDv05_decompressBegin Legacy06_ZSTDv05_decompressBegin +#define ZSTDv05_decompressBegin_usingDict Legacy06_ZSTDv05_decompressBegin_usingDict +#define ZSTDv05_decompressBlock Legacy06_ZSTDv05_decompressBlock +#define ZSTDv05_decompressContinue Legacy06_ZSTDv05_decompressContinue +#define ZSTDv05_decompressDCtx Legacy06_ZSTDv05_decompressDCtx +#define ZSTDv05_decompress_usingDict Legacy06_ZSTDv05_decompress_usingDict +#define ZSTDv05_decompress_usingPreparedDCtx Legacy06_ZSTDv05_decompress_usingPreparedDCtx +#define ZSTDv05_freeDCtx Legacy06_ZSTDv05_freeDCtx +#define ZSTDv05_getcBlockSize Legacy06_ZSTDv05_getcBlockSize +#define ZSTDv05_getError Legacy06_ZSTDv05_getError +#define ZSTDv05_getErrorName Legacy06_ZSTDv05_getErrorName +#define ZSTDv05_getFrameParams Legacy06_ZSTDv05_getFrameParams +#define ZSTDv05_isError Legacy06_ZSTDv05_isError +#define ZSTDv05_nextSrcSizeToDecompress Legacy06_ZSTDv05_nextSrcSizeToDecompress +#define ZSTDv05_sizeofDCtx Legacy06_ZSTDv05_sizeofDCtx +#define ZBUFFv04_createDCtx Legacy06_ZBUFFv04_createDCtx +#define ZBUFFv04_decompressContinue Legacy06_ZBUFFv04_decompressContinue +#define ZBUFFv04_decompressInit Legacy06_ZBUFFv04_decompressInit +#define ZBUFFv04_decompressWithDictionary Legacy06_ZBUFFv04_decompressWithDictionary +#define ZBUFFv04_freeDCtx Legacy06_ZBUFFv04_freeDCtx +#define ZBUFFv04_getErrorName Legacy06_ZBUFFv04_getErrorName +#define ZBUFFv04_isError Legacy06_ZBUFFv04_isError +#define ZBUFFv04_recommendedDInSize Legacy06_ZBUFFv04_recommendedDInSize +#define ZBUFFv04_recommendedDOutSize Legacy06_ZBUFFv04_recommendedDOutSize +#define ZSTDv04_createDCtx Legacy06_ZSTDv04_createDCtx +#define ZSTDv04_decompress Legacy06_ZSTDv04_decompress +#define ZSTDv04_decompressContinue Legacy06_ZSTDv04_decompressContinue +#define ZSTDv04_decompressDCtx Legacy06_ZSTDv04_decompressDCtx +#define ZSTDv04_freeDCtx Legacy06_ZSTDv04_freeDCtx +#define ZSTDv04_getFrameParams Legacy06_ZSTDv04_getFrameParams +#define ZSTDv04_nextSrcSizeToDecompress Legacy06_ZSTDv04_nextSrcSizeToDecompress +#define ZSTDv04_resetDCtx Legacy06_ZSTDv04_resetDCtx +#define ZSTDv03_createDCtx Legacy06_ZSTDv03_createDCtx +#define ZSTDv03_decompress Legacy06_ZSTDv03_decompress +#define ZSTDv03_decompressContinue Legacy06_ZSTDv03_decompressContinue +#define ZSTDv03_freeDCtx Legacy06_ZSTDv03_freeDCtx +#define ZSTDv03_isError Legacy06_ZSTDv03_isError +#define ZSTDv03_nextSrcSizeToDecompress Legacy06_ZSTDv03_nextSrcSizeToDecompress +#define ZSTDv03_resetDCtx Legacy06_ZSTDv03_resetDCtx +#define ZSTDv02_createDCtx Legacy06_ZSTDv02_createDCtx +#define ZSTDv02_decompress Legacy06_ZSTDv02_decompress +#define ZSTDv02_decompressContinue Legacy06_ZSTDv02_decompressContinue +#define ZSTDv02_freeDCtx Legacy06_ZSTDv02_freeDCtx +#define ZSTDv02_isError Legacy06_ZSTDv02_isError +#define ZSTDv02_nextSrcSizeToDecompress Legacy06_ZSTDv02_nextSrcSizeToDecompress +#define ZSTDv02_resetDCtx Legacy06_ZSTDv02_resetDCtx +#define ZSTDv01_createDCtx Legacy06_ZSTDv01_createDCtx +#define ZSTDv01_decompress Legacy06_ZSTDv01_decompress +#define ZSTDv01_decompressContinue Legacy06_ZSTDv01_decompressContinue +#define ZSTDv01_decompressDCtx Legacy06_ZSTDv01_decompressDCtx +#define ZSTDv01_freeDCtx Legacy06_ZSTDv01_freeDCtx +#define ZSTDv01_isError Legacy06_ZSTDv01_isError +#define ZSTDv01_nextSrcSizeToDecompress Legacy06_ZSTDv01_nextSrcSizeToDecompress +#define ZSTDv01_resetDCtx Legacy06_ZSTDv01_resetDCtx +#define ZDICT_getErrorName Legacy06_ZDICT_getErrorName +#define ZDICT_isError Legacy06_ZDICT_isError +#define ZDICT_trainFromBuffer Legacy06_ZDICT_trainFromBuffer +#define ZDICT_trainFromBuffer_advanced Legacy06_ZDICT_trainFromBuffer_advanced +#define ZDICT_trainFromBuffer_unsafe Legacy06_ZDICT_trainFromBuffer_unsafe +#define divbwt Legacy06_divbwt +#define divsufsort Legacy06_divsufsort +#define ZSTD_copyDCtx Legacy06_ZSTD_copyDCtx +#define ZSTD_createDCtx Legacy06_ZSTD_createDCtx +#define ZSTD_decodeLiteralsBlock Legacy06_ZSTD_decodeLiteralsBlock +#define ZSTD_decodeSeqHeaders Legacy06_ZSTD_decodeSeqHeaders +#define ZSTD_decompress Legacy06_ZSTD_decompress +#define ZSTD_decompressBegin Legacy06_ZSTD_decompressBegin +#define ZSTD_decompressBegin_usingDict Legacy06_ZSTD_decompressBegin_usingDict +#define ZSTD_decompressBlock Legacy06_ZSTD_decompressBlock +#define ZSTD_decompressContinue Legacy06_ZSTD_decompressContinue +#define ZSTD_decompressDCtx Legacy06_ZSTD_decompressDCtx +#define ZSTD_decompress_usingDict Legacy06_ZSTD_decompress_usingDict +#define ZSTD_decompress_usingPreparedDCtx Legacy06_ZSTD_decompress_usingPreparedDCtx +#define ZSTD_freeDCtx Legacy06_ZSTD_freeDCtx +#define ZSTD_getcBlockSize Legacy06_ZSTD_getcBlockSize +#define ZSTD_getFrameParams Legacy06_ZSTD_getFrameParams +#define ZSTD_nextSrcSizeToDecompress Legacy06_ZSTD_nextSrcSizeToDecompress +#define ZSTD_sizeofDCtx Legacy06_ZSTD_sizeofDCtx +#define ZBUFF_createDCtx Legacy06_ZBUFF_createDCtx +#define ZBUFF_decompressContinue Legacy06_ZBUFF_decompressContinue +#define ZBUFF_decompressInit Legacy06_ZBUFF_decompressInit +#define ZBUFF_decompressInitDictionary Legacy06_ZBUFF_decompressInitDictionary +#define ZBUFF_freeDCtx Legacy06_ZBUFF_freeDCtx +#define ZBUFF_recommendedDInSize Legacy06_ZBUFF_recommendedDInSize +#define ZBUFF_recommendedDOutSize Legacy06_ZBUFF_recommendedDOutSize +#define HUF_decompress Legacy06_HUF_decompress +#define HUF_decompress1X2 Legacy06_HUF_decompress1X2 +#define HUF_decompress1X2_usingDTable Legacy06_HUF_decompress1X2_usingDTable +#define HUF_decompress1X4 Legacy06_HUF_decompress1X4 +#define HUF_decompress1X4_usingDTable Legacy06_HUF_decompress1X4_usingDTable +#define HUF_decompress4X2 Legacy06_HUF_decompress4X2 +#define HUF_decompress4X2_usingDTable Legacy06_HUF_decompress4X2_usingDTable +#define HUF_decompress4X4 Legacy06_HUF_decompress4X4 +#define HUF_decompress4X4_usingDTable Legacy06_HUF_decompress4X4_usingDTable +#define HUF_readDTableX2 Legacy06_HUF_readDTableX2 +#define HUF_readDTableX4 Legacy06_HUF_readDTableX4 +#define ZSTD_adjustCParams Legacy06_ZSTD_adjustCParams +#define ZSTD_checkCParams Legacy06_ZSTD_checkCParams +#define ZSTD_checkCParams_advanced Legacy06_ZSTD_checkCParams_advanced +#define ZSTD_compress Legacy06_ZSTD_compress +#define ZSTD_compress_advanced Legacy06_ZSTD_compress_advanced +#define ZSTD_compressBegin Legacy06_ZSTD_compressBegin +#define ZSTD_compressBegin_advanced Legacy06_ZSTD_compressBegin_advanced +#define ZSTD_compressBegin_usingDict Legacy06_ZSTD_compressBegin_usingDict +#define ZSTD_compressBlock Legacy06_ZSTD_compressBlock +#define ZSTD_compressBlock_greedy_extDict Legacy06_ZSTD_compressBlock_greedy_extDict +#define ZSTD_compressBound Legacy06_ZSTD_compressBound +#define ZSTD_compressCCtx Legacy06_ZSTD_compressCCtx +#define ZSTD_compressContinue Legacy06_ZSTD_compressContinue +#define ZSTD_compressEnd Legacy06_ZSTD_compressEnd +#define ZSTD_compressSequences Legacy06_ZSTD_compressSequences +#define ZSTD_compress_usingDict Legacy06_ZSTD_compress_usingDict +#define ZSTD_compress_usingPreparedCCtx Legacy06_ZSTD_compress_usingPreparedCCtx +#define ZSTD_copyCCtx Legacy06_ZSTD_copyCCtx +#define ZSTD_createCCtx Legacy06_ZSTD_createCCtx +#define ZSTD_freeCCtx Legacy06_ZSTD_freeCCtx +#define ZSTD_getCParams Legacy06_ZSTD_getCParams +#define ZSTD_getSeqStore Legacy06_ZSTD_getSeqStore +#define ZSTD_maxCLevel Legacy06_ZSTD_maxCLevel +#define ZSTD_noCompressBlock Legacy06_ZSTD_noCompressBlock +#define ZSTD_seqToCodes Legacy06_ZSTD_seqToCodes +#define ZSTD_sizeofCCtx Legacy06_ZSTD_sizeofCCtx +#define ZBUFF_compressContinue Legacy06_ZBUFF_compressContinue +#define ZBUFF_compressEnd Legacy06_ZBUFF_compressEnd +#define ZBUFF_compressFlush Legacy06_ZBUFF_compressFlush +#define ZBUFF_compressInit Legacy06_ZBUFF_compressInit +#define ZBUFF_compressInit_advanced Legacy06_ZBUFF_compressInit_advanced +#define ZBUFF_compressInitDictionary Legacy06_ZBUFF_compressInitDictionary +#define ZBUFF_createCCtx Legacy06_ZBUFF_createCCtx +#define ZBUFF_freeCCtx Legacy06_ZBUFF_freeCCtx +#define ZBUFF_recommendedCInSize Legacy06_ZBUFF_recommendedCInSize +#define ZBUFF_recommendedCOutSize Legacy06_ZBUFF_recommendedCOutSize +#define HUF_buildCTable Legacy06_HUF_buildCTable +#define HUF_compress Legacy06_HUF_compress +#define HUF_compress1X Legacy06_HUF_compress1X +#define HUF_compress1X_usingCTable Legacy06_HUF_compress1X_usingCTable +#define HUF_compress2 Legacy06_HUF_compress2 +#define HUF_compress4X_usingCTable Legacy06_HUF_compress4X_usingCTable +#define HUF_compressBound Legacy06_HUF_compressBound +#define HUF_readCTable Legacy06_HUF_readCTable +#define HUF_writeCTable Legacy06_HUF_writeCTable +#define FSE_buildCTable Legacy06_FSE_buildCTable +#define FSE_buildCTable_raw Legacy06_FSE_buildCTable_raw +#define FSE_buildCTable_rle Legacy06_FSE_buildCTable_rle +#define FSE_compress Legacy06_FSE_compress +#define FSE_compress2 Legacy06_FSE_compress2 +#define FSE_compressBound Legacy06_FSE_compressBound +#define FSE_compress_usingCTable Legacy06_FSE_compress_usingCTable +#define FSE_count Legacy06_FSE_count +#define FSE_countFast Legacy06_FSE_countFast +#define FSE_createCTable Legacy06_FSE_createCTable +#define FSE_freeCTable Legacy06_FSE_freeCTable +#define FSE_NCountWriteBound Legacy06_FSE_NCountWriteBound +#define FSE_normalizeCount Legacy06_FSE_normalizeCount +#define FSE_optimalTableLog Legacy06_FSE_optimalTableLog +#define FSE_sizeof_CTable Legacy06_FSE_sizeof_CTable +#define FSE_writeNCount Legacy06_FSE_writeNCount +#define ZBUFF_getErrorName Legacy06_ZBUFF_getErrorName +#define ZBUFF_isError Legacy06_ZBUFF_isError +#define ZSTD_getErrorCode Legacy06_ZSTD_getErrorCode +#define ZSTD_getErrorName Legacy06_ZSTD_getErrorName +#define ZSTD_getErrorString Legacy06_ZSTD_getErrorString +#define ZSTD_isError Legacy06_ZSTD_isError +#define ZSTD_versionNumber Legacy06_ZSTD_versionNumber +#define XXH32 Legacy06_XXH32 +#define XXH32_canonicalFromHash Legacy06_XXH32_canonicalFromHash +#define XXH32_createState Legacy06_XXH32_createState +#define XXH32_digest Legacy06_XXH32_digest +#define XXH32_freeState Legacy06_XXH32_freeState +#define XXH32_hashFromCanonical Legacy06_XXH32_hashFromCanonical +#define XXH32_reset Legacy06_XXH32_reset +#define XXH32_update Legacy06_XXH32_update +#define XXH64 Legacy06_XXH64 +#define XXH64_canonicalFromHash Legacy06_XXH64_canonicalFromHash +#define XXH64_createState Legacy06_XXH64_createState +#define XXH64_digest Legacy06_XXH64_digest +#define XXH64_freeState Legacy06_XXH64_freeState +#define XXH64_hashFromCanonical Legacy06_XXH64_hashFromCanonical +#define XXH64_reset Legacy06_XXH64_reset +#define XXH64_update Legacy06_XXH64_update +#define XXH_versionNumber Legacy06_XXH_versionNumber +#define FSE_buildDTable Legacy06_FSE_buildDTable +#define FSE_buildDTable_raw Legacy06_FSE_buildDTable_raw +#define FSE_buildDTable_rle Legacy06_FSE_buildDTable_rle +#define FSE_createDTable Legacy06_FSE_createDTable +#define FSE_decompress Legacy06_FSE_decompress +#define FSE_decompress_usingDTable Legacy06_FSE_decompress_usingDTable +#define FSE_freeDTable Legacy06_FSE_freeDTable +#define FSE_getErrorName Legacy06_FSE_getErrorName +#define FSE_isError Legacy06_FSE_isError +#define FSE_readNCount Legacy06_FSE_readNCount +#define HUF_getErrorName Legacy06_HUF_getErrorName +#define HUF_isError Legacy06_HUF_isError diff --git a/contrib/libs/zstd06/ya.make b/contrib/libs/zstd06/ya.make index 33c77179b6..e35f69f443 100644 --- a/contrib/libs/zstd06/ya.make +++ b/contrib/libs/zstd06/ya.make @@ -18,40 +18,40 @@ OWNER( NO_UTIL() SRCS( - common/entropy_common.c - common/fse_decompress.c - common/xxhash.c - common/zstd_common.c - compress/fse_compress.c - compress/huf_compress.c - compress/zbuff_compress.c - compress/zstd_compress.c - decompress/huf_decompress.c - decompress/zbuff_decompress.c - decompress/zstd_decompress.c - dictBuilder/divsufsort.c - dictBuilder/zdict.c + common/entropy_common.c + common/fse_decompress.c + common/xxhash.c + common/zstd_common.c + compress/fse_compress.c + compress/huf_compress.c + compress/zbuff_compress.c + compress/zstd_compress.c + decompress/huf_decompress.c + decompress/zbuff_decompress.c + decompress/zstd_decompress.c + dictBuilder/divsufsort.c + dictBuilder/zdict.c legacy/zstd_v01.c legacy/zstd_v02.c legacy/zstd_v03.c legacy/zstd_v04.c - legacy/zstd_v05.c - legacy/zstd_v07.c - legacy/zstd_v08.c + legacy/zstd_v05.c + legacy/zstd_v07.c + legacy/zstd_v08.c ) -NO_COMPILER_WARNINGS() - +NO_COMPILER_WARNINGS() + CFLAGS( -DZSTD_LEGACY_SUPPORT=1 ) ADDINCL( contrib/libs/zstd06 - contrib/libs/zstd06/common - contrib/libs/zstd06/compress - contrib/libs/zstd06/decompress - contrib/libs/zstd06/dictBuilder + contrib/libs/zstd06/common + contrib/libs/zstd06/compress + contrib/libs/zstd06/decompress + contrib/libs/zstd06/dictBuilder contrib/libs/zstd06/legacy ) |