diff options
author | thegeorg <thegeorg@yandex-team.ru> | 2022-02-10 16:45:08 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:45:08 +0300 |
commit | 4e839db24a3bbc9f1c610c43d6faaaa99824dcca (patch) | |
tree | 506dac10f5df94fab310584ee51b24fc5a081c22 /contrib/libs/zstd/lib/compress/zstd_compress_internal.h | |
parent | 2d37894b1b037cf24231090eda8589bbb44fb6fc (diff) | |
download | ydb-4e839db24a3bbc9f1c610c43d6faaaa99824dcca.tar.gz |
Restoring authorship annotation for <thegeorg@yandex-team.ru>. Commit 1 of 2.
Diffstat (limited to 'contrib/libs/zstd/lib/compress/zstd_compress_internal.h')
-rw-r--r-- | contrib/libs/zstd/lib/compress/zstd_compress_internal.h | 568 |
1 files changed, 284 insertions, 284 deletions
diff --git a/contrib/libs/zstd/lib/compress/zstd_compress_internal.h b/contrib/libs/zstd/lib/compress/zstd_compress_internal.h index c406e794bd..e51904afa4 100644 --- a/contrib/libs/zstd/lib/compress/zstd_compress_internal.h +++ b/contrib/libs/zstd/lib/compress/zstd_compress_internal.h @@ -1,5 +1,5 @@ /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Yann Collet, Facebook, Inc. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -81,76 +81,76 @@ typedef struct { ZSTD_fseCTables_t fse; } ZSTD_entropyCTables_t; -/*********************************************** -* Entropy buffer statistics structs and funcs * -***********************************************/ -/** ZSTD_hufCTablesMetadata_t : - * Stores Literals Block Type for a super-block in hType, and - * huffman tree description in hufDesBuffer. - * hufDesSize refers to the size of huffman tree description in bytes. - * This metadata is populated in ZSTD_buildBlockEntropyStats_literals() */ -typedef struct { - symbolEncodingType_e hType; - BYTE hufDesBuffer[ZSTD_MAX_HUF_HEADER_SIZE]; - size_t hufDesSize; -} ZSTD_hufCTablesMetadata_t; - -/** ZSTD_fseCTablesMetadata_t : - * Stores symbol compression modes for a super-block in {ll, ol, ml}Type, and - * fse tables in fseTablesBuffer. - * fseTablesSize refers to the size of fse tables in bytes. - * This metadata is populated in ZSTD_buildBlockEntropyStats_sequences() */ -typedef struct { - symbolEncodingType_e llType; - symbolEncodingType_e ofType; - symbolEncodingType_e mlType; - BYTE fseTablesBuffer[ZSTD_MAX_FSE_HEADERS_SIZE]; - size_t fseTablesSize; - size_t lastCountSize; /* This is to account for bug in 1.3.4. More detail in ZSTD_entropyCompressSeqStore_internal() */ -} ZSTD_fseCTablesMetadata_t; - -typedef struct { - ZSTD_hufCTablesMetadata_t hufMetadata; - ZSTD_fseCTablesMetadata_t fseMetadata; -} ZSTD_entropyCTablesMetadata_t; - -/** ZSTD_buildBlockEntropyStats() : - * Builds entropy for the block. - * @return : 0 on success or error code */ -size_t ZSTD_buildBlockEntropyStats(seqStore_t* seqStorePtr, - const ZSTD_entropyCTables_t* prevEntropy, - ZSTD_entropyCTables_t* nextEntropy, - const ZSTD_CCtx_params* cctxParams, - ZSTD_entropyCTablesMetadata_t* entropyMetadata, - void* workspace, size_t wkspSize); - -/********************************* -* Compression internals structs * -*********************************/ - +/*********************************************** +* Entropy buffer statistics structs and funcs * +***********************************************/ +/** ZSTD_hufCTablesMetadata_t : + * Stores Literals Block Type for a super-block in hType, and + * huffman tree description in hufDesBuffer. + * hufDesSize refers to the size of huffman tree description in bytes. + * This metadata is populated in ZSTD_buildBlockEntropyStats_literals() */ typedef struct { + symbolEncodingType_e hType; + BYTE hufDesBuffer[ZSTD_MAX_HUF_HEADER_SIZE]; + size_t hufDesSize; +} ZSTD_hufCTablesMetadata_t; + +/** ZSTD_fseCTablesMetadata_t : + * Stores symbol compression modes for a super-block in {ll, ol, ml}Type, and + * fse tables in fseTablesBuffer. + * fseTablesSize refers to the size of fse tables in bytes. + * This metadata is populated in ZSTD_buildBlockEntropyStats_sequences() */ +typedef struct { + symbolEncodingType_e llType; + symbolEncodingType_e ofType; + symbolEncodingType_e mlType; + BYTE fseTablesBuffer[ZSTD_MAX_FSE_HEADERS_SIZE]; + size_t fseTablesSize; + size_t lastCountSize; /* This is to account for bug in 1.3.4. More detail in ZSTD_entropyCompressSeqStore_internal() */ +} ZSTD_fseCTablesMetadata_t; + +typedef struct { + ZSTD_hufCTablesMetadata_t hufMetadata; + ZSTD_fseCTablesMetadata_t fseMetadata; +} ZSTD_entropyCTablesMetadata_t; + +/** ZSTD_buildBlockEntropyStats() : + * Builds entropy for the block. + * @return : 0 on success or error code */ +size_t ZSTD_buildBlockEntropyStats(seqStore_t* seqStorePtr, + const ZSTD_entropyCTables_t* prevEntropy, + ZSTD_entropyCTables_t* nextEntropy, + const ZSTD_CCtx_params* cctxParams, + ZSTD_entropyCTablesMetadata_t* entropyMetadata, + void* workspace, size_t wkspSize); + +/********************************* +* Compression internals structs * +*********************************/ + +typedef struct { U32 off; /* Offset sumtype code for the match, using ZSTD_storeSeq() format */ - U32 len; /* Raw length of match */ + U32 len; /* Raw length of match */ } ZSTD_match_t; typedef struct { - U32 offset; /* Offset of sequence */ - U32 litLength; /* Length of literals prior to match */ - U32 matchLength; /* Raw length of match */ -} rawSeq; - -typedef struct { - rawSeq* seq; /* The start of the sequences */ - size_t pos; /* The index in seq where reading stopped. pos <= size. */ - size_t posInSequence; /* The position within the sequence at seq[pos] where reading - stopped. posInSequence <= seq[pos].litLength + seq[pos].matchLength */ - size_t size; /* The number of sequences. <= capacity. */ - size_t capacity; /* The capacity starting from `seq` pointer */ -} rawSeqStore_t; - -UNUSED_ATTR static const rawSeqStore_t kNullRawSeqStore = {NULL, 0, 0, 0, 0}; - -typedef struct { + U32 offset; /* Offset of sequence */ + U32 litLength; /* Length of literals prior to match */ + U32 matchLength; /* Raw length of match */ +} rawSeq; + +typedef struct { + rawSeq* seq; /* The start of the sequences */ + size_t pos; /* The index in seq where reading stopped. pos <= size. */ + size_t posInSequence; /* The position within the sequence at seq[pos] where reading + stopped. posInSequence <= seq[pos].litLength + seq[pos].matchLength */ + size_t size; /* The number of sequences. <= capacity. */ + size_t capacity; /* The capacity starting from `seq` pointer */ +} rawSeqStore_t; + +UNUSED_ATTR static const rawSeqStore_t kNullRawSeqStore = {NULL, 0, 0, 0, 0}; + +typedef struct { int price; U32 off; U32 mlen; @@ -188,23 +188,23 @@ typedef struct { } ZSTD_compressedBlockState_t; typedef struct { - BYTE const* nextSrc; /* next block here to continue on current prefix */ - BYTE const* base; /* All regular indexes relative to this position */ - BYTE const* dictBase; /* extDict indexes relative to this position */ - U32 dictLimit; /* below that point, need extDict */ - U32 lowLimit; /* below that point, no more valid data */ - U32 nbOverflowCorrections; /* Number of times overflow correction has run since - * ZSTD_window_init(). Useful for debugging coredumps - * and for ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY. - */ + BYTE const* nextSrc; /* next block here to continue on current prefix */ + BYTE const* base; /* All regular indexes relative to this position */ + BYTE const* dictBase; /* extDict indexes relative to this position */ + U32 dictLimit; /* below that point, need extDict */ + U32 lowLimit; /* below that point, no more valid data */ + U32 nbOverflowCorrections; /* Number of times overflow correction has run since + * ZSTD_window_init(). Useful for debugging coredumps + * and for ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY. + */ } ZSTD_window_t; #define ZSTD_WINDOW_START_INDEX 2 typedef struct ZSTD_matchState_t ZSTD_matchState_t; - -#define ZSTD_ROW_HASH_CACHE_SIZE 8 /* Size of prefetching hash cache for row-based matchfinder */ - + +#define ZSTD_ROW_HASH_CACHE_SIZE 8 /* Size of prefetching hash cache for row-based matchfinder */ + struct ZSTD_matchState_t { ZSTD_window_t window; /* State for window round buffer management */ U32 loadedDictEnd; /* index of end of dictionary, within context's referential. @@ -216,24 +216,24 @@ struct ZSTD_matchState_t { */ U32 nextToUpdate; /* index from which to continue table update */ U32 hashLog3; /* dispatch table for matches of len==3 : larger == faster, more memory */ - - U32 rowHashLog; /* For row-based matchfinder: Hashlog based on nb of rows in the hashTable.*/ - U16* tagTable; /* For row-based matchFinder: A row-based table containing the hashes and head index. */ - U32 hashCache[ZSTD_ROW_HASH_CACHE_SIZE]; /* For row-based matchFinder: a cache of hashes to improve speed */ - + + U32 rowHashLog; /* For row-based matchfinder: Hashlog based on nb of rows in the hashTable.*/ + U16* tagTable; /* For row-based matchFinder: A row-based table containing the hashes and head index. */ + U32 hashCache[ZSTD_ROW_HASH_CACHE_SIZE]; /* For row-based matchFinder: a cache of hashes to improve speed */ + U32* hashTable; U32* hashTable3; U32* chainTable; - - U32 forceNonContiguous; /* Non-zero if we should force non-contiguous load for the next window update. */ - - int dedicatedDictSearch; /* Indicates whether this matchState is using the - * dedicated dictionary search structure. - */ + + U32 forceNonContiguous; /* Non-zero if we should force non-contiguous load for the next window update. */ + + int dedicatedDictSearch; /* Indicates whether this matchState is using the + * dedicated dictionary search structure. + */ optState_t opt; /* optimal parser state */ const ZSTD_matchState_t* dictMatchState; ZSTD_compressionParameters cParams; - const rawSeqStore_t* ldmSeqStore; + const rawSeqStore_t* ldmSeqStore; }; typedef struct { @@ -248,21 +248,21 @@ typedef struct { } ldmEntry_t; typedef struct { - BYTE const* split; - U32 hash; - U32 checksum; - ldmEntry_t* bucket; -} ldmMatchCandidate_t; - -#define LDM_BATCH_SIZE 64 - -typedef struct { + BYTE const* split; + U32 hash; + U32 checksum; + ldmEntry_t* bucket; +} ldmMatchCandidate_t; + +#define LDM_BATCH_SIZE 64 + +typedef struct { ZSTD_window_t window; /* State for the window round buffer management */ ldmEntry_t* hashTable; U32 loadedDictEnd; BYTE* bucketOffsets; /* Next position in bucket to insert entry */ - size_t splitIndices[LDM_BATCH_SIZE]; - ldmMatchCandidate_t matchCandidates[LDM_BATCH_SIZE]; + size_t splitIndices[LDM_BATCH_SIZE]; + ldmMatchCandidate_t matchCandidates[LDM_BATCH_SIZE]; } ldmState_t; typedef struct { @@ -308,43 +308,43 @@ struct ZSTD_CCtx_params_s { /* Long distance matching parameters */ ldmParams_t ldmParams; - /* Dedicated dict search algorithm trigger */ - int enableDedicatedDictSearch; - - /* Input/output buffer modes */ - ZSTD_bufferMode_e inBufferMode; - ZSTD_bufferMode_e outBufferMode; - - /* Sequence compression API */ - ZSTD_sequenceFormat_e blockDelimiters; - int validateSequences; - - /* Block splitting */ + /* Dedicated dict search algorithm trigger */ + int enableDedicatedDictSearch; + + /* Input/output buffer modes */ + ZSTD_bufferMode_e inBufferMode; + ZSTD_bufferMode_e outBufferMode; + + /* Sequence compression API */ + ZSTD_sequenceFormat_e blockDelimiters; + int validateSequences; + + /* Block splitting */ ZSTD_paramSwitch_e useBlockSplitter; - - /* Param for deciding whether to use row-based matchfinder */ + + /* Param for deciding whether to use row-based matchfinder */ ZSTD_paramSwitch_e useRowMatchFinder; - - /* Always load a dictionary in ext-dict mode (not prefix mode)? */ - int deterministicRefPrefix; - + + /* Always load a dictionary in ext-dict mode (not prefix mode)? */ + int deterministicRefPrefix; + /* Internal use, for createCCtxParams() and freeCCtxParams() only */ ZSTD_customMem customMem; }; /* typedef'd to ZSTD_CCtx_params within "zstd.h" */ -#define COMPRESS_SEQUENCES_WORKSPACE_SIZE (sizeof(unsigned) * (MaxSeq + 2)) -#define ENTROPY_WORKSPACE_SIZE (HUF_WORKSPACE_SIZE + COMPRESS_SEQUENCES_WORKSPACE_SIZE) - -/** - * Indicates whether this compression proceeds directly from user-provided - * source buffer to user-provided destination buffer (ZSTDb_not_buffered), or - * whether the context needs to buffer the input/output (ZSTDb_buffered). - */ -typedef enum { - ZSTDb_not_buffered, - ZSTDb_buffered -} ZSTD_buffered_policy_e; - +#define COMPRESS_SEQUENCES_WORKSPACE_SIZE (sizeof(unsigned) * (MaxSeq + 2)) +#define ENTROPY_WORKSPACE_SIZE (HUF_WORKSPACE_SIZE + COMPRESS_SEQUENCES_WORKSPACE_SIZE) + +/** + * Indicates whether this compression proceeds directly from user-provided + * source buffer to user-provided destination buffer (ZSTDb_not_buffered), or + * whether the context needs to buffer the input/output (ZSTDb_buffered). + */ +typedef enum { + ZSTDb_not_buffered, + ZSTDb_buffered +} ZSTD_buffered_policy_e; + /** * Struct that contains all elements of block splitter that should be allocated * in a wksp. @@ -367,9 +367,9 @@ struct ZSTD_CCtx_s { int bmi2; /* == 1 if the CPU supports BMI2 and 0 otherwise. CPU support is determined dynamically once per context lifetime. */ ZSTD_CCtx_params requestedParams; ZSTD_CCtx_params appliedParams; - ZSTD_CCtx_params simpleApiParams; /* Param storage used by the simple API - not sticky. Must only be used in top-level simple API functions for storage. */ + ZSTD_CCtx_params simpleApiParams; /* Param storage used by the simple API - not sticky. Must only be used in top-level simple API functions for storage. */ U32 dictID; - size_t dictContentSize; + size_t dictContentSize; ZSTD_cwksp workspace; /* manages buffer for dynamic allocations */ size_t blockSize; @@ -378,7 +378,7 @@ struct ZSTD_CCtx_s { unsigned long long producedCSize; XXH64_state_t xxhState; ZSTD_customMem customMem; - ZSTD_threadPool* pool; + ZSTD_threadPool* pool; size_t staticSize; SeqCollector seqCollector; int isFirstBlock; @@ -390,11 +390,11 @@ struct ZSTD_CCtx_s { size_t maxNbLdmSequences; rawSeqStore_t externSeqStore; /* Mutable reference to external sequences */ ZSTD_blockState_t blockState; - U32* entropyWorkspace; /* entropy workspace of ENTROPY_WORKSPACE_SIZE bytes */ + U32* entropyWorkspace; /* entropy workspace of ENTROPY_WORKSPACE_SIZE bytes */ /* Whether we are streaming or not */ - ZSTD_buffered_policy_e bufferedPolicy; - + ZSTD_buffered_policy_e bufferedPolicy; + /* streaming */ char* inBuff; size_t inBuffSize; @@ -408,10 +408,10 @@ struct ZSTD_CCtx_s { ZSTD_cStreamStage streamStage; U32 frameEnded; - /* Stable in/out buffer verification */ - ZSTD_inBuffer expectedInBuffer; - size_t expectedOutBufferSize; - + /* Stable in/out buffer verification */ + ZSTD_inBuffer expectedInBuffer; + size_t expectedOutBufferSize; + /* Dictionary */ ZSTD_localDict localDict; const ZSTD_CDict* cdict; @@ -421,11 +421,11 @@ struct ZSTD_CCtx_s { #ifdef ZSTD_MULTITHREAD ZSTDMT_CCtx* mtctx; #endif - - /* Tracing */ -#if ZSTD_TRACE - ZSTD_TraceCtx traceCtx; -#endif + + /* Tracing */ +#if ZSTD_TRACE + ZSTD_TraceCtx traceCtx; +#endif /* Workspace for block splitter */ ZSTD_blockSplitCtx blockSplitCtx; @@ -433,32 +433,32 @@ struct ZSTD_CCtx_s { typedef enum { ZSTD_dtlm_fast, ZSTD_dtlm_full } ZSTD_dictTableLoadMethod_e; -typedef enum { - ZSTD_noDict = 0, - ZSTD_extDict = 1, - ZSTD_dictMatchState = 2, - ZSTD_dedicatedDictSearch = 3 -} ZSTD_dictMode_e; - -typedef enum { - ZSTD_cpm_noAttachDict = 0, /* Compression with ZSTD_noDict or ZSTD_extDict. - * In this mode we use both the srcSize and the dictSize - * when selecting and adjusting parameters. - */ - ZSTD_cpm_attachDict = 1, /* Compression with ZSTD_dictMatchState or ZSTD_dedicatedDictSearch. - * In this mode we only take the srcSize into account when selecting - * and adjusting parameters. - */ - ZSTD_cpm_createCDict = 2, /* Creating a CDict. - * In this mode we take both the source size and the dictionary size - * into account when selecting and adjusting the parameters. - */ - ZSTD_cpm_unknown = 3, /* ZSTD_getCParams, ZSTD_getParams, ZSTD_adjustParams. - * We don't know what these parameters are for. We default to the legacy - * behavior of taking both the source size and the dict size into account - * when selecting and adjusting parameters. - */ -} ZSTD_cParamMode_e; +typedef enum { + ZSTD_noDict = 0, + ZSTD_extDict = 1, + ZSTD_dictMatchState = 2, + ZSTD_dedicatedDictSearch = 3 +} ZSTD_dictMode_e; + +typedef enum { + ZSTD_cpm_noAttachDict = 0, /* Compression with ZSTD_noDict or ZSTD_extDict. + * In this mode we use both the srcSize and the dictSize + * when selecting and adjusting parameters. + */ + ZSTD_cpm_attachDict = 1, /* Compression with ZSTD_dictMatchState or ZSTD_dedicatedDictSearch. + * In this mode we only take the srcSize into account when selecting + * and adjusting parameters. + */ + ZSTD_cpm_createCDict = 2, /* Creating a CDict. + * In this mode we take both the source size and the dictionary size + * into account when selecting and adjusting the parameters. + */ + ZSTD_cpm_unknown = 3, /* ZSTD_getCParams, ZSTD_getParams, ZSTD_adjustParams. + * We don't know what these parameters are for. We default to the legacy + * behavior of taking both the source size and the dict size into account + * when selecting and adjusting parameters. + */ +} ZSTD_cParamMode_e; typedef size_t (*ZSTD_blockCompressor) ( ZSTD_matchState_t* bs, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], @@ -518,7 +518,7 @@ MEM_STATIC size_t ZSTD_noCompressBlock (void* dst, size_t dstCapacity, const voi RETURN_ERROR_IF(srcSize + ZSTD_blockHeaderSize > dstCapacity, dstSize_tooSmall, "dst buf too small for uncompressed block"); MEM_writeLE24(dst, cBlockHeader24); - ZSTD_memcpy((BYTE*)dst + ZSTD_blockHeaderSize, src, srcSize); + ZSTD_memcpy((BYTE*)dst + ZSTD_blockHeaderSize, src, srcSize); return ZSTD_blockHeaderSize + srcSize; } @@ -633,8 +633,8 @@ ZSTD_storeSeq(seqStore_t* seqStorePtr, /* literal Length */ if (litLength>0xFFFF) { - assert(seqStorePtr->longLengthType == ZSTD_llt_none); /* there can only be a single long length */ - seqStorePtr->longLengthType = ZSTD_llt_literalLength; + assert(seqStorePtr->longLengthType == ZSTD_llt_none); /* there can only be a single long length */ + seqStorePtr->longLengthType = ZSTD_llt_literalLength; seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); } seqStorePtr->sequences[0].litLength = (U16)litLength; @@ -702,9 +702,9 @@ static unsigned ZSTD_NbCommonBytes (size_t val) if (MEM_isLittleEndian()) { if (MEM_64bits()) { # if defined(_MSC_VER) && defined(_WIN64) -# if STATIC_BMI2 - return _tzcnt_u64(val) >> 3; -# else +# if STATIC_BMI2 + return _tzcnt_u64(val) >> 3; +# else if (val != 0) { unsigned long r; _BitScanForward64(&r, (U64)val); @@ -713,7 +713,7 @@ static unsigned ZSTD_NbCommonBytes (size_t val) /* Should not reach this code path */ __assume(0); } -# endif +# endif # elif defined(__GNUC__) && (__GNUC__ >= 4) return (__builtin_ctzll((U64)val) >> 3); # else @@ -750,9 +750,9 @@ static unsigned ZSTD_NbCommonBytes (size_t val) } else { /* Big Endian CPU */ if (MEM_64bits()) { # if defined(_MSC_VER) && defined(_WIN64) -# if STATIC_BMI2 - return _lzcnt_u64(val) >> 3; -# else +# if STATIC_BMI2 + return _lzcnt_u64(val) >> 3; +# else if (val != 0) { unsigned long r; _BitScanReverse64(&r, (U64)val); @@ -761,7 +761,7 @@ static unsigned ZSTD_NbCommonBytes (size_t val) /* Should not reach this code path */ __assume(0); } -# endif +# endif # elif defined(__GNUC__) && (__GNUC__ >= 4) return (__builtin_clzll(val) >> 3); # else @@ -862,8 +862,8 @@ static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL; static size_t ZSTD_hash8(U64 u, U32 h) { return (size_t)(((u) * prime8bytes) >> (64-h)) ; } static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(MEM_readLE64(p), h); } -MEM_STATIC FORCE_INLINE_ATTR -size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls) +MEM_STATIC FORCE_INLINE_ATTR +size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls) { switch(mls) { @@ -960,13 +960,13 @@ MEM_STATIC void ZSTD_window_clear(ZSTD_window_t* window) window->dictLimit = end; } -MEM_STATIC U32 ZSTD_window_isEmpty(ZSTD_window_t const window) -{ +MEM_STATIC U32 ZSTD_window_isEmpty(ZSTD_window_t const window) +{ return window.dictLimit == ZSTD_WINDOW_START_INDEX && window.lowLimit == ZSTD_WINDOW_START_INDEX && (window.nextSrc - window.base) == ZSTD_WINDOW_START_INDEX; -} - +} + /** * ZSTD_window_hasExtDict(): * Returns non-zero if the window has a non-empty extDict. @@ -986,76 +986,76 @@ MEM_STATIC ZSTD_dictMode_e ZSTD_matchState_dictMode(const ZSTD_matchState_t *ms) return ZSTD_window_hasExtDict(ms->window) ? ZSTD_extDict : ms->dictMatchState != NULL ? - (ms->dictMatchState->dedicatedDictSearch ? ZSTD_dedicatedDictSearch : ZSTD_dictMatchState) : + (ms->dictMatchState->dedicatedDictSearch ? ZSTD_dedicatedDictSearch : ZSTD_dictMatchState) : ZSTD_noDict; } -/* Defining this macro to non-zero tells zstd to run the overflow correction - * code much more frequently. This is very inefficient, and should only be - * used for tests and fuzzers. - */ -#ifndef ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY -# ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -# define ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY 1 -# else -# define ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY 0 -# endif -#endif - +/* Defining this macro to non-zero tells zstd to run the overflow correction + * code much more frequently. This is very inefficient, and should only be + * used for tests and fuzzers. + */ +#ifndef ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY +# ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION +# define ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY 1 +# else +# define ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY 0 +# endif +#endif + /** - * ZSTD_window_canOverflowCorrect(): - * Returns non-zero if the indices are large enough for overflow correction - * to work correctly without impacting compression ratio. - */ -MEM_STATIC U32 ZSTD_window_canOverflowCorrect(ZSTD_window_t const window, - U32 cycleLog, - U32 maxDist, - U32 loadedDictEnd, - void const* src) -{ - U32 const cycleSize = 1u << cycleLog; - U32 const curr = (U32)((BYTE const*)src - window.base); + * ZSTD_window_canOverflowCorrect(): + * Returns non-zero if the indices are large enough for overflow correction + * to work correctly without impacting compression ratio. + */ +MEM_STATIC U32 ZSTD_window_canOverflowCorrect(ZSTD_window_t const window, + U32 cycleLog, + U32 maxDist, + U32 loadedDictEnd, + void const* src) +{ + U32 const cycleSize = 1u << cycleLog; + U32 const curr = (U32)((BYTE const*)src - window.base); U32 const minIndexToOverflowCorrect = cycleSize + MAX(maxDist, cycleSize) + ZSTD_WINDOW_START_INDEX; - - /* Adjust the min index to backoff the overflow correction frequency, - * so we don't waste too much CPU in overflow correction. If this - * computation overflows we don't really care, we just need to make - * sure it is at least minIndexToOverflowCorrect. - */ - U32 const adjustment = window.nbOverflowCorrections + 1; - U32 const adjustedIndex = MAX(minIndexToOverflowCorrect * adjustment, - minIndexToOverflowCorrect); - U32 const indexLargeEnough = curr > adjustedIndex; - - /* Only overflow correct early if the dictionary is invalidated already, - * so we don't hurt compression ratio. - */ - U32 const dictionaryInvalidated = curr > maxDist + loadedDictEnd; - - return indexLargeEnough && dictionaryInvalidated; -} - -/** + + /* Adjust the min index to backoff the overflow correction frequency, + * so we don't waste too much CPU in overflow correction. If this + * computation overflows we don't really care, we just need to make + * sure it is at least minIndexToOverflowCorrect. + */ + U32 const adjustment = window.nbOverflowCorrections + 1; + U32 const adjustedIndex = MAX(minIndexToOverflowCorrect * adjustment, + minIndexToOverflowCorrect); + U32 const indexLargeEnough = curr > adjustedIndex; + + /* Only overflow correct early if the dictionary is invalidated already, + * so we don't hurt compression ratio. + */ + U32 const dictionaryInvalidated = curr > maxDist + loadedDictEnd; + + return indexLargeEnough && dictionaryInvalidated; +} + +/** * ZSTD_window_needOverflowCorrection(): * Returns non-zero if the indices are getting too large and need overflow * protection. */ MEM_STATIC U32 ZSTD_window_needOverflowCorrection(ZSTD_window_t const window, - U32 cycleLog, - U32 maxDist, - U32 loadedDictEnd, - void const* src, + U32 cycleLog, + U32 maxDist, + U32 loadedDictEnd, + void const* src, void const* srcEnd) { - U32 const curr = (U32)((BYTE const*)srcEnd - window.base); - if (ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY) { - if (ZSTD_window_canOverflowCorrect(window, cycleLog, maxDist, loadedDictEnd, src)) { - return 1; - } - } - return curr > ZSTD_CURRENT_MAX; + U32 const curr = (U32)((BYTE const*)srcEnd - window.base); + if (ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY) { + if (ZSTD_window_canOverflowCorrect(window, cycleLog, maxDist, loadedDictEnd, src)) { + return 1; + } + } + return curr > ZSTD_CURRENT_MAX; } /** @@ -1089,9 +1089,9 @@ MEM_STATIC U32 ZSTD_window_correctOverflow(ZSTD_window_t* window, U32 cycleLog, * 3. (cctx->lowLimit + 1<<windowLog) < 1<<32: * windowLog <= 31 ==> 3<<29 + 1<<windowLog < 7<<29 < 1<<32. */ - U32 const cycleSize = 1u << cycleLog; - U32 const cycleMask = cycleSize - 1; - U32 const curr = (U32)((BYTE const*)src - window->base); + U32 const cycleSize = 1u << cycleLog; + U32 const cycleMask = cycleSize - 1; + U32 const curr = (U32)((BYTE const*)src - window->base); U32 const currentCycle = curr & cycleMask; /* Ensure newCurrent - maxDist >= ZSTD_WINDOW_START_INDEX. */ U32 const currentCycleCorrection = currentCycle < ZSTD_WINDOW_START_INDEX @@ -1100,18 +1100,18 @@ MEM_STATIC U32 ZSTD_window_correctOverflow(ZSTD_window_t* window, U32 cycleLog, U32 const newCurrent = currentCycle + currentCycleCorrection + MAX(maxDist, cycleSize); - U32 const correction = curr - newCurrent; - /* maxDist must be a power of two so that: - * (newCurrent & cycleMask) == (curr & cycleMask) - * This is required to not corrupt the chains / binary tree. - */ - assert((maxDist & (maxDist - 1)) == 0); - assert((curr & cycleMask) == (newCurrent & cycleMask)); - assert(curr > newCurrent); - if (!ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY) { - /* Loose bound, should be around 1<<29 (see above) */ - assert(correction > 1<<28); - } + U32 const correction = curr - newCurrent; + /* maxDist must be a power of two so that: + * (newCurrent & cycleMask) == (curr & cycleMask) + * This is required to not corrupt the chains / binary tree. + */ + assert((maxDist & (maxDist - 1)) == 0); + assert((curr & cycleMask) == (newCurrent & cycleMask)); + assert(curr > newCurrent); + if (!ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY) { + /* Loose bound, should be around 1<<29 (see above) */ + assert(correction > 1<<28); + } window->base += correction; window->dictBase += correction; @@ -1133,8 +1133,8 @@ MEM_STATIC U32 ZSTD_window_correctOverflow(ZSTD_window_t* window, U32 cycleLog, assert(window->lowLimit <= newCurrent); assert(window->dictLimit <= newCurrent); - ++window->nbOverflowCorrections; - + ++window->nbOverflowCorrections; + DEBUGLOG(4, "Correction of 0x%x bytes to lowLimit=0x%x", correction, window->lowLimit); return correction; @@ -1238,14 +1238,14 @@ ZSTD_checkDictValidity(const ZSTD_window_t* window, } MEM_STATIC void ZSTD_window_init(ZSTD_window_t* window) { - ZSTD_memset(window, 0, sizeof(*window)); + ZSTD_memset(window, 0, sizeof(*window)); window->base = (BYTE const*)" "; window->dictBase = (BYTE const*)" "; ZSTD_STATIC_ASSERT(ZSTD_DUBT_UNSORTED_MARK < ZSTD_WINDOW_START_INDEX); /* Start above ZSTD_DUBT_UNSORTED_MARK */ window->dictLimit = ZSTD_WINDOW_START_INDEX; /* start from >0, so that 1st position is valid */ window->lowLimit = ZSTD_WINDOW_START_INDEX; /* it ensures first and later CCtx usages compress the same */ window->nextSrc = window->base + ZSTD_WINDOW_START_INDEX; /* see issue #1241 */ - window->nbOverflowCorrections = 0; + window->nbOverflowCorrections = 0; } /** @@ -1256,8 +1256,8 @@ MEM_STATIC void ZSTD_window_init(ZSTD_window_t* window) { * Returns non-zero if the segment is contiguous. */ MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window, - void const* src, size_t srcSize, - int forceNonContiguous) + void const* src, size_t srcSize, + int forceNonContiguous) { BYTE const* const ip = (BYTE const*)src; U32 contiguous = 1; @@ -1267,7 +1267,7 @@ MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window, assert(window->base != NULL); assert(window->dictBase != NULL); /* Check if blocks follow each other */ - if (src != window->nextSrc || forceNonContiguous) { + if (src != window->nextSrc || forceNonContiguous) { /* not contiguous */ size_t const distanceFromBase = (size_t)(window->nextSrc - window->base); DEBUGLOG(5, "Non contiguous blocks, new segment starts at %u", window->dictLimit); @@ -1295,16 +1295,16 @@ MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window, /** * Returns the lowest allowed match index. It may either be in the ext-dict or the prefix. */ -MEM_STATIC U32 ZSTD_getLowestMatchIndex(const ZSTD_matchState_t* ms, U32 curr, unsigned windowLog) +MEM_STATIC U32 ZSTD_getLowestMatchIndex(const ZSTD_matchState_t* ms, U32 curr, unsigned windowLog) { U32 const maxDistance = 1U << windowLog; U32 const lowestValid = ms->window.lowLimit; U32 const withinWindow = (curr - lowestValid > maxDistance) ? curr - maxDistance : lowestValid; U32 const isDictionary = (ms->loadedDictEnd != 0); - /* When using a dictionary the entire dictionary is valid if a single byte of the dictionary - * is within the window. We invalidate the dictionary (and set loadedDictEnd to 0) when it isn't - * valid for the entire block. So this check is sufficient to find the lowest valid match index. - */ + /* When using a dictionary the entire dictionary is valid if a single byte of the dictionary + * is within the window. We invalidate the dictionary (and set loadedDictEnd to 0) when it isn't + * valid for the entire block. So this check is sufficient to find the lowest valid match index. + */ U32 const matchLowest = isDictionary ? lowestValid : withinWindow; return matchLowest; } @@ -1312,15 +1312,15 @@ MEM_STATIC U32 ZSTD_getLowestMatchIndex(const ZSTD_matchState_t* ms, U32 curr, u /** * Returns the lowest allowed match index in the prefix. */ -MEM_STATIC U32 ZSTD_getLowestPrefixIndex(const ZSTD_matchState_t* ms, U32 curr, unsigned windowLog) +MEM_STATIC U32 ZSTD_getLowestPrefixIndex(const ZSTD_matchState_t* ms, U32 curr, unsigned windowLog) { U32 const maxDistance = 1U << windowLog; U32 const lowestValid = ms->window.dictLimit; - U32 const withinWindow = (curr - lowestValid > maxDistance) ? curr - maxDistance : lowestValid; + U32 const withinWindow = (curr - lowestValid > maxDistance) ? curr - maxDistance : lowestValid; U32 const isDictionary = (ms->loadedDictEnd != 0); - /* When computing the lowest prefix index we need to take the dictionary into account to handle - * the edge case where the dictionary and the source are contiguous in memory. - */ + /* When computing the lowest prefix index we need to take the dictionary into account to handle + * the edge case where the dictionary and the source are contiguous in memory. + */ U32 const matchLowest = isDictionary ? lowestValid : withinWindow; return matchLowest; } @@ -1389,7 +1389,7 @@ void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t* bs); * Note: srcSizeHint == 0 means 0! */ ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams( - const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode); + const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode); /*! ZSTD_initCStream_internal() : * Private use only. Init streaming operation. @@ -1450,9 +1450,9 @@ size_t ZSTD_referenceExternalSequences(ZSTD_CCtx* cctx, rawSeq* seq, size_t nbSe * condition for correct operation : hashLog > 1 */ U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat); -/** ZSTD_CCtx_trace() : - * Trace the end of a compression call. - */ -void ZSTD_CCtx_trace(ZSTD_CCtx* cctx, size_t extraCSize); - +/** ZSTD_CCtx_trace() : + * Trace the end of a compression call. + */ +void ZSTD_CCtx_trace(ZSTD_CCtx* cctx, size_t extraCSize); + #endif /* ZSTD_COMPRESS_H */ |