diff options
author | thegeorg <thegeorg@yandex-team.ru> | 2022-02-10 16:45:08 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:45:08 +0300 |
commit | 4e839db24a3bbc9f1c610c43d6faaaa99824dcca (patch) | |
tree | 506dac10f5df94fab310584ee51b24fc5a081c22 /contrib/libs/zstd/programs/fileio.c | |
parent | 2d37894b1b037cf24231090eda8589bbb44fb6fc (diff) | |
download | ydb-4e839db24a3bbc9f1c610c43d6faaaa99824dcca.tar.gz |
Restoring authorship annotation for <thegeorg@yandex-team.ru>. Commit 1 of 2.
Diffstat (limited to 'contrib/libs/zstd/programs/fileio.c')
-rw-r--r-- | contrib/libs/zstd/programs/fileio.c | 6110 |
1 files changed, 3055 insertions, 3055 deletions
diff --git a/contrib/libs/zstd/programs/fileio.c b/contrib/libs/zstd/programs/fileio.c index 0a0dfc42cb..eec90d13e1 100644 --- a/contrib/libs/zstd/programs/fileio.c +++ b/contrib/libs/zstd/programs/fileio.c @@ -1,744 +1,744 @@ -/* - * Copyright (c) Yann Collet, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. - */ - - -/* ************************************* -* Compiler Options -***************************************/ -#ifdef _MSC_VER /* Visual */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -# pragma warning(disable : 4204) /* non-constant aggregate initializer */ -#endif -#if defined(__MINGW32__) && !defined(_POSIX_SOURCE) -# define _POSIX_SOURCE 1 /* disable %llu warnings with MinGW on Windows */ -#endif - -/*-************************************* -* Includes -***************************************/ -#include "platform.h" /* Large Files support, SET_BINARY_MODE */ -#include "util.h" /* UTIL_getFileSize, UTIL_isRegularFile, UTIL_isSameFile */ -#include <stdio.h> /* fprintf, open, fdopen, fread, _fileno, stdin, stdout */ -#include <stdlib.h> /* malloc, free */ -#include <string.h> /* strcmp, strlen */ -#include <fcntl.h> /* O_WRONLY */ -#include <assert.h> -#include <errno.h> /* errno */ -#include <limits.h> /* INT_MAX */ -#include <signal.h> -#include "timefn.h" /* UTIL_getTime, UTIL_clockSpanMicro */ - -#if defined (_MSC_VER) -# include <sys/stat.h> -# include <io.h> -#endif - -#include "../lib/common/mem.h" /* U32, U64 */ -#include "fileio.h" - -#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */ -#include "../lib/zstd.h" -#include "../lib/zstd_errors.h" /* ZSTD_error_frameParameter_windowTooLarge */ - -#if defined(ZSTD_GZCOMPRESS) || defined(ZSTD_GZDECOMPRESS) -# error #include <zlib.h> -# if !defined(z_const) -# define z_const -# endif -#endif - -#if defined(ZSTD_LZMACOMPRESS) || defined(ZSTD_LZMADECOMPRESS) -# error #include <lzma.h> -#endif - -#define LZ4_MAGICNUMBER 0x184D2204 -#if defined(ZSTD_LZ4COMPRESS) || defined(ZSTD_LZ4DECOMPRESS) -# define LZ4F_ENABLE_OBSOLETE_ENUMS -# error #include <lz4frame.h> -# error #include <lz4.h> -#endif - - -/*-************************************* -* Constants -***************************************/ -#define ADAPT_WINDOWLOG_DEFAULT 23 /* 8 MB */ -#define DICTSIZE_MAX (32 MB) /* protection against large input (attack scenario) */ - -#define FNSPACE 30 - -/* Default file permissions 0666 (modulated by umask) */ -#if !defined(_WIN32) -/* These macros aren't defined on windows. */ -#define DEFAULT_FILE_PERMISSIONS (S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH) -#else -#define DEFAULT_FILE_PERMISSIONS (0666) -#endif - -/*-************************************* -* Macros -***************************************/ -#define KB *(1 <<10) -#define MB *(1 <<20) -#define GB *(1U<<30) -#undef MAX -#define MAX(a,b) ((a)>(b) ? (a) : (b)) - -struct FIO_display_prefs_s { - int displayLevel; /* 0 : no display; 1: errors; 2: + result + interaction + warnings; 3: + progression; 4: + information */ - FIO_progressSetting_e progressSetting; -}; - -static FIO_display_prefs_t g_display_prefs = {2, FIO_ps_auto}; - -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) -#define DISPLAYOUT(...) fprintf(stdout, __VA_ARGS__) -#define DISPLAYLEVEL(l, ...) { if (g_display_prefs.displayLevel>=l) { DISPLAY(__VA_ARGS__); } } - -static const U64 g_refreshRate = SEC_TO_MICRO / 6; -static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; - -#define READY_FOR_UPDATE() ((g_display_prefs.progressSetting != FIO_ps_never) && UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) -#define DELAY_NEXT_UPDATE() { g_displayClock = UTIL_getTime(); } -#define DISPLAYUPDATE(l, ...) { \ - if (g_display_prefs.displayLevel>=l && (g_display_prefs.progressSetting != FIO_ps_never)) { \ - if (READY_FOR_UPDATE() || (g_display_prefs.displayLevel>=4)) { \ - DELAY_NEXT_UPDATE(); \ - DISPLAY(__VA_ARGS__); \ - if (g_display_prefs.displayLevel>=4) fflush(stderr); \ - } } } - -#undef MIN /* in case it would be already defined */ -#define MIN(a,b) ((a) < (b) ? (a) : (b)) - - -#define EXM_THROW(error, ...) \ -{ \ - DISPLAYLEVEL(1, "zstd: "); \ - DISPLAYLEVEL(5, "Error defined at %s, line %i : \n", __FILE__, __LINE__); \ - DISPLAYLEVEL(1, "error %i : ", error); \ - DISPLAYLEVEL(1, __VA_ARGS__); \ - DISPLAYLEVEL(1, " \n"); \ - exit(error); \ -} - -#define CHECK_V(v, f) \ - v = f; \ - if (ZSTD_isError(v)) { \ - DISPLAYLEVEL(5, "%s \n", #f); \ - EXM_THROW(11, "%s", ZSTD_getErrorName(v)); \ - } -#define CHECK(f) { size_t err; CHECK_V(err, f); } - - -/*-************************************ -* Signal (Ctrl-C trapping) -**************************************/ -static const char* g_artefact = NULL; -static void INThandler(int sig) -{ - assert(sig==SIGINT); (void)sig; -#if !defined(_MSC_VER) - signal(sig, SIG_IGN); /* this invocation generates a buggy warning in Visual Studio */ -#endif - if (g_artefact) { - assert(UTIL_isRegularFile(g_artefact)); - remove(g_artefact); - } - DISPLAY("\n"); - exit(2); -} -static void addHandler(char const* dstFileName) -{ - if (UTIL_isRegularFile(dstFileName)) { - g_artefact = dstFileName; - signal(SIGINT, INThandler); - } else { - g_artefact = NULL; - } -} -/* Idempotent */ -static void clearHandler(void) -{ - if (g_artefact) signal(SIGINT, SIG_DFL); - g_artefact = NULL; -} - - -/*-********************************************************* -* Termination signal trapping (Print debug stack trace) -***********************************************************/ -#if defined(__has_feature) && !defined(BACKTRACE_ENABLE) /* Clang compiler */ -# if (__has_feature(address_sanitizer)) -# define BACKTRACE_ENABLE 0 -# endif /* __has_feature(address_sanitizer) */ -#elif defined(__SANITIZE_ADDRESS__) && !defined(BACKTRACE_ENABLE) /* GCC compiler */ -# define BACKTRACE_ENABLE 0 -#endif - -#if !defined(BACKTRACE_ENABLE) -/* automatic detector : backtrace enabled by default on linux+glibc and osx */ -# if (defined(__linux__) && (defined(__GLIBC__) && !defined(__UCLIBC__))) \ - || (defined(__APPLE__) && defined(__MACH__)) -# define BACKTRACE_ENABLE 1 -# else -# define BACKTRACE_ENABLE 0 -# endif -#endif - -/* note : after this point, BACKTRACE_ENABLE is necessarily defined */ - - -#if BACKTRACE_ENABLE - -#include <execinfo.h> /* backtrace, backtrace_symbols */ - -#define MAX_STACK_FRAMES 50 - -static void ABRThandler(int sig) { - const char* name; - void* addrlist[MAX_STACK_FRAMES]; - char** symbollist; - int addrlen, i; - - switch (sig) { - case SIGABRT: name = "SIGABRT"; break; - case SIGFPE: name = "SIGFPE"; break; - case SIGILL: name = "SIGILL"; break; - case SIGINT: name = "SIGINT"; break; - case SIGSEGV: name = "SIGSEGV"; break; - default: name = "UNKNOWN"; - } - - DISPLAY("Caught %s signal, printing stack:\n", name); - /* Retrieve current stack addresses. */ - addrlen = backtrace(addrlist, MAX_STACK_FRAMES); - if (addrlen == 0) { - DISPLAY("\n"); - return; - } - /* Create readable strings to each frame. */ - symbollist = backtrace_symbols(addrlist, addrlen); - /* Print the stack trace, excluding calls handling the signal. */ - for (i = ZSTD_START_SYMBOLLIST_FRAME; i < addrlen; i++) { - DISPLAY("%s\n", symbollist[i]); - } - free(symbollist); - /* Reset and raise the signal so default handler runs. */ - signal(sig, SIG_DFL); - raise(sig); -} -#endif - -void FIO_addAbortHandler() -{ -#if BACKTRACE_ENABLE - signal(SIGABRT, ABRThandler); - signal(SIGFPE, ABRThandler); - signal(SIGILL, ABRThandler); - signal(SIGSEGV, ABRThandler); - signal(SIGBUS, ABRThandler); -#endif -} - - -/*-************************************************************ -* Avoid fseek()'s 2GiB barrier with MSVC, macOS, *BSD, MinGW -***************************************************************/ -#if defined(_MSC_VER) && _MSC_VER >= 1400 -# define LONG_SEEK _fseeki64 -# define LONG_TELL _ftelli64 -#elif !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */ -# define LONG_SEEK fseeko -# define LONG_TELL ftello -#elif defined(__MINGW32__) && !defined(__STRICT_ANSI__) && !defined(__NO_MINGW_LFS) && defined(__MSVCRT__) -# define LONG_SEEK fseeko64 -# define LONG_TELL ftello64 -#elif defined(_WIN32) && !defined(__DJGPP__) -# include <windows.h> - static int LONG_SEEK(FILE* file, __int64 offset, int origin) { - LARGE_INTEGER off; - DWORD method; - off.QuadPart = offset; - if (origin == SEEK_END) - method = FILE_END; - else if (origin == SEEK_CUR) - method = FILE_CURRENT; - else - method = FILE_BEGIN; - - if (SetFilePointerEx((HANDLE) _get_osfhandle(_fileno(file)), off, NULL, method)) - return 0; - else - return -1; - } - static __int64 LONG_TELL(FILE* file) { - LARGE_INTEGER off, newOff; - off.QuadPart = 0; - newOff.QuadPart = 0; - SetFilePointerEx((HANDLE) _get_osfhandle(_fileno(file)), off, &newOff, FILE_CURRENT); - return newOff.QuadPart; - } -#else -# define LONG_SEEK fseek -# define LONG_TELL ftell -#endif - - -/*-************************************* -* Parameters: FIO_prefs_t -***************************************/ - -/* typedef'd to FIO_prefs_t within fileio.h */ -struct FIO_prefs_s { - - /* Algorithm preferences */ - FIO_compressionType_t compressionType; - U32 sparseFileSupport; /* 0: no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */ - int dictIDFlag; - int checksumFlag; - int blockSize; - int overlapLog; - U32 adaptiveMode; - U32 useRowMatchFinder; - int rsyncable; - int minAdaptLevel; - int maxAdaptLevel; - int ldmFlag; - int ldmHashLog; - int ldmMinMatch; - int ldmBucketSizeLog; - int ldmHashRateLog; - size_t streamSrcSize; - size_t targetCBlockSize; - int srcSizeHint; - int testMode; +/* + * Copyright (c) Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +/* ************************************* +* Compiler Options +***************************************/ +#ifdef _MSC_VER /* Visual */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4204) /* non-constant aggregate initializer */ +#endif +#if defined(__MINGW32__) && !defined(_POSIX_SOURCE) +# define _POSIX_SOURCE 1 /* disable %llu warnings with MinGW on Windows */ +#endif + +/*-************************************* +* Includes +***************************************/ +#include "platform.h" /* Large Files support, SET_BINARY_MODE */ +#include "util.h" /* UTIL_getFileSize, UTIL_isRegularFile, UTIL_isSameFile */ +#include <stdio.h> /* fprintf, open, fdopen, fread, _fileno, stdin, stdout */ +#include <stdlib.h> /* malloc, free */ +#include <string.h> /* strcmp, strlen */ +#include <fcntl.h> /* O_WRONLY */ +#include <assert.h> +#include <errno.h> /* errno */ +#include <limits.h> /* INT_MAX */ +#include <signal.h> +#include "timefn.h" /* UTIL_getTime, UTIL_clockSpanMicro */ + +#if defined (_MSC_VER) +# include <sys/stat.h> +# include <io.h> +#endif + +#include "../lib/common/mem.h" /* U32, U64 */ +#include "fileio.h" + +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */ +#include "../lib/zstd.h" +#include "../lib/zstd_errors.h" /* ZSTD_error_frameParameter_windowTooLarge */ + +#if defined(ZSTD_GZCOMPRESS) || defined(ZSTD_GZDECOMPRESS) +# error #include <zlib.h> +# if !defined(z_const) +# define z_const +# endif +#endif + +#if defined(ZSTD_LZMACOMPRESS) || defined(ZSTD_LZMADECOMPRESS) +# error #include <lzma.h> +#endif + +#define LZ4_MAGICNUMBER 0x184D2204 +#if defined(ZSTD_LZ4COMPRESS) || defined(ZSTD_LZ4DECOMPRESS) +# define LZ4F_ENABLE_OBSOLETE_ENUMS +# error #include <lz4frame.h> +# error #include <lz4.h> +#endif + + +/*-************************************* +* Constants +***************************************/ +#define ADAPT_WINDOWLOG_DEFAULT 23 /* 8 MB */ +#define DICTSIZE_MAX (32 MB) /* protection against large input (attack scenario) */ + +#define FNSPACE 30 + +/* Default file permissions 0666 (modulated by umask) */ +#if !defined(_WIN32) +/* These macros aren't defined on windows. */ +#define DEFAULT_FILE_PERMISSIONS (S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH) +#else +#define DEFAULT_FILE_PERMISSIONS (0666) +#endif + +/*-************************************* +* Macros +***************************************/ +#define KB *(1 <<10) +#define MB *(1 <<20) +#define GB *(1U<<30) +#undef MAX +#define MAX(a,b) ((a)>(b) ? (a) : (b)) + +struct FIO_display_prefs_s { + int displayLevel; /* 0 : no display; 1: errors; 2: + result + interaction + warnings; 3: + progression; 4: + information */ + FIO_progressSetting_e progressSetting; +}; + +static FIO_display_prefs_t g_display_prefs = {2, FIO_ps_auto}; + +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAYOUT(...) fprintf(stdout, __VA_ARGS__) +#define DISPLAYLEVEL(l, ...) { if (g_display_prefs.displayLevel>=l) { DISPLAY(__VA_ARGS__); } } + +static const U64 g_refreshRate = SEC_TO_MICRO / 6; +static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; + +#define READY_FOR_UPDATE() ((g_display_prefs.progressSetting != FIO_ps_never) && UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) +#define DELAY_NEXT_UPDATE() { g_displayClock = UTIL_getTime(); } +#define DISPLAYUPDATE(l, ...) { \ + if (g_display_prefs.displayLevel>=l && (g_display_prefs.progressSetting != FIO_ps_never)) { \ + if (READY_FOR_UPDATE() || (g_display_prefs.displayLevel>=4)) { \ + DELAY_NEXT_UPDATE(); \ + DISPLAY(__VA_ARGS__); \ + if (g_display_prefs.displayLevel>=4) fflush(stderr); \ + } } } + +#undef MIN /* in case it would be already defined */ +#define MIN(a,b) ((a) < (b) ? (a) : (b)) + + +#define EXM_THROW(error, ...) \ +{ \ + DISPLAYLEVEL(1, "zstd: "); \ + DISPLAYLEVEL(5, "Error defined at %s, line %i : \n", __FILE__, __LINE__); \ + DISPLAYLEVEL(1, "error %i : ", error); \ + DISPLAYLEVEL(1, __VA_ARGS__); \ + DISPLAYLEVEL(1, " \n"); \ + exit(error); \ +} + +#define CHECK_V(v, f) \ + v = f; \ + if (ZSTD_isError(v)) { \ + DISPLAYLEVEL(5, "%s \n", #f); \ + EXM_THROW(11, "%s", ZSTD_getErrorName(v)); \ + } +#define CHECK(f) { size_t err; CHECK_V(err, f); } + + +/*-************************************ +* Signal (Ctrl-C trapping) +**************************************/ +static const char* g_artefact = NULL; +static void INThandler(int sig) +{ + assert(sig==SIGINT); (void)sig; +#if !defined(_MSC_VER) + signal(sig, SIG_IGN); /* this invocation generates a buggy warning in Visual Studio */ +#endif + if (g_artefact) { + assert(UTIL_isRegularFile(g_artefact)); + remove(g_artefact); + } + DISPLAY("\n"); + exit(2); +} +static void addHandler(char const* dstFileName) +{ + if (UTIL_isRegularFile(dstFileName)) { + g_artefact = dstFileName; + signal(SIGINT, INThandler); + } else { + g_artefact = NULL; + } +} +/* Idempotent */ +static void clearHandler(void) +{ + if (g_artefact) signal(SIGINT, SIG_DFL); + g_artefact = NULL; +} + + +/*-********************************************************* +* Termination signal trapping (Print debug stack trace) +***********************************************************/ +#if defined(__has_feature) && !defined(BACKTRACE_ENABLE) /* Clang compiler */ +# if (__has_feature(address_sanitizer)) +# define BACKTRACE_ENABLE 0 +# endif /* __has_feature(address_sanitizer) */ +#elif defined(__SANITIZE_ADDRESS__) && !defined(BACKTRACE_ENABLE) /* GCC compiler */ +# define BACKTRACE_ENABLE 0 +#endif + +#if !defined(BACKTRACE_ENABLE) +/* automatic detector : backtrace enabled by default on linux+glibc and osx */ +# if (defined(__linux__) && (defined(__GLIBC__) && !defined(__UCLIBC__))) \ + || (defined(__APPLE__) && defined(__MACH__)) +# define BACKTRACE_ENABLE 1 +# else +# define BACKTRACE_ENABLE 0 +# endif +#endif + +/* note : after this point, BACKTRACE_ENABLE is necessarily defined */ + + +#if BACKTRACE_ENABLE + +#include <execinfo.h> /* backtrace, backtrace_symbols */ + +#define MAX_STACK_FRAMES 50 + +static void ABRThandler(int sig) { + const char* name; + void* addrlist[MAX_STACK_FRAMES]; + char** symbollist; + int addrlen, i; + + switch (sig) { + case SIGABRT: name = "SIGABRT"; break; + case SIGFPE: name = "SIGFPE"; break; + case SIGILL: name = "SIGILL"; break; + case SIGINT: name = "SIGINT"; break; + case SIGSEGV: name = "SIGSEGV"; break; + default: name = "UNKNOWN"; + } + + DISPLAY("Caught %s signal, printing stack:\n", name); + /* Retrieve current stack addresses. */ + addrlen = backtrace(addrlist, MAX_STACK_FRAMES); + if (addrlen == 0) { + DISPLAY("\n"); + return; + } + /* Create readable strings to each frame. */ + symbollist = backtrace_symbols(addrlist, addrlen); + /* Print the stack trace, excluding calls handling the signal. */ + for (i = ZSTD_START_SYMBOLLIST_FRAME; i < addrlen; i++) { + DISPLAY("%s\n", symbollist[i]); + } + free(symbollist); + /* Reset and raise the signal so default handler runs. */ + signal(sig, SIG_DFL); + raise(sig); +} +#endif + +void FIO_addAbortHandler() +{ +#if BACKTRACE_ENABLE + signal(SIGABRT, ABRThandler); + signal(SIGFPE, ABRThandler); + signal(SIGILL, ABRThandler); + signal(SIGSEGV, ABRThandler); + signal(SIGBUS, ABRThandler); +#endif +} + + +/*-************************************************************ +* Avoid fseek()'s 2GiB barrier with MSVC, macOS, *BSD, MinGW +***************************************************************/ +#if defined(_MSC_VER) && _MSC_VER >= 1400 +# define LONG_SEEK _fseeki64 +# define LONG_TELL _ftelli64 +#elif !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */ +# define LONG_SEEK fseeko +# define LONG_TELL ftello +#elif defined(__MINGW32__) && !defined(__STRICT_ANSI__) && !defined(__NO_MINGW_LFS) && defined(__MSVCRT__) +# define LONG_SEEK fseeko64 +# define LONG_TELL ftello64 +#elif defined(_WIN32) && !defined(__DJGPP__) +# include <windows.h> + static int LONG_SEEK(FILE* file, __int64 offset, int origin) { + LARGE_INTEGER off; + DWORD method; + off.QuadPart = offset; + if (origin == SEEK_END) + method = FILE_END; + else if (origin == SEEK_CUR) + method = FILE_CURRENT; + else + method = FILE_BEGIN; + + if (SetFilePointerEx((HANDLE) _get_osfhandle(_fileno(file)), off, NULL, method)) + return 0; + else + return -1; + } + static __int64 LONG_TELL(FILE* file) { + LARGE_INTEGER off, newOff; + off.QuadPart = 0; + newOff.QuadPart = 0; + SetFilePointerEx((HANDLE) _get_osfhandle(_fileno(file)), off, &newOff, FILE_CURRENT); + return newOff.QuadPart; + } +#else +# define LONG_SEEK fseek +# define LONG_TELL ftell +#endif + + +/*-************************************* +* Parameters: FIO_prefs_t +***************************************/ + +/* typedef'd to FIO_prefs_t within fileio.h */ +struct FIO_prefs_s { + + /* Algorithm preferences */ + FIO_compressionType_t compressionType; + U32 sparseFileSupport; /* 0: no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */ + int dictIDFlag; + int checksumFlag; + int blockSize; + int overlapLog; + U32 adaptiveMode; + U32 useRowMatchFinder; + int rsyncable; + int minAdaptLevel; + int maxAdaptLevel; + int ldmFlag; + int ldmHashLog; + int ldmMinMatch; + int ldmBucketSizeLog; + int ldmHashRateLog; + size_t streamSrcSize; + size_t targetCBlockSize; + int srcSizeHint; + int testMode; ZSTD_paramSwitch_e literalCompressionMode; - - /* IO preferences */ - U32 removeSrcFile; - U32 overwrite; - - /* Computation resources preferences */ - unsigned memLimit; - int nbWorkers; - - int excludeCompressedFiles; - int patchFromMode; - int contentSize; - int allowBlockDevices; -}; - -/*-************************************* -* Parameters: FIO_ctx_t -***************************************/ - -/* typedef'd to FIO_ctx_t within fileio.h */ -struct FIO_ctx_s { - - /* file i/o info */ - int nbFilesTotal; - int hasStdinInput; - int hasStdoutOutput; - - /* file i/o state */ - int currFileIdx; - int nbFilesProcessed; - size_t totalBytesInput; - size_t totalBytesOutput; -}; - - -/*-************************************* -* Parameters: Initialization -***************************************/ - -#define FIO_OVERLAP_LOG_NOTSET 9999 -#define FIO_LDM_PARAM_NOTSET 9999 - - -FIO_prefs_t* FIO_createPreferences(void) -{ - FIO_prefs_t* const ret = (FIO_prefs_t*)malloc(sizeof(FIO_prefs_t)); - if (!ret) EXM_THROW(21, "Allocation error : not enough memory"); - - ret->compressionType = FIO_zstdCompression; - ret->overwrite = 0; - ret->sparseFileSupport = ZSTD_SPARSE_DEFAULT; - ret->dictIDFlag = 1; - ret->checksumFlag = 1; - ret->removeSrcFile = 0; - ret->memLimit = 0; - ret->nbWorkers = 1; - ret->blockSize = 0; - ret->overlapLog = FIO_OVERLAP_LOG_NOTSET; - ret->adaptiveMode = 0; - ret->rsyncable = 0; - ret->minAdaptLevel = -50; /* initializing this value requires a constant, so ZSTD_minCLevel() doesn't work */ - ret->maxAdaptLevel = 22; /* initializing this value requires a constant, so ZSTD_maxCLevel() doesn't work */ - ret->ldmFlag = 0; - ret->ldmHashLog = 0; - ret->ldmMinMatch = 0; - ret->ldmBucketSizeLog = FIO_LDM_PARAM_NOTSET; - ret->ldmHashRateLog = FIO_LDM_PARAM_NOTSET; - ret->streamSrcSize = 0; - ret->targetCBlockSize = 0; - ret->srcSizeHint = 0; - ret->testMode = 0; + + /* IO preferences */ + U32 removeSrcFile; + U32 overwrite; + + /* Computation resources preferences */ + unsigned memLimit; + int nbWorkers; + + int excludeCompressedFiles; + int patchFromMode; + int contentSize; + int allowBlockDevices; +}; + +/*-************************************* +* Parameters: FIO_ctx_t +***************************************/ + +/* typedef'd to FIO_ctx_t within fileio.h */ +struct FIO_ctx_s { + + /* file i/o info */ + int nbFilesTotal; + int hasStdinInput; + int hasStdoutOutput; + + /* file i/o state */ + int currFileIdx; + int nbFilesProcessed; + size_t totalBytesInput; + size_t totalBytesOutput; +}; + + +/*-************************************* +* Parameters: Initialization +***************************************/ + +#define FIO_OVERLAP_LOG_NOTSET 9999 +#define FIO_LDM_PARAM_NOTSET 9999 + + +FIO_prefs_t* FIO_createPreferences(void) +{ + FIO_prefs_t* const ret = (FIO_prefs_t*)malloc(sizeof(FIO_prefs_t)); + if (!ret) EXM_THROW(21, "Allocation error : not enough memory"); + + ret->compressionType = FIO_zstdCompression; + ret->overwrite = 0; + ret->sparseFileSupport = ZSTD_SPARSE_DEFAULT; + ret->dictIDFlag = 1; + ret->checksumFlag = 1; + ret->removeSrcFile = 0; + ret->memLimit = 0; + ret->nbWorkers = 1; + ret->blockSize = 0; + ret->overlapLog = FIO_OVERLAP_LOG_NOTSET; + ret->adaptiveMode = 0; + ret->rsyncable = 0; + ret->minAdaptLevel = -50; /* initializing this value requires a constant, so ZSTD_minCLevel() doesn't work */ + ret->maxAdaptLevel = 22; /* initializing this value requires a constant, so ZSTD_maxCLevel() doesn't work */ + ret->ldmFlag = 0; + ret->ldmHashLog = 0; + ret->ldmMinMatch = 0; + ret->ldmBucketSizeLog = FIO_LDM_PARAM_NOTSET; + ret->ldmHashRateLog = FIO_LDM_PARAM_NOTSET; + ret->streamSrcSize = 0; + ret->targetCBlockSize = 0; + ret->srcSizeHint = 0; + ret->testMode = 0; ret->literalCompressionMode = ZSTD_ps_auto; - ret->excludeCompressedFiles = 0; - ret->allowBlockDevices = 0; - return ret; -} - -FIO_ctx_t* FIO_createContext(void) -{ - FIO_ctx_t* const ret = (FIO_ctx_t*)malloc(sizeof(FIO_ctx_t)); - if (!ret) EXM_THROW(21, "Allocation error : not enough memory"); - - ret->currFileIdx = 0; - ret->hasStdinInput = 0; - ret->hasStdoutOutput = 0; - ret->nbFilesTotal = 1; - ret->nbFilesProcessed = 0; - ret->totalBytesInput = 0; - ret->totalBytesOutput = 0; - return ret; -} - -void FIO_freePreferences(FIO_prefs_t* const prefs) -{ - free(prefs); -} - -void FIO_freeContext(FIO_ctx_t* const fCtx) -{ - free(fCtx); -} - - -/*-************************************* -* Parameters: Display Options -***************************************/ - -void FIO_setNotificationLevel(int level) { g_display_prefs.displayLevel=level; } - -void FIO_setProgressSetting(FIO_progressSetting_e setting) { g_display_prefs.progressSetting = setting; } - - -/*-************************************* -* Parameters: Setters -***************************************/ - -/* FIO_prefs_t functions */ - -void FIO_setCompressionType(FIO_prefs_t* const prefs, FIO_compressionType_t compressionType) { prefs->compressionType = compressionType; } - -void FIO_overwriteMode(FIO_prefs_t* const prefs) { prefs->overwrite = 1; } - -void FIO_setSparseWrite(FIO_prefs_t* const prefs, unsigned sparse) { prefs->sparseFileSupport = sparse; } - -void FIO_setDictIDFlag(FIO_prefs_t* const prefs, int dictIDFlag) { prefs->dictIDFlag = dictIDFlag; } - -void FIO_setChecksumFlag(FIO_prefs_t* const prefs, int checksumFlag) { prefs->checksumFlag = checksumFlag; } - -void FIO_setRemoveSrcFile(FIO_prefs_t* const prefs, unsigned flag) { prefs->removeSrcFile = (flag>0); } - -void FIO_setMemLimit(FIO_prefs_t* const prefs, unsigned memLimit) { prefs->memLimit = memLimit; } - -void FIO_setNbWorkers(FIO_prefs_t* const prefs, int nbWorkers) { -#ifndef ZSTD_MULTITHREAD - if (nbWorkers > 0) DISPLAYLEVEL(2, "Note : multi-threading is disabled \n"); -#endif - prefs->nbWorkers = nbWorkers; -} - -void FIO_setExcludeCompressedFile(FIO_prefs_t* const prefs, int excludeCompressedFiles) { prefs->excludeCompressedFiles = excludeCompressedFiles; } - -void FIO_setAllowBlockDevices(FIO_prefs_t* const prefs, int allowBlockDevices) { prefs->allowBlockDevices = allowBlockDevices; } - -void FIO_setBlockSize(FIO_prefs_t* const prefs, int blockSize) { - if (blockSize && prefs->nbWorkers==0) - DISPLAYLEVEL(2, "Setting block size is useless in single-thread mode \n"); - prefs->blockSize = blockSize; -} - -void FIO_setOverlapLog(FIO_prefs_t* const prefs, int overlapLog){ - if (overlapLog && prefs->nbWorkers==0) - DISPLAYLEVEL(2, "Setting overlapLog is useless in single-thread mode \n"); - prefs->overlapLog = overlapLog; -} - -void FIO_setAdaptiveMode(FIO_prefs_t* const prefs, unsigned adapt) { - if ((adapt>0) && (prefs->nbWorkers==0)) - EXM_THROW(1, "Adaptive mode is not compatible with single thread mode \n"); - prefs->adaptiveMode = adapt; -} - -void FIO_setUseRowMatchFinder(FIO_prefs_t* const prefs, int useRowMatchFinder) { - prefs->useRowMatchFinder = useRowMatchFinder; -} - -void FIO_setRsyncable(FIO_prefs_t* const prefs, int rsyncable) { - if ((rsyncable>0) && (prefs->nbWorkers==0)) - EXM_THROW(1, "Rsyncable mode is not compatible with single thread mode \n"); - prefs->rsyncable = rsyncable; -} - -void FIO_setStreamSrcSize(FIO_prefs_t* const prefs, size_t streamSrcSize) { - prefs->streamSrcSize = streamSrcSize; -} - -void FIO_setTargetCBlockSize(FIO_prefs_t* const prefs, size_t targetCBlockSize) { - prefs->targetCBlockSize = targetCBlockSize; -} - -void FIO_setSrcSizeHint(FIO_prefs_t* const prefs, size_t srcSizeHint) { - prefs->srcSizeHint = (int)MIN((size_t)INT_MAX, srcSizeHint); -} - -void FIO_setTestMode(FIO_prefs_t* const prefs, int testMode) { - prefs->testMode = (testMode!=0); -} - -void FIO_setLiteralCompressionMode( - FIO_prefs_t* const prefs, + ret->excludeCompressedFiles = 0; + ret->allowBlockDevices = 0; + return ret; +} + +FIO_ctx_t* FIO_createContext(void) +{ + FIO_ctx_t* const ret = (FIO_ctx_t*)malloc(sizeof(FIO_ctx_t)); + if (!ret) EXM_THROW(21, "Allocation error : not enough memory"); + + ret->currFileIdx = 0; + ret->hasStdinInput = 0; + ret->hasStdoutOutput = 0; + ret->nbFilesTotal = 1; + ret->nbFilesProcessed = 0; + ret->totalBytesInput = 0; + ret->totalBytesOutput = 0; + return ret; +} + +void FIO_freePreferences(FIO_prefs_t* const prefs) +{ + free(prefs); +} + +void FIO_freeContext(FIO_ctx_t* const fCtx) +{ + free(fCtx); +} + + +/*-************************************* +* Parameters: Display Options +***************************************/ + +void FIO_setNotificationLevel(int level) { g_display_prefs.displayLevel=level; } + +void FIO_setProgressSetting(FIO_progressSetting_e setting) { g_display_prefs.progressSetting = setting; } + + +/*-************************************* +* Parameters: Setters +***************************************/ + +/* FIO_prefs_t functions */ + +void FIO_setCompressionType(FIO_prefs_t* const prefs, FIO_compressionType_t compressionType) { prefs->compressionType = compressionType; } + +void FIO_overwriteMode(FIO_prefs_t* const prefs) { prefs->overwrite = 1; } + +void FIO_setSparseWrite(FIO_prefs_t* const prefs, unsigned sparse) { prefs->sparseFileSupport = sparse; } + +void FIO_setDictIDFlag(FIO_prefs_t* const prefs, int dictIDFlag) { prefs->dictIDFlag = dictIDFlag; } + +void FIO_setChecksumFlag(FIO_prefs_t* const prefs, int checksumFlag) { prefs->checksumFlag = checksumFlag; } + +void FIO_setRemoveSrcFile(FIO_prefs_t* const prefs, unsigned flag) { prefs->removeSrcFile = (flag>0); } + +void FIO_setMemLimit(FIO_prefs_t* const prefs, unsigned memLimit) { prefs->memLimit = memLimit; } + +void FIO_setNbWorkers(FIO_prefs_t* const prefs, int nbWorkers) { +#ifndef ZSTD_MULTITHREAD + if (nbWorkers > 0) DISPLAYLEVEL(2, "Note : multi-threading is disabled \n"); +#endif + prefs->nbWorkers = nbWorkers; +} + +void FIO_setExcludeCompressedFile(FIO_prefs_t* const prefs, int excludeCompressedFiles) { prefs->excludeCompressedFiles = excludeCompressedFiles; } + +void FIO_setAllowBlockDevices(FIO_prefs_t* const prefs, int allowBlockDevices) { prefs->allowBlockDevices = allowBlockDevices; } + +void FIO_setBlockSize(FIO_prefs_t* const prefs, int blockSize) { + if (blockSize && prefs->nbWorkers==0) + DISPLAYLEVEL(2, "Setting block size is useless in single-thread mode \n"); + prefs->blockSize = blockSize; +} + +void FIO_setOverlapLog(FIO_prefs_t* const prefs, int overlapLog){ + if (overlapLog && prefs->nbWorkers==0) + DISPLAYLEVEL(2, "Setting overlapLog is useless in single-thread mode \n"); + prefs->overlapLog = overlapLog; +} + +void FIO_setAdaptiveMode(FIO_prefs_t* const prefs, unsigned adapt) { + if ((adapt>0) && (prefs->nbWorkers==0)) + EXM_THROW(1, "Adaptive mode is not compatible with single thread mode \n"); + prefs->adaptiveMode = adapt; +} + +void FIO_setUseRowMatchFinder(FIO_prefs_t* const prefs, int useRowMatchFinder) { + prefs->useRowMatchFinder = useRowMatchFinder; +} + +void FIO_setRsyncable(FIO_prefs_t* const prefs, int rsyncable) { + if ((rsyncable>0) && (prefs->nbWorkers==0)) + EXM_THROW(1, "Rsyncable mode is not compatible with single thread mode \n"); + prefs->rsyncable = rsyncable; +} + +void FIO_setStreamSrcSize(FIO_prefs_t* const prefs, size_t streamSrcSize) { + prefs->streamSrcSize = streamSrcSize; +} + +void FIO_setTargetCBlockSize(FIO_prefs_t* const prefs, size_t targetCBlockSize) { + prefs->targetCBlockSize = targetCBlockSize; +} + +void FIO_setSrcSizeHint(FIO_prefs_t* const prefs, size_t srcSizeHint) { + prefs->srcSizeHint = (int)MIN((size_t)INT_MAX, srcSizeHint); +} + +void FIO_setTestMode(FIO_prefs_t* const prefs, int testMode) { + prefs->testMode = (testMode!=0); +} + +void FIO_setLiteralCompressionMode( + FIO_prefs_t* const prefs, ZSTD_paramSwitch_e mode) { - prefs->literalCompressionMode = mode; -} - -void FIO_setAdaptMin(FIO_prefs_t* const prefs, int minCLevel) -{ -#ifndef ZSTD_NOCOMPRESS - assert(minCLevel >= ZSTD_minCLevel()); -#endif - prefs->minAdaptLevel = minCLevel; -} - -void FIO_setAdaptMax(FIO_prefs_t* const prefs, int maxCLevel) -{ - prefs->maxAdaptLevel = maxCLevel; -} - -void FIO_setLdmFlag(FIO_prefs_t* const prefs, unsigned ldmFlag) { - prefs->ldmFlag = (ldmFlag>0); -} - -void FIO_setLdmHashLog(FIO_prefs_t* const prefs, int ldmHashLog) { - prefs->ldmHashLog = ldmHashLog; -} - -void FIO_setLdmMinMatch(FIO_prefs_t* const prefs, int ldmMinMatch) { - prefs->ldmMinMatch = ldmMinMatch; -} - -void FIO_setLdmBucketSizeLog(FIO_prefs_t* const prefs, int ldmBucketSizeLog) { - prefs->ldmBucketSizeLog = ldmBucketSizeLog; -} - - -void FIO_setLdmHashRateLog(FIO_prefs_t* const prefs, int ldmHashRateLog) { - prefs->ldmHashRateLog = ldmHashRateLog; -} - -void FIO_setPatchFromMode(FIO_prefs_t* const prefs, int value) -{ - prefs->patchFromMode = value != 0; -} - -void FIO_setContentSize(FIO_prefs_t* const prefs, int value) -{ - prefs->contentSize = value != 0; -} - -/* FIO_ctx_t functions */ - -void FIO_setHasStdoutOutput(FIO_ctx_t* const fCtx, int value) { - fCtx->hasStdoutOutput = value; -} - -void FIO_setNbFilesTotal(FIO_ctx_t* const fCtx, int value) -{ - fCtx->nbFilesTotal = value; -} - -void FIO_determineHasStdinInput(FIO_ctx_t* const fCtx, const FileNamesTable* const filenames) { - size_t i = 0; - for ( ; i < filenames->tableSize; ++i) { - if (!strcmp(stdinmark, filenames->fileNames[i])) { - fCtx->hasStdinInput = 1; - return; - } - } -} - -/*-************************************* -* Functions -***************************************/ -/** FIO_removeFile() : - * @result : Unlink `fileName`, even if it's read-only */ -static int FIO_removeFile(const char* path) -{ - stat_t statbuf; - if (!UTIL_stat(path, &statbuf)) { - DISPLAYLEVEL(2, "zstd: Failed to stat %s while trying to remove it\n", path); - return 0; - } - if (!UTIL_isRegularFileStat(&statbuf)) { - DISPLAYLEVEL(2, "zstd: Refusing to remove non-regular file %s\n", path); - return 0; - } -#if defined(_WIN32) || defined(WIN32) - /* windows doesn't allow remove read-only files, - * so try to make it writable first */ - if (!(statbuf.st_mode & _S_IWRITE)) { - UTIL_chmod(path, &statbuf, _S_IWRITE); - } -#endif - return remove(path); -} - -/** FIO_openSrcFile() : - * condition : `srcFileName` must be non-NULL. `prefs` may be NULL. - * @result : FILE* to `srcFileName`, or NULL if it fails */ -static FILE* FIO_openSrcFile(const FIO_prefs_t* const prefs, const char* srcFileName) -{ - stat_t statbuf; - int allowBlockDevices = prefs != NULL ? prefs->allowBlockDevices : 0; - assert(srcFileName != NULL); - if (!strcmp (srcFileName, stdinmark)) { - DISPLAYLEVEL(4,"Using stdin for input \n"); - SET_BINARY_MODE(stdin); - return stdin; - } - - if (!UTIL_stat(srcFileName, &statbuf)) { - DISPLAYLEVEL(1, "zstd: can't stat %s : %s -- ignored \n", - srcFileName, strerror(errno)); - return NULL; - } - - if (!UTIL_isRegularFileStat(&statbuf) - && !UTIL_isFIFOStat(&statbuf) - && !(allowBlockDevices && UTIL_isBlockDevStat(&statbuf)) - ) { - DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n", - srcFileName); - return NULL; - } - - { FILE* const f = fopen(srcFileName, "rb"); - if (f == NULL) - DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno)); - return f; - } -} - -/** FIO_openDstFile() : - * condition : `dstFileName` must be non-NULL. - * @result : FILE* to `dstFileName`, or NULL if it fails */ -static FILE* -FIO_openDstFile(FIO_ctx_t* fCtx, FIO_prefs_t* const prefs, - const char* srcFileName, const char* dstFileName, - const int mode) -{ - if (prefs->testMode) return NULL; /* do not open file in test mode */ - - assert(dstFileName != NULL); - if (!strcmp (dstFileName, stdoutmark)) { - DISPLAYLEVEL(4,"Using stdout for output \n"); - SET_BINARY_MODE(stdout); - if (prefs->sparseFileSupport == 1) { - prefs->sparseFileSupport = 0; - DISPLAYLEVEL(4, "Sparse File Support is automatically disabled on stdout ; try --sparse \n"); - } - return stdout; - } - - /* ensure dst is not the same as src */ - if (srcFileName != NULL && UTIL_isSameFile(srcFileName, dstFileName)) { - DISPLAYLEVEL(1, "zstd: Refusing to open an output file which will overwrite the input file \n"); - return NULL; - } - - if (prefs->sparseFileSupport == 1) { - prefs->sparseFileSupport = ZSTD_SPARSE_DEFAULT; - } - - if (UTIL_isRegularFile(dstFileName)) { - /* Check if destination file already exists */ -#if !defined(_WIN32) - /* this test does not work on Windows : - * `NUL` and `nul` are detected as regular files */ - if (!strcmp(dstFileName, nulmark)) { - EXM_THROW(40, "%s is unexpectedly categorized as a regular file", - dstFileName); - } -#endif - if (!prefs->overwrite) { - if (g_display_prefs.displayLevel <= 1) { - /* No interaction possible */ - DISPLAY("zstd: %s already exists; not overwritten \n", - dstFileName); - return NULL; - } - DISPLAY("zstd: %s already exists; ", dstFileName); - if (UTIL_requireUserConfirmation("overwrite (y/n) ? ", "Not overwritten \n", "yY", fCtx->hasStdinInput)) - return NULL; - } - /* need to unlink */ - FIO_removeFile(dstFileName); - } - - { -#if defined(_WIN32) - /* Windows requires opening the file as a "binary" file to avoid - * mangling. This macro doesn't exist on unix. */ - const int openflags = O_WRONLY|O_CREAT|O_TRUNC|O_BINARY; - const int fd = _open(dstFileName, openflags, mode); - FILE* f = NULL; - if (fd != -1) { - f = _fdopen(fd, "wb"); - } -#else - const int openflags = O_WRONLY|O_CREAT|O_TRUNC; - const int fd = open(dstFileName, openflags, mode); - FILE* f = NULL; - if (fd != -1) { - f = fdopen(fd, "wb"); - } -#endif - if (f == NULL) { - DISPLAYLEVEL(1, "zstd: %s: %s\n", dstFileName, strerror(errno)); - } - return f; - } -} - -/*! FIO_createDictBuffer() : - * creates a buffer, pointed by `*bufferPtr`, - * loads `filename` content into it, up to DICTSIZE_MAX bytes. - * @return : loaded size - * if fileName==NULL, returns 0 and a NULL pointer - */ -static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName, FIO_prefs_t* const prefs) -{ - FILE* fileHandle; - U64 fileSize; + prefs->literalCompressionMode = mode; +} + +void FIO_setAdaptMin(FIO_prefs_t* const prefs, int minCLevel) +{ +#ifndef ZSTD_NOCOMPRESS + assert(minCLevel >= ZSTD_minCLevel()); +#endif + prefs->minAdaptLevel = minCLevel; +} + +void FIO_setAdaptMax(FIO_prefs_t* const prefs, int maxCLevel) +{ + prefs->maxAdaptLevel = maxCLevel; +} + +void FIO_setLdmFlag(FIO_prefs_t* const prefs, unsigned ldmFlag) { + prefs->ldmFlag = (ldmFlag>0); +} + +void FIO_setLdmHashLog(FIO_prefs_t* const prefs, int ldmHashLog) { + prefs->ldmHashLog = ldmHashLog; +} + +void FIO_setLdmMinMatch(FIO_prefs_t* const prefs, int ldmMinMatch) { + prefs->ldmMinMatch = ldmMinMatch; +} + +void FIO_setLdmBucketSizeLog(FIO_prefs_t* const prefs, int ldmBucketSizeLog) { + prefs->ldmBucketSizeLog = ldmBucketSizeLog; +} + + +void FIO_setLdmHashRateLog(FIO_prefs_t* const prefs, int ldmHashRateLog) { + prefs->ldmHashRateLog = ldmHashRateLog; +} + +void FIO_setPatchFromMode(FIO_prefs_t* const prefs, int value) +{ + prefs->patchFromMode = value != 0; +} + +void FIO_setContentSize(FIO_prefs_t* const prefs, int value) +{ + prefs->contentSize = value != 0; +} + +/* FIO_ctx_t functions */ + +void FIO_setHasStdoutOutput(FIO_ctx_t* const fCtx, int value) { + fCtx->hasStdoutOutput = value; +} + +void FIO_setNbFilesTotal(FIO_ctx_t* const fCtx, int value) +{ + fCtx->nbFilesTotal = value; +} + +void FIO_determineHasStdinInput(FIO_ctx_t* const fCtx, const FileNamesTable* const filenames) { + size_t i = 0; + for ( ; i < filenames->tableSize; ++i) { + if (!strcmp(stdinmark, filenames->fileNames[i])) { + fCtx->hasStdinInput = 1; + return; + } + } +} + +/*-************************************* +* Functions +***************************************/ +/** FIO_removeFile() : + * @result : Unlink `fileName`, even if it's read-only */ +static int FIO_removeFile(const char* path) +{ + stat_t statbuf; + if (!UTIL_stat(path, &statbuf)) { + DISPLAYLEVEL(2, "zstd: Failed to stat %s while trying to remove it\n", path); + return 0; + } + if (!UTIL_isRegularFileStat(&statbuf)) { + DISPLAYLEVEL(2, "zstd: Refusing to remove non-regular file %s\n", path); + return 0; + } +#if defined(_WIN32) || defined(WIN32) + /* windows doesn't allow remove read-only files, + * so try to make it writable first */ + if (!(statbuf.st_mode & _S_IWRITE)) { + UTIL_chmod(path, &statbuf, _S_IWRITE); + } +#endif + return remove(path); +} + +/** FIO_openSrcFile() : + * condition : `srcFileName` must be non-NULL. `prefs` may be NULL. + * @result : FILE* to `srcFileName`, or NULL if it fails */ +static FILE* FIO_openSrcFile(const FIO_prefs_t* const prefs, const char* srcFileName) +{ + stat_t statbuf; + int allowBlockDevices = prefs != NULL ? prefs->allowBlockDevices : 0; + assert(srcFileName != NULL); + if (!strcmp (srcFileName, stdinmark)) { + DISPLAYLEVEL(4,"Using stdin for input \n"); + SET_BINARY_MODE(stdin); + return stdin; + } + + if (!UTIL_stat(srcFileName, &statbuf)) { + DISPLAYLEVEL(1, "zstd: can't stat %s : %s -- ignored \n", + srcFileName, strerror(errno)); + return NULL; + } + + if (!UTIL_isRegularFileStat(&statbuf) + && !UTIL_isFIFOStat(&statbuf) + && !(allowBlockDevices && UTIL_isBlockDevStat(&statbuf)) + ) { + DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n", + srcFileName); + return NULL; + } + + { FILE* const f = fopen(srcFileName, "rb"); + if (f == NULL) + DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno)); + return f; + } +} + +/** FIO_openDstFile() : + * condition : `dstFileName` must be non-NULL. + * @result : FILE* to `dstFileName`, or NULL if it fails */ +static FILE* +FIO_openDstFile(FIO_ctx_t* fCtx, FIO_prefs_t* const prefs, + const char* srcFileName, const char* dstFileName, + const int mode) +{ + if (prefs->testMode) return NULL; /* do not open file in test mode */ + + assert(dstFileName != NULL); + if (!strcmp (dstFileName, stdoutmark)) { + DISPLAYLEVEL(4,"Using stdout for output \n"); + SET_BINARY_MODE(stdout); + if (prefs->sparseFileSupport == 1) { + prefs->sparseFileSupport = 0; + DISPLAYLEVEL(4, "Sparse File Support is automatically disabled on stdout ; try --sparse \n"); + } + return stdout; + } + + /* ensure dst is not the same as src */ + if (srcFileName != NULL && UTIL_isSameFile(srcFileName, dstFileName)) { + DISPLAYLEVEL(1, "zstd: Refusing to open an output file which will overwrite the input file \n"); + return NULL; + } + + if (prefs->sparseFileSupport == 1) { + prefs->sparseFileSupport = ZSTD_SPARSE_DEFAULT; + } + + if (UTIL_isRegularFile(dstFileName)) { + /* Check if destination file already exists */ +#if !defined(_WIN32) + /* this test does not work on Windows : + * `NUL` and `nul` are detected as regular files */ + if (!strcmp(dstFileName, nulmark)) { + EXM_THROW(40, "%s is unexpectedly categorized as a regular file", + dstFileName); + } +#endif + if (!prefs->overwrite) { + if (g_display_prefs.displayLevel <= 1) { + /* No interaction possible */ + DISPLAY("zstd: %s already exists; not overwritten \n", + dstFileName); + return NULL; + } + DISPLAY("zstd: %s already exists; ", dstFileName); + if (UTIL_requireUserConfirmation("overwrite (y/n) ? ", "Not overwritten \n", "yY", fCtx->hasStdinInput)) + return NULL; + } + /* need to unlink */ + FIO_removeFile(dstFileName); + } + + { +#if defined(_WIN32) + /* Windows requires opening the file as a "binary" file to avoid + * mangling. This macro doesn't exist on unix. */ + const int openflags = O_WRONLY|O_CREAT|O_TRUNC|O_BINARY; + const int fd = _open(dstFileName, openflags, mode); + FILE* f = NULL; + if (fd != -1) { + f = _fdopen(fd, "wb"); + } +#else + const int openflags = O_WRONLY|O_CREAT|O_TRUNC; + const int fd = open(dstFileName, openflags, mode); + FILE* f = NULL; + if (fd != -1) { + f = fdopen(fd, "wb"); + } +#endif + if (f == NULL) { + DISPLAYLEVEL(1, "zstd: %s: %s\n", dstFileName, strerror(errno)); + } + return f; + } +} + +/*! FIO_createDictBuffer() : + * creates a buffer, pointed by `*bufferPtr`, + * loads `filename` content into it, up to DICTSIZE_MAX bytes. + * @return : loaded size + * if fileName==NULL, returns 0 and a NULL pointer + */ +static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName, FIO_prefs_t* const prefs) +{ + FILE* fileHandle; + U64 fileSize; stat_t statbuf; - - assert(bufferPtr != NULL); - *bufferPtr = NULL; - if (fileName == NULL) return 0; - - DISPLAYLEVEL(4,"Loading %s as dictionary \n", fileName); + + assert(bufferPtr != NULL); + *bufferPtr = NULL; + if (fileName == NULL) return 0; + + DISPLAYLEVEL(4,"Loading %s as dictionary \n", fileName); if (!UTIL_stat(fileName, &statbuf)) { EXM_THROW(31, "Stat failed on dictionary file %s: %s", fileName, strerror(errno)); @@ -748,627 +748,627 @@ static size_t FIO_createDictBuffer(void** bufferPtr, const char* fileName, FIO_p EXM_THROW(32, "Dictionary %s must be a regular file.", fileName); } - fileHandle = fopen(fileName, "rb"); - + fileHandle = fopen(fileName, "rb"); + if (fileHandle == NULL) { EXM_THROW(33, "Couldn't open dictionary %s: %s", fileName, strerror(errno)); } fileSize = UTIL_getFileSizeStat(&statbuf); - { - size_t const dictSizeMax = prefs->patchFromMode ? prefs->memLimit : DICTSIZE_MAX; - if (fileSize > dictSizeMax) { + { + size_t const dictSizeMax = prefs->patchFromMode ? prefs->memLimit : DICTSIZE_MAX; + if (fileSize > dictSizeMax) { EXM_THROW(34, "Dictionary file %s is too large (> %u bytes)", - fileName, (unsigned)dictSizeMax); /* avoid extreme cases */ - } - } - *bufferPtr = malloc((size_t)fileSize); - if (*bufferPtr==NULL) EXM_THROW(34, "%s", strerror(errno)); - { size_t const readSize = fread(*bufferPtr, 1, (size_t)fileSize, fileHandle); + fileName, (unsigned)dictSizeMax); /* avoid extreme cases */ + } + } + *bufferPtr = malloc((size_t)fileSize); + if (*bufferPtr==NULL) EXM_THROW(34, "%s", strerror(errno)); + { size_t const readSize = fread(*bufferPtr, 1, (size_t)fileSize, fileHandle); if (readSize != fileSize) { - EXM_THROW(35, "Error reading dictionary file %s : %s", - fileName, strerror(errno)); - } - } - fclose(fileHandle); - return (size_t)fileSize; -} - - - -/* FIO_checkFilenameCollisions() : - * Checks for and warns if there are any files that would have the same output path - */ -int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) { - const char **filenameTableSorted, *prevElem, *filename; - unsigned u; - - filenameTableSorted = (const char**) malloc(sizeof(char*) * nbFiles); - if (!filenameTableSorted) { - DISPLAY("Unable to malloc new str array, not checking for name collisions\n"); - return 1; - } - - for (u = 0; u < nbFiles; ++u) { - filename = strrchr(filenameTable[u], PATH_SEP); - if (filename == NULL) { - filenameTableSorted[u] = filenameTable[u]; - } else { - filenameTableSorted[u] = filename+1; + EXM_THROW(35, "Error reading dictionary file %s : %s", + fileName, strerror(errno)); } - } - - qsort((void*)filenameTableSorted, nbFiles, sizeof(char*), UTIL_compareStr); - prevElem = filenameTableSorted[0]; - for (u = 1; u < nbFiles; ++u) { - if (strcmp(prevElem, filenameTableSorted[u]) == 0) { - DISPLAY("WARNING: Two files have same filename: %s\n", prevElem); - } - prevElem = filenameTableSorted[u]; - } - - free((void*)filenameTableSorted); - return 0; -} - -static const char* -extractFilename(const char* path, char separator) -{ - const char* search = strrchr(path, separator); - if (search == NULL) return path; - return search+1; -} - -/* FIO_createFilename_fromOutDir() : - * Takes a source file name and specified output directory, and - * allocates memory for and returns a pointer to final path. - * This function never returns an error (it may abort() in case of pb) - */ -static char* -FIO_createFilename_fromOutDir(const char* path, const char* outDirName, const size_t suffixLen) -{ - const char* filenameStart; - char separator; - char* result; - -#if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */ - separator = '\\'; -#else - separator = '/'; -#endif - - filenameStart = extractFilename(path, separator); -#if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */ - filenameStart = extractFilename(filenameStart, '/'); /* sometimes, '/' separator is also used on Windows (mingw+msys2) */ -#endif - - result = (char*) calloc(1, strlen(outDirName) + 1 + strlen(filenameStart) + suffixLen + 1); - if (!result) { - EXM_THROW(30, "zstd: FIO_createFilename_fromOutDir: %s", strerror(errno)); - } - - memcpy(result, outDirName, strlen(outDirName)); - if (outDirName[strlen(outDirName)-1] == separator) { - memcpy(result + strlen(outDirName), filenameStart, strlen(filenameStart)); - } else { - memcpy(result + strlen(outDirName), &separator, 1); - memcpy(result + strlen(outDirName) + 1, filenameStart, strlen(filenameStart)); - } - - return result; -} - -/* FIO_highbit64() : - * gives position of highest bit. - * note : only works for v > 0 ! - */ -static unsigned FIO_highbit64(unsigned long long v) -{ - unsigned count = 0; - assert(v != 0); - v >>= 1; - while (v) { v >>= 1; count++; } - return count; -} - -static void FIO_adjustMemLimitForPatchFromMode(FIO_prefs_t* const prefs, - unsigned long long const dictSize, - unsigned long long const maxSrcFileSize) -{ - unsigned long long maxSize = MAX(prefs->memLimit, MAX(dictSize, maxSrcFileSize)); - unsigned const maxWindowSize = (1U << ZSTD_WINDOWLOG_MAX); - if (maxSize == UTIL_FILESIZE_UNKNOWN) - EXM_THROW(42, "Using --patch-from with stdin requires --stream-size"); - assert(maxSize != UTIL_FILESIZE_UNKNOWN); - if (maxSize > maxWindowSize) - EXM_THROW(42, "Can't handle files larger than %u GB\n", maxWindowSize/(1 GB)); - FIO_setMemLimit(prefs, (unsigned)maxSize); -} - -/* FIO_removeMultiFilesWarning() : - * Returns 1 if the console should abort, 0 if console should proceed. - * This function handles logic when processing multiple files with -o, displaying the appropriate warnings/prompts. - * - * If -f is specified, or there is just 1 file, zstd will always proceed as usual. - * If --rm is specified, there will be a prompt asking for user confirmation. - * If -f is specified with --rm, zstd will proceed as usual - * If -q is specified with --rm, zstd will abort pre-emptively - * If neither flag is specified, zstd will prompt the user for confirmation to proceed. - * If --rm is not specified, then zstd will print a warning to the user (which can be silenced with -q). - * However, if the output is stdout, we will always abort rather than displaying the warning prompt. - */ -static int FIO_removeMultiFilesWarning(FIO_ctx_t* const fCtx, const FIO_prefs_t* const prefs, const char* outFileName, int displayLevelCutoff) -{ - int error = 0; - if (fCtx->nbFilesTotal > 1 && !prefs->overwrite) { - if (g_display_prefs.displayLevel <= displayLevelCutoff) { - if (prefs->removeSrcFile) { + } + fclose(fileHandle); + return (size_t)fileSize; +} + + + +/* FIO_checkFilenameCollisions() : + * Checks for and warns if there are any files that would have the same output path + */ +int FIO_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles) { + const char **filenameTableSorted, *prevElem, *filename; + unsigned u; + + filenameTableSorted = (const char**) malloc(sizeof(char*) * nbFiles); + if (!filenameTableSorted) { + DISPLAY("Unable to malloc new str array, not checking for name collisions\n"); + return 1; + } + + for (u = 0; u < nbFiles; ++u) { + filename = strrchr(filenameTable[u], PATH_SEP); + if (filename == NULL) { + filenameTableSorted[u] = filenameTable[u]; + } else { + filenameTableSorted[u] = filename+1; + } + } + + qsort((void*)filenameTableSorted, nbFiles, sizeof(char*), UTIL_compareStr); + prevElem = filenameTableSorted[0]; + for (u = 1; u < nbFiles; ++u) { + if (strcmp(prevElem, filenameTableSorted[u]) == 0) { + DISPLAY("WARNING: Two files have same filename: %s\n", prevElem); + } + prevElem = filenameTableSorted[u]; + } + + free((void*)filenameTableSorted); + return 0; +} + +static const char* +extractFilename(const char* path, char separator) +{ + const char* search = strrchr(path, separator); + if (search == NULL) return path; + return search+1; +} + +/* FIO_createFilename_fromOutDir() : + * Takes a source file name and specified output directory, and + * allocates memory for and returns a pointer to final path. + * This function never returns an error (it may abort() in case of pb) + */ +static char* +FIO_createFilename_fromOutDir(const char* path, const char* outDirName, const size_t suffixLen) +{ + const char* filenameStart; + char separator; + char* result; + +#if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */ + separator = '\\'; +#else + separator = '/'; +#endif + + filenameStart = extractFilename(path, separator); +#if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) /* windows support */ + filenameStart = extractFilename(filenameStart, '/'); /* sometimes, '/' separator is also used on Windows (mingw+msys2) */ +#endif + + result = (char*) calloc(1, strlen(outDirName) + 1 + strlen(filenameStart) + suffixLen + 1); + if (!result) { + EXM_THROW(30, "zstd: FIO_createFilename_fromOutDir: %s", strerror(errno)); + } + + memcpy(result, outDirName, strlen(outDirName)); + if (outDirName[strlen(outDirName)-1] == separator) { + memcpy(result + strlen(outDirName), filenameStart, strlen(filenameStart)); + } else { + memcpy(result + strlen(outDirName), &separator, 1); + memcpy(result + strlen(outDirName) + 1, filenameStart, strlen(filenameStart)); + } + + return result; +} + +/* FIO_highbit64() : + * gives position of highest bit. + * note : only works for v > 0 ! + */ +static unsigned FIO_highbit64(unsigned long long v) +{ + unsigned count = 0; + assert(v != 0); + v >>= 1; + while (v) { v >>= 1; count++; } + return count; +} + +static void FIO_adjustMemLimitForPatchFromMode(FIO_prefs_t* const prefs, + unsigned long long const dictSize, + unsigned long long const maxSrcFileSize) +{ + unsigned long long maxSize = MAX(prefs->memLimit, MAX(dictSize, maxSrcFileSize)); + unsigned const maxWindowSize = (1U << ZSTD_WINDOWLOG_MAX); + if (maxSize == UTIL_FILESIZE_UNKNOWN) + EXM_THROW(42, "Using --patch-from with stdin requires --stream-size"); + assert(maxSize != UTIL_FILESIZE_UNKNOWN); + if (maxSize > maxWindowSize) + EXM_THROW(42, "Can't handle files larger than %u GB\n", maxWindowSize/(1 GB)); + FIO_setMemLimit(prefs, (unsigned)maxSize); +} + +/* FIO_removeMultiFilesWarning() : + * Returns 1 if the console should abort, 0 if console should proceed. + * This function handles logic when processing multiple files with -o, displaying the appropriate warnings/prompts. + * + * If -f is specified, or there is just 1 file, zstd will always proceed as usual. + * If --rm is specified, there will be a prompt asking for user confirmation. + * If -f is specified with --rm, zstd will proceed as usual + * If -q is specified with --rm, zstd will abort pre-emptively + * If neither flag is specified, zstd will prompt the user for confirmation to proceed. + * If --rm is not specified, then zstd will print a warning to the user (which can be silenced with -q). + * However, if the output is stdout, we will always abort rather than displaying the warning prompt. + */ +static int FIO_removeMultiFilesWarning(FIO_ctx_t* const fCtx, const FIO_prefs_t* const prefs, const char* outFileName, int displayLevelCutoff) +{ + int error = 0; + if (fCtx->nbFilesTotal > 1 && !prefs->overwrite) { + if (g_display_prefs.displayLevel <= displayLevelCutoff) { + if (prefs->removeSrcFile) { DISPLAYLEVEL(1, "zstd: Aborting... not deleting files and processing into dst: %s\n", outFileName); - error = 1; - } - } else { - if (!strcmp(outFileName, stdoutmark)) { + error = 1; + } + } else { + if (!strcmp(outFileName, stdoutmark)) { DISPLAYLEVEL(2, "zstd: WARNING: all input files will be processed and concatenated into stdout. \n"); - } else { + } else { DISPLAYLEVEL(2, "zstd: WARNING: all input files will be processed and concatenated into a single output file: %s \n", outFileName); - } + } DISPLAYLEVEL(2, "The concatenated output CANNOT regenerate the original directory tree. \n") - if (prefs->removeSrcFile) { - if (fCtx->hasStdoutOutput) { + if (prefs->removeSrcFile) { + if (fCtx->hasStdoutOutput) { DISPLAYLEVEL(1, "Aborting. Use -f if you really want to delete the files and output to stdout\n"); - error = 1; - } else { - error = g_display_prefs.displayLevel > displayLevelCutoff && UTIL_requireUserConfirmation("This is a destructive operation. Proceed? (y/n): ", "Aborting...", "yY", fCtx->hasStdinInput); - } - } - } - } - return error; -} - -#ifndef ZSTD_NOCOMPRESS - -/* ********************************************************************** - * Compression - ************************************************************************/ -typedef struct { - FILE* srcFile; - FILE* dstFile; - void* srcBuffer; - size_t srcBufferSize; - void* dstBuffer; - size_t dstBufferSize; - void* dictBuffer; - size_t dictBufferSize; - const char* dictFileName; - ZSTD_CStream* cctx; -} cRess_t; - -/** ZSTD_cycleLog() : - * condition for correct operation : hashLog > 1 */ -static U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat) -{ - U32 const btScale = ((U32)strat >= (U32)ZSTD_btlazy2); - assert(hashLog > 1); - return hashLog - btScale; -} - -static void FIO_adjustParamsForPatchFromMode(FIO_prefs_t* const prefs, - ZSTD_compressionParameters* comprParams, - unsigned long long const dictSize, - unsigned long long const maxSrcFileSize, - int cLevel) -{ - unsigned const fileWindowLog = FIO_highbit64(maxSrcFileSize) + 1; - ZSTD_compressionParameters const cParams = ZSTD_getCParams(cLevel, (size_t)maxSrcFileSize, (size_t)dictSize); - FIO_adjustMemLimitForPatchFromMode(prefs, dictSize, maxSrcFileSize); - if (fileWindowLog > ZSTD_WINDOWLOG_MAX) - DISPLAYLEVEL(1, "Max window log exceeded by file (compression ratio will suffer)\n"); - comprParams->windowLog = MAX(ZSTD_WINDOWLOG_MIN, MIN(ZSTD_WINDOWLOG_MAX, fileWindowLog)); - if (fileWindowLog > ZSTD_cycleLog(cParams.chainLog, cParams.strategy)) { - if (!prefs->ldmFlag) - DISPLAYLEVEL(1, "long mode automatically triggered\n"); - FIO_setLdmFlag(prefs, 1); - } - if (cParams.strategy >= ZSTD_btopt) { - DISPLAYLEVEL(1, "[Optimal parser notes] Consider the following to improve patch size at the cost of speed:\n"); - DISPLAYLEVEL(1, "- Use --single-thread mode in the zstd cli\n"); - DISPLAYLEVEL(1, "- Set a larger targetLength (eg. --zstd=targetLength=4096)\n"); - DISPLAYLEVEL(1, "- Set a larger chainLog (eg. --zstd=chainLog=%u)\n", ZSTD_CHAINLOG_MAX); + error = 1; + } else { + error = g_display_prefs.displayLevel > displayLevelCutoff && UTIL_requireUserConfirmation("This is a destructive operation. Proceed? (y/n): ", "Aborting...", "yY", fCtx->hasStdinInput); + } + } + } + } + return error; +} + +#ifndef ZSTD_NOCOMPRESS + +/* ********************************************************************** + * Compression + ************************************************************************/ +typedef struct { + FILE* srcFile; + FILE* dstFile; + void* srcBuffer; + size_t srcBufferSize; + void* dstBuffer; + size_t dstBufferSize; + void* dictBuffer; + size_t dictBufferSize; + const char* dictFileName; + ZSTD_CStream* cctx; +} cRess_t; + +/** ZSTD_cycleLog() : + * condition for correct operation : hashLog > 1 */ +static U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat) +{ + U32 const btScale = ((U32)strat >= (U32)ZSTD_btlazy2); + assert(hashLog > 1); + return hashLog - btScale; +} + +static void FIO_adjustParamsForPatchFromMode(FIO_prefs_t* const prefs, + ZSTD_compressionParameters* comprParams, + unsigned long long const dictSize, + unsigned long long const maxSrcFileSize, + int cLevel) +{ + unsigned const fileWindowLog = FIO_highbit64(maxSrcFileSize) + 1; + ZSTD_compressionParameters const cParams = ZSTD_getCParams(cLevel, (size_t)maxSrcFileSize, (size_t)dictSize); + FIO_adjustMemLimitForPatchFromMode(prefs, dictSize, maxSrcFileSize); + if (fileWindowLog > ZSTD_WINDOWLOG_MAX) + DISPLAYLEVEL(1, "Max window log exceeded by file (compression ratio will suffer)\n"); + comprParams->windowLog = MAX(ZSTD_WINDOWLOG_MIN, MIN(ZSTD_WINDOWLOG_MAX, fileWindowLog)); + if (fileWindowLog > ZSTD_cycleLog(cParams.chainLog, cParams.strategy)) { + if (!prefs->ldmFlag) + DISPLAYLEVEL(1, "long mode automatically triggered\n"); + FIO_setLdmFlag(prefs, 1); + } + if (cParams.strategy >= ZSTD_btopt) { + DISPLAYLEVEL(1, "[Optimal parser notes] Consider the following to improve patch size at the cost of speed:\n"); + DISPLAYLEVEL(1, "- Use --single-thread mode in the zstd cli\n"); + DISPLAYLEVEL(1, "- Set a larger targetLength (eg. --zstd=targetLength=4096)\n"); + DISPLAYLEVEL(1, "- Set a larger chainLog (eg. --zstd=chainLog=%u)\n", ZSTD_CHAINLOG_MAX); DISPLAYLEVEL(1, "Also consider playing around with searchLog and hashLog\n"); - } -} - -static cRess_t FIO_createCResources(FIO_prefs_t* const prefs, - const char* dictFileName, unsigned long long const maxSrcFileSize, - int cLevel, ZSTD_compressionParameters comprParams) { - cRess_t ress; - memset(&ress, 0, sizeof(ress)); - - DISPLAYLEVEL(6, "FIO_createCResources \n"); - ress.cctx = ZSTD_createCCtx(); - if (ress.cctx == NULL) - EXM_THROW(30, "allocation error (%s): can't create ZSTD_CCtx", - strerror(errno)); - ress.srcBufferSize = ZSTD_CStreamInSize(); - ress.srcBuffer = malloc(ress.srcBufferSize); - ress.dstBufferSize = ZSTD_CStreamOutSize(); - - /* need to update memLimit before calling createDictBuffer - * because of memLimit check inside it */ - if (prefs->patchFromMode) { - unsigned long long const ssSize = (unsigned long long)prefs->streamSrcSize; - FIO_adjustParamsForPatchFromMode(prefs, &comprParams, UTIL_getFileSize(dictFileName), ssSize > 0 ? ssSize : maxSrcFileSize, cLevel); - } - ress.dstBuffer = malloc(ress.dstBufferSize); - ress.dictBufferSize = FIO_createDictBuffer(&ress.dictBuffer, dictFileName, prefs); /* works with dictFileName==NULL */ - if (!ress.srcBuffer || !ress.dstBuffer) - EXM_THROW(31, "allocation error : not enough memory"); - - /* Advanced parameters, including dictionary */ - if (dictFileName && (ress.dictBuffer==NULL)) - EXM_THROW(32, "allocation error : can't create dictBuffer"); - ress.dictFileName = dictFileName; - - if (prefs->adaptiveMode && !prefs->ldmFlag && !comprParams.windowLog) - comprParams.windowLog = ADAPT_WINDOWLOG_DEFAULT; - - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_contentSizeFlag, prefs->contentSize) ); /* always enable content size when available (note: supposed to be default) */ - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_dictIDFlag, prefs->dictIDFlag) ); - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_checksumFlag, prefs->checksumFlag) ); - /* compression level */ - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, cLevel) ); - /* max compressed block size */ - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_targetCBlockSize, (int)prefs->targetCBlockSize) ); - /* source size hint */ - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_srcSizeHint, (int)prefs->srcSizeHint) ); - /* long distance matching */ - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_enableLongDistanceMatching, prefs->ldmFlag) ); - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmHashLog, prefs->ldmHashLog) ); - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmMinMatch, prefs->ldmMinMatch) ); - if (prefs->ldmBucketSizeLog != FIO_LDM_PARAM_NOTSET) { - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmBucketSizeLog, prefs->ldmBucketSizeLog) ); - } - if (prefs->ldmHashRateLog != FIO_LDM_PARAM_NOTSET) { - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmHashRateLog, prefs->ldmHashRateLog) ); - } - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_useRowMatchFinder, prefs->useRowMatchFinder)); - /* compression parameters */ - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_windowLog, (int)comprParams.windowLog) ); - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_chainLog, (int)comprParams.chainLog) ); - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_hashLog, (int)comprParams.hashLog) ); - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_searchLog, (int)comprParams.searchLog) ); - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_minMatch, (int)comprParams.minMatch) ); - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_targetLength, (int)comprParams.targetLength) ); - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_strategy, (int)comprParams.strategy) ); - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_literalCompressionMode, (int)prefs->literalCompressionMode) ); - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_enableDedicatedDictSearch, 1) ); - /* multi-threading */ -#ifdef ZSTD_MULTITHREAD - DISPLAYLEVEL(5,"set nb workers = %u \n", prefs->nbWorkers); - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_nbWorkers, prefs->nbWorkers) ); - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_jobSize, prefs->blockSize) ); - if (prefs->overlapLog != FIO_OVERLAP_LOG_NOTSET) { - DISPLAYLEVEL(3,"set overlapLog = %u \n", prefs->overlapLog); - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_overlapLog, prefs->overlapLog) ); - } - CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_rsyncable, prefs->rsyncable) ); -#endif - /* dictionary */ - if (prefs->patchFromMode) { - CHECK( ZSTD_CCtx_refPrefix(ress.cctx, ress.dictBuffer, ress.dictBufferSize) ); - } else { - CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, ress.dictBuffer, ress.dictBufferSize) ); - } - - return ress; -} - -static void FIO_freeCResources(const cRess_t* const ress) -{ - free(ress->srcBuffer); - free(ress->dstBuffer); - free(ress->dictBuffer); - ZSTD_freeCStream(ress->cctx); /* never fails */ -} - - -#ifdef ZSTD_GZCOMPRESS -static unsigned long long -FIO_compressGzFrame(const cRess_t* ress, /* buffers & handlers are used, but not changed */ - const char* srcFileName, U64 const srcFileSize, - int compressionLevel, U64* readsize) -{ - unsigned long long inFileSize = 0, outFileSize = 0; - z_stream strm; - - if (compressionLevel > Z_BEST_COMPRESSION) - compressionLevel = Z_BEST_COMPRESSION; - - strm.zalloc = Z_NULL; - strm.zfree = Z_NULL; - strm.opaque = Z_NULL; - - { int const ret = deflateInit2(&strm, compressionLevel, Z_DEFLATED, - 15 /* maxWindowLogSize */ + 16 /* gzip only */, - 8, Z_DEFAULT_STRATEGY); /* see http://www.zlib.net/manual.html */ - if (ret != Z_OK) { - EXM_THROW(71, "zstd: %s: deflateInit2 error %d \n", srcFileName, ret); - } } - - strm.next_in = 0; - strm.avail_in = 0; - strm.next_out = (Bytef*)ress->dstBuffer; - strm.avail_out = (uInt)ress->dstBufferSize; - - while (1) { - int ret; - if (strm.avail_in == 0) { - size_t const inSize = fread(ress->srcBuffer, 1, ress->srcBufferSize, ress->srcFile); - if (inSize == 0) break; - inFileSize += inSize; - strm.next_in = (z_const unsigned char*)ress->srcBuffer; - strm.avail_in = (uInt)inSize; - } - ret = deflate(&strm, Z_NO_FLUSH); - if (ret != Z_OK) - EXM_THROW(72, "zstd: %s: deflate error %d \n", srcFileName, ret); - { size_t const cSize = ress->dstBufferSize - strm.avail_out; - if (cSize) { - if (fwrite(ress->dstBuffer, 1, cSize, ress->dstFile) != cSize) - EXM_THROW(73, "Write error : cannot write to output file : %s ", strerror(errno)); - outFileSize += cSize; - strm.next_out = (Bytef*)ress->dstBuffer; - strm.avail_out = (uInt)ress->dstBufferSize; - } } - if (srcFileSize == UTIL_FILESIZE_UNKNOWN) { - DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", - (unsigned)(inFileSize>>20), - (double)outFileSize/inFileSize*100) - } else { - DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%% ", - (unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20), - (double)outFileSize/inFileSize*100); - } } - - while (1) { - int const ret = deflate(&strm, Z_FINISH); - { size_t const cSize = ress->dstBufferSize - strm.avail_out; - if (cSize) { - if (fwrite(ress->dstBuffer, 1, cSize, ress->dstFile) != cSize) - EXM_THROW(75, "Write error : %s ", strerror(errno)); - outFileSize += cSize; - strm.next_out = (Bytef*)ress->dstBuffer; - strm.avail_out = (uInt)ress->dstBufferSize; - } } - if (ret == Z_STREAM_END) break; - if (ret != Z_BUF_ERROR) - EXM_THROW(77, "zstd: %s: deflate error %d \n", srcFileName, ret); - } - - { int const ret = deflateEnd(&strm); - if (ret != Z_OK) { - EXM_THROW(79, "zstd: %s: deflateEnd error %d \n", srcFileName, ret); - } } - *readsize = inFileSize; - return outFileSize; -} -#endif - - -#ifdef ZSTD_LZMACOMPRESS -static unsigned long long -FIO_compressLzmaFrame(cRess_t* ress, - const char* srcFileName, U64 const srcFileSize, - int compressionLevel, U64* readsize, int plain_lzma) -{ - unsigned long long inFileSize = 0, outFileSize = 0; - lzma_stream strm = LZMA_STREAM_INIT; - lzma_action action = LZMA_RUN; - lzma_ret ret; - - if (compressionLevel < 0) compressionLevel = 0; - if (compressionLevel > 9) compressionLevel = 9; - - if (plain_lzma) { - lzma_options_lzma opt_lzma; - if (lzma_lzma_preset(&opt_lzma, compressionLevel)) - EXM_THROW(81, "zstd: %s: lzma_lzma_preset error", srcFileName); - ret = lzma_alone_encoder(&strm, &opt_lzma); /* LZMA */ - if (ret != LZMA_OK) - EXM_THROW(82, "zstd: %s: lzma_alone_encoder error %d", srcFileName, ret); - } else { - ret = lzma_easy_encoder(&strm, compressionLevel, LZMA_CHECK_CRC64); /* XZ */ - if (ret != LZMA_OK) - EXM_THROW(83, "zstd: %s: lzma_easy_encoder error %d", srcFileName, ret); - } - - strm.next_in = 0; - strm.avail_in = 0; - strm.next_out = (BYTE*)ress->dstBuffer; - strm.avail_out = ress->dstBufferSize; - - while (1) { - if (strm.avail_in == 0) { - size_t const inSize = fread(ress->srcBuffer, 1, ress->srcBufferSize, ress->srcFile); - if (inSize == 0) action = LZMA_FINISH; - inFileSize += inSize; - strm.next_in = (BYTE const*)ress->srcBuffer; - strm.avail_in = inSize; - } - - ret = lzma_code(&strm, action); - - if (ret != LZMA_OK && ret != LZMA_STREAM_END) - EXM_THROW(84, "zstd: %s: lzma_code encoding error %d", srcFileName, ret); - { size_t const compBytes = ress->dstBufferSize - strm.avail_out; - if (compBytes) { - if (fwrite(ress->dstBuffer, 1, compBytes, ress->dstFile) != compBytes) - EXM_THROW(85, "Write error : %s", strerror(errno)); - outFileSize += compBytes; - strm.next_out = (BYTE*)ress->dstBuffer; - strm.avail_out = ress->dstBufferSize; - } } - if (srcFileSize == UTIL_FILESIZE_UNKNOWN) - DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%", - (unsigned)(inFileSize>>20), - (double)outFileSize/inFileSize*100) - else - DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%", - (unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20), - (double)outFileSize/inFileSize*100); - if (ret == LZMA_STREAM_END) break; - } - - lzma_end(&strm); - *readsize = inFileSize; - - return outFileSize; -} -#endif - -#ifdef ZSTD_LZ4COMPRESS - -#if LZ4_VERSION_NUMBER <= 10600 -#define LZ4F_blockLinked blockLinked -#define LZ4F_max64KB max64KB -#endif - -static int FIO_LZ4_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id))); } - -static unsigned long long -FIO_compressLz4Frame(cRess_t* ress, - const char* srcFileName, U64 const srcFileSize, - int compressionLevel, int checksumFlag, - U64* readsize) -{ - const size_t blockSize = FIO_LZ4_GetBlockSize_FromBlockId(LZ4F_max64KB); - unsigned long long inFileSize = 0, outFileSize = 0; - - LZ4F_preferences_t prefs; - LZ4F_compressionContext_t ctx; - - LZ4F_errorCode_t const errorCode = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION); - if (LZ4F_isError(errorCode)) - EXM_THROW(31, "zstd: failed to create lz4 compression context"); - - memset(&prefs, 0, sizeof(prefs)); - - assert(blockSize <= ress->srcBufferSize); - - prefs.autoFlush = 1; - prefs.compressionLevel = compressionLevel; - prefs.frameInfo.blockMode = LZ4F_blockLinked; - prefs.frameInfo.blockSizeID = LZ4F_max64KB; - prefs.frameInfo.contentChecksumFlag = (contentChecksum_t)checksumFlag; -#if LZ4_VERSION_NUMBER >= 10600 - prefs.frameInfo.contentSize = (srcFileSize==UTIL_FILESIZE_UNKNOWN) ? 0 : srcFileSize; -#endif - assert(LZ4F_compressBound(blockSize, &prefs) <= ress->dstBufferSize); - - { - size_t readSize; - size_t headerSize = LZ4F_compressBegin(ctx, ress->dstBuffer, ress->dstBufferSize, &prefs); - if (LZ4F_isError(headerSize)) - EXM_THROW(33, "File header generation failed : %s", - LZ4F_getErrorName(headerSize)); - if (fwrite(ress->dstBuffer, 1, headerSize, ress->dstFile) != headerSize) - EXM_THROW(34, "Write error : %s (cannot write header)", strerror(errno)); - outFileSize += headerSize; - - /* Read first block */ - readSize = fread(ress->srcBuffer, (size_t)1, (size_t)blockSize, ress->srcFile); - inFileSize += readSize; - - /* Main Loop */ - while (readSize>0) { - size_t const outSize = LZ4F_compressUpdate(ctx, - ress->dstBuffer, ress->dstBufferSize, - ress->srcBuffer, readSize, NULL); - if (LZ4F_isError(outSize)) - EXM_THROW(35, "zstd: %s: lz4 compression failed : %s", - srcFileName, LZ4F_getErrorName(outSize)); - outFileSize += outSize; - if (srcFileSize == UTIL_FILESIZE_UNKNOWN) { - DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%", - (unsigned)(inFileSize>>20), - (double)outFileSize/inFileSize*100) - } else { - DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%", - (unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20), - (double)outFileSize/inFileSize*100); - } - - /* Write Block */ - { size_t const sizeCheck = fwrite(ress->dstBuffer, 1, outSize, ress->dstFile); - if (sizeCheck != outSize) - EXM_THROW(36, "Write error : %s", strerror(errno)); - } - - /* Read next block */ - readSize = fread(ress->srcBuffer, (size_t)1, (size_t)blockSize, ress->srcFile); - inFileSize += readSize; - } - if (ferror(ress->srcFile)) EXM_THROW(37, "Error reading %s ", srcFileName); - - /* End of Stream mark */ - headerSize = LZ4F_compressEnd(ctx, ress->dstBuffer, ress->dstBufferSize, NULL); - if (LZ4F_isError(headerSize)) - EXM_THROW(38, "zstd: %s: lz4 end of file generation failed : %s", - srcFileName, LZ4F_getErrorName(headerSize)); - - { size_t const sizeCheck = fwrite(ress->dstBuffer, 1, headerSize, ress->dstFile); - if (sizeCheck != headerSize) - EXM_THROW(39, "Write error : %s (cannot write end of stream)", - strerror(errno)); - } - outFileSize += headerSize; - } - - *readsize = inFileSize; - LZ4F_freeCompressionContext(ctx); - - return outFileSize; -} -#endif - - -static unsigned long long -FIO_compressZstdFrame(FIO_ctx_t* const fCtx, - FIO_prefs_t* const prefs, - const cRess_t* ressPtr, - const char* srcFileName, U64 fileSize, - int compressionLevel, U64* readsize) -{ - cRess_t const ress = *ressPtr; - FILE* const srcFile = ress.srcFile; - FILE* const dstFile = ress.dstFile; - U64 compressedfilesize = 0; - ZSTD_EndDirective directive = ZSTD_e_continue; + } +} + +static cRess_t FIO_createCResources(FIO_prefs_t* const prefs, + const char* dictFileName, unsigned long long const maxSrcFileSize, + int cLevel, ZSTD_compressionParameters comprParams) { + cRess_t ress; + memset(&ress, 0, sizeof(ress)); + + DISPLAYLEVEL(6, "FIO_createCResources \n"); + ress.cctx = ZSTD_createCCtx(); + if (ress.cctx == NULL) + EXM_THROW(30, "allocation error (%s): can't create ZSTD_CCtx", + strerror(errno)); + ress.srcBufferSize = ZSTD_CStreamInSize(); + ress.srcBuffer = malloc(ress.srcBufferSize); + ress.dstBufferSize = ZSTD_CStreamOutSize(); + + /* need to update memLimit before calling createDictBuffer + * because of memLimit check inside it */ + if (prefs->patchFromMode) { + unsigned long long const ssSize = (unsigned long long)prefs->streamSrcSize; + FIO_adjustParamsForPatchFromMode(prefs, &comprParams, UTIL_getFileSize(dictFileName), ssSize > 0 ? ssSize : maxSrcFileSize, cLevel); + } + ress.dstBuffer = malloc(ress.dstBufferSize); + ress.dictBufferSize = FIO_createDictBuffer(&ress.dictBuffer, dictFileName, prefs); /* works with dictFileName==NULL */ + if (!ress.srcBuffer || !ress.dstBuffer) + EXM_THROW(31, "allocation error : not enough memory"); + + /* Advanced parameters, including dictionary */ + if (dictFileName && (ress.dictBuffer==NULL)) + EXM_THROW(32, "allocation error : can't create dictBuffer"); + ress.dictFileName = dictFileName; + + if (prefs->adaptiveMode && !prefs->ldmFlag && !comprParams.windowLog) + comprParams.windowLog = ADAPT_WINDOWLOG_DEFAULT; + + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_contentSizeFlag, prefs->contentSize) ); /* always enable content size when available (note: supposed to be default) */ + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_dictIDFlag, prefs->dictIDFlag) ); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_checksumFlag, prefs->checksumFlag) ); + /* compression level */ + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, cLevel) ); + /* max compressed block size */ + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_targetCBlockSize, (int)prefs->targetCBlockSize) ); + /* source size hint */ + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_srcSizeHint, (int)prefs->srcSizeHint) ); + /* long distance matching */ + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_enableLongDistanceMatching, prefs->ldmFlag) ); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmHashLog, prefs->ldmHashLog) ); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmMinMatch, prefs->ldmMinMatch) ); + if (prefs->ldmBucketSizeLog != FIO_LDM_PARAM_NOTSET) { + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmBucketSizeLog, prefs->ldmBucketSizeLog) ); + } + if (prefs->ldmHashRateLog != FIO_LDM_PARAM_NOTSET) { + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmHashRateLog, prefs->ldmHashRateLog) ); + } + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_useRowMatchFinder, prefs->useRowMatchFinder)); + /* compression parameters */ + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_windowLog, (int)comprParams.windowLog) ); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_chainLog, (int)comprParams.chainLog) ); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_hashLog, (int)comprParams.hashLog) ); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_searchLog, (int)comprParams.searchLog) ); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_minMatch, (int)comprParams.minMatch) ); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_targetLength, (int)comprParams.targetLength) ); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_strategy, (int)comprParams.strategy) ); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_literalCompressionMode, (int)prefs->literalCompressionMode) ); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_enableDedicatedDictSearch, 1) ); + /* multi-threading */ +#ifdef ZSTD_MULTITHREAD + DISPLAYLEVEL(5,"set nb workers = %u \n", prefs->nbWorkers); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_nbWorkers, prefs->nbWorkers) ); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_jobSize, prefs->blockSize) ); + if (prefs->overlapLog != FIO_OVERLAP_LOG_NOTSET) { + DISPLAYLEVEL(3,"set overlapLog = %u \n", prefs->overlapLog); + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_overlapLog, prefs->overlapLog) ); + } + CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_rsyncable, prefs->rsyncable) ); +#endif + /* dictionary */ + if (prefs->patchFromMode) { + CHECK( ZSTD_CCtx_refPrefix(ress.cctx, ress.dictBuffer, ress.dictBufferSize) ); + } else { + CHECK( ZSTD_CCtx_loadDictionary(ress.cctx, ress.dictBuffer, ress.dictBufferSize) ); + } + + return ress; +} + +static void FIO_freeCResources(const cRess_t* const ress) +{ + free(ress->srcBuffer); + free(ress->dstBuffer); + free(ress->dictBuffer); + ZSTD_freeCStream(ress->cctx); /* never fails */ +} + + +#ifdef ZSTD_GZCOMPRESS +static unsigned long long +FIO_compressGzFrame(const cRess_t* ress, /* buffers & handlers are used, but not changed */ + const char* srcFileName, U64 const srcFileSize, + int compressionLevel, U64* readsize) +{ + unsigned long long inFileSize = 0, outFileSize = 0; + z_stream strm; + + if (compressionLevel > Z_BEST_COMPRESSION) + compressionLevel = Z_BEST_COMPRESSION; + + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + + { int const ret = deflateInit2(&strm, compressionLevel, Z_DEFLATED, + 15 /* maxWindowLogSize */ + 16 /* gzip only */, + 8, Z_DEFAULT_STRATEGY); /* see http://www.zlib.net/manual.html */ + if (ret != Z_OK) { + EXM_THROW(71, "zstd: %s: deflateInit2 error %d \n", srcFileName, ret); + } } + + strm.next_in = 0; + strm.avail_in = 0; + strm.next_out = (Bytef*)ress->dstBuffer; + strm.avail_out = (uInt)ress->dstBufferSize; + + while (1) { + int ret; + if (strm.avail_in == 0) { + size_t const inSize = fread(ress->srcBuffer, 1, ress->srcBufferSize, ress->srcFile); + if (inSize == 0) break; + inFileSize += inSize; + strm.next_in = (z_const unsigned char*)ress->srcBuffer; + strm.avail_in = (uInt)inSize; + } + ret = deflate(&strm, Z_NO_FLUSH); + if (ret != Z_OK) + EXM_THROW(72, "zstd: %s: deflate error %d \n", srcFileName, ret); + { size_t const cSize = ress->dstBufferSize - strm.avail_out; + if (cSize) { + if (fwrite(ress->dstBuffer, 1, cSize, ress->dstFile) != cSize) + EXM_THROW(73, "Write error : cannot write to output file : %s ", strerror(errno)); + outFileSize += cSize; + strm.next_out = (Bytef*)ress->dstBuffer; + strm.avail_out = (uInt)ress->dstBufferSize; + } } + if (srcFileSize == UTIL_FILESIZE_UNKNOWN) { + DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%% ", + (unsigned)(inFileSize>>20), + (double)outFileSize/inFileSize*100) + } else { + DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%% ", + (unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20), + (double)outFileSize/inFileSize*100); + } } + + while (1) { + int const ret = deflate(&strm, Z_FINISH); + { size_t const cSize = ress->dstBufferSize - strm.avail_out; + if (cSize) { + if (fwrite(ress->dstBuffer, 1, cSize, ress->dstFile) != cSize) + EXM_THROW(75, "Write error : %s ", strerror(errno)); + outFileSize += cSize; + strm.next_out = (Bytef*)ress->dstBuffer; + strm.avail_out = (uInt)ress->dstBufferSize; + } } + if (ret == Z_STREAM_END) break; + if (ret != Z_BUF_ERROR) + EXM_THROW(77, "zstd: %s: deflate error %d \n", srcFileName, ret); + } + + { int const ret = deflateEnd(&strm); + if (ret != Z_OK) { + EXM_THROW(79, "zstd: %s: deflateEnd error %d \n", srcFileName, ret); + } } + *readsize = inFileSize; + return outFileSize; +} +#endif + + +#ifdef ZSTD_LZMACOMPRESS +static unsigned long long +FIO_compressLzmaFrame(cRess_t* ress, + const char* srcFileName, U64 const srcFileSize, + int compressionLevel, U64* readsize, int plain_lzma) +{ + unsigned long long inFileSize = 0, outFileSize = 0; + lzma_stream strm = LZMA_STREAM_INIT; + lzma_action action = LZMA_RUN; + lzma_ret ret; + + if (compressionLevel < 0) compressionLevel = 0; + if (compressionLevel > 9) compressionLevel = 9; + + if (plain_lzma) { + lzma_options_lzma opt_lzma; + if (lzma_lzma_preset(&opt_lzma, compressionLevel)) + EXM_THROW(81, "zstd: %s: lzma_lzma_preset error", srcFileName); + ret = lzma_alone_encoder(&strm, &opt_lzma); /* LZMA */ + if (ret != LZMA_OK) + EXM_THROW(82, "zstd: %s: lzma_alone_encoder error %d", srcFileName, ret); + } else { + ret = lzma_easy_encoder(&strm, compressionLevel, LZMA_CHECK_CRC64); /* XZ */ + if (ret != LZMA_OK) + EXM_THROW(83, "zstd: %s: lzma_easy_encoder error %d", srcFileName, ret); + } + + strm.next_in = 0; + strm.avail_in = 0; + strm.next_out = (BYTE*)ress->dstBuffer; + strm.avail_out = ress->dstBufferSize; + + while (1) { + if (strm.avail_in == 0) { + size_t const inSize = fread(ress->srcBuffer, 1, ress->srcBufferSize, ress->srcFile); + if (inSize == 0) action = LZMA_FINISH; + inFileSize += inSize; + strm.next_in = (BYTE const*)ress->srcBuffer; + strm.avail_in = inSize; + } + + ret = lzma_code(&strm, action); + + if (ret != LZMA_OK && ret != LZMA_STREAM_END) + EXM_THROW(84, "zstd: %s: lzma_code encoding error %d", srcFileName, ret); + { size_t const compBytes = ress->dstBufferSize - strm.avail_out; + if (compBytes) { + if (fwrite(ress->dstBuffer, 1, compBytes, ress->dstFile) != compBytes) + EXM_THROW(85, "Write error : %s", strerror(errno)); + outFileSize += compBytes; + strm.next_out = (BYTE*)ress->dstBuffer; + strm.avail_out = ress->dstBufferSize; + } } + if (srcFileSize == UTIL_FILESIZE_UNKNOWN) + DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%", + (unsigned)(inFileSize>>20), + (double)outFileSize/inFileSize*100) + else + DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%", + (unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20), + (double)outFileSize/inFileSize*100); + if (ret == LZMA_STREAM_END) break; + } + + lzma_end(&strm); + *readsize = inFileSize; + + return outFileSize; +} +#endif + +#ifdef ZSTD_LZ4COMPRESS + +#if LZ4_VERSION_NUMBER <= 10600 +#define LZ4F_blockLinked blockLinked +#define LZ4F_max64KB max64KB +#endif + +static int FIO_LZ4_GetBlockSize_FromBlockId (int id) { return (1 << (8 + (2 * id))); } + +static unsigned long long +FIO_compressLz4Frame(cRess_t* ress, + const char* srcFileName, U64 const srcFileSize, + int compressionLevel, int checksumFlag, + U64* readsize) +{ + const size_t blockSize = FIO_LZ4_GetBlockSize_FromBlockId(LZ4F_max64KB); + unsigned long long inFileSize = 0, outFileSize = 0; + + LZ4F_preferences_t prefs; + LZ4F_compressionContext_t ctx; + + LZ4F_errorCode_t const errorCode = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION); + if (LZ4F_isError(errorCode)) + EXM_THROW(31, "zstd: failed to create lz4 compression context"); + + memset(&prefs, 0, sizeof(prefs)); + + assert(blockSize <= ress->srcBufferSize); + + prefs.autoFlush = 1; + prefs.compressionLevel = compressionLevel; + prefs.frameInfo.blockMode = LZ4F_blockLinked; + prefs.frameInfo.blockSizeID = LZ4F_max64KB; + prefs.frameInfo.contentChecksumFlag = (contentChecksum_t)checksumFlag; +#if LZ4_VERSION_NUMBER >= 10600 + prefs.frameInfo.contentSize = (srcFileSize==UTIL_FILESIZE_UNKNOWN) ? 0 : srcFileSize; +#endif + assert(LZ4F_compressBound(blockSize, &prefs) <= ress->dstBufferSize); + + { + size_t readSize; + size_t headerSize = LZ4F_compressBegin(ctx, ress->dstBuffer, ress->dstBufferSize, &prefs); + if (LZ4F_isError(headerSize)) + EXM_THROW(33, "File header generation failed : %s", + LZ4F_getErrorName(headerSize)); + if (fwrite(ress->dstBuffer, 1, headerSize, ress->dstFile) != headerSize) + EXM_THROW(34, "Write error : %s (cannot write header)", strerror(errno)); + outFileSize += headerSize; + + /* Read first block */ + readSize = fread(ress->srcBuffer, (size_t)1, (size_t)blockSize, ress->srcFile); + inFileSize += readSize; + + /* Main Loop */ + while (readSize>0) { + size_t const outSize = LZ4F_compressUpdate(ctx, + ress->dstBuffer, ress->dstBufferSize, + ress->srcBuffer, readSize, NULL); + if (LZ4F_isError(outSize)) + EXM_THROW(35, "zstd: %s: lz4 compression failed : %s", + srcFileName, LZ4F_getErrorName(outSize)); + outFileSize += outSize; + if (srcFileSize == UTIL_FILESIZE_UNKNOWN) { + DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%", + (unsigned)(inFileSize>>20), + (double)outFileSize/inFileSize*100) + } else { + DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%", + (unsigned)(inFileSize>>20), (unsigned)(srcFileSize>>20), + (double)outFileSize/inFileSize*100); + } + + /* Write Block */ + { size_t const sizeCheck = fwrite(ress->dstBuffer, 1, outSize, ress->dstFile); + if (sizeCheck != outSize) + EXM_THROW(36, "Write error : %s", strerror(errno)); + } + + /* Read next block */ + readSize = fread(ress->srcBuffer, (size_t)1, (size_t)blockSize, ress->srcFile); + inFileSize += readSize; + } + if (ferror(ress->srcFile)) EXM_THROW(37, "Error reading %s ", srcFileName); + + /* End of Stream mark */ + headerSize = LZ4F_compressEnd(ctx, ress->dstBuffer, ress->dstBufferSize, NULL); + if (LZ4F_isError(headerSize)) + EXM_THROW(38, "zstd: %s: lz4 end of file generation failed : %s", + srcFileName, LZ4F_getErrorName(headerSize)); + + { size_t const sizeCheck = fwrite(ress->dstBuffer, 1, headerSize, ress->dstFile); + if (sizeCheck != headerSize) + EXM_THROW(39, "Write error : %s (cannot write end of stream)", + strerror(errno)); + } + outFileSize += headerSize; + } + + *readsize = inFileSize; + LZ4F_freeCompressionContext(ctx); + + return outFileSize; +} +#endif + + +static unsigned long long +FIO_compressZstdFrame(FIO_ctx_t* const fCtx, + FIO_prefs_t* const prefs, + const cRess_t* ressPtr, + const char* srcFileName, U64 fileSize, + int compressionLevel, U64* readsize) +{ + cRess_t const ress = *ressPtr; + FILE* const srcFile = ress.srcFile; + FILE* const dstFile = ress.dstFile; + U64 compressedfilesize = 0; + ZSTD_EndDirective directive = ZSTD_e_continue; U64 pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN; - - /* stats */ - ZSTD_frameProgression previous_zfp_update = { 0, 0, 0, 0, 0, 0 }; - ZSTD_frameProgression previous_zfp_correction = { 0, 0, 0, 0, 0, 0 }; - typedef enum { noChange, slower, faster } speedChange_e; - speedChange_e speedChange = noChange; - unsigned flushWaiting = 0; - unsigned inputPresented = 0; - unsigned inputBlocked = 0; - unsigned lastJobID = 0; + + /* stats */ + ZSTD_frameProgression previous_zfp_update = { 0, 0, 0, 0, 0, 0 }; + ZSTD_frameProgression previous_zfp_correction = { 0, 0, 0, 0, 0, 0 }; + typedef enum { noChange, slower, faster } speedChange_e; + speedChange_e speedChange = noChange; + unsigned flushWaiting = 0; + unsigned inputPresented = 0; + unsigned inputBlocked = 0; + unsigned lastJobID = 0; UTIL_HumanReadableSize_t const file_hrs = UTIL_makeHumanReadableSize(fileSize); - - DISPLAYLEVEL(6, "compression using zstd format \n"); - - /* init */ - if (fileSize != UTIL_FILESIZE_UNKNOWN) { + + DISPLAYLEVEL(6, "compression using zstd format \n"); + + /* init */ + if (fileSize != UTIL_FILESIZE_UNKNOWN) { pledgedSrcSize = fileSize; - CHECK(ZSTD_CCtx_setPledgedSrcSize(ress.cctx, fileSize)); - } else if (prefs->streamSrcSize > 0) { - /* unknown source size; use the declared stream size */ + CHECK(ZSTD_CCtx_setPledgedSrcSize(ress.cctx, fileSize)); + } else if (prefs->streamSrcSize > 0) { + /* unknown source size; use the declared stream size */ pledgedSrcSize = prefs->streamSrcSize; - CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, prefs->streamSrcSize) ); - } + CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, prefs->streamSrcSize) ); + } { int windowLog; @@ -1381,421 +1381,421 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx, windowSize = UTIL_makeHumanReadableSize(MAX(1ULL, MIN(1ULL << windowLog, pledgedSrcSize))); DISPLAYLEVEL(4, "Decompression will require %.*f%s of memory\n", windowSize.precision, windowSize.value, windowSize.suffix); } - (void)srcFileName; - - /* Main compression loop */ - do { - size_t stillToFlush; - /* Fill input Buffer */ - size_t const inSize = fread(ress.srcBuffer, (size_t)1, ress.srcBufferSize, srcFile); - ZSTD_inBuffer inBuff = { ress.srcBuffer, inSize, 0 }; - DISPLAYLEVEL(6, "fread %u bytes from source \n", (unsigned)inSize); - *readsize += inSize; - - if ((inSize == 0) || (*readsize == fileSize)) - directive = ZSTD_e_end; - - stillToFlush = 1; - while ((inBuff.pos != inBuff.size) /* input buffer must be entirely ingested */ - || (directive == ZSTD_e_end && stillToFlush != 0) ) { - - size_t const oldIPos = inBuff.pos; - ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 }; - size_t const toFlushNow = ZSTD_toFlushNow(ress.cctx); - CHECK_V(stillToFlush, ZSTD_compressStream2(ress.cctx, &outBuff, &inBuff, directive)); - - /* count stats */ - inputPresented++; - if (oldIPos == inBuff.pos) inputBlocked++; /* input buffer is full and can't take any more : input speed is faster than consumption rate */ - if (!toFlushNow) flushWaiting = 1; - - /* Write compressed stream */ - DISPLAYLEVEL(6, "ZSTD_compress_generic(end:%u) => input pos(%u)<=(%u)size ; output generated %u bytes \n", - (unsigned)directive, (unsigned)inBuff.pos, (unsigned)inBuff.size, (unsigned)outBuff.pos); - if (outBuff.pos) { - size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile); - if (sizeCheck != outBuff.pos) - EXM_THROW(25, "Write error : %s (cannot write compressed block)", - strerror(errno)); - compressedfilesize += outBuff.pos; - } - - /* display notification; and adapt compression level */ - if (READY_FOR_UPDATE()) { - ZSTD_frameProgression const zfp = ZSTD_getFrameProgression(ress.cctx); - double const cShare = (double)zfp.produced / (double)(zfp.consumed + !zfp.consumed/*avoid div0*/) * 100; + (void)srcFileName; + + /* Main compression loop */ + do { + size_t stillToFlush; + /* Fill input Buffer */ + size_t const inSize = fread(ress.srcBuffer, (size_t)1, ress.srcBufferSize, srcFile); + ZSTD_inBuffer inBuff = { ress.srcBuffer, inSize, 0 }; + DISPLAYLEVEL(6, "fread %u bytes from source \n", (unsigned)inSize); + *readsize += inSize; + + if ((inSize == 0) || (*readsize == fileSize)) + directive = ZSTD_e_end; + + stillToFlush = 1; + while ((inBuff.pos != inBuff.size) /* input buffer must be entirely ingested */ + || (directive == ZSTD_e_end && stillToFlush != 0) ) { + + size_t const oldIPos = inBuff.pos; + ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 }; + size_t const toFlushNow = ZSTD_toFlushNow(ress.cctx); + CHECK_V(stillToFlush, ZSTD_compressStream2(ress.cctx, &outBuff, &inBuff, directive)); + + /* count stats */ + inputPresented++; + if (oldIPos == inBuff.pos) inputBlocked++; /* input buffer is full and can't take any more : input speed is faster than consumption rate */ + if (!toFlushNow) flushWaiting = 1; + + /* Write compressed stream */ + DISPLAYLEVEL(6, "ZSTD_compress_generic(end:%u) => input pos(%u)<=(%u)size ; output generated %u bytes \n", + (unsigned)directive, (unsigned)inBuff.pos, (unsigned)inBuff.size, (unsigned)outBuff.pos); + if (outBuff.pos) { + size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile); + if (sizeCheck != outBuff.pos) + EXM_THROW(25, "Write error : %s (cannot write compressed block)", + strerror(errno)); + compressedfilesize += outBuff.pos; + } + + /* display notification; and adapt compression level */ + if (READY_FOR_UPDATE()) { + ZSTD_frameProgression const zfp = ZSTD_getFrameProgression(ress.cctx); + double const cShare = (double)zfp.produced / (double)(zfp.consumed + !zfp.consumed/*avoid div0*/) * 100; UTIL_HumanReadableSize_t const buffered_hrs = UTIL_makeHumanReadableSize(zfp.ingested - zfp.consumed); UTIL_HumanReadableSize_t const consumed_hrs = UTIL_makeHumanReadableSize(zfp.consumed); UTIL_HumanReadableSize_t const produced_hrs = UTIL_makeHumanReadableSize(zfp.produced); - - /* display progress notifications */ - if (g_display_prefs.displayLevel >= 3) { + + /* display progress notifications */ + if (g_display_prefs.displayLevel >= 3) { DISPLAYUPDATE(3, "\r(L%i) Buffered :%6.*f%4s - Consumed :%6.*f%4s - Compressed :%6.*f%4s => %.2f%% ", - compressionLevel, + compressionLevel, buffered_hrs.precision, buffered_hrs.value, buffered_hrs.suffix, consumed_hrs.precision, consumed_hrs.value, consumed_hrs.suffix, produced_hrs.precision, produced_hrs.value, produced_hrs.suffix, - cShare ); - } else if (g_display_prefs.displayLevel >= 2 || g_display_prefs.progressSetting == FIO_ps_always) { - /* Require level 2 or forcibly displayed progress counter for summarized updates */ - DISPLAYLEVEL(1, "\r%79s\r", ""); /* Clear out the current displayed line */ - if (fCtx->nbFilesTotal > 1) { - size_t srcFileNameSize = strlen(srcFileName); - /* Ensure that the string we print is roughly the same size each time */ - if (srcFileNameSize > 18) { - const char* truncatedSrcFileName = srcFileName + srcFileNameSize - 15; - DISPLAYLEVEL(1, "Compress: %u/%u files. Current: ...%s ", - fCtx->currFileIdx+1, fCtx->nbFilesTotal, truncatedSrcFileName); - } else { - DISPLAYLEVEL(1, "Compress: %u/%u files. Current: %*s ", - fCtx->currFileIdx+1, fCtx->nbFilesTotal, (int)(18-srcFileNameSize), srcFileName); - } - } + cShare ); + } else if (g_display_prefs.displayLevel >= 2 || g_display_prefs.progressSetting == FIO_ps_always) { + /* Require level 2 or forcibly displayed progress counter for summarized updates */ + DISPLAYLEVEL(1, "\r%79s\r", ""); /* Clear out the current displayed line */ + if (fCtx->nbFilesTotal > 1) { + size_t srcFileNameSize = strlen(srcFileName); + /* Ensure that the string we print is roughly the same size each time */ + if (srcFileNameSize > 18) { + const char* truncatedSrcFileName = srcFileName + srcFileNameSize - 15; + DISPLAYLEVEL(1, "Compress: %u/%u files. Current: ...%s ", + fCtx->currFileIdx+1, fCtx->nbFilesTotal, truncatedSrcFileName); + } else { + DISPLAYLEVEL(1, "Compress: %u/%u files. Current: %*s ", + fCtx->currFileIdx+1, fCtx->nbFilesTotal, (int)(18-srcFileNameSize), srcFileName); + } + } DISPLAYLEVEL(1, "Read:%6.*f%4s ", consumed_hrs.precision, consumed_hrs.value, consumed_hrs.suffix); - if (fileSize != UTIL_FILESIZE_UNKNOWN) + if (fileSize != UTIL_FILESIZE_UNKNOWN) DISPLAYLEVEL(2, "/%6.*f%4s", file_hrs.precision, file_hrs.value, file_hrs.suffix); DISPLAYLEVEL(1, " ==> %2.f%%", cShare); - DELAY_NEXT_UPDATE(); - } - - /* adaptive mode : statistics measurement and speed correction */ - if (prefs->adaptiveMode) { - - /* check output speed */ - if (zfp.currentJobID > 1) { /* only possible if nbWorkers >= 1 */ - - unsigned long long newlyProduced = zfp.produced - previous_zfp_update.produced; - unsigned long long newlyFlushed = zfp.flushed - previous_zfp_update.flushed; - assert(zfp.produced >= previous_zfp_update.produced); - assert(prefs->nbWorkers >= 1); - - /* test if compression is blocked - * either because output is slow and all buffers are full - * or because input is slow and no job can start while waiting for at least one buffer to be filled. - * note : exclude starting part, since currentJobID > 1 */ - if ( (zfp.consumed == previous_zfp_update.consumed) /* no data compressed : no data available, or no more buffer to compress to, OR compression is really slow (compression of a single block is slower than update rate)*/ - && (zfp.nbActiveWorkers == 0) /* confirmed : no compression ongoing */ - ) { - DISPLAYLEVEL(6, "all buffers full : compression stopped => slow down \n") - speedChange = slower; - } - - previous_zfp_update = zfp; - - if ( (newlyProduced > (newlyFlushed * 9 / 8)) /* compression produces more data than output can flush (though production can be spiky, due to work unit : (N==4)*block sizes) */ - && (flushWaiting == 0) /* flush speed was never slowed by lack of production, so it's operating at max capacity */ - ) { - DISPLAYLEVEL(6, "compression faster than flush (%llu > %llu), and flushed was never slowed down by lack of production => slow down \n", newlyProduced, newlyFlushed); - speedChange = slower; - } - flushWaiting = 0; - } - - /* course correct only if there is at least one new job completed */ - if (zfp.currentJobID > lastJobID) { - DISPLAYLEVEL(6, "compression level adaptation check \n") - - /* check input speed */ - if (zfp.currentJobID > (unsigned)(prefs->nbWorkers+1)) { /* warm up period, to fill all workers */ - if (inputBlocked <= 0) { - DISPLAYLEVEL(6, "input is never blocked => input is slower than ingestion \n"); - speedChange = slower; - } else if (speedChange == noChange) { - unsigned long long newlyIngested = zfp.ingested - previous_zfp_correction.ingested; - unsigned long long newlyConsumed = zfp.consumed - previous_zfp_correction.consumed; - unsigned long long newlyProduced = zfp.produced - previous_zfp_correction.produced; - unsigned long long newlyFlushed = zfp.flushed - previous_zfp_correction.flushed; - previous_zfp_correction = zfp; - assert(inputPresented > 0); - DISPLAYLEVEL(6, "input blocked %u/%u(%.2f) - ingested:%u vs %u:consumed - flushed:%u vs %u:produced \n", - inputBlocked, inputPresented, (double)inputBlocked/inputPresented*100, - (unsigned)newlyIngested, (unsigned)newlyConsumed, - (unsigned)newlyFlushed, (unsigned)newlyProduced); - if ( (inputBlocked > inputPresented / 8) /* input is waiting often, because input buffers is full : compression or output too slow */ - && (newlyFlushed * 33 / 32 > newlyProduced) /* flush everything that is produced */ - && (newlyIngested * 33 / 32 > newlyConsumed) /* input speed as fast or faster than compression speed */ - ) { - DISPLAYLEVEL(6, "recommend faster as in(%llu) >= (%llu)comp(%llu) <= out(%llu) \n", - newlyIngested, newlyConsumed, newlyProduced, newlyFlushed); - speedChange = faster; - } - } - inputBlocked = 0; - inputPresented = 0; - } - - if (speedChange == slower) { - DISPLAYLEVEL(6, "slower speed , higher compression \n") - compressionLevel ++; - if (compressionLevel > ZSTD_maxCLevel()) compressionLevel = ZSTD_maxCLevel(); - if (compressionLevel > prefs->maxAdaptLevel) compressionLevel = prefs->maxAdaptLevel; - compressionLevel += (compressionLevel == 0); /* skip 0 */ - ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, compressionLevel); - } - if (speedChange == faster) { - DISPLAYLEVEL(6, "faster speed , lighter compression \n") - compressionLevel --; - if (compressionLevel < prefs->minAdaptLevel) compressionLevel = prefs->minAdaptLevel; - compressionLevel -= (compressionLevel == 0); /* skip 0 */ - ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, compressionLevel); - } - speedChange = noChange; - - lastJobID = zfp.currentJobID; - } /* if (zfp.currentJobID > lastJobID) */ - } /* if (g_adaptiveMode) */ - } /* if (READY_FOR_UPDATE()) */ - } /* while ((inBuff.pos != inBuff.size) */ - } while (directive != ZSTD_e_end); - - if (ferror(srcFile)) { - EXM_THROW(26, "Read error : I/O error"); - } - if (fileSize != UTIL_FILESIZE_UNKNOWN && *readsize != fileSize) { - EXM_THROW(27, "Read error : Incomplete read : %llu / %llu B", - (unsigned long long)*readsize, (unsigned long long)fileSize); - } - - return compressedfilesize; -} - -/*! FIO_compressFilename_internal() : - * same as FIO_compressFilename_extRess(), with `ress.desFile` already opened. - * @return : 0 : compression completed correctly, - * 1 : missing or pb opening srcFileName - */ -static int -FIO_compressFilename_internal(FIO_ctx_t* const fCtx, - FIO_prefs_t* const prefs, - cRess_t ress, - const char* dstFileName, const char* srcFileName, - int compressionLevel) -{ - UTIL_time_t const timeStart = UTIL_getTime(); - clock_t const cpuStart = clock(); - U64 readsize = 0; - U64 compressedfilesize = 0; - U64 const fileSize = UTIL_getFileSize(srcFileName); - DISPLAYLEVEL(5, "%s: %llu bytes \n", srcFileName, (unsigned long long)fileSize); - - /* compression format selection */ - switch (prefs->compressionType) { - default: - case FIO_zstdCompression: - compressedfilesize = FIO_compressZstdFrame(fCtx, prefs, &ress, srcFileName, fileSize, compressionLevel, &readsize); - break; - - case FIO_gzipCompression: -#ifdef ZSTD_GZCOMPRESS - compressedfilesize = FIO_compressGzFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize); -#else - (void)compressionLevel; - EXM_THROW(20, "zstd: %s: file cannot be compressed as gzip (zstd compiled without ZSTD_GZCOMPRESS) -- ignored \n", - srcFileName); -#endif - break; - - case FIO_xzCompression: - case FIO_lzmaCompression: -#ifdef ZSTD_LZMACOMPRESS - compressedfilesize = FIO_compressLzmaFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize, prefs->compressionType==FIO_lzmaCompression); -#else - (void)compressionLevel; - EXM_THROW(20, "zstd: %s: file cannot be compressed as xz/lzma (zstd compiled without ZSTD_LZMACOMPRESS) -- ignored \n", - srcFileName); -#endif - break; - - case FIO_lz4Compression: -#ifdef ZSTD_LZ4COMPRESS - compressedfilesize = FIO_compressLz4Frame(&ress, srcFileName, fileSize, compressionLevel, prefs->checksumFlag, &readsize); -#else - (void)compressionLevel; - EXM_THROW(20, "zstd: %s: file cannot be compressed as lz4 (zstd compiled without ZSTD_LZ4COMPRESS) -- ignored \n", - srcFileName); -#endif - break; - } - - /* Status */ - fCtx->totalBytesInput += (size_t)readsize; - fCtx->totalBytesOutput += (size_t)compressedfilesize; - DISPLAYLEVEL(2, "\r%79s\r", ""); - if (g_display_prefs.displayLevel >= 2 && - !fCtx->hasStdoutOutput && - (g_display_prefs.displayLevel >= 3 || fCtx->nbFilesTotal <= 1)) { + DELAY_NEXT_UPDATE(); + } + + /* adaptive mode : statistics measurement and speed correction */ + if (prefs->adaptiveMode) { + + /* check output speed */ + if (zfp.currentJobID > 1) { /* only possible if nbWorkers >= 1 */ + + unsigned long long newlyProduced = zfp.produced - previous_zfp_update.produced; + unsigned long long newlyFlushed = zfp.flushed - previous_zfp_update.flushed; + assert(zfp.produced >= previous_zfp_update.produced); + assert(prefs->nbWorkers >= 1); + + /* test if compression is blocked + * either because output is slow and all buffers are full + * or because input is slow and no job can start while waiting for at least one buffer to be filled. + * note : exclude starting part, since currentJobID > 1 */ + if ( (zfp.consumed == previous_zfp_update.consumed) /* no data compressed : no data available, or no more buffer to compress to, OR compression is really slow (compression of a single block is slower than update rate)*/ + && (zfp.nbActiveWorkers == 0) /* confirmed : no compression ongoing */ + ) { + DISPLAYLEVEL(6, "all buffers full : compression stopped => slow down \n") + speedChange = slower; + } + + previous_zfp_update = zfp; + + if ( (newlyProduced > (newlyFlushed * 9 / 8)) /* compression produces more data than output can flush (though production can be spiky, due to work unit : (N==4)*block sizes) */ + && (flushWaiting == 0) /* flush speed was never slowed by lack of production, so it's operating at max capacity */ + ) { + DISPLAYLEVEL(6, "compression faster than flush (%llu > %llu), and flushed was never slowed down by lack of production => slow down \n", newlyProduced, newlyFlushed); + speedChange = slower; + } + flushWaiting = 0; + } + + /* course correct only if there is at least one new job completed */ + if (zfp.currentJobID > lastJobID) { + DISPLAYLEVEL(6, "compression level adaptation check \n") + + /* check input speed */ + if (zfp.currentJobID > (unsigned)(prefs->nbWorkers+1)) { /* warm up period, to fill all workers */ + if (inputBlocked <= 0) { + DISPLAYLEVEL(6, "input is never blocked => input is slower than ingestion \n"); + speedChange = slower; + } else if (speedChange == noChange) { + unsigned long long newlyIngested = zfp.ingested - previous_zfp_correction.ingested; + unsigned long long newlyConsumed = zfp.consumed - previous_zfp_correction.consumed; + unsigned long long newlyProduced = zfp.produced - previous_zfp_correction.produced; + unsigned long long newlyFlushed = zfp.flushed - previous_zfp_correction.flushed; + previous_zfp_correction = zfp; + assert(inputPresented > 0); + DISPLAYLEVEL(6, "input blocked %u/%u(%.2f) - ingested:%u vs %u:consumed - flushed:%u vs %u:produced \n", + inputBlocked, inputPresented, (double)inputBlocked/inputPresented*100, + (unsigned)newlyIngested, (unsigned)newlyConsumed, + (unsigned)newlyFlushed, (unsigned)newlyProduced); + if ( (inputBlocked > inputPresented / 8) /* input is waiting often, because input buffers is full : compression or output too slow */ + && (newlyFlushed * 33 / 32 > newlyProduced) /* flush everything that is produced */ + && (newlyIngested * 33 / 32 > newlyConsumed) /* input speed as fast or faster than compression speed */ + ) { + DISPLAYLEVEL(6, "recommend faster as in(%llu) >= (%llu)comp(%llu) <= out(%llu) \n", + newlyIngested, newlyConsumed, newlyProduced, newlyFlushed); + speedChange = faster; + } + } + inputBlocked = 0; + inputPresented = 0; + } + + if (speedChange == slower) { + DISPLAYLEVEL(6, "slower speed , higher compression \n") + compressionLevel ++; + if (compressionLevel > ZSTD_maxCLevel()) compressionLevel = ZSTD_maxCLevel(); + if (compressionLevel > prefs->maxAdaptLevel) compressionLevel = prefs->maxAdaptLevel; + compressionLevel += (compressionLevel == 0); /* skip 0 */ + ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, compressionLevel); + } + if (speedChange == faster) { + DISPLAYLEVEL(6, "faster speed , lighter compression \n") + compressionLevel --; + if (compressionLevel < prefs->minAdaptLevel) compressionLevel = prefs->minAdaptLevel; + compressionLevel -= (compressionLevel == 0); /* skip 0 */ + ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, compressionLevel); + } + speedChange = noChange; + + lastJobID = zfp.currentJobID; + } /* if (zfp.currentJobID > lastJobID) */ + } /* if (g_adaptiveMode) */ + } /* if (READY_FOR_UPDATE()) */ + } /* while ((inBuff.pos != inBuff.size) */ + } while (directive != ZSTD_e_end); + + if (ferror(srcFile)) { + EXM_THROW(26, "Read error : I/O error"); + } + if (fileSize != UTIL_FILESIZE_UNKNOWN && *readsize != fileSize) { + EXM_THROW(27, "Read error : Incomplete read : %llu / %llu B", + (unsigned long long)*readsize, (unsigned long long)fileSize); + } + + return compressedfilesize; +} + +/*! FIO_compressFilename_internal() : + * same as FIO_compressFilename_extRess(), with `ress.desFile` already opened. + * @return : 0 : compression completed correctly, + * 1 : missing or pb opening srcFileName + */ +static int +FIO_compressFilename_internal(FIO_ctx_t* const fCtx, + FIO_prefs_t* const prefs, + cRess_t ress, + const char* dstFileName, const char* srcFileName, + int compressionLevel) +{ + UTIL_time_t const timeStart = UTIL_getTime(); + clock_t const cpuStart = clock(); + U64 readsize = 0; + U64 compressedfilesize = 0; + U64 const fileSize = UTIL_getFileSize(srcFileName); + DISPLAYLEVEL(5, "%s: %llu bytes \n", srcFileName, (unsigned long long)fileSize); + + /* compression format selection */ + switch (prefs->compressionType) { + default: + case FIO_zstdCompression: + compressedfilesize = FIO_compressZstdFrame(fCtx, prefs, &ress, srcFileName, fileSize, compressionLevel, &readsize); + break; + + case FIO_gzipCompression: +#ifdef ZSTD_GZCOMPRESS + compressedfilesize = FIO_compressGzFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize); +#else + (void)compressionLevel; + EXM_THROW(20, "zstd: %s: file cannot be compressed as gzip (zstd compiled without ZSTD_GZCOMPRESS) -- ignored \n", + srcFileName); +#endif + break; + + case FIO_xzCompression: + case FIO_lzmaCompression: +#ifdef ZSTD_LZMACOMPRESS + compressedfilesize = FIO_compressLzmaFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize, prefs->compressionType==FIO_lzmaCompression); +#else + (void)compressionLevel; + EXM_THROW(20, "zstd: %s: file cannot be compressed as xz/lzma (zstd compiled without ZSTD_LZMACOMPRESS) -- ignored \n", + srcFileName); +#endif + break; + + case FIO_lz4Compression: +#ifdef ZSTD_LZ4COMPRESS + compressedfilesize = FIO_compressLz4Frame(&ress, srcFileName, fileSize, compressionLevel, prefs->checksumFlag, &readsize); +#else + (void)compressionLevel; + EXM_THROW(20, "zstd: %s: file cannot be compressed as lz4 (zstd compiled without ZSTD_LZ4COMPRESS) -- ignored \n", + srcFileName); +#endif + break; + } + + /* Status */ + fCtx->totalBytesInput += (size_t)readsize; + fCtx->totalBytesOutput += (size_t)compressedfilesize; + DISPLAYLEVEL(2, "\r%79s\r", ""); + if (g_display_prefs.displayLevel >= 2 && + !fCtx->hasStdoutOutput && + (g_display_prefs.displayLevel >= 3 || fCtx->nbFilesTotal <= 1)) { UTIL_HumanReadableSize_t hr_isize = UTIL_makeHumanReadableSize((U64) readsize); UTIL_HumanReadableSize_t hr_osize = UTIL_makeHumanReadableSize((U64) compressedfilesize); - if (readsize == 0) { + if (readsize == 0) { DISPLAYLEVEL(2,"%-20s : (%6.*f%4s => %6.*f%4s, %s) \n", - srcFileName, + srcFileName, hr_isize.precision, hr_isize.value, hr_isize.suffix, hr_osize.precision, hr_osize.value, hr_osize.suffix, - dstFileName); - } else { + dstFileName); + } else { DISPLAYLEVEL(2,"%-20s :%6.2f%% (%6.*f%4s => %6.*f%4s, %s) \n", - srcFileName, - (double)compressedfilesize / (double)readsize * 100, + srcFileName, + (double)compressedfilesize / (double)readsize * 100, hr_isize.precision, hr_isize.value, hr_isize.suffix, hr_osize.precision, hr_osize.value, hr_osize.suffix, - dstFileName); - } - } - - /* Elapsed Time and CPU Load */ - { clock_t const cpuEnd = clock(); - double const cpuLoad_s = (double)(cpuEnd - cpuStart) / CLOCKS_PER_SEC; - U64 const timeLength_ns = UTIL_clockSpanNano(timeStart); - double const timeLength_s = (double)timeLength_ns / 1000000000; - double const cpuLoad_pct = (cpuLoad_s / timeLength_s) * 100; - DISPLAYLEVEL(4, "%-20s : Completed in %.2f sec (cpu load : %.0f%%)\n", - srcFileName, timeLength_s, cpuLoad_pct); - } - return 0; -} - - -/*! FIO_compressFilename_dstFile() : - * open dstFileName, or pass-through if ress.dstFile != NULL, - * then start compression with FIO_compressFilename_internal(). - * Manages source removal (--rm) and file permissions transfer. - * note : ress.srcFile must be != NULL, - * so reach this function through FIO_compressFilename_srcFile(). - * @return : 0 : compression completed correctly, - * 1 : pb - */ -static int FIO_compressFilename_dstFile(FIO_ctx_t* const fCtx, - FIO_prefs_t* const prefs, - cRess_t ress, - const char* dstFileName, - const char* srcFileName, - int compressionLevel) -{ - int closeDstFile = 0; - int result; - stat_t statbuf; + dstFileName); + } + } + + /* Elapsed Time and CPU Load */ + { clock_t const cpuEnd = clock(); + double const cpuLoad_s = (double)(cpuEnd - cpuStart) / CLOCKS_PER_SEC; + U64 const timeLength_ns = UTIL_clockSpanNano(timeStart); + double const timeLength_s = (double)timeLength_ns / 1000000000; + double const cpuLoad_pct = (cpuLoad_s / timeLength_s) * 100; + DISPLAYLEVEL(4, "%-20s : Completed in %.2f sec (cpu load : %.0f%%)\n", + srcFileName, timeLength_s, cpuLoad_pct); + } + return 0; +} + + +/*! FIO_compressFilename_dstFile() : + * open dstFileName, or pass-through if ress.dstFile != NULL, + * then start compression with FIO_compressFilename_internal(). + * Manages source removal (--rm) and file permissions transfer. + * note : ress.srcFile must be != NULL, + * so reach this function through FIO_compressFilename_srcFile(). + * @return : 0 : compression completed correctly, + * 1 : pb + */ +static int FIO_compressFilename_dstFile(FIO_ctx_t* const fCtx, + FIO_prefs_t* const prefs, + cRess_t ress, + const char* dstFileName, + const char* srcFileName, + int compressionLevel) +{ + int closeDstFile = 0; + int result; + stat_t statbuf; int transferMTime = 0; - assert(ress.srcFile != NULL); - if (ress.dstFile == NULL) { - int dstFilePermissions = DEFAULT_FILE_PERMISSIONS; - if ( strcmp (srcFileName, stdinmark) + assert(ress.srcFile != NULL); + if (ress.dstFile == NULL) { + int dstFilePermissions = DEFAULT_FILE_PERMISSIONS; + if ( strcmp (srcFileName, stdinmark) && strcmp (dstFileName, stdoutmark) - && UTIL_stat(srcFileName, &statbuf) - && UTIL_isRegularFileStat(&statbuf) ) { - dstFilePermissions = statbuf.st_mode; + && UTIL_stat(srcFileName, &statbuf) + && UTIL_isRegularFileStat(&statbuf) ) { + dstFilePermissions = statbuf.st_mode; transferMTime = 1; - } - - closeDstFile = 1; - DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: opening dst: %s \n", dstFileName); - ress.dstFile = FIO_openDstFile(fCtx, prefs, srcFileName, dstFileName, dstFilePermissions); - if (ress.dstFile==NULL) return 1; /* could not open dstFileName */ - /* Must only be added after FIO_openDstFile() succeeds. - * Otherwise we may delete the destination file if it already exists, - * and the user presses Ctrl-C when asked if they wish to overwrite. - */ - addHandler(dstFileName); - } - - result = FIO_compressFilename_internal(fCtx, prefs, ress, dstFileName, srcFileName, compressionLevel); - - if (closeDstFile) { - FILE* const dstFile = ress.dstFile; - ress.dstFile = NULL; - - clearHandler(); - - DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: closing dst: %s \n", dstFileName); - if (fclose(dstFile)) { /* error closing dstFile */ - DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno)); - result=1; - } + } + + closeDstFile = 1; + DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: opening dst: %s \n", dstFileName); + ress.dstFile = FIO_openDstFile(fCtx, prefs, srcFileName, dstFileName, dstFilePermissions); + if (ress.dstFile==NULL) return 1; /* could not open dstFileName */ + /* Must only be added after FIO_openDstFile() succeeds. + * Otherwise we may delete the destination file if it already exists, + * and the user presses Ctrl-C when asked if they wish to overwrite. + */ + addHandler(dstFileName); + } + + result = FIO_compressFilename_internal(fCtx, prefs, ress, dstFileName, srcFileName, compressionLevel); + + if (closeDstFile) { + FILE* const dstFile = ress.dstFile; + ress.dstFile = NULL; + + clearHandler(); + + DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: closing dst: %s \n", dstFileName); + if (fclose(dstFile)) { /* error closing dstFile */ + DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno)); + result=1; + } if (transferMTime) { UTIL_utime(dstFileName, &statbuf); } - if ( (result != 0) /* operation failure */ - && strcmp(dstFileName, stdoutmark) /* special case : don't remove() stdout */ - ) { - FIO_removeFile(dstFileName); /* remove compression artefact; note don't do anything special if remove() fails */ - } - } - - return result; -} - -/* List used to compare file extensions (used with --exclude-compressed flag) -* Different from the suffixList and should only apply to ZSTD compress operationResult -*/ -static const char *compressedFileExtensions[] = { - ZSTD_EXTENSION, - TZSTD_EXTENSION, - GZ_EXTENSION, - TGZ_EXTENSION, - LZMA_EXTENSION, - XZ_EXTENSION, - TXZ_EXTENSION, - LZ4_EXTENSION, - TLZ4_EXTENSION, - NULL -}; - -/*! FIO_compressFilename_srcFile() : - * @return : 0 : compression completed correctly, - * 1 : missing or pb opening srcFileName - */ -static int -FIO_compressFilename_srcFile(FIO_ctx_t* const fCtx, - FIO_prefs_t* const prefs, - cRess_t ress, - const char* dstFileName, - const char* srcFileName, - int compressionLevel) -{ - int result; - DISPLAYLEVEL(6, "FIO_compressFilename_srcFile: %s \n", srcFileName); - - /* ensure src is not a directory */ - if (UTIL_isDirectory(srcFileName)) { - DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName); - return 1; - } - - /* ensure src is not the same as dict (if present) */ - if (ress.dictFileName != NULL && UTIL_isSameFile(srcFileName, ress.dictFileName)) { - DISPLAYLEVEL(1, "zstd: cannot use %s as an input file and dictionary \n", srcFileName); - return 1; - } - - /* Check if "srcFile" is compressed. Only done if --exclude-compressed flag is used - * YES => ZSTD will skip compression of the file and will return 0. - * NO => ZSTD will resume with compress operation. - */ - if (prefs->excludeCompressedFiles == 1 && UTIL_isCompressedFile(srcFileName, compressedFileExtensions)) { - DISPLAYLEVEL(4, "File is already compressed : %s \n", srcFileName); - return 0; - } - - ress.srcFile = FIO_openSrcFile(prefs, srcFileName); - if (ress.srcFile == NULL) return 1; /* srcFile could not be opened */ - - result = FIO_compressFilename_dstFile(fCtx, prefs, ress, dstFileName, srcFileName, compressionLevel); - - fclose(ress.srcFile); - ress.srcFile = NULL; - if ( prefs->removeSrcFile /* --rm */ - && result == 0 /* success */ - && strcmp(srcFileName, stdinmark) /* exception : don't erase stdin */ - ) { - /* We must clear the handler, since after this point calling it would - * delete both the source and destination files. - */ - clearHandler(); - if (FIO_removeFile(srcFileName)) - EXM_THROW(1, "zstd: %s: %s", srcFileName, strerror(errno)); - } - return result; -} - + if ( (result != 0) /* operation failure */ + && strcmp(dstFileName, stdoutmark) /* special case : don't remove() stdout */ + ) { + FIO_removeFile(dstFileName); /* remove compression artefact; note don't do anything special if remove() fails */ + } + } + + return result; +} + +/* List used to compare file extensions (used with --exclude-compressed flag) +* Different from the suffixList and should only apply to ZSTD compress operationResult +*/ +static const char *compressedFileExtensions[] = { + ZSTD_EXTENSION, + TZSTD_EXTENSION, + GZ_EXTENSION, + TGZ_EXTENSION, + LZMA_EXTENSION, + XZ_EXTENSION, + TXZ_EXTENSION, + LZ4_EXTENSION, + TLZ4_EXTENSION, + NULL +}; + +/*! FIO_compressFilename_srcFile() : + * @return : 0 : compression completed correctly, + * 1 : missing or pb opening srcFileName + */ +static int +FIO_compressFilename_srcFile(FIO_ctx_t* const fCtx, + FIO_prefs_t* const prefs, + cRess_t ress, + const char* dstFileName, + const char* srcFileName, + int compressionLevel) +{ + int result; + DISPLAYLEVEL(6, "FIO_compressFilename_srcFile: %s \n", srcFileName); + + /* ensure src is not a directory */ + if (UTIL_isDirectory(srcFileName)) { + DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName); + return 1; + } + + /* ensure src is not the same as dict (if present) */ + if (ress.dictFileName != NULL && UTIL_isSameFile(srcFileName, ress.dictFileName)) { + DISPLAYLEVEL(1, "zstd: cannot use %s as an input file and dictionary \n", srcFileName); + return 1; + } + + /* Check if "srcFile" is compressed. Only done if --exclude-compressed flag is used + * YES => ZSTD will skip compression of the file and will return 0. + * NO => ZSTD will resume with compress operation. + */ + if (prefs->excludeCompressedFiles == 1 && UTIL_isCompressedFile(srcFileName, compressedFileExtensions)) { + DISPLAYLEVEL(4, "File is already compressed : %s \n", srcFileName); + return 0; + } + + ress.srcFile = FIO_openSrcFile(prefs, srcFileName); + if (ress.srcFile == NULL) return 1; /* srcFile could not be opened */ + + result = FIO_compressFilename_dstFile(fCtx, prefs, ress, dstFileName, srcFileName, compressionLevel); + + fclose(ress.srcFile); + ress.srcFile = NULL; + if ( prefs->removeSrcFile /* --rm */ + && result == 0 /* success */ + && strcmp(srcFileName, stdinmark) /* exception : don't erase stdin */ + ) { + /* We must clear the handler, since after this point calling it would + * delete both the source and destination files. + */ + clearHandler(); + if (FIO_removeFile(srcFileName)) + EXM_THROW(1, "zstd: %s: %s", srcFileName, strerror(errno)); + } + return result; +} + static const char* checked_index(const char* options[], size_t length, size_t index) { assert(index < length); // Necessary to avoid warnings since -O3 will omit the above `assert` @@ -1840,1382 +1840,1382 @@ void FIO_displayCompressionParameters(const FIO_prefs_t* prefs) { #undef INDEX -int FIO_compressFilename(FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs, const char* dstFileName, - const char* srcFileName, const char* dictFileName, - int compressionLevel, ZSTD_compressionParameters comprParams) -{ - cRess_t const ress = FIO_createCResources(prefs, dictFileName, UTIL_getFileSize(srcFileName), compressionLevel, comprParams); - int const result = FIO_compressFilename_srcFile(fCtx, prefs, ress, dstFileName, srcFileName, compressionLevel); - -#define DISPLAY_LEVEL_DEFAULT 2 - - FIO_freeCResources(&ress); - return result; -} - -/* FIO_determineCompressedName() : - * create a destination filename for compressed srcFileName. - * @return a pointer to it. - * This function never returns an error (it may abort() in case of pb) - */ -static const char* -FIO_determineCompressedName(const char* srcFileName, const char* outDirName, const char* suffix) -{ - static size_t dfnbCapacity = 0; - static char* dstFileNameBuffer = NULL; /* using static allocation : this function cannot be multi-threaded */ - char* outDirFilename = NULL; - size_t sfnSize = strlen(srcFileName); - size_t const srcSuffixLen = strlen(suffix); - if (outDirName) { - outDirFilename = FIO_createFilename_fromOutDir(srcFileName, outDirName, srcSuffixLen); - sfnSize = strlen(outDirFilename); - assert(outDirFilename != NULL); - } - - if (dfnbCapacity <= sfnSize+srcSuffixLen+1) { - /* resize buffer for dstName */ - free(dstFileNameBuffer); - dfnbCapacity = sfnSize + srcSuffixLen + 30; - dstFileNameBuffer = (char*)malloc(dfnbCapacity); - if (!dstFileNameBuffer) { - EXM_THROW(30, "zstd: %s", strerror(errno)); - } - } - assert(dstFileNameBuffer != NULL); - - if (outDirFilename) { - memcpy(dstFileNameBuffer, outDirFilename, sfnSize); - free(outDirFilename); - } else { - memcpy(dstFileNameBuffer, srcFileName, sfnSize); - } - memcpy(dstFileNameBuffer+sfnSize, suffix, srcSuffixLen+1 /* Include terminating null */); - return dstFileNameBuffer; -} - -static unsigned long long FIO_getLargestFileSize(const char** inFileNames, unsigned nbFiles) -{ - size_t i; - unsigned long long fileSize, maxFileSize = 0; - for (i = 0; i < nbFiles; i++) { - fileSize = UTIL_getFileSize(inFileNames[i]); - maxFileSize = fileSize > maxFileSize ? fileSize : maxFileSize; - } - return maxFileSize; -} - -/* FIO_compressMultipleFilenames() : - * compress nbFiles files - * into either one destination (outFileName), - * or into one file each (outFileName == NULL, but suffix != NULL), - * or into a destination folder (specified with -O) - */ -int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx, - FIO_prefs_t* const prefs, - const char** inFileNamesTable, - const char* outMirroredRootDirName, - const char* outDirName, - const char* outFileName, const char* suffix, - const char* dictFileName, int compressionLevel, - ZSTD_compressionParameters comprParams) -{ - int status; - int error = 0; - cRess_t ress = FIO_createCResources(prefs, dictFileName, - FIO_getLargestFileSize(inFileNamesTable, (unsigned)fCtx->nbFilesTotal), - compressionLevel, comprParams); - - /* init */ - assert(outFileName != NULL || suffix != NULL); - if (outFileName != NULL) { /* output into a single destination (stdout typically) */ - if (FIO_removeMultiFilesWarning(fCtx, prefs, outFileName, 1 /* displayLevelCutoff */)) { - FIO_freeCResources(&ress); - return 1; - } - ress.dstFile = FIO_openDstFile(fCtx, prefs, NULL, outFileName, DEFAULT_FILE_PERMISSIONS); - if (ress.dstFile == NULL) { /* could not open outFileName */ - error = 1; - } else { - for (; fCtx->currFileIdx < fCtx->nbFilesTotal; ++fCtx->currFileIdx) { - status = FIO_compressFilename_srcFile(fCtx, prefs, ress, outFileName, inFileNamesTable[fCtx->currFileIdx], compressionLevel); - if (!status) fCtx->nbFilesProcessed++; - error |= status; - } - if (fclose(ress.dstFile)) - EXM_THROW(29, "Write error (%s) : cannot properly close %s", - strerror(errno), outFileName); - ress.dstFile = NULL; - } - } else { - if (outMirroredRootDirName) - UTIL_mirrorSourceFilesDirectories(inFileNamesTable, (unsigned)fCtx->nbFilesTotal, outMirroredRootDirName); - - for (; fCtx->currFileIdx < fCtx->nbFilesTotal; ++fCtx->currFileIdx) { - const char* const srcFileName = inFileNamesTable[fCtx->currFileIdx]; - const char* dstFileName = NULL; - if (outMirroredRootDirName) { - char* validMirroredDirName = UTIL_createMirroredDestDirName(srcFileName, outMirroredRootDirName); - if (validMirroredDirName) { - dstFileName = FIO_determineCompressedName(srcFileName, validMirroredDirName, suffix); - free(validMirroredDirName); - } else { - DISPLAYLEVEL(2, "zstd: --output-dir-mirror cannot compress '%s' into '%s' \n", srcFileName, outMirroredRootDirName); - error=1; - continue; - } - } else { - dstFileName = FIO_determineCompressedName(srcFileName, outDirName, suffix); /* cannot fail */ - } - status = FIO_compressFilename_srcFile(fCtx, prefs, ress, dstFileName, srcFileName, compressionLevel); - if (!status) fCtx->nbFilesProcessed++; - error |= status; - } - - if (outDirName) - FIO_checkFilenameCollisions(inFileNamesTable , (unsigned)fCtx->nbFilesTotal); - } - - if (fCtx->nbFilesProcessed >= 1 && fCtx->nbFilesTotal > 1 && fCtx->totalBytesInput != 0) { +int FIO_compressFilename(FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs, const char* dstFileName, + const char* srcFileName, const char* dictFileName, + int compressionLevel, ZSTD_compressionParameters comprParams) +{ + cRess_t const ress = FIO_createCResources(prefs, dictFileName, UTIL_getFileSize(srcFileName), compressionLevel, comprParams); + int const result = FIO_compressFilename_srcFile(fCtx, prefs, ress, dstFileName, srcFileName, compressionLevel); + +#define DISPLAY_LEVEL_DEFAULT 2 + + FIO_freeCResources(&ress); + return result; +} + +/* FIO_determineCompressedName() : + * create a destination filename for compressed srcFileName. + * @return a pointer to it. + * This function never returns an error (it may abort() in case of pb) + */ +static const char* +FIO_determineCompressedName(const char* srcFileName, const char* outDirName, const char* suffix) +{ + static size_t dfnbCapacity = 0; + static char* dstFileNameBuffer = NULL; /* using static allocation : this function cannot be multi-threaded */ + char* outDirFilename = NULL; + size_t sfnSize = strlen(srcFileName); + size_t const srcSuffixLen = strlen(suffix); + if (outDirName) { + outDirFilename = FIO_createFilename_fromOutDir(srcFileName, outDirName, srcSuffixLen); + sfnSize = strlen(outDirFilename); + assert(outDirFilename != NULL); + } + + if (dfnbCapacity <= sfnSize+srcSuffixLen+1) { + /* resize buffer for dstName */ + free(dstFileNameBuffer); + dfnbCapacity = sfnSize + srcSuffixLen + 30; + dstFileNameBuffer = (char*)malloc(dfnbCapacity); + if (!dstFileNameBuffer) { + EXM_THROW(30, "zstd: %s", strerror(errno)); + } + } + assert(dstFileNameBuffer != NULL); + + if (outDirFilename) { + memcpy(dstFileNameBuffer, outDirFilename, sfnSize); + free(outDirFilename); + } else { + memcpy(dstFileNameBuffer, srcFileName, sfnSize); + } + memcpy(dstFileNameBuffer+sfnSize, suffix, srcSuffixLen+1 /* Include terminating null */); + return dstFileNameBuffer; +} + +static unsigned long long FIO_getLargestFileSize(const char** inFileNames, unsigned nbFiles) +{ + size_t i; + unsigned long long fileSize, maxFileSize = 0; + for (i = 0; i < nbFiles; i++) { + fileSize = UTIL_getFileSize(inFileNames[i]); + maxFileSize = fileSize > maxFileSize ? fileSize : maxFileSize; + } + return maxFileSize; +} + +/* FIO_compressMultipleFilenames() : + * compress nbFiles files + * into either one destination (outFileName), + * or into one file each (outFileName == NULL, but suffix != NULL), + * or into a destination folder (specified with -O) + */ +int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx, + FIO_prefs_t* const prefs, + const char** inFileNamesTable, + const char* outMirroredRootDirName, + const char* outDirName, + const char* outFileName, const char* suffix, + const char* dictFileName, int compressionLevel, + ZSTD_compressionParameters comprParams) +{ + int status; + int error = 0; + cRess_t ress = FIO_createCResources(prefs, dictFileName, + FIO_getLargestFileSize(inFileNamesTable, (unsigned)fCtx->nbFilesTotal), + compressionLevel, comprParams); + + /* init */ + assert(outFileName != NULL || suffix != NULL); + if (outFileName != NULL) { /* output into a single destination (stdout typically) */ + if (FIO_removeMultiFilesWarning(fCtx, prefs, outFileName, 1 /* displayLevelCutoff */)) { + FIO_freeCResources(&ress); + return 1; + } + ress.dstFile = FIO_openDstFile(fCtx, prefs, NULL, outFileName, DEFAULT_FILE_PERMISSIONS); + if (ress.dstFile == NULL) { /* could not open outFileName */ + error = 1; + } else { + for (; fCtx->currFileIdx < fCtx->nbFilesTotal; ++fCtx->currFileIdx) { + status = FIO_compressFilename_srcFile(fCtx, prefs, ress, outFileName, inFileNamesTable[fCtx->currFileIdx], compressionLevel); + if (!status) fCtx->nbFilesProcessed++; + error |= status; + } + if (fclose(ress.dstFile)) + EXM_THROW(29, "Write error (%s) : cannot properly close %s", + strerror(errno), outFileName); + ress.dstFile = NULL; + } + } else { + if (outMirroredRootDirName) + UTIL_mirrorSourceFilesDirectories(inFileNamesTable, (unsigned)fCtx->nbFilesTotal, outMirroredRootDirName); + + for (; fCtx->currFileIdx < fCtx->nbFilesTotal; ++fCtx->currFileIdx) { + const char* const srcFileName = inFileNamesTable[fCtx->currFileIdx]; + const char* dstFileName = NULL; + if (outMirroredRootDirName) { + char* validMirroredDirName = UTIL_createMirroredDestDirName(srcFileName, outMirroredRootDirName); + if (validMirroredDirName) { + dstFileName = FIO_determineCompressedName(srcFileName, validMirroredDirName, suffix); + free(validMirroredDirName); + } else { + DISPLAYLEVEL(2, "zstd: --output-dir-mirror cannot compress '%s' into '%s' \n", srcFileName, outMirroredRootDirName); + error=1; + continue; + } + } else { + dstFileName = FIO_determineCompressedName(srcFileName, outDirName, suffix); /* cannot fail */ + } + status = FIO_compressFilename_srcFile(fCtx, prefs, ress, dstFileName, srcFileName, compressionLevel); + if (!status) fCtx->nbFilesProcessed++; + error |= status; + } + + if (outDirName) + FIO_checkFilenameCollisions(inFileNamesTable , (unsigned)fCtx->nbFilesTotal); + } + + if (fCtx->nbFilesProcessed >= 1 && fCtx->nbFilesTotal > 1 && fCtx->totalBytesInput != 0) { UTIL_HumanReadableSize_t hr_isize = UTIL_makeHumanReadableSize((U64) fCtx->totalBytesInput); UTIL_HumanReadableSize_t hr_osize = UTIL_makeHumanReadableSize((U64) fCtx->totalBytesOutput); - DISPLAYLEVEL(2, "\r%79s\r", ""); + DISPLAYLEVEL(2, "\r%79s\r", ""); DISPLAYLEVEL(2, "%3d files compressed :%.2f%% (%6.*f%4s => %6.*f%4s)\n", fCtx->nbFilesProcessed, - (double)fCtx->totalBytesOutput/((double)fCtx->totalBytesInput)*100, + (double)fCtx->totalBytesOutput/((double)fCtx->totalBytesInput)*100, hr_isize.precision, hr_isize.value, hr_isize.suffix, hr_osize.precision, hr_osize.value, hr_osize.suffix); - } - - FIO_freeCResources(&ress); - return error; -} - -#endif /* #ifndef ZSTD_NOCOMPRESS */ - - - -#ifndef ZSTD_NODECOMPRESS - -/* ************************************************************************** - * Decompression - ***************************************************************************/ -typedef struct { - void* srcBuffer; - size_t srcBufferSize; - size_t srcBufferLoaded; - void* dstBuffer; - size_t dstBufferSize; - ZSTD_DStream* dctx; - FILE* dstFile; -} dRess_t; - -static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFileName) -{ - dRess_t ress; - memset(&ress, 0, sizeof(ress)); - - if (prefs->patchFromMode) - FIO_adjustMemLimitForPatchFromMode(prefs, UTIL_getFileSize(dictFileName), 0 /* just use the dict size */); - - /* Allocation */ - ress.dctx = ZSTD_createDStream(); - if (ress.dctx==NULL) - EXM_THROW(60, "Error: %s : can't create ZSTD_DStream", strerror(errno)); - CHECK( ZSTD_DCtx_setMaxWindowSize(ress.dctx, prefs->memLimit) ); - CHECK( ZSTD_DCtx_setParameter(ress.dctx, ZSTD_d_forceIgnoreChecksum, !prefs->checksumFlag)); - - ress.srcBufferSize = ZSTD_DStreamInSize(); - ress.srcBuffer = malloc(ress.srcBufferSize); - ress.dstBufferSize = ZSTD_DStreamOutSize(); - ress.dstBuffer = malloc(ress.dstBufferSize); - if (!ress.srcBuffer || !ress.dstBuffer) - EXM_THROW(61, "Allocation error : not enough memory"); - - /* dictionary */ - { void* dictBuffer; - size_t const dictBufferSize = FIO_createDictBuffer(&dictBuffer, dictFileName, prefs); - CHECK( ZSTD_initDStream_usingDict(ress.dctx, dictBuffer, dictBufferSize) ); - free(dictBuffer); - } - - return ress; -} - -static void FIO_freeDResources(dRess_t ress) -{ - CHECK( ZSTD_freeDStream(ress.dctx) ); - free(ress.srcBuffer); - free(ress.dstBuffer); -} - - -/** FIO_fwriteSparse() : -* @return : storedSkips, -* argument for next call to FIO_fwriteSparse() or FIO_fwriteSparseEnd() */ -static unsigned -FIO_fwriteSparse(FILE* file, - const void* buffer, size_t bufferSize, - const FIO_prefs_t* const prefs, - unsigned storedSkips) -{ - const size_t* const bufferT = (const size_t*)buffer; /* Buffer is supposed malloc'ed, hence aligned on size_t */ - size_t bufferSizeT = bufferSize / sizeof(size_t); - const size_t* const bufferTEnd = bufferT + bufferSizeT; - const size_t* ptrT = bufferT; - static const size_t segmentSizeT = (32 KB) / sizeof(size_t); /* check every 32 KB */ - - if (prefs->testMode) return 0; /* do not output anything in test mode */ - - if (!prefs->sparseFileSupport) { /* normal write */ - size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file); - if (sizeCheck != bufferSize) - EXM_THROW(70, "Write error : cannot write decoded block : %s", - strerror(errno)); - return 0; - } - - /* avoid int overflow */ - if (storedSkips > 1 GB) { - if (LONG_SEEK(file, 1 GB, SEEK_CUR) != 0) - EXM_THROW(91, "1 GB skip error (sparse file support)"); - storedSkips -= 1 GB; - } - - while (ptrT < bufferTEnd) { - size_t nb0T; - - /* adjust last segment if < 32 KB */ - size_t seg0SizeT = segmentSizeT; - if (seg0SizeT > bufferSizeT) seg0SizeT = bufferSizeT; - bufferSizeT -= seg0SizeT; - - /* count leading zeroes */ - for (nb0T=0; (nb0T < seg0SizeT) && (ptrT[nb0T] == 0); nb0T++) ; - storedSkips += (unsigned)(nb0T * sizeof(size_t)); - - if (nb0T != seg0SizeT) { /* not all 0s */ - size_t const nbNon0ST = seg0SizeT - nb0T; - /* skip leading zeros */ - if (LONG_SEEK(file, storedSkips, SEEK_CUR) != 0) - EXM_THROW(92, "Sparse skip error ; try --no-sparse"); - storedSkips = 0; - /* write the rest */ - if (fwrite(ptrT + nb0T, sizeof(size_t), nbNon0ST, file) != nbNon0ST) - EXM_THROW(93, "Write error : cannot write decoded block : %s", - strerror(errno)); - } - ptrT += seg0SizeT; - } - - { static size_t const maskT = sizeof(size_t)-1; - if (bufferSize & maskT) { - /* size not multiple of sizeof(size_t) : implies end of block */ - const char* const restStart = (const char*)bufferTEnd; - const char* restPtr = restStart; - const char* const restEnd = (const char*)buffer + bufferSize; - assert(restEnd > restStart && restEnd < restStart + sizeof(size_t)); - for ( ; (restPtr < restEnd) && (*restPtr == 0); restPtr++) ; - storedSkips += (unsigned) (restPtr - restStart); - if (restPtr != restEnd) { - /* not all remaining bytes are 0 */ - size_t const restSize = (size_t)(restEnd - restPtr); - if (LONG_SEEK(file, storedSkips, SEEK_CUR) != 0) - EXM_THROW(92, "Sparse skip error ; try --no-sparse"); - if (fwrite(restPtr, 1, restSize, file) != restSize) - EXM_THROW(95, "Write error : cannot write end of decoded block : %s", - strerror(errno)); - storedSkips = 0; - } } } - - return storedSkips; -} - -static void -FIO_fwriteSparseEnd(const FIO_prefs_t* const prefs, FILE* file, unsigned storedSkips) -{ - if (prefs->testMode) assert(storedSkips == 0); - if (storedSkips>0) { - assert(prefs->sparseFileSupport > 0); /* storedSkips>0 implies sparse support is enabled */ - (void)prefs; /* assert can be disabled, in which case prefs becomes unused */ - if (LONG_SEEK(file, storedSkips-1, SEEK_CUR) != 0) - EXM_THROW(69, "Final skip error (sparse file support)"); - /* last zero must be explicitly written, - * so that skipped ones get implicitly translated as zero by FS */ - { const char lastZeroByte[1] = { 0 }; - if (fwrite(lastZeroByte, 1, 1, file) != 1) - EXM_THROW(69, "Write error : cannot write last zero : %s", strerror(errno)); - } } -} - - -/** FIO_passThrough() : just copy input into output, for compatibility with gzip -df mode - @return : 0 (no error) */ -static int FIO_passThrough(const FIO_prefs_t* const prefs, - FILE* foutput, FILE* finput, - void* buffer, size_t bufferSize, - size_t alreadyLoaded) -{ - size_t const blockSize = MIN(64 KB, bufferSize); - size_t readFromInput; - unsigned storedSkips = 0; - - /* assumption : ress->srcBufferLoaded bytes already loaded and stored within buffer */ - { size_t const sizeCheck = fwrite(buffer, 1, alreadyLoaded, foutput); - if (sizeCheck != alreadyLoaded) { - DISPLAYLEVEL(1, "Pass-through write error : %s\n", strerror(errno)); - return 1; - } } - - do { - readFromInput = fread(buffer, 1, blockSize, finput); - storedSkips = FIO_fwriteSparse(foutput, buffer, readFromInput, prefs, storedSkips); - } while (readFromInput == blockSize); - if (ferror(finput)) { - DISPLAYLEVEL(1, "Pass-through read error : %s\n", strerror(errno)); - return 1; - } - assert(feof(finput)); - - FIO_fwriteSparseEnd(prefs, foutput, storedSkips); - return 0; -} - -/* FIO_zstdErrorHelp() : - * detailed error message when requested window size is too large */ -static void -FIO_zstdErrorHelp(const FIO_prefs_t* const prefs, - const dRess_t* ress, - size_t err, const char* srcFileName) -{ - ZSTD_frameHeader header; - - /* Help message only for one specific error */ - if (ZSTD_getErrorCode(err) != ZSTD_error_frameParameter_windowTooLarge) - return; - - /* Try to decode the frame header */ - err = ZSTD_getFrameHeader(&header, ress->srcBuffer, ress->srcBufferLoaded); - if (err == 0) { - unsigned long long const windowSize = header.windowSize; - unsigned const windowLog = FIO_highbit64(windowSize) + ((windowSize & (windowSize - 1)) != 0); - assert(prefs->memLimit > 0); - DISPLAYLEVEL(1, "%s : Window size larger than maximum : %llu > %u \n", - srcFileName, windowSize, prefs->memLimit); - if (windowLog <= ZSTD_WINDOWLOG_MAX) { - unsigned const windowMB = (unsigned)((windowSize >> 20) + ((windowSize & ((1 MB) - 1)) != 0)); - assert(windowSize < (U64)(1ULL << 52)); /* ensure now overflow for windowMB */ - DISPLAYLEVEL(1, "%s : Use --long=%u or --memory=%uMB \n", - srcFileName, windowLog, windowMB); - return; - } } - DISPLAYLEVEL(1, "%s : Window log larger than ZSTD_WINDOWLOG_MAX=%u; not supported \n", - srcFileName, ZSTD_WINDOWLOG_MAX); -} - -/** FIO_decompressFrame() : - * @return : size of decoded zstd frame, or an error code - */ -#define FIO_ERROR_FRAME_DECODING ((unsigned long long)(-2)) -static unsigned long long -FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress, FILE* finput, - const FIO_prefs_t* const prefs, - const char* srcFileName, - U64 alreadyDecoded) /* for multi-frames streams */ -{ - U64 frameSize = 0; - U32 storedSkips = 0; - - /* display last 20 characters only */ - { size_t const srcFileLength = strlen(srcFileName); - if (srcFileLength>20) srcFileName += srcFileLength-20; - } - - ZSTD_DCtx_reset(ress->dctx, ZSTD_reset_session_only); - - /* Header loading : ensures ZSTD_getFrameHeader() will succeed */ - { size_t const toDecode = ZSTD_FRAMEHEADERSIZE_MAX; - if (ress->srcBufferLoaded < toDecode) { - size_t const toRead = toDecode - ress->srcBufferLoaded; - void* const startPosition = (char*)ress->srcBuffer + ress->srcBufferLoaded; - ress->srcBufferLoaded += fread(startPosition, 1, toRead, finput); - } } - - /* Main decompression Loop */ - while (1) { - ZSTD_inBuffer inBuff = { ress->srcBuffer, ress->srcBufferLoaded, 0 }; - ZSTD_outBuffer outBuff= { ress->dstBuffer, ress->dstBufferSize, 0 }; - size_t const readSizeHint = ZSTD_decompressStream(ress->dctx, &outBuff, &inBuff); + } + + FIO_freeCResources(&ress); + return error; +} + +#endif /* #ifndef ZSTD_NOCOMPRESS */ + + + +#ifndef ZSTD_NODECOMPRESS + +/* ************************************************************************** + * Decompression + ***************************************************************************/ +typedef struct { + void* srcBuffer; + size_t srcBufferSize; + size_t srcBufferLoaded; + void* dstBuffer; + size_t dstBufferSize; + ZSTD_DStream* dctx; + FILE* dstFile; +} dRess_t; + +static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFileName) +{ + dRess_t ress; + memset(&ress, 0, sizeof(ress)); + + if (prefs->patchFromMode) + FIO_adjustMemLimitForPatchFromMode(prefs, UTIL_getFileSize(dictFileName), 0 /* just use the dict size */); + + /* Allocation */ + ress.dctx = ZSTD_createDStream(); + if (ress.dctx==NULL) + EXM_THROW(60, "Error: %s : can't create ZSTD_DStream", strerror(errno)); + CHECK( ZSTD_DCtx_setMaxWindowSize(ress.dctx, prefs->memLimit) ); + CHECK( ZSTD_DCtx_setParameter(ress.dctx, ZSTD_d_forceIgnoreChecksum, !prefs->checksumFlag)); + + ress.srcBufferSize = ZSTD_DStreamInSize(); + ress.srcBuffer = malloc(ress.srcBufferSize); + ress.dstBufferSize = ZSTD_DStreamOutSize(); + ress.dstBuffer = malloc(ress.dstBufferSize); + if (!ress.srcBuffer || !ress.dstBuffer) + EXM_THROW(61, "Allocation error : not enough memory"); + + /* dictionary */ + { void* dictBuffer; + size_t const dictBufferSize = FIO_createDictBuffer(&dictBuffer, dictFileName, prefs); + CHECK( ZSTD_initDStream_usingDict(ress.dctx, dictBuffer, dictBufferSize) ); + free(dictBuffer); + } + + return ress; +} + +static void FIO_freeDResources(dRess_t ress) +{ + CHECK( ZSTD_freeDStream(ress.dctx) ); + free(ress.srcBuffer); + free(ress.dstBuffer); +} + + +/** FIO_fwriteSparse() : +* @return : storedSkips, +* argument for next call to FIO_fwriteSparse() or FIO_fwriteSparseEnd() */ +static unsigned +FIO_fwriteSparse(FILE* file, + const void* buffer, size_t bufferSize, + const FIO_prefs_t* const prefs, + unsigned storedSkips) +{ + const size_t* const bufferT = (const size_t*)buffer; /* Buffer is supposed malloc'ed, hence aligned on size_t */ + size_t bufferSizeT = bufferSize / sizeof(size_t); + const size_t* const bufferTEnd = bufferT + bufferSizeT; + const size_t* ptrT = bufferT; + static const size_t segmentSizeT = (32 KB) / sizeof(size_t); /* check every 32 KB */ + + if (prefs->testMode) return 0; /* do not output anything in test mode */ + + if (!prefs->sparseFileSupport) { /* normal write */ + size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file); + if (sizeCheck != bufferSize) + EXM_THROW(70, "Write error : cannot write decoded block : %s", + strerror(errno)); + return 0; + } + + /* avoid int overflow */ + if (storedSkips > 1 GB) { + if (LONG_SEEK(file, 1 GB, SEEK_CUR) != 0) + EXM_THROW(91, "1 GB skip error (sparse file support)"); + storedSkips -= 1 GB; + } + + while (ptrT < bufferTEnd) { + size_t nb0T; + + /* adjust last segment if < 32 KB */ + size_t seg0SizeT = segmentSizeT; + if (seg0SizeT > bufferSizeT) seg0SizeT = bufferSizeT; + bufferSizeT -= seg0SizeT; + + /* count leading zeroes */ + for (nb0T=0; (nb0T < seg0SizeT) && (ptrT[nb0T] == 0); nb0T++) ; + storedSkips += (unsigned)(nb0T * sizeof(size_t)); + + if (nb0T != seg0SizeT) { /* not all 0s */ + size_t const nbNon0ST = seg0SizeT - nb0T; + /* skip leading zeros */ + if (LONG_SEEK(file, storedSkips, SEEK_CUR) != 0) + EXM_THROW(92, "Sparse skip error ; try --no-sparse"); + storedSkips = 0; + /* write the rest */ + if (fwrite(ptrT + nb0T, sizeof(size_t), nbNon0ST, file) != nbNon0ST) + EXM_THROW(93, "Write error : cannot write decoded block : %s", + strerror(errno)); + } + ptrT += seg0SizeT; + } + + { static size_t const maskT = sizeof(size_t)-1; + if (bufferSize & maskT) { + /* size not multiple of sizeof(size_t) : implies end of block */ + const char* const restStart = (const char*)bufferTEnd; + const char* restPtr = restStart; + const char* const restEnd = (const char*)buffer + bufferSize; + assert(restEnd > restStart && restEnd < restStart + sizeof(size_t)); + for ( ; (restPtr < restEnd) && (*restPtr == 0); restPtr++) ; + storedSkips += (unsigned) (restPtr - restStart); + if (restPtr != restEnd) { + /* not all remaining bytes are 0 */ + size_t const restSize = (size_t)(restEnd - restPtr); + if (LONG_SEEK(file, storedSkips, SEEK_CUR) != 0) + EXM_THROW(92, "Sparse skip error ; try --no-sparse"); + if (fwrite(restPtr, 1, restSize, file) != restSize) + EXM_THROW(95, "Write error : cannot write end of decoded block : %s", + strerror(errno)); + storedSkips = 0; + } } } + + return storedSkips; +} + +static void +FIO_fwriteSparseEnd(const FIO_prefs_t* const prefs, FILE* file, unsigned storedSkips) +{ + if (prefs->testMode) assert(storedSkips == 0); + if (storedSkips>0) { + assert(prefs->sparseFileSupport > 0); /* storedSkips>0 implies sparse support is enabled */ + (void)prefs; /* assert can be disabled, in which case prefs becomes unused */ + if (LONG_SEEK(file, storedSkips-1, SEEK_CUR) != 0) + EXM_THROW(69, "Final skip error (sparse file support)"); + /* last zero must be explicitly written, + * so that skipped ones get implicitly translated as zero by FS */ + { const char lastZeroByte[1] = { 0 }; + if (fwrite(lastZeroByte, 1, 1, file) != 1) + EXM_THROW(69, "Write error : cannot write last zero : %s", strerror(errno)); + } } +} + + +/** FIO_passThrough() : just copy input into output, for compatibility with gzip -df mode + @return : 0 (no error) */ +static int FIO_passThrough(const FIO_prefs_t* const prefs, + FILE* foutput, FILE* finput, + void* buffer, size_t bufferSize, + size_t alreadyLoaded) +{ + size_t const blockSize = MIN(64 KB, bufferSize); + size_t readFromInput; + unsigned storedSkips = 0; + + /* assumption : ress->srcBufferLoaded bytes already loaded and stored within buffer */ + { size_t const sizeCheck = fwrite(buffer, 1, alreadyLoaded, foutput); + if (sizeCheck != alreadyLoaded) { + DISPLAYLEVEL(1, "Pass-through write error : %s\n", strerror(errno)); + return 1; + } } + + do { + readFromInput = fread(buffer, 1, blockSize, finput); + storedSkips = FIO_fwriteSparse(foutput, buffer, readFromInput, prefs, storedSkips); + } while (readFromInput == blockSize); + if (ferror(finput)) { + DISPLAYLEVEL(1, "Pass-through read error : %s\n", strerror(errno)); + return 1; + } + assert(feof(finput)); + + FIO_fwriteSparseEnd(prefs, foutput, storedSkips); + return 0; +} + +/* FIO_zstdErrorHelp() : + * detailed error message when requested window size is too large */ +static void +FIO_zstdErrorHelp(const FIO_prefs_t* const prefs, + const dRess_t* ress, + size_t err, const char* srcFileName) +{ + ZSTD_frameHeader header; + + /* Help message only for one specific error */ + if (ZSTD_getErrorCode(err) != ZSTD_error_frameParameter_windowTooLarge) + return; + + /* Try to decode the frame header */ + err = ZSTD_getFrameHeader(&header, ress->srcBuffer, ress->srcBufferLoaded); + if (err == 0) { + unsigned long long const windowSize = header.windowSize; + unsigned const windowLog = FIO_highbit64(windowSize) + ((windowSize & (windowSize - 1)) != 0); + assert(prefs->memLimit > 0); + DISPLAYLEVEL(1, "%s : Window size larger than maximum : %llu > %u \n", + srcFileName, windowSize, prefs->memLimit); + if (windowLog <= ZSTD_WINDOWLOG_MAX) { + unsigned const windowMB = (unsigned)((windowSize >> 20) + ((windowSize & ((1 MB) - 1)) != 0)); + assert(windowSize < (U64)(1ULL << 52)); /* ensure now overflow for windowMB */ + DISPLAYLEVEL(1, "%s : Use --long=%u or --memory=%uMB \n", + srcFileName, windowLog, windowMB); + return; + } } + DISPLAYLEVEL(1, "%s : Window log larger than ZSTD_WINDOWLOG_MAX=%u; not supported \n", + srcFileName, ZSTD_WINDOWLOG_MAX); +} + +/** FIO_decompressFrame() : + * @return : size of decoded zstd frame, or an error code + */ +#define FIO_ERROR_FRAME_DECODING ((unsigned long long)(-2)) +static unsigned long long +FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress, FILE* finput, + const FIO_prefs_t* const prefs, + const char* srcFileName, + U64 alreadyDecoded) /* for multi-frames streams */ +{ + U64 frameSize = 0; + U32 storedSkips = 0; + + /* display last 20 characters only */ + { size_t const srcFileLength = strlen(srcFileName); + if (srcFileLength>20) srcFileName += srcFileLength-20; + } + + ZSTD_DCtx_reset(ress->dctx, ZSTD_reset_session_only); + + /* Header loading : ensures ZSTD_getFrameHeader() will succeed */ + { size_t const toDecode = ZSTD_FRAMEHEADERSIZE_MAX; + if (ress->srcBufferLoaded < toDecode) { + size_t const toRead = toDecode - ress->srcBufferLoaded; + void* const startPosition = (char*)ress->srcBuffer + ress->srcBufferLoaded; + ress->srcBufferLoaded += fread(startPosition, 1, toRead, finput); + } } + + /* Main decompression Loop */ + while (1) { + ZSTD_inBuffer inBuff = { ress->srcBuffer, ress->srcBufferLoaded, 0 }; + ZSTD_outBuffer outBuff= { ress->dstBuffer, ress->dstBufferSize, 0 }; + size_t const readSizeHint = ZSTD_decompressStream(ress->dctx, &outBuff, &inBuff); const int displayLevel = (g_display_prefs.progressSetting == FIO_ps_always) ? 1 : 2; UTIL_HumanReadableSize_t const hrs = UTIL_makeHumanReadableSize(alreadyDecoded+frameSize); - if (ZSTD_isError(readSizeHint)) { - DISPLAYLEVEL(1, "%s : Decoding error (36) : %s \n", - srcFileName, ZSTD_getErrorName(readSizeHint)); - FIO_zstdErrorHelp(prefs, ress, readSizeHint, srcFileName); - return FIO_ERROR_FRAME_DECODING; - } - - /* Write block */ - storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, outBuff.pos, prefs, storedSkips); - frameSize += outBuff.pos; + if (ZSTD_isError(readSizeHint)) { + DISPLAYLEVEL(1, "%s : Decoding error (36) : %s \n", + srcFileName, ZSTD_getErrorName(readSizeHint)); + FIO_zstdErrorHelp(prefs, ress, readSizeHint, srcFileName); + return FIO_ERROR_FRAME_DECODING; + } + + /* Write block */ + storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, outBuff.pos, prefs, storedSkips); + frameSize += outBuff.pos; if (fCtx->nbFilesTotal > 1) { size_t srcFileNameSize = strlen(srcFileName); if (srcFileNameSize > 18) { const char* truncatedSrcFileName = srcFileName + srcFileNameSize - 15; DISPLAYUPDATE(displayLevel, "\rDecompress: %2u/%2u files. Current: ...%s : %.*f%s... ", fCtx->currFileIdx+1, fCtx->nbFilesTotal, truncatedSrcFileName, hrs.precision, hrs.value, hrs.suffix); - } else { + } else { DISPLAYUPDATE(displayLevel, "\rDecompress: %2u/%2u files. Current: %s : %.*f%s... ", fCtx->currFileIdx+1, fCtx->nbFilesTotal, srcFileName, hrs.precision, hrs.value, hrs.suffix); - } + } } else { DISPLAYUPDATE(displayLevel, "\r%-20.20s : %.*f%s... ", srcFileName, hrs.precision, hrs.value, hrs.suffix); - } - - if (inBuff.pos > 0) { - memmove(ress->srcBuffer, (char*)ress->srcBuffer + inBuff.pos, inBuff.size - inBuff.pos); - ress->srcBufferLoaded -= inBuff.pos; - } - - if (readSizeHint == 0) break; /* end of frame */ - - /* Fill input buffer */ - { size_t const toDecode = MIN(readSizeHint, ress->srcBufferSize); /* support large skippable frames */ - if (ress->srcBufferLoaded < toDecode) { - size_t const toRead = toDecode - ress->srcBufferLoaded; /* > 0 */ - void* const startPosition = (char*)ress->srcBuffer + ress->srcBufferLoaded; - size_t const readSize = fread(startPosition, 1, toRead, finput); - if (readSize==0) { - DISPLAYLEVEL(1, "%s : Read error (39) : premature end \n", - srcFileName); - return FIO_ERROR_FRAME_DECODING; - } - ress->srcBufferLoaded += readSize; - } } } - - FIO_fwriteSparseEnd(prefs, ress->dstFile, storedSkips); - - return frameSize; -} - - -#ifdef ZSTD_GZDECOMPRESS -static unsigned long long -FIO_decompressGzFrame(dRess_t* ress, FILE* srcFile, - const FIO_prefs_t* const prefs, - const char* srcFileName) -{ - unsigned long long outFileSize = 0; - z_stream strm; - int flush = Z_NO_FLUSH; - int decodingError = 0; - unsigned storedSkips = 0; - - strm.zalloc = Z_NULL; - strm.zfree = Z_NULL; - strm.opaque = Z_NULL; - strm.next_in = 0; - strm.avail_in = 0; - /* see http://www.zlib.net/manual.html */ - if (inflateInit2(&strm, 15 /* maxWindowLogSize */ + 16 /* gzip only */) != Z_OK) - return FIO_ERROR_FRAME_DECODING; - - strm.next_out = (Bytef*)ress->dstBuffer; - strm.avail_out = (uInt)ress->dstBufferSize; - strm.avail_in = (uInt)ress->srcBufferLoaded; - strm.next_in = (z_const unsigned char*)ress->srcBuffer; - - for ( ; ; ) { - int ret; - if (strm.avail_in == 0) { - ress->srcBufferLoaded = fread(ress->srcBuffer, 1, ress->srcBufferSize, srcFile); - if (ress->srcBufferLoaded == 0) flush = Z_FINISH; - strm.next_in = (z_const unsigned char*)ress->srcBuffer; - strm.avail_in = (uInt)ress->srcBufferLoaded; - } - ret = inflate(&strm, flush); - if (ret == Z_BUF_ERROR) { - DISPLAYLEVEL(1, "zstd: %s: premature gz end \n", srcFileName); - decodingError = 1; break; - } - if (ret != Z_OK && ret != Z_STREAM_END) { - DISPLAYLEVEL(1, "zstd: %s: inflate error %d \n", srcFileName, ret); - decodingError = 1; break; - } - { size_t const decompBytes = ress->dstBufferSize - strm.avail_out; - if (decompBytes) { - storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, decompBytes, prefs, storedSkips); - outFileSize += decompBytes; - strm.next_out = (Bytef*)ress->dstBuffer; - strm.avail_out = (uInt)ress->dstBufferSize; - } - } - if (ret == Z_STREAM_END) break; - } - - if (strm.avail_in > 0) - memmove(ress->srcBuffer, strm.next_in, strm.avail_in); - ress->srcBufferLoaded = strm.avail_in; - if ( (inflateEnd(&strm) != Z_OK) /* release resources ; error detected */ - && (decodingError==0) ) { - DISPLAYLEVEL(1, "zstd: %s: inflateEnd error \n", srcFileName); - decodingError = 1; - } - FIO_fwriteSparseEnd(prefs, ress->dstFile, storedSkips); - return decodingError ? FIO_ERROR_FRAME_DECODING : outFileSize; -} -#endif - - -#ifdef ZSTD_LZMADECOMPRESS -static unsigned long long -FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile, - const FIO_prefs_t* const prefs, - const char* srcFileName, int plain_lzma) -{ - unsigned long long outFileSize = 0; - lzma_stream strm = LZMA_STREAM_INIT; - lzma_action action = LZMA_RUN; - lzma_ret initRet; - int decodingError = 0; - unsigned storedSkips = 0; - - strm.next_in = 0; - strm.avail_in = 0; - if (plain_lzma) { - initRet = lzma_alone_decoder(&strm, UINT64_MAX); /* LZMA */ - } else { - initRet = lzma_stream_decoder(&strm, UINT64_MAX, 0); /* XZ */ - } - - if (initRet != LZMA_OK) { - DISPLAYLEVEL(1, "zstd: %s: %s error %d \n", - plain_lzma ? "lzma_alone_decoder" : "lzma_stream_decoder", - srcFileName, initRet); - return FIO_ERROR_FRAME_DECODING; - } - - strm.next_out = (BYTE*)ress->dstBuffer; - strm.avail_out = ress->dstBufferSize; - strm.next_in = (BYTE const*)ress->srcBuffer; - strm.avail_in = ress->srcBufferLoaded; - - for ( ; ; ) { - lzma_ret ret; - if (strm.avail_in == 0) { - ress->srcBufferLoaded = fread(ress->srcBuffer, 1, ress->srcBufferSize, srcFile); - if (ress->srcBufferLoaded == 0) action = LZMA_FINISH; - strm.next_in = (BYTE const*)ress->srcBuffer; - strm.avail_in = ress->srcBufferLoaded; - } - ret = lzma_code(&strm, action); - - if (ret == LZMA_BUF_ERROR) { - DISPLAYLEVEL(1, "zstd: %s: premature lzma end \n", srcFileName); - decodingError = 1; break; - } - if (ret != LZMA_OK && ret != LZMA_STREAM_END) { - DISPLAYLEVEL(1, "zstd: %s: lzma_code decoding error %d \n", - srcFileName, ret); - decodingError = 1; break; - } - { size_t const decompBytes = ress->dstBufferSize - strm.avail_out; - if (decompBytes) { - storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, decompBytes, prefs, storedSkips); - outFileSize += decompBytes; - strm.next_out = (BYTE*)ress->dstBuffer; - strm.avail_out = ress->dstBufferSize; - } } - if (ret == LZMA_STREAM_END) break; - } - - if (strm.avail_in > 0) - memmove(ress->srcBuffer, strm.next_in, strm.avail_in); - ress->srcBufferLoaded = strm.avail_in; - lzma_end(&strm); - FIO_fwriteSparseEnd(prefs, ress->dstFile, storedSkips); - return decodingError ? FIO_ERROR_FRAME_DECODING : outFileSize; -} -#endif - -#ifdef ZSTD_LZ4DECOMPRESS -static unsigned long long -FIO_decompressLz4Frame(dRess_t* ress, FILE* srcFile, - const FIO_prefs_t* const prefs, - const char* srcFileName) -{ - unsigned long long filesize = 0; - LZ4F_errorCode_t nextToLoad; - LZ4F_decompressionContext_t dCtx; - LZ4F_errorCode_t const errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); - int decodingError = 0; - unsigned storedSkips = 0; - - if (LZ4F_isError(errorCode)) { - DISPLAYLEVEL(1, "zstd: failed to create lz4 decompression context \n"); - return FIO_ERROR_FRAME_DECODING; - } - - /* Init feed with magic number (already consumed from FILE* sFile) */ - { size_t inSize = 4; - size_t outSize= 0; - MEM_writeLE32(ress->srcBuffer, LZ4_MAGICNUMBER); - nextToLoad = LZ4F_decompress(dCtx, ress->dstBuffer, &outSize, ress->srcBuffer, &inSize, NULL); - if (LZ4F_isError(nextToLoad)) { - DISPLAYLEVEL(1, "zstd: %s: lz4 header error : %s \n", - srcFileName, LZ4F_getErrorName(nextToLoad)); - LZ4F_freeDecompressionContext(dCtx); - return FIO_ERROR_FRAME_DECODING; - } } - - /* Main Loop */ - for (;nextToLoad;) { - size_t readSize; - size_t pos = 0; - size_t decodedBytes = ress->dstBufferSize; - - /* Read input */ - if (nextToLoad > ress->srcBufferSize) nextToLoad = ress->srcBufferSize; - readSize = fread(ress->srcBuffer, 1, nextToLoad, srcFile); - if (!readSize) break; /* reached end of file or stream */ - - while ((pos < readSize) || (decodedBytes == ress->dstBufferSize)) { /* still to read, or still to flush */ - /* Decode Input (at least partially) */ - size_t remaining = readSize - pos; - decodedBytes = ress->dstBufferSize; - nextToLoad = LZ4F_decompress(dCtx, ress->dstBuffer, &decodedBytes, (char*)(ress->srcBuffer)+pos, &remaining, NULL); - if (LZ4F_isError(nextToLoad)) { - DISPLAYLEVEL(1, "zstd: %s: lz4 decompression error : %s \n", - srcFileName, LZ4F_getErrorName(nextToLoad)); - decodingError = 1; nextToLoad = 0; break; - } - pos += remaining; - - /* Write Block */ - if (decodedBytes) { + } + + if (inBuff.pos > 0) { + memmove(ress->srcBuffer, (char*)ress->srcBuffer + inBuff.pos, inBuff.size - inBuff.pos); + ress->srcBufferLoaded -= inBuff.pos; + } + + if (readSizeHint == 0) break; /* end of frame */ + + /* Fill input buffer */ + { size_t const toDecode = MIN(readSizeHint, ress->srcBufferSize); /* support large skippable frames */ + if (ress->srcBufferLoaded < toDecode) { + size_t const toRead = toDecode - ress->srcBufferLoaded; /* > 0 */ + void* const startPosition = (char*)ress->srcBuffer + ress->srcBufferLoaded; + size_t const readSize = fread(startPosition, 1, toRead, finput); + if (readSize==0) { + DISPLAYLEVEL(1, "%s : Read error (39) : premature end \n", + srcFileName); + return FIO_ERROR_FRAME_DECODING; + } + ress->srcBufferLoaded += readSize; + } } } + + FIO_fwriteSparseEnd(prefs, ress->dstFile, storedSkips); + + return frameSize; +} + + +#ifdef ZSTD_GZDECOMPRESS +static unsigned long long +FIO_decompressGzFrame(dRess_t* ress, FILE* srcFile, + const FIO_prefs_t* const prefs, + const char* srcFileName) +{ + unsigned long long outFileSize = 0; + z_stream strm; + int flush = Z_NO_FLUSH; + int decodingError = 0; + unsigned storedSkips = 0; + + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + strm.next_in = 0; + strm.avail_in = 0; + /* see http://www.zlib.net/manual.html */ + if (inflateInit2(&strm, 15 /* maxWindowLogSize */ + 16 /* gzip only */) != Z_OK) + return FIO_ERROR_FRAME_DECODING; + + strm.next_out = (Bytef*)ress->dstBuffer; + strm.avail_out = (uInt)ress->dstBufferSize; + strm.avail_in = (uInt)ress->srcBufferLoaded; + strm.next_in = (z_const unsigned char*)ress->srcBuffer; + + for ( ; ; ) { + int ret; + if (strm.avail_in == 0) { + ress->srcBufferLoaded = fread(ress->srcBuffer, 1, ress->srcBufferSize, srcFile); + if (ress->srcBufferLoaded == 0) flush = Z_FINISH; + strm.next_in = (z_const unsigned char*)ress->srcBuffer; + strm.avail_in = (uInt)ress->srcBufferLoaded; + } + ret = inflate(&strm, flush); + if (ret == Z_BUF_ERROR) { + DISPLAYLEVEL(1, "zstd: %s: premature gz end \n", srcFileName); + decodingError = 1; break; + } + if (ret != Z_OK && ret != Z_STREAM_END) { + DISPLAYLEVEL(1, "zstd: %s: inflate error %d \n", srcFileName, ret); + decodingError = 1; break; + } + { size_t const decompBytes = ress->dstBufferSize - strm.avail_out; + if (decompBytes) { + storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, decompBytes, prefs, storedSkips); + outFileSize += decompBytes; + strm.next_out = (Bytef*)ress->dstBuffer; + strm.avail_out = (uInt)ress->dstBufferSize; + } + } + if (ret == Z_STREAM_END) break; + } + + if (strm.avail_in > 0) + memmove(ress->srcBuffer, strm.next_in, strm.avail_in); + ress->srcBufferLoaded = strm.avail_in; + if ( (inflateEnd(&strm) != Z_OK) /* release resources ; error detected */ + && (decodingError==0) ) { + DISPLAYLEVEL(1, "zstd: %s: inflateEnd error \n", srcFileName); + decodingError = 1; + } + FIO_fwriteSparseEnd(prefs, ress->dstFile, storedSkips); + return decodingError ? FIO_ERROR_FRAME_DECODING : outFileSize; +} +#endif + + +#ifdef ZSTD_LZMADECOMPRESS +static unsigned long long +FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile, + const FIO_prefs_t* const prefs, + const char* srcFileName, int plain_lzma) +{ + unsigned long long outFileSize = 0; + lzma_stream strm = LZMA_STREAM_INIT; + lzma_action action = LZMA_RUN; + lzma_ret initRet; + int decodingError = 0; + unsigned storedSkips = 0; + + strm.next_in = 0; + strm.avail_in = 0; + if (plain_lzma) { + initRet = lzma_alone_decoder(&strm, UINT64_MAX); /* LZMA */ + } else { + initRet = lzma_stream_decoder(&strm, UINT64_MAX, 0); /* XZ */ + } + + if (initRet != LZMA_OK) { + DISPLAYLEVEL(1, "zstd: %s: %s error %d \n", + plain_lzma ? "lzma_alone_decoder" : "lzma_stream_decoder", + srcFileName, initRet); + return FIO_ERROR_FRAME_DECODING; + } + + strm.next_out = (BYTE*)ress->dstBuffer; + strm.avail_out = ress->dstBufferSize; + strm.next_in = (BYTE const*)ress->srcBuffer; + strm.avail_in = ress->srcBufferLoaded; + + for ( ; ; ) { + lzma_ret ret; + if (strm.avail_in == 0) { + ress->srcBufferLoaded = fread(ress->srcBuffer, 1, ress->srcBufferSize, srcFile); + if (ress->srcBufferLoaded == 0) action = LZMA_FINISH; + strm.next_in = (BYTE const*)ress->srcBuffer; + strm.avail_in = ress->srcBufferLoaded; + } + ret = lzma_code(&strm, action); + + if (ret == LZMA_BUF_ERROR) { + DISPLAYLEVEL(1, "zstd: %s: premature lzma end \n", srcFileName); + decodingError = 1; break; + } + if (ret != LZMA_OK && ret != LZMA_STREAM_END) { + DISPLAYLEVEL(1, "zstd: %s: lzma_code decoding error %d \n", + srcFileName, ret); + decodingError = 1; break; + } + { size_t const decompBytes = ress->dstBufferSize - strm.avail_out; + if (decompBytes) { + storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, decompBytes, prefs, storedSkips); + outFileSize += decompBytes; + strm.next_out = (BYTE*)ress->dstBuffer; + strm.avail_out = ress->dstBufferSize; + } } + if (ret == LZMA_STREAM_END) break; + } + + if (strm.avail_in > 0) + memmove(ress->srcBuffer, strm.next_in, strm.avail_in); + ress->srcBufferLoaded = strm.avail_in; + lzma_end(&strm); + FIO_fwriteSparseEnd(prefs, ress->dstFile, storedSkips); + return decodingError ? FIO_ERROR_FRAME_DECODING : outFileSize; +} +#endif + +#ifdef ZSTD_LZ4DECOMPRESS +static unsigned long long +FIO_decompressLz4Frame(dRess_t* ress, FILE* srcFile, + const FIO_prefs_t* const prefs, + const char* srcFileName) +{ + unsigned long long filesize = 0; + LZ4F_errorCode_t nextToLoad; + LZ4F_decompressionContext_t dCtx; + LZ4F_errorCode_t const errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION); + int decodingError = 0; + unsigned storedSkips = 0; + + if (LZ4F_isError(errorCode)) { + DISPLAYLEVEL(1, "zstd: failed to create lz4 decompression context \n"); + return FIO_ERROR_FRAME_DECODING; + } + + /* Init feed with magic number (already consumed from FILE* sFile) */ + { size_t inSize = 4; + size_t outSize= 0; + MEM_writeLE32(ress->srcBuffer, LZ4_MAGICNUMBER); + nextToLoad = LZ4F_decompress(dCtx, ress->dstBuffer, &outSize, ress->srcBuffer, &inSize, NULL); + if (LZ4F_isError(nextToLoad)) { + DISPLAYLEVEL(1, "zstd: %s: lz4 header error : %s \n", + srcFileName, LZ4F_getErrorName(nextToLoad)); + LZ4F_freeDecompressionContext(dCtx); + return FIO_ERROR_FRAME_DECODING; + } } + + /* Main Loop */ + for (;nextToLoad;) { + size_t readSize; + size_t pos = 0; + size_t decodedBytes = ress->dstBufferSize; + + /* Read input */ + if (nextToLoad > ress->srcBufferSize) nextToLoad = ress->srcBufferSize; + readSize = fread(ress->srcBuffer, 1, nextToLoad, srcFile); + if (!readSize) break; /* reached end of file or stream */ + + while ((pos < readSize) || (decodedBytes == ress->dstBufferSize)) { /* still to read, or still to flush */ + /* Decode Input (at least partially) */ + size_t remaining = readSize - pos; + decodedBytes = ress->dstBufferSize; + nextToLoad = LZ4F_decompress(dCtx, ress->dstBuffer, &decodedBytes, (char*)(ress->srcBuffer)+pos, &remaining, NULL); + if (LZ4F_isError(nextToLoad)) { + DISPLAYLEVEL(1, "zstd: %s: lz4 decompression error : %s \n", + srcFileName, LZ4F_getErrorName(nextToLoad)); + decodingError = 1; nextToLoad = 0; break; + } + pos += remaining; + + /* Write Block */ + if (decodedBytes) { UTIL_HumanReadableSize_t hrs; - storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, decodedBytes, prefs, storedSkips); - filesize += decodedBytes; + storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, decodedBytes, prefs, storedSkips); + filesize += decodedBytes; hrs = UTIL_makeHumanReadableSize(filesize); DISPLAYUPDATE(2, "\rDecompressed : %.*f%s ", hrs.precision, hrs.value, hrs.suffix); - } - - if (!nextToLoad) break; - } - } - /* can be out because readSize == 0, which could be an fread() error */ - if (ferror(srcFile)) { - DISPLAYLEVEL(1, "zstd: %s: read error \n", srcFileName); - decodingError=1; - } - - if (nextToLoad!=0) { - DISPLAYLEVEL(1, "zstd: %s: unfinished lz4 stream \n", srcFileName); - decodingError=1; - } - - LZ4F_freeDecompressionContext(dCtx); - ress->srcBufferLoaded = 0; /* LZ4F will reach exact frame boundary */ - FIO_fwriteSparseEnd(prefs, ress->dstFile, storedSkips); - - return decodingError ? FIO_ERROR_FRAME_DECODING : filesize; -} -#endif - - - -/** FIO_decompressFrames() : - * Find and decode frames inside srcFile - * srcFile presumed opened and valid - * @return : 0 : OK - * 1 : error - */ -static int FIO_decompressFrames(FIO_ctx_t* const fCtx, - dRess_t ress, FILE* srcFile, - const FIO_prefs_t* const prefs, - const char* dstFileName, const char* srcFileName) -{ - unsigned readSomething = 0; - unsigned long long filesize = 0; - assert(srcFile != NULL); - - /* for each frame */ - for ( ; ; ) { - /* check magic number -> version */ - size_t const toRead = 4; - const BYTE* const buf = (const BYTE*)ress.srcBuffer; - if (ress.srcBufferLoaded < toRead) /* load up to 4 bytes for header */ - ress.srcBufferLoaded += fread((char*)ress.srcBuffer + ress.srcBufferLoaded, - (size_t)1, toRead - ress.srcBufferLoaded, srcFile); - if (ress.srcBufferLoaded==0) { - if (readSomething==0) { /* srcFile is empty (which is invalid) */ - DISPLAYLEVEL(1, "zstd: %s: unexpected end of file \n", srcFileName); - return 1; - } /* else, just reached frame boundary */ - break; /* no more input */ - } - readSomething = 1; /* there is at least 1 byte in srcFile */ - if (ress.srcBufferLoaded < toRead) { - DISPLAYLEVEL(1, "zstd: %s: unknown header \n", srcFileName); - return 1; - } - if (ZSTD_isFrame(buf, ress.srcBufferLoaded)) { - unsigned long long const frameSize = FIO_decompressZstdFrame(fCtx, &ress, srcFile, prefs, srcFileName, filesize); - if (frameSize == FIO_ERROR_FRAME_DECODING) return 1; - filesize += frameSize; - } else if (buf[0] == 31 && buf[1] == 139) { /* gz magic number */ -#ifdef ZSTD_GZDECOMPRESS - unsigned long long const frameSize = FIO_decompressGzFrame(&ress, srcFile, prefs, srcFileName); - if (frameSize == FIO_ERROR_FRAME_DECODING) return 1; - filesize += frameSize; -#else - DISPLAYLEVEL(1, "zstd: %s: gzip file cannot be uncompressed (zstd compiled without HAVE_ZLIB) -- ignored \n", srcFileName); - return 1; -#endif - } else if ((buf[0] == 0xFD && buf[1] == 0x37) /* xz magic number */ - || (buf[0] == 0x5D && buf[1] == 0x00)) { /* lzma header (no magic number) */ -#ifdef ZSTD_LZMADECOMPRESS - unsigned long long const frameSize = FIO_decompressLzmaFrame(&ress, srcFile, prefs, srcFileName, buf[0] != 0xFD); - if (frameSize == FIO_ERROR_FRAME_DECODING) return 1; - filesize += frameSize; -#else - DISPLAYLEVEL(1, "zstd: %s: xz/lzma file cannot be uncompressed (zstd compiled without HAVE_LZMA) -- ignored \n", srcFileName); - return 1; -#endif - } else if (MEM_readLE32(buf) == LZ4_MAGICNUMBER) { -#ifdef ZSTD_LZ4DECOMPRESS - unsigned long long const frameSize = FIO_decompressLz4Frame(&ress, srcFile, prefs, srcFileName); - if (frameSize == FIO_ERROR_FRAME_DECODING) return 1; - filesize += frameSize; -#else - DISPLAYLEVEL(1, "zstd: %s: lz4 file cannot be uncompressed (zstd compiled without HAVE_LZ4) -- ignored \n", srcFileName); - return 1; -#endif - } else if ((prefs->overwrite) && !strcmp (dstFileName, stdoutmark)) { /* pass-through mode */ - return FIO_passThrough(prefs, - ress.dstFile, srcFile, - ress.srcBuffer, ress.srcBufferSize, - ress.srcBufferLoaded); - } else { - DISPLAYLEVEL(1, "zstd: %s: unsupported format \n", srcFileName); - return 1; - } } /* for each frame */ - - /* Final Status */ - fCtx->totalBytesOutput += (size_t)filesize; - DISPLAYLEVEL(2, "\r%79s\r", ""); - /* No status message in pipe mode (stdin - stdout) or multi-files mode */ + } + + if (!nextToLoad) break; + } + } + /* can be out because readSize == 0, which could be an fread() error */ + if (ferror(srcFile)) { + DISPLAYLEVEL(1, "zstd: %s: read error \n", srcFileName); + decodingError=1; + } + + if (nextToLoad!=0) { + DISPLAYLEVEL(1, "zstd: %s: unfinished lz4 stream \n", srcFileName); + decodingError=1; + } + + LZ4F_freeDecompressionContext(dCtx); + ress->srcBufferLoaded = 0; /* LZ4F will reach exact frame boundary */ + FIO_fwriteSparseEnd(prefs, ress->dstFile, storedSkips); + + return decodingError ? FIO_ERROR_FRAME_DECODING : filesize; +} +#endif + + + +/** FIO_decompressFrames() : + * Find and decode frames inside srcFile + * srcFile presumed opened and valid + * @return : 0 : OK + * 1 : error + */ +static int FIO_decompressFrames(FIO_ctx_t* const fCtx, + dRess_t ress, FILE* srcFile, + const FIO_prefs_t* const prefs, + const char* dstFileName, const char* srcFileName) +{ + unsigned readSomething = 0; + unsigned long long filesize = 0; + assert(srcFile != NULL); + + /* for each frame */ + for ( ; ; ) { + /* check magic number -> version */ + size_t const toRead = 4; + const BYTE* const buf = (const BYTE*)ress.srcBuffer; + if (ress.srcBufferLoaded < toRead) /* load up to 4 bytes for header */ + ress.srcBufferLoaded += fread((char*)ress.srcBuffer + ress.srcBufferLoaded, + (size_t)1, toRead - ress.srcBufferLoaded, srcFile); + if (ress.srcBufferLoaded==0) { + if (readSomething==0) { /* srcFile is empty (which is invalid) */ + DISPLAYLEVEL(1, "zstd: %s: unexpected end of file \n", srcFileName); + return 1; + } /* else, just reached frame boundary */ + break; /* no more input */ + } + readSomething = 1; /* there is at least 1 byte in srcFile */ + if (ress.srcBufferLoaded < toRead) { + DISPLAYLEVEL(1, "zstd: %s: unknown header \n", srcFileName); + return 1; + } + if (ZSTD_isFrame(buf, ress.srcBufferLoaded)) { + unsigned long long const frameSize = FIO_decompressZstdFrame(fCtx, &ress, srcFile, prefs, srcFileName, filesize); + if (frameSize == FIO_ERROR_FRAME_DECODING) return 1; + filesize += frameSize; + } else if (buf[0] == 31 && buf[1] == 139) { /* gz magic number */ +#ifdef ZSTD_GZDECOMPRESS + unsigned long long const frameSize = FIO_decompressGzFrame(&ress, srcFile, prefs, srcFileName); + if (frameSize == FIO_ERROR_FRAME_DECODING) return 1; + filesize += frameSize; +#else + DISPLAYLEVEL(1, "zstd: %s: gzip file cannot be uncompressed (zstd compiled without HAVE_ZLIB) -- ignored \n", srcFileName); + return 1; +#endif + } else if ((buf[0] == 0xFD && buf[1] == 0x37) /* xz magic number */ + || (buf[0] == 0x5D && buf[1] == 0x00)) { /* lzma header (no magic number) */ +#ifdef ZSTD_LZMADECOMPRESS + unsigned long long const frameSize = FIO_decompressLzmaFrame(&ress, srcFile, prefs, srcFileName, buf[0] != 0xFD); + if (frameSize == FIO_ERROR_FRAME_DECODING) return 1; + filesize += frameSize; +#else + DISPLAYLEVEL(1, "zstd: %s: xz/lzma file cannot be uncompressed (zstd compiled without HAVE_LZMA) -- ignored \n", srcFileName); + return 1; +#endif + } else if (MEM_readLE32(buf) == LZ4_MAGICNUMBER) { +#ifdef ZSTD_LZ4DECOMPRESS + unsigned long long const frameSize = FIO_decompressLz4Frame(&ress, srcFile, prefs, srcFileName); + if (frameSize == FIO_ERROR_FRAME_DECODING) return 1; + filesize += frameSize; +#else + DISPLAYLEVEL(1, "zstd: %s: lz4 file cannot be uncompressed (zstd compiled without HAVE_LZ4) -- ignored \n", srcFileName); + return 1; +#endif + } else if ((prefs->overwrite) && !strcmp (dstFileName, stdoutmark)) { /* pass-through mode */ + return FIO_passThrough(prefs, + ress.dstFile, srcFile, + ress.srcBuffer, ress.srcBufferSize, + ress.srcBufferLoaded); + } else { + DISPLAYLEVEL(1, "zstd: %s: unsupported format \n", srcFileName); + return 1; + } } /* for each frame */ + + /* Final Status */ + fCtx->totalBytesOutput += (size_t)filesize; + DISPLAYLEVEL(2, "\r%79s\r", ""); + /* No status message in pipe mode (stdin - stdout) or multi-files mode */ if ((g_display_prefs.displayLevel >= 2 && fCtx->nbFilesTotal <= 1) || g_display_prefs.displayLevel >= 3 || g_display_prefs.progressSetting == FIO_ps_always) { DISPLAYLEVEL(1, "\r%-20s: %llu bytes \n", srcFileName, filesize); - } - - return 0; -} - -/** FIO_decompressDstFile() : - open `dstFileName`, - or path-through if ress.dstFile is already != 0, - then start decompression process (FIO_decompressFrames()). - @return : 0 : OK - 1 : operation aborted -*/ -static int FIO_decompressDstFile(FIO_ctx_t* const fCtx, - FIO_prefs_t* const prefs, - dRess_t ress, FILE* srcFile, - const char* dstFileName, const char* srcFileName) -{ - int result; - stat_t statbuf; - int releaseDstFile = 0; + } + + return 0; +} + +/** FIO_decompressDstFile() : + open `dstFileName`, + or path-through if ress.dstFile is already != 0, + then start decompression process (FIO_decompressFrames()). + @return : 0 : OK + 1 : operation aborted +*/ +static int FIO_decompressDstFile(FIO_ctx_t* const fCtx, + FIO_prefs_t* const prefs, + dRess_t ress, FILE* srcFile, + const char* dstFileName, const char* srcFileName) +{ + int result; + stat_t statbuf; + int releaseDstFile = 0; int transferMTime = 0; - - if ((ress.dstFile == NULL) && (prefs->testMode==0)) { - int dstFilePermissions = DEFAULT_FILE_PERMISSIONS; - if ( strcmp(srcFileName, stdinmark) /* special case : don't transfer permissions from stdin */ + + if ((ress.dstFile == NULL) && (prefs->testMode==0)) { + int dstFilePermissions = DEFAULT_FILE_PERMISSIONS; + if ( strcmp(srcFileName, stdinmark) /* special case : don't transfer permissions from stdin */ && strcmp(dstFileName, stdoutmark) - && UTIL_stat(srcFileName, &statbuf) - && UTIL_isRegularFileStat(&statbuf) ) { - dstFilePermissions = statbuf.st_mode; + && UTIL_stat(srcFileName, &statbuf) + && UTIL_isRegularFileStat(&statbuf) ) { + dstFilePermissions = statbuf.st_mode; transferMTime = 1; - } - - releaseDstFile = 1; - - ress.dstFile = FIO_openDstFile(fCtx, prefs, srcFileName, dstFileName, dstFilePermissions); - if (ress.dstFile==NULL) return 1; - - /* Must only be added after FIO_openDstFile() succeeds. - * Otherwise we may delete the destination file if it already exists, - * and the user presses Ctrl-C when asked if they wish to overwrite. - */ - addHandler(dstFileName); - } - - result = FIO_decompressFrames(fCtx, ress, srcFile, prefs, dstFileName, srcFileName); - - if (releaseDstFile) { - FILE* const dstFile = ress.dstFile; - clearHandler(); - ress.dstFile = NULL; - if (fclose(dstFile)) { - DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno)); - result = 1; - } - + } + + releaseDstFile = 1; + + ress.dstFile = FIO_openDstFile(fCtx, prefs, srcFileName, dstFileName, dstFilePermissions); + if (ress.dstFile==NULL) return 1; + + /* Must only be added after FIO_openDstFile() succeeds. + * Otherwise we may delete the destination file if it already exists, + * and the user presses Ctrl-C when asked if they wish to overwrite. + */ + addHandler(dstFileName); + } + + result = FIO_decompressFrames(fCtx, ress, srcFile, prefs, dstFileName, srcFileName); + + if (releaseDstFile) { + FILE* const dstFile = ress.dstFile; + clearHandler(); + ress.dstFile = NULL; + if (fclose(dstFile)) { + DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno)); + result = 1; + } + if (transferMTime) { UTIL_utime(dstFileName, &statbuf); } - if ( (result != 0) /* operation failure */ - && strcmp(dstFileName, stdoutmark) /* special case : don't remove() stdout */ - ) { - FIO_removeFile(dstFileName); /* remove decompression artefact; note: don't do anything special if remove() fails */ - } - } - - return result; -} - - -/** FIO_decompressSrcFile() : - Open `srcFileName`, transfer control to decompressDstFile() - @return : 0 : OK - 1 : error -*/ -static int FIO_decompressSrcFile(FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs, dRess_t ress, const char* dstFileName, const char* srcFileName) -{ - FILE* srcFile; - int result; - - if (UTIL_isDirectory(srcFileName)) { - DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName); - return 1; - } - - srcFile = FIO_openSrcFile(prefs, srcFileName); - if (srcFile==NULL) return 1; - ress.srcBufferLoaded = 0; - - result = FIO_decompressDstFile(fCtx, prefs, ress, srcFile, dstFileName, srcFileName); - - /* Close file */ - if (fclose(srcFile)) { - DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno)); /* error should not happen */ - return 1; - } - if ( prefs->removeSrcFile /* --rm */ - && (result==0) /* decompression successful */ - && strcmp(srcFileName, stdinmark) ) /* not stdin */ { - /* We must clear the handler, since after this point calling it would - * delete both the source and destination files. - */ - clearHandler(); - if (FIO_removeFile(srcFileName)) { - /* failed to remove src file */ - DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno)); - return 1; - } } - return result; -} - - - -int FIO_decompressFilename(FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs, - const char* dstFileName, const char* srcFileName, - const char* dictFileName) -{ - dRess_t const ress = FIO_createDResources(prefs, dictFileName); - - int const decodingError = FIO_decompressSrcFile(fCtx, prefs, ress, dstFileName, srcFileName); - - FIO_freeDResources(ress); - return decodingError; -} - -static const char *suffixList[] = { - ZSTD_EXTENSION, - TZSTD_EXTENSION, -#ifndef ZSTD_NODECOMPRESS - ZSTD_ALT_EXTENSION, -#endif -#ifdef ZSTD_GZDECOMPRESS - GZ_EXTENSION, - TGZ_EXTENSION, -#endif -#ifdef ZSTD_LZMADECOMPRESS - LZMA_EXTENSION, - XZ_EXTENSION, - TXZ_EXTENSION, -#endif -#ifdef ZSTD_LZ4DECOMPRESS - LZ4_EXTENSION, - TLZ4_EXTENSION, -#endif - NULL -}; - -static const char *suffixListStr = - ZSTD_EXTENSION "/" TZSTD_EXTENSION -#ifdef ZSTD_GZDECOMPRESS - "/" GZ_EXTENSION "/" TGZ_EXTENSION -#endif -#ifdef ZSTD_LZMADECOMPRESS - "/" LZMA_EXTENSION "/" XZ_EXTENSION "/" TXZ_EXTENSION -#endif -#ifdef ZSTD_LZ4DECOMPRESS - "/" LZ4_EXTENSION "/" TLZ4_EXTENSION -#endif -; - -/* FIO_determineDstName() : - * create a destination filename from a srcFileName. - * @return a pointer to it. - * @return == NULL if there is an error */ -static const char* -FIO_determineDstName(const char* srcFileName, const char* outDirName) -{ - static size_t dfnbCapacity = 0; - static char* dstFileNameBuffer = NULL; /* using static allocation : this function cannot be multi-threaded */ - size_t dstFileNameEndPos; - char* outDirFilename = NULL; - const char* dstSuffix = ""; - size_t dstSuffixLen = 0; - - size_t sfnSize = strlen(srcFileName); - - size_t srcSuffixLen; - const char* const srcSuffix = strrchr(srcFileName, '.'); - if (srcSuffix == NULL) { - DISPLAYLEVEL(1, - "zstd: %s: unknown suffix (%s expected). " - "Can't derive the output file name. " - "Specify it with -o dstFileName. Ignoring.\n", - srcFileName, suffixListStr); - return NULL; - } - srcSuffixLen = strlen(srcSuffix); - - { - const char** matchedSuffixPtr; - for (matchedSuffixPtr = suffixList; *matchedSuffixPtr != NULL; matchedSuffixPtr++) { - if (!strcmp(*matchedSuffixPtr, srcSuffix)) { - break; - } - } - - /* check suffix is authorized */ - if (sfnSize <= srcSuffixLen || *matchedSuffixPtr == NULL) { - DISPLAYLEVEL(1, - "zstd: %s: unknown suffix (%s expected). " - "Can't derive the output file name. " - "Specify it with -o dstFileName. Ignoring.\n", - srcFileName, suffixListStr); - return NULL; - } - - if ((*matchedSuffixPtr)[1] == 't') { - dstSuffix = ".tar"; - dstSuffixLen = strlen(dstSuffix); - } - } - - if (outDirName) { - outDirFilename = FIO_createFilename_fromOutDir(srcFileName, outDirName, 0); - sfnSize = strlen(outDirFilename); - assert(outDirFilename != NULL); - } - - if (dfnbCapacity+srcSuffixLen <= sfnSize+1+dstSuffixLen) { - /* allocate enough space to write dstFilename into it */ - free(dstFileNameBuffer); - dfnbCapacity = sfnSize + 20; - dstFileNameBuffer = (char*)malloc(dfnbCapacity); - if (dstFileNameBuffer==NULL) - EXM_THROW(74, "%s : not enough memory for dstFileName", - strerror(errno)); - } - - /* return dst name == src name truncated from suffix */ - assert(dstFileNameBuffer != NULL); - dstFileNameEndPos = sfnSize - srcSuffixLen; - if (outDirFilename) { - memcpy(dstFileNameBuffer, outDirFilename, dstFileNameEndPos); - free(outDirFilename); - } else { - memcpy(dstFileNameBuffer, srcFileName, dstFileNameEndPos); - } - - /* The short tar extensions tzst, tgz, txz and tlz4 files should have "tar" - * extension on decompression. Also writes terminating null. */ - strcpy(dstFileNameBuffer + dstFileNameEndPos, dstSuffix); - return dstFileNameBuffer; - - /* note : dstFileNameBuffer memory is not going to be free */ -} - -int -FIO_decompressMultipleFilenames(FIO_ctx_t* const fCtx, - FIO_prefs_t* const prefs, - const char** srcNamesTable, - const char* outMirroredRootDirName, - const char* outDirName, const char* outFileName, - const char* dictFileName) -{ - int status; - int error = 0; - dRess_t ress = FIO_createDResources(prefs, dictFileName); - - if (outFileName) { - if (FIO_removeMultiFilesWarning(fCtx, prefs, outFileName, 1 /* displayLevelCutoff */)) { - FIO_freeDResources(ress); - return 1; - } - if (!prefs->testMode) { - ress.dstFile = FIO_openDstFile(fCtx, prefs, NULL, outFileName, DEFAULT_FILE_PERMISSIONS); - if (ress.dstFile == 0) EXM_THROW(19, "cannot open %s", outFileName); - } - for (; fCtx->currFileIdx < fCtx->nbFilesTotal; fCtx->currFileIdx++) { - status = FIO_decompressSrcFile(fCtx, prefs, ress, outFileName, srcNamesTable[fCtx->currFileIdx]); - if (!status) fCtx->nbFilesProcessed++; - error |= status; - } - if ((!prefs->testMode) && (fclose(ress.dstFile))) - EXM_THROW(72, "Write error : %s : cannot properly close output file", - strerror(errno)); - } else { - if (outMirroredRootDirName) - UTIL_mirrorSourceFilesDirectories(srcNamesTable, (unsigned)fCtx->nbFilesTotal, outMirroredRootDirName); - - for (; fCtx->currFileIdx < fCtx->nbFilesTotal; fCtx->currFileIdx++) { /* create dstFileName */ - const char* const srcFileName = srcNamesTable[fCtx->currFileIdx]; - const char* dstFileName = NULL; - if (outMirroredRootDirName) { - char* validMirroredDirName = UTIL_createMirroredDestDirName(srcFileName, outMirroredRootDirName); - if (validMirroredDirName) { - dstFileName = FIO_determineDstName(srcFileName, validMirroredDirName); - free(validMirroredDirName); - } else { - DISPLAYLEVEL(2, "zstd: --output-dir-mirror cannot decompress '%s' into '%s'\n", srcFileName, outMirroredRootDirName); - } - } else { - dstFileName = FIO_determineDstName(srcFileName, outDirName); - } - if (dstFileName == NULL) { error=1; continue; } - status = FIO_decompressSrcFile(fCtx, prefs, ress, dstFileName, srcFileName); - if (!status) fCtx->nbFilesProcessed++; - error |= status; - } - if (outDirName) - FIO_checkFilenameCollisions(srcNamesTable , (unsigned)fCtx->nbFilesTotal); - } - - if (fCtx->nbFilesProcessed >= 1 && fCtx->nbFilesTotal > 1 && fCtx->totalBytesOutput != 0) - DISPLAYLEVEL(2, "%d files decompressed : %6zu bytes total \n", fCtx->nbFilesProcessed, fCtx->totalBytesOutput); - - FIO_freeDResources(ress); - return error; -} - -/* ************************************************************************** - * .zst file info (--list command) - ***************************************************************************/ - -typedef struct { - U64 decompressedSize; - U64 compressedSize; - U64 windowSize; - int numActualFrames; - int numSkippableFrames; - int decompUnavailable; - int usesCheck; - U32 nbFiles; -} fileInfo_t; - -typedef enum { - info_success=0, - info_frame_error=1, - info_not_zstd=2, - info_file_error=3, - info_truncated_input=4, -} InfoError; - -#define ERROR_IF(c,n,...) { \ - if (c) { \ - DISPLAYLEVEL(1, __VA_ARGS__); \ - DISPLAYLEVEL(1, " \n"); \ - return n; \ - } \ -} - -static InfoError -FIO_analyzeFrames(fileInfo_t* info, FILE* const srcFile) -{ - /* begin analyzing frame */ - for ( ; ; ) { - BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; - size_t const numBytesRead = fread(headerBuffer, 1, sizeof(headerBuffer), srcFile); - if (numBytesRead < ZSTD_FRAMEHEADERSIZE_MIN(ZSTD_f_zstd1)) { - if ( feof(srcFile) - && (numBytesRead == 0) - && (info->compressedSize > 0) - && (info->compressedSize != UTIL_FILESIZE_UNKNOWN) ) { - unsigned long long file_position = (unsigned long long) LONG_TELL(srcFile); - unsigned long long file_size = (unsigned long long) info->compressedSize; - ERROR_IF(file_position != file_size, info_truncated_input, - "Error: seeked to position %llu, which is beyond file size of %llu\n", - file_position, - file_size); - break; /* correct end of file => success */ - } - ERROR_IF(feof(srcFile), info_not_zstd, "Error: reached end of file with incomplete frame"); - ERROR_IF(1, info_frame_error, "Error: did not reach end of file but ran out of frames"); - } - { U32 const magicNumber = MEM_readLE32(headerBuffer); - /* Zstandard frame */ - if (magicNumber == ZSTD_MAGICNUMBER) { - ZSTD_frameHeader header; - U64 const frameContentSize = ZSTD_getFrameContentSize(headerBuffer, numBytesRead); - if ( frameContentSize == ZSTD_CONTENTSIZE_ERROR - || frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN ) { - info->decompUnavailable = 1; - } else { - info->decompressedSize += frameContentSize; - } - ERROR_IF(ZSTD_getFrameHeader(&header, headerBuffer, numBytesRead) != 0, - info_frame_error, "Error: could not decode frame header"); - info->windowSize = header.windowSize; - /* move to the end of the frame header */ - { size_t const headerSize = ZSTD_frameHeaderSize(headerBuffer, numBytesRead); - ERROR_IF(ZSTD_isError(headerSize), info_frame_error, "Error: could not determine frame header size"); - ERROR_IF(fseek(srcFile, ((long)headerSize)-((long)numBytesRead), SEEK_CUR) != 0, - info_frame_error, "Error: could not move to end of frame header"); - } - - /* skip all blocks in the frame */ - { int lastBlock = 0; - do { - BYTE blockHeaderBuffer[3]; - ERROR_IF(fread(blockHeaderBuffer, 1, 3, srcFile) != 3, - info_frame_error, "Error while reading block header"); - { U32 const blockHeader = MEM_readLE24(blockHeaderBuffer); - U32 const blockTypeID = (blockHeader >> 1) & 3; - U32 const isRLE = (blockTypeID == 1); - U32 const isWrongBlock = (blockTypeID == 3); - long const blockSize = isRLE ? 1 : (long)(blockHeader >> 3); - ERROR_IF(isWrongBlock, info_frame_error, "Error: unsupported block type"); - lastBlock = blockHeader & 1; - ERROR_IF(fseek(srcFile, blockSize, SEEK_CUR) != 0, - info_frame_error, "Error: could not skip to end of block"); - } - } while (lastBlock != 1); - } - - /* check if checksum is used */ - { BYTE const frameHeaderDescriptor = headerBuffer[4]; - int const contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2; - if (contentChecksumFlag) { - info->usesCheck = 1; - ERROR_IF(fseek(srcFile, 4, SEEK_CUR) != 0, - info_frame_error, "Error: could not skip past checksum"); - } } - info->numActualFrames++; - } - /* Skippable frame */ - else if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { - U32 const frameSize = MEM_readLE32(headerBuffer + 4); - long const seek = (long)(8 + frameSize - numBytesRead); - ERROR_IF(LONG_SEEK(srcFile, seek, SEEK_CUR) != 0, - info_frame_error, "Error: could not find end of skippable frame"); - info->numSkippableFrames++; - } - /* unknown content */ - else { - return info_not_zstd; - } - } /* magic number analysis */ - } /* end analyzing frames */ - return info_success; -} - - -static InfoError -getFileInfo_fileConfirmed(fileInfo_t* info, const char* inFileName) -{ - InfoError status; - FILE* const srcFile = FIO_openSrcFile(NULL, inFileName); - ERROR_IF(srcFile == NULL, info_file_error, "Error: could not open source file %s", inFileName); - - info->compressedSize = UTIL_getFileSize(inFileName); - status = FIO_analyzeFrames(info, srcFile); - - fclose(srcFile); - info->nbFiles = 1; - return status; -} - - -/** getFileInfo() : - * Reads information from file, stores in *info - * @return : InfoError status - */ -static InfoError -getFileInfo(fileInfo_t* info, const char* srcFileName) -{ - ERROR_IF(!UTIL_isRegularFile(srcFileName), - info_file_error, "Error : %s is not a file", srcFileName); - return getFileInfo_fileConfirmed(info, srcFileName); -} - - -static void -displayInfo(const char* inFileName, const fileInfo_t* info, int displayLevel) -{ + if ( (result != 0) /* operation failure */ + && strcmp(dstFileName, stdoutmark) /* special case : don't remove() stdout */ + ) { + FIO_removeFile(dstFileName); /* remove decompression artefact; note: don't do anything special if remove() fails */ + } + } + + return result; +} + + +/** FIO_decompressSrcFile() : + Open `srcFileName`, transfer control to decompressDstFile() + @return : 0 : OK + 1 : error +*/ +static int FIO_decompressSrcFile(FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs, dRess_t ress, const char* dstFileName, const char* srcFileName) +{ + FILE* srcFile; + int result; + + if (UTIL_isDirectory(srcFileName)) { + DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName); + return 1; + } + + srcFile = FIO_openSrcFile(prefs, srcFileName); + if (srcFile==NULL) return 1; + ress.srcBufferLoaded = 0; + + result = FIO_decompressDstFile(fCtx, prefs, ress, srcFile, dstFileName, srcFileName); + + /* Close file */ + if (fclose(srcFile)) { + DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno)); /* error should not happen */ + return 1; + } + if ( prefs->removeSrcFile /* --rm */ + && (result==0) /* decompression successful */ + && strcmp(srcFileName, stdinmark) ) /* not stdin */ { + /* We must clear the handler, since after this point calling it would + * delete both the source and destination files. + */ + clearHandler(); + if (FIO_removeFile(srcFileName)) { + /* failed to remove src file */ + DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno)); + return 1; + } } + return result; +} + + + +int FIO_decompressFilename(FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs, + const char* dstFileName, const char* srcFileName, + const char* dictFileName) +{ + dRess_t const ress = FIO_createDResources(prefs, dictFileName); + + int const decodingError = FIO_decompressSrcFile(fCtx, prefs, ress, dstFileName, srcFileName); + + FIO_freeDResources(ress); + return decodingError; +} + +static const char *suffixList[] = { + ZSTD_EXTENSION, + TZSTD_EXTENSION, +#ifndef ZSTD_NODECOMPRESS + ZSTD_ALT_EXTENSION, +#endif +#ifdef ZSTD_GZDECOMPRESS + GZ_EXTENSION, + TGZ_EXTENSION, +#endif +#ifdef ZSTD_LZMADECOMPRESS + LZMA_EXTENSION, + XZ_EXTENSION, + TXZ_EXTENSION, +#endif +#ifdef ZSTD_LZ4DECOMPRESS + LZ4_EXTENSION, + TLZ4_EXTENSION, +#endif + NULL +}; + +static const char *suffixListStr = + ZSTD_EXTENSION "/" TZSTD_EXTENSION +#ifdef ZSTD_GZDECOMPRESS + "/" GZ_EXTENSION "/" TGZ_EXTENSION +#endif +#ifdef ZSTD_LZMADECOMPRESS + "/" LZMA_EXTENSION "/" XZ_EXTENSION "/" TXZ_EXTENSION +#endif +#ifdef ZSTD_LZ4DECOMPRESS + "/" LZ4_EXTENSION "/" TLZ4_EXTENSION +#endif +; + +/* FIO_determineDstName() : + * create a destination filename from a srcFileName. + * @return a pointer to it. + * @return == NULL if there is an error */ +static const char* +FIO_determineDstName(const char* srcFileName, const char* outDirName) +{ + static size_t dfnbCapacity = 0; + static char* dstFileNameBuffer = NULL; /* using static allocation : this function cannot be multi-threaded */ + size_t dstFileNameEndPos; + char* outDirFilename = NULL; + const char* dstSuffix = ""; + size_t dstSuffixLen = 0; + + size_t sfnSize = strlen(srcFileName); + + size_t srcSuffixLen; + const char* const srcSuffix = strrchr(srcFileName, '.'); + if (srcSuffix == NULL) { + DISPLAYLEVEL(1, + "zstd: %s: unknown suffix (%s expected). " + "Can't derive the output file name. " + "Specify it with -o dstFileName. Ignoring.\n", + srcFileName, suffixListStr); + return NULL; + } + srcSuffixLen = strlen(srcSuffix); + + { + const char** matchedSuffixPtr; + for (matchedSuffixPtr = suffixList; *matchedSuffixPtr != NULL; matchedSuffixPtr++) { + if (!strcmp(*matchedSuffixPtr, srcSuffix)) { + break; + } + } + + /* check suffix is authorized */ + if (sfnSize <= srcSuffixLen || *matchedSuffixPtr == NULL) { + DISPLAYLEVEL(1, + "zstd: %s: unknown suffix (%s expected). " + "Can't derive the output file name. " + "Specify it with -o dstFileName. Ignoring.\n", + srcFileName, suffixListStr); + return NULL; + } + + if ((*matchedSuffixPtr)[1] == 't') { + dstSuffix = ".tar"; + dstSuffixLen = strlen(dstSuffix); + } + } + + if (outDirName) { + outDirFilename = FIO_createFilename_fromOutDir(srcFileName, outDirName, 0); + sfnSize = strlen(outDirFilename); + assert(outDirFilename != NULL); + } + + if (dfnbCapacity+srcSuffixLen <= sfnSize+1+dstSuffixLen) { + /* allocate enough space to write dstFilename into it */ + free(dstFileNameBuffer); + dfnbCapacity = sfnSize + 20; + dstFileNameBuffer = (char*)malloc(dfnbCapacity); + if (dstFileNameBuffer==NULL) + EXM_THROW(74, "%s : not enough memory for dstFileName", + strerror(errno)); + } + + /* return dst name == src name truncated from suffix */ + assert(dstFileNameBuffer != NULL); + dstFileNameEndPos = sfnSize - srcSuffixLen; + if (outDirFilename) { + memcpy(dstFileNameBuffer, outDirFilename, dstFileNameEndPos); + free(outDirFilename); + } else { + memcpy(dstFileNameBuffer, srcFileName, dstFileNameEndPos); + } + + /* The short tar extensions tzst, tgz, txz and tlz4 files should have "tar" + * extension on decompression. Also writes terminating null. */ + strcpy(dstFileNameBuffer + dstFileNameEndPos, dstSuffix); + return dstFileNameBuffer; + + /* note : dstFileNameBuffer memory is not going to be free */ +} + +int +FIO_decompressMultipleFilenames(FIO_ctx_t* const fCtx, + FIO_prefs_t* const prefs, + const char** srcNamesTable, + const char* outMirroredRootDirName, + const char* outDirName, const char* outFileName, + const char* dictFileName) +{ + int status; + int error = 0; + dRess_t ress = FIO_createDResources(prefs, dictFileName); + + if (outFileName) { + if (FIO_removeMultiFilesWarning(fCtx, prefs, outFileName, 1 /* displayLevelCutoff */)) { + FIO_freeDResources(ress); + return 1; + } + if (!prefs->testMode) { + ress.dstFile = FIO_openDstFile(fCtx, prefs, NULL, outFileName, DEFAULT_FILE_PERMISSIONS); + if (ress.dstFile == 0) EXM_THROW(19, "cannot open %s", outFileName); + } + for (; fCtx->currFileIdx < fCtx->nbFilesTotal; fCtx->currFileIdx++) { + status = FIO_decompressSrcFile(fCtx, prefs, ress, outFileName, srcNamesTable[fCtx->currFileIdx]); + if (!status) fCtx->nbFilesProcessed++; + error |= status; + } + if ((!prefs->testMode) && (fclose(ress.dstFile))) + EXM_THROW(72, "Write error : %s : cannot properly close output file", + strerror(errno)); + } else { + if (outMirroredRootDirName) + UTIL_mirrorSourceFilesDirectories(srcNamesTable, (unsigned)fCtx->nbFilesTotal, outMirroredRootDirName); + + for (; fCtx->currFileIdx < fCtx->nbFilesTotal; fCtx->currFileIdx++) { /* create dstFileName */ + const char* const srcFileName = srcNamesTable[fCtx->currFileIdx]; + const char* dstFileName = NULL; + if (outMirroredRootDirName) { + char* validMirroredDirName = UTIL_createMirroredDestDirName(srcFileName, outMirroredRootDirName); + if (validMirroredDirName) { + dstFileName = FIO_determineDstName(srcFileName, validMirroredDirName); + free(validMirroredDirName); + } else { + DISPLAYLEVEL(2, "zstd: --output-dir-mirror cannot decompress '%s' into '%s'\n", srcFileName, outMirroredRootDirName); + } + } else { + dstFileName = FIO_determineDstName(srcFileName, outDirName); + } + if (dstFileName == NULL) { error=1; continue; } + status = FIO_decompressSrcFile(fCtx, prefs, ress, dstFileName, srcFileName); + if (!status) fCtx->nbFilesProcessed++; + error |= status; + } + if (outDirName) + FIO_checkFilenameCollisions(srcNamesTable , (unsigned)fCtx->nbFilesTotal); + } + + if (fCtx->nbFilesProcessed >= 1 && fCtx->nbFilesTotal > 1 && fCtx->totalBytesOutput != 0) + DISPLAYLEVEL(2, "%d files decompressed : %6zu bytes total \n", fCtx->nbFilesProcessed, fCtx->totalBytesOutput); + + FIO_freeDResources(ress); + return error; +} + +/* ************************************************************************** + * .zst file info (--list command) + ***************************************************************************/ + +typedef struct { + U64 decompressedSize; + U64 compressedSize; + U64 windowSize; + int numActualFrames; + int numSkippableFrames; + int decompUnavailable; + int usesCheck; + U32 nbFiles; +} fileInfo_t; + +typedef enum { + info_success=0, + info_frame_error=1, + info_not_zstd=2, + info_file_error=3, + info_truncated_input=4, +} InfoError; + +#define ERROR_IF(c,n,...) { \ + if (c) { \ + DISPLAYLEVEL(1, __VA_ARGS__); \ + DISPLAYLEVEL(1, " \n"); \ + return n; \ + } \ +} + +static InfoError +FIO_analyzeFrames(fileInfo_t* info, FILE* const srcFile) +{ + /* begin analyzing frame */ + for ( ; ; ) { + BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; + size_t const numBytesRead = fread(headerBuffer, 1, sizeof(headerBuffer), srcFile); + if (numBytesRead < ZSTD_FRAMEHEADERSIZE_MIN(ZSTD_f_zstd1)) { + if ( feof(srcFile) + && (numBytesRead == 0) + && (info->compressedSize > 0) + && (info->compressedSize != UTIL_FILESIZE_UNKNOWN) ) { + unsigned long long file_position = (unsigned long long) LONG_TELL(srcFile); + unsigned long long file_size = (unsigned long long) info->compressedSize; + ERROR_IF(file_position != file_size, info_truncated_input, + "Error: seeked to position %llu, which is beyond file size of %llu\n", + file_position, + file_size); + break; /* correct end of file => success */ + } + ERROR_IF(feof(srcFile), info_not_zstd, "Error: reached end of file with incomplete frame"); + ERROR_IF(1, info_frame_error, "Error: did not reach end of file but ran out of frames"); + } + { U32 const magicNumber = MEM_readLE32(headerBuffer); + /* Zstandard frame */ + if (magicNumber == ZSTD_MAGICNUMBER) { + ZSTD_frameHeader header; + U64 const frameContentSize = ZSTD_getFrameContentSize(headerBuffer, numBytesRead); + if ( frameContentSize == ZSTD_CONTENTSIZE_ERROR + || frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN ) { + info->decompUnavailable = 1; + } else { + info->decompressedSize += frameContentSize; + } + ERROR_IF(ZSTD_getFrameHeader(&header, headerBuffer, numBytesRead) != 0, + info_frame_error, "Error: could not decode frame header"); + info->windowSize = header.windowSize; + /* move to the end of the frame header */ + { size_t const headerSize = ZSTD_frameHeaderSize(headerBuffer, numBytesRead); + ERROR_IF(ZSTD_isError(headerSize), info_frame_error, "Error: could not determine frame header size"); + ERROR_IF(fseek(srcFile, ((long)headerSize)-((long)numBytesRead), SEEK_CUR) != 0, + info_frame_error, "Error: could not move to end of frame header"); + } + + /* skip all blocks in the frame */ + { int lastBlock = 0; + do { + BYTE blockHeaderBuffer[3]; + ERROR_IF(fread(blockHeaderBuffer, 1, 3, srcFile) != 3, + info_frame_error, "Error while reading block header"); + { U32 const blockHeader = MEM_readLE24(blockHeaderBuffer); + U32 const blockTypeID = (blockHeader >> 1) & 3; + U32 const isRLE = (blockTypeID == 1); + U32 const isWrongBlock = (blockTypeID == 3); + long const blockSize = isRLE ? 1 : (long)(blockHeader >> 3); + ERROR_IF(isWrongBlock, info_frame_error, "Error: unsupported block type"); + lastBlock = blockHeader & 1; + ERROR_IF(fseek(srcFile, blockSize, SEEK_CUR) != 0, + info_frame_error, "Error: could not skip to end of block"); + } + } while (lastBlock != 1); + } + + /* check if checksum is used */ + { BYTE const frameHeaderDescriptor = headerBuffer[4]; + int const contentChecksumFlag = (frameHeaderDescriptor & (1 << 2)) >> 2; + if (contentChecksumFlag) { + info->usesCheck = 1; + ERROR_IF(fseek(srcFile, 4, SEEK_CUR) != 0, + info_frame_error, "Error: could not skip past checksum"); + } } + info->numActualFrames++; + } + /* Skippable frame */ + else if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { + U32 const frameSize = MEM_readLE32(headerBuffer + 4); + long const seek = (long)(8 + frameSize - numBytesRead); + ERROR_IF(LONG_SEEK(srcFile, seek, SEEK_CUR) != 0, + info_frame_error, "Error: could not find end of skippable frame"); + info->numSkippableFrames++; + } + /* unknown content */ + else { + return info_not_zstd; + } + } /* magic number analysis */ + } /* end analyzing frames */ + return info_success; +} + + +static InfoError +getFileInfo_fileConfirmed(fileInfo_t* info, const char* inFileName) +{ + InfoError status; + FILE* const srcFile = FIO_openSrcFile(NULL, inFileName); + ERROR_IF(srcFile == NULL, info_file_error, "Error: could not open source file %s", inFileName); + + info->compressedSize = UTIL_getFileSize(inFileName); + status = FIO_analyzeFrames(info, srcFile); + + fclose(srcFile); + info->nbFiles = 1; + return status; +} + + +/** getFileInfo() : + * Reads information from file, stores in *info + * @return : InfoError status + */ +static InfoError +getFileInfo(fileInfo_t* info, const char* srcFileName) +{ + ERROR_IF(!UTIL_isRegularFile(srcFileName), + info_file_error, "Error : %s is not a file", srcFileName); + return getFileInfo_fileConfirmed(info, srcFileName); +} + + +static void +displayInfo(const char* inFileName, const fileInfo_t* info, int displayLevel) +{ UTIL_HumanReadableSize_t const window_hrs = UTIL_makeHumanReadableSize(info->windowSize); UTIL_HumanReadableSize_t const compressed_hrs = UTIL_makeHumanReadableSize(info->compressedSize); UTIL_HumanReadableSize_t const decompressed_hrs = UTIL_makeHumanReadableSize(info->decompressedSize); - double const ratio = (info->compressedSize == 0) ? 0 : ((double)info->decompressedSize)/(double)info->compressedSize; - const char* const checkString = (info->usesCheck ? "XXH64" : "None"); - if (displayLevel <= 2) { - if (!info->decompUnavailable) { + double const ratio = (info->compressedSize == 0) ? 0 : ((double)info->decompressedSize)/(double)info->compressedSize; + const char* const checkString = (info->usesCheck ? "XXH64" : "None"); + if (displayLevel <= 2) { + if (!info->decompUnavailable) { DISPLAYOUT("%6d %5d %6.*f%4s %8.*f%4s %5.3f %5s %s\n", - info->numSkippableFrames + info->numActualFrames, - info->numSkippableFrames, + info->numSkippableFrames + info->numActualFrames, + info->numSkippableFrames, compressed_hrs.precision, compressed_hrs.value, compressed_hrs.suffix, decompressed_hrs.precision, decompressed_hrs.value, decompressed_hrs.suffix, - ratio, checkString, inFileName); - } else { + ratio, checkString, inFileName); + } else { DISPLAYOUT("%6d %5d %6.*f%4s %5s %s\n", - info->numSkippableFrames + info->numActualFrames, - info->numSkippableFrames, + info->numSkippableFrames + info->numActualFrames, + info->numSkippableFrames, compressed_hrs.precision, compressed_hrs.value, compressed_hrs.suffix, - checkString, inFileName); - } - } else { - DISPLAYOUT("%s \n", inFileName); - DISPLAYOUT("# Zstandard Frames: %d\n", info->numActualFrames); - if (info->numSkippableFrames) - DISPLAYOUT("# Skippable Frames: %d\n", info->numSkippableFrames); + checkString, inFileName); + } + } else { + DISPLAYOUT("%s \n", inFileName); + DISPLAYOUT("# Zstandard Frames: %d\n", info->numActualFrames); + if (info->numSkippableFrames) + DISPLAYOUT("# Skippable Frames: %d\n", info->numSkippableFrames); DISPLAYOUT("Window Size: %.*f%s (%llu B)\n", window_hrs.precision, window_hrs.value, window_hrs.suffix, - (unsigned long long)info->windowSize); + (unsigned long long)info->windowSize); DISPLAYOUT("Compressed Size: %.*f%s (%llu B)\n", compressed_hrs.precision, compressed_hrs.value, compressed_hrs.suffix, - (unsigned long long)info->compressedSize); - if (!info->decompUnavailable) { + (unsigned long long)info->compressedSize); + if (!info->decompUnavailable) { DISPLAYOUT("Decompressed Size: %.*f%s (%llu B)\n", decompressed_hrs.precision, decompressed_hrs.value, decompressed_hrs.suffix, - (unsigned long long)info->decompressedSize); - DISPLAYOUT("Ratio: %.4f\n", ratio); - } - DISPLAYOUT("Check: %s\n", checkString); - DISPLAYOUT("\n"); - } -} - -static fileInfo_t FIO_addFInfo(fileInfo_t fi1, fileInfo_t fi2) -{ - fileInfo_t total; - memset(&total, 0, sizeof(total)); - total.numActualFrames = fi1.numActualFrames + fi2.numActualFrames; - total.numSkippableFrames = fi1.numSkippableFrames + fi2.numSkippableFrames; - total.compressedSize = fi1.compressedSize + fi2.compressedSize; - total.decompressedSize = fi1.decompressedSize + fi2.decompressedSize; - total.decompUnavailable = fi1.decompUnavailable | fi2.decompUnavailable; - total.usesCheck = fi1.usesCheck & fi2.usesCheck; - total.nbFiles = fi1.nbFiles + fi2.nbFiles; - return total; -} - -static int -FIO_listFile(fileInfo_t* total, const char* inFileName, int displayLevel) -{ - fileInfo_t info; - memset(&info, 0, sizeof(info)); - { InfoError const error = getFileInfo(&info, inFileName); - switch (error) { - case info_frame_error: - /* display error, but provide output */ - DISPLAYLEVEL(1, "Error while parsing \"%s\" \n", inFileName); - break; - case info_not_zstd: - DISPLAYOUT("File \"%s\" not compressed by zstd \n", inFileName); - if (displayLevel > 2) DISPLAYOUT("\n"); - return 1; - case info_file_error: - /* error occurred while opening the file */ - if (displayLevel > 2) DISPLAYOUT("\n"); - return 1; - case info_truncated_input: - DISPLAYOUT("File \"%s\" is truncated \n", inFileName); - if (displayLevel > 2) DISPLAYOUT("\n"); - return 1; - case info_success: - default: - break; - } - - displayInfo(inFileName, &info, displayLevel); - *total = FIO_addFInfo(*total, info); - assert(error == info_success || error == info_frame_error); - return (int)error; - } -} - -int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int displayLevel) -{ - /* ensure no specified input is stdin (needs fseek() capability) */ - { unsigned u; - for (u=0; u<numFiles;u++) { - ERROR_IF(!strcmp (filenameTable[u], stdinmark), - 1, "zstd: --list does not support reading from standard input"); - } } - - if (numFiles == 0) { - if (!IS_CONSOLE(stdin)) { - DISPLAYLEVEL(1, "zstd: --list does not support reading from standard input \n"); - } - DISPLAYLEVEL(1, "No files given \n"); - return 1; - } - - if (displayLevel <= 2) { - DISPLAYOUT("Frames Skips Compressed Uncompressed Ratio Check Filename\n"); - } - { int error = 0; - fileInfo_t total; - memset(&total, 0, sizeof(total)); - total.usesCheck = 1; - /* --list each file, and check for any error */ - { unsigned u; - for (u=0; u<numFiles;u++) { - error |= FIO_listFile(&total, filenameTable[u], displayLevel); - } } - if (numFiles > 1 && displayLevel <= 2) { /* display total */ + (unsigned long long)info->decompressedSize); + DISPLAYOUT("Ratio: %.4f\n", ratio); + } + DISPLAYOUT("Check: %s\n", checkString); + DISPLAYOUT("\n"); + } +} + +static fileInfo_t FIO_addFInfo(fileInfo_t fi1, fileInfo_t fi2) +{ + fileInfo_t total; + memset(&total, 0, sizeof(total)); + total.numActualFrames = fi1.numActualFrames + fi2.numActualFrames; + total.numSkippableFrames = fi1.numSkippableFrames + fi2.numSkippableFrames; + total.compressedSize = fi1.compressedSize + fi2.compressedSize; + total.decompressedSize = fi1.decompressedSize + fi2.decompressedSize; + total.decompUnavailable = fi1.decompUnavailable | fi2.decompUnavailable; + total.usesCheck = fi1.usesCheck & fi2.usesCheck; + total.nbFiles = fi1.nbFiles + fi2.nbFiles; + return total; +} + +static int +FIO_listFile(fileInfo_t* total, const char* inFileName, int displayLevel) +{ + fileInfo_t info; + memset(&info, 0, sizeof(info)); + { InfoError const error = getFileInfo(&info, inFileName); + switch (error) { + case info_frame_error: + /* display error, but provide output */ + DISPLAYLEVEL(1, "Error while parsing \"%s\" \n", inFileName); + break; + case info_not_zstd: + DISPLAYOUT("File \"%s\" not compressed by zstd \n", inFileName); + if (displayLevel > 2) DISPLAYOUT("\n"); + return 1; + case info_file_error: + /* error occurred while opening the file */ + if (displayLevel > 2) DISPLAYOUT("\n"); + return 1; + case info_truncated_input: + DISPLAYOUT("File \"%s\" is truncated \n", inFileName); + if (displayLevel > 2) DISPLAYOUT("\n"); + return 1; + case info_success: + default: + break; + } + + displayInfo(inFileName, &info, displayLevel); + *total = FIO_addFInfo(*total, info); + assert(error == info_success || error == info_frame_error); + return (int)error; + } +} + +int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int displayLevel) +{ + /* ensure no specified input is stdin (needs fseek() capability) */ + { unsigned u; + for (u=0; u<numFiles;u++) { + ERROR_IF(!strcmp (filenameTable[u], stdinmark), + 1, "zstd: --list does not support reading from standard input"); + } } + + if (numFiles == 0) { + if (!IS_CONSOLE(stdin)) { + DISPLAYLEVEL(1, "zstd: --list does not support reading from standard input \n"); + } + DISPLAYLEVEL(1, "No files given \n"); + return 1; + } + + if (displayLevel <= 2) { + DISPLAYOUT("Frames Skips Compressed Uncompressed Ratio Check Filename\n"); + } + { int error = 0; + fileInfo_t total; + memset(&total, 0, sizeof(total)); + total.usesCheck = 1; + /* --list each file, and check for any error */ + { unsigned u; + for (u=0; u<numFiles;u++) { + error |= FIO_listFile(&total, filenameTable[u], displayLevel); + } } + if (numFiles > 1 && displayLevel <= 2) { /* display total */ UTIL_HumanReadableSize_t const compressed_hrs = UTIL_makeHumanReadableSize(total.compressedSize); UTIL_HumanReadableSize_t const decompressed_hrs = UTIL_makeHumanReadableSize(total.decompressedSize); - double const ratio = (total.compressedSize == 0) ? 0 : ((double)total.decompressedSize)/(double)total.compressedSize; - const char* const checkString = (total.usesCheck ? "XXH64" : ""); - DISPLAYOUT("----------------------------------------------------------------- \n"); - if (total.decompUnavailable) { + double const ratio = (total.compressedSize == 0) ? 0 : ((double)total.decompressedSize)/(double)total.compressedSize; + const char* const checkString = (total.usesCheck ? "XXH64" : ""); + DISPLAYOUT("----------------------------------------------------------------- \n"); + if (total.decompUnavailable) { DISPLAYOUT("%6d %5d %6.*f%4s %5s %u files\n", - total.numSkippableFrames + total.numActualFrames, - total.numSkippableFrames, + total.numSkippableFrames + total.numActualFrames, + total.numSkippableFrames, compressed_hrs.precision, compressed_hrs.value, compressed_hrs.suffix, - checkString, (unsigned)total.nbFiles); - } else { + checkString, (unsigned)total.nbFiles); + } else { DISPLAYOUT("%6d %5d %6.*f%4s %8.*f%4s %5.3f %5s %u files\n", - total.numSkippableFrames + total.numActualFrames, - total.numSkippableFrames, + total.numSkippableFrames + total.numActualFrames, + total.numSkippableFrames, compressed_hrs.precision, compressed_hrs.value, compressed_hrs.suffix, decompressed_hrs.precision, decompressed_hrs.value, decompressed_hrs.suffix, - ratio, checkString, (unsigned)total.nbFiles); - } } - return error; - } -} - - -#endif /* #ifndef ZSTD_NODECOMPRESS */ + ratio, checkString, (unsigned)total.nbFiles); + } } + return error; + } +} + + +#endif /* #ifndef ZSTD_NODECOMPRESS */ |