diff options
author | thegeorg <thegeorg@yandex-team.com> | 2022-11-07 19:34:08 +0300 |
---|---|---|
committer | thegeorg <thegeorg@yandex-team.com> | 2022-11-07 19:34:08 +0300 |
commit | 50f76e264c70a223a34b24aa59e97bff97128f4c (patch) | |
tree | e604247a10f09df6158c172577b9bfa431f1e1b5 /contrib/libs | |
parent | 278a58c5af63dbd7f7a6d8b8d92dc246651242da (diff) | |
download | ydb-50f76e264c70a223a34b24aa59e97bff97128f4c.tar.gz |
Switch fuzz tests to contrib/libs/libfuzzer
Diffstat (limited to 'contrib/libs')
-rw-r--r-- | contrib/libs/libfuzzer/CODE_OWNERS.TXT (renamed from contrib/libs/libfuzzer12/CODE_OWNERS.TXT) | 0 | ||||
-rw-r--r-- | contrib/libs/libfuzzer/CREDITS.TXT (renamed from contrib/libs/libfuzzer12/CREDITS.TXT) | 0 | ||||
-rw-r--r-- | contrib/libs/libfuzzer/FuzzerBuiltins.h (renamed from contrib/libs/libfuzzer12/FuzzerBuiltins.h) | 1 | ||||
-rw-r--r-- | contrib/libs/libfuzzer/FuzzerBuiltinsMsvc.h (renamed from contrib/libs/libfuzzer12/FuzzerBuiltinsMsvc.h) | 9 | ||||
-rw-r--r-- | contrib/libs/libfuzzer/FuzzerCommand.h (renamed from contrib/libs/libfuzzer12/FuzzerCommand.h) | 12 | ||||
-rw-r--r-- | contrib/libs/libfuzzer/FuzzerCorpus.h (renamed from contrib/libs/libfuzzer12/FuzzerCorpus.h) | 63 | ||||
-rw-r--r-- | contrib/libs/libfuzzer/FuzzerCrossOver.cpp (renamed from contrib/libs/libfuzzer12/FuzzerCrossOver.cpp) | 0 | ||||
-rw-r--r-- | contrib/libs/libfuzzer/FuzzerDataFlowTrace.cpp (renamed from contrib/libs/libfuzzer12/FuzzerDataFlowTrace.cpp) | 36 | ||||
-rw-r--r-- | contrib/libs/libfuzzer/FuzzerDataFlowTrace.h (renamed from contrib/libs/libfuzzer12/FuzzerDataFlowTrace.h) | 30 | ||||
-rw-r--r-- | contrib/libs/libfuzzer/FuzzerDefs.h (renamed from contrib/libs/libfuzzer12/FuzzerDefs.h) | 24 | ||||
-rw-r--r-- | contrib/libs/libfuzzer/FuzzerDictionary.h (renamed from contrib/libs/libfuzzer12/FuzzerDictionary.h) | 19 | ||||
-rw-r--r-- | contrib/libs/libfuzzer/FuzzerDriver.cpp (renamed from contrib/libs/libfuzzer12/FuzzerDriver.cpp) | 86 | ||||
-rw-r--r-- | contrib/libs/libfuzzer/FuzzerExtFunctions.def (renamed from contrib/libs/libfuzzer12/FuzzerExtFunctions.def) | 0 | ||||
-rw-r--r-- | contrib/libs/libfuzzer/FuzzerExtFunctions.h (renamed from contrib/libs/libfuzzer12/FuzzerExtFunctions.h) | 0 | ||||
-rw-r--r-- | contrib/libs/libfuzzer/FuzzerExtFunctionsDlsym.cpp (renamed from contrib/libs/libfuzzer12/FuzzerExtFunctionsDlsym.cpp) | 0 | ||||
-rw-r--r-- | contrib/libs/libfuzzer/FuzzerExtFunctionsWeak.cpp (renamed from contrib/libs/libfuzzer12/FuzzerExtFunctionsWeak.cpp) | 0 | ||||
-rw-r--r-- | contrib/libs/libfuzzer/FuzzerExtFunctionsWindows.cpp (renamed from contrib/libs/libfuzzer12/FuzzerExtFunctionsWindows.cpp) | 0 | ||||
-rw-r--r-- | contrib/libs/libfuzzer/FuzzerExtraCounters.cpp (renamed from contrib/libs/libfuzzer12/FuzzerExtraCounters.cpp) | 8 | ||||
-rw-r--r-- | contrib/libs/libfuzzer/FuzzerExtraCountersDarwin.cpp | 22 | ||||
-rw-r--r-- | contrib/libs/libfuzzer/FuzzerExtraCountersWindows.cpp | 80 | ||||
-rw-r--r-- | contrib/libs/libfuzzer/FuzzerFlags.def (renamed from contrib/libs/libfuzzer12/FuzzerFlags.def) | 9 | ||||
-rw-r--r-- | contrib/libs/libfuzzer/FuzzerFork.cpp (renamed from contrib/libs/libfuzzer12/FuzzerFork.cpp) | 131 | ||||
-rw-r--r-- | contrib/libs/libfuzzer/FuzzerFork.h (renamed from contrib/libs/libfuzzer12/FuzzerFork.h) | 4 | ||||
-rw-r--r-- | contrib/libs/libfuzzer/FuzzerIO.cpp (renamed from contrib/libs/libfuzzer12/FuzzerIO.cpp) | 25 | ||||
-rw-r--r-- | contrib/libs/libfuzzer/FuzzerIO.h (renamed from contrib/libs/libfuzzer12/FuzzerIO.h) | 13 | ||||
-rw-r--r-- | contrib/libs/libfuzzer/FuzzerIOPosix.cpp (renamed from contrib/libs/libfuzzer12/FuzzerIOPosix.cpp) | 3 | ||||
-rw-r--r-- | contrib/libs/libfuzzer/FuzzerIOWindows.cpp (renamed from contrib/libs/libfuzzer12/FuzzerIOWindows.cpp) | 8 | ||||
-rw-r--r-- | contrib/libs/libfuzzer/FuzzerInterface.h (renamed from contrib/libs/libfuzzer12/FuzzerInterface.h) | 0 | ||||
-rw-r--r-- | contrib/libs/libfuzzer/FuzzerInternal.h (renamed from contrib/libs/libfuzzer12/FuzzerInternal.h) | 11 | ||||
-rw-r--r-- | contrib/libs/libfuzzer/FuzzerLoop.cpp (renamed from contrib/libs/libfuzzer12/FuzzerLoop.cpp) | 42 | ||||
-rw-r--r-- | contrib/libs/libfuzzer/FuzzerMain.cpp (renamed from contrib/libs/libfuzzer12/FuzzerMain.cpp) | 0 | ||||
-rw-r--r-- | contrib/libs/libfuzzer/FuzzerMerge.cpp (renamed from contrib/libs/libfuzzer12/FuzzerMerge.cpp) | 219 | ||||
-rw-r--r-- | contrib/libs/libfuzzer/FuzzerMerge.h (renamed from contrib/libs/libfuzzer12/FuzzerMerge.h) | 38 | ||||
-rw-r--r-- | contrib/libs/libfuzzer/FuzzerMutate.cpp (renamed from contrib/libs/libfuzzer12/FuzzerMutate.cpp) | 54 | ||||
-rw-r--r-- | contrib/libs/libfuzzer/FuzzerMutate.h (renamed from contrib/libs/libfuzzer12/FuzzerMutate.h) | 18 | ||||
-rw-r--r-- | contrib/libs/libfuzzer/FuzzerOptions.h (renamed from contrib/libs/libfuzzer12/FuzzerOptions.h) | 1 | ||||
-rw-r--r-- | contrib/libs/libfuzzer/FuzzerPlatform.h (renamed from contrib/libs/libfuzzer12/FuzzerPlatform.h) | 0 | ||||
-rw-r--r-- | contrib/libs/libfuzzer/FuzzerRandom.h (renamed from contrib/libs/libfuzzer12/FuzzerRandom.h) | 23 | ||||
-rw-r--r-- | contrib/libs/libfuzzer/FuzzerSHA1.cpp (renamed from contrib/libs/libfuzzer12/FuzzerSHA1.cpp) | 9 | ||||
-rw-r--r-- | contrib/libs/libfuzzer/FuzzerSHA1.h (renamed from contrib/libs/libfuzzer12/FuzzerSHA1.h) | 0 | ||||
-rw-r--r-- | contrib/libs/libfuzzer/FuzzerTracePC.cpp (renamed from contrib/libs/libfuzzer12/FuzzerTracePC.cpp) | 21 | ||||
-rw-r--r-- | contrib/libs/libfuzzer/FuzzerTracePC.h (renamed from contrib/libs/libfuzzer12/FuzzerTracePC.h) | 41 | ||||
-rw-r--r-- | contrib/libs/libfuzzer/FuzzerUtil.cpp (renamed from contrib/libs/libfuzzer12/FuzzerUtil.cpp) | 13 | ||||
-rw-r--r-- | contrib/libs/libfuzzer/FuzzerUtil.h (renamed from contrib/libs/libfuzzer12/FuzzerUtil.h) | 10 | ||||
-rw-r--r-- | contrib/libs/libfuzzer/FuzzerUtilDarwin.cpp (renamed from contrib/libs/libfuzzer12/FuzzerUtilDarwin.cpp) | 0 | ||||
-rw-r--r-- | contrib/libs/libfuzzer/FuzzerUtilLinux.cpp (renamed from contrib/libs/libfuzzer12/FuzzerUtilLinux.cpp) | 0 | ||||
-rw-r--r-- | contrib/libs/libfuzzer/FuzzerUtilPosix.cpp (renamed from contrib/libs/libfuzzer12/FuzzerUtilPosix.cpp) | 11 | ||||
-rw-r--r-- | contrib/libs/libfuzzer/FuzzerUtilWindows.cpp (renamed from contrib/libs/libfuzzer12/FuzzerUtilWindows.cpp) | 2 | ||||
-rw-r--r-- | contrib/libs/libfuzzer/FuzzerValueBitMap.h (renamed from contrib/libs/libfuzzer12/FuzzerValueBitMap.h) | 0 | ||||
-rw-r--r-- | contrib/libs/libfuzzer/LICENSE.TXT (renamed from contrib/libs/libfuzzer12/LICENSE.TXT) | 0 | ||||
-rw-r--r-- | contrib/libs/libfuzzer/README.txt | 11 | ||||
-rw-r--r-- | contrib/libs/libfuzzer12/README.txt | 1 |
52 files changed, 759 insertions, 349 deletions
diff --git a/contrib/libs/libfuzzer12/CODE_OWNERS.TXT b/contrib/libs/libfuzzer/CODE_OWNERS.TXT index 125487816b..125487816b 100644 --- a/contrib/libs/libfuzzer12/CODE_OWNERS.TXT +++ b/contrib/libs/libfuzzer/CODE_OWNERS.TXT diff --git a/contrib/libs/libfuzzer12/CREDITS.TXT b/contrib/libs/libfuzzer/CREDITS.TXT index 6964eba020..6964eba020 100644 --- a/contrib/libs/libfuzzer12/CREDITS.TXT +++ b/contrib/libs/libfuzzer/CREDITS.TXT diff --git a/contrib/libs/libfuzzer12/FuzzerBuiltins.h b/contrib/libs/libfuzzer/FuzzerBuiltins.h index 4c0ada8266..ce0bd5cb47 100644 --- a/contrib/libs/libfuzzer12/FuzzerBuiltins.h +++ b/contrib/libs/libfuzzer/FuzzerBuiltins.h @@ -26,7 +26,6 @@ inline uint32_t Bswap(uint32_t x) { return __builtin_bswap32(x); } inline uint64_t Bswap(uint64_t x) { return __builtin_bswap64(x); } inline uint32_t Clzll(unsigned long long X) { return __builtin_clzll(X); } -inline uint32_t Clz(unsigned long long X) { return __builtin_clz(X); } inline int Popcountll(unsigned long long X) { return __builtin_popcountll(X); } } // namespace fuzzer diff --git a/contrib/libs/libfuzzer12/FuzzerBuiltinsMsvc.h b/contrib/libs/libfuzzer/FuzzerBuiltinsMsvc.h index c5bec9787d..421dee7f66 100644 --- a/contrib/libs/libfuzzer12/FuzzerBuiltinsMsvc.h +++ b/contrib/libs/libfuzzer/FuzzerBuiltinsMsvc.h @@ -41,7 +41,8 @@ inline uint32_t Clzll(uint64_t X) { #if !defined(_M_ARM) && !defined(_M_X64) // Scan the high 32 bits. if (_BitScanReverse(&LeadZeroIdx, static_cast<unsigned long>(X >> 32))) - return static_cast<int>(63 - (LeadZeroIdx + 32)); // Create a bit offset from the MSB. + return static_cast<int>( + 63 - (LeadZeroIdx + 32)); // Create a bit offset from the MSB. // Scan the low 32 bits. if (_BitScanReverse(&LeadZeroIdx, static_cast<unsigned long>(X))) return static_cast<int>(63 - LeadZeroIdx); @@ -52,12 +53,6 @@ inline uint32_t Clzll(uint64_t X) { return 64; } -inline uint32_t Clz(uint32_t X) { - unsigned long LeadZeroIdx = 0; - if (_BitScanReverse(&LeadZeroIdx, X)) return 31 - LeadZeroIdx; - return 32; -} - inline int Popcountll(unsigned long long X) { #if !defined(_M_ARM) && !defined(_M_X64) return __popcnt(X) + __popcnt(X >> 32); diff --git a/contrib/libs/libfuzzer12/FuzzerCommand.h b/contrib/libs/libfuzzer/FuzzerCommand.h index 87308864af..f653fe3587 100644 --- a/contrib/libs/libfuzzer12/FuzzerCommand.h +++ b/contrib/libs/libfuzzer/FuzzerCommand.h @@ -33,7 +33,7 @@ public: Command() : CombinedOutAndErr(false) {} - explicit Command(const Vector<std::string> &ArgsToAdd) + explicit Command(const std::vector<std::string> &ArgsToAdd) : Args(ArgsToAdd), CombinedOutAndErr(false) {} explicit Command(const Command &Other) @@ -58,7 +58,7 @@ public: // Gets all of the current command line arguments, **including** those after // "-ignore-remaining-args=1". - const Vector<std::string> &getArguments() const { return Args; } + const std::vector<std::string> &getArguments() const { return Args; } // Adds the given argument before "-ignore_remaining_args=1", or at the end // if that flag isn't present. @@ -68,7 +68,7 @@ public: // Adds all given arguments before "-ignore_remaining_args=1", or at the end // if that flag isn't present. - void addArguments(const Vector<std::string> &ArgsToAdd) { + void addArguments(const std::vector<std::string> &ArgsToAdd) { Args.insert(endMutableArgs(), ArgsToAdd.begin(), ArgsToAdd.end()); } @@ -155,16 +155,16 @@ private: Command(Command &&Other) = delete; Command &operator=(Command &&Other) = delete; - Vector<std::string>::iterator endMutableArgs() { + std::vector<std::string>::iterator endMutableArgs() { return std::find(Args.begin(), Args.end(), ignoreRemainingArgs()); } - Vector<std::string>::const_iterator endMutableArgs() const { + std::vector<std::string>::const_iterator endMutableArgs() const { return std::find(Args.begin(), Args.end(), ignoreRemainingArgs()); } // The command arguments. Args[0] is the command name. - Vector<std::string> Args; + std::vector<std::string> Args; // True indicates stderr is redirected to stdout. bool CombinedOutAndErr; diff --git a/contrib/libs/libfuzzer12/FuzzerCorpus.h b/contrib/libs/libfuzzer/FuzzerCorpus.h index daea4f5213..e01891e18f 100644 --- a/contrib/libs/libfuzzer12/FuzzerCorpus.h +++ b/contrib/libs/libfuzzer/FuzzerCorpus.h @@ -39,13 +39,13 @@ struct InputInfo { bool MayDeleteFile = false; bool Reduced = false; bool HasFocusFunction = false; - Vector<uint32_t> UniqFeatureSet; - Vector<uint8_t> DataFlowTraceForFocusFunction; + std::vector<uint32_t> UniqFeatureSet; + std::vector<uint8_t> DataFlowTraceForFocusFunction; // Power schedule. bool NeedsEnergyUpdate = false; double Energy = 0.0; - size_t SumIncidence = 0; - Vector<std::pair<uint32_t, uint16_t>> FeatureFreqs; + double SumIncidence = 0.0; + std::vector<std::pair<uint32_t, uint16_t>> FeatureFreqs; // Delete feature Idx and its frequency from FeatureFreqs. bool DeleteFeatureFreq(uint32_t Idx) { @@ -74,27 +74,28 @@ struct InputInfo { void UpdateEnergy(size_t GlobalNumberOfFeatures, bool ScalePerExecTime, std::chrono::microseconds AverageUnitExecutionTime) { Energy = 0.0; - SumIncidence = 0; + SumIncidence = 0.0; // Apply add-one smoothing to locally discovered features. for (auto F : FeatureFreqs) { - size_t LocalIncidence = F.second + 1; - Energy -= LocalIncidence * logl(LocalIncidence); + double LocalIncidence = F.second + 1; + Energy -= LocalIncidence * log(LocalIncidence); SumIncidence += LocalIncidence; } // Apply add-one smoothing to locally undiscovered features. - // PreciseEnergy -= 0; // since logl(1.0) == 0) - SumIncidence += (GlobalNumberOfFeatures - FeatureFreqs.size()); + // PreciseEnergy -= 0; // since log(1.0) == 0) + SumIncidence += + static_cast<double>(GlobalNumberOfFeatures - FeatureFreqs.size()); // Add a single locally abundant feature apply add-one smoothing. - size_t AbdIncidence = NumExecutedMutations + 1; - Energy -= AbdIncidence * logl(AbdIncidence); + double AbdIncidence = static_cast<double>(NumExecutedMutations + 1); + Energy -= AbdIncidence * log(AbdIncidence); SumIncidence += AbdIncidence; // Normalize. if (SumIncidence != 0) - Energy = (Energy / SumIncidence) + logl(SumIncidence); + Energy = Energy / SumIncidence + log(SumIncidence); if (ScalePerExecTime) { // Scaling to favor inputs with lower execution time. @@ -208,11 +209,13 @@ public: InputInfo *AddToCorpus(const Unit &U, size_t NumFeatures, bool MayDeleteFile, bool HasFocusFunction, bool NeverReduce, std::chrono::microseconds TimeOfUnit, - const Vector<uint32_t> &FeatureSet, + const std::vector<uint32_t> &FeatureSet, const DataFlowTrace &DFT, const InputInfo *BaseII) { assert(!U.empty()); if (FeatureDebug) Printf("ADD_TO_CORPUS %zd NF %zd\n", Inputs.size(), NumFeatures); + // Inputs.size() is cast to uint32_t below. + assert(Inputs.size() < std::numeric_limits<uint32_t>::max()); Inputs.push_back(new InputInfo()); InputInfo &II = *Inputs.back(); II.U = U; @@ -224,7 +227,7 @@ public: II.HasFocusFunction = HasFocusFunction; // Assign maximal energy to the new seed. II.Energy = RareFeatures.empty() ? 1.0 : log(RareFeatures.size()); - II.SumIncidence = RareFeatures.size(); + II.SumIncidence = static_cast<double>(RareFeatures.size()); II.NeedsEnergyUpdate = false; std::sort(II.UniqFeatureSet.begin(), II.UniqFeatureSet.end()); ComputeSHA1(U.data(), U.size(), II.Sha1); @@ -255,7 +258,7 @@ public: } // Debug-only - void PrintFeatureSet(const Vector<uint32_t> &FeatureSet) { + void PrintFeatureSet(const std::vector<uint32_t> &FeatureSet) { if (!FeatureDebug) return; Printf("{"); for (uint32_t Feature: FeatureSet) @@ -281,7 +284,8 @@ public: } } - void Replace(InputInfo *II, const Unit &U) { + void Replace(InputInfo *II, const Unit &U, + std::chrono::microseconds TimeOfUnit) { assert(II->U.size() > U.size()); Hashes.erase(Sha1ToString(II->Sha1)); DeleteFile(*II); @@ -289,6 +293,7 @@ public: Hashes.insert(Sha1ToString(II->Sha1)); II->U = U; II->Reduced = true; + II->TimeOfUnit = TimeOfUnit; DistributionNeedsUpdate = true; } @@ -322,7 +327,8 @@ public: const auto &II = *Inputs[i]; Printf(" [% 3zd %s] sz: % 5zd runs: % 5zd succ: % 5zd focus: %d\n", i, Sha1ToString(II.Sha1).c_str(), II.U.size(), - II.NumExecutedMutations, II.NumSuccessfullMutations, II.HasFocusFunction); + II.NumExecutedMutations, II.NumSuccessfullMutations, + II.HasFocusFunction); } } @@ -399,7 +405,7 @@ public: // Zero energy seeds will never be fuzzed and remain zero energy. if (II->Energy > 0.0) { II->SumIncidence += 1; - II->Energy += logl(II->SumIncidence) / II->SumIncidence; + II->Energy += log(II->SumIncidence) / II->SumIncidence; } } @@ -426,7 +432,8 @@ public: NumUpdatedFeatures++; if (FeatureDebug) Printf("ADD FEATURE %zd sz %d\n", Idx, NewSize); - SmallestElementPerFeature[Idx] = Inputs.size(); + // Inputs.size() is guaranteed to be less than UINT32_MAX by AddToCorpus. + SmallestElementPerFeature[Idx] = static_cast<uint32_t>(Inputs.size()); InputSizesPerFeature[Idx] = NewSize; return true; } @@ -464,7 +471,7 @@ private: static const bool FeatureDebug = false; - size_t GetFeature(size_t Idx) const { return InputSizesPerFeature[Idx]; } + uint32_t GetFeature(size_t Idx) const { return InputSizesPerFeature[Idx]; } void ValidateFeatureSet() { if (FeatureDebug) @@ -539,9 +546,11 @@ private: if (VanillaSchedule) { for (size_t i = 0; i < N; i++) - Weights[i] = Inputs[i]->NumFeatures - ? (i + 1) * (Inputs[i]->HasFocusFunction ? 1000 : 1) - : 0.; + Weights[i] = + Inputs[i]->NumFeatures + ? static_cast<double>((i + 1) * + (Inputs[i]->HasFocusFunction ? 1000 : 1)) + : 0.; } if (FeatureDebug) { @@ -557,11 +566,11 @@ private: } std::piecewise_constant_distribution<double> CorpusDistribution; - Vector<double> Intervals; - Vector<double> Weights; + std::vector<double> Intervals; + std::vector<double> Weights; std::unordered_set<std::string> Hashes; - Vector<InputInfo*> Inputs; + std::vector<InputInfo *> Inputs; size_t NumAddedFeatures = 0; size_t NumUpdatedFeatures = 0; @@ -571,7 +580,7 @@ private: bool DistributionNeedsUpdate = true; uint16_t FreqOfMostAbundantRareFeature = 0; uint16_t GlobalFeatureFreqs[kFeatureSetSize] = {}; - Vector<uint32_t> RareFeatures; + std::vector<uint32_t> RareFeatures; std::string OutputCorpus; }; diff --git a/contrib/libs/libfuzzer12/FuzzerCrossOver.cpp b/contrib/libs/libfuzzer/FuzzerCrossOver.cpp index 83d9f8d47c..83d9f8d47c 100644 --- a/contrib/libs/libfuzzer12/FuzzerCrossOver.cpp +++ b/contrib/libs/libfuzzer/FuzzerCrossOver.cpp diff --git a/contrib/libs/libfuzzer12/FuzzerDataFlowTrace.cpp b/contrib/libs/libfuzzer/FuzzerDataFlowTrace.cpp index 0e9cdf7e66..2f9a4d2d7a 100644 --- a/contrib/libs/libfuzzer12/FuzzerDataFlowTrace.cpp +++ b/contrib/libs/libfuzzer/FuzzerDataFlowTrace.cpp @@ -37,7 +37,7 @@ bool BlockCoverage::AppendCoverage(const std::string &S) { // Coverage lines have this form: // CN X Y Z T // where N is the number of the function, T is the total number of instrumented -// BBs, and X,Y,Z, if present, are the indecies of covered BB. +// BBs, and X,Y,Z, if present, are the indices of covered BB. // BB #0, which is the entry block, is not explicitly listed. bool BlockCoverage::AppendCoverage(std::istream &IN) { std::string L; @@ -52,7 +52,7 @@ bool BlockCoverage::AppendCoverage(std::istream &IN) { continue; } if (L[0] != 'C') continue; - Vector<uint32_t> CoveredBlocks; + std::vector<uint32_t> CoveredBlocks; while (true) { uint32_t BB = 0; SS >> BB; @@ -60,6 +60,7 @@ bool BlockCoverage::AppendCoverage(std::istream &IN) { CoveredBlocks.push_back(BB); } if (CoveredBlocks.empty()) return false; + // Ensures no CoverageVector is longer than UINT32_MAX. uint32_t NumBlocks = CoveredBlocks.back(); CoveredBlocks.pop_back(); for (auto BB : CoveredBlocks) @@ -67,7 +68,7 @@ bool BlockCoverage::AppendCoverage(std::istream &IN) { auto It = Functions.find(FunctionId); auto &Counters = It == Functions.end() - ? Functions.insert({FunctionId, Vector<uint32_t>(NumBlocks)}) + ? Functions.insert({FunctionId, std::vector<uint32_t>(NumBlocks)}) .first->second : It->second; @@ -85,8 +86,8 @@ bool BlockCoverage::AppendCoverage(std::istream &IN) { // * any uncovered function gets weight 0. // * a function with lots of uncovered blocks gets bigger weight. // * a function with a less frequently executed code gets bigger weight. -Vector<double> BlockCoverage::FunctionWeights(size_t NumFunctions) const { - Vector<double> Res(NumFunctions); +std::vector<double> BlockCoverage::FunctionWeights(size_t NumFunctions) const { + std::vector<double> Res(NumFunctions); for (auto It : Functions) { auto FunctionID = It.first; auto Counters = It.second; @@ -103,7 +104,7 @@ Vector<double> BlockCoverage::FunctionWeights(size_t NumFunctions) const { } void DataFlowTrace::ReadCoverage(const std::string &DirPath) { - Vector<SizedFile> Files; + std::vector<SizedFile> Files; GetSizedFilesFromDir(DirPath, &Files); for (auto &SF : Files) { auto Name = Basename(SF.File); @@ -114,16 +115,16 @@ void DataFlowTrace::ReadCoverage(const std::string &DirPath) { } } -static void DFTStringAppendToVector(Vector<uint8_t> *DFT, +static void DFTStringAppendToVector(std::vector<uint8_t> *DFT, const std::string &DFTString) { assert(DFT->size() == DFTString.size()); for (size_t I = 0, Len = DFT->size(); I < Len; I++) (*DFT)[I] = DFTString[I] == '1'; } -// converts a string of '0' and '1' into a Vector<uint8_t> -static Vector<uint8_t> DFTStringToVector(const std::string &DFTString) { - Vector<uint8_t> DFT(DFTString.size()); +// converts a string of '0' and '1' into a std::vector<uint8_t> +static std::vector<uint8_t> DFTStringToVector(const std::string &DFTString) { + std::vector<uint8_t> DFT(DFTString.size()); DFTStringAppendToVector(&DFT, DFTString); return DFT; } @@ -158,14 +159,14 @@ static bool ParseDFTLine(const std::string &Line, size_t *FunctionNum, } bool DataFlowTrace::Init(const std::string &DirPath, std::string *FocusFunction, - Vector<SizedFile> &CorporaFiles, Random &Rand) { + std::vector<SizedFile> &CorporaFiles, Random &Rand) { if (DirPath.empty()) return false; Printf("INFO: DataFlowTrace: reading from '%s'\n", DirPath.c_str()); - Vector<SizedFile> Files; + std::vector<SizedFile> Files; GetSizedFilesFromDir(DirPath, &Files); std::string L; size_t FocusFuncIdx = SIZE_MAX; - Vector<std::string> FunctionNames; + std::vector<std::string> FunctionNames; // Collect the hashes of the corpus files. for (auto &SF : CorporaFiles) @@ -190,7 +191,7 @@ bool DataFlowTrace::Init(const std::string &DirPath, std::string *FocusFunction, // * chooses a random function according to the weights. ReadCoverage(DirPath); auto Weights = Coverage.FunctionWeights(NumFunctions); - Vector<double> Intervals(NumFunctions + 1); + std::vector<double> Intervals(NumFunctions + 1); std::iota(Intervals.begin(), Intervals.end(), 0); auto Distribution = std::piecewise_constant_distribution<double>( Intervals.begin(), Intervals.end(), Weights.begin()); @@ -200,7 +201,8 @@ bool DataFlowTrace::Init(const std::string &DirPath, std::string *FocusFunction, Printf("INFO: AUTOFOCUS: %zd %s\n", FocusFuncIdx, FunctionNames[FocusFuncIdx].c_str()); for (size_t i = 0; i < NumFunctions; i++) { - if (!Weights[i]) continue; + if (Weights[i] == 0.0) + continue; Printf(" [%zd] W %g\tBB-tot %u\tBB-cov %u\tEntryFreq %u:\t%s\n", i, Weights[i], Coverage.GetNumberOfBlocks(i), Coverage.GetNumberOfCoveredBlocks(i), Coverage.GetCounter(i, 0), @@ -245,7 +247,7 @@ bool DataFlowTrace::Init(const std::string &DirPath, std::string *FocusFunction, } int CollectDataFlow(const std::string &DFTBinary, const std::string &DirPath, - const Vector<SizedFile> &CorporaFiles) { + const std::vector<SizedFile> &CorporaFiles) { Printf("INFO: collecting data flow: bin: %s dir: %s files: %zd\n", DFTBinary.c_str(), DirPath.c_str(), CorporaFiles.size()); if (CorporaFiles.empty()) { @@ -263,7 +265,7 @@ int CollectDataFlow(const std::string &DFTBinary, const std::string &DirPath, // we then request tags in [0,Size/2) and [Size/2, Size), and so on. // Function number => DFT. auto OutPath = DirPlusFile(DirPath, Hash(FileToVector(F.File))); - std::unordered_map<size_t, Vector<uint8_t>> DFTMap; + std::unordered_map<size_t, std::vector<uint8_t>> DFTMap; std::unordered_set<std::string> Cov; Command Cmd; Cmd.addArgument(DFTBinary); diff --git a/contrib/libs/libfuzzer12/FuzzerDataFlowTrace.h b/contrib/libs/libfuzzer/FuzzerDataFlowTrace.h index d6e3de30a4..054dce1bdc 100644 --- a/contrib/libs/libfuzzer12/FuzzerDataFlowTrace.h +++ b/contrib/libs/libfuzzer/FuzzerDataFlowTrace.h @@ -39,10 +39,11 @@ namespace fuzzer { int CollectDataFlow(const std::string &DFTBinary, const std::string &DirPath, - const Vector<SizedFile> &CorporaFiles); + const std::vector<SizedFile> &CorporaFiles); class BlockCoverage { - public: +public: + // These functions guarantee no CoverageVector is longer than UINT32_MAX. bool AppendCoverage(std::istream &IN); bool AppendCoverage(const std::string &S); @@ -50,7 +51,8 @@ class BlockCoverage { uint32_t GetCounter(size_t FunctionId, size_t BasicBlockId) { auto It = Functions.find(FunctionId); - if (It == Functions.end()) return 0; + if (It == Functions.end()) + return 0; const auto &Counters = It->second; if (BasicBlockId < Counters.size()) return Counters[BasicBlockId]; @@ -61,7 +63,7 @@ class BlockCoverage { auto It = Functions.find(FunctionId); if (It == Functions.end()) return 0; const auto &Counters = It->second; - return Counters.size(); + return static_cast<uint32_t>(Counters.size()); } uint32_t GetNumberOfCoveredBlocks(size_t FunctionId) { @@ -75,12 +77,11 @@ class BlockCoverage { return Result; } - Vector<double> FunctionWeights(size_t NumFunctions) const; + std::vector<double> FunctionWeights(size_t NumFunctions) const; void clear() { Functions.clear(); } - private: - - typedef Vector<uint32_t> CoverageVector; +private: + typedef std::vector<uint32_t> CoverageVector; uint32_t NumberOfCoveredBlocks(const CoverageVector &Counters) const { uint32_t Res = 0; @@ -91,7 +92,8 @@ class BlockCoverage { } uint32_t NumberOfUncoveredBlocks(const CoverageVector &Counters) const { - return Counters.size() - NumberOfCoveredBlocks(Counters); + return static_cast<uint32_t>(Counters.size()) - + NumberOfCoveredBlocks(Counters); } uint32_t SmallestNonZeroCounter(const CoverageVector &Counters) const { @@ -115,9 +117,9 @@ class DataFlowTrace { public: void ReadCoverage(const std::string &DirPath); bool Init(const std::string &DirPath, std::string *FocusFunction, - Vector<SizedFile> &CorporaFiles, Random &Rand); + std::vector<SizedFile> &CorporaFiles, Random &Rand); void Clear() { Traces.clear(); } - const Vector<uint8_t> *Get(const std::string &InputSha1) const { + const std::vector<uint8_t> *Get(const std::string &InputSha1) const { auto It = Traces.find(InputSha1); if (It != Traces.end()) return &It->second; @@ -126,9 +128,9 @@ class DataFlowTrace { private: // Input's sha1 => DFT for the FocusFunction. - std::unordered_map<std::string, Vector<uint8_t> > Traces; - BlockCoverage Coverage; - std::unordered_set<std::string> CorporaHashes; + std::unordered_map<std::string, std::vector<uint8_t>> Traces; + BlockCoverage Coverage; + std::unordered_set<std::string> CorporaHashes; }; } // namespace fuzzer diff --git a/contrib/libs/libfuzzer12/FuzzerDefs.h b/contrib/libs/libfuzzer/FuzzerDefs.h index 1bc89c9448..369978d0d3 100644 --- a/contrib/libs/libfuzzer12/FuzzerDefs.h +++ b/contrib/libs/libfuzzer/FuzzerDefs.h @@ -38,28 +38,8 @@ struct ExternalFunctions; // Global interface to functions that may or may not be available. extern ExternalFunctions *EF; -// We are using a custom allocator to give a different symbol name to STL -// containers in order to avoid ODR violations. -template<typename T> - class fuzzer_allocator: public std::allocator<T> { - public: - fuzzer_allocator() = default; - - template<class U> - fuzzer_allocator(const fuzzer_allocator<U>&) {} - - template<class Other> - struct rebind { typedef fuzzer_allocator<Other> other; }; - }; - -template<typename T> -using Vector = std::vector<T, fuzzer_allocator<T>>; - -template<typename T> -using Set = std::set<T, std::less<T>, fuzzer_allocator<T>>; - -typedef Vector<uint8_t> Unit; -typedef Vector<Unit> UnitVector; +typedef std::vector<uint8_t> Unit; +typedef std::vector<Unit> UnitVector; typedef int (*UserCallback)(const uint8_t *Data, size_t Size); #define exit(status) FuzzerExit(status) diff --git a/contrib/libs/libfuzzer12/FuzzerDictionary.h b/contrib/libs/libfuzzer/FuzzerDictionary.h index 301c5d9afe..48f063c7ee 100644 --- a/contrib/libs/libfuzzer12/FuzzerDictionary.h +++ b/contrib/libs/libfuzzer/FuzzerDictionary.h @@ -23,12 +23,14 @@ template <size_t kMaxSizeT> class FixedWord { public: static const size_t kMaxSize = kMaxSizeT; FixedWord() {} - FixedWord(const uint8_t *B, uint8_t S) { Set(B, S); } + FixedWord(const uint8_t *B, size_t S) { Set(B, S); } - void Set(const uint8_t *B, uint8_t S) { + void Set(const uint8_t *B, size_t S) { + static_assert(kMaxSizeT <= std::numeric_limits<uint8_t>::max(), + "FixedWord::kMaxSizeT cannot fit in a uint8_t."); assert(S <= kMaxSize); memcpy(Data, B, S); - Size = S; + Size = static_cast<uint8_t>(S); } bool operator==(const FixedWord<kMaxSize> &w) const { @@ -50,10 +52,13 @@ class DictionaryEntry { public: DictionaryEntry() {} DictionaryEntry(Word W) : W(W) {} - DictionaryEntry(Word W, size_t PositionHint) : W(W), PositionHint(PositionHint) {} + DictionaryEntry(Word W, size_t PositionHint) + : W(W), PositionHint(PositionHint) {} const Word &GetW() const { return W; } - bool HasPositionHint() const { return PositionHint != std::numeric_limits<size_t>::max(); } + bool HasPositionHint() const { + return PositionHint != std::numeric_limits<size_t>::max(); + } size_t GetPositionHint() const { assert(HasPositionHint()); return PositionHint; @@ -106,12 +111,12 @@ private: }; // Parses one dictionary entry. -// If successful, write the enty to Unit and returns true, +// If successful, writes the entry to Unit and returns true, // otherwise returns false. bool ParseOneDictionaryEntry(const std::string &Str, Unit *U); // Parses the dictionary file, fills Units, returns true iff all lines // were parsed successfully. -bool ParseDictionaryFile(const std::string &Text, Vector<Unit> *Units); +bool ParseDictionaryFile(const std::string &Text, std::vector<Unit> *Units); } // namespace fuzzer diff --git a/contrib/libs/libfuzzer12/FuzzerDriver.cpp b/contrib/libs/libfuzzer/FuzzerDriver.cpp index 44b8e23cfb..462e147c58 100644 --- a/contrib/libs/libfuzzer12/FuzzerDriver.cpp +++ b/contrib/libs/libfuzzer/FuzzerDriver.cpp @@ -86,7 +86,7 @@ static const FlagDescription FlagDescriptions [] { static const size_t kNumFlags = sizeof(FlagDescriptions) / sizeof(FlagDescriptions[0]); -static Vector<std::string> *Inputs; +static std::vector<std::string> *Inputs; static std::string *ProgName; static void PrintHelp() { @@ -159,14 +159,14 @@ static bool ParseOneFlag(const char *Param) { const char *Str = FlagValue(Param, Name); if (Str) { if (FlagDescriptions[F].IntFlag) { - int Val = MyStol(Str); - *FlagDescriptions[F].IntFlag = Val; + auto Val = MyStol(Str); + *FlagDescriptions[F].IntFlag = static_cast<int>(Val); if (Flags.verbosity >= 2) Printf("Flag: %s %d\n", Name, Val); return true; } else if (FlagDescriptions[F].UIntFlag) { - unsigned int Val = std::stoul(Str); - *FlagDescriptions[F].UIntFlag = Val; + auto Val = std::stoul(Str); + *FlagDescriptions[F].UIntFlag = static_cast<unsigned int>(Val); if (Flags.verbosity >= 2) Printf("Flag: %s %u\n", Name, Val); return true; @@ -187,7 +187,7 @@ static bool ParseOneFlag(const char *Param) { } // We don't use any library to minimize dependencies. -static void ParseFlags(const Vector<std::string> &Args, +static void ParseFlags(const std::vector<std::string> &Args, const ExternalFunctions *EF) { for (size_t F = 0; F < kNumFlags; F++) { if (FlagDescriptions[F].IntFlag) @@ -206,7 +206,7 @@ static void ParseFlags(const Vector<std::string> &Args, "Disabling -len_control by default.\n", EF->LLVMFuzzerCustomMutator); } - Inputs = new Vector<std::string>; + Inputs = new std::vector<std::string>; for (size_t A = 1; A < Args.size(); A++) { if (ParseOneFlag(Args[A].c_str())) { if (Flags.ignore_remaining_args) @@ -272,7 +272,7 @@ static void ValidateDirectoryExists(const std::string &Path, exit(1); } -std::string CloneArgsWithoutX(const Vector<std::string> &Args, +std::string CloneArgsWithoutX(const std::vector<std::string> &Args, const char *X1, const char *X2) { std::string Cmd; for (auto &S : Args) { @@ -283,18 +283,19 @@ std::string CloneArgsWithoutX(const Vector<std::string> &Args, return Cmd; } -static int RunInMultipleProcesses(const Vector<std::string> &Args, +static int RunInMultipleProcesses(const std::vector<std::string> &Args, unsigned NumWorkers, unsigned NumJobs) { std::atomic<unsigned> Counter(0); std::atomic<bool> HasErrors(false); Command Cmd(Args); Cmd.removeFlag("jobs"); Cmd.removeFlag("workers"); - Vector<std::thread> V; + std::vector<std::thread> V; std::thread Pulse(PulseThread); Pulse.detach(); for (unsigned i = 0; i < NumWorkers; i++) - V.push_back(std::thread(WorkerThread, std::ref(Cmd), &Counter, NumJobs, &HasErrors)); + V.push_back(std::thread(WorkerThread, std::ref(Cmd), &Counter, NumJobs, + &HasErrors)); for (auto &T : V) T.join(); return HasErrors ? 1 : 0; @@ -348,8 +349,8 @@ static std::string GetDedupTokenFromCmdOutput(const std::string &S) { return S.substr(Beg, End - Beg); } -int CleanseCrashInput(const Vector<std::string> &Args, - const FuzzingOptions &Options) { +int CleanseCrashInput(const std::vector<std::string> &Args, + const FuzzingOptions &Options) { if (Inputs->size() != 1 || !Flags.exact_artifact_path) { Printf("ERROR: -cleanse_crash should be given one input file and" " -exact_artifact_path\n"); @@ -372,7 +373,7 @@ int CleanseCrashInput(const Vector<std::string> &Args, auto U = FileToVector(CurrentFilePath); size_t Size = U.size(); - const Vector<uint8_t> ReplacementBytes = {' ', 0xff}; + const std::vector<uint8_t> ReplacementBytes = {' ', 0xff}; for (int NumAttempts = 0; NumAttempts < 5; NumAttempts++) { bool Changed = false; for (size_t Idx = 0; Idx < Size; Idx++) { @@ -403,7 +404,7 @@ int CleanseCrashInput(const Vector<std::string> &Args, return 0; } -int MinimizeCrashInput(const Vector<std::string> &Args, +int MinimizeCrashInput(const std::vector<std::string> &Args, const FuzzingOptions &Options) { if (Inputs->size() != 1) { Printf("ERROR: -minimize_crash should be given one input file\n"); @@ -503,14 +504,15 @@ int MinimizeCrashInputInternalStep(Fuzzer *F, InputCorpus *Corpus) { return 0; } -void Merge(Fuzzer *F, FuzzingOptions &Options, const Vector<std::string> &Args, - const Vector<std::string> &Corpora, const char *CFPathOrNull) { +void Merge(Fuzzer *F, FuzzingOptions &Options, + const std::vector<std::string> &Args, + const std::vector<std::string> &Corpora, const char *CFPathOrNull) { if (Corpora.size() < 2) { Printf("INFO: Merge requires two or more corpus dirs\n"); exit(0); } - Vector<SizedFile> OldCorpus, NewCorpus; + std::vector<SizedFile> OldCorpus, NewCorpus; GetSizedFilesFromDir(Corpora[0], &OldCorpus); for (size_t i = 1; i < Corpora.size(); i++) GetSizedFilesFromDir(Corpora[i], &NewCorpus); @@ -518,10 +520,10 @@ void Merge(Fuzzer *F, FuzzingOptions &Options, const Vector<std::string> &Args, std::sort(NewCorpus.begin(), NewCorpus.end()); std::string CFPath = CFPathOrNull ? CFPathOrNull : TempPath("Merge", ".txt"); - Vector<std::string> NewFiles; - Set<uint32_t> NewFeatures, NewCov; + std::vector<std::string> NewFiles; + std::set<uint32_t> NewFeatures, NewCov; CrashResistantMerge(Args, OldCorpus, NewCorpus, &NewFiles, {}, &NewFeatures, - {}, &NewCov, CFPath, true); + {}, &NewCov, CFPath, true, Flags.set_cover_merge); for (auto &Path : NewFiles) F->WriteToOutputCorpus(FileToVector(Path, Options.MaxLen)); // We are done, delete the control file if it was a temporary one. @@ -531,17 +533,17 @@ void Merge(Fuzzer *F, FuzzingOptions &Options, const Vector<std::string> &Args, exit(0); } -int AnalyzeDictionary(Fuzzer *F, const Vector<Unit>& Dict, - UnitVector& Corpus) { +int AnalyzeDictionary(Fuzzer *F, const std::vector<Unit> &Dict, + UnitVector &Corpus) { Printf("Started dictionary minimization (up to %d tests)\n", Dict.size() * Corpus.size() * 2); // Scores and usage count for each dictionary unit. - Vector<int> Scores(Dict.size()); - Vector<int> Usages(Dict.size()); + std::vector<int> Scores(Dict.size()); + std::vector<int> Usages(Dict.size()); - Vector<size_t> InitialFeatures; - Vector<size_t> ModifiedFeatures; + std::vector<size_t> InitialFeatures; + std::vector<size_t> ModifiedFeatures; for (auto &C : Corpus) { // Get coverage for the testcase without modifications. F->ExecuteCallback(C.data(), C.size()); @@ -551,7 +553,7 @@ int AnalyzeDictionary(Fuzzer *F, const Vector<Unit>& Dict, }); for (size_t i = 0; i < Dict.size(); ++i) { - Vector<uint8_t> Data = C; + std::vector<uint8_t> Data = C; auto StartPos = std::search(Data.begin(), Data.end(), Dict[i].begin(), Dict[i].end()); // Skip dictionary unit, if the testcase does not contain it. @@ -597,9 +599,9 @@ int AnalyzeDictionary(Fuzzer *F, const Vector<Unit>& Dict, return 0; } -Vector<std::string> ParseSeedInuts(const char *seed_inputs) { +std::vector<std::string> ParseSeedInuts(const char *seed_inputs) { // Parse -seed_inputs=file1,file2,... or -seed_inputs=@seed_inputs_file - Vector<std::string> Files; + std::vector<std::string> Files; if (!seed_inputs) return Files; std::string SeedInputs; if (Flags.seed_inputs[0] == '@') @@ -620,9 +622,10 @@ Vector<std::string> ParseSeedInuts(const char *seed_inputs) { return Files; } -static Vector<SizedFile> ReadCorpora(const Vector<std::string> &CorpusDirs, - const Vector<std::string> &ExtraSeedFiles) { - Vector<SizedFile> SizedFiles; +static std::vector<SizedFile> +ReadCorpora(const std::vector<std::string> &CorpusDirs, + const std::vector<std::string> &ExtraSeedFiles) { + std::vector<SizedFile> SizedFiles; size_t LastNumFiles = 0; for (auto &Dir : CorpusDirs) { GetSizedFilesFromDir(Dir, &SizedFiles); @@ -651,7 +654,7 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { EF->LLVMFuzzerInitialize(argc, argv); if (EF->__msan_scoped_disable_interceptor_checks) EF->__msan_scoped_disable_interceptor_checks(); - const Vector<std::string> Args(*argv, *argv + *argc); + const std::vector<std::string> Args(*argv, *argv + *argc); assert(!Args.empty()); ProgName = new std::string(Args[0]); if (Argv0 != *ProgName) { @@ -742,7 +745,7 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { ValidateDirectoryExists(DirName(Options.ExactArtifactPath), Flags.create_missing_dirs); } - Vector<Unit> Dictionary; + std::vector<Unit> Dictionary; if (Flags.dict) if (!ParseDictionaryFile(FileToString(Flags.dict), &Dictionary)) return 1; @@ -797,12 +800,13 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { unsigned Seed = Flags.seed; // Initialize Seed. if (Seed == 0) - Seed = - std::chrono::system_clock::now().time_since_epoch().count() + GetPid(); + Seed = static_cast<unsigned>( + std::chrono::system_clock::now().time_since_epoch().count() + GetPid()); if (Flags.verbosity) Printf("INFO: Seed: %u\n", Seed); - if (Flags.collect_data_flow && !Flags.fork && !Flags.merge) { + if (Flags.collect_data_flow && !Flags.fork && + !(Flags.merge || Flags.set_cover_merge)) { if (RunIndividualFiles) return CollectDataFlow(Flags.collect_data_flow, Flags.data_flow_trace, ReadCorpora({}, *Inputs)); @@ -874,10 +878,11 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { exit(0); } + Options.ForkCorpusGroups = Flags.fork_corpus_groups; if (Flags.fork) FuzzWithFork(F->GetMD().GetRand(), Options, Args, *Inputs, Flags.fork); - if (Flags.merge) + if (Flags.merge || Flags.set_cover_merge) Merge(F, Options, Args, *Inputs, Flags.merge_control_file); if (Flags.merge_inner) { @@ -885,7 +890,8 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { if (Options.MaxLen == 0) F->SetMaxInputLen(kDefaultMaxMergeLen); assert(Flags.merge_control_file); - F->CrashResistantMergeInternalStep(Flags.merge_control_file); + F->CrashResistantMergeInternalStep(Flags.merge_control_file, + !strncmp(Flags.merge_inner, "2", 1)); exit(0); } diff --git a/contrib/libs/libfuzzer12/FuzzerExtFunctions.def b/contrib/libs/libfuzzer/FuzzerExtFunctions.def index 40b1e73945..40b1e73945 100644 --- a/contrib/libs/libfuzzer12/FuzzerExtFunctions.def +++ b/contrib/libs/libfuzzer/FuzzerExtFunctions.def diff --git a/contrib/libs/libfuzzer12/FuzzerExtFunctions.h b/contrib/libs/libfuzzer/FuzzerExtFunctions.h index c88aac4e67..c88aac4e67 100644 --- a/contrib/libs/libfuzzer12/FuzzerExtFunctions.h +++ b/contrib/libs/libfuzzer/FuzzerExtFunctions.h diff --git a/contrib/libs/libfuzzer12/FuzzerExtFunctionsDlsym.cpp b/contrib/libs/libfuzzer/FuzzerExtFunctionsDlsym.cpp index 95233d2a10..95233d2a10 100644 --- a/contrib/libs/libfuzzer12/FuzzerExtFunctionsDlsym.cpp +++ b/contrib/libs/libfuzzer/FuzzerExtFunctionsDlsym.cpp diff --git a/contrib/libs/libfuzzer12/FuzzerExtFunctionsWeak.cpp b/contrib/libs/libfuzzer/FuzzerExtFunctionsWeak.cpp index 3ef758daa7..3ef758daa7 100644 --- a/contrib/libs/libfuzzer12/FuzzerExtFunctionsWeak.cpp +++ b/contrib/libs/libfuzzer/FuzzerExtFunctionsWeak.cpp diff --git a/contrib/libs/libfuzzer12/FuzzerExtFunctionsWindows.cpp b/contrib/libs/libfuzzer/FuzzerExtFunctionsWindows.cpp index 688bad1d51..688bad1d51 100644 --- a/contrib/libs/libfuzzer12/FuzzerExtFunctionsWindows.cpp +++ b/contrib/libs/libfuzzer/FuzzerExtFunctionsWindows.cpp diff --git a/contrib/libs/libfuzzer12/FuzzerExtraCounters.cpp b/contrib/libs/libfuzzer/FuzzerExtraCounters.cpp index 04f569a1a8..54ecbf7c62 100644 --- a/contrib/libs/libfuzzer12/FuzzerExtraCounters.cpp +++ b/contrib/libs/libfuzzer/FuzzerExtraCounters.cpp @@ -31,12 +31,4 @@ void ClearExtraCounters() { // hand-written memset, don't asan-ify. } // namespace fuzzer -#else -// TODO: implement for other platforms. -namespace fuzzer { -uint8_t *ExtraCountersBegin() { return nullptr; } -uint8_t *ExtraCountersEnd() { return nullptr; } -void ClearExtraCounters() {} -} // namespace fuzzer - #endif diff --git a/contrib/libs/libfuzzer/FuzzerExtraCountersDarwin.cpp b/contrib/libs/libfuzzer/FuzzerExtraCountersDarwin.cpp new file mode 100644 index 0000000000..2321ba8a3d --- /dev/null +++ b/contrib/libs/libfuzzer/FuzzerExtraCountersDarwin.cpp @@ -0,0 +1,22 @@ +//===- FuzzerExtraCountersDarwin.cpp - Extra coverage counters for Darwin -===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Extra coverage counters defined by user code for Darwin. +//===----------------------------------------------------------------------===// + +#include "FuzzerPlatform.h" +#include <cstdint> + +#if LIBFUZZER_APPLE + +namespace fuzzer { +uint8_t *ExtraCountersBegin() { return nullptr; } +uint8_t *ExtraCountersEnd() { return nullptr; } +void ClearExtraCounters() {} +} // namespace fuzzer + +#endif diff --git a/contrib/libs/libfuzzer/FuzzerExtraCountersWindows.cpp b/contrib/libs/libfuzzer/FuzzerExtraCountersWindows.cpp new file mode 100644 index 0000000000..102f5febda --- /dev/null +++ b/contrib/libs/libfuzzer/FuzzerExtraCountersWindows.cpp @@ -0,0 +1,80 @@ +//===- FuzzerExtraCountersWindows.cpp - Extra coverage counters for Win32 -===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Extra coverage counters defined by user code for Windows. +//===----------------------------------------------------------------------===// + +#include "FuzzerPlatform.h" +#include <cstdint> + +#if LIBFUZZER_WINDOWS +#include <windows.h> + +namespace fuzzer { + +// +// The __start___libfuzzer_extra_counters variable is align 16, size 16 to +// ensure the padding between it and the next variable in this section (either +// __libfuzzer_extra_counters or __stop___libfuzzer_extra_counters) will be +// located at (__start___libfuzzer_extra_counters + +// sizeof(__start___libfuzzer_extra_counters)). Otherwise, the calculation of +// (stop - (start + sizeof(start))) might be skewed. +// +// The section name, __libfuzzer_extra_countaaa ends with "aaa", so it sorts +// before __libfuzzer_extra_counters alphabetically. We want the start symbol to +// be placed in the section just before the user supplied counters (if present). +// +#pragma section(".data$__libfuzzer_extra_countaaa") +ATTRIBUTE_ALIGNED(16) +__declspec(allocate(".data$__libfuzzer_extra_countaaa")) uint8_t + __start___libfuzzer_extra_counters[16] = {0}; + +// +// Example of what the user-supplied counters should look like. First, the +// pragma to create the section name. It will fall alphabetically between +// ".data$__libfuzzer_extra_countaaa" and ".data$__libfuzzer_extra_countzzz". +// Next, the declspec to allocate the variable inside the specified section. +// Finally, some array, struct, whatever that is used to track the counter data. +// The size of this variable is computed at runtime by finding the difference of +// __stop___libfuzzer_extra_counters and __start___libfuzzer_extra_counters + +// sizeof(__start___libfuzzer_extra_counters). +// + +// +// #pragma section(".data$__libfuzzer_extra_counters") +// __declspec(allocate(".data$__libfuzzer_extra_counters")) +// uint8_t any_name_variable[64 * 1024]; +// + +// +// Here, the section name, __libfuzzer_extra_countzzz ends with "zzz", so it +// sorts after __libfuzzer_extra_counters alphabetically. We want the stop +// symbol to be placed in the section just after the user supplied counters (if +// present). Align to 1 so there isn't any padding placed between this and the +// previous variable. +// +#pragma section(".data$__libfuzzer_extra_countzzz") +ATTRIBUTE_ALIGNED(1) +__declspec(allocate(".data$__libfuzzer_extra_countzzz")) uint8_t + __stop___libfuzzer_extra_counters = 0; + +uint8_t *ExtraCountersBegin() { + return __start___libfuzzer_extra_counters + + sizeof(__start___libfuzzer_extra_counters); +} + +uint8_t *ExtraCountersEnd() { return &__stop___libfuzzer_extra_counters; } + +ATTRIBUTE_NO_SANITIZE_ALL +void ClearExtraCounters() { + uint8_t *Beg = ExtraCountersBegin(); + SecureZeroMemory(Beg, ExtraCountersEnd() - Beg); +} + +} // namespace fuzzer + +#endif diff --git a/contrib/libs/libfuzzer12/FuzzerFlags.def b/contrib/libs/libfuzzer/FuzzerFlags.def index 078333e81f..dacfa84a6c 100644 --- a/contrib/libs/libfuzzer12/FuzzerFlags.def +++ b/contrib/libs/libfuzzer/FuzzerFlags.def @@ -61,12 +61,21 @@ FUZZER_FLAG_INT(max_total_time, 0, "If positive, indicates the maximal total " FUZZER_FLAG_INT(help, 0, "Print help.") FUZZER_FLAG_INT(fork, 0, "Experimental mode where fuzzing happens " "in a subprocess") +FUZZER_FLAG_INT(fork_corpus_groups, 0, "For fork mode, enable the corpus-group " + "strategy, The main corpus will be grouped according to size, " + "and each sub-process will randomly select seeds from different " + "groups as the sub-corpus.") FUZZER_FLAG_INT(ignore_timeouts, 1, "Ignore timeouts in fork mode") FUZZER_FLAG_INT(ignore_ooms, 1, "Ignore OOMs in fork mode") FUZZER_FLAG_INT(ignore_crashes, 0, "Ignore crashes in fork mode") FUZZER_FLAG_INT(merge, 0, "If 1, the 2-nd, 3-rd, etc corpora will be " "merged into the 1-st corpus. Only interesting units will be taken. " "This flag can be used to minimize a corpus.") +FUZZER_FLAG_INT(set_cover_merge, 0, "If 1, the 2-nd, 3-rd, etc corpora will be " + "merged into the 1-st corpus. Same as the 'merge' flag, but uses the " + "standard greedy algorithm for the set cover problem to " + "compute an approximation of the minimum set of testcases that " + "provide the same coverage as the initial corpora") FUZZER_FLAG_STRING(stop_file, "Stop fuzzing ASAP if this file exists") FUZZER_FLAG_STRING(merge_inner, "internal flag") FUZZER_FLAG_STRING(merge_control_file, diff --git a/contrib/libs/libfuzzer12/FuzzerFork.cpp b/contrib/libs/libfuzzer/FuzzerFork.cpp index 84725d22a9..d59d513842 100644 --- a/contrib/libs/libfuzzer12/FuzzerFork.cpp +++ b/contrib/libs/libfuzzer/FuzzerFork.cpp @@ -86,18 +86,21 @@ struct FuzzJob { }; struct GlobalEnv { - Vector<std::string> Args; - Vector<std::string> CorpusDirs; + std::vector<std::string> Args; + std::vector<std::string> CorpusDirs; std::string MainCorpusDir; std::string TempDir; std::string DFTDir; std::string DataFlowBinary; - Set<uint32_t> Features, Cov; - Set<std::string> FilesWithDFT; - Vector<std::string> Files; + std::set<uint32_t> Features, Cov; + std::set<std::string> FilesWithDFT; + std::vector<std::string> Files; + std::vector<std::size_t> FilesSizes; Random *Rand; std::chrono::system_clock::time_point ProcessStartTime; int Verbosity = 0; + int Group = 0; + int NumCorpuses = 8; size_t NumTimeouts = 0; size_t NumOOMs = 0; @@ -136,13 +139,29 @@ struct GlobalEnv { if (size_t CorpusSubsetSize = std::min(Files.size(), (size_t)sqrt(Files.size() + 2))) { auto Time1 = std::chrono::system_clock::now(); - for (size_t i = 0; i < CorpusSubsetSize; i++) { - auto &SF = Files[Rand->SkewTowardsLast(Files.size())]; - Seeds += (Seeds.empty() ? "" : ",") + SF; - CollectDFT(SF); + if (Group) { // whether to group the corpus. + size_t AverageCorpusSize = Files.size() / NumCorpuses + 1; + size_t StartIndex = ((JobId - 1) % NumCorpuses) * AverageCorpusSize; + for (size_t i = 0; i < CorpusSubsetSize; i++) { + size_t RandNum = (*Rand)(AverageCorpusSize); + size_t Index = RandNum + StartIndex; + Index = Index < Files.size() ? Index + : Rand->SkewTowardsLast(Files.size()); + auto &SF = Files[Index]; + Seeds += (Seeds.empty() ? "" : ",") + SF; + CollectDFT(SF); + } + } else { + for (size_t i = 0; i < CorpusSubsetSize; i++) { + auto &SF = Files[Rand->SkewTowardsLast(Files.size())]; + Seeds += (Seeds.empty() ? "" : ",") + SF; + CollectDFT(SF); + } } auto Time2 = std::chrono::system_clock::now(); - Job->DftTimeInSeconds = duration_cast<seconds>(Time2 - Time1).count(); + auto DftTimeInSeconds = duration_cast<seconds>(Time2 - Time1).count(); + assert(DftTimeInSeconds < std::numeric_limits<int>::max()); + Job->DftTimeInSeconds = static_cast<int>(DftTimeInSeconds); } if (!Seeds.empty()) { Job->SeedListPath = @@ -181,7 +200,7 @@ struct GlobalEnv { auto Stats = ParseFinalStatsFromLog(Job->LogPath); NumRuns += Stats.number_of_executed_units; - Vector<SizedFile> TempFiles, MergeCandidates; + std::vector<SizedFile> TempFiles, MergeCandidates; // Read all newly created inputs and their feature sets. // Choose only those inputs that have new features. GetSizedFilesFromDir(Job->CorpusDir, &TempFiles); @@ -191,7 +210,7 @@ struct GlobalEnv { FeatureFile.replace(0, Job->CorpusDir.size(), Job->FeaturesDir); auto FeatureBytes = FileToVector(FeatureFile, 0, false); assert((FeatureBytes.size() % sizeof(uint32_t)) == 0); - Vector<uint32_t> NewFeatures(FeatureBytes.size() / sizeof(uint32_t)); + std::vector<uint32_t> NewFeatures(FeatureBytes.size() / sizeof(uint32_t)); memcpy(NewFeatures.data(), FeatureBytes.data(), FeatureBytes.size()); for (auto Ft : NewFeatures) { if (!Features.count(Ft)) { @@ -209,15 +228,27 @@ struct GlobalEnv { if (MergeCandidates.empty()) return; - Vector<std::string> FilesToAdd; - Set<uint32_t> NewFeatures, NewCov; + std::vector<std::string> FilesToAdd; + std::set<uint32_t> NewFeatures, NewCov; + bool IsSetCoverMerge = + !Job->Cmd.getFlagValue("set_cover_merge").compare("1"); CrashResistantMerge(Args, {}, MergeCandidates, &FilesToAdd, Features, - &NewFeatures, Cov, &NewCov, Job->CFPath, false); + &NewFeatures, Cov, &NewCov, Job->CFPath, false, + IsSetCoverMerge); for (auto &Path : FilesToAdd) { auto U = FileToVector(Path); auto NewPath = DirPlusFile(MainCorpusDir, Hash(U)); WriteToFile(U, NewPath); - Files.push_back(NewPath); + if (Group) { // Insert the queue according to the size of the seed. + size_t UnitSize = U.size(); + auto Idx = + std::upper_bound(FilesSizes.begin(), FilesSizes.end(), UnitSize) - + FilesSizes.begin(); + FilesSizes.insert(FilesSizes.begin() + Idx, UnitSize); + Files.insert(Files.begin() + Idx, NewPath); + } else { + Files.push_back(NewPath); + } } Features.insert(NewFeatures.begin(), NewFeatures.end()); Cov.insert(NewCov.begin(), NewCov.end()); @@ -226,10 +257,8 @@ struct GlobalEnv { if (TPC.PcIsFuncEntry(TE)) PrintPC(" NEW_FUNC: %p %F %L\n", "", TPC.GetNextInstructionPc(TE->PC)); - } - void CollectDFT(const std::string &InputPath) { if (DataFlowBinary.empty()) return; if (!FilesWithDFT.insert(InputPath).second) return; @@ -281,8 +310,8 @@ void WorkerThread(JobQueue *FuzzQ, JobQueue *MergeQ) { // This is just a skeleton of an experimental -fork=1 feature. void FuzzWithFork(Random &Rand, const FuzzingOptions &Options, - const Vector<std::string> &Args, - const Vector<std::string> &CorpusDirs, int NumJobs) { + const std::vector<std::string> &Args, + const std::vector<std::string> &CorpusDirs, int NumJobs) { Printf("INFO: -fork=%d: fuzzing in separate process(s)\n", NumJobs); GlobalEnv Env; @@ -292,8 +321,9 @@ void FuzzWithFork(Random &Rand, const FuzzingOptions &Options, Env.Verbosity = Options.Verbosity; Env.ProcessStartTime = std::chrono::system_clock::now(); Env.DataFlowBinary = Options.CollectDataFlow; + Env.Group = Options.ForkCorpusGroups; - Vector<SizedFile> SeedFiles; + std::vector<SizedFile> SeedFiles; for (auto &Dir : CorpusDirs) GetSizedFilesFromDir(Dir, &SeedFiles); std::sort(SeedFiles.begin(), SeedFiles.end()); @@ -314,10 +344,20 @@ void FuzzWithFork(Random &Rand, const FuzzingOptions &Options, Env.Files.push_back(File.File); } else { auto CFPath = DirPlusFile(Env.TempDir, "merge.txt"); - CrashResistantMerge(Env.Args, {}, SeedFiles, &Env.Files, {}, &Env.Features, - {}, &Env.Cov, CFPath, false); + std::set<uint32_t> NewFeatures, NewCov; + CrashResistantMerge(Env.Args, {}, SeedFiles, &Env.Files, Env.Features, + &NewFeatures, Env.Cov, &NewCov, CFPath, + /*Verbose=*/false, /*IsSetCoverMerge=*/false); + Env.Features.insert(NewFeatures.begin(), NewFeatures.end()); + Env.Cov.insert(NewFeatures.begin(), NewFeatures.end()); RemoveFile(CFPath); } + + if (Env.Group) { + for (auto &path : Env.Files) + Env.FilesSizes.push_back(FileSize(path)); + } + Printf("INFO: -fork=%d: %zd seed inputs, starting to fuzz in %s\n", NumJobs, Env.Files.size(), Env.TempDir.c_str()); @@ -332,8 +372,10 @@ void FuzzWithFork(Random &Rand, const FuzzingOptions &Options, WriteToFile(Unit({1}), Env.StopFile()); }; + size_t MergeCycle = 20; + size_t JobExecuted = 0; size_t JobId = 1; - Vector<std::thread> Threads; + std::vector<std::thread> Threads; for (int t = 0; t < NumJobs; t++) { Threads.push_back(std::thread(WorkerThread, &FuzzQ, &MergeQ)); FuzzQ.Push(Env.CreateNewJob(JobId++)); @@ -353,7 +395,46 @@ void FuzzWithFork(Random &Rand, const FuzzingOptions &Options, Env.RunOneMergeJob(Job.get()); - // Continue if our crash is one of the ignorred ones. + // merge the corpus . + JobExecuted++; + if (Env.Group && JobExecuted >= MergeCycle) { + std::vector<SizedFile> CurrentSeedFiles; + for (auto &Dir : CorpusDirs) + GetSizedFilesFromDir(Dir, &CurrentSeedFiles); + std::sort(CurrentSeedFiles.begin(), CurrentSeedFiles.end()); + + auto CFPath = DirPlusFile(Env.TempDir, "merge.txt"); + std::set<uint32_t> TmpNewFeatures, TmpNewCov; + std::set<uint32_t> TmpFeatures, TmpCov; + Env.Files.clear(); + Env.FilesSizes.clear(); + CrashResistantMerge(Env.Args, {}, CurrentSeedFiles, &Env.Files, + TmpFeatures, &TmpNewFeatures, TmpCov, &TmpNewCov, + CFPath, /*Verbose=*/false, /*IsSetCoverMerge=*/false); + for (auto &path : Env.Files) + Env.FilesSizes.push_back(FileSize(path)); + RemoveFile(CFPath); + JobExecuted = 0; + MergeCycle += 5; + } + + // Since the number of corpus seeds will gradually increase, in order to + // control the number in each group to be about three times the number of + // seeds selected each time, the number of groups is dynamically adjusted. + if (Env.Files.size() < 2000) + Env.NumCorpuses = 12; + else if (Env.Files.size() < 6000) + Env.NumCorpuses = 20; + else if (Env.Files.size() < 12000) + Env.NumCorpuses = 32; + else if (Env.Files.size() < 16000) + Env.NumCorpuses = 40; + else if (Env.Files.size() < 24000) + Env.NumCorpuses = 60; + else + Env.NumCorpuses = 80; + + // Continue if our crash is one of the ignored ones. if (Options.IgnoreTimeouts && ExitCode == Options.TimeoutExitCode) Env.NumTimeouts++; else if (Options.IgnoreOOMs && ExitCode == Options.OOMExitCode) diff --git a/contrib/libs/libfuzzer12/FuzzerFork.h b/contrib/libs/libfuzzer/FuzzerFork.h index b29a43e13f..fc3e9d636c 100644 --- a/contrib/libs/libfuzzer12/FuzzerFork.h +++ b/contrib/libs/libfuzzer/FuzzerFork.h @@ -17,8 +17,8 @@ namespace fuzzer { void FuzzWithFork(Random &Rand, const FuzzingOptions &Options, - const Vector<std::string> &Args, - const Vector<std::string> &CorpusDirs, int NumJobs); + const std::vector<std::string> &Args, + const std::vector<std::string> &CorpusDirs, int NumJobs); } // namespace fuzzer #endif // LLVM_FUZZER_FORK_H diff --git a/contrib/libs/libfuzzer12/FuzzerIO.cpp b/contrib/libs/libfuzzer/FuzzerIO.cpp index 54a7219fc0..0a58c5377b 100644 --- a/contrib/libs/libfuzzer12/FuzzerIO.cpp +++ b/contrib/libs/libfuzzer/FuzzerIO.cpp @@ -23,6 +23,14 @@ namespace fuzzer { static FILE *OutputFile = stderr; +FILE *GetOutputFile() { + return OutputFile; +} + +void SetOutputFile(FILE *NewOutputFile) { + OutputFile = NewOutputFile; +} + long GetEpoch(const std::string &Path) { struct stat St; if (stat(Path.c_str(), &St)) @@ -90,10 +98,11 @@ void AppendToFile(const uint8_t *Data, size_t Size, const std::string &Path) { fclose(Out); } -void ReadDirToVectorOfUnits(const char *Path, Vector<Unit> *V, - long *Epoch, size_t MaxSize, bool ExitOnError) { +void ReadDirToVectorOfUnits(const char *Path, std::vector<Unit> *V, long *Epoch, + size_t MaxSize, bool ExitOnError, + std::vector<std::string> *VPaths) { long E = Epoch ? *Epoch : 0; - Vector<std::string> Files; + std::vector<std::string> Files; ListFilesInDirRecursive(Path, Epoch, &Files, /*TopDir*/true); size_t NumLoaded = 0; for (size_t i = 0; i < Files.size(); i++) { @@ -103,14 +112,16 @@ void ReadDirToVectorOfUnits(const char *Path, Vector<Unit> *V, if ((NumLoaded & (NumLoaded - 1)) == 0 && NumLoaded >= 1024) Printf("Loaded %zd/%zd files from %s\n", NumLoaded, Files.size(), Path); auto S = FileToVector(X, MaxSize, ExitOnError); - if (!S.empty()) + if (!S.empty()) { V->push_back(S); + if (VPaths) + VPaths->push_back(X); + } } } - -void GetSizedFilesFromDir(const std::string &Dir, Vector<SizedFile> *V) { - Vector<std::string> Files; +void GetSizedFilesFromDir(const std::string &Dir, std::vector<SizedFile> *V) { + std::vector<std::string> Files; ListFilesInDirRecursive(Dir, 0, &Files, /*TopDir*/true); for (auto &File : Files) if (size_t Size = FileSize(File)) diff --git a/contrib/libs/libfuzzer12/FuzzerIO.h b/contrib/libs/libfuzzer/FuzzerIO.h index abd25110d0..401afa0b44 100644 --- a/contrib/libs/libfuzzer12/FuzzerIO.h +++ b/contrib/libs/libfuzzer/FuzzerIO.h @@ -32,8 +32,9 @@ void WriteToFile(const Unit &U, const std::string &Path); void AppendToFile(const uint8_t *Data, size_t Size, const std::string &Path); void AppendToFile(const std::string &Data, const std::string &Path); -void ReadDirToVectorOfUnits(const char *Path, Vector<Unit> *V, - long *Epoch, size_t MaxSize, bool ExitOnError); +void ReadDirToVectorOfUnits(const char *Path, std::vector<Unit> *V, long *Epoch, + size_t MaxSize, bool ExitOnError, + std::vector<std::string> *VPaths = 0); // Returns "Dir/FileName" or equivalent for the current OS. std::string DirPlusFile(const std::string &DirPath, @@ -53,6 +54,10 @@ void DupAndCloseStderr(); void CloseStdout(); +// For testing. +FILE *GetOutputFile(); +void SetOutputFile(FILE *NewOutputFile); + void Printf(const char *Fmt, ...); void VPrintf(bool Verbose, const char *Fmt, ...); @@ -65,7 +70,7 @@ bool IsDirectory(const std::string &Path); size_t FileSize(const std::string &Path); void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, - Vector<std::string> *V, bool TopDir); + std::vector<std::string> *V, bool TopDir); bool MkDirRecursive(const std::string &Dir); void RmDirRecursive(const std::string &Dir); @@ -84,7 +89,7 @@ struct SizedFile { bool operator<(const SizedFile &B) const { return Size < B.Size; } }; -void GetSizedFilesFromDir(const std::string &Dir, Vector<SizedFile> *V); +void GetSizedFilesFromDir(const std::string &Dir, std::vector<SizedFile> *V); char GetSeparator(); bool IsSeparator(char C); diff --git a/contrib/libs/libfuzzer12/FuzzerIOPosix.cpp b/contrib/libs/libfuzzer/FuzzerIOPosix.cpp index 4706a40959..3700fb098e 100644 --- a/contrib/libs/libfuzzer12/FuzzerIOPosix.cpp +++ b/contrib/libs/libfuzzer/FuzzerIOPosix.cpp @@ -53,7 +53,7 @@ std::string Basename(const std::string &Path) { } void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, - Vector<std::string> *V, bool TopDir) { + std::vector<std::string> *V, bool TopDir) { auto E = GetEpoch(Dir); if (Epoch) if (E && *Epoch >= E) return; @@ -78,7 +78,6 @@ void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, *Epoch = E; } - void IterateDirRecursive(const std::string &Dir, void (*DirPreCallback)(const std::string &Dir), void (*DirPostCallback)(const std::string &Dir), diff --git a/contrib/libs/libfuzzer12/FuzzerIOWindows.cpp b/contrib/libs/libfuzzer/FuzzerIOWindows.cpp index 61ad35e281..6771fc173c 100644 --- a/contrib/libs/libfuzzer12/FuzzerIOWindows.cpp +++ b/contrib/libs/libfuzzer/FuzzerIOWindows.cpp @@ -111,7 +111,7 @@ size_t FileSize(const std::string &Path) { } void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, - Vector<std::string> *V, bool TopDir) { + std::vector<std::string> *V, bool TopDir) { auto E = GetEpoch(Dir); if (Epoch) if (E && *Epoch >= E) return; @@ -159,7 +159,6 @@ void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, *Epoch = E; } - void IterateDirRecursive(const std::string &Dir, void (*DirPreCallback)(const std::string &Dir), void (*DirPostCallback)(const std::string &Dir), @@ -297,9 +296,8 @@ static size_t ParseServerAndShare(const std::string &FileName, return Pos - Offset; } -// Parse the given Ref string from the position Offset, to exactly match the given -// string Patt. -// Returns number of characters considered if successful. +// Parse the given Ref string from the position Offset, to exactly match the +// given string Patt. Returns number of characters considered if successful. static size_t ParseCustomString(const std::string &Ref, size_t Offset, const char *Patt) { size_t Len = strlen(Patt); diff --git a/contrib/libs/libfuzzer12/FuzzerInterface.h b/contrib/libs/libfuzzer/FuzzerInterface.h index f1130dc396..f1130dc396 100644 --- a/contrib/libs/libfuzzer12/FuzzerInterface.h +++ b/contrib/libs/libfuzzer/FuzzerInterface.h diff --git a/contrib/libs/libfuzzer12/FuzzerInternal.h b/contrib/libs/libfuzzer/FuzzerInternal.h index 37c8a01dc3..6637b0034e 100644 --- a/contrib/libs/libfuzzer12/FuzzerInternal.h +++ b/contrib/libs/libfuzzer/FuzzerInternal.h @@ -35,8 +35,8 @@ public: Fuzzer(UserCallback CB, InputCorpus &Corpus, MutationDispatcher &MD, FuzzingOptions Options); ~Fuzzer(); - void Loop(Vector<SizedFile> &CorporaFiles); - void ReadAndExecuteSeedCorpora(Vector<SizedFile> &CorporaFiles); + void Loop(std::vector<SizedFile> &CorporaFiles); + void ReadAndExecuteSeedCorpora(std::vector<SizedFile> &CorporaFiles); void MinimizeCrashLoop(const Unit &U); void RereadOutputCorpus(size_t MaxSize); @@ -72,8 +72,9 @@ public: void TPCUpdateObservedPCs(); // Merge Corpora[1:] into Corpora[0]. - void Merge(const Vector<std::string> &Corpora); - void CrashResistantMergeInternalStep(const std::string &ControlFilePath); + void Merge(const std::vector<std::string> &Corpora); + void CrashResistantMergeInternalStep(const std::string &ControlFilePath, + bool IsSetCoverMerge); MutationDispatcher &GetMD() { return MD; } void PrintFinalStats(); void SetMaxInputLen(size_t MaxInputLen); @@ -141,7 +142,7 @@ private: size_t MaxMutationLen = 0; size_t TmpMaxMutationLen = 0; - Vector<uint32_t> UniqFeatureSetTmp; + std::vector<uint32_t> UniqFeatureSetTmp; // Need to know our own thread. static thread_local bool IsMyThread; diff --git a/contrib/libs/libfuzzer12/FuzzerLoop.cpp b/contrib/libs/libfuzzer/FuzzerLoop.cpp index fc53ac086b..cbfe73d813 100644 --- a/contrib/libs/libfuzzer12/FuzzerLoop.cpp +++ b/contrib/libs/libfuzzer/FuzzerLoop.cpp @@ -390,7 +390,7 @@ void Fuzzer::SetMaxMutationLen(size_t MaxMutationLen) { void Fuzzer::CheckExitOnSrcPosOrItem() { if (!Options.ExitOnSrcPos.empty()) { - static auto *PCsSet = new Set<uintptr_t>; + static auto *PCsSet = new std::set<uintptr_t>; auto HandlePC = [&](const TracePC::PCTableEntry *TE) { if (!PCsSet->insert(TE->PC).second) return; @@ -415,20 +415,26 @@ void Fuzzer::CheckExitOnSrcPosOrItem() { void Fuzzer::RereadOutputCorpus(size_t MaxSize) { if (Options.OutputCorpus.empty() || !Options.ReloadIntervalSec) return; - Vector<Unit> AdditionalCorpus; - ReadDirToVectorOfUnits(Options.OutputCorpus.c_str(), &AdditionalCorpus, - &EpochOfLastReadOfOutputCorpus, MaxSize, - /*ExitOnError*/ false); + std::vector<Unit> AdditionalCorpus; + std::vector<std::string> AdditionalCorpusPaths; + ReadDirToVectorOfUnits( + Options.OutputCorpus.c_str(), &AdditionalCorpus, + &EpochOfLastReadOfOutputCorpus, MaxSize, + /*ExitOnError*/ false, + (Options.Verbosity >= 2 ? &AdditionalCorpusPaths : nullptr)); if (Options.Verbosity >= 2) Printf("Reload: read %zd new units.\n", AdditionalCorpus.size()); bool Reloaded = false; - for (auto &U : AdditionalCorpus) { + for (size_t i = 0; i != AdditionalCorpus.size(); ++i) { + auto &U = AdditionalCorpus[i]; if (U.size() > MaxSize) U.resize(MaxSize); if (!Corpus.HasUnit(U)) { if (RunOne(U.data(), U.size())) { CheckExitOnSrcPosOrItem(); Reloaded = true; + if (Options.Verbosity >= 2) + Printf("Reloaded %s\n", AdditionalCorpusPaths[i].c_str()); } } } @@ -442,8 +448,9 @@ void Fuzzer::PrintPulseAndReportSlowInput(const uint8_t *Data, size_t Size) { if (!(TotalNumberOfRuns & (TotalNumberOfRuns - 1)) && secondsSinceProcessStartUp() >= 2) PrintStats("pulse "); - if (TimeOfUnit > TimeOfLongestUnitInSeconds * 1.1 && - TimeOfUnit >= Options.ReportSlowUnits) { + auto Threshhold = + static_cast<long>(static_cast<double>(TimeOfLongestUnitInSeconds) * 1.1); + if (TimeOfUnit > Threshhold && TimeOfUnit >= Options.ReportSlowUnits) { TimeOfLongestUnitInSeconds = TimeOfUnit; Printf("Slowest unit: %zd s:\n", TimeOfLongestUnitInSeconds); WriteUnitToFileWithPrefix({Data, Data + Size}, "slow-unit-"); @@ -452,7 +459,7 @@ void Fuzzer::PrintPulseAndReportSlowInput(const uint8_t *Data, size_t Size) { static void WriteFeatureSetToFile(const std::string &FeaturesDir, const std::string &FileName, - const Vector<uint32_t> &FeatureSet) { + const std::vector<uint32_t> &FeatureSet) { if (FeaturesDir.empty() || FeatureSet.empty()) return; WriteToFile(reinterpret_cast<const uint8_t *>(FeatureSet.data()), FeatureSet.size() * sizeof(FeatureSet[0]), @@ -503,6 +510,8 @@ bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile, bool *FoundUniqFeatures) { if (!Size) return false; + // Largest input length should be INT_MAX. + assert(Size < std::numeric_limits<uint32_t>::max()); ExecuteCallback(Data, Size); auto TimeOfUnit = duration_cast<microseconds>(UnitStopTime - UnitStartTime); @@ -510,8 +519,8 @@ bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile, UniqFeatureSetTmp.clear(); size_t FoundUniqFeaturesOfII = 0; size_t NumUpdatesBefore = Corpus.NumFeatureUpdates(); - TPC.CollectFeatures([&](size_t Feature) { - if (Corpus.AddFeature(Feature, Size, Options.Shrink)) + TPC.CollectFeatures([&](uint32_t Feature) { + if (Corpus.AddFeature(Feature, static_cast<uint32_t>(Size), Options.Shrink)) UniqFeatureSetTmp.push_back(Feature); if (Options.Entropic) Corpus.UpdateFeatureFrequency(II, Feature); @@ -541,7 +550,7 @@ bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile, FoundUniqFeaturesOfII == II->UniqFeatureSet.size() && II->U.size() > Size) { auto OldFeaturesFile = Sha1ToString(II->Sha1); - Corpus.Replace(II, {Data, Data + Size}); + Corpus.Replace(II, {Data, Data + Size}, TimeOfUnit); RenameFeatureSetFile(Options.FeaturesDir, OldFeaturesFile, Sha1ToString(II->Sha1)); return true; @@ -577,7 +586,10 @@ static bool LooseMemeq(const uint8_t *A, const uint8_t *B, size_t Size) { !memcmp(A + Size - Limit / 2, B + Size - Limit / 2, Limit / 2); } -void Fuzzer::ExecuteCallback(const uint8_t *Data, size_t Size) { +// This method is not inlined because it would cause a test to fail where it +// is part of the stack unwinding. See D97975 for details. +ATTRIBUTE_NOINLINE void Fuzzer::ExecuteCallback(const uint8_t *Data, + size_t Size) { TPC.RecordInitialStack(); TotalNumberOfRuns++; assert(InFuzzingThread()); @@ -774,7 +786,7 @@ void Fuzzer::PurgeAllocator() { LastAllocatorPurgeAttemptTime = system_clock::now(); } -void Fuzzer::ReadAndExecuteSeedCorpora(Vector<SizedFile> &CorporaFiles) { +void Fuzzer::ReadAndExecuteSeedCorpora(std::vector<SizedFile> &CorporaFiles) { const size_t kMaxSaneLen = 1 << 20; const size_t kMinDefaultLen = 4096; size_t MaxSize = 0; @@ -839,7 +851,7 @@ void Fuzzer::ReadAndExecuteSeedCorpora(Vector<SizedFile> &CorporaFiles) { } } -void Fuzzer::Loop(Vector<SizedFile> &CorporaFiles) { +void Fuzzer::Loop(std::vector<SizedFile> &CorporaFiles) { auto FocusFunctionOrAuto = Options.FocusFunction; DFT.Init(Options.DataFlowTrace, &FocusFunctionOrAuto, CorporaFiles, MD.GetRand()); diff --git a/contrib/libs/libfuzzer12/FuzzerMain.cpp b/contrib/libs/libfuzzer/FuzzerMain.cpp index 75f2f8e75c..75f2f8e75c 100644 --- a/contrib/libs/libfuzzer12/FuzzerMain.cpp +++ b/contrib/libs/libfuzzer/FuzzerMain.cpp diff --git a/contrib/libs/libfuzzer12/FuzzerMerge.cpp b/contrib/libs/libfuzzer/FuzzerMerge.cpp index e3ad8b3851..24bd11958e 100644 --- a/contrib/libs/libfuzzer12/FuzzerMerge.cpp +++ b/contrib/libs/libfuzzer/FuzzerMerge.cpp @@ -77,14 +77,14 @@ bool Merger::Parse(std::istream &IS, bool ParseCoverage) { size_t ExpectedStartMarker = 0; const size_t kInvalidStartMarker = -1; size_t LastSeenStartMarker = kInvalidStartMarker; - Vector<uint32_t> TmpFeatures; - Set<uint32_t> PCs; + std::vector<uint32_t> TmpFeatures; + std::set<uint32_t> PCs; while (std::getline(IS, Line, '\n')) { std::istringstream ISS1(Line); std::string Marker; - size_t N; - ISS1 >> Marker; - ISS1 >> N; + uint32_t N; + if (!(ISS1 >> Marker) || !(ISS1 >> N)) + return false; if (Marker == "STARTED") { // STARTED FILE_ID FILE_SIZE if (ExpectedStartMarker != N) @@ -132,13 +132,16 @@ size_t Merger::ApproximateMemoryConsumption() const { // Decides which files need to be merged (add those to NewFiles). // Returns the number of new features added. -size_t Merger::Merge(const Set<uint32_t> &InitialFeatures, - Set<uint32_t> *NewFeatures, - const Set<uint32_t> &InitialCov, Set<uint32_t> *NewCov, - Vector<std::string> *NewFiles) { +size_t Merger::Merge(const std::set<uint32_t> &InitialFeatures, + std::set<uint32_t> *NewFeatures, + const std::set<uint32_t> &InitialCov, + std::set<uint32_t> *NewCov, + std::vector<std::string> *NewFiles) { NewFiles->clear(); + NewFeatures->clear(); + NewCov->clear(); assert(NumFilesInFirstCorpus <= Files.size()); - Set<uint32_t> AllFeatures = InitialFeatures; + std::set<uint32_t> AllFeatures = InitialFeatures; // What features are in the initial corpus? for (size_t i = 0; i < NumFilesInFirstCorpus; i++) { @@ -148,7 +151,7 @@ size_t Merger::Merge(const Set<uint32_t> &InitialFeatures, // Remove all features that we already know from all other inputs. for (size_t i = NumFilesInFirstCorpus; i < Files.size(); i++) { auto &Cur = Files[i].Features; - Vector<uint32_t> Tmp; + std::vector<uint32_t> Tmp; std::set_difference(Cur.begin(), Cur.end(), AllFeatures.begin(), AllFeatures.end(), std::inserter(Tmp, Tmp.begin())); Cur.swap(Tmp); @@ -186,15 +189,16 @@ size_t Merger::Merge(const Set<uint32_t> &InitialFeatures, return NewFeatures->size(); } -Set<uint32_t> Merger::AllFeatures() const { - Set<uint32_t> S; +std::set<uint32_t> Merger::AllFeatures() const { + std::set<uint32_t> S; for (auto &File : Files) S.insert(File.Features.begin(), File.Features.end()); return S; } // Inner process. May crash if the target crashes. -void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) { +void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath, + bool IsSetCoverMerge) { Printf("MERGE-INNER: using the control file '%s'\n", CFPath.c_str()); Merger M; std::ifstream IF(CFPath); @@ -210,11 +214,11 @@ void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) { M.Files.size() - M.FirstNotProcessedFile); std::ofstream OF(CFPath, std::ofstream::out | std::ofstream::app); - Set<size_t> AllFeatures; + std::set<size_t> AllFeatures; auto PrintStatsWrapper = [this, &AllFeatures](const char* Where) { this->PrintStats(Where, "\n", 0, AllFeatures.size()); }; - Set<const TracePC::PCTableEntry *> AllPCs; + std::set<const TracePC::PCTableEntry *> AllPCs; for (size_t i = M.FirstNotProcessedFile; i < M.Files.size(); i++) { Fuzzer::MaybeExitGracefully(); auto U = FileToVector(M.Files[i].Name); @@ -232,13 +236,14 @@ void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) { // Collect coverage. We are iterating over the files in this order: // * First, files in the initial corpus ordered by size, smallest first. // * Then, all other files, smallest first. - // So it makes no sense to record all features for all files, instead we - // only record features that were not seen before. - Set<size_t> UniqFeatures; - TPC.CollectFeatures([&](size_t Feature) { - if (AllFeatures.insert(Feature).second) - UniqFeatures.insert(Feature); - }); + std::set<size_t> Features; + if (IsSetCoverMerge) + TPC.CollectFeatures([&](size_t Feature) { Features.insert(Feature); }); + else + TPC.CollectFeatures([&](size_t Feature) { + if (AllFeatures.insert(Feature).second) + Features.insert(Feature); + }); TPC.UpdateObservedPCs(); // Show stats. if (!(TotalNumberOfRuns & (TotalNumberOfRuns - 1))) @@ -247,7 +252,7 @@ void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) { PrintStatsWrapper("LOADED"); // Write the post-run marker and the coverage. OF << "FT " << i; - for (size_t F : UniqFeatures) + for (size_t F : Features) OF << " " << F; OF << "\n"; OF << "COV " << i; @@ -261,15 +266,137 @@ void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) { PrintStatsWrapper("DONE "); } -static size_t WriteNewControlFile(const std::string &CFPath, - const Vector<SizedFile> &OldCorpus, - const Vector<SizedFile> &NewCorpus, - const Vector<MergeFileInfo> &KnownFiles) { +// Merges all corpora into the first corpus. A file is added into +// the first corpus only if it adds new features. Unlike `Merger::Merge`, +// this implementation calculates an approximation of the minimum set +// of corpora files, that cover all known features (set cover problem). +// Generally, this means that files with more features are preferred for +// merge into the first corpus. When two files have the same number of +// features, the smaller one is preferred. +size_t Merger::SetCoverMerge(const std::set<uint32_t> &InitialFeatures, + std::set<uint32_t> *NewFeatures, + const std::set<uint32_t> &InitialCov, + std::set<uint32_t> *NewCov, + std::vector<std::string> *NewFiles) { + assert(NumFilesInFirstCorpus <= Files.size()); + NewFiles->clear(); + NewFeatures->clear(); + NewCov->clear(); + std::set<uint32_t> AllFeatures; + // 1 << 21 - 1 is the maximum feature index. + // See 'kFeatureSetSize' in 'FuzzerCorpus.h'. + const uint32_t kFeatureSetSize = 1 << 21; + std::vector<bool> Covered(kFeatureSetSize, false); + size_t NumCovered = 0; + + std::set<uint32_t> ExistingFeatures = InitialFeatures; + for (size_t i = 0; i < NumFilesInFirstCorpus; ++i) + ExistingFeatures.insert(Files[i].Features.begin(), Files[i].Features.end()); + + // Mark the existing features as covered. + for (const auto &F : ExistingFeatures) { + if (!Covered[F % kFeatureSetSize]) { + ++NumCovered; + Covered[F % kFeatureSetSize] = true; + } + // Calculate an underestimation of the set of covered features + // since the `Covered` bitvector is smaller than the feature range. + AllFeatures.insert(F % kFeatureSetSize); + } + + std::set<size_t> RemainingFiles; + for (size_t i = NumFilesInFirstCorpus; i < Files.size(); ++i) { + // Construct an incremental sequence which represent the + // indices to all files (excluding those in the initial corpus). + // RemainingFiles = range(NumFilesInFirstCorpus..Files.size()). + RemainingFiles.insert(i); + // Insert this file's unique features to all features. + for (const auto &F : Files[i].Features) + AllFeatures.insert(F % kFeatureSetSize); + } + + // Integrate files into Covered until set is complete. + while (NumCovered != AllFeatures.size()) { + // Index to file with largest number of unique features. + size_t MaxFeaturesIndex = NumFilesInFirstCorpus; + // Indices to remove from RemainingFiles. + std::set<size_t> RemoveIndices; + // Running max unique feature count. + // Updated upon finding a file with more features. + size_t MaxNumFeatures = 0; + + // Iterate over all files not yet integrated into Covered, + // to find the file which has the largest number of + // features that are not already in Covered. + for (const auto &i : RemainingFiles) { + const auto &File = Files[i]; + size_t CurrentUnique = 0; + // Count number of features in this file + // which are not yet in Covered. + for (const auto &F : File.Features) + if (!Covered[F % kFeatureSetSize]) + ++CurrentUnique; + + if (CurrentUnique == 0) { + // All features in this file are already in Covered: skip next time. + RemoveIndices.insert(i); + } else if (CurrentUnique > MaxNumFeatures || + (CurrentUnique == MaxNumFeatures && + File.Size < Files[MaxFeaturesIndex].Size)) { + // Update the max features file based on unique features + // Break ties by selecting smaller files. + MaxNumFeatures = CurrentUnique; + MaxFeaturesIndex = i; + } + } + // Must be a valid index/ + assert(MaxFeaturesIndex < Files.size()); + // Remove any feature-less files found. + for (const auto &i : RemoveIndices) + RemainingFiles.erase(i); + if (MaxNumFeatures == 0) { + // Did not find a file that adds unique features. + // This means that we should have no remaining files. + assert(RemainingFiles.size() == 0); + assert(NumCovered == AllFeatures.size()); + break; + } + + // MaxFeaturesIndex must be an element of Remaining. + assert(RemainingFiles.find(MaxFeaturesIndex) != RemainingFiles.end()); + // Remove the file with the most features from Remaining. + RemainingFiles.erase(MaxFeaturesIndex); + const auto &MaxFeatureFile = Files[MaxFeaturesIndex]; + // Add the features of the max feature file to Covered. + for (const auto &F : MaxFeatureFile.Features) { + if (!Covered[F % kFeatureSetSize]) { + ++NumCovered; + Covered[F % kFeatureSetSize] = true; + NewFeatures->insert(F); + } + } + // Add the index to this file to the result. + NewFiles->push_back(MaxFeatureFile.Name); + // Update NewCov with the additional coverage + // that MaxFeatureFile provides. + for (const auto &C : MaxFeatureFile.Cov) + if (InitialCov.find(C) == InitialCov.end()) + NewCov->insert(C); + } + + return NewFeatures->size(); +} + +static size_t +WriteNewControlFile(const std::string &CFPath, + const std::vector<SizedFile> &OldCorpus, + const std::vector<SizedFile> &NewCorpus, + const std::vector<MergeFileInfo> &KnownFiles) { std::unordered_set<std::string> FilesToSkip; for (auto &SF: KnownFiles) FilesToSkip.insert(SF.Name); - Vector<std::string> FilesToUse; + std::vector<std::string> FilesToUse; auto MaybeUseFile = [=, &FilesToUse](std::string Name) { if (FilesToSkip.find(Name) == FilesToSkip.end()) FilesToUse.push_back(Name); @@ -297,19 +424,19 @@ static size_t WriteNewControlFile(const std::string &CFPath, } // Outer process. Does not call the target code and thus should not fail. -void CrashResistantMerge(const Vector<std::string> &Args, - const Vector<SizedFile> &OldCorpus, - const Vector<SizedFile> &NewCorpus, - Vector<std::string> *NewFiles, - const Set<uint32_t> &InitialFeatures, - Set<uint32_t> *NewFeatures, - const Set<uint32_t> &InitialCov, - Set<uint32_t> *NewCov, - const std::string &CFPath, - bool V /*Verbose*/) { +void CrashResistantMerge(const std::vector<std::string> &Args, + const std::vector<SizedFile> &OldCorpus, + const std::vector<SizedFile> &NewCorpus, + std::vector<std::string> *NewFiles, + const std::set<uint32_t> &InitialFeatures, + std::set<uint32_t> *NewFeatures, + const std::set<uint32_t> &InitialCov, + std::set<uint32_t> *NewCov, const std::string &CFPath, + bool V, /*Verbose*/ + bool IsSetCoverMerge) { if (NewCorpus.empty() && OldCorpus.empty()) return; // Nothing to merge. size_t NumAttempts = 0; - Vector<MergeFileInfo> KnownFiles; + std::vector<MergeFileInfo> KnownFiles; if (FileSize(CFPath)) { VPrintf(V, "MERGE-OUTER: non-empty control file provided: '%s'\n", CFPath.c_str()); @@ -361,6 +488,7 @@ void CrashResistantMerge(const Vector<std::string> &Args, // Every inner process should execute at least one input. Command BaseCmd(Args); BaseCmd.removeFlag("merge"); + BaseCmd.removeFlag("set_cover_merge"); BaseCmd.removeFlag("fork"); BaseCmd.removeFlag("collect_data_flow"); for (size_t Attempt = 1; Attempt <= NumAttempts; Attempt++) { @@ -368,14 +496,16 @@ void CrashResistantMerge(const Vector<std::string> &Args, VPrintf(V, "MERGE-OUTER: attempt %zd\n", Attempt); Command Cmd(BaseCmd); Cmd.addFlag("merge_control_file", CFPath); - Cmd.addFlag("merge_inner", "1"); + // If we are going to use the set cover implementation for + // minimization add the merge_inner=2 internal flag. + Cmd.addFlag("merge_inner", IsSetCoverMerge ? "2" : "1"); if (!V) { Cmd.setOutputFile(getDevNull()); Cmd.combineOutAndErr(); } auto ExitCode = ExecuteCommand(Cmd); if (!ExitCode) { - VPrintf(V, "MERGE-OUTER: succesfull in %zd attempt(s)\n", Attempt); + VPrintf(V, "MERGE-OUTER: successful in %zd attempt(s)\n", Attempt); break; } } @@ -393,7 +523,10 @@ void CrashResistantMerge(const Vector<std::string> &Args, M.ApproximateMemoryConsumption() >> 20, GetPeakRSSMb()); M.Files.insert(M.Files.end(), KnownFiles.begin(), KnownFiles.end()); - M.Merge(InitialFeatures, NewFeatures, InitialCov, NewCov, NewFiles); + if (IsSetCoverMerge) + M.SetCoverMerge(InitialFeatures, NewFeatures, InitialCov, NewCov, NewFiles); + else + M.Merge(InitialFeatures, NewFeatures, InitialCov, NewCov, NewFiles); VPrintf(V, "MERGE-OUTER: %zd new files with %zd new features added; " "%zd new coverage edges\n", NewFiles->size(), NewFeatures->size(), NewCov->size()); diff --git a/contrib/libs/libfuzzer12/FuzzerMerge.h b/contrib/libs/libfuzzer/FuzzerMerge.h index e0c6bc539b..42f798e1da 100644 --- a/contrib/libs/libfuzzer12/FuzzerMerge.h +++ b/contrib/libs/libfuzzer/FuzzerMerge.h @@ -41,6 +41,7 @@ #define LLVM_FUZZER_MERGE_H #include "FuzzerDefs.h" +#include "FuzzerIO.h" #include <istream> #include <ostream> @@ -52,11 +53,11 @@ namespace fuzzer { struct MergeFileInfo { std::string Name; size_t Size = 0; - Vector<uint32_t> Features, Cov; + std::vector<uint32_t> Features, Cov; }; struct Merger { - Vector<MergeFileInfo> Files; + std::vector<MergeFileInfo> Files; size_t NumFilesInFirstCorpus = 0; size_t FirstNotProcessedFile = 0; std::string LastFailure; @@ -64,23 +65,28 @@ struct Merger { bool Parse(std::istream &IS, bool ParseCoverage); bool Parse(const std::string &Str, bool ParseCoverage); void ParseOrExit(std::istream &IS, bool ParseCoverage); - size_t Merge(const Set<uint32_t> &InitialFeatures, Set<uint32_t> *NewFeatures, - const Set<uint32_t> &InitialCov, Set<uint32_t> *NewCov, - Vector<std::string> *NewFiles); + size_t Merge(const std::set<uint32_t> &InitialFeatures, + std::set<uint32_t> *NewFeatures, + const std::set<uint32_t> &InitialCov, std::set<uint32_t> *NewCov, + std::vector<std::string> *NewFiles); + size_t SetCoverMerge(const std::set<uint32_t> &InitialFeatures, + std::set<uint32_t> *NewFeatures, + const std::set<uint32_t> &InitialCov, + std::set<uint32_t> *NewCov, + std::vector<std::string> *NewFiles); size_t ApproximateMemoryConsumption() const; - Set<uint32_t> AllFeatures() const; + std::set<uint32_t> AllFeatures() const; }; -void CrashResistantMerge(const Vector<std::string> &Args, - const Vector<SizedFile> &OldCorpus, - const Vector<SizedFile> &NewCorpus, - Vector<std::string> *NewFiles, - const Set<uint32_t> &InitialFeatures, - Set<uint32_t> *NewFeatures, - const Set<uint32_t> &InitialCov, - Set<uint32_t> *NewCov, - const std::string &CFPath, - bool Verbose); +void CrashResistantMerge(const std::vector<std::string> &Args, + const std::vector<SizedFile> &OldCorpus, + const std::vector<SizedFile> &NewCorpus, + std::vector<std::string> *NewFiles, + const std::set<uint32_t> &InitialFeatures, + std::set<uint32_t> *NewFeatures, + const std::set<uint32_t> &InitialCov, + std::set<uint32_t> *NewCov, const std::string &CFPath, + bool Verbose, bool IsSetCoverMerge); } // namespace fuzzer diff --git a/contrib/libs/libfuzzer12/FuzzerMutate.cpp b/contrib/libs/libfuzzer/FuzzerMutate.cpp index cf34a9fe8e..d663900fdc 100644 --- a/contrib/libs/libfuzzer12/FuzzerMutate.cpp +++ b/contrib/libs/libfuzzer/FuzzerMutate.cpp @@ -61,14 +61,20 @@ MutationDispatcher::MutationDispatcher(Random &Rand, } static char RandCh(Random &Rand) { - if (Rand.RandBool()) return Rand(256); + if (Rand.RandBool()) + return static_cast<char>(Rand(256)); const char Special[] = "!*'();:@&=+$,/?%#[]012Az-`~.\xff\x00"; return Special[Rand(sizeof(Special) - 1)]; } size_t MutationDispatcher::Mutate_Custom(uint8_t *Data, size_t Size, size_t MaxSize) { - return EF->LLVMFuzzerCustomMutator(Data, Size, MaxSize, Rand.Rand()); + if (EF->__msan_unpoison) + EF->__msan_unpoison(Data, Size); + if (EF->__msan_unpoison_param) + EF->__msan_unpoison_param(4); + return EF->LLVMFuzzerCustomMutator(Data, Size, MaxSize, + Rand.Rand<unsigned int>()); } size_t MutationDispatcher::Mutate_CustomCrossOver(uint8_t *Data, size_t Size, @@ -81,8 +87,18 @@ size_t MutationDispatcher::Mutate_CustomCrossOver(uint8_t *Data, size_t Size, return 0; CustomCrossOverInPlaceHere.resize(MaxSize); auto &U = CustomCrossOverInPlaceHere; + + if (EF->__msan_unpoison) { + EF->__msan_unpoison(Data, Size); + EF->__msan_unpoison(Other.data(), Other.size()); + EF->__msan_unpoison(U.data(), U.size()); + } + if (EF->__msan_unpoison_param) + EF->__msan_unpoison_param(7); size_t NewSize = EF->LLVMFuzzerCustomCrossOver( - Data, Size, Other.data(), Other.size(), U.data(), U.size(), Rand.Rand()); + Data, Size, Other.data(), Other.size(), U.data(), U.size(), + Rand.Rand<unsigned int>()); + if (!NewSize) return 0; assert(NewSize <= MaxSize && "CustomCrossOver returned overisized unit"); @@ -135,7 +151,8 @@ size_t MutationDispatcher::Mutate_InsertRepeatedBytes(uint8_t *Data, // Insert new values at Data[Idx]. memmove(Data + Idx + N, Data + Idx, Size - Idx); // Give preference to 0x00 and 0xff. - uint8_t Byte = Rand.RandBool() ? Rand(256) : (Rand.RandBool() ? 0 : 255); + uint8_t Byte = static_cast<uint8_t>( + Rand.RandBool() ? Rand(256) : (Rand.RandBool() ? 0 : 255)); for (size_t i = 0; i < N; i++) Data[Idx + i] = Byte; return Size + N; @@ -178,7 +195,8 @@ size_t MutationDispatcher::ApplyDictionaryEntry(uint8_t *Data, size_t Size, Size += W.size(); } else { // Overwrite some bytes with W. if (W.size() > Size) return 0; - size_t Idx = UsePositionHint ? DE.GetPositionHint() : Rand(Size - W.size()); + size_t Idx = + UsePositionHint ? DE.GetPositionHint() : Rand(Size + 1 - W.size()); memcpy(Data + Idx, W.data(), W.size()); } return Size; @@ -227,8 +245,8 @@ DictionaryEntry MutationDispatcher::MakeDictionaryEntryFromCMP( T Arg1, T Arg2, const uint8_t *Data, size_t Size) { if (Rand.RandBool()) Arg1 = Bswap(Arg1); if (Rand.RandBool()) Arg2 = Bswap(Arg2); - T Arg1Mutation = Arg1 + Rand(-1, 1); - T Arg2Mutation = Arg2 + Rand(-1, 1); + T Arg1Mutation = static_cast<T>(Arg1 + Rand(-1, 1)); + T Arg2Mutation = static_cast<T>(Arg2 + Rand(-1, 1)); return MakeDictionaryEntryFromCMP(&Arg1, &Arg2, &Arg1Mutation, &Arg2Mutation, sizeof(Arg1), Data, Size); } @@ -245,23 +263,23 @@ size_t MutationDispatcher::Mutate_AddWordFromTORC( DictionaryEntry DE; switch (Rand(4)) { case 0: { - auto X = TPC.TORC8.Get(Rand.Rand()); + auto X = TPC.TORC8.Get(Rand.Rand<size_t>()); DE = MakeDictionaryEntryFromCMP(X.A, X.B, Data, Size); } break; case 1: { - auto X = TPC.TORC4.Get(Rand.Rand()); + auto X = TPC.TORC4.Get(Rand.Rand<size_t>()); if ((X.A >> 16) == 0 && (X.B >> 16) == 0 && Rand.RandBool()) DE = MakeDictionaryEntryFromCMP((uint16_t)X.A, (uint16_t)X.B, Data, Size); else DE = MakeDictionaryEntryFromCMP(X.A, X.B, Data, Size); } break; case 2: { - auto X = TPC.TORCW.Get(Rand.Rand()); + auto X = TPC.TORCW.Get(Rand.Rand<size_t>()); DE = MakeDictionaryEntryFromCMP(X.A, X.B, Data, Size); } break; case 3: if (Options.UseMemmem) { - auto X = TPC.MMT.Get(Rand.Rand()); - DE = DictionaryEntry(X); + auto X = TPC.MMT.Get(Rand.Rand<size_t>()); + DE = DictionaryEntry(X); } break; default: assert(0); @@ -387,12 +405,12 @@ size_t ChangeBinaryInteger(uint8_t *Data, size_t Size, Random &Rand) { assert(Off + sizeof(T) <= Size); T Val; if (Off < 64 && !Rand(4)) { - Val = Size; + Val = static_cast<T>(Size); if (Rand.RandBool()) Val = Bswap(Val); } else { memcpy(&Val, Data + Off, sizeof(Val)); - T Add = Rand(21); + T Add = static_cast<T>(Rand(21)); Add -= 10; if (Rand.RandBool()) Val = Bswap(T(Bswap(Val) + Add)); // Add assuming different endiannes. @@ -462,12 +480,12 @@ void MutationDispatcher::RecordSuccessfulMutationSequence() { assert(DE->GetW().size()); // Linear search is fine here as this happens seldom. if (!PersistentAutoDictionary.ContainsWord(DE->GetW())) - PersistentAutoDictionary.push_back({DE->GetW(), 1}); + PersistentAutoDictionary.push_back(*DE); } } void MutationDispatcher::PrintRecommendedDictionary() { - Vector<DictionaryEntry> V; + std::vector<DictionaryEntry> V; for (auto &DE : PersistentAutoDictionary) if (!ManualDictionary.ContainsWord(DE.GetW())) V.push_back(DE); @@ -522,7 +540,7 @@ size_t MutationDispatcher::DefaultMutate(uint8_t *Data, size_t Size, // Mutates Data in place, returns new size. size_t MutationDispatcher::MutateImpl(uint8_t *Data, size_t Size, size_t MaxSize, - Vector<Mutator> &Mutators) { + std::vector<Mutator> &Mutators) { assert(MaxSize > 0); // Some mutations may fail (e.g. can't insert more bytes if Size == MaxSize), // in which case they will return 0. @@ -544,7 +562,7 @@ size_t MutationDispatcher::MutateImpl(uint8_t *Data, size_t Size, // Mask represents the set of Data bytes that are worth mutating. size_t MutationDispatcher::MutateWithMask(uint8_t *Data, size_t Size, size_t MaxSize, - const Vector<uint8_t> &Mask) { + const std::vector<uint8_t> &Mask) { size_t MaskedSize = std::min(Size, Mask.size()); // * Copy the worthy bytes into a temporary array T // * Mutate T diff --git a/contrib/libs/libfuzzer12/FuzzerMutate.h b/contrib/libs/libfuzzer/FuzzerMutate.h index fd37191156..97704e2160 100644 --- a/contrib/libs/libfuzzer12/FuzzerMutate.h +++ b/contrib/libs/libfuzzer/FuzzerMutate.h @@ -77,7 +77,7 @@ public: /// that have '1' in Mask. /// Mask.size() should be >= Size. size_t MutateWithMask(uint8_t *Data, size_t Size, size_t MaxSize, - const Vector<uint8_t> &Mask); + const std::vector<uint8_t> &Mask); /// Applies one of the default mutations. Provided as a service /// to mutation authors. @@ -104,7 +104,7 @@ public: size_t AddWordFromDictionary(Dictionary &D, uint8_t *Data, size_t Size, size_t MaxSize); size_t MutateImpl(uint8_t *Data, size_t Size, size_t MaxSize, - Vector<Mutator> &Mutators); + std::vector<Mutator> &Mutators); size_t InsertPartOf(const uint8_t *From, size_t FromSize, uint8_t *To, size_t ToSize, size_t MaxToSize); @@ -133,22 +133,22 @@ public: // entries that led to successful discoveries in the past mutations. Dictionary PersistentAutoDictionary; - Vector<DictionaryEntry *> CurrentDictionaryEntrySequence; + std::vector<DictionaryEntry *> CurrentDictionaryEntrySequence; static const size_t kCmpDictionaryEntriesDequeSize = 16; DictionaryEntry CmpDictionaryEntriesDeque[kCmpDictionaryEntriesDequeSize]; size_t CmpDictionaryEntriesDequeIdx = 0; const Unit *CrossOverWith = nullptr; - Vector<uint8_t> MutateInPlaceHere; - Vector<uint8_t> MutateWithMaskTemp; + std::vector<uint8_t> MutateInPlaceHere; + std::vector<uint8_t> MutateWithMaskTemp; // CustomCrossOver needs its own buffer as a custom implementation may call // LLVMFuzzerMutate, which in turn may resize MutateInPlaceHere. - Vector<uint8_t> CustomCrossOverInPlaceHere; + std::vector<uint8_t> CustomCrossOverInPlaceHere; - Vector<Mutator> Mutators; - Vector<Mutator> DefaultMutators; - Vector<Mutator> CurrentMutatorSequence; + std::vector<Mutator> Mutators; + std::vector<Mutator> DefaultMutators; + std::vector<Mutator> CurrentMutatorSequence; }; } // namespace fuzzer diff --git a/contrib/libs/libfuzzer12/FuzzerOptions.h b/contrib/libs/libfuzzer/FuzzerOptions.h index 2e8501626b..83d64c2b9e 100644 --- a/contrib/libs/libfuzzer12/FuzzerOptions.h +++ b/contrib/libs/libfuzzer/FuzzerOptions.h @@ -48,6 +48,7 @@ struct FuzzingOptions { int ReportSlowUnits = 10; bool OnlyASCII = false; bool Entropic = true; + bool ForkCorpusGroups = false; size_t EntropicFeatureFrequencyThreshold = 0xFF; size_t EntropicNumberOfRarestFeatures = 100; bool EntropicScalePerExecTime = false; diff --git a/contrib/libs/libfuzzer12/FuzzerPlatform.h b/contrib/libs/libfuzzer/FuzzerPlatform.h index 1602e67895..1602e67895 100644 --- a/contrib/libs/libfuzzer12/FuzzerPlatform.h +++ b/contrib/libs/libfuzzer/FuzzerPlatform.h diff --git a/contrib/libs/libfuzzer12/FuzzerRandom.h b/contrib/libs/libfuzzer/FuzzerRandom.h index 659283eee2..ad6c07eb5e 100644 --- a/contrib/libs/libfuzzer12/FuzzerRandom.h +++ b/contrib/libs/libfuzzer/FuzzerRandom.h @@ -18,18 +18,27 @@ class Random : public std::minstd_rand { public: Random(unsigned int seed) : std::minstd_rand(seed) {} result_type operator()() { return this->std::minstd_rand::operator()(); } - size_t Rand() { return this->operator()(); } - size_t RandBool() { return Rand() % 2; } + template <typename T> + typename std::enable_if<std::is_integral<T>::value, T>::type Rand() { + return static_cast<T>(this->operator()()); + } + size_t RandBool() { return this->operator()() % 2; } size_t SkewTowardsLast(size_t n) { size_t T = this->operator()(n * n); - size_t Res = sqrt(T); + size_t Res = static_cast<size_t>(sqrt(T)); return Res; } - size_t operator()(size_t n) { return n ? Rand() % n : 0; } - intptr_t operator()(intptr_t From, intptr_t To) { + template <typename T> + typename std::enable_if<std::is_integral<T>::value, T>::type operator()(T n) { + return n ? Rand<T>() % n : 0; + } + template <typename T> + typename std::enable_if<std::is_integral<T>::value, T>::type + operator()(T From, T To) { assert(From < To); - intptr_t RangeSize = To - From + 1; - return operator()(RangeSize) + From; + auto RangeSize = static_cast<unsigned long long>(To) - + static_cast<unsigned long long>(From) + 1; + return static_cast<T>(this->operator()(RangeSize) + From); } }; diff --git a/contrib/libs/libfuzzer12/FuzzerSHA1.cpp b/contrib/libs/libfuzzer/FuzzerSHA1.cpp index 2005dc7003..b05655c8ef 100644 --- a/contrib/libs/libfuzzer12/FuzzerSHA1.cpp +++ b/contrib/libs/libfuzzer/FuzzerSHA1.cpp @@ -134,12 +134,13 @@ void sha1_hashBlock(sha1nfo *s) { s->state[4] += e; } -void sha1_addUncounted(sha1nfo *s, uint8_t data) { - uint8_t * const b = (uint8_t*) s->buffer; +// Adds the least significant byte of |data|. +void sha1_addUncounted(sha1nfo *s, uint32_t data) { + uint8_t *const b = (uint8_t *)s->buffer; #ifdef SHA_BIG_ENDIAN - b[s->bufferOffset] = data; + b[s->bufferOffset] = static_cast<uint8_t>(data); #else - b[s->bufferOffset ^ 3] = data; + b[s->bufferOffset ^ 3] = static_cast<uint8_t>(data); #endif s->bufferOffset++; if (s->bufferOffset == BLOCK_LENGTH) { diff --git a/contrib/libs/libfuzzer12/FuzzerSHA1.h b/contrib/libs/libfuzzer/FuzzerSHA1.h index 05cbacda87..05cbacda87 100644 --- a/contrib/libs/libfuzzer12/FuzzerSHA1.h +++ b/contrib/libs/libfuzzer/FuzzerSHA1.h diff --git a/contrib/libs/libfuzzer12/FuzzerTracePC.cpp b/contrib/libs/libfuzzer/FuzzerTracePC.cpp index 91e94d8240..af8d1ce50f 100644 --- a/contrib/libs/libfuzzer12/FuzzerTracePC.cpp +++ b/contrib/libs/libfuzzer/FuzzerTracePC.cpp @@ -106,6 +106,15 @@ void TracePC::PrintModuleInfo() { } if (size_t NumExtraCounters = ExtraCountersEnd() - ExtraCountersBegin()) Printf("INFO: %zd Extra Counters\n", NumExtraCounters); + + size_t MaxFeatures = CollectFeatures([](uint32_t) {}); + if (MaxFeatures > std::numeric_limits<uint32_t>::max()) + Printf("WARNING: The coverage PC tables may produce up to %zu features.\n" + "This exceeds the maximum 32-bit value. Some features may be\n" + "ignored, and fuzzing may become less precise. If possible,\n" + "consider refactoring the fuzzer into several smaller fuzzers\n" + "linked against only a portion of the current target.\n", + MaxFeatures); } ATTRIBUTE_NO_SANITIZE_ALL @@ -148,7 +157,7 @@ ALWAYS_INLINE uintptr_t TracePC::GetNextInstructionPc(uintptr_t PC) { } void TracePC::UpdateObservedPCs() { - Vector<uintptr_t> CoveredFuncs; + std::vector<uintptr_t> CoveredFuncs; auto ObservePC = [&](const PCTableEntry *TE) { if (ObservedPCs.insert(TE).second && DoPrintNewPCs) { PrintPC("\tNEW_PC: %p %F %L", "\tNEW_PC: %p", @@ -291,8 +300,8 @@ void TracePC::PrintCoverage(bool PrintAllCounters) { FunctionStr = FunctionStr.substr(3); std::string LineStr = DescribePC("%l", VisualizePC); size_t NumEdges = Last - First; - Vector<uintptr_t> UncoveredPCs; - Vector<uintptr_t> CoveredPCs; + std::vector<uintptr_t> UncoveredPCs; + std::vector<uintptr_t> CoveredPCs; for (auto TE = First; TE < Last; TE++) if (!ObservedPCs.count(TE)) UncoveredPCs.push_back(TE->PC); @@ -356,7 +365,7 @@ void TracePC::AddValueForMemcmp(void *caller_pc, const void *s1, const void *s2, uint8_t HammingDistance = 0; for (; I < Len; I++) { if (B1[I] != B2[I] || (StopAtZero && B1[I] == 0)) { - HammingDistance = Popcountll(B1[I] ^ B2[I]); + HammingDistance = static_cast<uint8_t>(Popcountll(B1[I] ^ B2[I])); break; } } @@ -382,6 +391,7 @@ void TracePC::HandleCmp(uintptr_t PC, T Arg1, T Arg2) { ValueProfileMap.AddValue(PC * 128 + 64 + AbsoluteDistance); } +ATTRIBUTE_NO_SANITIZE_MEMORY static size_t InternalStrnlen(const char *S, size_t MaxLen) { size_t Len = 0; for (; Len < MaxLen && S[Len]; Len++) {} @@ -389,7 +399,8 @@ static size_t InternalStrnlen(const char *S, size_t MaxLen) { } // Finds min of (strlen(S1), strlen(S2)). -// Needed bacause one of these strings may actually be non-zero terminated. +// Needed because one of these strings may actually be non-zero terminated. +ATTRIBUTE_NO_SANITIZE_MEMORY static size_t InternalStrnlen2(const char *S1, const char *S2) { size_t Len = 0; for (; S1[Len] && S2[Len]; Len++) {} diff --git a/contrib/libs/libfuzzer12/FuzzerTracePC.h b/contrib/libs/libfuzzer/FuzzerTracePC.h index 0090923073..af1f9d81e9 100644 --- a/contrib/libs/libfuzzer12/FuzzerTracePC.h +++ b/contrib/libs/libfuzzer/FuzzerTracePC.h @@ -54,7 +54,7 @@ struct MemMemTable { void Add(const uint8_t *Data, size_t Size) { if (Size <= 2) return; Size = std::min(Size, Word::GetMaxSize()); - size_t Idx = SimpleFastHash(Data, Size) % kSize; + auto Idx = SimpleFastHash(Data, Size) % kSize; MemMemWords[Idx].Set(Data, Size); } const Word &Get(size_t Idx) { @@ -79,7 +79,7 @@ class TracePC { void SetPrintNewPCs(bool P) { DoPrintNewPCs = P; } void SetPrintNewFuncs(size_t P) { NumPrintNewFuncs = P; } void UpdateObservedPCs(); - template <class Callback> void CollectFeatures(Callback CB) const; + template <class Callback> size_t CollectFeatures(Callback CB) const; void ResetMaps() { ValueProfileMap.Reset(); @@ -169,7 +169,7 @@ private: size_t NumPCTables; size_t NumPCsInPCTables; - Set<const PCTableEntry*> ObservedPCs; + std::set<const PCTableEntry *> ObservedPCs; std::unordered_map<uintptr_t, uintptr_t> ObservedFuncs; // PC => Counter. uint8_t *FocusFunctionCounterPtr = nullptr; @@ -193,7 +193,7 @@ size_t ForEachNonZeroByte(const uint8_t *Begin, const uint8_t *End, Handle8bitCounter(FirstFeature, P - Begin, V); // Iterate by Step bytes at a time. - for (; P < End; P += Step) + for (; P + Step <= End; P += Step) if (LargeType Bundle = *reinterpret_cast<const LargeType *>(P)) { Bundle = HostToLE(Bundle); for (size_t I = 0; I < Step; I++, Bundle >>= 8) @@ -234,16 +234,16 @@ unsigned CounterToFeature(T Counter) { return Bit; } -template <class Callback> // void Callback(size_t Feature) -ATTRIBUTE_NO_SANITIZE_ADDRESS -ATTRIBUTE_NOINLINE -void TracePC::CollectFeatures(Callback HandleFeature) const { +template <class Callback> // void Callback(uint32_t Feature) +ATTRIBUTE_NO_SANITIZE_ADDRESS ATTRIBUTE_NOINLINE size_t +TracePC::CollectFeatures(Callback HandleFeature) const { auto Handle8bitCounter = [&](size_t FirstFeature, size_t Idx, uint8_t Counter) { if (UseCounters) - HandleFeature(FirstFeature + Idx * 8 + CounterToFeature(Counter)); + HandleFeature(static_cast<uint32_t>(FirstFeature + Idx * 8 + + CounterToFeature(Counter))); else - HandleFeature(FirstFeature + Idx); + HandleFeature(static_cast<uint32_t>(FirstFeature + Idx)); }; size_t FirstFeature = 0; @@ -263,16 +263,18 @@ void TracePC::CollectFeatures(Callback HandleFeature) const { if (UseValueProfileMask) { ValueProfileMap.ForEach([&](size_t Idx) { - HandleFeature(FirstFeature + Idx); + HandleFeature(static_cast<uint32_t>(FirstFeature + Idx)); }); FirstFeature += ValueProfileMap.SizeInBits(); } // Step function, grows similar to 8 * Log_2(A). - auto StackDepthStepFunction = [](uint32_t A) -> uint32_t { - if (!A) return A; - uint32_t Log2 = Log(A); - if (Log2 < 3) return A; + auto StackDepthStepFunction = [](size_t A) -> size_t { + if (!A) + return A; + auto Log2 = Log(A); + if (Log2 < 3) + return A; Log2 -= 3; return (Log2 + 1) * 8 + ((A >> Log2) & 7); }; @@ -280,8 +282,13 @@ void TracePC::CollectFeatures(Callback HandleFeature) const { assert(StackDepthStepFunction(1024 * 4) == 80); assert(StackDepthStepFunction(1024 * 1024) == 144); - if (auto MaxStackOffset = GetMaxStackOffset()) - HandleFeature(FirstFeature + StackDepthStepFunction(MaxStackOffset / 8)); + if (auto MaxStackOffset = GetMaxStackOffset()) { + HandleFeature(static_cast<uint32_t>( + FirstFeature + StackDepthStepFunction(MaxStackOffset / 8))); + FirstFeature += StackDepthStepFunction(std::numeric_limits<size_t>::max()); + } + + return FirstFeature; } extern TracePC TPC; diff --git a/contrib/libs/libfuzzer12/FuzzerUtil.cpp b/contrib/libs/libfuzzer/FuzzerUtil.cpp index 7eecb68d07..aeab70f20c 100644 --- a/contrib/libs/libfuzzer12/FuzzerUtil.cpp +++ b/contrib/libs/libfuzzer/FuzzerUtil.cpp @@ -43,7 +43,7 @@ void PrintASCIIByte(uint8_t Byte) { else if (Byte >= 32 && Byte < 127) Printf("%c", Byte); else - Printf("\\x%02x", Byte); + Printf("\\%03o", Byte); } void PrintASCII(const uint8_t *Data, size_t Size, const char *PrintAfter) { @@ -111,7 +111,7 @@ bool ParseOneDictionaryEntry(const std::string &Str, Unit *U) { char Hex[] = "0xAA"; Hex[2] = Str[Pos + 2]; Hex[3] = Str[Pos + 3]; - U->push_back(strtol(Hex, nullptr, 16)); + U->push_back(static_cast<uint8_t>(strtol(Hex, nullptr, 16))); Pos += 3; continue; } @@ -124,7 +124,7 @@ bool ParseOneDictionaryEntry(const std::string &Str, Unit *U) { return true; } -bool ParseDictionaryFile(const std::string &Text, Vector<Unit> *Units) { +bool ParseDictionaryFile(const std::string &Text, std::vector<Unit> *Units) { if (Text.empty()) { Printf("ParseDictionaryFile: file does not exist or is empty\n"); return false; @@ -226,10 +226,11 @@ unsigned NumberOfCpuCores() { return N; } -size_t SimpleFastHash(const uint8_t *Data, size_t Size) { - size_t Res = 0; +uint64_t SimpleFastHash(const void *Data, size_t Size, uint64_t Initial) { + uint64_t Res = Initial; + const uint8_t *Bytes = static_cast<const uint8_t *>(Data); for (size_t i = 0; i < Size; i++) - Res = Res * 11 + Data[i]; + Res = Res * 11 + Bytes[i]; return Res; } diff --git a/contrib/libs/libfuzzer12/FuzzerUtil.h b/contrib/libs/libfuzzer/FuzzerUtil.h index e90be08500..71d49097e5 100644 --- a/contrib/libs/libfuzzer12/FuzzerUtil.h +++ b/contrib/libs/libfuzzer/FuzzerUtil.h @@ -66,10 +66,10 @@ int CloseProcessPipe(FILE *F); const void *SearchMemory(const void *haystack, size_t haystacklen, const void *needle, size_t needlelen); -std::string CloneArgsWithoutX(const Vector<std::string> &Args, +std::string CloneArgsWithoutX(const std::vector<std::string> &Args, const char *X1, const char *X2); -inline std::string CloneArgsWithoutX(const Vector<std::string> &Args, +inline std::string CloneArgsWithoutX(const std::vector<std::string> &Args, const char *X) { return CloneArgsWithoutX(Args, X, X); } @@ -88,9 +88,11 @@ std::string DisassembleCmd(const std::string &FileName); std::string SearchRegexCmd(const std::string &Regex); -size_t SimpleFastHash(const uint8_t *Data, size_t Size); +uint64_t SimpleFastHash(const void *Data, size_t Size, uint64_t Initial = 0); -inline uint32_t Log(uint32_t X) { return 32 - Clz(X) - 1; } +inline size_t Log(size_t X) { + return static_cast<size_t>((sizeof(unsigned long long) * 8) - Clzll(X) - 1); +} inline size_t PageSize() { return 4096; } inline uint8_t *RoundUpByPage(uint8_t *P) { diff --git a/contrib/libs/libfuzzer12/FuzzerUtilDarwin.cpp b/contrib/libs/libfuzzer/FuzzerUtilDarwin.cpp index a5bed658a4..a5bed658a4 100644 --- a/contrib/libs/libfuzzer12/FuzzerUtilDarwin.cpp +++ b/contrib/libs/libfuzzer/FuzzerUtilDarwin.cpp diff --git a/contrib/libs/libfuzzer12/FuzzerUtilLinux.cpp b/contrib/libs/libfuzzer/FuzzerUtilLinux.cpp index 981f9a8b42..981f9a8b42 100644 --- a/contrib/libs/libfuzzer12/FuzzerUtilLinux.cpp +++ b/contrib/libs/libfuzzer/FuzzerUtilLinux.cpp diff --git a/contrib/libs/libfuzzer12/FuzzerUtilPosix.cpp b/contrib/libs/libfuzzer/FuzzerUtilPosix.cpp index afb733409a..0446d732a9 100644 --- a/contrib/libs/libfuzzer12/FuzzerUtilPosix.cpp +++ b/contrib/libs/libfuzzer/FuzzerUtilPosix.cpp @@ -77,10 +77,13 @@ static void SetSigaction(int signum, return; } - sigact = {}; - sigact.sa_flags = SA_SIGINFO; - sigact.sa_sigaction = callback; - if (sigaction(signum, &sigact, 0)) { + struct sigaction new_sigact = {}; + // Address sanitizer needs SA_ONSTACK (causing the signal handler to run on a + // dedicated stack) in order to be able to detect stack overflows; keep the + // flag if it's set. + new_sigact.sa_flags = SA_SIGINFO | (sigact.sa_flags & SA_ONSTACK); + new_sigact.sa_sigaction = callback; + if (sigaction(signum, &new_sigact, nullptr)) { Printf("libFuzzer: sigaction failed with %d\n", errno); exit(1); } diff --git a/contrib/libs/libfuzzer12/FuzzerUtilWindows.cpp b/contrib/libs/libfuzzer/FuzzerUtilWindows.cpp index 1a54bb569e..3598758dbb 100644 --- a/contrib/libs/libfuzzer12/FuzzerUtilWindows.cpp +++ b/contrib/libs/libfuzzer/FuzzerUtilWindows.cpp @@ -204,7 +204,7 @@ const void *SearchMemory(const void *Data, size_t DataLen, const void *Patt, } std::string DisassembleCmd(const std::string &FileName) { - Vector<std::string> command_vector; + std::vector<std::string> command_vector; command_vector.push_back("dumpbin /summary > nul"); if (ExecuteCommand(Command(command_vector)) == 0) return "dumpbin /disasm " + FileName; diff --git a/contrib/libs/libfuzzer12/FuzzerValueBitMap.h b/contrib/libs/libfuzzer/FuzzerValueBitMap.h index ddbfe200af..ddbfe200af 100644 --- a/contrib/libs/libfuzzer12/FuzzerValueBitMap.h +++ b/contrib/libs/libfuzzer/FuzzerValueBitMap.h diff --git a/contrib/libs/libfuzzer12/LICENSE.TXT b/contrib/libs/libfuzzer/LICENSE.TXT index 5a79a1b9d5..5a79a1b9d5 100644 --- a/contrib/libs/libfuzzer12/LICENSE.TXT +++ b/contrib/libs/libfuzzer/LICENSE.TXT diff --git a/contrib/libs/libfuzzer/README.txt b/contrib/libs/libfuzzer/README.txt new file mode 100644 index 0000000000..fc8843246e --- /dev/null +++ b/contrib/libs/libfuzzer/README.txt @@ -0,0 +1,11 @@ +Compiler-RT +================================ + +This directory and its subdirectories contain source code for the compiler +support routines. + +Compiler-RT is open source software. You may freely distribute it under the +terms of the license agreement found in LICENSE.txt. + +================================ + diff --git a/contrib/libs/libfuzzer12/README.txt b/contrib/libs/libfuzzer12/README.txt deleted file mode 100644 index 3eee01c776..0000000000 --- a/contrib/libs/libfuzzer12/README.txt +++ /dev/null @@ -1 +0,0 @@ -See http://llvm.org/docs/LibFuzzer.html |