aboutsummaryrefslogtreecommitdiffstats
path: root/contrib/libs
diff options
context:
space:
mode:
authorthegeorg <thegeorg@yandex-team.com>2022-11-07 19:34:08 +0300
committerthegeorg <thegeorg@yandex-team.com>2022-11-07 19:34:08 +0300
commit50f76e264c70a223a34b24aa59e97bff97128f4c (patch)
treee604247a10f09df6158c172577b9bfa431f1e1b5 /contrib/libs
parent278a58c5af63dbd7f7a6d8b8d92dc246651242da (diff)
downloadydb-50f76e264c70a223a34b24aa59e97bff97128f4c.tar.gz
Switch fuzz tests to contrib/libs/libfuzzer
Diffstat (limited to 'contrib/libs')
-rw-r--r--contrib/libs/libfuzzer/CODE_OWNERS.TXT (renamed from contrib/libs/libfuzzer12/CODE_OWNERS.TXT)0
-rw-r--r--contrib/libs/libfuzzer/CREDITS.TXT (renamed from contrib/libs/libfuzzer12/CREDITS.TXT)0
-rw-r--r--contrib/libs/libfuzzer/FuzzerBuiltins.h (renamed from contrib/libs/libfuzzer12/FuzzerBuiltins.h)1
-rw-r--r--contrib/libs/libfuzzer/FuzzerBuiltinsMsvc.h (renamed from contrib/libs/libfuzzer12/FuzzerBuiltinsMsvc.h)9
-rw-r--r--contrib/libs/libfuzzer/FuzzerCommand.h (renamed from contrib/libs/libfuzzer12/FuzzerCommand.h)12
-rw-r--r--contrib/libs/libfuzzer/FuzzerCorpus.h (renamed from contrib/libs/libfuzzer12/FuzzerCorpus.h)63
-rw-r--r--contrib/libs/libfuzzer/FuzzerCrossOver.cpp (renamed from contrib/libs/libfuzzer12/FuzzerCrossOver.cpp)0
-rw-r--r--contrib/libs/libfuzzer/FuzzerDataFlowTrace.cpp (renamed from contrib/libs/libfuzzer12/FuzzerDataFlowTrace.cpp)36
-rw-r--r--contrib/libs/libfuzzer/FuzzerDataFlowTrace.h (renamed from contrib/libs/libfuzzer12/FuzzerDataFlowTrace.h)30
-rw-r--r--contrib/libs/libfuzzer/FuzzerDefs.h (renamed from contrib/libs/libfuzzer12/FuzzerDefs.h)24
-rw-r--r--contrib/libs/libfuzzer/FuzzerDictionary.h (renamed from contrib/libs/libfuzzer12/FuzzerDictionary.h)19
-rw-r--r--contrib/libs/libfuzzer/FuzzerDriver.cpp (renamed from contrib/libs/libfuzzer12/FuzzerDriver.cpp)86
-rw-r--r--contrib/libs/libfuzzer/FuzzerExtFunctions.def (renamed from contrib/libs/libfuzzer12/FuzzerExtFunctions.def)0
-rw-r--r--contrib/libs/libfuzzer/FuzzerExtFunctions.h (renamed from contrib/libs/libfuzzer12/FuzzerExtFunctions.h)0
-rw-r--r--contrib/libs/libfuzzer/FuzzerExtFunctionsDlsym.cpp (renamed from contrib/libs/libfuzzer12/FuzzerExtFunctionsDlsym.cpp)0
-rw-r--r--contrib/libs/libfuzzer/FuzzerExtFunctionsWeak.cpp (renamed from contrib/libs/libfuzzer12/FuzzerExtFunctionsWeak.cpp)0
-rw-r--r--contrib/libs/libfuzzer/FuzzerExtFunctionsWindows.cpp (renamed from contrib/libs/libfuzzer12/FuzzerExtFunctionsWindows.cpp)0
-rw-r--r--contrib/libs/libfuzzer/FuzzerExtraCounters.cpp (renamed from contrib/libs/libfuzzer12/FuzzerExtraCounters.cpp)8
-rw-r--r--contrib/libs/libfuzzer/FuzzerExtraCountersDarwin.cpp22
-rw-r--r--contrib/libs/libfuzzer/FuzzerExtraCountersWindows.cpp80
-rw-r--r--contrib/libs/libfuzzer/FuzzerFlags.def (renamed from contrib/libs/libfuzzer12/FuzzerFlags.def)9
-rw-r--r--contrib/libs/libfuzzer/FuzzerFork.cpp (renamed from contrib/libs/libfuzzer12/FuzzerFork.cpp)131
-rw-r--r--contrib/libs/libfuzzer/FuzzerFork.h (renamed from contrib/libs/libfuzzer12/FuzzerFork.h)4
-rw-r--r--contrib/libs/libfuzzer/FuzzerIO.cpp (renamed from contrib/libs/libfuzzer12/FuzzerIO.cpp)25
-rw-r--r--contrib/libs/libfuzzer/FuzzerIO.h (renamed from contrib/libs/libfuzzer12/FuzzerIO.h)13
-rw-r--r--contrib/libs/libfuzzer/FuzzerIOPosix.cpp (renamed from contrib/libs/libfuzzer12/FuzzerIOPosix.cpp)3
-rw-r--r--contrib/libs/libfuzzer/FuzzerIOWindows.cpp (renamed from contrib/libs/libfuzzer12/FuzzerIOWindows.cpp)8
-rw-r--r--contrib/libs/libfuzzer/FuzzerInterface.h (renamed from contrib/libs/libfuzzer12/FuzzerInterface.h)0
-rw-r--r--contrib/libs/libfuzzer/FuzzerInternal.h (renamed from contrib/libs/libfuzzer12/FuzzerInternal.h)11
-rw-r--r--contrib/libs/libfuzzer/FuzzerLoop.cpp (renamed from contrib/libs/libfuzzer12/FuzzerLoop.cpp)42
-rw-r--r--contrib/libs/libfuzzer/FuzzerMain.cpp (renamed from contrib/libs/libfuzzer12/FuzzerMain.cpp)0
-rw-r--r--contrib/libs/libfuzzer/FuzzerMerge.cpp (renamed from contrib/libs/libfuzzer12/FuzzerMerge.cpp)219
-rw-r--r--contrib/libs/libfuzzer/FuzzerMerge.h (renamed from contrib/libs/libfuzzer12/FuzzerMerge.h)38
-rw-r--r--contrib/libs/libfuzzer/FuzzerMutate.cpp (renamed from contrib/libs/libfuzzer12/FuzzerMutate.cpp)54
-rw-r--r--contrib/libs/libfuzzer/FuzzerMutate.h (renamed from contrib/libs/libfuzzer12/FuzzerMutate.h)18
-rw-r--r--contrib/libs/libfuzzer/FuzzerOptions.h (renamed from contrib/libs/libfuzzer12/FuzzerOptions.h)1
-rw-r--r--contrib/libs/libfuzzer/FuzzerPlatform.h (renamed from contrib/libs/libfuzzer12/FuzzerPlatform.h)0
-rw-r--r--contrib/libs/libfuzzer/FuzzerRandom.h (renamed from contrib/libs/libfuzzer12/FuzzerRandom.h)23
-rw-r--r--contrib/libs/libfuzzer/FuzzerSHA1.cpp (renamed from contrib/libs/libfuzzer12/FuzzerSHA1.cpp)9
-rw-r--r--contrib/libs/libfuzzer/FuzzerSHA1.h (renamed from contrib/libs/libfuzzer12/FuzzerSHA1.h)0
-rw-r--r--contrib/libs/libfuzzer/FuzzerTracePC.cpp (renamed from contrib/libs/libfuzzer12/FuzzerTracePC.cpp)21
-rw-r--r--contrib/libs/libfuzzer/FuzzerTracePC.h (renamed from contrib/libs/libfuzzer12/FuzzerTracePC.h)41
-rw-r--r--contrib/libs/libfuzzer/FuzzerUtil.cpp (renamed from contrib/libs/libfuzzer12/FuzzerUtil.cpp)13
-rw-r--r--contrib/libs/libfuzzer/FuzzerUtil.h (renamed from contrib/libs/libfuzzer12/FuzzerUtil.h)10
-rw-r--r--contrib/libs/libfuzzer/FuzzerUtilDarwin.cpp (renamed from contrib/libs/libfuzzer12/FuzzerUtilDarwin.cpp)0
-rw-r--r--contrib/libs/libfuzzer/FuzzerUtilLinux.cpp (renamed from contrib/libs/libfuzzer12/FuzzerUtilLinux.cpp)0
-rw-r--r--contrib/libs/libfuzzer/FuzzerUtilPosix.cpp (renamed from contrib/libs/libfuzzer12/FuzzerUtilPosix.cpp)11
-rw-r--r--contrib/libs/libfuzzer/FuzzerUtilWindows.cpp (renamed from contrib/libs/libfuzzer12/FuzzerUtilWindows.cpp)2
-rw-r--r--contrib/libs/libfuzzer/FuzzerValueBitMap.h (renamed from contrib/libs/libfuzzer12/FuzzerValueBitMap.h)0
-rw-r--r--contrib/libs/libfuzzer/LICENSE.TXT (renamed from contrib/libs/libfuzzer12/LICENSE.TXT)0
-rw-r--r--contrib/libs/libfuzzer/README.txt11
-rw-r--r--contrib/libs/libfuzzer12/README.txt1
52 files changed, 759 insertions, 349 deletions
diff --git a/contrib/libs/libfuzzer12/CODE_OWNERS.TXT b/contrib/libs/libfuzzer/CODE_OWNERS.TXT
index 125487816b..125487816b 100644
--- a/contrib/libs/libfuzzer12/CODE_OWNERS.TXT
+++ b/contrib/libs/libfuzzer/CODE_OWNERS.TXT
diff --git a/contrib/libs/libfuzzer12/CREDITS.TXT b/contrib/libs/libfuzzer/CREDITS.TXT
index 6964eba020..6964eba020 100644
--- a/contrib/libs/libfuzzer12/CREDITS.TXT
+++ b/contrib/libs/libfuzzer/CREDITS.TXT
diff --git a/contrib/libs/libfuzzer12/FuzzerBuiltins.h b/contrib/libs/libfuzzer/FuzzerBuiltins.h
index 4c0ada8266..ce0bd5cb47 100644
--- a/contrib/libs/libfuzzer12/FuzzerBuiltins.h
+++ b/contrib/libs/libfuzzer/FuzzerBuiltins.h
@@ -26,7 +26,6 @@ inline uint32_t Bswap(uint32_t x) { return __builtin_bswap32(x); }
inline uint64_t Bswap(uint64_t x) { return __builtin_bswap64(x); }
inline uint32_t Clzll(unsigned long long X) { return __builtin_clzll(X); }
-inline uint32_t Clz(unsigned long long X) { return __builtin_clz(X); }
inline int Popcountll(unsigned long long X) { return __builtin_popcountll(X); }
} // namespace fuzzer
diff --git a/contrib/libs/libfuzzer12/FuzzerBuiltinsMsvc.h b/contrib/libs/libfuzzer/FuzzerBuiltinsMsvc.h
index c5bec9787d..421dee7f66 100644
--- a/contrib/libs/libfuzzer12/FuzzerBuiltinsMsvc.h
+++ b/contrib/libs/libfuzzer/FuzzerBuiltinsMsvc.h
@@ -41,7 +41,8 @@ inline uint32_t Clzll(uint64_t X) {
#if !defined(_M_ARM) && !defined(_M_X64)
// Scan the high 32 bits.
if (_BitScanReverse(&LeadZeroIdx, static_cast<unsigned long>(X >> 32)))
- return static_cast<int>(63 - (LeadZeroIdx + 32)); // Create a bit offset from the MSB.
+ return static_cast<int>(
+ 63 - (LeadZeroIdx + 32)); // Create a bit offset from the MSB.
// Scan the low 32 bits.
if (_BitScanReverse(&LeadZeroIdx, static_cast<unsigned long>(X)))
return static_cast<int>(63 - LeadZeroIdx);
@@ -52,12 +53,6 @@ inline uint32_t Clzll(uint64_t X) {
return 64;
}
-inline uint32_t Clz(uint32_t X) {
- unsigned long LeadZeroIdx = 0;
- if (_BitScanReverse(&LeadZeroIdx, X)) return 31 - LeadZeroIdx;
- return 32;
-}
-
inline int Popcountll(unsigned long long X) {
#if !defined(_M_ARM) && !defined(_M_X64)
return __popcnt(X) + __popcnt(X >> 32);
diff --git a/contrib/libs/libfuzzer12/FuzzerCommand.h b/contrib/libs/libfuzzer/FuzzerCommand.h
index 87308864af..f653fe3587 100644
--- a/contrib/libs/libfuzzer12/FuzzerCommand.h
+++ b/contrib/libs/libfuzzer/FuzzerCommand.h
@@ -33,7 +33,7 @@ public:
Command() : CombinedOutAndErr(false) {}
- explicit Command(const Vector<std::string> &ArgsToAdd)
+ explicit Command(const std::vector<std::string> &ArgsToAdd)
: Args(ArgsToAdd), CombinedOutAndErr(false) {}
explicit Command(const Command &Other)
@@ -58,7 +58,7 @@ public:
// Gets all of the current command line arguments, **including** those after
// "-ignore-remaining-args=1".
- const Vector<std::string> &getArguments() const { return Args; }
+ const std::vector<std::string> &getArguments() const { return Args; }
// Adds the given argument before "-ignore_remaining_args=1", or at the end
// if that flag isn't present.
@@ -68,7 +68,7 @@ public:
// Adds all given arguments before "-ignore_remaining_args=1", or at the end
// if that flag isn't present.
- void addArguments(const Vector<std::string> &ArgsToAdd) {
+ void addArguments(const std::vector<std::string> &ArgsToAdd) {
Args.insert(endMutableArgs(), ArgsToAdd.begin(), ArgsToAdd.end());
}
@@ -155,16 +155,16 @@ private:
Command(Command &&Other) = delete;
Command &operator=(Command &&Other) = delete;
- Vector<std::string>::iterator endMutableArgs() {
+ std::vector<std::string>::iterator endMutableArgs() {
return std::find(Args.begin(), Args.end(), ignoreRemainingArgs());
}
- Vector<std::string>::const_iterator endMutableArgs() const {
+ std::vector<std::string>::const_iterator endMutableArgs() const {
return std::find(Args.begin(), Args.end(), ignoreRemainingArgs());
}
// The command arguments. Args[0] is the command name.
- Vector<std::string> Args;
+ std::vector<std::string> Args;
// True indicates stderr is redirected to stdout.
bool CombinedOutAndErr;
diff --git a/contrib/libs/libfuzzer12/FuzzerCorpus.h b/contrib/libs/libfuzzer/FuzzerCorpus.h
index daea4f5213..e01891e18f 100644
--- a/contrib/libs/libfuzzer12/FuzzerCorpus.h
+++ b/contrib/libs/libfuzzer/FuzzerCorpus.h
@@ -39,13 +39,13 @@ struct InputInfo {
bool MayDeleteFile = false;
bool Reduced = false;
bool HasFocusFunction = false;
- Vector<uint32_t> UniqFeatureSet;
- Vector<uint8_t> DataFlowTraceForFocusFunction;
+ std::vector<uint32_t> UniqFeatureSet;
+ std::vector<uint8_t> DataFlowTraceForFocusFunction;
// Power schedule.
bool NeedsEnergyUpdate = false;
double Energy = 0.0;
- size_t SumIncidence = 0;
- Vector<std::pair<uint32_t, uint16_t>> FeatureFreqs;
+ double SumIncidence = 0.0;
+ std::vector<std::pair<uint32_t, uint16_t>> FeatureFreqs;
// Delete feature Idx and its frequency from FeatureFreqs.
bool DeleteFeatureFreq(uint32_t Idx) {
@@ -74,27 +74,28 @@ struct InputInfo {
void UpdateEnergy(size_t GlobalNumberOfFeatures, bool ScalePerExecTime,
std::chrono::microseconds AverageUnitExecutionTime) {
Energy = 0.0;
- SumIncidence = 0;
+ SumIncidence = 0.0;
// Apply add-one smoothing to locally discovered features.
for (auto F : FeatureFreqs) {
- size_t LocalIncidence = F.second + 1;
- Energy -= LocalIncidence * logl(LocalIncidence);
+ double LocalIncidence = F.second + 1;
+ Energy -= LocalIncidence * log(LocalIncidence);
SumIncidence += LocalIncidence;
}
// Apply add-one smoothing to locally undiscovered features.
- // PreciseEnergy -= 0; // since logl(1.0) == 0)
- SumIncidence += (GlobalNumberOfFeatures - FeatureFreqs.size());
+ // PreciseEnergy -= 0; // since log(1.0) == 0)
+ SumIncidence +=
+ static_cast<double>(GlobalNumberOfFeatures - FeatureFreqs.size());
// Add a single locally abundant feature apply add-one smoothing.
- size_t AbdIncidence = NumExecutedMutations + 1;
- Energy -= AbdIncidence * logl(AbdIncidence);
+ double AbdIncidence = static_cast<double>(NumExecutedMutations + 1);
+ Energy -= AbdIncidence * log(AbdIncidence);
SumIncidence += AbdIncidence;
// Normalize.
if (SumIncidence != 0)
- Energy = (Energy / SumIncidence) + logl(SumIncidence);
+ Energy = Energy / SumIncidence + log(SumIncidence);
if (ScalePerExecTime) {
// Scaling to favor inputs with lower execution time.
@@ -208,11 +209,13 @@ public:
InputInfo *AddToCorpus(const Unit &U, size_t NumFeatures, bool MayDeleteFile,
bool HasFocusFunction, bool NeverReduce,
std::chrono::microseconds TimeOfUnit,
- const Vector<uint32_t> &FeatureSet,
+ const std::vector<uint32_t> &FeatureSet,
const DataFlowTrace &DFT, const InputInfo *BaseII) {
assert(!U.empty());
if (FeatureDebug)
Printf("ADD_TO_CORPUS %zd NF %zd\n", Inputs.size(), NumFeatures);
+ // Inputs.size() is cast to uint32_t below.
+ assert(Inputs.size() < std::numeric_limits<uint32_t>::max());
Inputs.push_back(new InputInfo());
InputInfo &II = *Inputs.back();
II.U = U;
@@ -224,7 +227,7 @@ public:
II.HasFocusFunction = HasFocusFunction;
// Assign maximal energy to the new seed.
II.Energy = RareFeatures.empty() ? 1.0 : log(RareFeatures.size());
- II.SumIncidence = RareFeatures.size();
+ II.SumIncidence = static_cast<double>(RareFeatures.size());
II.NeedsEnergyUpdate = false;
std::sort(II.UniqFeatureSet.begin(), II.UniqFeatureSet.end());
ComputeSHA1(U.data(), U.size(), II.Sha1);
@@ -255,7 +258,7 @@ public:
}
// Debug-only
- void PrintFeatureSet(const Vector<uint32_t> &FeatureSet) {
+ void PrintFeatureSet(const std::vector<uint32_t> &FeatureSet) {
if (!FeatureDebug) return;
Printf("{");
for (uint32_t Feature: FeatureSet)
@@ -281,7 +284,8 @@ public:
}
}
- void Replace(InputInfo *II, const Unit &U) {
+ void Replace(InputInfo *II, const Unit &U,
+ std::chrono::microseconds TimeOfUnit) {
assert(II->U.size() > U.size());
Hashes.erase(Sha1ToString(II->Sha1));
DeleteFile(*II);
@@ -289,6 +293,7 @@ public:
Hashes.insert(Sha1ToString(II->Sha1));
II->U = U;
II->Reduced = true;
+ II->TimeOfUnit = TimeOfUnit;
DistributionNeedsUpdate = true;
}
@@ -322,7 +327,8 @@ public:
const auto &II = *Inputs[i];
Printf(" [% 3zd %s] sz: % 5zd runs: % 5zd succ: % 5zd focus: %d\n", i,
Sha1ToString(II.Sha1).c_str(), II.U.size(),
- II.NumExecutedMutations, II.NumSuccessfullMutations, II.HasFocusFunction);
+ II.NumExecutedMutations, II.NumSuccessfullMutations,
+ II.HasFocusFunction);
}
}
@@ -399,7 +405,7 @@ public:
// Zero energy seeds will never be fuzzed and remain zero energy.
if (II->Energy > 0.0) {
II->SumIncidence += 1;
- II->Energy += logl(II->SumIncidence) / II->SumIncidence;
+ II->Energy += log(II->SumIncidence) / II->SumIncidence;
}
}
@@ -426,7 +432,8 @@ public:
NumUpdatedFeatures++;
if (FeatureDebug)
Printf("ADD FEATURE %zd sz %d\n", Idx, NewSize);
- SmallestElementPerFeature[Idx] = Inputs.size();
+ // Inputs.size() is guaranteed to be less than UINT32_MAX by AddToCorpus.
+ SmallestElementPerFeature[Idx] = static_cast<uint32_t>(Inputs.size());
InputSizesPerFeature[Idx] = NewSize;
return true;
}
@@ -464,7 +471,7 @@ private:
static const bool FeatureDebug = false;
- size_t GetFeature(size_t Idx) const { return InputSizesPerFeature[Idx]; }
+ uint32_t GetFeature(size_t Idx) const { return InputSizesPerFeature[Idx]; }
void ValidateFeatureSet() {
if (FeatureDebug)
@@ -539,9 +546,11 @@ private:
if (VanillaSchedule) {
for (size_t i = 0; i < N; i++)
- Weights[i] = Inputs[i]->NumFeatures
- ? (i + 1) * (Inputs[i]->HasFocusFunction ? 1000 : 1)
- : 0.;
+ Weights[i] =
+ Inputs[i]->NumFeatures
+ ? static_cast<double>((i + 1) *
+ (Inputs[i]->HasFocusFunction ? 1000 : 1))
+ : 0.;
}
if (FeatureDebug) {
@@ -557,11 +566,11 @@ private:
}
std::piecewise_constant_distribution<double> CorpusDistribution;
- Vector<double> Intervals;
- Vector<double> Weights;
+ std::vector<double> Intervals;
+ std::vector<double> Weights;
std::unordered_set<std::string> Hashes;
- Vector<InputInfo*> Inputs;
+ std::vector<InputInfo *> Inputs;
size_t NumAddedFeatures = 0;
size_t NumUpdatedFeatures = 0;
@@ -571,7 +580,7 @@ private:
bool DistributionNeedsUpdate = true;
uint16_t FreqOfMostAbundantRareFeature = 0;
uint16_t GlobalFeatureFreqs[kFeatureSetSize] = {};
- Vector<uint32_t> RareFeatures;
+ std::vector<uint32_t> RareFeatures;
std::string OutputCorpus;
};
diff --git a/contrib/libs/libfuzzer12/FuzzerCrossOver.cpp b/contrib/libs/libfuzzer/FuzzerCrossOver.cpp
index 83d9f8d47c..83d9f8d47c 100644
--- a/contrib/libs/libfuzzer12/FuzzerCrossOver.cpp
+++ b/contrib/libs/libfuzzer/FuzzerCrossOver.cpp
diff --git a/contrib/libs/libfuzzer12/FuzzerDataFlowTrace.cpp b/contrib/libs/libfuzzer/FuzzerDataFlowTrace.cpp
index 0e9cdf7e66..2f9a4d2d7a 100644
--- a/contrib/libs/libfuzzer12/FuzzerDataFlowTrace.cpp
+++ b/contrib/libs/libfuzzer/FuzzerDataFlowTrace.cpp
@@ -37,7 +37,7 @@ bool BlockCoverage::AppendCoverage(const std::string &S) {
// Coverage lines have this form:
// CN X Y Z T
// where N is the number of the function, T is the total number of instrumented
-// BBs, and X,Y,Z, if present, are the indecies of covered BB.
+// BBs, and X,Y,Z, if present, are the indices of covered BB.
// BB #0, which is the entry block, is not explicitly listed.
bool BlockCoverage::AppendCoverage(std::istream &IN) {
std::string L;
@@ -52,7 +52,7 @@ bool BlockCoverage::AppendCoverage(std::istream &IN) {
continue;
}
if (L[0] != 'C') continue;
- Vector<uint32_t> CoveredBlocks;
+ std::vector<uint32_t> CoveredBlocks;
while (true) {
uint32_t BB = 0;
SS >> BB;
@@ -60,6 +60,7 @@ bool BlockCoverage::AppendCoverage(std::istream &IN) {
CoveredBlocks.push_back(BB);
}
if (CoveredBlocks.empty()) return false;
+ // Ensures no CoverageVector is longer than UINT32_MAX.
uint32_t NumBlocks = CoveredBlocks.back();
CoveredBlocks.pop_back();
for (auto BB : CoveredBlocks)
@@ -67,7 +68,7 @@ bool BlockCoverage::AppendCoverage(std::istream &IN) {
auto It = Functions.find(FunctionId);
auto &Counters =
It == Functions.end()
- ? Functions.insert({FunctionId, Vector<uint32_t>(NumBlocks)})
+ ? Functions.insert({FunctionId, std::vector<uint32_t>(NumBlocks)})
.first->second
: It->second;
@@ -85,8 +86,8 @@ bool BlockCoverage::AppendCoverage(std::istream &IN) {
// * any uncovered function gets weight 0.
// * a function with lots of uncovered blocks gets bigger weight.
// * a function with a less frequently executed code gets bigger weight.
-Vector<double> BlockCoverage::FunctionWeights(size_t NumFunctions) const {
- Vector<double> Res(NumFunctions);
+std::vector<double> BlockCoverage::FunctionWeights(size_t NumFunctions) const {
+ std::vector<double> Res(NumFunctions);
for (auto It : Functions) {
auto FunctionID = It.first;
auto Counters = It.second;
@@ -103,7 +104,7 @@ Vector<double> BlockCoverage::FunctionWeights(size_t NumFunctions) const {
}
void DataFlowTrace::ReadCoverage(const std::string &DirPath) {
- Vector<SizedFile> Files;
+ std::vector<SizedFile> Files;
GetSizedFilesFromDir(DirPath, &Files);
for (auto &SF : Files) {
auto Name = Basename(SF.File);
@@ -114,16 +115,16 @@ void DataFlowTrace::ReadCoverage(const std::string &DirPath) {
}
}
-static void DFTStringAppendToVector(Vector<uint8_t> *DFT,
+static void DFTStringAppendToVector(std::vector<uint8_t> *DFT,
const std::string &DFTString) {
assert(DFT->size() == DFTString.size());
for (size_t I = 0, Len = DFT->size(); I < Len; I++)
(*DFT)[I] = DFTString[I] == '1';
}
-// converts a string of '0' and '1' into a Vector<uint8_t>
-static Vector<uint8_t> DFTStringToVector(const std::string &DFTString) {
- Vector<uint8_t> DFT(DFTString.size());
+// converts a string of '0' and '1' into a std::vector<uint8_t>
+static std::vector<uint8_t> DFTStringToVector(const std::string &DFTString) {
+ std::vector<uint8_t> DFT(DFTString.size());
DFTStringAppendToVector(&DFT, DFTString);
return DFT;
}
@@ -158,14 +159,14 @@ static bool ParseDFTLine(const std::string &Line, size_t *FunctionNum,
}
bool DataFlowTrace::Init(const std::string &DirPath, std::string *FocusFunction,
- Vector<SizedFile> &CorporaFiles, Random &Rand) {
+ std::vector<SizedFile> &CorporaFiles, Random &Rand) {
if (DirPath.empty()) return false;
Printf("INFO: DataFlowTrace: reading from '%s'\n", DirPath.c_str());
- Vector<SizedFile> Files;
+ std::vector<SizedFile> Files;
GetSizedFilesFromDir(DirPath, &Files);
std::string L;
size_t FocusFuncIdx = SIZE_MAX;
- Vector<std::string> FunctionNames;
+ std::vector<std::string> FunctionNames;
// Collect the hashes of the corpus files.
for (auto &SF : CorporaFiles)
@@ -190,7 +191,7 @@ bool DataFlowTrace::Init(const std::string &DirPath, std::string *FocusFunction,
// * chooses a random function according to the weights.
ReadCoverage(DirPath);
auto Weights = Coverage.FunctionWeights(NumFunctions);
- Vector<double> Intervals(NumFunctions + 1);
+ std::vector<double> Intervals(NumFunctions + 1);
std::iota(Intervals.begin(), Intervals.end(), 0);
auto Distribution = std::piecewise_constant_distribution<double>(
Intervals.begin(), Intervals.end(), Weights.begin());
@@ -200,7 +201,8 @@ bool DataFlowTrace::Init(const std::string &DirPath, std::string *FocusFunction,
Printf("INFO: AUTOFOCUS: %zd %s\n", FocusFuncIdx,
FunctionNames[FocusFuncIdx].c_str());
for (size_t i = 0; i < NumFunctions; i++) {
- if (!Weights[i]) continue;
+ if (Weights[i] == 0.0)
+ continue;
Printf(" [%zd] W %g\tBB-tot %u\tBB-cov %u\tEntryFreq %u:\t%s\n", i,
Weights[i], Coverage.GetNumberOfBlocks(i),
Coverage.GetNumberOfCoveredBlocks(i), Coverage.GetCounter(i, 0),
@@ -245,7 +247,7 @@ bool DataFlowTrace::Init(const std::string &DirPath, std::string *FocusFunction,
}
int CollectDataFlow(const std::string &DFTBinary, const std::string &DirPath,
- const Vector<SizedFile> &CorporaFiles) {
+ const std::vector<SizedFile> &CorporaFiles) {
Printf("INFO: collecting data flow: bin: %s dir: %s files: %zd\n",
DFTBinary.c_str(), DirPath.c_str(), CorporaFiles.size());
if (CorporaFiles.empty()) {
@@ -263,7 +265,7 @@ int CollectDataFlow(const std::string &DFTBinary, const std::string &DirPath,
// we then request tags in [0,Size/2) and [Size/2, Size), and so on.
// Function number => DFT.
auto OutPath = DirPlusFile(DirPath, Hash(FileToVector(F.File)));
- std::unordered_map<size_t, Vector<uint8_t>> DFTMap;
+ std::unordered_map<size_t, std::vector<uint8_t>> DFTMap;
std::unordered_set<std::string> Cov;
Command Cmd;
Cmd.addArgument(DFTBinary);
diff --git a/contrib/libs/libfuzzer12/FuzzerDataFlowTrace.h b/contrib/libs/libfuzzer/FuzzerDataFlowTrace.h
index d6e3de30a4..054dce1bdc 100644
--- a/contrib/libs/libfuzzer12/FuzzerDataFlowTrace.h
+++ b/contrib/libs/libfuzzer/FuzzerDataFlowTrace.h
@@ -39,10 +39,11 @@
namespace fuzzer {
int CollectDataFlow(const std::string &DFTBinary, const std::string &DirPath,
- const Vector<SizedFile> &CorporaFiles);
+ const std::vector<SizedFile> &CorporaFiles);
class BlockCoverage {
- public:
+public:
+ // These functions guarantee no CoverageVector is longer than UINT32_MAX.
bool AppendCoverage(std::istream &IN);
bool AppendCoverage(const std::string &S);
@@ -50,7 +51,8 @@ class BlockCoverage {
uint32_t GetCounter(size_t FunctionId, size_t BasicBlockId) {
auto It = Functions.find(FunctionId);
- if (It == Functions.end()) return 0;
+ if (It == Functions.end())
+ return 0;
const auto &Counters = It->second;
if (BasicBlockId < Counters.size())
return Counters[BasicBlockId];
@@ -61,7 +63,7 @@ class BlockCoverage {
auto It = Functions.find(FunctionId);
if (It == Functions.end()) return 0;
const auto &Counters = It->second;
- return Counters.size();
+ return static_cast<uint32_t>(Counters.size());
}
uint32_t GetNumberOfCoveredBlocks(size_t FunctionId) {
@@ -75,12 +77,11 @@ class BlockCoverage {
return Result;
}
- Vector<double> FunctionWeights(size_t NumFunctions) const;
+ std::vector<double> FunctionWeights(size_t NumFunctions) const;
void clear() { Functions.clear(); }
- private:
-
- typedef Vector<uint32_t> CoverageVector;
+private:
+ typedef std::vector<uint32_t> CoverageVector;
uint32_t NumberOfCoveredBlocks(const CoverageVector &Counters) const {
uint32_t Res = 0;
@@ -91,7 +92,8 @@ class BlockCoverage {
}
uint32_t NumberOfUncoveredBlocks(const CoverageVector &Counters) const {
- return Counters.size() - NumberOfCoveredBlocks(Counters);
+ return static_cast<uint32_t>(Counters.size()) -
+ NumberOfCoveredBlocks(Counters);
}
uint32_t SmallestNonZeroCounter(const CoverageVector &Counters) const {
@@ -115,9 +117,9 @@ class DataFlowTrace {
public:
void ReadCoverage(const std::string &DirPath);
bool Init(const std::string &DirPath, std::string *FocusFunction,
- Vector<SizedFile> &CorporaFiles, Random &Rand);
+ std::vector<SizedFile> &CorporaFiles, Random &Rand);
void Clear() { Traces.clear(); }
- const Vector<uint8_t> *Get(const std::string &InputSha1) const {
+ const std::vector<uint8_t> *Get(const std::string &InputSha1) const {
auto It = Traces.find(InputSha1);
if (It != Traces.end())
return &It->second;
@@ -126,9 +128,9 @@ class DataFlowTrace {
private:
// Input's sha1 => DFT for the FocusFunction.
- std::unordered_map<std::string, Vector<uint8_t> > Traces;
- BlockCoverage Coverage;
- std::unordered_set<std::string> CorporaHashes;
+ std::unordered_map<std::string, std::vector<uint8_t>> Traces;
+ BlockCoverage Coverage;
+ std::unordered_set<std::string> CorporaHashes;
};
} // namespace fuzzer
diff --git a/contrib/libs/libfuzzer12/FuzzerDefs.h b/contrib/libs/libfuzzer/FuzzerDefs.h
index 1bc89c9448..369978d0d3 100644
--- a/contrib/libs/libfuzzer12/FuzzerDefs.h
+++ b/contrib/libs/libfuzzer/FuzzerDefs.h
@@ -38,28 +38,8 @@ struct ExternalFunctions;
// Global interface to functions that may or may not be available.
extern ExternalFunctions *EF;
-// We are using a custom allocator to give a different symbol name to STL
-// containers in order to avoid ODR violations.
-template<typename T>
- class fuzzer_allocator: public std::allocator<T> {
- public:
- fuzzer_allocator() = default;
-
- template<class U>
- fuzzer_allocator(const fuzzer_allocator<U>&) {}
-
- template<class Other>
- struct rebind { typedef fuzzer_allocator<Other> other; };
- };
-
-template<typename T>
-using Vector = std::vector<T, fuzzer_allocator<T>>;
-
-template<typename T>
-using Set = std::set<T, std::less<T>, fuzzer_allocator<T>>;
-
-typedef Vector<uint8_t> Unit;
-typedef Vector<Unit> UnitVector;
+typedef std::vector<uint8_t> Unit;
+typedef std::vector<Unit> UnitVector;
typedef int (*UserCallback)(const uint8_t *Data, size_t Size);
#define exit(status) FuzzerExit(status)
diff --git a/contrib/libs/libfuzzer12/FuzzerDictionary.h b/contrib/libs/libfuzzer/FuzzerDictionary.h
index 301c5d9afe..48f063c7ee 100644
--- a/contrib/libs/libfuzzer12/FuzzerDictionary.h
+++ b/contrib/libs/libfuzzer/FuzzerDictionary.h
@@ -23,12 +23,14 @@ template <size_t kMaxSizeT> class FixedWord {
public:
static const size_t kMaxSize = kMaxSizeT;
FixedWord() {}
- FixedWord(const uint8_t *B, uint8_t S) { Set(B, S); }
+ FixedWord(const uint8_t *B, size_t S) { Set(B, S); }
- void Set(const uint8_t *B, uint8_t S) {
+ void Set(const uint8_t *B, size_t S) {
+ static_assert(kMaxSizeT <= std::numeric_limits<uint8_t>::max(),
+ "FixedWord::kMaxSizeT cannot fit in a uint8_t.");
assert(S <= kMaxSize);
memcpy(Data, B, S);
- Size = S;
+ Size = static_cast<uint8_t>(S);
}
bool operator==(const FixedWord<kMaxSize> &w) const {
@@ -50,10 +52,13 @@ class DictionaryEntry {
public:
DictionaryEntry() {}
DictionaryEntry(Word W) : W(W) {}
- DictionaryEntry(Word W, size_t PositionHint) : W(W), PositionHint(PositionHint) {}
+ DictionaryEntry(Word W, size_t PositionHint)
+ : W(W), PositionHint(PositionHint) {}
const Word &GetW() const { return W; }
- bool HasPositionHint() const { return PositionHint != std::numeric_limits<size_t>::max(); }
+ bool HasPositionHint() const {
+ return PositionHint != std::numeric_limits<size_t>::max();
+ }
size_t GetPositionHint() const {
assert(HasPositionHint());
return PositionHint;
@@ -106,12 +111,12 @@ private:
};
// Parses one dictionary entry.
-// If successful, write the enty to Unit and returns true,
+// If successful, writes the entry to Unit and returns true,
// otherwise returns false.
bool ParseOneDictionaryEntry(const std::string &Str, Unit *U);
// Parses the dictionary file, fills Units, returns true iff all lines
// were parsed successfully.
-bool ParseDictionaryFile(const std::string &Text, Vector<Unit> *Units);
+bool ParseDictionaryFile(const std::string &Text, std::vector<Unit> *Units);
} // namespace fuzzer
diff --git a/contrib/libs/libfuzzer12/FuzzerDriver.cpp b/contrib/libs/libfuzzer/FuzzerDriver.cpp
index 44b8e23cfb..462e147c58 100644
--- a/contrib/libs/libfuzzer12/FuzzerDriver.cpp
+++ b/contrib/libs/libfuzzer/FuzzerDriver.cpp
@@ -86,7 +86,7 @@ static const FlagDescription FlagDescriptions [] {
static const size_t kNumFlags =
sizeof(FlagDescriptions) / sizeof(FlagDescriptions[0]);
-static Vector<std::string> *Inputs;
+static std::vector<std::string> *Inputs;
static std::string *ProgName;
static void PrintHelp() {
@@ -159,14 +159,14 @@ static bool ParseOneFlag(const char *Param) {
const char *Str = FlagValue(Param, Name);
if (Str) {
if (FlagDescriptions[F].IntFlag) {
- int Val = MyStol(Str);
- *FlagDescriptions[F].IntFlag = Val;
+ auto Val = MyStol(Str);
+ *FlagDescriptions[F].IntFlag = static_cast<int>(Val);
if (Flags.verbosity >= 2)
Printf("Flag: %s %d\n", Name, Val);
return true;
} else if (FlagDescriptions[F].UIntFlag) {
- unsigned int Val = std::stoul(Str);
- *FlagDescriptions[F].UIntFlag = Val;
+ auto Val = std::stoul(Str);
+ *FlagDescriptions[F].UIntFlag = static_cast<unsigned int>(Val);
if (Flags.verbosity >= 2)
Printf("Flag: %s %u\n", Name, Val);
return true;
@@ -187,7 +187,7 @@ static bool ParseOneFlag(const char *Param) {
}
// We don't use any library to minimize dependencies.
-static void ParseFlags(const Vector<std::string> &Args,
+static void ParseFlags(const std::vector<std::string> &Args,
const ExternalFunctions *EF) {
for (size_t F = 0; F < kNumFlags; F++) {
if (FlagDescriptions[F].IntFlag)
@@ -206,7 +206,7 @@ static void ParseFlags(const Vector<std::string> &Args,
"Disabling -len_control by default.\n", EF->LLVMFuzzerCustomMutator);
}
- Inputs = new Vector<std::string>;
+ Inputs = new std::vector<std::string>;
for (size_t A = 1; A < Args.size(); A++) {
if (ParseOneFlag(Args[A].c_str())) {
if (Flags.ignore_remaining_args)
@@ -272,7 +272,7 @@ static void ValidateDirectoryExists(const std::string &Path,
exit(1);
}
-std::string CloneArgsWithoutX(const Vector<std::string> &Args,
+std::string CloneArgsWithoutX(const std::vector<std::string> &Args,
const char *X1, const char *X2) {
std::string Cmd;
for (auto &S : Args) {
@@ -283,18 +283,19 @@ std::string CloneArgsWithoutX(const Vector<std::string> &Args,
return Cmd;
}
-static int RunInMultipleProcesses(const Vector<std::string> &Args,
+static int RunInMultipleProcesses(const std::vector<std::string> &Args,
unsigned NumWorkers, unsigned NumJobs) {
std::atomic<unsigned> Counter(0);
std::atomic<bool> HasErrors(false);
Command Cmd(Args);
Cmd.removeFlag("jobs");
Cmd.removeFlag("workers");
- Vector<std::thread> V;
+ std::vector<std::thread> V;
std::thread Pulse(PulseThread);
Pulse.detach();
for (unsigned i = 0; i < NumWorkers; i++)
- V.push_back(std::thread(WorkerThread, std::ref(Cmd), &Counter, NumJobs, &HasErrors));
+ V.push_back(std::thread(WorkerThread, std::ref(Cmd), &Counter, NumJobs,
+ &HasErrors));
for (auto &T : V)
T.join();
return HasErrors ? 1 : 0;
@@ -348,8 +349,8 @@ static std::string GetDedupTokenFromCmdOutput(const std::string &S) {
return S.substr(Beg, End - Beg);
}
-int CleanseCrashInput(const Vector<std::string> &Args,
- const FuzzingOptions &Options) {
+int CleanseCrashInput(const std::vector<std::string> &Args,
+ const FuzzingOptions &Options) {
if (Inputs->size() != 1 || !Flags.exact_artifact_path) {
Printf("ERROR: -cleanse_crash should be given one input file and"
" -exact_artifact_path\n");
@@ -372,7 +373,7 @@ int CleanseCrashInput(const Vector<std::string> &Args,
auto U = FileToVector(CurrentFilePath);
size_t Size = U.size();
- const Vector<uint8_t> ReplacementBytes = {' ', 0xff};
+ const std::vector<uint8_t> ReplacementBytes = {' ', 0xff};
for (int NumAttempts = 0; NumAttempts < 5; NumAttempts++) {
bool Changed = false;
for (size_t Idx = 0; Idx < Size; Idx++) {
@@ -403,7 +404,7 @@ int CleanseCrashInput(const Vector<std::string> &Args,
return 0;
}
-int MinimizeCrashInput(const Vector<std::string> &Args,
+int MinimizeCrashInput(const std::vector<std::string> &Args,
const FuzzingOptions &Options) {
if (Inputs->size() != 1) {
Printf("ERROR: -minimize_crash should be given one input file\n");
@@ -503,14 +504,15 @@ int MinimizeCrashInputInternalStep(Fuzzer *F, InputCorpus *Corpus) {
return 0;
}
-void Merge(Fuzzer *F, FuzzingOptions &Options, const Vector<std::string> &Args,
- const Vector<std::string> &Corpora, const char *CFPathOrNull) {
+void Merge(Fuzzer *F, FuzzingOptions &Options,
+ const std::vector<std::string> &Args,
+ const std::vector<std::string> &Corpora, const char *CFPathOrNull) {
if (Corpora.size() < 2) {
Printf("INFO: Merge requires two or more corpus dirs\n");
exit(0);
}
- Vector<SizedFile> OldCorpus, NewCorpus;
+ std::vector<SizedFile> OldCorpus, NewCorpus;
GetSizedFilesFromDir(Corpora[0], &OldCorpus);
for (size_t i = 1; i < Corpora.size(); i++)
GetSizedFilesFromDir(Corpora[i], &NewCorpus);
@@ -518,10 +520,10 @@ void Merge(Fuzzer *F, FuzzingOptions &Options, const Vector<std::string> &Args,
std::sort(NewCorpus.begin(), NewCorpus.end());
std::string CFPath = CFPathOrNull ? CFPathOrNull : TempPath("Merge", ".txt");
- Vector<std::string> NewFiles;
- Set<uint32_t> NewFeatures, NewCov;
+ std::vector<std::string> NewFiles;
+ std::set<uint32_t> NewFeatures, NewCov;
CrashResistantMerge(Args, OldCorpus, NewCorpus, &NewFiles, {}, &NewFeatures,
- {}, &NewCov, CFPath, true);
+ {}, &NewCov, CFPath, true, Flags.set_cover_merge);
for (auto &Path : NewFiles)
F->WriteToOutputCorpus(FileToVector(Path, Options.MaxLen));
// We are done, delete the control file if it was a temporary one.
@@ -531,17 +533,17 @@ void Merge(Fuzzer *F, FuzzingOptions &Options, const Vector<std::string> &Args,
exit(0);
}
-int AnalyzeDictionary(Fuzzer *F, const Vector<Unit>& Dict,
- UnitVector& Corpus) {
+int AnalyzeDictionary(Fuzzer *F, const std::vector<Unit> &Dict,
+ UnitVector &Corpus) {
Printf("Started dictionary minimization (up to %d tests)\n",
Dict.size() * Corpus.size() * 2);
// Scores and usage count for each dictionary unit.
- Vector<int> Scores(Dict.size());
- Vector<int> Usages(Dict.size());
+ std::vector<int> Scores(Dict.size());
+ std::vector<int> Usages(Dict.size());
- Vector<size_t> InitialFeatures;
- Vector<size_t> ModifiedFeatures;
+ std::vector<size_t> InitialFeatures;
+ std::vector<size_t> ModifiedFeatures;
for (auto &C : Corpus) {
// Get coverage for the testcase without modifications.
F->ExecuteCallback(C.data(), C.size());
@@ -551,7 +553,7 @@ int AnalyzeDictionary(Fuzzer *F, const Vector<Unit>& Dict,
});
for (size_t i = 0; i < Dict.size(); ++i) {
- Vector<uint8_t> Data = C;
+ std::vector<uint8_t> Data = C;
auto StartPos = std::search(Data.begin(), Data.end(),
Dict[i].begin(), Dict[i].end());
// Skip dictionary unit, if the testcase does not contain it.
@@ -597,9 +599,9 @@ int AnalyzeDictionary(Fuzzer *F, const Vector<Unit>& Dict,
return 0;
}
-Vector<std::string> ParseSeedInuts(const char *seed_inputs) {
+std::vector<std::string> ParseSeedInuts(const char *seed_inputs) {
// Parse -seed_inputs=file1,file2,... or -seed_inputs=@seed_inputs_file
- Vector<std::string> Files;
+ std::vector<std::string> Files;
if (!seed_inputs) return Files;
std::string SeedInputs;
if (Flags.seed_inputs[0] == '@')
@@ -620,9 +622,10 @@ Vector<std::string> ParseSeedInuts(const char *seed_inputs) {
return Files;
}
-static Vector<SizedFile> ReadCorpora(const Vector<std::string> &CorpusDirs,
- const Vector<std::string> &ExtraSeedFiles) {
- Vector<SizedFile> SizedFiles;
+static std::vector<SizedFile>
+ReadCorpora(const std::vector<std::string> &CorpusDirs,
+ const std::vector<std::string> &ExtraSeedFiles) {
+ std::vector<SizedFile> SizedFiles;
size_t LastNumFiles = 0;
for (auto &Dir : CorpusDirs) {
GetSizedFilesFromDir(Dir, &SizedFiles);
@@ -651,7 +654,7 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) {
EF->LLVMFuzzerInitialize(argc, argv);
if (EF->__msan_scoped_disable_interceptor_checks)
EF->__msan_scoped_disable_interceptor_checks();
- const Vector<std::string> Args(*argv, *argv + *argc);
+ const std::vector<std::string> Args(*argv, *argv + *argc);
assert(!Args.empty());
ProgName = new std::string(Args[0]);
if (Argv0 != *ProgName) {
@@ -742,7 +745,7 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) {
ValidateDirectoryExists(DirName(Options.ExactArtifactPath),
Flags.create_missing_dirs);
}
- Vector<Unit> Dictionary;
+ std::vector<Unit> Dictionary;
if (Flags.dict)
if (!ParseDictionaryFile(FileToString(Flags.dict), &Dictionary))
return 1;
@@ -797,12 +800,13 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) {
unsigned Seed = Flags.seed;
// Initialize Seed.
if (Seed == 0)
- Seed =
- std::chrono::system_clock::now().time_since_epoch().count() + GetPid();
+ Seed = static_cast<unsigned>(
+ std::chrono::system_clock::now().time_since_epoch().count() + GetPid());
if (Flags.verbosity)
Printf("INFO: Seed: %u\n", Seed);
- if (Flags.collect_data_flow && !Flags.fork && !Flags.merge) {
+ if (Flags.collect_data_flow && !Flags.fork &&
+ !(Flags.merge || Flags.set_cover_merge)) {
if (RunIndividualFiles)
return CollectDataFlow(Flags.collect_data_flow, Flags.data_flow_trace,
ReadCorpora({}, *Inputs));
@@ -874,10 +878,11 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) {
exit(0);
}
+ Options.ForkCorpusGroups = Flags.fork_corpus_groups;
if (Flags.fork)
FuzzWithFork(F->GetMD().GetRand(), Options, Args, *Inputs, Flags.fork);
- if (Flags.merge)
+ if (Flags.merge || Flags.set_cover_merge)
Merge(F, Options, Args, *Inputs, Flags.merge_control_file);
if (Flags.merge_inner) {
@@ -885,7 +890,8 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) {
if (Options.MaxLen == 0)
F->SetMaxInputLen(kDefaultMaxMergeLen);
assert(Flags.merge_control_file);
- F->CrashResistantMergeInternalStep(Flags.merge_control_file);
+ F->CrashResistantMergeInternalStep(Flags.merge_control_file,
+ !strncmp(Flags.merge_inner, "2", 1));
exit(0);
}
diff --git a/contrib/libs/libfuzzer12/FuzzerExtFunctions.def b/contrib/libs/libfuzzer/FuzzerExtFunctions.def
index 40b1e73945..40b1e73945 100644
--- a/contrib/libs/libfuzzer12/FuzzerExtFunctions.def
+++ b/contrib/libs/libfuzzer/FuzzerExtFunctions.def
diff --git a/contrib/libs/libfuzzer12/FuzzerExtFunctions.h b/contrib/libs/libfuzzer/FuzzerExtFunctions.h
index c88aac4e67..c88aac4e67 100644
--- a/contrib/libs/libfuzzer12/FuzzerExtFunctions.h
+++ b/contrib/libs/libfuzzer/FuzzerExtFunctions.h
diff --git a/contrib/libs/libfuzzer12/FuzzerExtFunctionsDlsym.cpp b/contrib/libs/libfuzzer/FuzzerExtFunctionsDlsym.cpp
index 95233d2a10..95233d2a10 100644
--- a/contrib/libs/libfuzzer12/FuzzerExtFunctionsDlsym.cpp
+++ b/contrib/libs/libfuzzer/FuzzerExtFunctionsDlsym.cpp
diff --git a/contrib/libs/libfuzzer12/FuzzerExtFunctionsWeak.cpp b/contrib/libs/libfuzzer/FuzzerExtFunctionsWeak.cpp
index 3ef758daa7..3ef758daa7 100644
--- a/contrib/libs/libfuzzer12/FuzzerExtFunctionsWeak.cpp
+++ b/contrib/libs/libfuzzer/FuzzerExtFunctionsWeak.cpp
diff --git a/contrib/libs/libfuzzer12/FuzzerExtFunctionsWindows.cpp b/contrib/libs/libfuzzer/FuzzerExtFunctionsWindows.cpp
index 688bad1d51..688bad1d51 100644
--- a/contrib/libs/libfuzzer12/FuzzerExtFunctionsWindows.cpp
+++ b/contrib/libs/libfuzzer/FuzzerExtFunctionsWindows.cpp
diff --git a/contrib/libs/libfuzzer12/FuzzerExtraCounters.cpp b/contrib/libs/libfuzzer/FuzzerExtraCounters.cpp
index 04f569a1a8..54ecbf7c62 100644
--- a/contrib/libs/libfuzzer12/FuzzerExtraCounters.cpp
+++ b/contrib/libs/libfuzzer/FuzzerExtraCounters.cpp
@@ -31,12 +31,4 @@ void ClearExtraCounters() { // hand-written memset, don't asan-ify.
} // namespace fuzzer
-#else
-// TODO: implement for other platforms.
-namespace fuzzer {
-uint8_t *ExtraCountersBegin() { return nullptr; }
-uint8_t *ExtraCountersEnd() { return nullptr; }
-void ClearExtraCounters() {}
-} // namespace fuzzer
-
#endif
diff --git a/contrib/libs/libfuzzer/FuzzerExtraCountersDarwin.cpp b/contrib/libs/libfuzzer/FuzzerExtraCountersDarwin.cpp
new file mode 100644
index 0000000000..2321ba8a3d
--- /dev/null
+++ b/contrib/libs/libfuzzer/FuzzerExtraCountersDarwin.cpp
@@ -0,0 +1,22 @@
+//===- FuzzerExtraCountersDarwin.cpp - Extra coverage counters for Darwin -===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// Extra coverage counters defined by user code for Darwin.
+//===----------------------------------------------------------------------===//
+
+#include "FuzzerPlatform.h"
+#include <cstdint>
+
+#if LIBFUZZER_APPLE
+
+namespace fuzzer {
+uint8_t *ExtraCountersBegin() { return nullptr; }
+uint8_t *ExtraCountersEnd() { return nullptr; }
+void ClearExtraCounters() {}
+} // namespace fuzzer
+
+#endif
diff --git a/contrib/libs/libfuzzer/FuzzerExtraCountersWindows.cpp b/contrib/libs/libfuzzer/FuzzerExtraCountersWindows.cpp
new file mode 100644
index 0000000000..102f5febda
--- /dev/null
+++ b/contrib/libs/libfuzzer/FuzzerExtraCountersWindows.cpp
@@ -0,0 +1,80 @@
+//===- FuzzerExtraCountersWindows.cpp - Extra coverage counters for Win32 -===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// Extra coverage counters defined by user code for Windows.
+//===----------------------------------------------------------------------===//
+
+#include "FuzzerPlatform.h"
+#include <cstdint>
+
+#if LIBFUZZER_WINDOWS
+#include <windows.h>
+
+namespace fuzzer {
+
+//
+// The __start___libfuzzer_extra_counters variable is align 16, size 16 to
+// ensure the padding between it and the next variable in this section (either
+// __libfuzzer_extra_counters or __stop___libfuzzer_extra_counters) will be
+// located at (__start___libfuzzer_extra_counters +
+// sizeof(__start___libfuzzer_extra_counters)). Otherwise, the calculation of
+// (stop - (start + sizeof(start))) might be skewed.
+//
+// The section name, __libfuzzer_extra_countaaa ends with "aaa", so it sorts
+// before __libfuzzer_extra_counters alphabetically. We want the start symbol to
+// be placed in the section just before the user supplied counters (if present).
+//
+#pragma section(".data$__libfuzzer_extra_countaaa")
+ATTRIBUTE_ALIGNED(16)
+__declspec(allocate(".data$__libfuzzer_extra_countaaa")) uint8_t
+ __start___libfuzzer_extra_counters[16] = {0};
+
+//
+// Example of what the user-supplied counters should look like. First, the
+// pragma to create the section name. It will fall alphabetically between
+// ".data$__libfuzzer_extra_countaaa" and ".data$__libfuzzer_extra_countzzz".
+// Next, the declspec to allocate the variable inside the specified section.
+// Finally, some array, struct, whatever that is used to track the counter data.
+// The size of this variable is computed at runtime by finding the difference of
+// __stop___libfuzzer_extra_counters and __start___libfuzzer_extra_counters +
+// sizeof(__start___libfuzzer_extra_counters).
+//
+
+//
+// #pragma section(".data$__libfuzzer_extra_counters")
+// __declspec(allocate(".data$__libfuzzer_extra_counters"))
+// uint8_t any_name_variable[64 * 1024];
+//
+
+//
+// Here, the section name, __libfuzzer_extra_countzzz ends with "zzz", so it
+// sorts after __libfuzzer_extra_counters alphabetically. We want the stop
+// symbol to be placed in the section just after the user supplied counters (if
+// present). Align to 1 so there isn't any padding placed between this and the
+// previous variable.
+//
+#pragma section(".data$__libfuzzer_extra_countzzz")
+ATTRIBUTE_ALIGNED(1)
+__declspec(allocate(".data$__libfuzzer_extra_countzzz")) uint8_t
+ __stop___libfuzzer_extra_counters = 0;
+
+uint8_t *ExtraCountersBegin() {
+ return __start___libfuzzer_extra_counters +
+ sizeof(__start___libfuzzer_extra_counters);
+}
+
+uint8_t *ExtraCountersEnd() { return &__stop___libfuzzer_extra_counters; }
+
+ATTRIBUTE_NO_SANITIZE_ALL
+void ClearExtraCounters() {
+ uint8_t *Beg = ExtraCountersBegin();
+ SecureZeroMemory(Beg, ExtraCountersEnd() - Beg);
+}
+
+} // namespace fuzzer
+
+#endif
diff --git a/contrib/libs/libfuzzer12/FuzzerFlags.def b/contrib/libs/libfuzzer/FuzzerFlags.def
index 078333e81f..dacfa84a6c 100644
--- a/contrib/libs/libfuzzer12/FuzzerFlags.def
+++ b/contrib/libs/libfuzzer/FuzzerFlags.def
@@ -61,12 +61,21 @@ FUZZER_FLAG_INT(max_total_time, 0, "If positive, indicates the maximal total "
FUZZER_FLAG_INT(help, 0, "Print help.")
FUZZER_FLAG_INT(fork, 0, "Experimental mode where fuzzing happens "
"in a subprocess")
+FUZZER_FLAG_INT(fork_corpus_groups, 0, "For fork mode, enable the corpus-group "
+ "strategy, The main corpus will be grouped according to size, "
+ "and each sub-process will randomly select seeds from different "
+ "groups as the sub-corpus.")
FUZZER_FLAG_INT(ignore_timeouts, 1, "Ignore timeouts in fork mode")
FUZZER_FLAG_INT(ignore_ooms, 1, "Ignore OOMs in fork mode")
FUZZER_FLAG_INT(ignore_crashes, 0, "Ignore crashes in fork mode")
FUZZER_FLAG_INT(merge, 0, "If 1, the 2-nd, 3-rd, etc corpora will be "
"merged into the 1-st corpus. Only interesting units will be taken. "
"This flag can be used to minimize a corpus.")
+FUZZER_FLAG_INT(set_cover_merge, 0, "If 1, the 2-nd, 3-rd, etc corpora will be "
+ "merged into the 1-st corpus. Same as the 'merge' flag, but uses the "
+ "standard greedy algorithm for the set cover problem to "
+ "compute an approximation of the minimum set of testcases that "
+ "provide the same coverage as the initial corpora")
FUZZER_FLAG_STRING(stop_file, "Stop fuzzing ASAP if this file exists")
FUZZER_FLAG_STRING(merge_inner, "internal flag")
FUZZER_FLAG_STRING(merge_control_file,
diff --git a/contrib/libs/libfuzzer12/FuzzerFork.cpp b/contrib/libs/libfuzzer/FuzzerFork.cpp
index 84725d22a9..d59d513842 100644
--- a/contrib/libs/libfuzzer12/FuzzerFork.cpp
+++ b/contrib/libs/libfuzzer/FuzzerFork.cpp
@@ -86,18 +86,21 @@ struct FuzzJob {
};
struct GlobalEnv {
- Vector<std::string> Args;
- Vector<std::string> CorpusDirs;
+ std::vector<std::string> Args;
+ std::vector<std::string> CorpusDirs;
std::string MainCorpusDir;
std::string TempDir;
std::string DFTDir;
std::string DataFlowBinary;
- Set<uint32_t> Features, Cov;
- Set<std::string> FilesWithDFT;
- Vector<std::string> Files;
+ std::set<uint32_t> Features, Cov;
+ std::set<std::string> FilesWithDFT;
+ std::vector<std::string> Files;
+ std::vector<std::size_t> FilesSizes;
Random *Rand;
std::chrono::system_clock::time_point ProcessStartTime;
int Verbosity = 0;
+ int Group = 0;
+ int NumCorpuses = 8;
size_t NumTimeouts = 0;
size_t NumOOMs = 0;
@@ -136,13 +139,29 @@ struct GlobalEnv {
if (size_t CorpusSubsetSize =
std::min(Files.size(), (size_t)sqrt(Files.size() + 2))) {
auto Time1 = std::chrono::system_clock::now();
- for (size_t i = 0; i < CorpusSubsetSize; i++) {
- auto &SF = Files[Rand->SkewTowardsLast(Files.size())];
- Seeds += (Seeds.empty() ? "" : ",") + SF;
- CollectDFT(SF);
+ if (Group) { // whether to group the corpus.
+ size_t AverageCorpusSize = Files.size() / NumCorpuses + 1;
+ size_t StartIndex = ((JobId - 1) % NumCorpuses) * AverageCorpusSize;
+ for (size_t i = 0; i < CorpusSubsetSize; i++) {
+ size_t RandNum = (*Rand)(AverageCorpusSize);
+ size_t Index = RandNum + StartIndex;
+ Index = Index < Files.size() ? Index
+ : Rand->SkewTowardsLast(Files.size());
+ auto &SF = Files[Index];
+ Seeds += (Seeds.empty() ? "" : ",") + SF;
+ CollectDFT(SF);
+ }
+ } else {
+ for (size_t i = 0; i < CorpusSubsetSize; i++) {
+ auto &SF = Files[Rand->SkewTowardsLast(Files.size())];
+ Seeds += (Seeds.empty() ? "" : ",") + SF;
+ CollectDFT(SF);
+ }
}
auto Time2 = std::chrono::system_clock::now();
- Job->DftTimeInSeconds = duration_cast<seconds>(Time2 - Time1).count();
+ auto DftTimeInSeconds = duration_cast<seconds>(Time2 - Time1).count();
+ assert(DftTimeInSeconds < std::numeric_limits<int>::max());
+ Job->DftTimeInSeconds = static_cast<int>(DftTimeInSeconds);
}
if (!Seeds.empty()) {
Job->SeedListPath =
@@ -181,7 +200,7 @@ struct GlobalEnv {
auto Stats = ParseFinalStatsFromLog(Job->LogPath);
NumRuns += Stats.number_of_executed_units;
- Vector<SizedFile> TempFiles, MergeCandidates;
+ std::vector<SizedFile> TempFiles, MergeCandidates;
// Read all newly created inputs and their feature sets.
// Choose only those inputs that have new features.
GetSizedFilesFromDir(Job->CorpusDir, &TempFiles);
@@ -191,7 +210,7 @@ struct GlobalEnv {
FeatureFile.replace(0, Job->CorpusDir.size(), Job->FeaturesDir);
auto FeatureBytes = FileToVector(FeatureFile, 0, false);
assert((FeatureBytes.size() % sizeof(uint32_t)) == 0);
- Vector<uint32_t> NewFeatures(FeatureBytes.size() / sizeof(uint32_t));
+ std::vector<uint32_t> NewFeatures(FeatureBytes.size() / sizeof(uint32_t));
memcpy(NewFeatures.data(), FeatureBytes.data(), FeatureBytes.size());
for (auto Ft : NewFeatures) {
if (!Features.count(Ft)) {
@@ -209,15 +228,27 @@ struct GlobalEnv {
if (MergeCandidates.empty()) return;
- Vector<std::string> FilesToAdd;
- Set<uint32_t> NewFeatures, NewCov;
+ std::vector<std::string> FilesToAdd;
+ std::set<uint32_t> NewFeatures, NewCov;
+ bool IsSetCoverMerge =
+ !Job->Cmd.getFlagValue("set_cover_merge").compare("1");
CrashResistantMerge(Args, {}, MergeCandidates, &FilesToAdd, Features,
- &NewFeatures, Cov, &NewCov, Job->CFPath, false);
+ &NewFeatures, Cov, &NewCov, Job->CFPath, false,
+ IsSetCoverMerge);
for (auto &Path : FilesToAdd) {
auto U = FileToVector(Path);
auto NewPath = DirPlusFile(MainCorpusDir, Hash(U));
WriteToFile(U, NewPath);
- Files.push_back(NewPath);
+ if (Group) { // Insert the queue according to the size of the seed.
+ size_t UnitSize = U.size();
+ auto Idx =
+ std::upper_bound(FilesSizes.begin(), FilesSizes.end(), UnitSize) -
+ FilesSizes.begin();
+ FilesSizes.insert(FilesSizes.begin() + Idx, UnitSize);
+ Files.insert(Files.begin() + Idx, NewPath);
+ } else {
+ Files.push_back(NewPath);
+ }
}
Features.insert(NewFeatures.begin(), NewFeatures.end());
Cov.insert(NewCov.begin(), NewCov.end());
@@ -226,10 +257,8 @@ struct GlobalEnv {
if (TPC.PcIsFuncEntry(TE))
PrintPC(" NEW_FUNC: %p %F %L\n", "",
TPC.GetNextInstructionPc(TE->PC));
-
}
-
void CollectDFT(const std::string &InputPath) {
if (DataFlowBinary.empty()) return;
if (!FilesWithDFT.insert(InputPath).second) return;
@@ -281,8 +310,8 @@ void WorkerThread(JobQueue *FuzzQ, JobQueue *MergeQ) {
// This is just a skeleton of an experimental -fork=1 feature.
void FuzzWithFork(Random &Rand, const FuzzingOptions &Options,
- const Vector<std::string> &Args,
- const Vector<std::string> &CorpusDirs, int NumJobs) {
+ const std::vector<std::string> &Args,
+ const std::vector<std::string> &CorpusDirs, int NumJobs) {
Printf("INFO: -fork=%d: fuzzing in separate process(s)\n", NumJobs);
GlobalEnv Env;
@@ -292,8 +321,9 @@ void FuzzWithFork(Random &Rand, const FuzzingOptions &Options,
Env.Verbosity = Options.Verbosity;
Env.ProcessStartTime = std::chrono::system_clock::now();
Env.DataFlowBinary = Options.CollectDataFlow;
+ Env.Group = Options.ForkCorpusGroups;
- Vector<SizedFile> SeedFiles;
+ std::vector<SizedFile> SeedFiles;
for (auto &Dir : CorpusDirs)
GetSizedFilesFromDir(Dir, &SeedFiles);
std::sort(SeedFiles.begin(), SeedFiles.end());
@@ -314,10 +344,20 @@ void FuzzWithFork(Random &Rand, const FuzzingOptions &Options,
Env.Files.push_back(File.File);
} else {
auto CFPath = DirPlusFile(Env.TempDir, "merge.txt");
- CrashResistantMerge(Env.Args, {}, SeedFiles, &Env.Files, {}, &Env.Features,
- {}, &Env.Cov, CFPath, false);
+ std::set<uint32_t> NewFeatures, NewCov;
+ CrashResistantMerge(Env.Args, {}, SeedFiles, &Env.Files, Env.Features,
+ &NewFeatures, Env.Cov, &NewCov, CFPath,
+ /*Verbose=*/false, /*IsSetCoverMerge=*/false);
+ Env.Features.insert(NewFeatures.begin(), NewFeatures.end());
+ Env.Cov.insert(NewFeatures.begin(), NewFeatures.end());
RemoveFile(CFPath);
}
+
+ if (Env.Group) {
+ for (auto &path : Env.Files)
+ Env.FilesSizes.push_back(FileSize(path));
+ }
+
Printf("INFO: -fork=%d: %zd seed inputs, starting to fuzz in %s\n", NumJobs,
Env.Files.size(), Env.TempDir.c_str());
@@ -332,8 +372,10 @@ void FuzzWithFork(Random &Rand, const FuzzingOptions &Options,
WriteToFile(Unit({1}), Env.StopFile());
};
+ size_t MergeCycle = 20;
+ size_t JobExecuted = 0;
size_t JobId = 1;
- Vector<std::thread> Threads;
+ std::vector<std::thread> Threads;
for (int t = 0; t < NumJobs; t++) {
Threads.push_back(std::thread(WorkerThread, &FuzzQ, &MergeQ));
FuzzQ.Push(Env.CreateNewJob(JobId++));
@@ -353,7 +395,46 @@ void FuzzWithFork(Random &Rand, const FuzzingOptions &Options,
Env.RunOneMergeJob(Job.get());
- // Continue if our crash is one of the ignorred ones.
+ // merge the corpus .
+ JobExecuted++;
+ if (Env.Group && JobExecuted >= MergeCycle) {
+ std::vector<SizedFile> CurrentSeedFiles;
+ for (auto &Dir : CorpusDirs)
+ GetSizedFilesFromDir(Dir, &CurrentSeedFiles);
+ std::sort(CurrentSeedFiles.begin(), CurrentSeedFiles.end());
+
+ auto CFPath = DirPlusFile(Env.TempDir, "merge.txt");
+ std::set<uint32_t> TmpNewFeatures, TmpNewCov;
+ std::set<uint32_t> TmpFeatures, TmpCov;
+ Env.Files.clear();
+ Env.FilesSizes.clear();
+ CrashResistantMerge(Env.Args, {}, CurrentSeedFiles, &Env.Files,
+ TmpFeatures, &TmpNewFeatures, TmpCov, &TmpNewCov,
+ CFPath, /*Verbose=*/false, /*IsSetCoverMerge=*/false);
+ for (auto &path : Env.Files)
+ Env.FilesSizes.push_back(FileSize(path));
+ RemoveFile(CFPath);
+ JobExecuted = 0;
+ MergeCycle += 5;
+ }
+
+ // Since the number of corpus seeds will gradually increase, in order to
+ // control the number in each group to be about three times the number of
+ // seeds selected each time, the number of groups is dynamically adjusted.
+ if (Env.Files.size() < 2000)
+ Env.NumCorpuses = 12;
+ else if (Env.Files.size() < 6000)
+ Env.NumCorpuses = 20;
+ else if (Env.Files.size() < 12000)
+ Env.NumCorpuses = 32;
+ else if (Env.Files.size() < 16000)
+ Env.NumCorpuses = 40;
+ else if (Env.Files.size() < 24000)
+ Env.NumCorpuses = 60;
+ else
+ Env.NumCorpuses = 80;
+
+ // Continue if our crash is one of the ignored ones.
if (Options.IgnoreTimeouts && ExitCode == Options.TimeoutExitCode)
Env.NumTimeouts++;
else if (Options.IgnoreOOMs && ExitCode == Options.OOMExitCode)
diff --git a/contrib/libs/libfuzzer12/FuzzerFork.h b/contrib/libs/libfuzzer/FuzzerFork.h
index b29a43e13f..fc3e9d636c 100644
--- a/contrib/libs/libfuzzer12/FuzzerFork.h
+++ b/contrib/libs/libfuzzer/FuzzerFork.h
@@ -17,8 +17,8 @@
namespace fuzzer {
void FuzzWithFork(Random &Rand, const FuzzingOptions &Options,
- const Vector<std::string> &Args,
- const Vector<std::string> &CorpusDirs, int NumJobs);
+ const std::vector<std::string> &Args,
+ const std::vector<std::string> &CorpusDirs, int NumJobs);
} // namespace fuzzer
#endif // LLVM_FUZZER_FORK_H
diff --git a/contrib/libs/libfuzzer12/FuzzerIO.cpp b/contrib/libs/libfuzzer/FuzzerIO.cpp
index 54a7219fc0..0a58c5377b 100644
--- a/contrib/libs/libfuzzer12/FuzzerIO.cpp
+++ b/contrib/libs/libfuzzer/FuzzerIO.cpp
@@ -23,6 +23,14 @@ namespace fuzzer {
static FILE *OutputFile = stderr;
+FILE *GetOutputFile() {
+ return OutputFile;
+}
+
+void SetOutputFile(FILE *NewOutputFile) {
+ OutputFile = NewOutputFile;
+}
+
long GetEpoch(const std::string &Path) {
struct stat St;
if (stat(Path.c_str(), &St))
@@ -90,10 +98,11 @@ void AppendToFile(const uint8_t *Data, size_t Size, const std::string &Path) {
fclose(Out);
}
-void ReadDirToVectorOfUnits(const char *Path, Vector<Unit> *V,
- long *Epoch, size_t MaxSize, bool ExitOnError) {
+void ReadDirToVectorOfUnits(const char *Path, std::vector<Unit> *V, long *Epoch,
+ size_t MaxSize, bool ExitOnError,
+ std::vector<std::string> *VPaths) {
long E = Epoch ? *Epoch : 0;
- Vector<std::string> Files;
+ std::vector<std::string> Files;
ListFilesInDirRecursive(Path, Epoch, &Files, /*TopDir*/true);
size_t NumLoaded = 0;
for (size_t i = 0; i < Files.size(); i++) {
@@ -103,14 +112,16 @@ void ReadDirToVectorOfUnits(const char *Path, Vector<Unit> *V,
if ((NumLoaded & (NumLoaded - 1)) == 0 && NumLoaded >= 1024)
Printf("Loaded %zd/%zd files from %s\n", NumLoaded, Files.size(), Path);
auto S = FileToVector(X, MaxSize, ExitOnError);
- if (!S.empty())
+ if (!S.empty()) {
V->push_back(S);
+ if (VPaths)
+ VPaths->push_back(X);
+ }
}
}
-
-void GetSizedFilesFromDir(const std::string &Dir, Vector<SizedFile> *V) {
- Vector<std::string> Files;
+void GetSizedFilesFromDir(const std::string &Dir, std::vector<SizedFile> *V) {
+ std::vector<std::string> Files;
ListFilesInDirRecursive(Dir, 0, &Files, /*TopDir*/true);
for (auto &File : Files)
if (size_t Size = FileSize(File))
diff --git a/contrib/libs/libfuzzer12/FuzzerIO.h b/contrib/libs/libfuzzer/FuzzerIO.h
index abd25110d0..401afa0b44 100644
--- a/contrib/libs/libfuzzer12/FuzzerIO.h
+++ b/contrib/libs/libfuzzer/FuzzerIO.h
@@ -32,8 +32,9 @@ void WriteToFile(const Unit &U, const std::string &Path);
void AppendToFile(const uint8_t *Data, size_t Size, const std::string &Path);
void AppendToFile(const std::string &Data, const std::string &Path);
-void ReadDirToVectorOfUnits(const char *Path, Vector<Unit> *V,
- long *Epoch, size_t MaxSize, bool ExitOnError);
+void ReadDirToVectorOfUnits(const char *Path, std::vector<Unit> *V, long *Epoch,
+ size_t MaxSize, bool ExitOnError,
+ std::vector<std::string> *VPaths = 0);
// Returns "Dir/FileName" or equivalent for the current OS.
std::string DirPlusFile(const std::string &DirPath,
@@ -53,6 +54,10 @@ void DupAndCloseStderr();
void CloseStdout();
+// For testing.
+FILE *GetOutputFile();
+void SetOutputFile(FILE *NewOutputFile);
+
void Printf(const char *Fmt, ...);
void VPrintf(bool Verbose, const char *Fmt, ...);
@@ -65,7 +70,7 @@ bool IsDirectory(const std::string &Path);
size_t FileSize(const std::string &Path);
void ListFilesInDirRecursive(const std::string &Dir, long *Epoch,
- Vector<std::string> *V, bool TopDir);
+ std::vector<std::string> *V, bool TopDir);
bool MkDirRecursive(const std::string &Dir);
void RmDirRecursive(const std::string &Dir);
@@ -84,7 +89,7 @@ struct SizedFile {
bool operator<(const SizedFile &B) const { return Size < B.Size; }
};
-void GetSizedFilesFromDir(const std::string &Dir, Vector<SizedFile> *V);
+void GetSizedFilesFromDir(const std::string &Dir, std::vector<SizedFile> *V);
char GetSeparator();
bool IsSeparator(char C);
diff --git a/contrib/libs/libfuzzer12/FuzzerIOPosix.cpp b/contrib/libs/libfuzzer/FuzzerIOPosix.cpp
index 4706a40959..3700fb098e 100644
--- a/contrib/libs/libfuzzer12/FuzzerIOPosix.cpp
+++ b/contrib/libs/libfuzzer/FuzzerIOPosix.cpp
@@ -53,7 +53,7 @@ std::string Basename(const std::string &Path) {
}
void ListFilesInDirRecursive(const std::string &Dir, long *Epoch,
- Vector<std::string> *V, bool TopDir) {
+ std::vector<std::string> *V, bool TopDir) {
auto E = GetEpoch(Dir);
if (Epoch)
if (E && *Epoch >= E) return;
@@ -78,7 +78,6 @@ void ListFilesInDirRecursive(const std::string &Dir, long *Epoch,
*Epoch = E;
}
-
void IterateDirRecursive(const std::string &Dir,
void (*DirPreCallback)(const std::string &Dir),
void (*DirPostCallback)(const std::string &Dir),
diff --git a/contrib/libs/libfuzzer12/FuzzerIOWindows.cpp b/contrib/libs/libfuzzer/FuzzerIOWindows.cpp
index 61ad35e281..6771fc173c 100644
--- a/contrib/libs/libfuzzer12/FuzzerIOWindows.cpp
+++ b/contrib/libs/libfuzzer/FuzzerIOWindows.cpp
@@ -111,7 +111,7 @@ size_t FileSize(const std::string &Path) {
}
void ListFilesInDirRecursive(const std::string &Dir, long *Epoch,
- Vector<std::string> *V, bool TopDir) {
+ std::vector<std::string> *V, bool TopDir) {
auto E = GetEpoch(Dir);
if (Epoch)
if (E && *Epoch >= E) return;
@@ -159,7 +159,6 @@ void ListFilesInDirRecursive(const std::string &Dir, long *Epoch,
*Epoch = E;
}
-
void IterateDirRecursive(const std::string &Dir,
void (*DirPreCallback)(const std::string &Dir),
void (*DirPostCallback)(const std::string &Dir),
@@ -297,9 +296,8 @@ static size_t ParseServerAndShare(const std::string &FileName,
return Pos - Offset;
}
-// Parse the given Ref string from the position Offset, to exactly match the given
-// string Patt.
-// Returns number of characters considered if successful.
+// Parse the given Ref string from the position Offset, to exactly match the
+// given string Patt. Returns number of characters considered if successful.
static size_t ParseCustomString(const std::string &Ref, size_t Offset,
const char *Patt) {
size_t Len = strlen(Patt);
diff --git a/contrib/libs/libfuzzer12/FuzzerInterface.h b/contrib/libs/libfuzzer/FuzzerInterface.h
index f1130dc396..f1130dc396 100644
--- a/contrib/libs/libfuzzer12/FuzzerInterface.h
+++ b/contrib/libs/libfuzzer/FuzzerInterface.h
diff --git a/contrib/libs/libfuzzer12/FuzzerInternal.h b/contrib/libs/libfuzzer/FuzzerInternal.h
index 37c8a01dc3..6637b0034e 100644
--- a/contrib/libs/libfuzzer12/FuzzerInternal.h
+++ b/contrib/libs/libfuzzer/FuzzerInternal.h
@@ -35,8 +35,8 @@ public:
Fuzzer(UserCallback CB, InputCorpus &Corpus, MutationDispatcher &MD,
FuzzingOptions Options);
~Fuzzer();
- void Loop(Vector<SizedFile> &CorporaFiles);
- void ReadAndExecuteSeedCorpora(Vector<SizedFile> &CorporaFiles);
+ void Loop(std::vector<SizedFile> &CorporaFiles);
+ void ReadAndExecuteSeedCorpora(std::vector<SizedFile> &CorporaFiles);
void MinimizeCrashLoop(const Unit &U);
void RereadOutputCorpus(size_t MaxSize);
@@ -72,8 +72,9 @@ public:
void TPCUpdateObservedPCs();
// Merge Corpora[1:] into Corpora[0].
- void Merge(const Vector<std::string> &Corpora);
- void CrashResistantMergeInternalStep(const std::string &ControlFilePath);
+ void Merge(const std::vector<std::string> &Corpora);
+ void CrashResistantMergeInternalStep(const std::string &ControlFilePath,
+ bool IsSetCoverMerge);
MutationDispatcher &GetMD() { return MD; }
void PrintFinalStats();
void SetMaxInputLen(size_t MaxInputLen);
@@ -141,7 +142,7 @@ private:
size_t MaxMutationLen = 0;
size_t TmpMaxMutationLen = 0;
- Vector<uint32_t> UniqFeatureSetTmp;
+ std::vector<uint32_t> UniqFeatureSetTmp;
// Need to know our own thread.
static thread_local bool IsMyThread;
diff --git a/contrib/libs/libfuzzer12/FuzzerLoop.cpp b/contrib/libs/libfuzzer/FuzzerLoop.cpp
index fc53ac086b..cbfe73d813 100644
--- a/contrib/libs/libfuzzer12/FuzzerLoop.cpp
+++ b/contrib/libs/libfuzzer/FuzzerLoop.cpp
@@ -390,7 +390,7 @@ void Fuzzer::SetMaxMutationLen(size_t MaxMutationLen) {
void Fuzzer::CheckExitOnSrcPosOrItem() {
if (!Options.ExitOnSrcPos.empty()) {
- static auto *PCsSet = new Set<uintptr_t>;
+ static auto *PCsSet = new std::set<uintptr_t>;
auto HandlePC = [&](const TracePC::PCTableEntry *TE) {
if (!PCsSet->insert(TE->PC).second)
return;
@@ -415,20 +415,26 @@ void Fuzzer::CheckExitOnSrcPosOrItem() {
void Fuzzer::RereadOutputCorpus(size_t MaxSize) {
if (Options.OutputCorpus.empty() || !Options.ReloadIntervalSec)
return;
- Vector<Unit> AdditionalCorpus;
- ReadDirToVectorOfUnits(Options.OutputCorpus.c_str(), &AdditionalCorpus,
- &EpochOfLastReadOfOutputCorpus, MaxSize,
- /*ExitOnError*/ false);
+ std::vector<Unit> AdditionalCorpus;
+ std::vector<std::string> AdditionalCorpusPaths;
+ ReadDirToVectorOfUnits(
+ Options.OutputCorpus.c_str(), &AdditionalCorpus,
+ &EpochOfLastReadOfOutputCorpus, MaxSize,
+ /*ExitOnError*/ false,
+ (Options.Verbosity >= 2 ? &AdditionalCorpusPaths : nullptr));
if (Options.Verbosity >= 2)
Printf("Reload: read %zd new units.\n", AdditionalCorpus.size());
bool Reloaded = false;
- for (auto &U : AdditionalCorpus) {
+ for (size_t i = 0; i != AdditionalCorpus.size(); ++i) {
+ auto &U = AdditionalCorpus[i];
if (U.size() > MaxSize)
U.resize(MaxSize);
if (!Corpus.HasUnit(U)) {
if (RunOne(U.data(), U.size())) {
CheckExitOnSrcPosOrItem();
Reloaded = true;
+ if (Options.Verbosity >= 2)
+ Printf("Reloaded %s\n", AdditionalCorpusPaths[i].c_str());
}
}
}
@@ -442,8 +448,9 @@ void Fuzzer::PrintPulseAndReportSlowInput(const uint8_t *Data, size_t Size) {
if (!(TotalNumberOfRuns & (TotalNumberOfRuns - 1)) &&
secondsSinceProcessStartUp() >= 2)
PrintStats("pulse ");
- if (TimeOfUnit > TimeOfLongestUnitInSeconds * 1.1 &&
- TimeOfUnit >= Options.ReportSlowUnits) {
+ auto Threshhold =
+ static_cast<long>(static_cast<double>(TimeOfLongestUnitInSeconds) * 1.1);
+ if (TimeOfUnit > Threshhold && TimeOfUnit >= Options.ReportSlowUnits) {
TimeOfLongestUnitInSeconds = TimeOfUnit;
Printf("Slowest unit: %zd s:\n", TimeOfLongestUnitInSeconds);
WriteUnitToFileWithPrefix({Data, Data + Size}, "slow-unit-");
@@ -452,7 +459,7 @@ void Fuzzer::PrintPulseAndReportSlowInput(const uint8_t *Data, size_t Size) {
static void WriteFeatureSetToFile(const std::string &FeaturesDir,
const std::string &FileName,
- const Vector<uint32_t> &FeatureSet) {
+ const std::vector<uint32_t> &FeatureSet) {
if (FeaturesDir.empty() || FeatureSet.empty()) return;
WriteToFile(reinterpret_cast<const uint8_t *>(FeatureSet.data()),
FeatureSet.size() * sizeof(FeatureSet[0]),
@@ -503,6 +510,8 @@ bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile,
bool *FoundUniqFeatures) {
if (!Size)
return false;
+ // Largest input length should be INT_MAX.
+ assert(Size < std::numeric_limits<uint32_t>::max());
ExecuteCallback(Data, Size);
auto TimeOfUnit = duration_cast<microseconds>(UnitStopTime - UnitStartTime);
@@ -510,8 +519,8 @@ bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile,
UniqFeatureSetTmp.clear();
size_t FoundUniqFeaturesOfII = 0;
size_t NumUpdatesBefore = Corpus.NumFeatureUpdates();
- TPC.CollectFeatures([&](size_t Feature) {
- if (Corpus.AddFeature(Feature, Size, Options.Shrink))
+ TPC.CollectFeatures([&](uint32_t Feature) {
+ if (Corpus.AddFeature(Feature, static_cast<uint32_t>(Size), Options.Shrink))
UniqFeatureSetTmp.push_back(Feature);
if (Options.Entropic)
Corpus.UpdateFeatureFrequency(II, Feature);
@@ -541,7 +550,7 @@ bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile,
FoundUniqFeaturesOfII == II->UniqFeatureSet.size() &&
II->U.size() > Size) {
auto OldFeaturesFile = Sha1ToString(II->Sha1);
- Corpus.Replace(II, {Data, Data + Size});
+ Corpus.Replace(II, {Data, Data + Size}, TimeOfUnit);
RenameFeatureSetFile(Options.FeaturesDir, OldFeaturesFile,
Sha1ToString(II->Sha1));
return true;
@@ -577,7 +586,10 @@ static bool LooseMemeq(const uint8_t *A, const uint8_t *B, size_t Size) {
!memcmp(A + Size - Limit / 2, B + Size - Limit / 2, Limit / 2);
}
-void Fuzzer::ExecuteCallback(const uint8_t *Data, size_t Size) {
+// This method is not inlined because it would cause a test to fail where it
+// is part of the stack unwinding. See D97975 for details.
+ATTRIBUTE_NOINLINE void Fuzzer::ExecuteCallback(const uint8_t *Data,
+ size_t Size) {
TPC.RecordInitialStack();
TotalNumberOfRuns++;
assert(InFuzzingThread());
@@ -774,7 +786,7 @@ void Fuzzer::PurgeAllocator() {
LastAllocatorPurgeAttemptTime = system_clock::now();
}
-void Fuzzer::ReadAndExecuteSeedCorpora(Vector<SizedFile> &CorporaFiles) {
+void Fuzzer::ReadAndExecuteSeedCorpora(std::vector<SizedFile> &CorporaFiles) {
const size_t kMaxSaneLen = 1 << 20;
const size_t kMinDefaultLen = 4096;
size_t MaxSize = 0;
@@ -839,7 +851,7 @@ void Fuzzer::ReadAndExecuteSeedCorpora(Vector<SizedFile> &CorporaFiles) {
}
}
-void Fuzzer::Loop(Vector<SizedFile> &CorporaFiles) {
+void Fuzzer::Loop(std::vector<SizedFile> &CorporaFiles) {
auto FocusFunctionOrAuto = Options.FocusFunction;
DFT.Init(Options.DataFlowTrace, &FocusFunctionOrAuto, CorporaFiles,
MD.GetRand());
diff --git a/contrib/libs/libfuzzer12/FuzzerMain.cpp b/contrib/libs/libfuzzer/FuzzerMain.cpp
index 75f2f8e75c..75f2f8e75c 100644
--- a/contrib/libs/libfuzzer12/FuzzerMain.cpp
+++ b/contrib/libs/libfuzzer/FuzzerMain.cpp
diff --git a/contrib/libs/libfuzzer12/FuzzerMerge.cpp b/contrib/libs/libfuzzer/FuzzerMerge.cpp
index e3ad8b3851..24bd11958e 100644
--- a/contrib/libs/libfuzzer12/FuzzerMerge.cpp
+++ b/contrib/libs/libfuzzer/FuzzerMerge.cpp
@@ -77,14 +77,14 @@ bool Merger::Parse(std::istream &IS, bool ParseCoverage) {
size_t ExpectedStartMarker = 0;
const size_t kInvalidStartMarker = -1;
size_t LastSeenStartMarker = kInvalidStartMarker;
- Vector<uint32_t> TmpFeatures;
- Set<uint32_t> PCs;
+ std::vector<uint32_t> TmpFeatures;
+ std::set<uint32_t> PCs;
while (std::getline(IS, Line, '\n')) {
std::istringstream ISS1(Line);
std::string Marker;
- size_t N;
- ISS1 >> Marker;
- ISS1 >> N;
+ uint32_t N;
+ if (!(ISS1 >> Marker) || !(ISS1 >> N))
+ return false;
if (Marker == "STARTED") {
// STARTED FILE_ID FILE_SIZE
if (ExpectedStartMarker != N)
@@ -132,13 +132,16 @@ size_t Merger::ApproximateMemoryConsumption() const {
// Decides which files need to be merged (add those to NewFiles).
// Returns the number of new features added.
-size_t Merger::Merge(const Set<uint32_t> &InitialFeatures,
- Set<uint32_t> *NewFeatures,
- const Set<uint32_t> &InitialCov, Set<uint32_t> *NewCov,
- Vector<std::string> *NewFiles) {
+size_t Merger::Merge(const std::set<uint32_t> &InitialFeatures,
+ std::set<uint32_t> *NewFeatures,
+ const std::set<uint32_t> &InitialCov,
+ std::set<uint32_t> *NewCov,
+ std::vector<std::string> *NewFiles) {
NewFiles->clear();
+ NewFeatures->clear();
+ NewCov->clear();
assert(NumFilesInFirstCorpus <= Files.size());
- Set<uint32_t> AllFeatures = InitialFeatures;
+ std::set<uint32_t> AllFeatures = InitialFeatures;
// What features are in the initial corpus?
for (size_t i = 0; i < NumFilesInFirstCorpus; i++) {
@@ -148,7 +151,7 @@ size_t Merger::Merge(const Set<uint32_t> &InitialFeatures,
// Remove all features that we already know from all other inputs.
for (size_t i = NumFilesInFirstCorpus; i < Files.size(); i++) {
auto &Cur = Files[i].Features;
- Vector<uint32_t> Tmp;
+ std::vector<uint32_t> Tmp;
std::set_difference(Cur.begin(), Cur.end(), AllFeatures.begin(),
AllFeatures.end(), std::inserter(Tmp, Tmp.begin()));
Cur.swap(Tmp);
@@ -186,15 +189,16 @@ size_t Merger::Merge(const Set<uint32_t> &InitialFeatures,
return NewFeatures->size();
}
-Set<uint32_t> Merger::AllFeatures() const {
- Set<uint32_t> S;
+std::set<uint32_t> Merger::AllFeatures() const {
+ std::set<uint32_t> S;
for (auto &File : Files)
S.insert(File.Features.begin(), File.Features.end());
return S;
}
// Inner process. May crash if the target crashes.
-void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) {
+void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath,
+ bool IsSetCoverMerge) {
Printf("MERGE-INNER: using the control file '%s'\n", CFPath.c_str());
Merger M;
std::ifstream IF(CFPath);
@@ -210,11 +214,11 @@ void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) {
M.Files.size() - M.FirstNotProcessedFile);
std::ofstream OF(CFPath, std::ofstream::out | std::ofstream::app);
- Set<size_t> AllFeatures;
+ std::set<size_t> AllFeatures;
auto PrintStatsWrapper = [this, &AllFeatures](const char* Where) {
this->PrintStats(Where, "\n", 0, AllFeatures.size());
};
- Set<const TracePC::PCTableEntry *> AllPCs;
+ std::set<const TracePC::PCTableEntry *> AllPCs;
for (size_t i = M.FirstNotProcessedFile; i < M.Files.size(); i++) {
Fuzzer::MaybeExitGracefully();
auto U = FileToVector(M.Files[i].Name);
@@ -232,13 +236,14 @@ void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) {
// Collect coverage. We are iterating over the files in this order:
// * First, files in the initial corpus ordered by size, smallest first.
// * Then, all other files, smallest first.
- // So it makes no sense to record all features for all files, instead we
- // only record features that were not seen before.
- Set<size_t> UniqFeatures;
- TPC.CollectFeatures([&](size_t Feature) {
- if (AllFeatures.insert(Feature).second)
- UniqFeatures.insert(Feature);
- });
+ std::set<size_t> Features;
+ if (IsSetCoverMerge)
+ TPC.CollectFeatures([&](size_t Feature) { Features.insert(Feature); });
+ else
+ TPC.CollectFeatures([&](size_t Feature) {
+ if (AllFeatures.insert(Feature).second)
+ Features.insert(Feature);
+ });
TPC.UpdateObservedPCs();
// Show stats.
if (!(TotalNumberOfRuns & (TotalNumberOfRuns - 1)))
@@ -247,7 +252,7 @@ void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) {
PrintStatsWrapper("LOADED");
// Write the post-run marker and the coverage.
OF << "FT " << i;
- for (size_t F : UniqFeatures)
+ for (size_t F : Features)
OF << " " << F;
OF << "\n";
OF << "COV " << i;
@@ -261,15 +266,137 @@ void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) {
PrintStatsWrapper("DONE ");
}
-static size_t WriteNewControlFile(const std::string &CFPath,
- const Vector<SizedFile> &OldCorpus,
- const Vector<SizedFile> &NewCorpus,
- const Vector<MergeFileInfo> &KnownFiles) {
+// Merges all corpora into the first corpus. A file is added into
+// the first corpus only if it adds new features. Unlike `Merger::Merge`,
+// this implementation calculates an approximation of the minimum set
+// of corpora files, that cover all known features (set cover problem).
+// Generally, this means that files with more features are preferred for
+// merge into the first corpus. When two files have the same number of
+// features, the smaller one is preferred.
+size_t Merger::SetCoverMerge(const std::set<uint32_t> &InitialFeatures,
+ std::set<uint32_t> *NewFeatures,
+ const std::set<uint32_t> &InitialCov,
+ std::set<uint32_t> *NewCov,
+ std::vector<std::string> *NewFiles) {
+ assert(NumFilesInFirstCorpus <= Files.size());
+ NewFiles->clear();
+ NewFeatures->clear();
+ NewCov->clear();
+ std::set<uint32_t> AllFeatures;
+ // 1 << 21 - 1 is the maximum feature index.
+ // See 'kFeatureSetSize' in 'FuzzerCorpus.h'.
+ const uint32_t kFeatureSetSize = 1 << 21;
+ std::vector<bool> Covered(kFeatureSetSize, false);
+ size_t NumCovered = 0;
+
+ std::set<uint32_t> ExistingFeatures = InitialFeatures;
+ for (size_t i = 0; i < NumFilesInFirstCorpus; ++i)
+ ExistingFeatures.insert(Files[i].Features.begin(), Files[i].Features.end());
+
+ // Mark the existing features as covered.
+ for (const auto &F : ExistingFeatures) {
+ if (!Covered[F % kFeatureSetSize]) {
+ ++NumCovered;
+ Covered[F % kFeatureSetSize] = true;
+ }
+ // Calculate an underestimation of the set of covered features
+ // since the `Covered` bitvector is smaller than the feature range.
+ AllFeatures.insert(F % kFeatureSetSize);
+ }
+
+ std::set<size_t> RemainingFiles;
+ for (size_t i = NumFilesInFirstCorpus; i < Files.size(); ++i) {
+ // Construct an incremental sequence which represent the
+ // indices to all files (excluding those in the initial corpus).
+ // RemainingFiles = range(NumFilesInFirstCorpus..Files.size()).
+ RemainingFiles.insert(i);
+ // Insert this file's unique features to all features.
+ for (const auto &F : Files[i].Features)
+ AllFeatures.insert(F % kFeatureSetSize);
+ }
+
+ // Integrate files into Covered until set is complete.
+ while (NumCovered != AllFeatures.size()) {
+ // Index to file with largest number of unique features.
+ size_t MaxFeaturesIndex = NumFilesInFirstCorpus;
+ // Indices to remove from RemainingFiles.
+ std::set<size_t> RemoveIndices;
+ // Running max unique feature count.
+ // Updated upon finding a file with more features.
+ size_t MaxNumFeatures = 0;
+
+ // Iterate over all files not yet integrated into Covered,
+ // to find the file which has the largest number of
+ // features that are not already in Covered.
+ for (const auto &i : RemainingFiles) {
+ const auto &File = Files[i];
+ size_t CurrentUnique = 0;
+ // Count number of features in this file
+ // which are not yet in Covered.
+ for (const auto &F : File.Features)
+ if (!Covered[F % kFeatureSetSize])
+ ++CurrentUnique;
+
+ if (CurrentUnique == 0) {
+ // All features in this file are already in Covered: skip next time.
+ RemoveIndices.insert(i);
+ } else if (CurrentUnique > MaxNumFeatures ||
+ (CurrentUnique == MaxNumFeatures &&
+ File.Size < Files[MaxFeaturesIndex].Size)) {
+ // Update the max features file based on unique features
+ // Break ties by selecting smaller files.
+ MaxNumFeatures = CurrentUnique;
+ MaxFeaturesIndex = i;
+ }
+ }
+ // Must be a valid index/
+ assert(MaxFeaturesIndex < Files.size());
+ // Remove any feature-less files found.
+ for (const auto &i : RemoveIndices)
+ RemainingFiles.erase(i);
+ if (MaxNumFeatures == 0) {
+ // Did not find a file that adds unique features.
+ // This means that we should have no remaining files.
+ assert(RemainingFiles.size() == 0);
+ assert(NumCovered == AllFeatures.size());
+ break;
+ }
+
+ // MaxFeaturesIndex must be an element of Remaining.
+ assert(RemainingFiles.find(MaxFeaturesIndex) != RemainingFiles.end());
+ // Remove the file with the most features from Remaining.
+ RemainingFiles.erase(MaxFeaturesIndex);
+ const auto &MaxFeatureFile = Files[MaxFeaturesIndex];
+ // Add the features of the max feature file to Covered.
+ for (const auto &F : MaxFeatureFile.Features) {
+ if (!Covered[F % kFeatureSetSize]) {
+ ++NumCovered;
+ Covered[F % kFeatureSetSize] = true;
+ NewFeatures->insert(F);
+ }
+ }
+ // Add the index to this file to the result.
+ NewFiles->push_back(MaxFeatureFile.Name);
+ // Update NewCov with the additional coverage
+ // that MaxFeatureFile provides.
+ for (const auto &C : MaxFeatureFile.Cov)
+ if (InitialCov.find(C) == InitialCov.end())
+ NewCov->insert(C);
+ }
+
+ return NewFeatures->size();
+}
+
+static size_t
+WriteNewControlFile(const std::string &CFPath,
+ const std::vector<SizedFile> &OldCorpus,
+ const std::vector<SizedFile> &NewCorpus,
+ const std::vector<MergeFileInfo> &KnownFiles) {
std::unordered_set<std::string> FilesToSkip;
for (auto &SF: KnownFiles)
FilesToSkip.insert(SF.Name);
- Vector<std::string> FilesToUse;
+ std::vector<std::string> FilesToUse;
auto MaybeUseFile = [=, &FilesToUse](std::string Name) {
if (FilesToSkip.find(Name) == FilesToSkip.end())
FilesToUse.push_back(Name);
@@ -297,19 +424,19 @@ static size_t WriteNewControlFile(const std::string &CFPath,
}
// Outer process. Does not call the target code and thus should not fail.
-void CrashResistantMerge(const Vector<std::string> &Args,
- const Vector<SizedFile> &OldCorpus,
- const Vector<SizedFile> &NewCorpus,
- Vector<std::string> *NewFiles,
- const Set<uint32_t> &InitialFeatures,
- Set<uint32_t> *NewFeatures,
- const Set<uint32_t> &InitialCov,
- Set<uint32_t> *NewCov,
- const std::string &CFPath,
- bool V /*Verbose*/) {
+void CrashResistantMerge(const std::vector<std::string> &Args,
+ const std::vector<SizedFile> &OldCorpus,
+ const std::vector<SizedFile> &NewCorpus,
+ std::vector<std::string> *NewFiles,
+ const std::set<uint32_t> &InitialFeatures,
+ std::set<uint32_t> *NewFeatures,
+ const std::set<uint32_t> &InitialCov,
+ std::set<uint32_t> *NewCov, const std::string &CFPath,
+ bool V, /*Verbose*/
+ bool IsSetCoverMerge) {
if (NewCorpus.empty() && OldCorpus.empty()) return; // Nothing to merge.
size_t NumAttempts = 0;
- Vector<MergeFileInfo> KnownFiles;
+ std::vector<MergeFileInfo> KnownFiles;
if (FileSize(CFPath)) {
VPrintf(V, "MERGE-OUTER: non-empty control file provided: '%s'\n",
CFPath.c_str());
@@ -361,6 +488,7 @@ void CrashResistantMerge(const Vector<std::string> &Args,
// Every inner process should execute at least one input.
Command BaseCmd(Args);
BaseCmd.removeFlag("merge");
+ BaseCmd.removeFlag("set_cover_merge");
BaseCmd.removeFlag("fork");
BaseCmd.removeFlag("collect_data_flow");
for (size_t Attempt = 1; Attempt <= NumAttempts; Attempt++) {
@@ -368,14 +496,16 @@ void CrashResistantMerge(const Vector<std::string> &Args,
VPrintf(V, "MERGE-OUTER: attempt %zd\n", Attempt);
Command Cmd(BaseCmd);
Cmd.addFlag("merge_control_file", CFPath);
- Cmd.addFlag("merge_inner", "1");
+ // If we are going to use the set cover implementation for
+ // minimization add the merge_inner=2 internal flag.
+ Cmd.addFlag("merge_inner", IsSetCoverMerge ? "2" : "1");
if (!V) {
Cmd.setOutputFile(getDevNull());
Cmd.combineOutAndErr();
}
auto ExitCode = ExecuteCommand(Cmd);
if (!ExitCode) {
- VPrintf(V, "MERGE-OUTER: succesfull in %zd attempt(s)\n", Attempt);
+ VPrintf(V, "MERGE-OUTER: successful in %zd attempt(s)\n", Attempt);
break;
}
}
@@ -393,7 +523,10 @@ void CrashResistantMerge(const Vector<std::string> &Args,
M.ApproximateMemoryConsumption() >> 20, GetPeakRSSMb());
M.Files.insert(M.Files.end(), KnownFiles.begin(), KnownFiles.end());
- M.Merge(InitialFeatures, NewFeatures, InitialCov, NewCov, NewFiles);
+ if (IsSetCoverMerge)
+ M.SetCoverMerge(InitialFeatures, NewFeatures, InitialCov, NewCov, NewFiles);
+ else
+ M.Merge(InitialFeatures, NewFeatures, InitialCov, NewCov, NewFiles);
VPrintf(V, "MERGE-OUTER: %zd new files with %zd new features added; "
"%zd new coverage edges\n",
NewFiles->size(), NewFeatures->size(), NewCov->size());
diff --git a/contrib/libs/libfuzzer12/FuzzerMerge.h b/contrib/libs/libfuzzer/FuzzerMerge.h
index e0c6bc539b..42f798e1da 100644
--- a/contrib/libs/libfuzzer12/FuzzerMerge.h
+++ b/contrib/libs/libfuzzer/FuzzerMerge.h
@@ -41,6 +41,7 @@
#define LLVM_FUZZER_MERGE_H
#include "FuzzerDefs.h"
+#include "FuzzerIO.h"
#include <istream>
#include <ostream>
@@ -52,11 +53,11 @@ namespace fuzzer {
struct MergeFileInfo {
std::string Name;
size_t Size = 0;
- Vector<uint32_t> Features, Cov;
+ std::vector<uint32_t> Features, Cov;
};
struct Merger {
- Vector<MergeFileInfo> Files;
+ std::vector<MergeFileInfo> Files;
size_t NumFilesInFirstCorpus = 0;
size_t FirstNotProcessedFile = 0;
std::string LastFailure;
@@ -64,23 +65,28 @@ struct Merger {
bool Parse(std::istream &IS, bool ParseCoverage);
bool Parse(const std::string &Str, bool ParseCoverage);
void ParseOrExit(std::istream &IS, bool ParseCoverage);
- size_t Merge(const Set<uint32_t> &InitialFeatures, Set<uint32_t> *NewFeatures,
- const Set<uint32_t> &InitialCov, Set<uint32_t> *NewCov,
- Vector<std::string> *NewFiles);
+ size_t Merge(const std::set<uint32_t> &InitialFeatures,
+ std::set<uint32_t> *NewFeatures,
+ const std::set<uint32_t> &InitialCov, std::set<uint32_t> *NewCov,
+ std::vector<std::string> *NewFiles);
+ size_t SetCoverMerge(const std::set<uint32_t> &InitialFeatures,
+ std::set<uint32_t> *NewFeatures,
+ const std::set<uint32_t> &InitialCov,
+ std::set<uint32_t> *NewCov,
+ std::vector<std::string> *NewFiles);
size_t ApproximateMemoryConsumption() const;
- Set<uint32_t> AllFeatures() const;
+ std::set<uint32_t> AllFeatures() const;
};
-void CrashResistantMerge(const Vector<std::string> &Args,
- const Vector<SizedFile> &OldCorpus,
- const Vector<SizedFile> &NewCorpus,
- Vector<std::string> *NewFiles,
- const Set<uint32_t> &InitialFeatures,
- Set<uint32_t> *NewFeatures,
- const Set<uint32_t> &InitialCov,
- Set<uint32_t> *NewCov,
- const std::string &CFPath,
- bool Verbose);
+void CrashResistantMerge(const std::vector<std::string> &Args,
+ const std::vector<SizedFile> &OldCorpus,
+ const std::vector<SizedFile> &NewCorpus,
+ std::vector<std::string> *NewFiles,
+ const std::set<uint32_t> &InitialFeatures,
+ std::set<uint32_t> *NewFeatures,
+ const std::set<uint32_t> &InitialCov,
+ std::set<uint32_t> *NewCov, const std::string &CFPath,
+ bool Verbose, bool IsSetCoverMerge);
} // namespace fuzzer
diff --git a/contrib/libs/libfuzzer12/FuzzerMutate.cpp b/contrib/libs/libfuzzer/FuzzerMutate.cpp
index cf34a9fe8e..d663900fdc 100644
--- a/contrib/libs/libfuzzer12/FuzzerMutate.cpp
+++ b/contrib/libs/libfuzzer/FuzzerMutate.cpp
@@ -61,14 +61,20 @@ MutationDispatcher::MutationDispatcher(Random &Rand,
}
static char RandCh(Random &Rand) {
- if (Rand.RandBool()) return Rand(256);
+ if (Rand.RandBool())
+ return static_cast<char>(Rand(256));
const char Special[] = "!*'();:@&=+$,/?%#[]012Az-`~.\xff\x00";
return Special[Rand(sizeof(Special) - 1)];
}
size_t MutationDispatcher::Mutate_Custom(uint8_t *Data, size_t Size,
size_t MaxSize) {
- return EF->LLVMFuzzerCustomMutator(Data, Size, MaxSize, Rand.Rand());
+ if (EF->__msan_unpoison)
+ EF->__msan_unpoison(Data, Size);
+ if (EF->__msan_unpoison_param)
+ EF->__msan_unpoison_param(4);
+ return EF->LLVMFuzzerCustomMutator(Data, Size, MaxSize,
+ Rand.Rand<unsigned int>());
}
size_t MutationDispatcher::Mutate_CustomCrossOver(uint8_t *Data, size_t Size,
@@ -81,8 +87,18 @@ size_t MutationDispatcher::Mutate_CustomCrossOver(uint8_t *Data, size_t Size,
return 0;
CustomCrossOverInPlaceHere.resize(MaxSize);
auto &U = CustomCrossOverInPlaceHere;
+
+ if (EF->__msan_unpoison) {
+ EF->__msan_unpoison(Data, Size);
+ EF->__msan_unpoison(Other.data(), Other.size());
+ EF->__msan_unpoison(U.data(), U.size());
+ }
+ if (EF->__msan_unpoison_param)
+ EF->__msan_unpoison_param(7);
size_t NewSize = EF->LLVMFuzzerCustomCrossOver(
- Data, Size, Other.data(), Other.size(), U.data(), U.size(), Rand.Rand());
+ Data, Size, Other.data(), Other.size(), U.data(), U.size(),
+ Rand.Rand<unsigned int>());
+
if (!NewSize)
return 0;
assert(NewSize <= MaxSize && "CustomCrossOver returned overisized unit");
@@ -135,7 +151,8 @@ size_t MutationDispatcher::Mutate_InsertRepeatedBytes(uint8_t *Data,
// Insert new values at Data[Idx].
memmove(Data + Idx + N, Data + Idx, Size - Idx);
// Give preference to 0x00 and 0xff.
- uint8_t Byte = Rand.RandBool() ? Rand(256) : (Rand.RandBool() ? 0 : 255);
+ uint8_t Byte = static_cast<uint8_t>(
+ Rand.RandBool() ? Rand(256) : (Rand.RandBool() ? 0 : 255));
for (size_t i = 0; i < N; i++)
Data[Idx + i] = Byte;
return Size + N;
@@ -178,7 +195,8 @@ size_t MutationDispatcher::ApplyDictionaryEntry(uint8_t *Data, size_t Size,
Size += W.size();
} else { // Overwrite some bytes with W.
if (W.size() > Size) return 0;
- size_t Idx = UsePositionHint ? DE.GetPositionHint() : Rand(Size - W.size());
+ size_t Idx =
+ UsePositionHint ? DE.GetPositionHint() : Rand(Size + 1 - W.size());
memcpy(Data + Idx, W.data(), W.size());
}
return Size;
@@ -227,8 +245,8 @@ DictionaryEntry MutationDispatcher::MakeDictionaryEntryFromCMP(
T Arg1, T Arg2, const uint8_t *Data, size_t Size) {
if (Rand.RandBool()) Arg1 = Bswap(Arg1);
if (Rand.RandBool()) Arg2 = Bswap(Arg2);
- T Arg1Mutation = Arg1 + Rand(-1, 1);
- T Arg2Mutation = Arg2 + Rand(-1, 1);
+ T Arg1Mutation = static_cast<T>(Arg1 + Rand(-1, 1));
+ T Arg2Mutation = static_cast<T>(Arg2 + Rand(-1, 1));
return MakeDictionaryEntryFromCMP(&Arg1, &Arg2, &Arg1Mutation, &Arg2Mutation,
sizeof(Arg1), Data, Size);
}
@@ -245,23 +263,23 @@ size_t MutationDispatcher::Mutate_AddWordFromTORC(
DictionaryEntry DE;
switch (Rand(4)) {
case 0: {
- auto X = TPC.TORC8.Get(Rand.Rand());
+ auto X = TPC.TORC8.Get(Rand.Rand<size_t>());
DE = MakeDictionaryEntryFromCMP(X.A, X.B, Data, Size);
} break;
case 1: {
- auto X = TPC.TORC4.Get(Rand.Rand());
+ auto X = TPC.TORC4.Get(Rand.Rand<size_t>());
if ((X.A >> 16) == 0 && (X.B >> 16) == 0 && Rand.RandBool())
DE = MakeDictionaryEntryFromCMP((uint16_t)X.A, (uint16_t)X.B, Data, Size);
else
DE = MakeDictionaryEntryFromCMP(X.A, X.B, Data, Size);
} break;
case 2: {
- auto X = TPC.TORCW.Get(Rand.Rand());
+ auto X = TPC.TORCW.Get(Rand.Rand<size_t>());
DE = MakeDictionaryEntryFromCMP(X.A, X.B, Data, Size);
} break;
case 3: if (Options.UseMemmem) {
- auto X = TPC.MMT.Get(Rand.Rand());
- DE = DictionaryEntry(X);
+ auto X = TPC.MMT.Get(Rand.Rand<size_t>());
+ DE = DictionaryEntry(X);
} break;
default:
assert(0);
@@ -387,12 +405,12 @@ size_t ChangeBinaryInteger(uint8_t *Data, size_t Size, Random &Rand) {
assert(Off + sizeof(T) <= Size);
T Val;
if (Off < 64 && !Rand(4)) {
- Val = Size;
+ Val = static_cast<T>(Size);
if (Rand.RandBool())
Val = Bswap(Val);
} else {
memcpy(&Val, Data + Off, sizeof(Val));
- T Add = Rand(21);
+ T Add = static_cast<T>(Rand(21));
Add -= 10;
if (Rand.RandBool())
Val = Bswap(T(Bswap(Val) + Add)); // Add assuming different endiannes.
@@ -462,12 +480,12 @@ void MutationDispatcher::RecordSuccessfulMutationSequence() {
assert(DE->GetW().size());
// Linear search is fine here as this happens seldom.
if (!PersistentAutoDictionary.ContainsWord(DE->GetW()))
- PersistentAutoDictionary.push_back({DE->GetW(), 1});
+ PersistentAutoDictionary.push_back(*DE);
}
}
void MutationDispatcher::PrintRecommendedDictionary() {
- Vector<DictionaryEntry> V;
+ std::vector<DictionaryEntry> V;
for (auto &DE : PersistentAutoDictionary)
if (!ManualDictionary.ContainsWord(DE.GetW()))
V.push_back(DE);
@@ -522,7 +540,7 @@ size_t MutationDispatcher::DefaultMutate(uint8_t *Data, size_t Size,
// Mutates Data in place, returns new size.
size_t MutationDispatcher::MutateImpl(uint8_t *Data, size_t Size,
size_t MaxSize,
- Vector<Mutator> &Mutators) {
+ std::vector<Mutator> &Mutators) {
assert(MaxSize > 0);
// Some mutations may fail (e.g. can't insert more bytes if Size == MaxSize),
// in which case they will return 0.
@@ -544,7 +562,7 @@ size_t MutationDispatcher::MutateImpl(uint8_t *Data, size_t Size,
// Mask represents the set of Data bytes that are worth mutating.
size_t MutationDispatcher::MutateWithMask(uint8_t *Data, size_t Size,
size_t MaxSize,
- const Vector<uint8_t> &Mask) {
+ const std::vector<uint8_t> &Mask) {
size_t MaskedSize = std::min(Size, Mask.size());
// * Copy the worthy bytes into a temporary array T
// * Mutate T
diff --git a/contrib/libs/libfuzzer12/FuzzerMutate.h b/contrib/libs/libfuzzer/FuzzerMutate.h
index fd37191156..97704e2160 100644
--- a/contrib/libs/libfuzzer12/FuzzerMutate.h
+++ b/contrib/libs/libfuzzer/FuzzerMutate.h
@@ -77,7 +77,7 @@ public:
/// that have '1' in Mask.
/// Mask.size() should be >= Size.
size_t MutateWithMask(uint8_t *Data, size_t Size, size_t MaxSize,
- const Vector<uint8_t> &Mask);
+ const std::vector<uint8_t> &Mask);
/// Applies one of the default mutations. Provided as a service
/// to mutation authors.
@@ -104,7 +104,7 @@ public:
size_t AddWordFromDictionary(Dictionary &D, uint8_t *Data, size_t Size,
size_t MaxSize);
size_t MutateImpl(uint8_t *Data, size_t Size, size_t MaxSize,
- Vector<Mutator> &Mutators);
+ std::vector<Mutator> &Mutators);
size_t InsertPartOf(const uint8_t *From, size_t FromSize, uint8_t *To,
size_t ToSize, size_t MaxToSize);
@@ -133,22 +133,22 @@ public:
// entries that led to successful discoveries in the past mutations.
Dictionary PersistentAutoDictionary;
- Vector<DictionaryEntry *> CurrentDictionaryEntrySequence;
+ std::vector<DictionaryEntry *> CurrentDictionaryEntrySequence;
static const size_t kCmpDictionaryEntriesDequeSize = 16;
DictionaryEntry CmpDictionaryEntriesDeque[kCmpDictionaryEntriesDequeSize];
size_t CmpDictionaryEntriesDequeIdx = 0;
const Unit *CrossOverWith = nullptr;
- Vector<uint8_t> MutateInPlaceHere;
- Vector<uint8_t> MutateWithMaskTemp;
+ std::vector<uint8_t> MutateInPlaceHere;
+ std::vector<uint8_t> MutateWithMaskTemp;
// CustomCrossOver needs its own buffer as a custom implementation may call
// LLVMFuzzerMutate, which in turn may resize MutateInPlaceHere.
- Vector<uint8_t> CustomCrossOverInPlaceHere;
+ std::vector<uint8_t> CustomCrossOverInPlaceHere;
- Vector<Mutator> Mutators;
- Vector<Mutator> DefaultMutators;
- Vector<Mutator> CurrentMutatorSequence;
+ std::vector<Mutator> Mutators;
+ std::vector<Mutator> DefaultMutators;
+ std::vector<Mutator> CurrentMutatorSequence;
};
} // namespace fuzzer
diff --git a/contrib/libs/libfuzzer12/FuzzerOptions.h b/contrib/libs/libfuzzer/FuzzerOptions.h
index 2e8501626b..83d64c2b9e 100644
--- a/contrib/libs/libfuzzer12/FuzzerOptions.h
+++ b/contrib/libs/libfuzzer/FuzzerOptions.h
@@ -48,6 +48,7 @@ struct FuzzingOptions {
int ReportSlowUnits = 10;
bool OnlyASCII = false;
bool Entropic = true;
+ bool ForkCorpusGroups = false;
size_t EntropicFeatureFrequencyThreshold = 0xFF;
size_t EntropicNumberOfRarestFeatures = 100;
bool EntropicScalePerExecTime = false;
diff --git a/contrib/libs/libfuzzer12/FuzzerPlatform.h b/contrib/libs/libfuzzer/FuzzerPlatform.h
index 1602e67895..1602e67895 100644
--- a/contrib/libs/libfuzzer12/FuzzerPlatform.h
+++ b/contrib/libs/libfuzzer/FuzzerPlatform.h
diff --git a/contrib/libs/libfuzzer12/FuzzerRandom.h b/contrib/libs/libfuzzer/FuzzerRandom.h
index 659283eee2..ad6c07eb5e 100644
--- a/contrib/libs/libfuzzer12/FuzzerRandom.h
+++ b/contrib/libs/libfuzzer/FuzzerRandom.h
@@ -18,18 +18,27 @@ class Random : public std::minstd_rand {
public:
Random(unsigned int seed) : std::minstd_rand(seed) {}
result_type operator()() { return this->std::minstd_rand::operator()(); }
- size_t Rand() { return this->operator()(); }
- size_t RandBool() { return Rand() % 2; }
+ template <typename T>
+ typename std::enable_if<std::is_integral<T>::value, T>::type Rand() {
+ return static_cast<T>(this->operator()());
+ }
+ size_t RandBool() { return this->operator()() % 2; }
size_t SkewTowardsLast(size_t n) {
size_t T = this->operator()(n * n);
- size_t Res = sqrt(T);
+ size_t Res = static_cast<size_t>(sqrt(T));
return Res;
}
- size_t operator()(size_t n) { return n ? Rand() % n : 0; }
- intptr_t operator()(intptr_t From, intptr_t To) {
+ template <typename T>
+ typename std::enable_if<std::is_integral<T>::value, T>::type operator()(T n) {
+ return n ? Rand<T>() % n : 0;
+ }
+ template <typename T>
+ typename std::enable_if<std::is_integral<T>::value, T>::type
+ operator()(T From, T To) {
assert(From < To);
- intptr_t RangeSize = To - From + 1;
- return operator()(RangeSize) + From;
+ auto RangeSize = static_cast<unsigned long long>(To) -
+ static_cast<unsigned long long>(From) + 1;
+ return static_cast<T>(this->operator()(RangeSize) + From);
}
};
diff --git a/contrib/libs/libfuzzer12/FuzzerSHA1.cpp b/contrib/libs/libfuzzer/FuzzerSHA1.cpp
index 2005dc7003..b05655c8ef 100644
--- a/contrib/libs/libfuzzer12/FuzzerSHA1.cpp
+++ b/contrib/libs/libfuzzer/FuzzerSHA1.cpp
@@ -134,12 +134,13 @@ void sha1_hashBlock(sha1nfo *s) {
s->state[4] += e;
}
-void sha1_addUncounted(sha1nfo *s, uint8_t data) {
- uint8_t * const b = (uint8_t*) s->buffer;
+// Adds the least significant byte of |data|.
+void sha1_addUncounted(sha1nfo *s, uint32_t data) {
+ uint8_t *const b = (uint8_t *)s->buffer;
#ifdef SHA_BIG_ENDIAN
- b[s->bufferOffset] = data;
+ b[s->bufferOffset] = static_cast<uint8_t>(data);
#else
- b[s->bufferOffset ^ 3] = data;
+ b[s->bufferOffset ^ 3] = static_cast<uint8_t>(data);
#endif
s->bufferOffset++;
if (s->bufferOffset == BLOCK_LENGTH) {
diff --git a/contrib/libs/libfuzzer12/FuzzerSHA1.h b/contrib/libs/libfuzzer/FuzzerSHA1.h
index 05cbacda87..05cbacda87 100644
--- a/contrib/libs/libfuzzer12/FuzzerSHA1.h
+++ b/contrib/libs/libfuzzer/FuzzerSHA1.h
diff --git a/contrib/libs/libfuzzer12/FuzzerTracePC.cpp b/contrib/libs/libfuzzer/FuzzerTracePC.cpp
index 91e94d8240..af8d1ce50f 100644
--- a/contrib/libs/libfuzzer12/FuzzerTracePC.cpp
+++ b/contrib/libs/libfuzzer/FuzzerTracePC.cpp
@@ -106,6 +106,15 @@ void TracePC::PrintModuleInfo() {
}
if (size_t NumExtraCounters = ExtraCountersEnd() - ExtraCountersBegin())
Printf("INFO: %zd Extra Counters\n", NumExtraCounters);
+
+ size_t MaxFeatures = CollectFeatures([](uint32_t) {});
+ if (MaxFeatures > std::numeric_limits<uint32_t>::max())
+ Printf("WARNING: The coverage PC tables may produce up to %zu features.\n"
+ "This exceeds the maximum 32-bit value. Some features may be\n"
+ "ignored, and fuzzing may become less precise. If possible,\n"
+ "consider refactoring the fuzzer into several smaller fuzzers\n"
+ "linked against only a portion of the current target.\n",
+ MaxFeatures);
}
ATTRIBUTE_NO_SANITIZE_ALL
@@ -148,7 +157,7 @@ ALWAYS_INLINE uintptr_t TracePC::GetNextInstructionPc(uintptr_t PC) {
}
void TracePC::UpdateObservedPCs() {
- Vector<uintptr_t> CoveredFuncs;
+ std::vector<uintptr_t> CoveredFuncs;
auto ObservePC = [&](const PCTableEntry *TE) {
if (ObservedPCs.insert(TE).second && DoPrintNewPCs) {
PrintPC("\tNEW_PC: %p %F %L", "\tNEW_PC: %p",
@@ -291,8 +300,8 @@ void TracePC::PrintCoverage(bool PrintAllCounters) {
FunctionStr = FunctionStr.substr(3);
std::string LineStr = DescribePC("%l", VisualizePC);
size_t NumEdges = Last - First;
- Vector<uintptr_t> UncoveredPCs;
- Vector<uintptr_t> CoveredPCs;
+ std::vector<uintptr_t> UncoveredPCs;
+ std::vector<uintptr_t> CoveredPCs;
for (auto TE = First; TE < Last; TE++)
if (!ObservedPCs.count(TE))
UncoveredPCs.push_back(TE->PC);
@@ -356,7 +365,7 @@ void TracePC::AddValueForMemcmp(void *caller_pc, const void *s1, const void *s2,
uint8_t HammingDistance = 0;
for (; I < Len; I++) {
if (B1[I] != B2[I] || (StopAtZero && B1[I] == 0)) {
- HammingDistance = Popcountll(B1[I] ^ B2[I]);
+ HammingDistance = static_cast<uint8_t>(Popcountll(B1[I] ^ B2[I]));
break;
}
}
@@ -382,6 +391,7 @@ void TracePC::HandleCmp(uintptr_t PC, T Arg1, T Arg2) {
ValueProfileMap.AddValue(PC * 128 + 64 + AbsoluteDistance);
}
+ATTRIBUTE_NO_SANITIZE_MEMORY
static size_t InternalStrnlen(const char *S, size_t MaxLen) {
size_t Len = 0;
for (; Len < MaxLen && S[Len]; Len++) {}
@@ -389,7 +399,8 @@ static size_t InternalStrnlen(const char *S, size_t MaxLen) {
}
// Finds min of (strlen(S1), strlen(S2)).
-// Needed bacause one of these strings may actually be non-zero terminated.
+// Needed because one of these strings may actually be non-zero terminated.
+ATTRIBUTE_NO_SANITIZE_MEMORY
static size_t InternalStrnlen2(const char *S1, const char *S2) {
size_t Len = 0;
for (; S1[Len] && S2[Len]; Len++) {}
diff --git a/contrib/libs/libfuzzer12/FuzzerTracePC.h b/contrib/libs/libfuzzer/FuzzerTracePC.h
index 0090923073..af1f9d81e9 100644
--- a/contrib/libs/libfuzzer12/FuzzerTracePC.h
+++ b/contrib/libs/libfuzzer/FuzzerTracePC.h
@@ -54,7 +54,7 @@ struct MemMemTable {
void Add(const uint8_t *Data, size_t Size) {
if (Size <= 2) return;
Size = std::min(Size, Word::GetMaxSize());
- size_t Idx = SimpleFastHash(Data, Size) % kSize;
+ auto Idx = SimpleFastHash(Data, Size) % kSize;
MemMemWords[Idx].Set(Data, Size);
}
const Word &Get(size_t Idx) {
@@ -79,7 +79,7 @@ class TracePC {
void SetPrintNewPCs(bool P) { DoPrintNewPCs = P; }
void SetPrintNewFuncs(size_t P) { NumPrintNewFuncs = P; }
void UpdateObservedPCs();
- template <class Callback> void CollectFeatures(Callback CB) const;
+ template <class Callback> size_t CollectFeatures(Callback CB) const;
void ResetMaps() {
ValueProfileMap.Reset();
@@ -169,7 +169,7 @@ private:
size_t NumPCTables;
size_t NumPCsInPCTables;
- Set<const PCTableEntry*> ObservedPCs;
+ std::set<const PCTableEntry *> ObservedPCs;
std::unordered_map<uintptr_t, uintptr_t> ObservedFuncs; // PC => Counter.
uint8_t *FocusFunctionCounterPtr = nullptr;
@@ -193,7 +193,7 @@ size_t ForEachNonZeroByte(const uint8_t *Begin, const uint8_t *End,
Handle8bitCounter(FirstFeature, P - Begin, V);
// Iterate by Step bytes at a time.
- for (; P < End; P += Step)
+ for (; P + Step <= End; P += Step)
if (LargeType Bundle = *reinterpret_cast<const LargeType *>(P)) {
Bundle = HostToLE(Bundle);
for (size_t I = 0; I < Step; I++, Bundle >>= 8)
@@ -234,16 +234,16 @@ unsigned CounterToFeature(T Counter) {
return Bit;
}
-template <class Callback> // void Callback(size_t Feature)
-ATTRIBUTE_NO_SANITIZE_ADDRESS
-ATTRIBUTE_NOINLINE
-void TracePC::CollectFeatures(Callback HandleFeature) const {
+template <class Callback> // void Callback(uint32_t Feature)
+ATTRIBUTE_NO_SANITIZE_ADDRESS ATTRIBUTE_NOINLINE size_t
+TracePC::CollectFeatures(Callback HandleFeature) const {
auto Handle8bitCounter = [&](size_t FirstFeature,
size_t Idx, uint8_t Counter) {
if (UseCounters)
- HandleFeature(FirstFeature + Idx * 8 + CounterToFeature(Counter));
+ HandleFeature(static_cast<uint32_t>(FirstFeature + Idx * 8 +
+ CounterToFeature(Counter)));
else
- HandleFeature(FirstFeature + Idx);
+ HandleFeature(static_cast<uint32_t>(FirstFeature + Idx));
};
size_t FirstFeature = 0;
@@ -263,16 +263,18 @@ void TracePC::CollectFeatures(Callback HandleFeature) const {
if (UseValueProfileMask) {
ValueProfileMap.ForEach([&](size_t Idx) {
- HandleFeature(FirstFeature + Idx);
+ HandleFeature(static_cast<uint32_t>(FirstFeature + Idx));
});
FirstFeature += ValueProfileMap.SizeInBits();
}
// Step function, grows similar to 8 * Log_2(A).
- auto StackDepthStepFunction = [](uint32_t A) -> uint32_t {
- if (!A) return A;
- uint32_t Log2 = Log(A);
- if (Log2 < 3) return A;
+ auto StackDepthStepFunction = [](size_t A) -> size_t {
+ if (!A)
+ return A;
+ auto Log2 = Log(A);
+ if (Log2 < 3)
+ return A;
Log2 -= 3;
return (Log2 + 1) * 8 + ((A >> Log2) & 7);
};
@@ -280,8 +282,13 @@ void TracePC::CollectFeatures(Callback HandleFeature) const {
assert(StackDepthStepFunction(1024 * 4) == 80);
assert(StackDepthStepFunction(1024 * 1024) == 144);
- if (auto MaxStackOffset = GetMaxStackOffset())
- HandleFeature(FirstFeature + StackDepthStepFunction(MaxStackOffset / 8));
+ if (auto MaxStackOffset = GetMaxStackOffset()) {
+ HandleFeature(static_cast<uint32_t>(
+ FirstFeature + StackDepthStepFunction(MaxStackOffset / 8)));
+ FirstFeature += StackDepthStepFunction(std::numeric_limits<size_t>::max());
+ }
+
+ return FirstFeature;
}
extern TracePC TPC;
diff --git a/contrib/libs/libfuzzer12/FuzzerUtil.cpp b/contrib/libs/libfuzzer/FuzzerUtil.cpp
index 7eecb68d07..aeab70f20c 100644
--- a/contrib/libs/libfuzzer12/FuzzerUtil.cpp
+++ b/contrib/libs/libfuzzer/FuzzerUtil.cpp
@@ -43,7 +43,7 @@ void PrintASCIIByte(uint8_t Byte) {
else if (Byte >= 32 && Byte < 127)
Printf("%c", Byte);
else
- Printf("\\x%02x", Byte);
+ Printf("\\%03o", Byte);
}
void PrintASCII(const uint8_t *Data, size_t Size, const char *PrintAfter) {
@@ -111,7 +111,7 @@ bool ParseOneDictionaryEntry(const std::string &Str, Unit *U) {
char Hex[] = "0xAA";
Hex[2] = Str[Pos + 2];
Hex[3] = Str[Pos + 3];
- U->push_back(strtol(Hex, nullptr, 16));
+ U->push_back(static_cast<uint8_t>(strtol(Hex, nullptr, 16)));
Pos += 3;
continue;
}
@@ -124,7 +124,7 @@ bool ParseOneDictionaryEntry(const std::string &Str, Unit *U) {
return true;
}
-bool ParseDictionaryFile(const std::string &Text, Vector<Unit> *Units) {
+bool ParseDictionaryFile(const std::string &Text, std::vector<Unit> *Units) {
if (Text.empty()) {
Printf("ParseDictionaryFile: file does not exist or is empty\n");
return false;
@@ -226,10 +226,11 @@ unsigned NumberOfCpuCores() {
return N;
}
-size_t SimpleFastHash(const uint8_t *Data, size_t Size) {
- size_t Res = 0;
+uint64_t SimpleFastHash(const void *Data, size_t Size, uint64_t Initial) {
+ uint64_t Res = Initial;
+ const uint8_t *Bytes = static_cast<const uint8_t *>(Data);
for (size_t i = 0; i < Size; i++)
- Res = Res * 11 + Data[i];
+ Res = Res * 11 + Bytes[i];
return Res;
}
diff --git a/contrib/libs/libfuzzer12/FuzzerUtil.h b/contrib/libs/libfuzzer/FuzzerUtil.h
index e90be08500..71d49097e5 100644
--- a/contrib/libs/libfuzzer12/FuzzerUtil.h
+++ b/contrib/libs/libfuzzer/FuzzerUtil.h
@@ -66,10 +66,10 @@ int CloseProcessPipe(FILE *F);
const void *SearchMemory(const void *haystack, size_t haystacklen,
const void *needle, size_t needlelen);
-std::string CloneArgsWithoutX(const Vector<std::string> &Args,
+std::string CloneArgsWithoutX(const std::vector<std::string> &Args,
const char *X1, const char *X2);
-inline std::string CloneArgsWithoutX(const Vector<std::string> &Args,
+inline std::string CloneArgsWithoutX(const std::vector<std::string> &Args,
const char *X) {
return CloneArgsWithoutX(Args, X, X);
}
@@ -88,9 +88,11 @@ std::string DisassembleCmd(const std::string &FileName);
std::string SearchRegexCmd(const std::string &Regex);
-size_t SimpleFastHash(const uint8_t *Data, size_t Size);
+uint64_t SimpleFastHash(const void *Data, size_t Size, uint64_t Initial = 0);
-inline uint32_t Log(uint32_t X) { return 32 - Clz(X) - 1; }
+inline size_t Log(size_t X) {
+ return static_cast<size_t>((sizeof(unsigned long long) * 8) - Clzll(X) - 1);
+}
inline size_t PageSize() { return 4096; }
inline uint8_t *RoundUpByPage(uint8_t *P) {
diff --git a/contrib/libs/libfuzzer12/FuzzerUtilDarwin.cpp b/contrib/libs/libfuzzer/FuzzerUtilDarwin.cpp
index a5bed658a4..a5bed658a4 100644
--- a/contrib/libs/libfuzzer12/FuzzerUtilDarwin.cpp
+++ b/contrib/libs/libfuzzer/FuzzerUtilDarwin.cpp
diff --git a/contrib/libs/libfuzzer12/FuzzerUtilLinux.cpp b/contrib/libs/libfuzzer/FuzzerUtilLinux.cpp
index 981f9a8b42..981f9a8b42 100644
--- a/contrib/libs/libfuzzer12/FuzzerUtilLinux.cpp
+++ b/contrib/libs/libfuzzer/FuzzerUtilLinux.cpp
diff --git a/contrib/libs/libfuzzer12/FuzzerUtilPosix.cpp b/contrib/libs/libfuzzer/FuzzerUtilPosix.cpp
index afb733409a..0446d732a9 100644
--- a/contrib/libs/libfuzzer12/FuzzerUtilPosix.cpp
+++ b/contrib/libs/libfuzzer/FuzzerUtilPosix.cpp
@@ -77,10 +77,13 @@ static void SetSigaction(int signum,
return;
}
- sigact = {};
- sigact.sa_flags = SA_SIGINFO;
- sigact.sa_sigaction = callback;
- if (sigaction(signum, &sigact, 0)) {
+ struct sigaction new_sigact = {};
+ // Address sanitizer needs SA_ONSTACK (causing the signal handler to run on a
+ // dedicated stack) in order to be able to detect stack overflows; keep the
+ // flag if it's set.
+ new_sigact.sa_flags = SA_SIGINFO | (sigact.sa_flags & SA_ONSTACK);
+ new_sigact.sa_sigaction = callback;
+ if (sigaction(signum, &new_sigact, nullptr)) {
Printf("libFuzzer: sigaction failed with %d\n", errno);
exit(1);
}
diff --git a/contrib/libs/libfuzzer12/FuzzerUtilWindows.cpp b/contrib/libs/libfuzzer/FuzzerUtilWindows.cpp
index 1a54bb569e..3598758dbb 100644
--- a/contrib/libs/libfuzzer12/FuzzerUtilWindows.cpp
+++ b/contrib/libs/libfuzzer/FuzzerUtilWindows.cpp
@@ -204,7 +204,7 @@ const void *SearchMemory(const void *Data, size_t DataLen, const void *Patt,
}
std::string DisassembleCmd(const std::string &FileName) {
- Vector<std::string> command_vector;
+ std::vector<std::string> command_vector;
command_vector.push_back("dumpbin /summary > nul");
if (ExecuteCommand(Command(command_vector)) == 0)
return "dumpbin /disasm " + FileName;
diff --git a/contrib/libs/libfuzzer12/FuzzerValueBitMap.h b/contrib/libs/libfuzzer/FuzzerValueBitMap.h
index ddbfe200af..ddbfe200af 100644
--- a/contrib/libs/libfuzzer12/FuzzerValueBitMap.h
+++ b/contrib/libs/libfuzzer/FuzzerValueBitMap.h
diff --git a/contrib/libs/libfuzzer12/LICENSE.TXT b/contrib/libs/libfuzzer/LICENSE.TXT
index 5a79a1b9d5..5a79a1b9d5 100644
--- a/contrib/libs/libfuzzer12/LICENSE.TXT
+++ b/contrib/libs/libfuzzer/LICENSE.TXT
diff --git a/contrib/libs/libfuzzer/README.txt b/contrib/libs/libfuzzer/README.txt
new file mode 100644
index 0000000000..fc8843246e
--- /dev/null
+++ b/contrib/libs/libfuzzer/README.txt
@@ -0,0 +1,11 @@
+Compiler-RT
+================================
+
+This directory and its subdirectories contain source code for the compiler
+support routines.
+
+Compiler-RT is open source software. You may freely distribute it under the
+terms of the license agreement found in LICENSE.txt.
+
+================================
+
diff --git a/contrib/libs/libfuzzer12/README.txt b/contrib/libs/libfuzzer12/README.txt
deleted file mode 100644
index 3eee01c776..0000000000
--- a/contrib/libs/libfuzzer12/README.txt
+++ /dev/null
@@ -1 +0,0 @@
-See http://llvm.org/docs/LibFuzzer.html