aboutsummaryrefslogtreecommitdiffstats
path: root/contrib/libs/llvm12/lib/Transforms/Instrumentation
diff options
context:
space:
mode:
authorshadchin <shadchin@yandex-team.ru>2022-02-10 16:44:39 +0300
committerDaniil Cherednik <dcherednik@yandex-team.ru>2022-02-10 16:44:39 +0300
commite9656aae26e0358d5378e5b63dcac5c8dbe0e4d0 (patch)
tree64175d5cadab313b3e7039ebaa06c5bc3295e274 /contrib/libs/llvm12/lib/Transforms/Instrumentation
parent2598ef1d0aee359b4b6d5fdd1758916d5907d04f (diff)
downloadydb-e9656aae26e0358d5378e5b63dcac5c8dbe0e4d0.tar.gz
Restoring authorship annotation for <shadchin@yandex-team.ru>. Commit 2 of 2.
Diffstat (limited to 'contrib/libs/llvm12/lib/Transforms/Instrumentation')
-rw-r--r--contrib/libs/llvm12/lib/Transforms/Instrumentation/AddressSanitizer.cpp160
-rw-r--r--contrib/libs/llvm12/lib/Transforms/Instrumentation/CFGMST.h18
-rw-r--r--contrib/libs/llvm12/lib/Transforms/Instrumentation/CGProfile.cpp4
-rw-r--r--contrib/libs/llvm12/lib/Transforms/Instrumentation/ControlHeightReduction.cpp34
-rw-r--r--contrib/libs/llvm12/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp1272
-rw-r--r--contrib/libs/llvm12/lib/Transforms/Instrumentation/GCOVProfiling.cpp788
-rw-r--r--contrib/libs/llvm12/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp348
-rw-r--r--contrib/libs/llvm12/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp16
-rw-r--r--contrib/libs/llvm12/lib/Transforms/Instrumentation/InstrProfiling.cpp108
-rw-r--r--contrib/libs/llvm12/lib/Transforms/Instrumentation/Instrumentation.cpp6
-rw-r--r--contrib/libs/llvm12/lib/Transforms/Instrumentation/MemProfiler.cpp1276
-rw-r--r--contrib/libs/llvm12/lib/Transforms/Instrumentation/MemorySanitizer.cpp620
-rw-r--r--contrib/libs/llvm12/lib/Transforms/Instrumentation/PGOInstrumentation.cpp566
-rw-r--r--contrib/libs/llvm12/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp18
-rw-r--r--contrib/libs/llvm12/lib/Transforms/Instrumentation/PoisonChecking.cpp12
-rw-r--r--contrib/libs/llvm12/lib/Transforms/Instrumentation/SanitizerCoverage.cpp104
-rw-r--r--contrib/libs/llvm12/lib/Transforms/Instrumentation/ThreadSanitizer.cpp250
-rw-r--r--contrib/libs/llvm12/lib/Transforms/Instrumentation/ValueProfileCollector.cpp2
-rw-r--r--contrib/libs/llvm12/lib/Transforms/Instrumentation/ValueProfileCollector.h12
-rw-r--r--contrib/libs/llvm12/lib/Transforms/Instrumentation/ya.make18
20 files changed, 2816 insertions, 2816 deletions
diff --git a/contrib/libs/llvm12/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/contrib/libs/llvm12/lib/Transforms/Instrumentation/AddressSanitizer.cpp
index bb82bf2337..f4e471706d 100644
--- a/contrib/libs/llvm12/lib/Transforms/Instrumentation/AddressSanitizer.cpp
+++ b/contrib/libs/llvm12/lib/Transforms/Instrumentation/AddressSanitizer.cpp
@@ -105,7 +105,7 @@ static const uint64_t kSystemZ_ShadowOffset64 = 1ULL << 52;
static const uint64_t kMIPS32_ShadowOffset32 = 0x0aaa0000;
static const uint64_t kMIPS64_ShadowOffset64 = 1ULL << 37;
static const uint64_t kAArch64_ShadowOffset64 = 1ULL << 36;
-static const uint64_t kRISCV64_ShadowOffset64 = 0x20000000;
+static const uint64_t kRISCV64_ShadowOffset64 = 0x20000000;
static const uint64_t kFreeBSD_ShadowOffset32 = 1ULL << 30;
static const uint64_t kFreeBSD_ShadowOffset64 = 1ULL << 46;
static const uint64_t kNetBSD_ShadowOffset32 = 1ULL << 30;
@@ -130,48 +130,48 @@ static const size_t kMaxStackMallocSize = 1 << 16; // 64K
static const uintptr_t kCurrentStackFrameMagic = 0x41B58AB3;
static const uintptr_t kRetiredStackFrameMagic = 0x45E0360E;
-const char kAsanModuleCtorName[] = "asan.module_ctor";
-const char kAsanModuleDtorName[] = "asan.module_dtor";
+const char kAsanModuleCtorName[] = "asan.module_ctor";
+const char kAsanModuleDtorName[] = "asan.module_dtor";
static const uint64_t kAsanCtorAndDtorPriority = 1;
// On Emscripten, the system needs more than one priorities for constructors.
static const uint64_t kAsanEmscriptenCtorAndDtorPriority = 50;
-const char kAsanReportErrorTemplate[] = "__asan_report_";
-const char kAsanRegisterGlobalsName[] = "__asan_register_globals";
-const char kAsanUnregisterGlobalsName[] = "__asan_unregister_globals";
-const char kAsanRegisterImageGlobalsName[] = "__asan_register_image_globals";
-const char kAsanUnregisterImageGlobalsName[] =
- "__asan_unregister_image_globals";
-const char kAsanRegisterElfGlobalsName[] = "__asan_register_elf_globals";
-const char kAsanUnregisterElfGlobalsName[] = "__asan_unregister_elf_globals";
-const char kAsanPoisonGlobalsName[] = "__asan_before_dynamic_init";
-const char kAsanUnpoisonGlobalsName[] = "__asan_after_dynamic_init";
-const char kAsanInitName[] = "__asan_init";
-const char kAsanVersionCheckNamePrefix[] = "__asan_version_mismatch_check_v";
-const char kAsanPtrCmp[] = "__sanitizer_ptr_cmp";
-const char kAsanPtrSub[] = "__sanitizer_ptr_sub";
-const char kAsanHandleNoReturnName[] = "__asan_handle_no_return";
+const char kAsanReportErrorTemplate[] = "__asan_report_";
+const char kAsanRegisterGlobalsName[] = "__asan_register_globals";
+const char kAsanUnregisterGlobalsName[] = "__asan_unregister_globals";
+const char kAsanRegisterImageGlobalsName[] = "__asan_register_image_globals";
+const char kAsanUnregisterImageGlobalsName[] =
+ "__asan_unregister_image_globals";
+const char kAsanRegisterElfGlobalsName[] = "__asan_register_elf_globals";
+const char kAsanUnregisterElfGlobalsName[] = "__asan_unregister_elf_globals";
+const char kAsanPoisonGlobalsName[] = "__asan_before_dynamic_init";
+const char kAsanUnpoisonGlobalsName[] = "__asan_after_dynamic_init";
+const char kAsanInitName[] = "__asan_init";
+const char kAsanVersionCheckNamePrefix[] = "__asan_version_mismatch_check_v";
+const char kAsanPtrCmp[] = "__sanitizer_ptr_cmp";
+const char kAsanPtrSub[] = "__sanitizer_ptr_sub";
+const char kAsanHandleNoReturnName[] = "__asan_handle_no_return";
static const int kMaxAsanStackMallocSizeClass = 10;
-const char kAsanStackMallocNameTemplate[] = "__asan_stack_malloc_";
-const char kAsanStackFreeNameTemplate[] = "__asan_stack_free_";
-const char kAsanGenPrefix[] = "___asan_gen_";
-const char kODRGenPrefix[] = "__odr_asan_gen_";
-const char kSanCovGenPrefix[] = "__sancov_gen_";
-const char kAsanSetShadowPrefix[] = "__asan_set_shadow_";
-const char kAsanPoisonStackMemoryName[] = "__asan_poison_stack_memory";
-const char kAsanUnpoisonStackMemoryName[] = "__asan_unpoison_stack_memory";
+const char kAsanStackMallocNameTemplate[] = "__asan_stack_malloc_";
+const char kAsanStackFreeNameTemplate[] = "__asan_stack_free_";
+const char kAsanGenPrefix[] = "___asan_gen_";
+const char kODRGenPrefix[] = "__odr_asan_gen_";
+const char kSanCovGenPrefix[] = "__sancov_gen_";
+const char kAsanSetShadowPrefix[] = "__asan_set_shadow_";
+const char kAsanPoisonStackMemoryName[] = "__asan_poison_stack_memory";
+const char kAsanUnpoisonStackMemoryName[] = "__asan_unpoison_stack_memory";
// ASan version script has __asan_* wildcard. Triple underscore prevents a
// linker (gold) warning about attempting to export a local symbol.
-const char kAsanGlobalsRegisteredFlagName[] = "___asan_globals_registered";
+const char kAsanGlobalsRegisteredFlagName[] = "___asan_globals_registered";
-const char kAsanOptionDetectUseAfterReturn[] =
+const char kAsanOptionDetectUseAfterReturn[] =
"__asan_option_detect_stack_use_after_return";
-const char kAsanShadowMemoryDynamicAddress[] =
+const char kAsanShadowMemoryDynamicAddress[] =
"__asan_shadow_memory_dynamic_address";
-const char kAsanAllocaPoison[] = "__asan_alloca_poison";
-const char kAsanAllocasUnpoison[] = "__asan_allocas_unpoison";
+const char kAsanAllocaPoison[] = "__asan_alloca_poison";
+const char kAsanAllocasUnpoison[] = "__asan_allocas_unpoison";
// Accesses sizes are powers of two: 1, 2, 4, 8, 16.
static const size_t kNumberOfAccessSizes = 5;
@@ -427,7 +427,7 @@ static ShadowMapping getShadowMapping(Triple &TargetTriple, int LongSize,
bool IsKasan) {
bool IsAndroid = TargetTriple.isAndroid();
bool IsIOS = TargetTriple.isiOS() || TargetTriple.isWatchOS();
- bool IsMacOS = TargetTriple.isMacOSX();
+ bool IsMacOS = TargetTriple.isMacOSX();
bool IsFreeBSD = TargetTriple.isOSFreeBSD();
bool IsNetBSD = TargetTriple.isOSNetBSD();
bool IsPS4CPU = TargetTriple.isPS4CPU();
@@ -440,7 +440,7 @@ static ShadowMapping getShadowMapping(Triple &TargetTriple, int LongSize,
bool IsMIPS64 = TargetTriple.isMIPS64();
bool IsArmOrThumb = TargetTriple.isARM() || TargetTriple.isThumb();
bool IsAArch64 = TargetTriple.getArch() == Triple::aarch64;
- bool IsRISCV64 = TargetTriple.getArch() == Triple::riscv64;
+ bool IsRISCV64 = TargetTriple.getArch() == Triple::riscv64;
bool IsWindows = TargetTriple.isOSWindows();
bool IsFuchsia = TargetTriple.isOSFuchsia();
bool IsMyriad = TargetTriple.getVendor() == llvm::Triple::Myriad;
@@ -505,12 +505,12 @@ static ShadowMapping getShadowMapping(Triple &TargetTriple, int LongSize,
Mapping.Offset = kMIPS64_ShadowOffset64;
else if (IsIOS)
Mapping.Offset = kDynamicShadowSentinel;
- else if (IsMacOS && IsAArch64)
- Mapping.Offset = kDynamicShadowSentinel;
+ else if (IsMacOS && IsAArch64)
+ Mapping.Offset = kDynamicShadowSentinel;
else if (IsAArch64)
Mapping.Offset = kAArch64_ShadowOffset64;
- else if (IsRISCV64)
- Mapping.Offset = kRISCV64_ShadowOffset64;
+ else if (IsRISCV64)
+ Mapping.Offset = kRISCV64_ShadowOffset64;
else
Mapping.Offset = kDefaultShadowOffset64;
}
@@ -529,7 +529,7 @@ static ShadowMapping getShadowMapping(Triple &TargetTriple, int LongSize,
// we could OR the constant in a single instruction, but it's more
// efficient to load it once and use indexed addressing.
Mapping.OrShadowOffset = !IsAArch64 && !IsPPC64 && !IsSystemZ && !IsPS4CPU &&
- !IsRISCV64 &&
+ !IsRISCV64 &&
!(Mapping.Offset & (Mapping.Offset - 1)) &&
Mapping.Offset != kDynamicShadowSentinel;
bool IsAndroidWithIfuncSupport =
@@ -961,14 +961,14 @@ struct FunctionStackPoisoner : public InstVisitor<FunctionStackPoisoner> {
void createDynamicAllocasInitStorage();
// ----------------------- Visitors.
- /// Collect all Ret instructions, or the musttail call instruction if it
- /// precedes the return instruction.
- void visitReturnInst(ReturnInst &RI) {
- if (CallInst *CI = RI.getParent()->getTerminatingMustTailCall())
- RetVec.push_back(CI);
- else
- RetVec.push_back(&RI);
- }
+ /// Collect all Ret instructions, or the musttail call instruction if it
+ /// precedes the return instruction.
+ void visitReturnInst(ReturnInst &RI) {
+ if (CallInst *CI = RI.getParent()->getTerminatingMustTailCall())
+ RetVec.push_back(CI);
+ else
+ RetVec.push_back(&RI);
+ }
/// Collect all Resume instructions.
void visitResumeInst(ResumeInst &RI) { RetVec.push_back(&RI); }
@@ -1002,10 +1002,10 @@ struct FunctionStackPoisoner : public InstVisitor<FunctionStackPoisoner> {
// Unpoison dynamic allocas redzones.
void unpoisonDynamicAllocas() {
- for (Instruction *Ret : RetVec)
+ for (Instruction *Ret : RetVec)
unpoisonDynamicAllocasBeforeInst(Ret, DynamicAllocaLayout);
- for (Instruction *StackRestoreInst : StackRestoreVec)
+ for (Instruction *StackRestoreInst : StackRestoreVec)
unpoisonDynamicAllocasBeforeInst(StackRestoreInst,
StackRestoreInst->getOperand(0));
}
@@ -1064,9 +1064,9 @@ struct FunctionStackPoisoner : public InstVisitor<FunctionStackPoisoner> {
!ConstantInt::isValueValidForType(IntptrTy, SizeValue))
return;
// Find alloca instruction that corresponds to llvm.lifetime argument.
- // Currently we can only handle lifetime markers pointing to the
- // beginning of the alloca.
- AllocaInst *AI = findAllocaForValue(II.getArgOperand(1), true);
+ // Currently we can only handle lifetime markers pointing to the
+ // beginning of the alloca.
+ AllocaInst *AI = findAllocaForValue(II.getArgOperand(1), true);
if (!AI) {
HasUntracedLifetimeIntrinsic = true;
return;
@@ -1561,7 +1561,7 @@ void AddressSanitizer::instrumentMop(ObjectSizeOffsetVisitor &ObjSizeVis,
if (ClOpt && ClOptGlobals) {
// If initialization order checking is disabled, a simple access to a
// dynamically initialized global is always valid.
- GlobalVariable *G = dyn_cast<GlobalVariable>(getUnderlyingObject(Addr));
+ GlobalVariable *G = dyn_cast<GlobalVariable>(getUnderlyingObject(Addr));
if (G && (!ClInitializers || GlobalIsLinkerInitialized(G)) &&
isSafeAccess(ObjSizeVis, Addr, O.TypeSize)) {
NumOptimizedAccessesToGlobalVar++;
@@ -1571,7 +1571,7 @@ void AddressSanitizer::instrumentMop(ObjectSizeOffsetVisitor &ObjSizeVis,
if (ClOpt && ClOptStack) {
// A direct inbounds access to a stack variable is always valid.
- if (isa<AllocaInst>(getUnderlyingObject(Addr)) &&
+ if (isa<AllocaInst>(getUnderlyingObject(Addr)) &&
isSafeAccess(ObjSizeVis, Addr, O.TypeSize)) {
NumOptimizedAccessesToStackVar++;
return;
@@ -1873,14 +1873,14 @@ bool ModuleAddressSanitizer::shouldInstrumentGlobal(GlobalVariable *G) const {
return false;
}
- // Do not instrument user-defined sections (with names resembling
- // valid C identifiers)
- if (TargetTriple.isOSBinFormatELF()) {
- if (llvm::all_of(Section,
- [](char c) { return llvm::isAlnum(c) || c == '_'; }))
- return false;
- }
-
+ // Do not instrument user-defined sections (with names resembling
+ // valid C identifiers)
+ if (TargetTriple.isOSBinFormatELF()) {
+ if (llvm::all_of(Section,
+ [](char c) { return llvm::isAlnum(c) || c == '_'; }))
+ return false;
+ }
+
// On COFF, if the section name contains '$', it is highly likely that the
// user is using section sorting to create an array of globals similar to
// the way initialization callbacks are registered in .init_array and
@@ -1965,10 +1965,10 @@ StringRef ModuleAddressSanitizer::getGlobalMetadataSection() const {
case Triple::ELF: return "asan_globals";
case Triple::MachO: return "__DATA,__asan_globals,regular";
case Triple::Wasm:
- case Triple::GOFF:
+ case Triple::GOFF:
case Triple::XCOFF:
report_fatal_error(
- "ModuleAddressSanitizer not implemented for object file format");
+ "ModuleAddressSanitizer not implemented for object file format");
case Triple::UnknownObjectFormat:
break;
}
@@ -2119,8 +2119,8 @@ void ModuleAddressSanitizer::InstrumentGlobalsELF(
// Update llvm.compiler.used, adding the new metadata globals. This is
// needed so that during LTO these variables stay alive.
- if (!MetadataGlobals.empty())
- appendToCompilerUsed(M, MetadataGlobals);
+ if (!MetadataGlobals.empty())
+ appendToCompilerUsed(M, MetadataGlobals);
// RegisteredFlag serves two purposes. First, we can pass it to dladdr()
// to look up the loaded image that contains it. Second, we can store in it
@@ -2133,15 +2133,15 @@ void ModuleAddressSanitizer::InstrumentGlobalsELF(
ConstantInt::get(IntptrTy, 0), kAsanGlobalsRegisteredFlagName);
RegisteredFlag->setVisibility(GlobalVariable::HiddenVisibility);
- // Create start and stop symbols.
- GlobalVariable *StartELFMetadata = new GlobalVariable(
- M, IntptrTy, false, GlobalVariable::ExternalWeakLinkage, nullptr,
- "__start_" + getGlobalMetadataSection());
- StartELFMetadata->setVisibility(GlobalVariable::HiddenVisibility);
- GlobalVariable *StopELFMetadata = new GlobalVariable(
- M, IntptrTy, false, GlobalVariable::ExternalWeakLinkage, nullptr,
- "__stop_" + getGlobalMetadataSection());
- StopELFMetadata->setVisibility(GlobalVariable::HiddenVisibility);
+ // Create start and stop symbols.
+ GlobalVariable *StartELFMetadata = new GlobalVariable(
+ M, IntptrTy, false, GlobalVariable::ExternalWeakLinkage, nullptr,
+ "__start_" + getGlobalMetadataSection());
+ StartELFMetadata->setVisibility(GlobalVariable::HiddenVisibility);
+ GlobalVariable *StopELFMetadata = new GlobalVariable(
+ M, IntptrTy, false, GlobalVariable::ExternalWeakLinkage, nullptr,
+ "__stop_" + getGlobalMetadataSection());
+ StopELFMetadata->setVisibility(GlobalVariable::HiddenVisibility);
// Create a call to register the globals with the runtime.
IRB.CreateCall(AsanRegisterElfGlobals,
@@ -2345,9 +2345,9 @@ bool ModuleAddressSanitizer::InstrumentGlobals(IRBuilder<> &IRB, Module &M,
NewGlobal->setSection("__TEXT,__asan_cstring,regular");
}
- // Transfer the debug info and type metadata. The payload starts at offset
- // zero so we can copy the metadata over as is.
- NewGlobal->copyMetadata(G, 0);
+ // Transfer the debug info and type metadata. The payload starts at offset
+ // zero so we can copy the metadata over as is.
+ NewGlobal->copyMetadata(G, 0);
Value *Indices2[2];
Indices2[0] = IRB.getInt32(0);
@@ -3105,8 +3105,8 @@ void FunctionStackPoisoner::processStaticAllocas() {
int StackMallocIdx = -1;
DebugLoc EntryDebugLocation;
if (auto SP = F.getSubprogram())
- EntryDebugLocation =
- DILocation::get(SP->getContext(), SP->getScopeLine(), 0, SP);
+ EntryDebugLocation =
+ DILocation::get(SP->getContext(), SP->getScopeLine(), 0, SP);
Instruction *InsBefore = AllocaVec[0];
IRBuilder<> IRB(InsBefore);
@@ -3314,7 +3314,7 @@ void FunctionStackPoisoner::processStaticAllocas() {
SmallVector<uint8_t, 64> ShadowAfterReturn;
// (Un)poison the stack before all ret instructions.
- for (Instruction *Ret : RetVec) {
+ for (Instruction *Ret : RetVec) {
IRBuilder<> IRBRet(Ret);
// Mark the current frame as retired.
IRBRet.CreateStore(ConstantInt::get(IntptrTy, kRetiredStackFrameMagic),
diff --git a/contrib/libs/llvm12/lib/Transforms/Instrumentation/CFGMST.h b/contrib/libs/llvm12/lib/Transforms/Instrumentation/CFGMST.h
index 77f525c020..6580b6d7d7 100644
--- a/contrib/libs/llvm12/lib/Transforms/Instrumentation/CFGMST.h
+++ b/contrib/libs/llvm12/lib/Transforms/Instrumentation/CFGMST.h
@@ -103,7 +103,7 @@ public:
const BasicBlock *Entry = &(F.getEntryBlock());
uint64_t EntryWeight = (BFI != nullptr ? BFI->getEntryFreq() : 2);
// If we want to instrument the entry count, lower the weight to 0.
- if (InstrumentFuncEntry)
+ if (InstrumentFuncEntry)
EntryWeight = 0;
Edge *EntryIncoming = nullptr, *EntryOutgoing = nullptr,
*ExitOutgoing = nullptr, *ExitIncoming = nullptr;
@@ -278,19 +278,19 @@ public:
BranchProbabilityInfo *BPI;
BlockFrequencyInfo *BFI;
- // If function entry will be always instrumented.
- bool InstrumentFuncEntry;
-
+ // If function entry will be always instrumented.
+ bool InstrumentFuncEntry;
+
public:
- CFGMST(Function &Func, bool InstrumentFuncEntry_,
- BranchProbabilityInfo *BPI_ = nullptr,
+ CFGMST(Function &Func, bool InstrumentFuncEntry_,
+ BranchProbabilityInfo *BPI_ = nullptr,
BlockFrequencyInfo *BFI_ = nullptr)
- : F(Func), BPI(BPI_), BFI(BFI_),
- InstrumentFuncEntry(InstrumentFuncEntry_) {
+ : F(Func), BPI(BPI_), BFI(BFI_),
+ InstrumentFuncEntry(InstrumentFuncEntry_) {
buildEdges();
sortEdgesByWeight();
computeMinimumSpanningTree();
- if (AllEdges.size() > 1 && InstrumentFuncEntry)
+ if (AllEdges.size() > 1 && InstrumentFuncEntry)
std::iter_swap(std::move(AllEdges.begin()),
std::move(AllEdges.begin() + AllEdges.size() - 1));
}
diff --git a/contrib/libs/llvm12/lib/Transforms/Instrumentation/CGProfile.cpp b/contrib/libs/llvm12/lib/Transforms/Instrumentation/CGProfile.cpp
index 80ad16c631..9acd82c005 100644
--- a/contrib/libs/llvm12/lib/Transforms/Instrumentation/CGProfile.cpp
+++ b/contrib/libs/llvm12/lib/Transforms/Instrumentation/CGProfile.cpp
@@ -53,8 +53,8 @@ static bool runCGProfilePass(
InstrProfSymtab Symtab;
auto UpdateCounts = [&](TargetTransformInfo &TTI, Function *F,
Function *CalledF, uint64_t NewCount) {
- if (!CalledF || !TTI.isLoweredToCall(CalledF) ||
- CalledF->hasDLLImportStorageClass())
+ if (!CalledF || !TTI.isLoweredToCall(CalledF) ||
+ CalledF->hasDLLImportStorageClass())
return;
uint64_t &Count = Counts[std::make_pair(F, CalledF)];
Count = SaturatingAdd(Count, NewCount);
diff --git a/contrib/libs/llvm12/lib/Transforms/Instrumentation/ControlHeightReduction.cpp b/contrib/libs/llvm12/lib/Transforms/Instrumentation/ControlHeightReduction.cpp
index 13e82eee95..927c34180d 100644
--- a/contrib/libs/llvm12/lib/Transforms/Instrumentation/ControlHeightReduction.cpp
+++ b/contrib/libs/llvm12/lib/Transforms/Instrumentation/ControlHeightReduction.cpp
@@ -260,9 +260,9 @@ class CHRScope {
if (TailRegionSet.count(Parent))
return false;
- assert(llvm::any_of(
- RegInfos,
- [&Parent](const RegInfo &RI) { return Parent == RI.R; }) &&
+ assert(llvm::any_of(
+ RegInfos,
+ [&Parent](const RegInfo &RI) { return Parent == RI.R; }) &&
"Must be in head");
return true;
});
@@ -731,7 +731,7 @@ static Instruction* getBranchInsertPoint(RegInfo &RI) {
}
}
for (Instruction &I : *EntryBB) {
- if (EntryBlockSelectSet.contains(&I)) {
+ if (EntryBlockSelectSet.contains(&I)) {
assert(&I == HoistPoint &&
"HoistPoint must be the first one in Selects");
break;
@@ -949,9 +949,9 @@ void CHR::checkScopeHoistable(CHRScope *Scope) {
<< "Dropped select due to unhoistable branch";
});
}
- llvm::erase_if(Selects, [EntryBB](SelectInst *SI) {
- return SI->getParent() == EntryBB;
- });
+ llvm::erase_if(Selects, [EntryBB](SelectInst *SI) {
+ return SI->getParent() == EntryBB;
+ });
Unhoistables.clear();
InsertPoint = Branch;
}
@@ -1247,7 +1247,7 @@ SmallVector<CHRScope *, 8> CHR::splitScope(
SmallVector<CHRScope *, 8> SubSplits = splitScope(
Sub, Split, &SplitConditionValues, SplitInsertPoint, Output,
SplitUnhoistables);
- llvm::append_range(NewSubs, SubSplits);
+ llvm::append_range(NewSubs, SubSplits);
}
Split->Subs = NewSubs;
}
@@ -1304,17 +1304,17 @@ void CHR::classifyBiasedScopes(CHRScope *Scope, CHRScope *OutermostScope) {
for (RegInfo &RI : Scope->RegInfos) {
if (RI.HasBranch) {
Region *R = RI.R;
- if (TrueBiasedRegionsGlobal.contains(R))
+ if (TrueBiasedRegionsGlobal.contains(R))
OutermostScope->TrueBiasedRegions.insert(R);
- else if (FalseBiasedRegionsGlobal.contains(R))
+ else if (FalseBiasedRegionsGlobal.contains(R))
OutermostScope->FalseBiasedRegions.insert(R);
else
llvm_unreachable("Must be biased");
}
for (SelectInst *SI : RI.Selects) {
- if (TrueBiasedSelectsGlobal.contains(SI))
+ if (TrueBiasedSelectsGlobal.contains(SI))
OutermostScope->TrueBiasedSelects.insert(SI);
- else if (FalseBiasedSelectsGlobal.contains(SI))
+ else if (FalseBiasedSelectsGlobal.contains(SI))
OutermostScope->FalseBiasedSelects.insert(SI);
else
llvm_unreachable("Must be biased");
@@ -1397,8 +1397,8 @@ void CHR::setCHRRegions(CHRScope *Scope, CHRScope *OutermostScope) {
DenseSet<Instruction *> HoistStops;
bool IsHoisted = false;
if (RI.HasBranch) {
- assert((OutermostScope->TrueBiasedRegions.contains(R) ||
- OutermostScope->FalseBiasedRegions.contains(R)) &&
+ assert((OutermostScope->TrueBiasedRegions.contains(R) ||
+ OutermostScope->FalseBiasedRegions.contains(R)) &&
"Must be truthy or falsy");
auto *BI = cast<BranchInst>(R->getEntry()->getTerminator());
// Note checkHoistValue fills in HoistStops.
@@ -1410,8 +1410,8 @@ void CHR::setCHRRegions(CHRScope *Scope, CHRScope *OutermostScope) {
IsHoisted = true;
}
for (SelectInst *SI : RI.Selects) {
- assert((OutermostScope->TrueBiasedSelects.contains(SI) ||
- OutermostScope->FalseBiasedSelects.contains(SI)) &&
+ assert((OutermostScope->TrueBiasedSelects.contains(SI) ||
+ OutermostScope->FalseBiasedSelects.contains(SI)) &&
"Must be true or false biased");
// Note checkHoistValue fills in HoistStops.
DenseMap<Instruction *, bool> Visited;
@@ -1607,7 +1607,7 @@ static void insertTrivialPHIs(CHRScope *Scope,
// Insert a trivial phi for I (phi [&I, P0], [&I, P1], ...) at
// ExitBlock. Replace I with the new phi in UI unless UI is another
// phi at ExitBlock.
- PHINode *PN = PHINode::Create(I.getType(), pred_size(ExitBlock), "",
+ PHINode *PN = PHINode::Create(I.getType(), pred_size(ExitBlock), "",
&ExitBlock->front());
for (BasicBlock *Pred : predecessors(ExitBlock)) {
PN->addIncoming(&I, Pred);
diff --git a/contrib/libs/llvm12/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp b/contrib/libs/llvm12/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
index 6112174f3c..1b14b8d569 100644
--- a/contrib/libs/llvm12/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
+++ b/contrib/libs/llvm12/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
@@ -46,7 +46,7 @@
//
//===----------------------------------------------------------------------===//
-#include "llvm/Transforms/Instrumentation/DataFlowSanitizer.h"
+#include "llvm/Transforms/Instrumentation/DataFlowSanitizer.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/DepthFirstIterator.h"
@@ -79,7 +79,7 @@
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/MDBuilder.h"
#include "llvm/IR/Module.h"
-#include "llvm/IR/PassManager.h"
+#include "llvm/IR/PassManager.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/User.h"
#include "llvm/IR/Value.h"
@@ -106,18 +106,18 @@
using namespace llvm;
-// This must be consistent with ShadowWidthBits.
-static const Align kShadowTLSAlignment = Align(2);
-
-// The size of TLS variables. These constants must be kept in sync with the ones
-// in dfsan.cpp.
-static const unsigned kArgTLSSize = 800;
-static const unsigned kRetvalTLSSize = 800;
-
+// This must be consistent with ShadowWidthBits.
+static const Align kShadowTLSAlignment = Align(2);
+
+// The size of TLS variables. These constants must be kept in sync with the ones
+// in dfsan.cpp.
+static const unsigned kArgTLSSize = 800;
+static const unsigned kRetvalTLSSize = 800;
+
// External symbol to be used when generating the shadow address for
// architectures with multiple VMAs. Instead of using a constant integer
// the runtime will set the external mask based on the VMA range.
-const char kDFSanExternShadowPtrMask[] = "__dfsan_shadow_ptr_mask";
+const char kDFSanExternShadowPtrMask[] = "__dfsan_shadow_ptr_mask";
// The -dfsan-preserve-alignment flag controls whether this pass assumes that
// alignment requirements provided by the input IR are correct. For example,
@@ -177,8 +177,8 @@ static cl::opt<bool> ClDebugNonzeroLabels(
//
// If this flag is set to true, the user must provide definitions for the
// following callback functions:
-// void __dfsan_load_callback(dfsan_label Label, void* addr);
-// void __dfsan_store_callback(dfsan_label Label, void* addr);
+// void __dfsan_load_callback(dfsan_label Label, void* addr);
+// void __dfsan_store_callback(dfsan_label Label, void* addr);
// void __dfsan_mem_transfer_callback(dfsan_label *Start, size_t Len);
// void __dfsan_cmp_callback(dfsan_label CombinedLabel);
static cl::opt<bool> ClEventCallbacks(
@@ -186,21 +186,21 @@ static cl::opt<bool> ClEventCallbacks(
cl::desc("Insert calls to __dfsan_*_callback functions on data events."),
cl::Hidden, cl::init(false));
-// Use a distinct bit for each base label, enabling faster unions with less
-// instrumentation. Limits the max number of base labels to 16.
-static cl::opt<bool> ClFast16Labels(
- "dfsan-fast-16-labels",
- cl::desc("Use more efficient instrumentation, limiting the number of "
- "labels to 16."),
- cl::Hidden, cl::init(false));
-
-// Controls whether the pass tracks the control flow of select instructions.
-static cl::opt<bool> ClTrackSelectControlFlow(
- "dfsan-track-select-control-flow",
- cl::desc("Propagate labels from condition values of select instructions "
- "to results."),
- cl::Hidden, cl::init(true));
-
+// Use a distinct bit for each base label, enabling faster unions with less
+// instrumentation. Limits the max number of base labels to 16.
+static cl::opt<bool> ClFast16Labels(
+ "dfsan-fast-16-labels",
+ cl::desc("Use more efficient instrumentation, limiting the number of "
+ "labels to 16."),
+ cl::Hidden, cl::init(false));
+
+// Controls whether the pass tracks the control flow of select instructions.
+static cl::opt<bool> ClTrackSelectControlFlow(
+ "dfsan-track-select-control-flow",
+ cl::desc("Propagate labels from condition values of select instructions "
+ "to results."),
+ cl::Hidden, cl::init(true));
+
static StringRef GetGlobalTypeString(const GlobalValue &G) {
// Types of GlobalVariables are always pointer types.
Type *GType = G.getValueType();
@@ -317,7 +317,7 @@ AttributeList TransformFunctionAttributes(
llvm::makeArrayRef(ArgumentAttributes));
}
-class DataFlowSanitizer {
+class DataFlowSanitizer {
friend struct DFSanFunction;
friend class DFSanVisitor;
@@ -361,12 +361,12 @@ class DataFlowSanitizer {
Module *Mod;
LLVMContext *Ctx;
- Type *Int8Ptr;
- /// The shadow type for all primitive types and vector types.
- IntegerType *PrimitiveShadowTy;
- PointerType *PrimitiveShadowPtrTy;
+ Type *Int8Ptr;
+ /// The shadow type for all primitive types and vector types.
+ IntegerType *PrimitiveShadowTy;
+ PointerType *PrimitiveShadowPtrTy;
IntegerType *IntptrTy;
- ConstantInt *ZeroPrimitiveShadow;
+ ConstantInt *ZeroPrimitiveShadow;
ConstantInt *ShadowPtrMask;
ConstantInt *ShadowPtrMul;
Constant *ArgTLS;
@@ -378,13 +378,13 @@ class DataFlowSanitizer {
FunctionType *DFSanSetLabelFnTy;
FunctionType *DFSanNonzeroLabelFnTy;
FunctionType *DFSanVarargWrapperFnTy;
- FunctionType *DFSanCmpCallbackFnTy;
- FunctionType *DFSanLoadStoreCallbackFnTy;
+ FunctionType *DFSanCmpCallbackFnTy;
+ FunctionType *DFSanLoadStoreCallbackFnTy;
FunctionType *DFSanMemTransferCallbackFnTy;
FunctionCallee DFSanUnionFn;
FunctionCallee DFSanCheckedUnionFn;
FunctionCallee DFSanUnionLoadFn;
- FunctionCallee DFSanUnionLoadFast16LabelsFn;
+ FunctionCallee DFSanUnionLoadFast16LabelsFn;
FunctionCallee DFSanUnimplementedFn;
FunctionCallee DFSanSetLabelFn;
FunctionCallee DFSanNonzeroLabelFn;
@@ -415,43 +415,43 @@ class DataFlowSanitizer {
void initializeCallbackFunctions(Module &M);
void initializeRuntimeFunctions(Module &M);
- bool init(Module &M);
-
- /// Returns whether the pass tracks labels for struct fields and array
- /// indices. Support only fast16 mode in TLS ABI mode.
- bool shouldTrackFieldsAndIndices();
-
- /// Returns a zero constant with the shadow type of OrigTy.
- ///
- /// getZeroShadow({T1,T2,...}) = {getZeroShadow(T1),getZeroShadow(T2,...}
- /// getZeroShadow([n x T]) = [n x getZeroShadow(T)]
- /// getZeroShadow(other type) = i16(0)
- ///
- /// Note that a zero shadow is always i16(0) when shouldTrackFieldsAndIndices
- /// returns false.
- Constant *getZeroShadow(Type *OrigTy);
- /// Returns a zero constant with the shadow type of V's type.
- Constant *getZeroShadow(Value *V);
-
- /// Checks if V is a zero shadow.
- bool isZeroShadow(Value *V);
-
- /// Returns the shadow type of OrigTy.
- ///
- /// getShadowTy({T1,T2,...}) = {getShadowTy(T1),getShadowTy(T2),...}
- /// getShadowTy([n x T]) = [n x getShadowTy(T)]
- /// getShadowTy(other type) = i16
- ///
- /// Note that a shadow type is always i16 when shouldTrackFieldsAndIndices
- /// returns false.
- Type *getShadowTy(Type *OrigTy);
- /// Returns the shadow type of of V's type.
- Type *getShadowTy(Value *V);
-
+ bool init(Module &M);
+
+ /// Returns whether the pass tracks labels for struct fields and array
+ /// indices. Support only fast16 mode in TLS ABI mode.
+ bool shouldTrackFieldsAndIndices();
+
+ /// Returns a zero constant with the shadow type of OrigTy.
+ ///
+ /// getZeroShadow({T1,T2,...}) = {getZeroShadow(T1),getZeroShadow(T2,...}
+ /// getZeroShadow([n x T]) = [n x getZeroShadow(T)]
+ /// getZeroShadow(other type) = i16(0)
+ ///
+ /// Note that a zero shadow is always i16(0) when shouldTrackFieldsAndIndices
+ /// returns false.
+ Constant *getZeroShadow(Type *OrigTy);
+ /// Returns a zero constant with the shadow type of V's type.
+ Constant *getZeroShadow(Value *V);
+
+ /// Checks if V is a zero shadow.
+ bool isZeroShadow(Value *V);
+
+ /// Returns the shadow type of OrigTy.
+ ///
+ /// getShadowTy({T1,T2,...}) = {getShadowTy(T1),getShadowTy(T2),...}
+ /// getShadowTy([n x T]) = [n x getShadowTy(T)]
+ /// getShadowTy(other type) = i16
+ ///
+ /// Note that a shadow type is always i16 when shouldTrackFieldsAndIndices
+ /// returns false.
+ Type *getShadowTy(Type *OrigTy);
+ /// Returns the shadow type of of V's type.
+ Type *getShadowTy(Value *V);
+
public:
- DataFlowSanitizer(const std::vector<std::string> &ABIListFiles);
+ DataFlowSanitizer(const std::vector<std::string> &ABIListFiles);
- bool runImpl(Module &M);
+ bool runImpl(Module &M);
};
struct DFSanFunction {
@@ -468,17 +468,17 @@ struct DFSanFunction {
std::vector<Value *> NonZeroChecks;
bool AvoidNewBlocks;
- struct CachedShadow {
- BasicBlock *Block; // The block where Shadow is defined.
+ struct CachedShadow {
+ BasicBlock *Block; // The block where Shadow is defined.
Value *Shadow;
};
- /// Maps a value to its latest shadow value in terms of domination tree.
- DenseMap<std::pair<Value *, Value *>, CachedShadow> CachedShadows;
- /// Maps a value to its latest collapsed shadow value it was converted to in
- /// terms of domination tree. When ClDebugNonzeroLabels is on, this cache is
- /// used at a post process where CFG blocks are split. So it does not cache
- /// BasicBlock like CachedShadows, but uses domination between values.
- DenseMap<Value *, Value *> CachedCollapsedShadows;
+ /// Maps a value to its latest shadow value in terms of domination tree.
+ DenseMap<std::pair<Value *, Value *>, CachedShadow> CachedShadows;
+ /// Maps a value to its latest collapsed shadow value it was converted to in
+ /// terms of domination tree. When ClDebugNonzeroLabels is on, this cache is
+ /// used at a post process where CFG blocks are split. So it does not cache
+ /// BasicBlock like CachedShadows, but uses domination between values.
+ DenseMap<Value *, Value *> CachedCollapsedShadows;
DenseMap<Value *, std::set<Value *>> ShadowElements;
DFSanFunction(DataFlowSanitizer &DFS, Function *F, bool IsNativeABI)
@@ -489,56 +489,56 @@ struct DFSanFunction {
AvoidNewBlocks = F->size() > 1000;
}
- /// Computes the shadow address for a given function argument.
- ///
- /// Shadow = ArgTLS+ArgOffset.
- Value *getArgTLS(Type *T, unsigned ArgOffset, IRBuilder<> &IRB);
-
- /// Computes the shadow address for a retval.
- Value *getRetvalTLS(Type *T, IRBuilder<> &IRB);
-
+ /// Computes the shadow address for a given function argument.
+ ///
+ /// Shadow = ArgTLS+ArgOffset.
+ Value *getArgTLS(Type *T, unsigned ArgOffset, IRBuilder<> &IRB);
+
+ /// Computes the shadow address for a retval.
+ Value *getRetvalTLS(Type *T, IRBuilder<> &IRB);
+
Value *getShadow(Value *V);
void setShadow(Instruction *I, Value *Shadow);
- /// Generates IR to compute the union of the two given shadows, inserting it
- /// before Pos. The combined value is with primitive type.
+ /// Generates IR to compute the union of the two given shadows, inserting it
+ /// before Pos. The combined value is with primitive type.
Value *combineShadows(Value *V1, Value *V2, Instruction *Pos);
- /// Combines the shadow values of V1 and V2, then converts the combined value
- /// with primitive type into a shadow value with the original type T.
- Value *combineShadowsThenConvert(Type *T, Value *V1, Value *V2,
- Instruction *Pos);
+ /// Combines the shadow values of V1 and V2, then converts the combined value
+ /// with primitive type into a shadow value with the original type T.
+ Value *combineShadowsThenConvert(Type *T, Value *V1, Value *V2,
+ Instruction *Pos);
Value *combineOperandShadows(Instruction *Inst);
Value *loadShadow(Value *ShadowAddr, uint64_t Size, uint64_t Align,
Instruction *Pos);
- void storePrimitiveShadow(Value *Addr, uint64_t Size, Align Alignment,
- Value *PrimitiveShadow, Instruction *Pos);
- /// Applies PrimitiveShadow to all primitive subtypes of T, returning
- /// the expanded shadow value.
- ///
- /// EFP({T1,T2, ...}, PS) = {EFP(T1,PS),EFP(T2,PS),...}
- /// EFP([n x T], PS) = [n x EFP(T,PS)]
- /// EFP(other types, PS) = PS
- Value *expandFromPrimitiveShadow(Type *T, Value *PrimitiveShadow,
- Instruction *Pos);
- /// Collapses Shadow into a single primitive shadow value, unioning all
- /// primitive shadow values in the process. Returns the final primitive
- /// shadow value.
- ///
- /// CTP({V1,V2, ...}) = UNION(CFP(V1,PS),CFP(V2,PS),...)
- /// CTP([V1,V2,...]) = UNION(CFP(V1,PS),CFP(V2,PS),...)
- /// CTP(other types, PS) = PS
- Value *collapseToPrimitiveShadow(Value *Shadow, Instruction *Pos);
-
-private:
- /// Collapses the shadow with aggregate type into a single primitive shadow
- /// value.
- template <class AggregateType>
- Value *collapseAggregateShadow(AggregateType *AT, Value *Shadow,
- IRBuilder<> &IRB);
-
- Value *collapseToPrimitiveShadow(Value *Shadow, IRBuilder<> &IRB);
-
- /// Returns the shadow value of an argument A.
- Value *getShadowForTLSArgument(Argument *A);
+ void storePrimitiveShadow(Value *Addr, uint64_t Size, Align Alignment,
+ Value *PrimitiveShadow, Instruction *Pos);
+ /// Applies PrimitiveShadow to all primitive subtypes of T, returning
+ /// the expanded shadow value.
+ ///
+ /// EFP({T1,T2, ...}, PS) = {EFP(T1,PS),EFP(T2,PS),...}
+ /// EFP([n x T], PS) = [n x EFP(T,PS)]
+ /// EFP(other types, PS) = PS
+ Value *expandFromPrimitiveShadow(Type *T, Value *PrimitiveShadow,
+ Instruction *Pos);
+ /// Collapses Shadow into a single primitive shadow value, unioning all
+ /// primitive shadow values in the process. Returns the final primitive
+ /// shadow value.
+ ///
+ /// CTP({V1,V2, ...}) = UNION(CFP(V1,PS),CFP(V2,PS),...)
+ /// CTP([V1,V2,...]) = UNION(CFP(V1,PS),CFP(V2,PS),...)
+ /// CTP(other types, PS) = PS
+ Value *collapseToPrimitiveShadow(Value *Shadow, Instruction *Pos);
+
+private:
+ /// Collapses the shadow with aggregate type into a single primitive shadow
+ /// value.
+ template <class AggregateType>
+ Value *collapseAggregateShadow(AggregateType *AT, Value *Shadow,
+ IRBuilder<> &IRB);
+
+ Value *collapseToPrimitiveShadow(Value *Shadow, IRBuilder<> &IRB);
+
+ /// Returns the shadow value of an argument A.
+ Value *getShadowForTLSArgument(Argument *A);
};
class DFSanVisitor : public InstVisitor<DFSanVisitor> {
@@ -579,9 +579,9 @@ public:
} // end anonymous namespace
DataFlowSanitizer::DataFlowSanitizer(
- const std::vector<std::string> &ABIListFiles) {
+ const std::vector<std::string> &ABIListFiles) {
std::vector<std::string> AllABIListFiles(std::move(ABIListFiles));
- llvm::append_range(AllABIListFiles, ClABIListFiles);
+ llvm::append_range(AllABIListFiles, ClABIListFiles);
// FIXME: should we propagate vfs::FileSystem to this constructor?
ABIList.set(
SpecialCaseList::createOrDie(AllABIListFiles, *vfs::getRealFileSystem()));
@@ -589,12 +589,12 @@ DataFlowSanitizer::DataFlowSanitizer(
FunctionType *DataFlowSanitizer::getArgsFunctionType(FunctionType *T) {
SmallVector<Type *, 4> ArgTypes(T->param_begin(), T->param_end());
- ArgTypes.append(T->getNumParams(), PrimitiveShadowTy);
+ ArgTypes.append(T->getNumParams(), PrimitiveShadowTy);
if (T->isVarArg())
- ArgTypes.push_back(PrimitiveShadowPtrTy);
+ ArgTypes.push_back(PrimitiveShadowPtrTy);
Type *RetType = T->getReturnType();
if (!RetType->isVoidTy())
- RetType = StructType::get(RetType, PrimitiveShadowTy);
+ RetType = StructType::get(RetType, PrimitiveShadowTy);
return FunctionType::get(RetType, ArgTypes, T->isVarArg());
}
@@ -603,10 +603,10 @@ FunctionType *DataFlowSanitizer::getTrampolineFunctionType(FunctionType *T) {
SmallVector<Type *, 4> ArgTypes;
ArgTypes.push_back(T->getPointerTo());
ArgTypes.append(T->param_begin(), T->param_end());
- ArgTypes.append(T->getNumParams(), PrimitiveShadowTy);
+ ArgTypes.append(T->getNumParams(), PrimitiveShadowTy);
Type *RetType = T->getReturnType();
if (!RetType->isVoidTy())
- ArgTypes.push_back(PrimitiveShadowPtrTy);
+ ArgTypes.push_back(PrimitiveShadowPtrTy);
return FunctionType::get(T->getReturnType(), ArgTypes, false);
}
@@ -632,174 +632,174 @@ TransformedFunction DataFlowSanitizer::getCustomFunctionType(FunctionType *T) {
}
}
for (unsigned i = 0, e = T->getNumParams(); i != e; ++i)
- ArgTypes.push_back(PrimitiveShadowTy);
+ ArgTypes.push_back(PrimitiveShadowTy);
if (T->isVarArg())
- ArgTypes.push_back(PrimitiveShadowPtrTy);
+ ArgTypes.push_back(PrimitiveShadowPtrTy);
Type *RetType = T->getReturnType();
if (!RetType->isVoidTy())
- ArgTypes.push_back(PrimitiveShadowPtrTy);
+ ArgTypes.push_back(PrimitiveShadowPtrTy);
return TransformedFunction(
T, FunctionType::get(T->getReturnType(), ArgTypes, T->isVarArg()),
ArgumentIndexMapping);
}
-bool DataFlowSanitizer::isZeroShadow(Value *V) {
- if (!shouldTrackFieldsAndIndices())
- return ZeroPrimitiveShadow == V;
-
- Type *T = V->getType();
- if (!isa<ArrayType>(T) && !isa<StructType>(T)) {
- if (const ConstantInt *CI = dyn_cast<ConstantInt>(V))
- return CI->isZero();
- return false;
- }
-
- return isa<ConstantAggregateZero>(V);
-}
-
-bool DataFlowSanitizer::shouldTrackFieldsAndIndices() {
- return getInstrumentedABI() == DataFlowSanitizer::IA_TLS && ClFast16Labels;
-}
-
-Constant *DataFlowSanitizer::getZeroShadow(Type *OrigTy) {
- if (!shouldTrackFieldsAndIndices())
- return ZeroPrimitiveShadow;
-
- if (!isa<ArrayType>(OrigTy) && !isa<StructType>(OrigTy))
- return ZeroPrimitiveShadow;
- Type *ShadowTy = getShadowTy(OrigTy);
- return ConstantAggregateZero::get(ShadowTy);
-}
-
-Constant *DataFlowSanitizer::getZeroShadow(Value *V) {
- return getZeroShadow(V->getType());
-}
-
-static Value *expandFromPrimitiveShadowRecursive(
- Value *Shadow, SmallVector<unsigned, 4> &Indices, Type *SubShadowTy,
- Value *PrimitiveShadow, IRBuilder<> &IRB) {
- if (!isa<ArrayType>(SubShadowTy) && !isa<StructType>(SubShadowTy))
- return IRB.CreateInsertValue(Shadow, PrimitiveShadow, Indices);
-
- if (ArrayType *AT = dyn_cast<ArrayType>(SubShadowTy)) {
- for (unsigned Idx = 0; Idx < AT->getNumElements(); Idx++) {
- Indices.push_back(Idx);
- Shadow = expandFromPrimitiveShadowRecursive(
- Shadow, Indices, AT->getElementType(), PrimitiveShadow, IRB);
- Indices.pop_back();
- }
- return Shadow;
- }
-
- if (StructType *ST = dyn_cast<StructType>(SubShadowTy)) {
- for (unsigned Idx = 0; Idx < ST->getNumElements(); Idx++) {
- Indices.push_back(Idx);
- Shadow = expandFromPrimitiveShadowRecursive(
- Shadow, Indices, ST->getElementType(Idx), PrimitiveShadow, IRB);
- Indices.pop_back();
- }
- return Shadow;
- }
- llvm_unreachable("Unexpected shadow type");
-}
-
-Value *DFSanFunction::expandFromPrimitiveShadow(Type *T, Value *PrimitiveShadow,
- Instruction *Pos) {
- Type *ShadowTy = DFS.getShadowTy(T);
-
- if (!isa<ArrayType>(ShadowTy) && !isa<StructType>(ShadowTy))
- return PrimitiveShadow;
-
- if (DFS.isZeroShadow(PrimitiveShadow))
- return DFS.getZeroShadow(ShadowTy);
-
- IRBuilder<> IRB(Pos);
- SmallVector<unsigned, 4> Indices;
- Value *Shadow = UndefValue::get(ShadowTy);
- Shadow = expandFromPrimitiveShadowRecursive(Shadow, Indices, ShadowTy,
- PrimitiveShadow, IRB);
-
- // Caches the primitive shadow value that built the shadow value.
- CachedCollapsedShadows[Shadow] = PrimitiveShadow;
- return Shadow;
-}
-
-template <class AggregateType>
-Value *DFSanFunction::collapseAggregateShadow(AggregateType *AT, Value *Shadow,
- IRBuilder<> &IRB) {
- if (!AT->getNumElements())
- return DFS.ZeroPrimitiveShadow;
-
- Value *FirstItem = IRB.CreateExtractValue(Shadow, 0);
- Value *Aggregator = collapseToPrimitiveShadow(FirstItem, IRB);
-
- for (unsigned Idx = 1; Idx < AT->getNumElements(); Idx++) {
- Value *ShadowItem = IRB.CreateExtractValue(Shadow, Idx);
- Value *ShadowInner = collapseToPrimitiveShadow(ShadowItem, IRB);
- Aggregator = IRB.CreateOr(Aggregator, ShadowInner);
- }
- return Aggregator;
-}
-
-Value *DFSanFunction::collapseToPrimitiveShadow(Value *Shadow,
- IRBuilder<> &IRB) {
- Type *ShadowTy = Shadow->getType();
- if (!isa<ArrayType>(ShadowTy) && !isa<StructType>(ShadowTy))
- return Shadow;
- if (ArrayType *AT = dyn_cast<ArrayType>(ShadowTy))
- return collapseAggregateShadow<>(AT, Shadow, IRB);
- if (StructType *ST = dyn_cast<StructType>(ShadowTy))
- return collapseAggregateShadow<>(ST, Shadow, IRB);
- llvm_unreachable("Unexpected shadow type");
-}
-
-Value *DFSanFunction::collapseToPrimitiveShadow(Value *Shadow,
- Instruction *Pos) {
- Type *ShadowTy = Shadow->getType();
- if (!isa<ArrayType>(ShadowTy) && !isa<StructType>(ShadowTy))
- return Shadow;
-
- assert(DFS.shouldTrackFieldsAndIndices());
-
- // Checks if the cached collapsed shadow value dominates Pos.
- Value *&CS = CachedCollapsedShadows[Shadow];
- if (CS && DT.dominates(CS, Pos))
- return CS;
-
- IRBuilder<> IRB(Pos);
- Value *PrimitiveShadow = collapseToPrimitiveShadow(Shadow, IRB);
- // Caches the converted primitive shadow value.
- CS = PrimitiveShadow;
- return PrimitiveShadow;
-}
-
-Type *DataFlowSanitizer::getShadowTy(Type *OrigTy) {
- if (!shouldTrackFieldsAndIndices())
- return PrimitiveShadowTy;
-
- if (!OrigTy->isSized())
- return PrimitiveShadowTy;
- if (isa<IntegerType>(OrigTy))
- return PrimitiveShadowTy;
- if (isa<VectorType>(OrigTy))
- return PrimitiveShadowTy;
- if (ArrayType *AT = dyn_cast<ArrayType>(OrigTy))
- return ArrayType::get(getShadowTy(AT->getElementType()),
- AT->getNumElements());
- if (StructType *ST = dyn_cast<StructType>(OrigTy)) {
- SmallVector<Type *, 4> Elements;
- for (unsigned I = 0, N = ST->getNumElements(); I < N; ++I)
- Elements.push_back(getShadowTy(ST->getElementType(I)));
- return StructType::get(*Ctx, Elements);
- }
- return PrimitiveShadowTy;
-}
-
-Type *DataFlowSanitizer::getShadowTy(Value *V) {
- return getShadowTy(V->getType());
-}
-
-bool DataFlowSanitizer::init(Module &M) {
+bool DataFlowSanitizer::isZeroShadow(Value *V) {
+ if (!shouldTrackFieldsAndIndices())
+ return ZeroPrimitiveShadow == V;
+
+ Type *T = V->getType();
+ if (!isa<ArrayType>(T) && !isa<StructType>(T)) {
+ if (const ConstantInt *CI = dyn_cast<ConstantInt>(V))
+ return CI->isZero();
+ return false;
+ }
+
+ return isa<ConstantAggregateZero>(V);
+}
+
+bool DataFlowSanitizer::shouldTrackFieldsAndIndices() {
+ return getInstrumentedABI() == DataFlowSanitizer::IA_TLS && ClFast16Labels;
+}
+
+Constant *DataFlowSanitizer::getZeroShadow(Type *OrigTy) {
+ if (!shouldTrackFieldsAndIndices())
+ return ZeroPrimitiveShadow;
+
+ if (!isa<ArrayType>(OrigTy) && !isa<StructType>(OrigTy))
+ return ZeroPrimitiveShadow;
+ Type *ShadowTy = getShadowTy(OrigTy);
+ return ConstantAggregateZero::get(ShadowTy);
+}
+
+Constant *DataFlowSanitizer::getZeroShadow(Value *V) {
+ return getZeroShadow(V->getType());
+}
+
+static Value *expandFromPrimitiveShadowRecursive(
+ Value *Shadow, SmallVector<unsigned, 4> &Indices, Type *SubShadowTy,
+ Value *PrimitiveShadow, IRBuilder<> &IRB) {
+ if (!isa<ArrayType>(SubShadowTy) && !isa<StructType>(SubShadowTy))
+ return IRB.CreateInsertValue(Shadow, PrimitiveShadow, Indices);
+
+ if (ArrayType *AT = dyn_cast<ArrayType>(SubShadowTy)) {
+ for (unsigned Idx = 0; Idx < AT->getNumElements(); Idx++) {
+ Indices.push_back(Idx);
+ Shadow = expandFromPrimitiveShadowRecursive(
+ Shadow, Indices, AT->getElementType(), PrimitiveShadow, IRB);
+ Indices.pop_back();
+ }
+ return Shadow;
+ }
+
+ if (StructType *ST = dyn_cast<StructType>(SubShadowTy)) {
+ for (unsigned Idx = 0; Idx < ST->getNumElements(); Idx++) {
+ Indices.push_back(Idx);
+ Shadow = expandFromPrimitiveShadowRecursive(
+ Shadow, Indices, ST->getElementType(Idx), PrimitiveShadow, IRB);
+ Indices.pop_back();
+ }
+ return Shadow;
+ }
+ llvm_unreachable("Unexpected shadow type");
+}
+
+Value *DFSanFunction::expandFromPrimitiveShadow(Type *T, Value *PrimitiveShadow,
+ Instruction *Pos) {
+ Type *ShadowTy = DFS.getShadowTy(T);
+
+ if (!isa<ArrayType>(ShadowTy) && !isa<StructType>(ShadowTy))
+ return PrimitiveShadow;
+
+ if (DFS.isZeroShadow(PrimitiveShadow))
+ return DFS.getZeroShadow(ShadowTy);
+
+ IRBuilder<> IRB(Pos);
+ SmallVector<unsigned, 4> Indices;
+ Value *Shadow = UndefValue::get(ShadowTy);
+ Shadow = expandFromPrimitiveShadowRecursive(Shadow, Indices, ShadowTy,
+ PrimitiveShadow, IRB);
+
+ // Caches the primitive shadow value that built the shadow value.
+ CachedCollapsedShadows[Shadow] = PrimitiveShadow;
+ return Shadow;
+}
+
+template <class AggregateType>
+Value *DFSanFunction::collapseAggregateShadow(AggregateType *AT, Value *Shadow,
+ IRBuilder<> &IRB) {
+ if (!AT->getNumElements())
+ return DFS.ZeroPrimitiveShadow;
+
+ Value *FirstItem = IRB.CreateExtractValue(Shadow, 0);
+ Value *Aggregator = collapseToPrimitiveShadow(FirstItem, IRB);
+
+ for (unsigned Idx = 1; Idx < AT->getNumElements(); Idx++) {
+ Value *ShadowItem = IRB.CreateExtractValue(Shadow, Idx);
+ Value *ShadowInner = collapseToPrimitiveShadow(ShadowItem, IRB);
+ Aggregator = IRB.CreateOr(Aggregator, ShadowInner);
+ }
+ return Aggregator;
+}
+
+Value *DFSanFunction::collapseToPrimitiveShadow(Value *Shadow,
+ IRBuilder<> &IRB) {
+ Type *ShadowTy = Shadow->getType();
+ if (!isa<ArrayType>(ShadowTy) && !isa<StructType>(ShadowTy))
+ return Shadow;
+ if (ArrayType *AT = dyn_cast<ArrayType>(ShadowTy))
+ return collapseAggregateShadow<>(AT, Shadow, IRB);
+ if (StructType *ST = dyn_cast<StructType>(ShadowTy))
+ return collapseAggregateShadow<>(ST, Shadow, IRB);
+ llvm_unreachable("Unexpected shadow type");
+}
+
+Value *DFSanFunction::collapseToPrimitiveShadow(Value *Shadow,
+ Instruction *Pos) {
+ Type *ShadowTy = Shadow->getType();
+ if (!isa<ArrayType>(ShadowTy) && !isa<StructType>(ShadowTy))
+ return Shadow;
+
+ assert(DFS.shouldTrackFieldsAndIndices());
+
+ // Checks if the cached collapsed shadow value dominates Pos.
+ Value *&CS = CachedCollapsedShadows[Shadow];
+ if (CS && DT.dominates(CS, Pos))
+ return CS;
+
+ IRBuilder<> IRB(Pos);
+ Value *PrimitiveShadow = collapseToPrimitiveShadow(Shadow, IRB);
+ // Caches the converted primitive shadow value.
+ CS = PrimitiveShadow;
+ return PrimitiveShadow;
+}
+
+Type *DataFlowSanitizer::getShadowTy(Type *OrigTy) {
+ if (!shouldTrackFieldsAndIndices())
+ return PrimitiveShadowTy;
+
+ if (!OrigTy->isSized())
+ return PrimitiveShadowTy;
+ if (isa<IntegerType>(OrigTy))
+ return PrimitiveShadowTy;
+ if (isa<VectorType>(OrigTy))
+ return PrimitiveShadowTy;
+ if (ArrayType *AT = dyn_cast<ArrayType>(OrigTy))
+ return ArrayType::get(getShadowTy(AT->getElementType()),
+ AT->getNumElements());
+ if (StructType *ST = dyn_cast<StructType>(OrigTy)) {
+ SmallVector<Type *, 4> Elements;
+ for (unsigned I = 0, N = ST->getNumElements(); I < N; ++I)
+ Elements.push_back(getShadowTy(ST->getElementType(I)));
+ return StructType::get(*Ctx, Elements);
+ }
+ return PrimitiveShadowTy;
+}
+
+Type *DataFlowSanitizer::getShadowTy(Value *V) {
+ return getShadowTy(V->getType());
+}
+
+bool DataFlowSanitizer::init(Module &M) {
Triple TargetTriple(M.getTargetTriple());
bool IsX86_64 = TargetTriple.getArch() == Triple::x86_64;
bool IsMIPS64 = TargetTriple.isMIPS64();
@@ -810,11 +810,11 @@ bool DataFlowSanitizer::init(Module &M) {
Mod = &M;
Ctx = &M.getContext();
- Int8Ptr = Type::getInt8PtrTy(*Ctx);
- PrimitiveShadowTy = IntegerType::get(*Ctx, ShadowWidthBits);
- PrimitiveShadowPtrTy = PointerType::getUnqual(PrimitiveShadowTy);
+ Int8Ptr = Type::getInt8PtrTy(*Ctx);
+ PrimitiveShadowTy = IntegerType::get(*Ctx, ShadowWidthBits);
+ PrimitiveShadowPtrTy = PointerType::getUnqual(PrimitiveShadowTy);
IntptrTy = DL.getIntPtrType(*Ctx);
- ZeroPrimitiveShadow = ConstantInt::getSigned(PrimitiveShadowTy, 0);
+ ZeroPrimitiveShadow = ConstantInt::getSigned(PrimitiveShadowTy, 0);
ShadowPtrMul = ConstantInt::getSigned(IntptrTy, ShadowWidthBytes);
if (IsX86_64)
ShadowPtrMask = ConstantInt::getSigned(IntptrTy, ~0x700000000000LL);
@@ -826,30 +826,30 @@ bool DataFlowSanitizer::init(Module &M) {
else
report_fatal_error("unsupported triple");
- Type *DFSanUnionArgs[2] = {PrimitiveShadowTy, PrimitiveShadowTy};
+ Type *DFSanUnionArgs[2] = {PrimitiveShadowTy, PrimitiveShadowTy};
DFSanUnionFnTy =
- FunctionType::get(PrimitiveShadowTy, DFSanUnionArgs, /*isVarArg=*/false);
- Type *DFSanUnionLoadArgs[2] = {PrimitiveShadowPtrTy, IntptrTy};
- DFSanUnionLoadFnTy = FunctionType::get(PrimitiveShadowTy, DFSanUnionLoadArgs,
- /*isVarArg=*/false);
+ FunctionType::get(PrimitiveShadowTy, DFSanUnionArgs, /*isVarArg=*/false);
+ Type *DFSanUnionLoadArgs[2] = {PrimitiveShadowPtrTy, IntptrTy};
+ DFSanUnionLoadFnTy = FunctionType::get(PrimitiveShadowTy, DFSanUnionLoadArgs,
+ /*isVarArg=*/false);
DFSanUnimplementedFnTy = FunctionType::get(
Type::getVoidTy(*Ctx), Type::getInt8PtrTy(*Ctx), /*isVarArg=*/false);
- Type *DFSanSetLabelArgs[3] = {PrimitiveShadowTy, Type::getInt8PtrTy(*Ctx),
- IntptrTy};
+ Type *DFSanSetLabelArgs[3] = {PrimitiveShadowTy, Type::getInt8PtrTy(*Ctx),
+ IntptrTy};
DFSanSetLabelFnTy = FunctionType::get(Type::getVoidTy(*Ctx),
DFSanSetLabelArgs, /*isVarArg=*/false);
- DFSanNonzeroLabelFnTy =
- FunctionType::get(Type::getVoidTy(*Ctx), None, /*isVarArg=*/false);
+ DFSanNonzeroLabelFnTy =
+ FunctionType::get(Type::getVoidTy(*Ctx), None, /*isVarArg=*/false);
DFSanVarargWrapperFnTy = FunctionType::get(
Type::getVoidTy(*Ctx), Type::getInt8PtrTy(*Ctx), /*isVarArg=*/false);
- DFSanCmpCallbackFnTy =
- FunctionType::get(Type::getVoidTy(*Ctx), PrimitiveShadowTy,
- /*isVarArg=*/false);
- Type *DFSanLoadStoreCallbackArgs[2] = {PrimitiveShadowTy, Int8Ptr};
- DFSanLoadStoreCallbackFnTy =
- FunctionType::get(Type::getVoidTy(*Ctx), DFSanLoadStoreCallbackArgs,
- /*isVarArg=*/false);
- Type *DFSanMemTransferCallbackArgs[2] = {PrimitiveShadowPtrTy, IntptrTy};
+ DFSanCmpCallbackFnTy =
+ FunctionType::get(Type::getVoidTy(*Ctx), PrimitiveShadowTy,
+ /*isVarArg=*/false);
+ Type *DFSanLoadStoreCallbackArgs[2] = {PrimitiveShadowTy, Int8Ptr};
+ DFSanLoadStoreCallbackFnTy =
+ FunctionType::get(Type::getVoidTy(*Ctx), DFSanLoadStoreCallbackArgs,
+ /*isVarArg=*/false);
+ Type *DFSanMemTransferCallbackArgs[2] = {PrimitiveShadowPtrTy, IntptrTy};
DFSanMemTransferCallbackFnTy =
FunctionType::get(Type::getVoidTy(*Ctx), DFSanMemTransferCallbackArgs,
/*isVarArg=*/false);
@@ -954,21 +954,21 @@ Constant *DataFlowSanitizer::getOrBuildTrampolineFunction(FunctionType *FT,
else
RI = ReturnInst::Create(*Ctx, CI, BB);
- // F is called by a wrapped custom function with primitive shadows. So
- // its arguments and return value need conversion.
+ // F is called by a wrapped custom function with primitive shadows. So
+ // its arguments and return value need conversion.
DFSanFunction DFSF(*this, F, /*IsNativeABI=*/true);
Function::arg_iterator ValAI = F->arg_begin(), ShadowAI = AI; ++ValAI;
- for (unsigned N = FT->getNumParams(); N != 0; ++ValAI, ++ShadowAI, --N) {
- Value *Shadow =
- DFSF.expandFromPrimitiveShadow(ValAI->getType(), &*ShadowAI, CI);
- DFSF.ValShadowMap[&*ValAI] = Shadow;
- }
+ for (unsigned N = FT->getNumParams(); N != 0; ++ValAI, ++ShadowAI, --N) {
+ Value *Shadow =
+ DFSF.expandFromPrimitiveShadow(ValAI->getType(), &*ShadowAI, CI);
+ DFSF.ValShadowMap[&*ValAI] = Shadow;
+ }
DFSanVisitor(DFSF).visitCallInst(*CI);
- if (!FT->getReturnType()->isVoidTy()) {
- Value *PrimitiveShadow = DFSF.collapseToPrimitiveShadow(
- DFSF.getShadow(RI->getReturnValue()), RI);
- new StoreInst(PrimitiveShadow, &*std::prev(F->arg_end()), RI);
- }
+ if (!FT->getReturnType()->isVoidTy()) {
+ Value *PrimitiveShadow = DFSF.collapseToPrimitiveShadow(
+ DFSF.getShadow(RI->getReturnValue()), RI);
+ new StoreInst(PrimitiveShadow, &*std::prev(F->arg_end()), RI);
+ }
}
return cast<Constant>(C.getCallee());
@@ -1013,17 +1013,17 @@ void DataFlowSanitizer::initializeRuntimeFunctions(Module &M) {
DFSanUnionLoadFn =
Mod->getOrInsertFunction("__dfsan_union_load", DFSanUnionLoadFnTy, AL);
}
- {
- AttributeList AL;
- AL = AL.addAttribute(M.getContext(), AttributeList::FunctionIndex,
- Attribute::NoUnwind);
- AL = AL.addAttribute(M.getContext(), AttributeList::FunctionIndex,
- Attribute::ReadOnly);
- AL = AL.addAttribute(M.getContext(), AttributeList::ReturnIndex,
- Attribute::ZExt);
- DFSanUnionLoadFast16LabelsFn = Mod->getOrInsertFunction(
- "__dfsan_union_load_fast16labels", DFSanUnionLoadFnTy, AL);
- }
+ {
+ AttributeList AL;
+ AL = AL.addAttribute(M.getContext(), AttributeList::FunctionIndex,
+ Attribute::NoUnwind);
+ AL = AL.addAttribute(M.getContext(), AttributeList::FunctionIndex,
+ Attribute::ReadOnly);
+ AL = AL.addAttribute(M.getContext(), AttributeList::ReturnIndex,
+ Attribute::ZExt);
+ DFSanUnionLoadFast16LabelsFn = Mod->getOrInsertFunction(
+ "__dfsan_union_load_fast16labels", DFSanUnionLoadFnTy, AL);
+ }
DFSanUnimplementedFn =
Mod->getOrInsertFunction("__dfsan_unimplemented", DFSanUnimplementedFnTy);
{
@@ -1041,18 +1041,18 @@ void DataFlowSanitizer::initializeRuntimeFunctions(Module &M) {
// Initializes event callback functions and declare them in the module
void DataFlowSanitizer::initializeCallbackFunctions(Module &M) {
DFSanLoadCallbackFn = Mod->getOrInsertFunction("__dfsan_load_callback",
- DFSanLoadStoreCallbackFnTy);
- DFSanStoreCallbackFn = Mod->getOrInsertFunction("__dfsan_store_callback",
- DFSanLoadStoreCallbackFnTy);
+ DFSanLoadStoreCallbackFnTy);
+ DFSanStoreCallbackFn = Mod->getOrInsertFunction("__dfsan_store_callback",
+ DFSanLoadStoreCallbackFnTy);
DFSanMemTransferCallbackFn = Mod->getOrInsertFunction(
"__dfsan_mem_transfer_callback", DFSanMemTransferCallbackFnTy);
- DFSanCmpCallbackFn =
- Mod->getOrInsertFunction("__dfsan_cmp_callback", DFSanCmpCallbackFnTy);
+ DFSanCmpCallbackFn =
+ Mod->getOrInsertFunction("__dfsan_cmp_callback", DFSanCmpCallbackFnTy);
}
-bool DataFlowSanitizer::runImpl(Module &M) {
- init(M);
-
+bool DataFlowSanitizer::runImpl(Module &M) {
+ init(M);
+
if (ABIList.isIn(M, "skip"))
return false;
@@ -1061,18 +1061,18 @@ bool DataFlowSanitizer::runImpl(Module &M) {
bool Changed = false;
- Type *ArgTLSTy = ArrayType::get(Type::getInt64Ty(*Ctx), kArgTLSSize / 8);
- ArgTLS = Mod->getOrInsertGlobal("__dfsan_arg_tls", ArgTLSTy);
- if (GlobalVariable *G = dyn_cast<GlobalVariable>(ArgTLS)) {
- Changed |= G->getThreadLocalMode() != GlobalVariable::InitialExecTLSModel;
- G->setThreadLocalMode(GlobalVariable::InitialExecTLSModel);
+ Type *ArgTLSTy = ArrayType::get(Type::getInt64Ty(*Ctx), kArgTLSSize / 8);
+ ArgTLS = Mod->getOrInsertGlobal("__dfsan_arg_tls", ArgTLSTy);
+ if (GlobalVariable *G = dyn_cast<GlobalVariable>(ArgTLS)) {
+ Changed |= G->getThreadLocalMode() != GlobalVariable::InitialExecTLSModel;
+ G->setThreadLocalMode(GlobalVariable::InitialExecTLSModel);
}
- Type *RetvalTLSTy =
- ArrayType::get(Type::getInt64Ty(*Ctx), kRetvalTLSSize / 8);
- RetvalTLS = Mod->getOrInsertGlobal("__dfsan_retval_tls", RetvalTLSTy);
- if (GlobalVariable *G = dyn_cast<GlobalVariable>(RetvalTLS)) {
- Changed |= G->getThreadLocalMode() != GlobalVariable::InitialExecTLSModel;
- G->setThreadLocalMode(GlobalVariable::InitialExecTLSModel);
+ Type *RetvalTLSTy =
+ ArrayType::get(Type::getInt64Ty(*Ctx), kRetvalTLSSize / 8);
+ RetvalTLS = Mod->getOrInsertGlobal("__dfsan_retval_tls", RetvalTLSTy);
+ if (GlobalVariable *G = dyn_cast<GlobalVariable>(RetvalTLS)) {
+ Changed |= G->getThreadLocalMode() != GlobalVariable::InitialExecTLSModel;
+ G->setThreadLocalMode(GlobalVariable::InitialExecTLSModel);
}
ExternalShadowMask =
@@ -1088,7 +1088,7 @@ bool DataFlowSanitizer::runImpl(Module &M) {
&i != DFSanUnionFn.getCallee()->stripPointerCasts() &&
&i != DFSanCheckedUnionFn.getCallee()->stripPointerCasts() &&
&i != DFSanUnionLoadFn.getCallee()->stripPointerCasts() &&
- &i != DFSanUnionLoadFast16LabelsFn.getCallee()->stripPointerCasts() &&
+ &i != DFSanUnionLoadFast16LabelsFn.getCallee()->stripPointerCasts() &&
&i != DFSanUnimplementedFn.getCallee()->stripPointerCasts() &&
&i != DFSanSetLabelFn.getCallee()->stripPointerCasts() &&
&i != DFSanNonzeroLabelFn.getCallee()->stripPointerCasts() &&
@@ -1288,9 +1288,9 @@ bool DataFlowSanitizer::runImpl(Module &M) {
while (isa<PHINode>(Pos) || isa<AllocaInst>(Pos))
Pos = Pos->getNextNode();
IRBuilder<> IRB(Pos);
- Value *PrimitiveShadow = DFSF.collapseToPrimitiveShadow(V, Pos);
- Value *Ne =
- IRB.CreateICmpNE(PrimitiveShadow, DFSF.DFS.ZeroPrimitiveShadow);
+ Value *PrimitiveShadow = DFSF.collapseToPrimitiveShadow(V, Pos);
+ Value *Ne =
+ IRB.CreateICmpNE(PrimitiveShadow, DFSF.DFS.ZeroPrimitiveShadow);
BranchInst *BI = cast<BranchInst>(SplitBlockAndInsertIfThen(
Ne, Pos, /*Unreachable=*/false, ColdCallWeights));
IRBuilder<> ThenIRB(BI);
@@ -1303,61 +1303,61 @@ bool DataFlowSanitizer::runImpl(Module &M) {
M.global_size() != InitialGlobalSize || M.size() != InitialModuleSize;
}
-Value *DFSanFunction::getArgTLS(Type *T, unsigned ArgOffset, IRBuilder<> &IRB) {
- Value *Base = IRB.CreatePointerCast(DFS.ArgTLS, DFS.IntptrTy);
- if (ArgOffset)
- Base = IRB.CreateAdd(Base, ConstantInt::get(DFS.IntptrTy, ArgOffset));
- return IRB.CreateIntToPtr(Base, PointerType::get(DFS.getShadowTy(T), 0),
- "_dfsarg");
-}
-
-Value *DFSanFunction::getRetvalTLS(Type *T, IRBuilder<> &IRB) {
- return IRB.CreatePointerCast(
- DFS.RetvalTLS, PointerType::get(DFS.getShadowTy(T), 0), "_dfsret");
+Value *DFSanFunction::getArgTLS(Type *T, unsigned ArgOffset, IRBuilder<> &IRB) {
+ Value *Base = IRB.CreatePointerCast(DFS.ArgTLS, DFS.IntptrTy);
+ if (ArgOffset)
+ Base = IRB.CreateAdd(Base, ConstantInt::get(DFS.IntptrTy, ArgOffset));
+ return IRB.CreateIntToPtr(Base, PointerType::get(DFS.getShadowTy(T), 0),
+ "_dfsarg");
}
-Value *DFSanFunction::getShadowForTLSArgument(Argument *A) {
- unsigned ArgOffset = 0;
- const DataLayout &DL = F->getParent()->getDataLayout();
- for (auto &FArg : F->args()) {
- if (!FArg.getType()->isSized()) {
- if (A == &FArg)
- break;
- continue;
- }
-
- unsigned Size = DL.getTypeAllocSize(DFS.getShadowTy(&FArg));
- if (A != &FArg) {
- ArgOffset += alignTo(Size, kShadowTLSAlignment);
- if (ArgOffset > kArgTLSSize)
- break; // ArgTLS overflows, uses a zero shadow.
- continue;
- }
-
- if (ArgOffset + Size > kArgTLSSize)
- break; // ArgTLS overflows, uses a zero shadow.
-
- Instruction *ArgTLSPos = &*F->getEntryBlock().begin();
- IRBuilder<> IRB(ArgTLSPos);
- Value *ArgShadowPtr = getArgTLS(FArg.getType(), ArgOffset, IRB);
- return IRB.CreateAlignedLoad(DFS.getShadowTy(&FArg), ArgShadowPtr,
- kShadowTLSAlignment);
- }
-
- return DFS.getZeroShadow(A);
+Value *DFSanFunction::getRetvalTLS(Type *T, IRBuilder<> &IRB) {
+ return IRB.CreatePointerCast(
+ DFS.RetvalTLS, PointerType::get(DFS.getShadowTy(T), 0), "_dfsret");
+}
+
+Value *DFSanFunction::getShadowForTLSArgument(Argument *A) {
+ unsigned ArgOffset = 0;
+ const DataLayout &DL = F->getParent()->getDataLayout();
+ for (auto &FArg : F->args()) {
+ if (!FArg.getType()->isSized()) {
+ if (A == &FArg)
+ break;
+ continue;
+ }
+
+ unsigned Size = DL.getTypeAllocSize(DFS.getShadowTy(&FArg));
+ if (A != &FArg) {
+ ArgOffset += alignTo(Size, kShadowTLSAlignment);
+ if (ArgOffset > kArgTLSSize)
+ break; // ArgTLS overflows, uses a zero shadow.
+ continue;
+ }
+
+ if (ArgOffset + Size > kArgTLSSize)
+ break; // ArgTLS overflows, uses a zero shadow.
+
+ Instruction *ArgTLSPos = &*F->getEntryBlock().begin();
+ IRBuilder<> IRB(ArgTLSPos);
+ Value *ArgShadowPtr = getArgTLS(FArg.getType(), ArgOffset, IRB);
+ return IRB.CreateAlignedLoad(DFS.getShadowTy(&FArg), ArgShadowPtr,
+ kShadowTLSAlignment);
+ }
+
+ return DFS.getZeroShadow(A);
}
Value *DFSanFunction::getShadow(Value *V) {
if (!isa<Argument>(V) && !isa<Instruction>(V))
- return DFS.getZeroShadow(V);
+ return DFS.getZeroShadow(V);
Value *&Shadow = ValShadowMap[V];
if (!Shadow) {
if (Argument *A = dyn_cast<Argument>(V)) {
if (IsNativeABI)
- return DFS.getZeroShadow(V);
+ return DFS.getZeroShadow(V);
switch (IA) {
case DataFlowSanitizer::IA_TLS: {
- Shadow = getShadowForTLSArgument(A);
+ Shadow = getShadowForTLSArgument(A);
break;
}
case DataFlowSanitizer::IA_Args: {
@@ -1366,13 +1366,13 @@ Value *DFSanFunction::getShadow(Value *V) {
while (ArgIdx--)
++i;
Shadow = &*i;
- assert(Shadow->getType() == DFS.PrimitiveShadowTy);
+ assert(Shadow->getType() == DFS.PrimitiveShadowTy);
break;
}
}
NonZeroChecks.push_back(Shadow);
} else {
- Shadow = DFS.getZeroShadow(V);
+ Shadow = DFS.getZeroShadow(V);
}
}
return Shadow;
@@ -1380,8 +1380,8 @@ Value *DFSanFunction::getShadow(Value *V) {
void DFSanFunction::setShadow(Instruction *I, Value *Shadow) {
assert(!ValShadowMap.count(I));
- assert(DFS.shouldTrackFieldsAndIndices() ||
- Shadow->getType() == DFS.PrimitiveShadowTy);
+ assert(DFS.shouldTrackFieldsAndIndices() ||
+ Shadow->getType() == DFS.PrimitiveShadowTy);
ValShadowMap[I] = Shadow;
}
@@ -1398,60 +1398,60 @@ Value *DataFlowSanitizer::getShadowAddress(Value *Addr, Instruction *Pos) {
IRB.CreateAnd(IRB.CreatePtrToInt(Addr, IntptrTy),
IRB.CreatePtrToInt(ShadowPtrMaskValue, IntptrTy)),
ShadowPtrMul),
- PrimitiveShadowPtrTy);
+ PrimitiveShadowPtrTy);
+}
+
+Value *DFSanFunction::combineShadowsThenConvert(Type *T, Value *V1, Value *V2,
+ Instruction *Pos) {
+ Value *PrimitiveValue = combineShadows(V1, V2, Pos);
+ return expandFromPrimitiveShadow(T, PrimitiveValue, Pos);
}
-Value *DFSanFunction::combineShadowsThenConvert(Type *T, Value *V1, Value *V2,
- Instruction *Pos) {
- Value *PrimitiveValue = combineShadows(V1, V2, Pos);
- return expandFromPrimitiveShadow(T, PrimitiveValue, Pos);
-}
-
// Generates IR to compute the union of the two given shadows, inserting it
-// before Pos. The combined value is with primitive type.
+// before Pos. The combined value is with primitive type.
Value *DFSanFunction::combineShadows(Value *V1, Value *V2, Instruction *Pos) {
- if (DFS.isZeroShadow(V1))
- return collapseToPrimitiveShadow(V2, Pos);
- if (DFS.isZeroShadow(V2))
- return collapseToPrimitiveShadow(V1, Pos);
+ if (DFS.isZeroShadow(V1))
+ return collapseToPrimitiveShadow(V2, Pos);
+ if (DFS.isZeroShadow(V2))
+ return collapseToPrimitiveShadow(V1, Pos);
if (V1 == V2)
- return collapseToPrimitiveShadow(V1, Pos);
+ return collapseToPrimitiveShadow(V1, Pos);
auto V1Elems = ShadowElements.find(V1);
auto V2Elems = ShadowElements.find(V2);
if (V1Elems != ShadowElements.end() && V2Elems != ShadowElements.end()) {
if (std::includes(V1Elems->second.begin(), V1Elems->second.end(),
V2Elems->second.begin(), V2Elems->second.end())) {
- return collapseToPrimitiveShadow(V1, Pos);
+ return collapseToPrimitiveShadow(V1, Pos);
} else if (std::includes(V2Elems->second.begin(), V2Elems->second.end(),
V1Elems->second.begin(), V1Elems->second.end())) {
- return collapseToPrimitiveShadow(V2, Pos);
+ return collapseToPrimitiveShadow(V2, Pos);
}
} else if (V1Elems != ShadowElements.end()) {
if (V1Elems->second.count(V2))
- return collapseToPrimitiveShadow(V1, Pos);
+ return collapseToPrimitiveShadow(V1, Pos);
} else if (V2Elems != ShadowElements.end()) {
if (V2Elems->second.count(V1))
- return collapseToPrimitiveShadow(V2, Pos);
+ return collapseToPrimitiveShadow(V2, Pos);
}
auto Key = std::make_pair(V1, V2);
if (V1 > V2)
std::swap(Key.first, Key.second);
- CachedShadow &CCS = CachedShadows[Key];
+ CachedShadow &CCS = CachedShadows[Key];
if (CCS.Block && DT.dominates(CCS.Block, Pos->getParent()))
return CCS.Shadow;
- // Converts inputs shadows to shadows with primitive types.
- Value *PV1 = collapseToPrimitiveShadow(V1, Pos);
- Value *PV2 = collapseToPrimitiveShadow(V2, Pos);
-
+ // Converts inputs shadows to shadows with primitive types.
+ Value *PV1 = collapseToPrimitiveShadow(V1, Pos);
+ Value *PV2 = collapseToPrimitiveShadow(V2, Pos);
+
IRBuilder<> IRB(Pos);
- if (ClFast16Labels) {
- CCS.Block = Pos->getParent();
- CCS.Shadow = IRB.CreateOr(PV1, PV2);
- } else if (AvoidNewBlocks) {
- CallInst *Call = IRB.CreateCall(DFS.DFSanCheckedUnionFn, {PV1, PV2});
+ if (ClFast16Labels) {
+ CCS.Block = Pos->getParent();
+ CCS.Shadow = IRB.CreateOr(PV1, PV2);
+ } else if (AvoidNewBlocks) {
+ CallInst *Call = IRB.CreateCall(DFS.DFSanCheckedUnionFn, {PV1, PV2});
Call->addAttribute(AttributeList::ReturnIndex, Attribute::ZExt);
Call->addParamAttr(0, Attribute::ZExt);
Call->addParamAttr(1, Attribute::ZExt);
@@ -1460,20 +1460,20 @@ Value *DFSanFunction::combineShadows(Value *V1, Value *V2, Instruction *Pos) {
CCS.Shadow = Call;
} else {
BasicBlock *Head = Pos->getParent();
- Value *Ne = IRB.CreateICmpNE(PV1, PV2);
+ Value *Ne = IRB.CreateICmpNE(PV1, PV2);
BranchInst *BI = cast<BranchInst>(SplitBlockAndInsertIfThen(
Ne, Pos, /*Unreachable=*/false, DFS.ColdCallWeights, &DT));
IRBuilder<> ThenIRB(BI);
- CallInst *Call = ThenIRB.CreateCall(DFS.DFSanUnionFn, {PV1, PV2});
+ CallInst *Call = ThenIRB.CreateCall(DFS.DFSanUnionFn, {PV1, PV2});
Call->addAttribute(AttributeList::ReturnIndex, Attribute::ZExt);
Call->addParamAttr(0, Attribute::ZExt);
Call->addParamAttr(1, Attribute::ZExt);
BasicBlock *Tail = BI->getSuccessor(0);
- PHINode *Phi =
- PHINode::Create(DFS.PrimitiveShadowTy, 2, "", &Tail->front());
+ PHINode *Phi =
+ PHINode::Create(DFS.PrimitiveShadowTy, 2, "", &Tail->front());
Phi->addIncoming(Call, Call->getParent());
- Phi->addIncoming(PV1, Head);
+ Phi->addIncoming(PV1, Head);
CCS.Block = Tail;
CCS.Shadow = Phi;
@@ -1500,13 +1500,13 @@ Value *DFSanFunction::combineShadows(Value *V1, Value *V2, Instruction *Pos) {
// the computed union Value.
Value *DFSanFunction::combineOperandShadows(Instruction *Inst) {
if (Inst->getNumOperands() == 0)
- return DFS.getZeroShadow(Inst);
+ return DFS.getZeroShadow(Inst);
Value *Shadow = getShadow(Inst->getOperand(0));
for (unsigned i = 1, n = Inst->getNumOperands(); i != n; ++i) {
Shadow = combineShadows(Shadow, getShadow(Inst->getOperand(i)), Inst);
}
- return expandFromPrimitiveShadow(Inst->getType(), Shadow, Inst);
+ return expandFromPrimitiveShadow(Inst->getType(), Shadow, Inst);
}
Value *DFSanVisitor::visitOperandShadowInst(Instruction &I) {
@@ -1516,21 +1516,21 @@ Value *DFSanVisitor::visitOperandShadowInst(Instruction &I) {
}
// Generates IR to load shadow corresponding to bytes [Addr, Addr+Size), where
-// Addr has alignment Align, and take the union of each of those shadows. The
-// returned shadow always has primitive type.
+// Addr has alignment Align, and take the union of each of those shadows. The
+// returned shadow always has primitive type.
Value *DFSanFunction::loadShadow(Value *Addr, uint64_t Size, uint64_t Align,
Instruction *Pos) {
if (AllocaInst *AI = dyn_cast<AllocaInst>(Addr)) {
const auto i = AllocaShadowMap.find(AI);
if (i != AllocaShadowMap.end()) {
IRBuilder<> IRB(Pos);
- return IRB.CreateLoad(DFS.PrimitiveShadowTy, i->second);
+ return IRB.CreateLoad(DFS.PrimitiveShadowTy, i->second);
}
}
const llvm::Align ShadowAlign(Align * DFS.ShadowWidthBytes);
SmallVector<const Value *, 2> Objs;
- getUnderlyingObjects(Addr, Objs);
+ getUnderlyingObjects(Addr, Objs);
bool AllConstants = true;
for (const Value *Obj : Objs) {
if (isa<Function>(Obj) || isa<BlockAddress>(Obj))
@@ -1542,51 +1542,51 @@ Value *DFSanFunction::loadShadow(Value *Addr, uint64_t Size, uint64_t Align,
break;
}
if (AllConstants)
- return DFS.ZeroPrimitiveShadow;
+ return DFS.ZeroPrimitiveShadow;
Value *ShadowAddr = DFS.getShadowAddress(Addr, Pos);
switch (Size) {
case 0:
- return DFS.ZeroPrimitiveShadow;
+ return DFS.ZeroPrimitiveShadow;
case 1: {
- LoadInst *LI = new LoadInst(DFS.PrimitiveShadowTy, ShadowAddr, "", Pos);
+ LoadInst *LI = new LoadInst(DFS.PrimitiveShadowTy, ShadowAddr, "", Pos);
LI->setAlignment(ShadowAlign);
return LI;
}
case 2: {
IRBuilder<> IRB(Pos);
- Value *ShadowAddr1 = IRB.CreateGEP(DFS.PrimitiveShadowTy, ShadowAddr,
+ Value *ShadowAddr1 = IRB.CreateGEP(DFS.PrimitiveShadowTy, ShadowAddr,
ConstantInt::get(DFS.IntptrTy, 1));
return combineShadows(
- IRB.CreateAlignedLoad(DFS.PrimitiveShadowTy, ShadowAddr, ShadowAlign),
- IRB.CreateAlignedLoad(DFS.PrimitiveShadowTy, ShadowAddr1, ShadowAlign),
- Pos);
- }
- }
-
- if (ClFast16Labels && Size % (64 / DFS.ShadowWidthBits) == 0) {
- // First OR all the WideShadows, then OR individual shadows within the
- // combined WideShadow. This is fewer instructions than ORing shadows
- // individually.
- IRBuilder<> IRB(Pos);
- Value *WideAddr =
- IRB.CreateBitCast(ShadowAddr, Type::getInt64PtrTy(*DFS.Ctx));
- Value *CombinedWideShadow =
- IRB.CreateAlignedLoad(IRB.getInt64Ty(), WideAddr, ShadowAlign);
- for (uint64_t Ofs = 64 / DFS.ShadowWidthBits; Ofs != Size;
- Ofs += 64 / DFS.ShadowWidthBits) {
- WideAddr = IRB.CreateGEP(Type::getInt64Ty(*DFS.Ctx), WideAddr,
- ConstantInt::get(DFS.IntptrTy, 1));
- Value *NextWideShadow =
- IRB.CreateAlignedLoad(IRB.getInt64Ty(), WideAddr, ShadowAlign);
- CombinedWideShadow = IRB.CreateOr(CombinedWideShadow, NextWideShadow);
- }
- for (unsigned Width = 32; Width >= DFS.ShadowWidthBits; Width >>= 1) {
- Value *ShrShadow = IRB.CreateLShr(CombinedWideShadow, Width);
- CombinedWideShadow = IRB.CreateOr(CombinedWideShadow, ShrShadow);
- }
- return IRB.CreateTrunc(CombinedWideShadow, DFS.PrimitiveShadowTy);
- }
+ IRB.CreateAlignedLoad(DFS.PrimitiveShadowTy, ShadowAddr, ShadowAlign),
+ IRB.CreateAlignedLoad(DFS.PrimitiveShadowTy, ShadowAddr1, ShadowAlign),
+ Pos);
+ }
+ }
+
+ if (ClFast16Labels && Size % (64 / DFS.ShadowWidthBits) == 0) {
+ // First OR all the WideShadows, then OR individual shadows within the
+ // combined WideShadow. This is fewer instructions than ORing shadows
+ // individually.
+ IRBuilder<> IRB(Pos);
+ Value *WideAddr =
+ IRB.CreateBitCast(ShadowAddr, Type::getInt64PtrTy(*DFS.Ctx));
+ Value *CombinedWideShadow =
+ IRB.CreateAlignedLoad(IRB.getInt64Ty(), WideAddr, ShadowAlign);
+ for (uint64_t Ofs = 64 / DFS.ShadowWidthBits; Ofs != Size;
+ Ofs += 64 / DFS.ShadowWidthBits) {
+ WideAddr = IRB.CreateGEP(Type::getInt64Ty(*DFS.Ctx), WideAddr,
+ ConstantInt::get(DFS.IntptrTy, 1));
+ Value *NextWideShadow =
+ IRB.CreateAlignedLoad(IRB.getInt64Ty(), WideAddr, ShadowAlign);
+ CombinedWideShadow = IRB.CreateOr(CombinedWideShadow, NextWideShadow);
+ }
+ for (unsigned Width = 32; Width >= DFS.ShadowWidthBits; Width >>= 1) {
+ Value *ShrShadow = IRB.CreateLShr(CombinedWideShadow, Width);
+ CombinedWideShadow = IRB.CreateOr(CombinedWideShadow, ShrShadow);
+ }
+ return IRB.CreateTrunc(CombinedWideShadow, DFS.PrimitiveShadowTy);
+ }
if (!AvoidNewBlocks && Size % (64 / DFS.ShadowWidthBits) == 0) {
// Fast path for the common case where each byte has identical shadow: load
// shadow 64 bits at a time, fall out to a __dfsan_union_load call if any
@@ -1605,7 +1605,7 @@ Value *DFSanFunction::loadShadow(Value *Addr, uint64_t Size, uint64_t Align,
IRB.CreateBitCast(ShadowAddr, Type::getInt64PtrTy(*DFS.Ctx));
Value *WideShadow =
IRB.CreateAlignedLoad(IRB.getInt64Ty(), WideAddr, ShadowAlign);
- Value *TruncShadow = IRB.CreateTrunc(WideShadow, DFS.PrimitiveShadowTy);
+ Value *TruncShadow = IRB.CreateTrunc(WideShadow, DFS.PrimitiveShadowTy);
Value *ShlShadow = IRB.CreateShl(WideShadow, DFS.ShadowWidthBits);
Value *ShrShadow = IRB.CreateLShr(WideShadow, 64 - DFS.ShadowWidthBits);
Value *RotShadow = IRB.CreateOr(ShlShadow, ShrShadow);
@@ -1646,18 +1646,18 @@ Value *DFSanFunction::loadShadow(Value *Addr, uint64_t Size, uint64_t Align,
LastBr->setSuccessor(0, Tail);
FallbackIRB.CreateBr(Tail);
- PHINode *Shadow =
- PHINode::Create(DFS.PrimitiveShadowTy, 2, "", &Tail->front());
+ PHINode *Shadow =
+ PHINode::Create(DFS.PrimitiveShadowTy, 2, "", &Tail->front());
Shadow->addIncoming(FallbackCall, FallbackBB);
Shadow->addIncoming(TruncShadow, LastBr->getParent());
return Shadow;
}
IRBuilder<> IRB(Pos);
- FunctionCallee &UnionLoadFn =
- ClFast16Labels ? DFS.DFSanUnionLoadFast16LabelsFn : DFS.DFSanUnionLoadFn;
+ FunctionCallee &UnionLoadFn =
+ ClFast16Labels ? DFS.DFSanUnionLoadFast16LabelsFn : DFS.DFSanUnionLoadFn;
CallInst *FallbackCall = IRB.CreateCall(
- UnionLoadFn, {ShadowAddr, ConstantInt::get(DFS.IntptrTy, Size)});
+ UnionLoadFn, {ShadowAddr, ConstantInt::get(DFS.IntptrTy, Size)});
FallbackCall->addAttribute(AttributeList::ReturnIndex, Attribute::ZExt);
return FallbackCall;
}
@@ -1666,39 +1666,39 @@ void DFSanVisitor::visitLoadInst(LoadInst &LI) {
auto &DL = LI.getModule()->getDataLayout();
uint64_t Size = DL.getTypeStoreSize(LI.getType());
if (Size == 0) {
- DFSF.setShadow(&LI, DFSF.DFS.getZeroShadow(&LI));
+ DFSF.setShadow(&LI, DFSF.DFS.getZeroShadow(&LI));
return;
}
Align Alignment = ClPreserveAlignment ? LI.getAlign() : Align(1);
- Value *PrimitiveShadow =
+ Value *PrimitiveShadow =
DFSF.loadShadow(LI.getPointerOperand(), Size, Alignment.value(), &LI);
if (ClCombinePointerLabelsOnLoad) {
Value *PtrShadow = DFSF.getShadow(LI.getPointerOperand());
- PrimitiveShadow = DFSF.combineShadows(PrimitiveShadow, PtrShadow, &LI);
+ PrimitiveShadow = DFSF.combineShadows(PrimitiveShadow, PtrShadow, &LI);
}
- if (!DFSF.DFS.isZeroShadow(PrimitiveShadow))
- DFSF.NonZeroChecks.push_back(PrimitiveShadow);
+ if (!DFSF.DFS.isZeroShadow(PrimitiveShadow))
+ DFSF.NonZeroChecks.push_back(PrimitiveShadow);
- Value *Shadow =
- DFSF.expandFromPrimitiveShadow(LI.getType(), PrimitiveShadow, &LI);
+ Value *Shadow =
+ DFSF.expandFromPrimitiveShadow(LI.getType(), PrimitiveShadow, &LI);
DFSF.setShadow(&LI, Shadow);
if (ClEventCallbacks) {
IRBuilder<> IRB(&LI);
- Value *Addr8 = IRB.CreateBitCast(LI.getPointerOperand(), DFSF.DFS.Int8Ptr);
- IRB.CreateCall(DFSF.DFS.DFSanLoadCallbackFn, {PrimitiveShadow, Addr8});
+ Value *Addr8 = IRB.CreateBitCast(LI.getPointerOperand(), DFSF.DFS.Int8Ptr);
+ IRB.CreateCall(DFSF.DFS.DFSanLoadCallbackFn, {PrimitiveShadow, Addr8});
}
}
-void DFSanFunction::storePrimitiveShadow(Value *Addr, uint64_t Size,
- Align Alignment,
- Value *PrimitiveShadow,
- Instruction *Pos) {
+void DFSanFunction::storePrimitiveShadow(Value *Addr, uint64_t Size,
+ Align Alignment,
+ Value *PrimitiveShadow,
+ Instruction *Pos) {
if (AllocaInst *AI = dyn_cast<AllocaInst>(Addr)) {
const auto i = AllocaShadowMap.find(AI);
if (i != AllocaShadowMap.end()) {
IRBuilder<> IRB(Pos);
- IRB.CreateStore(PrimitiveShadow, i->second);
+ IRB.CreateStore(PrimitiveShadow, i->second);
return;
}
}
@@ -1706,7 +1706,7 @@ void DFSanFunction::storePrimitiveShadow(Value *Addr, uint64_t Size,
const Align ShadowAlign(Alignment.value() * DFS.ShadowWidthBytes);
IRBuilder<> IRB(Pos);
Value *ShadowAddr = DFS.getShadowAddress(Addr, Pos);
- if (DFS.isZeroShadow(PrimitiveShadow)) {
+ if (DFS.isZeroShadow(PrimitiveShadow)) {
IntegerType *ShadowTy =
IntegerType::get(*DFS.Ctx, Size * DFS.ShadowWidthBits);
Value *ExtZeroShadow = ConstantInt::get(ShadowTy, 0);
@@ -1719,13 +1719,13 @@ void DFSanFunction::storePrimitiveShadow(Value *Addr, uint64_t Size,
const unsigned ShadowVecSize = 128 / DFS.ShadowWidthBits;
uint64_t Offset = 0;
if (Size >= ShadowVecSize) {
- auto *ShadowVecTy =
- FixedVectorType::get(DFS.PrimitiveShadowTy, ShadowVecSize);
+ auto *ShadowVecTy =
+ FixedVectorType::get(DFS.PrimitiveShadowTy, ShadowVecSize);
Value *ShadowVec = UndefValue::get(ShadowVecTy);
for (unsigned i = 0; i != ShadowVecSize; ++i) {
ShadowVec = IRB.CreateInsertElement(
- ShadowVec, PrimitiveShadow,
- ConstantInt::get(Type::getInt32Ty(*DFS.Ctx), i));
+ ShadowVec, PrimitiveShadow,
+ ConstantInt::get(Type::getInt32Ty(*DFS.Ctx), i));
}
Value *ShadowVecAddr =
IRB.CreateBitCast(ShadowAddr, PointerType::getUnqual(ShadowVecTy));
@@ -1740,8 +1740,8 @@ void DFSanFunction::storePrimitiveShadow(Value *Addr, uint64_t Size,
}
while (Size > 0) {
Value *CurShadowAddr =
- IRB.CreateConstGEP1_32(DFS.PrimitiveShadowTy, ShadowAddr, Offset);
- IRB.CreateAlignedStore(PrimitiveShadow, CurShadowAddr, ShadowAlign);
+ IRB.CreateConstGEP1_32(DFS.PrimitiveShadowTy, ShadowAddr, Offset);
+ IRB.CreateAlignedStore(PrimitiveShadow, CurShadowAddr, ShadowAlign);
--Size;
++Offset;
}
@@ -1756,19 +1756,19 @@ void DFSanVisitor::visitStoreInst(StoreInst &SI) {
const Align Alignment = ClPreserveAlignment ? SI.getAlign() : Align(1);
Value* Shadow = DFSF.getShadow(SI.getValueOperand());
- Value *PrimitiveShadow;
+ Value *PrimitiveShadow;
if (ClCombinePointerLabelsOnStore) {
Value *PtrShadow = DFSF.getShadow(SI.getPointerOperand());
- PrimitiveShadow = DFSF.combineShadows(Shadow, PtrShadow, &SI);
- } else {
- PrimitiveShadow = DFSF.collapseToPrimitiveShadow(Shadow, &SI);
+ PrimitiveShadow = DFSF.combineShadows(Shadow, PtrShadow, &SI);
+ } else {
+ PrimitiveShadow = DFSF.collapseToPrimitiveShadow(Shadow, &SI);
}
- DFSF.storePrimitiveShadow(SI.getPointerOperand(), Size, Alignment,
- PrimitiveShadow, &SI);
+ DFSF.storePrimitiveShadow(SI.getPointerOperand(), Size, Alignment,
+ PrimitiveShadow, &SI);
if (ClEventCallbacks) {
IRBuilder<> IRB(&SI);
- Value *Addr8 = IRB.CreateBitCast(SI.getPointerOperand(), DFSF.DFS.Int8Ptr);
- IRB.CreateCall(DFSF.DFS.DFSanStoreCallbackFn, {PrimitiveShadow, Addr8});
+ Value *Addr8 = IRB.CreateBitCast(SI.getPointerOperand(), DFSF.DFS.Int8Ptr);
+ IRB.CreateCall(DFSF.DFS.DFSanStoreCallbackFn, {PrimitiveShadow, Addr8});
}
}
@@ -1807,29 +1807,29 @@ void DFSanVisitor::visitShuffleVectorInst(ShuffleVectorInst &I) {
}
void DFSanVisitor::visitExtractValueInst(ExtractValueInst &I) {
- if (!DFSF.DFS.shouldTrackFieldsAndIndices()) {
- visitOperandShadowInst(I);
- return;
- }
-
- IRBuilder<> IRB(&I);
- Value *Agg = I.getAggregateOperand();
- Value *AggShadow = DFSF.getShadow(Agg);
- Value *ResShadow = IRB.CreateExtractValue(AggShadow, I.getIndices());
- DFSF.setShadow(&I, ResShadow);
+ if (!DFSF.DFS.shouldTrackFieldsAndIndices()) {
+ visitOperandShadowInst(I);
+ return;
+ }
+
+ IRBuilder<> IRB(&I);
+ Value *Agg = I.getAggregateOperand();
+ Value *AggShadow = DFSF.getShadow(Agg);
+ Value *ResShadow = IRB.CreateExtractValue(AggShadow, I.getIndices());
+ DFSF.setShadow(&I, ResShadow);
}
void DFSanVisitor::visitInsertValueInst(InsertValueInst &I) {
- if (!DFSF.DFS.shouldTrackFieldsAndIndices()) {
- visitOperandShadowInst(I);
- return;
- }
-
- IRBuilder<> IRB(&I);
- Value *AggShadow = DFSF.getShadow(I.getAggregateOperand());
- Value *InsShadow = DFSF.getShadow(I.getInsertedValueOperand());
- Value *Res = IRB.CreateInsertValue(AggShadow, InsShadow, I.getIndices());
- DFSF.setShadow(&I, Res);
+ if (!DFSF.DFS.shouldTrackFieldsAndIndices()) {
+ visitOperandShadowInst(I);
+ return;
+ }
+
+ IRBuilder<> IRB(&I);
+ Value *AggShadow = DFSF.getShadow(I.getAggregateOperand());
+ Value *InsShadow = DFSF.getShadow(I.getInsertedValueOperand());
+ Value *Res = IRB.CreateInsertValue(AggShadow, InsShadow, I.getIndices());
+ DFSF.setShadow(&I, Res);
}
void DFSanVisitor::visitAllocaInst(AllocaInst &I) {
@@ -1848,20 +1848,20 @@ void DFSanVisitor::visitAllocaInst(AllocaInst &I) {
}
if (AllLoadsStores) {
IRBuilder<> IRB(&I);
- DFSF.AllocaShadowMap[&I] = IRB.CreateAlloca(DFSF.DFS.PrimitiveShadowTy);
+ DFSF.AllocaShadowMap[&I] = IRB.CreateAlloca(DFSF.DFS.PrimitiveShadowTy);
}
- DFSF.setShadow(&I, DFSF.DFS.ZeroPrimitiveShadow);
+ DFSF.setShadow(&I, DFSF.DFS.ZeroPrimitiveShadow);
}
void DFSanVisitor::visitSelectInst(SelectInst &I) {
Value *CondShadow = DFSF.getShadow(I.getCondition());
Value *TrueShadow = DFSF.getShadow(I.getTrueValue());
Value *FalseShadow = DFSF.getShadow(I.getFalseValue());
- Value *ShadowSel = nullptr;
+ Value *ShadowSel = nullptr;
if (isa<VectorType>(I.getCondition()->getType())) {
- ShadowSel = DFSF.combineShadowsThenConvert(I.getType(), TrueShadow,
- FalseShadow, &I);
+ ShadowSel = DFSF.combineShadowsThenConvert(I.getType(), TrueShadow,
+ FalseShadow, &I);
} else {
if (TrueShadow == FalseShadow) {
ShadowSel = TrueShadow;
@@ -1870,10 +1870,10 @@ void DFSanVisitor::visitSelectInst(SelectInst &I) {
SelectInst::Create(I.getCondition(), TrueShadow, FalseShadow, "", &I);
}
}
- DFSF.setShadow(&I, ClTrackSelectControlFlow
- ? DFSF.combineShadowsThenConvert(
- I.getType(), CondShadow, ShadowSel, &I)
- : ShadowSel);
+ DFSF.setShadow(&I, ClTrackSelectControlFlow
+ ? DFSF.combineShadowsThenConvert(
+ I.getType(), CondShadow, ShadowSel, &I)
+ : ShadowSel);
}
void DFSanVisitor::visitMemSetInst(MemSetInst &I) {
@@ -1917,15 +1917,15 @@ void DFSanVisitor::visitReturnInst(ReturnInst &RI) {
case DataFlowSanitizer::IA_TLS: {
Value *S = DFSF.getShadow(RI.getReturnValue());
IRBuilder<> IRB(&RI);
- Type *RT = DFSF.F->getFunctionType()->getReturnType();
- unsigned Size =
- getDataLayout().getTypeAllocSize(DFSF.DFS.getShadowTy(RT));
- if (Size <= kRetvalTLSSize) {
- // If the size overflows, stores nothing. At callsite, oversized return
- // shadows are set to zero.
- IRB.CreateAlignedStore(S, DFSF.getRetvalTLS(RT, IRB),
- kShadowTLSAlignment);
- }
+ Type *RT = DFSF.F->getFunctionType()->getReturnType();
+ unsigned Size =
+ getDataLayout().getTypeAllocSize(DFSF.DFS.getShadowTy(RT));
+ if (Size <= kRetvalTLSSize) {
+ // If the size overflows, stores nothing. At callsite, oversized return
+ // shadows are set to zero.
+ IRB.CreateAlignedStore(S, DFSF.getRetvalTLS(RT, IRB),
+ kShadowTLSAlignment);
+ }
break;
}
case DataFlowSanitizer::IA_Args: {
@@ -1965,11 +1965,11 @@ void DFSanVisitor::visitCallBase(CallBase &CB) {
CB.setCalledFunction(F);
IRB.CreateCall(DFSF.DFS.DFSanUnimplementedFn,
IRB.CreateGlobalStringPtr(F->getName()));
- DFSF.setShadow(&CB, DFSF.DFS.getZeroShadow(&CB));
+ DFSF.setShadow(&CB, DFSF.DFS.getZeroShadow(&CB));
return;
case DataFlowSanitizer::WK_Discard:
CB.setCalledFunction(F);
- DFSF.setShadow(&CB, DFSF.DFS.getZeroShadow(&CB));
+ DFSF.setShadow(&CB, DFSF.DFS.getZeroShadow(&CB));
return;
case DataFlowSanitizer::WK_Functional:
CB.setCalledFunction(F);
@@ -2021,11 +2021,11 @@ void DFSanVisitor::visitCallBase(CallBase &CB) {
i = CB.arg_begin();
const unsigned ShadowArgStart = Args.size();
for (unsigned n = FT->getNumParams(); n != 0; ++i, --n)
- Args.push_back(
- DFSF.collapseToPrimitiveShadow(DFSF.getShadow(*i), &CB));
+ Args.push_back(
+ DFSF.collapseToPrimitiveShadow(DFSF.getShadow(*i), &CB));
if (FT->isVarArg()) {
- auto *LabelVATy = ArrayType::get(DFSF.DFS.PrimitiveShadowTy,
+ auto *LabelVATy = ArrayType::get(DFSF.DFS.PrimitiveShadowTy,
CB.arg_size() - FT->getNumParams());
auto *LabelVAAlloca = new AllocaInst(
LabelVATy, getDataLayout().getAllocaAddrSpace(),
@@ -2033,9 +2033,9 @@ void DFSanVisitor::visitCallBase(CallBase &CB) {
for (unsigned n = 0; i != CB.arg_end(); ++i, ++n) {
auto LabelVAPtr = IRB.CreateStructGEP(LabelVATy, LabelVAAlloca, n);
- IRB.CreateStore(
- DFSF.collapseToPrimitiveShadow(DFSF.getShadow(*i), &CB),
- LabelVAPtr);
+ IRB.CreateStore(
+ DFSF.collapseToPrimitiveShadow(DFSF.getShadow(*i), &CB),
+ LabelVAPtr);
}
Args.push_back(IRB.CreateStructGEP(LabelVATy, LabelVAAlloca, 0));
@@ -2044,9 +2044,9 @@ void DFSanVisitor::visitCallBase(CallBase &CB) {
if (!FT->getReturnType()->isVoidTy()) {
if (!DFSF.LabelReturnAlloca) {
DFSF.LabelReturnAlloca =
- new AllocaInst(DFSF.DFS.PrimitiveShadowTy,
- getDataLayout().getAllocaAddrSpace(),
- "labelreturn", &DFSF.F->getEntryBlock().front());
+ new AllocaInst(DFSF.DFS.PrimitiveShadowTy,
+ getDataLayout().getAllocaAddrSpace(),
+ "labelreturn", &DFSF.F->getEntryBlock().front());
}
Args.push_back(DFSF.LabelReturnAlloca);
}
@@ -2061,19 +2061,19 @@ void DFSanVisitor::visitCallBase(CallBase &CB) {
// Update the parameter attributes of the custom call instruction to
// zero extend the shadow parameters. This is required for targets
- // which consider PrimitiveShadowTy an illegal type.
+ // which consider PrimitiveShadowTy an illegal type.
for (unsigned n = 0; n < FT->getNumParams(); n++) {
const unsigned ArgNo = ShadowArgStart + n;
- if (CustomCI->getArgOperand(ArgNo)->getType() ==
- DFSF.DFS.PrimitiveShadowTy)
+ if (CustomCI->getArgOperand(ArgNo)->getType() ==
+ DFSF.DFS.PrimitiveShadowTy)
CustomCI->addParamAttr(ArgNo, Attribute::ZExt);
}
if (!FT->getReturnType()->isVoidTy()) {
- LoadInst *LabelLoad = IRB.CreateLoad(DFSF.DFS.PrimitiveShadowTy,
- DFSF.LabelReturnAlloca);
- DFSF.setShadow(CustomCI, DFSF.expandFromPrimitiveShadow(
- FT->getReturnType(), LabelLoad, &CB));
+ LoadInst *LabelLoad = IRB.CreateLoad(DFSF.DFS.PrimitiveShadowTy,
+ DFSF.LabelReturnAlloca);
+ DFSF.setShadow(CustomCI, DFSF.expandFromPrimitiveShadow(
+ FT->getReturnType(), LabelLoad, &CB));
}
CI->replaceAllUsesWith(CustomCI);
@@ -2086,20 +2086,20 @@ void DFSanVisitor::visitCallBase(CallBase &CB) {
FunctionType *FT = CB.getFunctionType();
if (DFSF.DFS.getInstrumentedABI() == DataFlowSanitizer::IA_TLS) {
- unsigned ArgOffset = 0;
- const DataLayout &DL = getDataLayout();
- for (unsigned I = 0, N = FT->getNumParams(); I != N; ++I) {
- unsigned Size =
- DL.getTypeAllocSize(DFSF.DFS.getShadowTy(FT->getParamType(I)));
- // Stop storing if arguments' size overflows. Inside a function, arguments
- // after overflow have zero shadow values.
- if (ArgOffset + Size > kArgTLSSize)
- break;
- IRB.CreateAlignedStore(
- DFSF.getShadow(CB.getArgOperand(I)),
- DFSF.getArgTLS(FT->getParamType(I), ArgOffset, IRB),
- kShadowTLSAlignment);
- ArgOffset += alignTo(Size, kShadowTLSAlignment);
+ unsigned ArgOffset = 0;
+ const DataLayout &DL = getDataLayout();
+ for (unsigned I = 0, N = FT->getNumParams(); I != N; ++I) {
+ unsigned Size =
+ DL.getTypeAllocSize(DFSF.DFS.getShadowTy(FT->getParamType(I)));
+ // Stop storing if arguments' size overflows. Inside a function, arguments
+ // after overflow have zero shadow values.
+ if (ArgOffset + Size > kArgTLSSize)
+ break;
+ IRB.CreateAlignedStore(
+ DFSF.getShadow(CB.getArgOperand(I)),
+ DFSF.getArgTLS(FT->getParamType(I), ArgOffset, IRB),
+ kShadowTLSAlignment);
+ ArgOffset += alignTo(Size, kShadowTLSAlignment);
}
}
@@ -2120,19 +2120,19 @@ void DFSanVisitor::visitCallBase(CallBase &CB) {
if (DFSF.DFS.getInstrumentedABI() == DataFlowSanitizer::IA_TLS) {
IRBuilder<> NextIRB(Next);
- const DataLayout &DL = getDataLayout();
- unsigned Size = DL.getTypeAllocSize(DFSF.DFS.getShadowTy(&CB));
- if (Size > kRetvalTLSSize) {
- // Set overflowed return shadow to be zero.
- DFSF.setShadow(&CB, DFSF.DFS.getZeroShadow(&CB));
- } else {
- LoadInst *LI = NextIRB.CreateAlignedLoad(
- DFSF.DFS.getShadowTy(&CB), DFSF.getRetvalTLS(CB.getType(), NextIRB),
- kShadowTLSAlignment, "_dfsret");
- DFSF.SkipInsts.insert(LI);
- DFSF.setShadow(&CB, LI);
- DFSF.NonZeroChecks.push_back(LI);
- }
+ const DataLayout &DL = getDataLayout();
+ unsigned Size = DL.getTypeAllocSize(DFSF.DFS.getShadowTy(&CB));
+ if (Size > kRetvalTLSSize) {
+ // Set overflowed return shadow to be zero.
+ DFSF.setShadow(&CB, DFSF.DFS.getZeroShadow(&CB));
+ } else {
+ LoadInst *LI = NextIRB.CreateAlignedLoad(
+ DFSF.DFS.getShadowTy(&CB), DFSF.getRetvalTLS(CB.getType(), NextIRB),
+ kShadowTLSAlignment, "_dfsret");
+ DFSF.SkipInsts.insert(LI);
+ DFSF.setShadow(&CB, LI);
+ DFSF.NonZeroChecks.push_back(LI);
+ }
}
}
@@ -2154,8 +2154,8 @@ void DFSanVisitor::visitCallBase(CallBase &CB) {
if (FT->isVarArg()) {
unsigned VarArgSize = CB.arg_size() - FT->getNumParams();
- ArrayType *VarArgArrayTy =
- ArrayType::get(DFSF.DFS.PrimitiveShadowTy, VarArgSize);
+ ArrayType *VarArgArrayTy =
+ ArrayType::get(DFSF.DFS.PrimitiveShadowTy, VarArgSize);
AllocaInst *VarArgShadow =
new AllocaInst(VarArgArrayTy, getDataLayout().getAllocaAddrSpace(),
"", &DFSF.F->getEntryBlock().front());
@@ -2196,12 +2196,12 @@ void DFSanVisitor::visitCallBase(CallBase &CB) {
}
void DFSanVisitor::visitPHINode(PHINode &PN) {
- Type *ShadowTy = DFSF.DFS.getShadowTy(&PN);
+ Type *ShadowTy = DFSF.DFS.getShadowTy(&PN);
PHINode *ShadowPN =
- PHINode::Create(ShadowTy, PN.getNumIncomingValues(), "", &PN);
+ PHINode::Create(ShadowTy, PN.getNumIncomingValues(), "", &PN);
// Give the shadow phi node valid predecessors to fool SplitEdge into working.
- Value *UndefShadow = UndefValue::get(ShadowTy);
+ Value *UndefShadow = UndefValue::get(ShadowTy);
for (PHINode::block_iterator i = PN.block_begin(), e = PN.block_end(); i != e;
++i) {
ShadowPN->addIncoming(UndefShadow, *i);
@@ -2210,39 +2210,39 @@ void DFSanVisitor::visitPHINode(PHINode &PN) {
DFSF.PHIFixups.push_back(std::make_pair(&PN, ShadowPN));
DFSF.setShadow(&PN, ShadowPN);
}
-
-namespace {
-class DataFlowSanitizerLegacyPass : public ModulePass {
-private:
- std::vector<std::string> ABIListFiles;
-
-public:
- static char ID;
-
- DataFlowSanitizerLegacyPass(
- const std::vector<std::string> &ABIListFiles = std::vector<std::string>())
- : ModulePass(ID), ABIListFiles(ABIListFiles) {}
-
- bool runOnModule(Module &M) override {
- return DataFlowSanitizer(ABIListFiles).runImpl(M);
- }
-};
-} // namespace
-
-char DataFlowSanitizerLegacyPass::ID;
-
-INITIALIZE_PASS(DataFlowSanitizerLegacyPass, "dfsan",
- "DataFlowSanitizer: dynamic data flow analysis.", false, false)
-
-ModulePass *llvm::createDataFlowSanitizerLegacyPassPass(
- const std::vector<std::string> &ABIListFiles) {
- return new DataFlowSanitizerLegacyPass(ABIListFiles);
-}
-
-PreservedAnalyses DataFlowSanitizerPass::run(Module &M,
- ModuleAnalysisManager &AM) {
- if (DataFlowSanitizer(ABIListFiles).runImpl(M)) {
- return PreservedAnalyses::none();
- }
- return PreservedAnalyses::all();
-}
+
+namespace {
+class DataFlowSanitizerLegacyPass : public ModulePass {
+private:
+ std::vector<std::string> ABIListFiles;
+
+public:
+ static char ID;
+
+ DataFlowSanitizerLegacyPass(
+ const std::vector<std::string> &ABIListFiles = std::vector<std::string>())
+ : ModulePass(ID), ABIListFiles(ABIListFiles) {}
+
+ bool runOnModule(Module &M) override {
+ return DataFlowSanitizer(ABIListFiles).runImpl(M);
+ }
+};
+} // namespace
+
+char DataFlowSanitizerLegacyPass::ID;
+
+INITIALIZE_PASS(DataFlowSanitizerLegacyPass, "dfsan",
+ "DataFlowSanitizer: dynamic data flow analysis.", false, false)
+
+ModulePass *llvm::createDataFlowSanitizerLegacyPassPass(
+ const std::vector<std::string> &ABIListFiles) {
+ return new DataFlowSanitizerLegacyPass(ABIListFiles);
+}
+
+PreservedAnalyses DataFlowSanitizerPass::run(Module &M,
+ ModuleAnalysisManager &AM) {
+ if (DataFlowSanitizer(ABIListFiles).runImpl(M)) {
+ return PreservedAnalyses::none();
+ }
+ return PreservedAnalyses::all();
+}
diff --git a/contrib/libs/llvm12/lib/Transforms/Instrumentation/GCOVProfiling.cpp b/contrib/libs/llvm12/lib/Transforms/Instrumentation/GCOVProfiling.cpp
index 2d52317d0d..527644a69d 100644
--- a/contrib/libs/llvm12/lib/Transforms/Instrumentation/GCOVProfiling.cpp
+++ b/contrib/libs/llvm12/lib/Transforms/Instrumentation/GCOVProfiling.cpp
@@ -13,17 +13,17 @@
//
//===----------------------------------------------------------------------===//
-#include "CFGMST.h"
+#include "CFGMST.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/Hashing.h"
-#include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/Sequence.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringMap.h"
-#include "llvm/Analysis/BlockFrequencyInfo.h"
-#include "llvm/Analysis/BranchProbabilityInfo.h"
+#include "llvm/Analysis/BlockFrequencyInfo.h"
+#include "llvm/Analysis/BranchProbabilityInfo.h"
#include "llvm/Analysis/EHPersonalities.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/IR/CFG.h"
@@ -36,7 +36,7 @@
#include "llvm/IR/Module.h"
#include "llvm/InitializePasses.h"
#include "llvm/Pass.h"
-#include "llvm/Support/CRC.h"
+#include "llvm/Support/CRC.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/FileSystem.h"
@@ -57,8 +57,8 @@ namespace endian = llvm::support::endian;
#define DEBUG_TYPE "insert-gcov-profiling"
enum : uint32_t {
- GCOV_ARC_ON_TREE = 1 << 0,
-
+ GCOV_ARC_ON_TREE = 1 << 0,
+
GCOV_TAG_FUNCTION = 0x01000000,
GCOV_TAG_BLOCKS = 0x01410000,
GCOV_TAG_ARCS = 0x01430000,
@@ -69,9 +69,9 @@ static cl::opt<std::string> DefaultGCOVVersion("default-gcov-version",
cl::init("408*"), cl::Hidden,
cl::ValueRequired);
-static cl::opt<bool> AtomicCounter("gcov-atomic-counter", cl::Hidden,
- cl::desc("Make counter updates atomic"));
-
+static cl::opt<bool> AtomicCounter("gcov-atomic-counter", cl::Hidden,
+ cl::desc("Make counter updates atomic"));
+
// Returns the number of words which will be used to represent this string.
static unsigned wordsOfString(StringRef s) {
// Length + NUL-terminated string + 0~3 padding NULs.
@@ -83,7 +83,7 @@ GCOVOptions GCOVOptions::getDefault() {
Options.EmitNotes = true;
Options.EmitData = true;
Options.NoRedZone = false;
- Options.Atomic = AtomicCounter;
+ Options.Atomic = AtomicCounter;
if (DefaultGCOVVersion.size() != 4) {
llvm::report_fatal_error(std::string("Invalid -default-gcov-version: ") +
@@ -101,8 +101,8 @@ public:
GCOVProfiler() : GCOVProfiler(GCOVOptions::getDefault()) {}
GCOVProfiler(const GCOVOptions &Opts) : Options(Opts) {}
bool
- runOnModule(Module &M, function_ref<BlockFrequencyInfo *(Function &F)> GetBFI,
- function_ref<BranchProbabilityInfo *(Function &F)> GetBPI,
+ runOnModule(Module &M, function_ref<BlockFrequencyInfo *(Function &F)> GetBFI,
+ function_ref<BranchProbabilityInfo *(Function &F)> GetBPI,
std::function<const TargetLibraryInfo &(Function &F)> GetTLI);
void write(uint32_t i) {
@@ -119,14 +119,14 @@ public:
private:
// Create the .gcno files for the Module based on DebugInfo.
- bool
- emitProfileNotes(NamedMDNode *CUNode, bool HasExecOrFork,
- function_ref<BlockFrequencyInfo *(Function &F)> GetBFI,
- function_ref<BranchProbabilityInfo *(Function &F)> GetBPI,
- function_ref<const TargetLibraryInfo &(Function &F)> GetTLI);
+ bool
+ emitProfileNotes(NamedMDNode *CUNode, bool HasExecOrFork,
+ function_ref<BlockFrequencyInfo *(Function &F)> GetBFI,
+ function_ref<BranchProbabilityInfo *(Function &F)> GetBPI,
+ function_ref<const TargetLibraryInfo &(Function &F)> GetTLI);
- void emitGlobalConstructor(
- SmallVectorImpl<std::pair<GlobalVariable *, MDNode *>> &CountersBySP);
+ void emitGlobalConstructor(
+ SmallVectorImpl<std::pair<GlobalVariable *, MDNode *>> &CountersBySP);
bool isFunctionInstrumented(const Function &F);
std::vector<Regex> createRegexesFromString(StringRef RegexesStr);
@@ -164,7 +164,7 @@ private:
SmallVector<std::unique_ptr<GCOVFunction>, 16> Funcs;
std::vector<Regex> FilterRe;
std::vector<Regex> ExcludeRe;
- DenseSet<const BasicBlock *> ExecBlocks;
+ DenseSet<const BasicBlock *> ExecBlocks;
StringMap<bool> InstrumentedFiles;
};
@@ -180,68 +180,68 @@ public:
StringRef getPassName() const override { return "GCOV Profiler"; }
bool runOnModule(Module &M) override {
- auto GetBFI = [this](Function &F) {
- return &this->getAnalysis<BlockFrequencyInfoWrapperPass>(F).getBFI();
- };
- auto GetBPI = [this](Function &F) {
- return &this->getAnalysis<BranchProbabilityInfoWrapperPass>(F).getBPI();
- };
- auto GetTLI = [this](Function &F) -> const TargetLibraryInfo & {
- return this->getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
- };
- return Profiler.runOnModule(M, GetBFI, GetBPI, GetTLI);
+ auto GetBFI = [this](Function &F) {
+ return &this->getAnalysis<BlockFrequencyInfoWrapperPass>(F).getBFI();
+ };
+ auto GetBPI = [this](Function &F) {
+ return &this->getAnalysis<BranchProbabilityInfoWrapperPass>(F).getBPI();
+ };
+ auto GetTLI = [this](Function &F) -> const TargetLibraryInfo & {
+ return this->getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
+ };
+ return Profiler.runOnModule(M, GetBFI, GetBPI, GetTLI);
}
void getAnalysisUsage(AnalysisUsage &AU) const override {
- AU.addRequired<BlockFrequencyInfoWrapperPass>();
+ AU.addRequired<BlockFrequencyInfoWrapperPass>();
AU.addRequired<TargetLibraryInfoWrapperPass>();
}
private:
GCOVProfiler Profiler;
};
-
-struct BBInfo {
- BBInfo *Group;
- uint32_t Index;
- uint32_t Rank = 0;
-
- BBInfo(unsigned Index) : Group(this), Index(Index) {}
- const std::string infoString() const {
- return (Twine("Index=") + Twine(Index)).str();
- }
-};
-
-struct Edge {
- // This class implements the CFG edges. Note the CFG can be a multi-graph.
- // So there might be multiple edges with same SrcBB and DestBB.
- const BasicBlock *SrcBB;
- const BasicBlock *DestBB;
- uint64_t Weight;
- BasicBlock *Place = nullptr;
- uint32_t SrcNumber, DstNumber;
- bool InMST = false;
- bool Removed = false;
- bool IsCritical = false;
-
- Edge(const BasicBlock *Src, const BasicBlock *Dest, uint64_t W = 1)
- : SrcBB(Src), DestBB(Dest), Weight(W) {}
-
- // Return the information string of an edge.
- const std::string infoString() const {
- return (Twine(Removed ? "-" : " ") + (InMST ? " " : "*") +
- (IsCritical ? "c" : " ") + " W=" + Twine(Weight))
- .str();
- }
-};
+
+struct BBInfo {
+ BBInfo *Group;
+ uint32_t Index;
+ uint32_t Rank = 0;
+
+ BBInfo(unsigned Index) : Group(this), Index(Index) {}
+ const std::string infoString() const {
+ return (Twine("Index=") + Twine(Index)).str();
+ }
+};
+
+struct Edge {
+ // This class implements the CFG edges. Note the CFG can be a multi-graph.
+ // So there might be multiple edges with same SrcBB and DestBB.
+ const BasicBlock *SrcBB;
+ const BasicBlock *DestBB;
+ uint64_t Weight;
+ BasicBlock *Place = nullptr;
+ uint32_t SrcNumber, DstNumber;
+ bool InMST = false;
+ bool Removed = false;
+ bool IsCritical = false;
+
+ Edge(const BasicBlock *Src, const BasicBlock *Dest, uint64_t W = 1)
+ : SrcBB(Src), DestBB(Dest), Weight(W) {}
+
+ // Return the information string of an edge.
+ const std::string infoString() const {
+ return (Twine(Removed ? "-" : " ") + (InMST ? " " : "*") +
+ (IsCritical ? "c" : " ") + " W=" + Twine(Weight))
+ .str();
+ }
+};
}
char GCOVProfilerLegacyPass::ID = 0;
INITIALIZE_PASS_BEGIN(
GCOVProfilerLegacyPass, "insert-gcov-profiling",
"Insert instrumentation for GCOV profiling", false, false)
-INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass)
-INITIALIZE_PASS_DEPENDENCY(BranchProbabilityInfoWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(BranchProbabilityInfoWrapperPass)
INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
INITIALIZE_PASS_END(
GCOVProfilerLegacyPass, "insert-gcov-profiling",
@@ -326,8 +326,8 @@ namespace {
return LinesByFile.try_emplace(Filename, P, Filename).first->second;
}
- void addEdge(GCOVBlock &Successor, uint32_t Flags) {
- OutEdges.emplace_back(&Successor, Flags);
+ void addEdge(GCOVBlock &Successor, uint32_t Flags) {
+ OutEdges.emplace_back(&Successor, Flags);
}
void writeOut() {
@@ -360,10 +360,10 @@ namespace {
assert(OutEdges.empty());
}
- uint32_t Number;
- SmallVector<std::pair<GCOVBlock *, uint32_t>, 4> OutEdges;
-
- private:
+ uint32_t Number;
+ SmallVector<std::pair<GCOVBlock *, uint32_t>, 4> OutEdges;
+
+ private:
friend class GCOVFunction;
GCOVBlock(GCOVProfiler *P, uint32_t Number)
@@ -380,11 +380,11 @@ namespace {
GCOVFunction(GCOVProfiler *P, Function *F, const DISubprogram *SP,
unsigned EndLine, uint32_t Ident, int Version)
: GCOVRecord(P), SP(SP), EndLine(EndLine), Ident(Ident),
- Version(Version), EntryBlock(P, 0), ReturnBlock(P, 1) {
+ Version(Version), EntryBlock(P, 0), ReturnBlock(P, 1) {
LLVM_DEBUG(dbgs() << "Function: " << getFunctionName(SP) << "\n");
bool ExitBlockBeforeBody = Version >= 48;
- uint32_t i = ExitBlockBeforeBody ? 2 : 1;
- for (BasicBlock &BB : *F)
+ uint32_t i = ExitBlockBeforeBody ? 2 : 1;
+ for (BasicBlock &BB : *F)
Blocks.insert(std::make_pair(&BB, GCOVBlock(P, i++)));
if (!ExitBlockBeforeBody)
ReturnBlock.Number = i;
@@ -396,11 +396,11 @@ namespace {
FuncChecksum = hash_value(FunctionNameAndLine);
}
- GCOVBlock &getBlock(const BasicBlock *BB) {
- return Blocks.find(const_cast<BasicBlock *>(BB))->second;
+ GCOVBlock &getBlock(const BasicBlock *BB) {
+ return Blocks.find(const_cast<BasicBlock *>(BB))->second;
}
- GCOVBlock &getEntryBlock() { return EntryBlock; }
+ GCOVBlock &getEntryBlock() { return EntryBlock; }
GCOVBlock &getReturnBlock() {
return ReturnBlock;
}
@@ -443,52 +443,52 @@ namespace {
// Emit count of blocks.
write(GCOV_TAG_BLOCKS);
if (Version < 80) {
- write(Blocks.size() + 2);
- for (int i = Blocks.size() + 2; i; --i)
+ write(Blocks.size() + 2);
+ for (int i = Blocks.size() + 2; i; --i)
write(0);
} else {
write(1);
- write(Blocks.size() + 2);
+ write(Blocks.size() + 2);
}
LLVM_DEBUG(dbgs() << (Blocks.size() + 1) << " blocks\n");
// Emit edges between blocks.
- const uint32_t Outgoing = EntryBlock.OutEdges.size();
- if (Outgoing) {
- write(GCOV_TAG_ARCS);
- write(Outgoing * 2 + 1);
- write(EntryBlock.Number);
- for (const auto &E : EntryBlock.OutEdges) {
- write(E.first->Number);
- write(E.second);
- }
- }
- for (auto &It : Blocks) {
- const GCOVBlock &Block = It.second;
+ const uint32_t Outgoing = EntryBlock.OutEdges.size();
+ if (Outgoing) {
+ write(GCOV_TAG_ARCS);
+ write(Outgoing * 2 + 1);
+ write(EntryBlock.Number);
+ for (const auto &E : EntryBlock.OutEdges) {
+ write(E.first->Number);
+ write(E.second);
+ }
+ }
+ for (auto &It : Blocks) {
+ const GCOVBlock &Block = It.second;
if (Block.OutEdges.empty()) continue;
write(GCOV_TAG_ARCS);
write(Block.OutEdges.size() * 2 + 1);
write(Block.Number);
- for (const auto &E : Block.OutEdges) {
- write(E.first->Number);
- write(E.second);
+ for (const auto &E : Block.OutEdges) {
+ write(E.first->Number);
+ write(E.second);
}
}
// Emit lines for each block.
- for (auto &It : Blocks)
- It.second.writeOut();
+ for (auto &It : Blocks)
+ It.second.writeOut();
}
- public:
+ public:
const DISubprogram *SP;
unsigned EndLine;
uint32_t Ident;
uint32_t FuncChecksum;
int Version;
- MapVector<BasicBlock *, GCOVBlock> Blocks;
- GCOVBlock EntryBlock;
+ MapVector<BasicBlock *, GCOVBlock> Blocks;
+ GCOVBlock EntryBlock;
GCOVBlock ReturnBlock;
};
}
@@ -602,23 +602,23 @@ std::string GCOVProfiler::mangleName(const DICompileUnit *CU,
}
bool GCOVProfiler::runOnModule(
- Module &M, function_ref<BlockFrequencyInfo *(Function &F)> GetBFI,
- function_ref<BranchProbabilityInfo *(Function &F)> GetBPI,
- std::function<const TargetLibraryInfo &(Function &F)> GetTLI) {
+ Module &M, function_ref<BlockFrequencyInfo *(Function &F)> GetBFI,
+ function_ref<BranchProbabilityInfo *(Function &F)> GetBPI,
+ std::function<const TargetLibraryInfo &(Function &F)> GetTLI) {
this->M = &M;
this->GetTLI = std::move(GetTLI);
Ctx = &M.getContext();
- NamedMDNode *CUNode = M.getNamedMetadata("llvm.dbg.cu");
- if (!CUNode || (!Options.EmitNotes && !Options.EmitData))
- return false;
+ NamedMDNode *CUNode = M.getNamedMetadata("llvm.dbg.cu");
+ if (!CUNode || (!Options.EmitNotes && !Options.EmitData))
+ return false;
+
+ bool HasExecOrFork = AddFlushBeforeForkAndExec();
- bool HasExecOrFork = AddFlushBeforeForkAndExec();
-
FilterRe = createRegexesFromString(Options.Filter);
ExcludeRe = createRegexesFromString(Options.Exclude);
- emitProfileNotes(CUNode, HasExecOrFork, GetBFI, GetBPI, this->GetTLI);
- return true;
+ emitProfileNotes(CUNode, HasExecOrFork, GetBFI, GetBPI, this->GetTLI);
+ return true;
}
PreservedAnalyses GCOVProfilerPass::run(Module &M,
@@ -628,17 +628,17 @@ PreservedAnalyses GCOVProfilerPass::run(Module &M,
FunctionAnalysisManager &FAM =
AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
- auto GetBFI = [&FAM](Function &F) {
- return &FAM.getResult<BlockFrequencyAnalysis>(F);
- };
- auto GetBPI = [&FAM](Function &F) {
- return &FAM.getResult<BranchProbabilityAnalysis>(F);
- };
- auto GetTLI = [&FAM](Function &F) -> const TargetLibraryInfo & {
- return FAM.getResult<TargetLibraryAnalysis>(F);
- };
-
- if (!Profiler.runOnModule(M, GetBFI, GetBPI, GetTLI))
+ auto GetBFI = [&FAM](Function &F) {
+ return &FAM.getResult<BlockFrequencyAnalysis>(F);
+ };
+ auto GetBPI = [&FAM](Function &F) {
+ return &FAM.getResult<BranchProbabilityAnalysis>(F);
+ };
+ auto GetTLI = [&FAM](Function &F) -> const TargetLibraryInfo & {
+ return FAM.getResult<TargetLibraryAnalysis>(F);
+ };
+
+ if (!Profiler.runOnModule(M, GetBFI, GetBPI, GetTLI))
return PreservedAnalyses::all();
return PreservedAnalyses::none();
@@ -744,7 +744,7 @@ bool GCOVProfiler::AddFlushBeforeForkAndExec() {
// dumped
FunctionCallee ResetF = M->getOrInsertFunction("llvm_reset_counters", FTy);
Builder.CreateCall(ResetF)->setDebugLoc(Loc);
- ExecBlocks.insert(Parent);
+ ExecBlocks.insert(Parent);
Parent->splitBasicBlock(NextInst);
Parent->back().setDebugLoc(Loc);
}
@@ -752,67 +752,67 @@ bool GCOVProfiler::AddFlushBeforeForkAndExec() {
return !Forks.empty() || !Execs.empty();
}
-static BasicBlock *getInstrBB(CFGMST<Edge, BBInfo> &MST, Edge &E,
- const DenseSet<const BasicBlock *> &ExecBlocks) {
- if (E.InMST || E.Removed)
- return nullptr;
-
- BasicBlock *SrcBB = const_cast<BasicBlock *>(E.SrcBB);
- BasicBlock *DestBB = const_cast<BasicBlock *>(E.DestBB);
- // For a fake edge, instrument the real BB.
- if (SrcBB == nullptr)
- return DestBB;
- if (DestBB == nullptr)
- return SrcBB;
-
- auto CanInstrument = [](BasicBlock *BB) -> BasicBlock * {
- // There are basic blocks (such as catchswitch) cannot be instrumented.
- // If the returned first insertion point is the end of BB, skip this BB.
- if (BB->getFirstInsertionPt() == BB->end())
- return nullptr;
- return BB;
- };
-
- // Instrument the SrcBB if it has a single successor,
- // otherwise, the DestBB if this is not a critical edge.
- Instruction *TI = SrcBB->getTerminator();
- if (TI->getNumSuccessors() <= 1 && !ExecBlocks.count(SrcBB))
- return CanInstrument(SrcBB);
- if (!E.IsCritical)
- return CanInstrument(DestBB);
-
- // Some IndirectBr critical edges cannot be split by the previous
- // SplitIndirectBrCriticalEdges call. Bail out.
- const unsigned SuccNum = GetSuccessorNumber(SrcBB, DestBB);
- BasicBlock *InstrBB =
- isa<IndirectBrInst>(TI) ? nullptr : SplitCriticalEdge(TI, SuccNum);
- if (!InstrBB)
- return nullptr;
-
- MST.addEdge(SrcBB, InstrBB, 0);
- MST.addEdge(InstrBB, DestBB, 0).InMST = true;
- E.Removed = true;
-
- return CanInstrument(InstrBB);
-}
-
-#ifndef NDEBUG
-static void dumpEdges(CFGMST<Edge, BBInfo> &MST, GCOVFunction &GF) {
- size_t ID = 0;
- for (auto &E : make_pointee_range(MST.AllEdges)) {
- GCOVBlock &Src = E.SrcBB ? GF.getBlock(E.SrcBB) : GF.getEntryBlock();
- GCOVBlock &Dst = E.DestBB ? GF.getBlock(E.DestBB) : GF.getReturnBlock();
- dbgs() << " Edge " << ID++ << ": " << Src.Number << "->" << Dst.Number
- << E.infoString() << "\n";
- }
-}
-#endif
-
-bool GCOVProfiler::emitProfileNotes(
- NamedMDNode *CUNode, bool HasExecOrFork,
- function_ref<BlockFrequencyInfo *(Function &F)> GetBFI,
- function_ref<BranchProbabilityInfo *(Function &F)> GetBPI,
- function_ref<const TargetLibraryInfo &(Function &F)> GetTLI) {
+static BasicBlock *getInstrBB(CFGMST<Edge, BBInfo> &MST, Edge &E,
+ const DenseSet<const BasicBlock *> &ExecBlocks) {
+ if (E.InMST || E.Removed)
+ return nullptr;
+
+ BasicBlock *SrcBB = const_cast<BasicBlock *>(E.SrcBB);
+ BasicBlock *DestBB = const_cast<BasicBlock *>(E.DestBB);
+ // For a fake edge, instrument the real BB.
+ if (SrcBB == nullptr)
+ return DestBB;
+ if (DestBB == nullptr)
+ return SrcBB;
+
+ auto CanInstrument = [](BasicBlock *BB) -> BasicBlock * {
+ // There are basic blocks (such as catchswitch) cannot be instrumented.
+ // If the returned first insertion point is the end of BB, skip this BB.
+ if (BB->getFirstInsertionPt() == BB->end())
+ return nullptr;
+ return BB;
+ };
+
+ // Instrument the SrcBB if it has a single successor,
+ // otherwise, the DestBB if this is not a critical edge.
+ Instruction *TI = SrcBB->getTerminator();
+ if (TI->getNumSuccessors() <= 1 && !ExecBlocks.count(SrcBB))
+ return CanInstrument(SrcBB);
+ if (!E.IsCritical)
+ return CanInstrument(DestBB);
+
+ // Some IndirectBr critical edges cannot be split by the previous
+ // SplitIndirectBrCriticalEdges call. Bail out.
+ const unsigned SuccNum = GetSuccessorNumber(SrcBB, DestBB);
+ BasicBlock *InstrBB =
+ isa<IndirectBrInst>(TI) ? nullptr : SplitCriticalEdge(TI, SuccNum);
+ if (!InstrBB)
+ return nullptr;
+
+ MST.addEdge(SrcBB, InstrBB, 0);
+ MST.addEdge(InstrBB, DestBB, 0).InMST = true;
+ E.Removed = true;
+
+ return CanInstrument(InstrBB);
+}
+
+#ifndef NDEBUG
+static void dumpEdges(CFGMST<Edge, BBInfo> &MST, GCOVFunction &GF) {
+ size_t ID = 0;
+ for (auto &E : make_pointee_range(MST.AllEdges)) {
+ GCOVBlock &Src = E.SrcBB ? GF.getBlock(E.SrcBB) : GF.getEntryBlock();
+ GCOVBlock &Dst = E.DestBB ? GF.getBlock(E.DestBB) : GF.getReturnBlock();
+ dbgs() << " Edge " << ID++ << ": " << Src.Number << "->" << Dst.Number
+ << E.infoString() << "\n";
+ }
+}
+#endif
+
+bool GCOVProfiler::emitProfileNotes(
+ NamedMDNode *CUNode, bool HasExecOrFork,
+ function_ref<BlockFrequencyInfo *(Function &F)> GetBFI,
+ function_ref<BranchProbabilityInfo *(Function &F)> GetBPI,
+ function_ref<const TargetLibraryInfo &(Function &F)> GetTLI) {
int Version;
{
uint8_t c3 = Options.Version[0];
@@ -822,20 +822,20 @@ bool GCOVProfiler::emitProfileNotes(
: (c3 - '0') * 10 + c1 - '0';
}
- bool EmitGCDA = Options.EmitData;
- for (unsigned i = 0, e = CUNode->getNumOperands(); i != e; ++i) {
+ bool EmitGCDA = Options.EmitData;
+ for (unsigned i = 0, e = CUNode->getNumOperands(); i != e; ++i) {
// Each compile unit gets its own .gcno file. This means that whether we run
// this pass over the original .o's as they're produced, or run it after
// LTO, we'll generate the same .gcno files.
- auto *CU = cast<DICompileUnit>(CUNode->getOperand(i));
+ auto *CU = cast<DICompileUnit>(CUNode->getOperand(i));
// Skip module skeleton (and module) CUs.
if (CU->getDWOId())
continue;
- std::vector<uint8_t> EdgeDestinations;
- SmallVector<std::pair<GlobalVariable *, MDNode *>, 8> CountersBySP;
+ std::vector<uint8_t> EdgeDestinations;
+ SmallVector<std::pair<GlobalVariable *, MDNode *>, 8> CountersBySP;
Endian = M->getDataLayout().isLittleEndian() ? support::endianness::little
: support::endianness::big;
@@ -849,82 +849,82 @@ bool GCOVProfiler::emitProfileNotes(
// TODO: Functions using scope-based EH are currently not supported.
if (isUsingScopeBasedEH(F)) continue;
- // Add the function line number to the lines of the entry block
- // to have a counter for the function definition.
- uint32_t Line = SP->getLine();
- auto Filename = getFilename(SP);
-
- BranchProbabilityInfo *BPI = GetBPI(F);
- BlockFrequencyInfo *BFI = GetBFI(F);
-
- // Split indirectbr critical edges here before computing the MST rather
- // than later in getInstrBB() to avoid invalidating it.
- SplitIndirectBrCriticalEdges(F, BPI, BFI);
-
- CFGMST<Edge, BBInfo> MST(F, /*InstrumentFuncEntry_=*/false, BPI, BFI);
-
- // getInstrBB can split basic blocks and push elements to AllEdges.
- for (size_t I : llvm::seq<size_t>(0, MST.AllEdges.size())) {
- auto &E = *MST.AllEdges[I];
- // For now, disable spanning tree optimization when fork or exec* is
- // used.
- if (HasExecOrFork)
- E.InMST = false;
- E.Place = getInstrBB(MST, E, ExecBlocks);
- }
- // Basic blocks in F are finalized at this point.
+ // Add the function line number to the lines of the entry block
+ // to have a counter for the function definition.
+ uint32_t Line = SP->getLine();
+ auto Filename = getFilename(SP);
+
+ BranchProbabilityInfo *BPI = GetBPI(F);
+ BlockFrequencyInfo *BFI = GetBFI(F);
+
+ // Split indirectbr critical edges here before computing the MST rather
+ // than later in getInstrBB() to avoid invalidating it.
+ SplitIndirectBrCriticalEdges(F, BPI, BFI);
+
+ CFGMST<Edge, BBInfo> MST(F, /*InstrumentFuncEntry_=*/false, BPI, BFI);
+
+ // getInstrBB can split basic blocks and push elements to AllEdges.
+ for (size_t I : llvm::seq<size_t>(0, MST.AllEdges.size())) {
+ auto &E = *MST.AllEdges[I];
+ // For now, disable spanning tree optimization when fork or exec* is
+ // used.
+ if (HasExecOrFork)
+ E.InMST = false;
+ E.Place = getInstrBB(MST, E, ExecBlocks);
+ }
+ // Basic blocks in F are finalized at this point.
BasicBlock &EntryBlock = F.getEntryBlock();
Funcs.push_back(std::make_unique<GCOVFunction>(this, &F, SP, EndLine,
FunctionIdent++, Version));
GCOVFunction &Func = *Funcs.back();
- // Some non-tree edges are IndirectBr which cannot be split. Ignore them
- // as well.
- llvm::erase_if(MST.AllEdges, [](std::unique_ptr<Edge> &E) {
- return E->Removed || (!E->InMST && !E->Place);
- });
- const size_t Measured =
- std::stable_partition(
- MST.AllEdges.begin(), MST.AllEdges.end(),
- [](std::unique_ptr<Edge> &E) { return E->Place; }) -
- MST.AllEdges.begin();
- for (size_t I : llvm::seq<size_t>(0, Measured)) {
- Edge &E = *MST.AllEdges[I];
- GCOVBlock &Src =
- E.SrcBB ? Func.getBlock(E.SrcBB) : Func.getEntryBlock();
- GCOVBlock &Dst =
- E.DestBB ? Func.getBlock(E.DestBB) : Func.getReturnBlock();
- E.SrcNumber = Src.Number;
- E.DstNumber = Dst.Number;
- }
- std::stable_sort(
- MST.AllEdges.begin(), MST.AllEdges.begin() + Measured,
- [](const std::unique_ptr<Edge> &L, const std::unique_ptr<Edge> &R) {
- return L->SrcNumber != R->SrcNumber ? L->SrcNumber < R->SrcNumber
- : L->DstNumber < R->DstNumber;
- });
-
- for (const Edge &E : make_pointee_range(MST.AllEdges)) {
- GCOVBlock &Src =
- E.SrcBB ? Func.getBlock(E.SrcBB) : Func.getEntryBlock();
- GCOVBlock &Dst =
- E.DestBB ? Func.getBlock(E.DestBB) : Func.getReturnBlock();
- Src.addEdge(Dst, E.Place ? 0 : uint32_t(GCOV_ARC_ON_TREE));
- }
-
+ // Some non-tree edges are IndirectBr which cannot be split. Ignore them
+ // as well.
+ llvm::erase_if(MST.AllEdges, [](std::unique_ptr<Edge> &E) {
+ return E->Removed || (!E->InMST && !E->Place);
+ });
+ const size_t Measured =
+ std::stable_partition(
+ MST.AllEdges.begin(), MST.AllEdges.end(),
+ [](std::unique_ptr<Edge> &E) { return E->Place; }) -
+ MST.AllEdges.begin();
+ for (size_t I : llvm::seq<size_t>(0, Measured)) {
+ Edge &E = *MST.AllEdges[I];
+ GCOVBlock &Src =
+ E.SrcBB ? Func.getBlock(E.SrcBB) : Func.getEntryBlock();
+ GCOVBlock &Dst =
+ E.DestBB ? Func.getBlock(E.DestBB) : Func.getReturnBlock();
+ E.SrcNumber = Src.Number;
+ E.DstNumber = Dst.Number;
+ }
+ std::stable_sort(
+ MST.AllEdges.begin(), MST.AllEdges.begin() + Measured,
+ [](const std::unique_ptr<Edge> &L, const std::unique_ptr<Edge> &R) {
+ return L->SrcNumber != R->SrcNumber ? L->SrcNumber < R->SrcNumber
+ : L->DstNumber < R->DstNumber;
+ });
+
+ for (const Edge &E : make_pointee_range(MST.AllEdges)) {
+ GCOVBlock &Src =
+ E.SrcBB ? Func.getBlock(E.SrcBB) : Func.getEntryBlock();
+ GCOVBlock &Dst =
+ E.DestBB ? Func.getBlock(E.DestBB) : Func.getReturnBlock();
+ Src.addEdge(Dst, E.Place ? 0 : uint32_t(GCOV_ARC_ON_TREE));
+ }
+
// Artificial functions such as global initializers
if (!SP->isArtificial())
Func.getBlock(&EntryBlock).getFile(Filename).addLine(Line);
- LLVM_DEBUG(dumpEdges(MST, Func));
-
- for (auto &GB : Func.Blocks) {
- const BasicBlock &BB = *GB.first;
- auto &Block = GB.second;
- for (auto Succ : Block.OutEdges) {
- uint32_t Idx = Succ.first->Number;
- do EdgeDestinations.push_back(Idx & 255);
- while ((Idx >>= 8) > 0);
+ LLVM_DEBUG(dumpEdges(MST, Func));
+
+ for (auto &GB : Func.Blocks) {
+ const BasicBlock &BB = *GB.first;
+ auto &Block = GB.second;
+ for (auto Succ : Block.OutEdges) {
+ uint32_t Idx = Succ.first->Number;
+ do EdgeDestinations.push_back(Idx & 255);
+ while ((Idx >>= 8) > 0);
}
for (auto &I : BB) {
@@ -950,110 +950,110 @@ bool GCOVProfiler::emitProfileNotes(
}
Line = 0;
}
- if (EmitGCDA) {
- DISubprogram *SP = F.getSubprogram();
- ArrayType *CounterTy = ArrayType::get(Type::getInt64Ty(*Ctx), Measured);
- GlobalVariable *Counters = new GlobalVariable(
- *M, CounterTy, false, GlobalValue::InternalLinkage,
- Constant::getNullValue(CounterTy), "__llvm_gcov_ctr");
- CountersBySP.emplace_back(Counters, SP);
-
- for (size_t I : llvm::seq<size_t>(0, Measured)) {
- const Edge &E = *MST.AllEdges[I];
- IRBuilder<> Builder(E.Place, E.Place->getFirstInsertionPt());
- Value *V = Builder.CreateConstInBoundsGEP2_64(
- Counters->getValueType(), Counters, 0, I);
- if (Options.Atomic) {
- Builder.CreateAtomicRMW(AtomicRMWInst::Add, V, Builder.getInt64(1),
- AtomicOrdering::Monotonic);
- } else {
- Value *Count =
- Builder.CreateLoad(Builder.getInt64Ty(), V, "gcov_ctr");
- Count = Builder.CreateAdd(Count, Builder.getInt64(1));
- Builder.CreateStore(Count, V);
- }
- }
- }
+ if (EmitGCDA) {
+ DISubprogram *SP = F.getSubprogram();
+ ArrayType *CounterTy = ArrayType::get(Type::getInt64Ty(*Ctx), Measured);
+ GlobalVariable *Counters = new GlobalVariable(
+ *M, CounterTy, false, GlobalValue::InternalLinkage,
+ Constant::getNullValue(CounterTy), "__llvm_gcov_ctr");
+ CountersBySP.emplace_back(Counters, SP);
+
+ for (size_t I : llvm::seq<size_t>(0, Measured)) {
+ const Edge &E = *MST.AllEdges[I];
+ IRBuilder<> Builder(E.Place, E.Place->getFirstInsertionPt());
+ Value *V = Builder.CreateConstInBoundsGEP2_64(
+ Counters->getValueType(), Counters, 0, I);
+ if (Options.Atomic) {
+ Builder.CreateAtomicRMW(AtomicRMWInst::Add, V, Builder.getInt64(1),
+ AtomicOrdering::Monotonic);
+ } else {
+ Value *Count =
+ Builder.CreateLoad(Builder.getInt64Ty(), V, "gcov_ctr");
+ Count = Builder.CreateAdd(Count, Builder.getInt64(1));
+ Builder.CreateStore(Count, V);
+ }
+ }
+ }
}
char Tmp[4];
- JamCRC JC;
- JC.update(EdgeDestinations);
- uint32_t Stamp = JC.getCRC();
+ JamCRC JC;
+ JC.update(EdgeDestinations);
+ uint32_t Stamp = JC.getCRC();
FileChecksums.push_back(Stamp);
- if (Options.EmitNotes) {
- std::error_code EC;
- raw_fd_ostream out(mangleName(CU, GCovFileType::GCNO), EC,
- sys::fs::OF_None);
- if (EC) {
- Ctx->emitError(
- Twine("failed to open coverage notes file for writing: ") +
- EC.message());
+ if (Options.EmitNotes) {
+ std::error_code EC;
+ raw_fd_ostream out(mangleName(CU, GCovFileType::GCNO), EC,
+ sys::fs::OF_None);
+ if (EC) {
+ Ctx->emitError(
+ Twine("failed to open coverage notes file for writing: ") +
+ EC.message());
continue;
}
- os = &out;
- if (Endian == support::endianness::big) {
- out.write("gcno", 4);
- out.write(Options.Version, 4);
- } else {
- out.write("oncg", 4);
- std::reverse_copy(Options.Version, Options.Version + 4, Tmp);
- out.write(Tmp, 4);
- }
- write(Stamp);
- if (Version >= 90)
- writeString(""); // unuseful current_working_directory
- if (Version >= 80)
- write(0); // unuseful has_unexecuted_blocks
-
- for (auto &Func : Funcs)
- Func->writeOut(Stamp);
-
- write(0);
- write(0);
- out.close();
- }
-
- if (EmitGCDA) {
- emitGlobalConstructor(CountersBySP);
- EmitGCDA = false;
+ os = &out;
+ if (Endian == support::endianness::big) {
+ out.write("gcno", 4);
+ out.write(Options.Version, 4);
+ } else {
+ out.write("oncg", 4);
+ std::reverse_copy(Options.Version, Options.Version + 4, Tmp);
+ out.write(Tmp, 4);
+ }
+ write(Stamp);
+ if (Version >= 90)
+ writeString(""); // unuseful current_working_directory
+ if (Version >= 80)
+ write(0); // unuseful has_unexecuted_blocks
+
+ for (auto &Func : Funcs)
+ Func->writeOut(Stamp);
+
+ write(0);
+ write(0);
+ out.close();
+ }
+
+ if (EmitGCDA) {
+ emitGlobalConstructor(CountersBySP);
+ EmitGCDA = false;
}
- }
- return true;
-}
-
-void GCOVProfiler::emitGlobalConstructor(
- SmallVectorImpl<std::pair<GlobalVariable *, MDNode *>> &CountersBySP) {
- Function *WriteoutF = insertCounterWriteout(CountersBySP);
- Function *ResetF = insertReset(CountersBySP);
-
- // Create a small bit of code that registers the "__llvm_gcov_writeout" to
- // be executed at exit and the "__llvm_gcov_flush" function to be executed
- // when "__gcov_flush" is called.
- FunctionType *FTy = FunctionType::get(Type::getVoidTy(*Ctx), false);
- Function *F = Function::Create(FTy, GlobalValue::InternalLinkage,
- "__llvm_gcov_init", M);
- F->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
- F->setLinkage(GlobalValue::InternalLinkage);
- F->addFnAttr(Attribute::NoInline);
- if (Options.NoRedZone)
- F->addFnAttr(Attribute::NoRedZone);
-
- BasicBlock *BB = BasicBlock::Create(*Ctx, "entry", F);
- IRBuilder<> Builder(BB);
-
- FTy = FunctionType::get(Type::getVoidTy(*Ctx), false);
- auto *PFTy = PointerType::get(FTy, 0);
- FTy = FunctionType::get(Builder.getVoidTy(), {PFTy, PFTy}, false);
-
- // Initialize the environment and register the local writeout, flush and
- // reset functions.
- FunctionCallee GCOVInit = M->getOrInsertFunction("llvm_gcov_init", FTy);
- Builder.CreateCall(GCOVInit, {WriteoutF, ResetF});
- Builder.CreateRetVoid();
-
- appendToGlobalCtors(*M, F, 0);
+ }
+ return true;
+}
+
+void GCOVProfiler::emitGlobalConstructor(
+ SmallVectorImpl<std::pair<GlobalVariable *, MDNode *>> &CountersBySP) {
+ Function *WriteoutF = insertCounterWriteout(CountersBySP);
+ Function *ResetF = insertReset(CountersBySP);
+
+ // Create a small bit of code that registers the "__llvm_gcov_writeout" to
+ // be executed at exit and the "__llvm_gcov_flush" function to be executed
+ // when "__gcov_flush" is called.
+ FunctionType *FTy = FunctionType::get(Type::getVoidTy(*Ctx), false);
+ Function *F = Function::Create(FTy, GlobalValue::InternalLinkage,
+ "__llvm_gcov_init", M);
+ F->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
+ F->setLinkage(GlobalValue::InternalLinkage);
+ F->addFnAttr(Attribute::NoInline);
+ if (Options.NoRedZone)
+ F->addFnAttr(Attribute::NoRedZone);
+
+ BasicBlock *BB = BasicBlock::Create(*Ctx, "entry", F);
+ IRBuilder<> Builder(BB);
+
+ FTy = FunctionType::get(Type::getVoidTy(*Ctx), false);
+ auto *PFTy = PointerType::get(FTy, 0);
+ FTy = FunctionType::get(Builder.getVoidTy(), {PFTy, PFTy}, false);
+
+ // Initialize the environment and register the local writeout, flush and
+ // reset functions.
+ FunctionCallee GCOVInit = M->getOrInsertFunction("llvm_gcov_init", FTy);
+ Builder.CreateCall(GCOVInit, {WriteoutF, ResetF});
+ Builder.CreateRetVoid();
+
+ appendToGlobalCtors(*M, F, 0);
}
FunctionCallee GCOVProfiler::getStartFileFunc(const TargetLibraryInfo *TLI) {
@@ -1140,19 +1140,19 @@ Function *GCOVProfiler::insertCounterWriteout(
// Collect the relevant data into a large constant data structure that we can
// walk to write out everything.
StructType *StartFileCallArgsTy = StructType::create(
- {Builder.getInt8PtrTy(), Builder.getInt32Ty(), Builder.getInt32Ty()},
- "start_file_args_ty");
+ {Builder.getInt8PtrTy(), Builder.getInt32Ty(), Builder.getInt32Ty()},
+ "start_file_args_ty");
StructType *EmitFunctionCallArgsTy = StructType::create(
- {Builder.getInt32Ty(), Builder.getInt32Ty(), Builder.getInt32Ty()},
- "emit_function_args_ty");
+ {Builder.getInt32Ty(), Builder.getInt32Ty(), Builder.getInt32Ty()},
+ "emit_function_args_ty");
StructType *EmitArcsCallArgsTy = StructType::create(
- {Builder.getInt32Ty(), Builder.getInt64Ty()->getPointerTo()},
- "emit_arcs_args_ty");
+ {Builder.getInt32Ty(), Builder.getInt64Ty()->getPointerTo()},
+ "emit_arcs_args_ty");
StructType *FileInfoTy =
StructType::create({StartFileCallArgsTy, Builder.getInt32Ty(),
EmitFunctionCallArgsTy->getPointerTo(),
- EmitArcsCallArgsTy->getPointerTo()},
- "file_info");
+ EmitArcsCallArgsTy->getPointerTo()},
+ "file_info");
Constant *Zero32 = Builder.getInt32(0);
// Build an explicit array of two zeros for use in ConstantExpr GEP building.
@@ -1262,46 +1262,46 @@ Function *GCOVProfiler::insertCounterWriteout(
// The index into the files structure is our loop induction variable.
Builder.SetInsertPoint(FileLoopHeader);
- PHINode *IV = Builder.CreatePHI(Builder.getInt32Ty(), /*NumReservedValues*/ 2,
- "file_idx");
+ PHINode *IV = Builder.CreatePHI(Builder.getInt32Ty(), /*NumReservedValues*/ 2,
+ "file_idx");
IV->addIncoming(Builder.getInt32(0), BB);
auto *FileInfoPtr = Builder.CreateInBoundsGEP(
FileInfoArrayTy, FileInfoArrayGV, {Builder.getInt32(0), IV});
auto *StartFileCallArgsPtr =
- Builder.CreateStructGEP(FileInfoTy, FileInfoPtr, 0, "start_file_args");
+ Builder.CreateStructGEP(FileInfoTy, FileInfoPtr, 0, "start_file_args");
auto *StartFileCall = Builder.CreateCall(
StartFile,
{Builder.CreateLoad(StartFileCallArgsTy->getElementType(0),
Builder.CreateStructGEP(StartFileCallArgsTy,
- StartFileCallArgsPtr, 0),
- "filename"),
+ StartFileCallArgsPtr, 0),
+ "filename"),
Builder.CreateLoad(StartFileCallArgsTy->getElementType(1),
Builder.CreateStructGEP(StartFileCallArgsTy,
- StartFileCallArgsPtr, 1),
- "version"),
+ StartFileCallArgsPtr, 1),
+ "version"),
Builder.CreateLoad(StartFileCallArgsTy->getElementType(2),
Builder.CreateStructGEP(StartFileCallArgsTy,
- StartFileCallArgsPtr, 2),
- "stamp")});
+ StartFileCallArgsPtr, 2),
+ "stamp")});
if (auto AK = TLI->getExtAttrForI32Param(false))
StartFileCall->addParamAttr(2, AK);
- auto *NumCounters = Builder.CreateLoad(
- FileInfoTy->getElementType(1),
- Builder.CreateStructGEP(FileInfoTy, FileInfoPtr, 1), "num_ctrs");
+ auto *NumCounters = Builder.CreateLoad(
+ FileInfoTy->getElementType(1),
+ Builder.CreateStructGEP(FileInfoTy, FileInfoPtr, 1), "num_ctrs");
auto *EmitFunctionCallArgsArray =
Builder.CreateLoad(FileInfoTy->getElementType(2),
- Builder.CreateStructGEP(FileInfoTy, FileInfoPtr, 2),
- "emit_function_args");
- auto *EmitArcsCallArgsArray = Builder.CreateLoad(
- FileInfoTy->getElementType(3),
- Builder.CreateStructGEP(FileInfoTy, FileInfoPtr, 3), "emit_arcs_args");
+ Builder.CreateStructGEP(FileInfoTy, FileInfoPtr, 2),
+ "emit_function_args");
+ auto *EmitArcsCallArgsArray = Builder.CreateLoad(
+ FileInfoTy->getElementType(3),
+ Builder.CreateStructGEP(FileInfoTy, FileInfoPtr, 3), "emit_arcs_args");
auto *EnterCounterLoopCond =
Builder.CreateICmpSLT(Builder.getInt32(0), NumCounters);
Builder.CreateCondBr(EnterCounterLoopCond, CounterLoopHeader, FileLoopLatch);
Builder.SetInsertPoint(CounterLoopHeader);
- auto *JV = Builder.CreatePHI(Builder.getInt32Ty(), /*NumReservedValues*/ 2,
- "ctr_idx");
+ auto *JV = Builder.CreatePHI(Builder.getInt32Ty(), /*NumReservedValues*/ 2,
+ "ctr_idx");
JV->addIncoming(Builder.getInt32(0), FileLoopHeader);
auto *EmitFunctionCallArgsPtr = Builder.CreateInBoundsGEP(
EmitFunctionCallArgsTy, EmitFunctionCallArgsArray, JV);
@@ -1309,16 +1309,16 @@ Function *GCOVProfiler::insertCounterWriteout(
EmitFunction,
{Builder.CreateLoad(EmitFunctionCallArgsTy->getElementType(0),
Builder.CreateStructGEP(EmitFunctionCallArgsTy,
- EmitFunctionCallArgsPtr, 0),
- "ident"),
+ EmitFunctionCallArgsPtr, 0),
+ "ident"),
Builder.CreateLoad(EmitFunctionCallArgsTy->getElementType(1),
Builder.CreateStructGEP(EmitFunctionCallArgsTy,
- EmitFunctionCallArgsPtr, 1),
- "func_checkssum"),
+ EmitFunctionCallArgsPtr, 1),
+ "func_checkssum"),
Builder.CreateLoad(EmitFunctionCallArgsTy->getElementType(2),
Builder.CreateStructGEP(EmitFunctionCallArgsTy,
- EmitFunctionCallArgsPtr, 2),
- "cfg_checksum")});
+ EmitFunctionCallArgsPtr, 2),
+ "cfg_checksum")});
if (auto AK = TLI->getExtAttrForI32Param(false)) {
EmitFunctionCall->addParamAttr(0, AK);
EmitFunctionCall->addParamAttr(1, AK);
@@ -1330,12 +1330,12 @@ Function *GCOVProfiler::insertCounterWriteout(
EmitArcs,
{Builder.CreateLoad(
EmitArcsCallArgsTy->getElementType(0),
- Builder.CreateStructGEP(EmitArcsCallArgsTy, EmitArcsCallArgsPtr, 0),
- "num_counters"),
- Builder.CreateLoad(
- EmitArcsCallArgsTy->getElementType(1),
- Builder.CreateStructGEP(EmitArcsCallArgsTy, EmitArcsCallArgsPtr, 1),
- "counters")});
+ Builder.CreateStructGEP(EmitArcsCallArgsTy, EmitArcsCallArgsPtr, 0),
+ "num_counters"),
+ Builder.CreateLoad(
+ EmitArcsCallArgsTy->getElementType(1),
+ Builder.CreateStructGEP(EmitArcsCallArgsTy, EmitArcsCallArgsPtr, 1),
+ "counters")});
if (auto AK = TLI->getExtAttrForI32Param(false))
EmitArcsCall->addParamAttr(0, AK);
auto *NextJV = Builder.CreateAdd(JV, Builder.getInt32(1));
@@ -1346,7 +1346,7 @@ Function *GCOVProfiler::insertCounterWriteout(
Builder.SetInsertPoint(FileLoopLatch);
Builder.CreateCall(SummaryInfo, {});
Builder.CreateCall(EndFile, {});
- auto *NextIV = Builder.CreateAdd(IV, Builder.getInt32(1), "next_file_idx");
+ auto *NextIV = Builder.CreateAdd(IV, Builder.getInt32(1), "next_file_idx");
auto *FileLoopCond =
Builder.CreateICmpSLT(NextIV, Builder.getInt32(FileInfos.size()));
Builder.CreateCondBr(FileLoopCond, FileLoopHeader, ExitBB);
diff --git a/contrib/libs/llvm12/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp b/contrib/libs/llvm12/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
index d668193c02..fedd9bfc97 100644
--- a/contrib/libs/llvm12/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
+++ b/contrib/libs/llvm12/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
@@ -55,12 +55,12 @@ using namespace llvm;
#define DEBUG_TYPE "hwasan"
-const char kHwasanModuleCtorName[] = "hwasan.module_ctor";
-const char kHwasanNoteName[] = "hwasan.note";
-const char kHwasanInitName[] = "__hwasan_init";
-const char kHwasanPersonalityThunkName[] = "__hwasan_personality_thunk";
+const char kHwasanModuleCtorName[] = "hwasan.module_ctor";
+const char kHwasanNoteName[] = "hwasan.note";
+const char kHwasanInitName[] = "__hwasan_init";
+const char kHwasanPersonalityThunkName[] = "__hwasan_personality_thunk";
-const char kHwasanShadowMemoryDynamicAddress[] =
+const char kHwasanShadowMemoryDynamicAddress[] =
"__hwasan_shadow_memory_dynamic_address";
// Accesses sizes are powers of two: 1, 2, 4, 8, 16.
@@ -202,14 +202,14 @@ public:
bool sanitizeFunction(Function &F);
void initializeModule();
- void createHwasanCtorComdat();
+ void createHwasanCtorComdat();
void initializeCallbacks(Module &M);
- Value *getOpaqueNoopCast(IRBuilder<> &IRB, Value *Val);
-
+ Value *getOpaqueNoopCast(IRBuilder<> &IRB, Value *Val);
+
Value *getDynamicShadowIfunc(IRBuilder<> &IRB);
- Value *getShadowNonTls(IRBuilder<> &IRB);
+ Value *getShadowNonTls(IRBuilder<> &IRB);
void untagPointerOperand(Instruction *I, Value *Addr);
Value *memToShadow(Value *Shadow, IRBuilder<> &IRB);
@@ -281,13 +281,13 @@ private:
bool CompileKernel;
bool Recover;
- bool OutlinedChecks;
+ bool OutlinedChecks;
bool UseShortGranules;
bool InstrumentLandingPads;
- bool HasMatchAllTag = false;
- uint8_t MatchAllTag = 0;
-
+ bool HasMatchAllTag = false;
+ uint8_t MatchAllTag = 0;
+
Function *HwasanCtorFunction;
FunctionCallee HwasanMemoryAccessCallback[2][kNumberOfAccessSizes];
@@ -298,7 +298,7 @@ private:
Constant *ShadowGlobal;
- Value *ShadowBase = nullptr;
+ Value *ShadowBase = nullptr;
Value *StackBaseTag = nullptr;
GlobalValue *ThreadPtrGlobal = nullptr;
};
@@ -370,106 +370,106 @@ PreservedAnalyses HWAddressSanitizerPass::run(Module &M,
return PreservedAnalyses::all();
}
-void HWAddressSanitizer::createHwasanCtorComdat() {
- std::tie(HwasanCtorFunction, std::ignore) =
- getOrCreateSanitizerCtorAndInitFunctions(
- M, kHwasanModuleCtorName, kHwasanInitName,
- /*InitArgTypes=*/{},
- /*InitArgs=*/{},
- // This callback is invoked when the functions are created the first
- // time. Hook them into the global ctors list in that case:
- [&](Function *Ctor, FunctionCallee) {
- Comdat *CtorComdat = M.getOrInsertComdat(kHwasanModuleCtorName);
- Ctor->setComdat(CtorComdat);
- appendToGlobalCtors(M, Ctor, 0, Ctor);
- });
-
- // Create a note that contains pointers to the list of global
- // descriptors. Adding a note to the output file will cause the linker to
- // create a PT_NOTE program header pointing to the note that we can use to
- // find the descriptor list starting from the program headers. A function
- // provided by the runtime initializes the shadow memory for the globals by
- // accessing the descriptor list via the note. The dynamic loader needs to
- // call this function whenever a library is loaded.
- //
- // The reason why we use a note for this instead of a more conventional
- // approach of having a global constructor pass a descriptor list pointer to
- // the runtime is because of an order of initialization problem. With
- // constructors we can encounter the following problematic scenario:
- //
- // 1) library A depends on library B and also interposes one of B's symbols
- // 2) B's constructors are called before A's (as required for correctness)
- // 3) during construction, B accesses one of its "own" globals (actually
- // interposed by A) and triggers a HWASAN failure due to the initialization
- // for A not having happened yet
- //
- // Even without interposition it is possible to run into similar situations in
- // cases where two libraries mutually depend on each other.
- //
- // We only need one note per binary, so put everything for the note in a
- // comdat. This needs to be a comdat with an .init_array section to prevent
- // newer versions of lld from discarding the note.
- //
- // Create the note even if we aren't instrumenting globals. This ensures that
- // binaries linked from object files with both instrumented and
- // non-instrumented globals will end up with a note, even if a comdat from an
- // object file with non-instrumented globals is selected. The note is harmless
- // if the runtime doesn't support it, since it will just be ignored.
- Comdat *NoteComdat = M.getOrInsertComdat(kHwasanModuleCtorName);
-
- Type *Int8Arr0Ty = ArrayType::get(Int8Ty, 0);
- auto Start =
- new GlobalVariable(M, Int8Arr0Ty, true, GlobalVariable::ExternalLinkage,
- nullptr, "__start_hwasan_globals");
- Start->setVisibility(GlobalValue::HiddenVisibility);
- Start->setDSOLocal(true);
- auto Stop =
- new GlobalVariable(M, Int8Arr0Ty, true, GlobalVariable::ExternalLinkage,
- nullptr, "__stop_hwasan_globals");
- Stop->setVisibility(GlobalValue::HiddenVisibility);
- Stop->setDSOLocal(true);
-
- // Null-terminated so actually 8 bytes, which are required in order to align
- // the note properly.
- auto *Name = ConstantDataArray::get(*C, "LLVM\0\0\0");
-
- auto *NoteTy = StructType::get(Int32Ty, Int32Ty, Int32Ty, Name->getType(),
- Int32Ty, Int32Ty);
- auto *Note =
- new GlobalVariable(M, NoteTy, /*isConstant=*/true,
- GlobalValue::PrivateLinkage, nullptr, kHwasanNoteName);
- Note->setSection(".note.hwasan.globals");
- Note->setComdat(NoteComdat);
- Note->setAlignment(Align(4));
- Note->setDSOLocal(true);
-
- // The pointers in the note need to be relative so that the note ends up being
- // placed in rodata, which is the standard location for notes.
- auto CreateRelPtr = [&](Constant *Ptr) {
- return ConstantExpr::getTrunc(
- ConstantExpr::getSub(ConstantExpr::getPtrToInt(Ptr, Int64Ty),
- ConstantExpr::getPtrToInt(Note, Int64Ty)),
- Int32Ty);
- };
- Note->setInitializer(ConstantStruct::getAnon(
- {ConstantInt::get(Int32Ty, 8), // n_namesz
- ConstantInt::get(Int32Ty, 8), // n_descsz
- ConstantInt::get(Int32Ty, ELF::NT_LLVM_HWASAN_GLOBALS), // n_type
- Name, CreateRelPtr(Start), CreateRelPtr(Stop)}));
- appendToCompilerUsed(M, Note);
-
- // Create a zero-length global in hwasan_globals so that the linker will
- // always create start and stop symbols.
- auto Dummy = new GlobalVariable(
- M, Int8Arr0Ty, /*isConstantGlobal*/ true, GlobalVariable::PrivateLinkage,
- Constant::getNullValue(Int8Arr0Ty), "hwasan.dummy.global");
- Dummy->setSection("hwasan_globals");
- Dummy->setComdat(NoteComdat);
- Dummy->setMetadata(LLVMContext::MD_associated,
- MDNode::get(*C, ValueAsMetadata::get(Note)));
- appendToCompilerUsed(M, Dummy);
-}
-
+void HWAddressSanitizer::createHwasanCtorComdat() {
+ std::tie(HwasanCtorFunction, std::ignore) =
+ getOrCreateSanitizerCtorAndInitFunctions(
+ M, kHwasanModuleCtorName, kHwasanInitName,
+ /*InitArgTypes=*/{},
+ /*InitArgs=*/{},
+ // This callback is invoked when the functions are created the first
+ // time. Hook them into the global ctors list in that case:
+ [&](Function *Ctor, FunctionCallee) {
+ Comdat *CtorComdat = M.getOrInsertComdat(kHwasanModuleCtorName);
+ Ctor->setComdat(CtorComdat);
+ appendToGlobalCtors(M, Ctor, 0, Ctor);
+ });
+
+ // Create a note that contains pointers to the list of global
+ // descriptors. Adding a note to the output file will cause the linker to
+ // create a PT_NOTE program header pointing to the note that we can use to
+ // find the descriptor list starting from the program headers. A function
+ // provided by the runtime initializes the shadow memory for the globals by
+ // accessing the descriptor list via the note. The dynamic loader needs to
+ // call this function whenever a library is loaded.
+ //
+ // The reason why we use a note for this instead of a more conventional
+ // approach of having a global constructor pass a descriptor list pointer to
+ // the runtime is because of an order of initialization problem. With
+ // constructors we can encounter the following problematic scenario:
+ //
+ // 1) library A depends on library B and also interposes one of B's symbols
+ // 2) B's constructors are called before A's (as required for correctness)
+ // 3) during construction, B accesses one of its "own" globals (actually
+ // interposed by A) and triggers a HWASAN failure due to the initialization
+ // for A not having happened yet
+ //
+ // Even without interposition it is possible to run into similar situations in
+ // cases where two libraries mutually depend on each other.
+ //
+ // We only need one note per binary, so put everything for the note in a
+ // comdat. This needs to be a comdat with an .init_array section to prevent
+ // newer versions of lld from discarding the note.
+ //
+ // Create the note even if we aren't instrumenting globals. This ensures that
+ // binaries linked from object files with both instrumented and
+ // non-instrumented globals will end up with a note, even if a comdat from an
+ // object file with non-instrumented globals is selected. The note is harmless
+ // if the runtime doesn't support it, since it will just be ignored.
+ Comdat *NoteComdat = M.getOrInsertComdat(kHwasanModuleCtorName);
+
+ Type *Int8Arr0Ty = ArrayType::get(Int8Ty, 0);
+ auto Start =
+ new GlobalVariable(M, Int8Arr0Ty, true, GlobalVariable::ExternalLinkage,
+ nullptr, "__start_hwasan_globals");
+ Start->setVisibility(GlobalValue::HiddenVisibility);
+ Start->setDSOLocal(true);
+ auto Stop =
+ new GlobalVariable(M, Int8Arr0Ty, true, GlobalVariable::ExternalLinkage,
+ nullptr, "__stop_hwasan_globals");
+ Stop->setVisibility(GlobalValue::HiddenVisibility);
+ Stop->setDSOLocal(true);
+
+ // Null-terminated so actually 8 bytes, which are required in order to align
+ // the note properly.
+ auto *Name = ConstantDataArray::get(*C, "LLVM\0\0\0");
+
+ auto *NoteTy = StructType::get(Int32Ty, Int32Ty, Int32Ty, Name->getType(),
+ Int32Ty, Int32Ty);
+ auto *Note =
+ new GlobalVariable(M, NoteTy, /*isConstant=*/true,
+ GlobalValue::PrivateLinkage, nullptr, kHwasanNoteName);
+ Note->setSection(".note.hwasan.globals");
+ Note->setComdat(NoteComdat);
+ Note->setAlignment(Align(4));
+ Note->setDSOLocal(true);
+
+ // The pointers in the note need to be relative so that the note ends up being
+ // placed in rodata, which is the standard location for notes.
+ auto CreateRelPtr = [&](Constant *Ptr) {
+ return ConstantExpr::getTrunc(
+ ConstantExpr::getSub(ConstantExpr::getPtrToInt(Ptr, Int64Ty),
+ ConstantExpr::getPtrToInt(Note, Int64Ty)),
+ Int32Ty);
+ };
+ Note->setInitializer(ConstantStruct::getAnon(
+ {ConstantInt::get(Int32Ty, 8), // n_namesz
+ ConstantInt::get(Int32Ty, 8), // n_descsz
+ ConstantInt::get(Int32Ty, ELF::NT_LLVM_HWASAN_GLOBALS), // n_type
+ Name, CreateRelPtr(Start), CreateRelPtr(Stop)}));
+ appendToCompilerUsed(M, Note);
+
+ // Create a zero-length global in hwasan_globals so that the linker will
+ // always create start and stop symbols.
+ auto Dummy = new GlobalVariable(
+ M, Int8Arr0Ty, /*isConstantGlobal*/ true, GlobalVariable::PrivateLinkage,
+ Constant::getNullValue(Int8Arr0Ty), "hwasan.dummy.global");
+ Dummy->setSection("hwasan_globals");
+ Dummy->setComdat(NoteComdat);
+ Dummy->setMetadata(LLVMContext::MD_associated,
+ MDNode::get(*C, ValueAsMetadata::get(Note)));
+ appendToCompilerUsed(M, Dummy);
+}
+
/// Module-level initialization.
///
/// inserts a call to __hwasan_init to the module's constructor list.
@@ -498,27 +498,27 @@ void HWAddressSanitizer::initializeModule() {
UseShortGranules =
ClUseShortGranules.getNumOccurrences() ? ClUseShortGranules : NewRuntime;
- OutlinedChecks =
- TargetTriple.isAArch64() && TargetTriple.isOSBinFormatELF() &&
- (ClInlineAllChecks.getNumOccurrences() ? !ClInlineAllChecks : !Recover);
-
- if (ClMatchAllTag.getNumOccurrences()) {
- if (ClMatchAllTag != -1) {
- HasMatchAllTag = true;
- MatchAllTag = ClMatchAllTag & 0xFF;
- }
- } else if (CompileKernel) {
- HasMatchAllTag = true;
- MatchAllTag = 0xFF;
- }
-
+ OutlinedChecks =
+ TargetTriple.isAArch64() && TargetTriple.isOSBinFormatELF() &&
+ (ClInlineAllChecks.getNumOccurrences() ? !ClInlineAllChecks : !Recover);
+
+ if (ClMatchAllTag.getNumOccurrences()) {
+ if (ClMatchAllTag != -1) {
+ HasMatchAllTag = true;
+ MatchAllTag = ClMatchAllTag & 0xFF;
+ }
+ } else if (CompileKernel) {
+ HasMatchAllTag = true;
+ MatchAllTag = 0xFF;
+ }
+
// If we don't have personality function support, fall back to landing pads.
InstrumentLandingPads = ClInstrumentLandingPads.getNumOccurrences()
? ClInstrumentLandingPads
: !NewRuntime;
if (!CompileKernel) {
- createHwasanCtorComdat();
+ createHwasanCtorComdat();
bool InstrumentGlobals =
ClGlobals.getNumOccurrences() ? ClGlobals : NewRuntime;
if (InstrumentGlobals)
@@ -589,27 +589,27 @@ void HWAddressSanitizer::initializeCallbacks(Module &M) {
M.getOrInsertFunction("__hwasan_handle_vfork", IRB.getVoidTy(), IntptrTy);
}
-Value *HWAddressSanitizer::getOpaqueNoopCast(IRBuilder<> &IRB, Value *Val) {
+Value *HWAddressSanitizer::getOpaqueNoopCast(IRBuilder<> &IRB, Value *Val) {
// An empty inline asm with input reg == output reg.
// An opaque no-op cast, basically.
- // This prevents code bloat as a result of rematerializing trivial definitions
- // such as constants or global addresses at every load and store.
- InlineAsm *Asm =
- InlineAsm::get(FunctionType::get(Int8PtrTy, {Val->getType()}, false),
- StringRef(""), StringRef("=r,0"),
- /*hasSideEffects=*/false);
- return IRB.CreateCall(Asm, {Val}, ".hwasan.shadow");
+ // This prevents code bloat as a result of rematerializing trivial definitions
+ // such as constants or global addresses at every load and store.
+ InlineAsm *Asm =
+ InlineAsm::get(FunctionType::get(Int8PtrTy, {Val->getType()}, false),
+ StringRef(""), StringRef("=r,0"),
+ /*hasSideEffects=*/false);
+ return IRB.CreateCall(Asm, {Val}, ".hwasan.shadow");
+}
+
+Value *HWAddressSanitizer::getDynamicShadowIfunc(IRBuilder<> &IRB) {
+ return getOpaqueNoopCast(IRB, ShadowGlobal);
}
-Value *HWAddressSanitizer::getDynamicShadowIfunc(IRBuilder<> &IRB) {
- return getOpaqueNoopCast(IRB, ShadowGlobal);
-}
-
-Value *HWAddressSanitizer::getShadowNonTls(IRBuilder<> &IRB) {
+Value *HWAddressSanitizer::getShadowNonTls(IRBuilder<> &IRB) {
if (Mapping.Offset != kDynamicShadowSentinel)
- return getOpaqueNoopCast(
- IRB, ConstantExpr::getIntToPtr(
- ConstantInt::get(IntptrTy, Mapping.Offset), Int8PtrTy));
+ return getOpaqueNoopCast(
+ IRB, ConstantExpr::getIntToPtr(
+ ConstantInt::get(IntptrTy, Mapping.Offset), Int8PtrTy));
if (Mapping.InGlobal) {
return getDynamicShadowIfunc(IRB);
@@ -645,7 +645,7 @@ void HWAddressSanitizer::getInterestingMemoryOperands(
return;
// Do not instrument the load fetching the dynamic shadow address.
- if (ShadowBase == I)
+ if (ShadowBase == I)
return;
if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
@@ -715,29 +715,29 @@ Value *HWAddressSanitizer::memToShadow(Value *Mem, IRBuilder<> &IRB) {
if (Mapping.Offset == 0)
return IRB.CreateIntToPtr(Shadow, Int8PtrTy);
// (Mem >> Scale) + Offset
- return IRB.CreateGEP(Int8Ty, ShadowBase, Shadow);
+ return IRB.CreateGEP(Int8Ty, ShadowBase, Shadow);
}
void HWAddressSanitizer::instrumentMemAccessInline(Value *Ptr, bool IsWrite,
unsigned AccessSizeIndex,
Instruction *InsertBefore) {
- const int64_t AccessInfo =
- (CompileKernel << HWASanAccessInfo::CompileKernelShift) +
- (HasMatchAllTag << HWASanAccessInfo::HasMatchAllShift) +
- (MatchAllTag << HWASanAccessInfo::MatchAllShift) +
- (Recover << HWASanAccessInfo::RecoverShift) +
- (IsWrite << HWASanAccessInfo::IsWriteShift) +
- (AccessSizeIndex << HWASanAccessInfo::AccessSizeShift);
+ const int64_t AccessInfo =
+ (CompileKernel << HWASanAccessInfo::CompileKernelShift) +
+ (HasMatchAllTag << HWASanAccessInfo::HasMatchAllShift) +
+ (MatchAllTag << HWASanAccessInfo::MatchAllShift) +
+ (Recover << HWASanAccessInfo::RecoverShift) +
+ (IsWrite << HWASanAccessInfo::IsWriteShift) +
+ (AccessSizeIndex << HWASanAccessInfo::AccessSizeShift);
IRBuilder<> IRB(InsertBefore);
- if (OutlinedChecks) {
+ if (OutlinedChecks) {
Module *M = IRB.GetInsertBlock()->getParent()->getParent();
Ptr = IRB.CreateBitCast(Ptr, Int8PtrTy);
IRB.CreateCall(Intrinsic::getDeclaration(
M, UseShortGranules
? Intrinsic::hwasan_check_memaccess_shortgranules
: Intrinsic::hwasan_check_memaccess),
- {ShadowBase, Ptr, ConstantInt::get(Int32Ty, AccessInfo)});
+ {ShadowBase, Ptr, ConstantInt::get(Int32Ty, AccessInfo)});
return;
}
@@ -749,9 +749,9 @@ void HWAddressSanitizer::instrumentMemAccessInline(Value *Ptr, bool IsWrite,
Value *MemTag = IRB.CreateLoad(Int8Ty, Shadow);
Value *TagMismatch = IRB.CreateICmpNE(PtrTag, MemTag);
- if (HasMatchAllTag) {
- Value *TagNotIgnored = IRB.CreateICmpNE(
- PtrTag, ConstantInt::get(PtrTag->getType(), MatchAllTag));
+ if (HasMatchAllTag) {
+ Value *TagNotIgnored = IRB.CreateICmpNE(
+ PtrTag, ConstantInt::get(PtrTag->getType(), MatchAllTag));
TagMismatch = IRB.CreateAnd(TagMismatch, TagNotIgnored);
}
@@ -773,8 +773,8 @@ void HWAddressSanitizer::instrumentMemAccessInline(Value *Ptr, bool IsWrite,
Value *PtrLowBitsOOB = IRB.CreateICmpUGE(PtrLowBits, MemTag);
SplitBlockAndInsertIfThen(PtrLowBitsOOB, CheckTerm, false,
MDBuilder(*C).createBranchWeights(1, 100000),
- (DomTreeUpdater *)nullptr, nullptr,
- CheckFailTerm->getParent());
+ (DomTreeUpdater *)nullptr, nullptr,
+ CheckFailTerm->getParent());
IRB.SetInsertPoint(CheckTerm);
Value *InlineTagAddr = IRB.CreateOr(AddrLong, 15);
@@ -783,8 +783,8 @@ void HWAddressSanitizer::instrumentMemAccessInline(Value *Ptr, bool IsWrite,
Value *InlineTagMismatch = IRB.CreateICmpNE(PtrTag, InlineTag);
SplitBlockAndInsertIfThen(InlineTagMismatch, CheckTerm, false,
MDBuilder(*C).createBranchWeights(1, 100000),
- (DomTreeUpdater *)nullptr, nullptr,
- CheckFailTerm->getParent());
+ (DomTreeUpdater *)nullptr, nullptr,
+ CheckFailTerm->getParent());
IRB.SetInsertPoint(CheckFailTerm);
InlineAsm *Asm;
@@ -793,9 +793,9 @@ void HWAddressSanitizer::instrumentMemAccessInline(Value *Ptr, bool IsWrite,
// The signal handler will find the data address in rdi.
Asm = InlineAsm::get(
FunctionType::get(IRB.getVoidTy(), {PtrLong->getType()}, false),
- "int3\nnopl " +
- itostr(0x40 + (AccessInfo & HWASanAccessInfo::RuntimeMask)) +
- "(%rax)",
+ "int3\nnopl " +
+ itostr(0x40 + (AccessInfo & HWASanAccessInfo::RuntimeMask)) +
+ "(%rax)",
"{rdi}",
/*hasSideEffects=*/true);
break;
@@ -804,8 +804,8 @@ void HWAddressSanitizer::instrumentMemAccessInline(Value *Ptr, bool IsWrite,
// The signal handler will find the data address in x0.
Asm = InlineAsm::get(
FunctionType::get(IRB.getVoidTy(), {PtrLong->getType()}, false),
- "brk #" +
- itostr(0x900 + (AccessInfo & HWASanAccessInfo::RuntimeMask)),
+ "brk #" +
+ itostr(0x900 + (AccessInfo & HWASanAccessInfo::RuntimeMask)),
"{x0}",
/*hasSideEffects=*/true);
break;
@@ -1028,12 +1028,12 @@ Value *HWAddressSanitizer::getHwasanThreadSlotPtr(IRBuilder<> &IRB, Type *Ty) {
void HWAddressSanitizer::emitPrologue(IRBuilder<> &IRB, bool WithFrameRecord) {
if (!Mapping.InTls) {
- ShadowBase = getShadowNonTls(IRB);
+ ShadowBase = getShadowNonTls(IRB);
return;
}
if (!WithFrameRecord && TargetTriple.isAndroid()) {
- ShadowBase = getDynamicShadowIfunc(IRB);
+ ShadowBase = getDynamicShadowIfunc(IRB);
return;
}
@@ -1094,12 +1094,12 @@ void HWAddressSanitizer::emitPrologue(IRBuilder<> &IRB, bool WithFrameRecord) {
// Get shadow base address by aligning RecordPtr up.
// Note: this is not correct if the pointer is already aligned.
// Runtime library will make sure this never happens.
- ShadowBase = IRB.CreateAdd(
+ ShadowBase = IRB.CreateAdd(
IRB.CreateOr(
ThreadLongMaybeUntagged,
ConstantInt::get(IntptrTy, (1ULL << kShadowBaseAlignment) - 1)),
ConstantInt::get(IntptrTy, 1), "hwasan.shadow");
- ShadowBase = IRB.CreateIntToPtr(ShadowBase, Int8PtrTy);
+ ShadowBase = IRB.CreateIntToPtr(ShadowBase, Int8PtrTy);
}
Value *HWAddressSanitizer::readRegister(IRBuilder<> &IRB, StringRef Name) {
@@ -1251,7 +1251,7 @@ bool HWAddressSanitizer::sanitizeFunction(Function &F) {
IntrinToInstrument.empty())
return Changed;
- assert(!ShadowBase);
+ assert(!ShadowBase);
Instruction *InsertPt = &*F.getEntryBlock().begin();
IRBuilder<> EntryIRB(InsertPt);
@@ -1331,7 +1331,7 @@ bool HWAddressSanitizer::sanitizeFunction(Function &F) {
instrumentMemIntrinsic(cast<MemIntrinsic>(Inst));
}
- ShadowBase = nullptr;
+ ShadowBase = nullptr;
StackBaseTag = nullptr;
return true;
diff --git a/contrib/libs/llvm12/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp b/contrib/libs/llvm12/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp
index 068cf578dd..5b9557a9b3 100644
--- a/contrib/libs/llvm12/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp
+++ b/contrib/libs/llvm12/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp
@@ -263,15 +263,15 @@ ICallPromotionFunc::getPromotionCandidatesForCallSite(
break;
}
- // Don't promote if the symbol is not defined in the module. This avoids
- // creating a reference to a symbol that doesn't exist in the module
- // This can happen when we compile with a sample profile collected from
- // one binary but used for another, which may have profiled targets that
- // aren't used in the new binary. We might have a declaration initially in
- // the case where the symbol is globally dead in the binary and removed by
- // ThinLTO.
+ // Don't promote if the symbol is not defined in the module. This avoids
+ // creating a reference to a symbol that doesn't exist in the module
+ // This can happen when we compile with a sample profile collected from
+ // one binary but used for another, which may have profiled targets that
+ // aren't used in the new binary. We might have a declaration initially in
+ // the case where the symbol is globally dead in the binary and removed by
+ // ThinLTO.
Function *TargetFunction = Symtab->getFunction(Target);
- if (TargetFunction == nullptr || TargetFunction->isDeclaration()) {
+ if (TargetFunction == nullptr || TargetFunction->isDeclaration()) {
LLVM_DEBUG(dbgs() << " Not promote: Cannot find the target\n");
ORE.emit([&]() {
return OptimizationRemarkMissed(DEBUG_TYPE, "UnableToFindTarget", &CB)
diff --git a/contrib/libs/llvm12/lib/Transforms/Instrumentation/InstrProfiling.cpp b/contrib/libs/llvm12/lib/Transforms/Instrumentation/InstrProfiling.cpp
index 83e048d010..9efc7d1ac5 100644
--- a/contrib/libs/llvm12/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/contrib/libs/llvm12/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -135,10 +135,10 @@ cl::opt<bool> IterativeCounterPromotion(
cl::ZeroOrMore, "iterative-counter-promotion", cl::init(true),
cl::desc("Allow counter promotion across the whole loop nest."));
-cl::opt<bool> SkipRetExitBlock(
- cl::ZeroOrMore, "skip-ret-exit-block", cl::init(true),
- cl::desc("Suppress counter promotion if exit blocks contain ret."));
-
+cl::opt<bool> SkipRetExitBlock(
+ cl::ZeroOrMore, "skip-ret-exit-block", cl::init(true),
+ cl::desc("Suppress counter promotion if exit blocks contain ret."));
+
class InstrProfilingLegacyPass : public ModulePass {
InstrProfiling InstrProf;
@@ -261,18 +261,18 @@ public:
// Skip 'infinite' loops:
if (ExitBlocks.size() == 0)
return false;
-
- // Skip if any of the ExitBlocks contains a ret instruction.
- // This is to prevent dumping of incomplete profile -- if the
- // the loop is a long running loop and dump is called in the middle
- // of the loop, the result profile is incomplete.
- // FIXME: add other heuristics to detect long running loops.
- if (SkipRetExitBlock) {
- for (auto BB : ExitBlocks)
- if (isa<ReturnInst>(BB->getTerminator()))
- return false;
- }
-
+
+ // Skip if any of the ExitBlocks contains a ret instruction.
+ // This is to prevent dumping of incomplete profile -- if the
+ // the loop is a long running loop and dump is called in the middle
+ // of the loop, the result profile is incomplete.
+ // FIXME: add other heuristics to detect long running loops.
+ if (SkipRetExitBlock) {
+ for (auto BB : ExitBlocks)
+ if (isa<ReturnInst>(BB->getTerminator()))
+ return false;
+ }
+
unsigned MaxProm = getMaxNumOfPromotionsInLoop(&L);
if (MaxProm == 0)
return false;
@@ -396,15 +396,15 @@ private:
BlockFrequencyInfo *BFI;
};
-enum class ValueProfilingCallType {
- // Individual values are tracked. Currently used for indiret call target
- // profiling.
- Default,
-
- // MemOp: the memop size value profiling.
- MemOp
-};
-
+enum class ValueProfilingCallType {
+ // Individual values are tracked. Currently used for indiret call target
+ // profiling.
+ Default,
+
+ // MemOp: the memop size value profiling.
+ MemOp
+};
+
} // end anonymous namespace
PreservedAnalyses InstrProfiling::run(Module &M, ModuleAnalysisManager &AM) {
@@ -587,9 +587,9 @@ bool InstrProfiling::run(
return true;
}
-static FunctionCallee getOrInsertValueProfilingCall(
- Module &M, const TargetLibraryInfo &TLI,
- ValueProfilingCallType CallType = ValueProfilingCallType::Default) {
+static FunctionCallee getOrInsertValueProfilingCall(
+ Module &M, const TargetLibraryInfo &TLI,
+ ValueProfilingCallType CallType = ValueProfilingCallType::Default) {
LLVMContext &Ctx = M.getContext();
auto *ReturnTy = Type::getVoidTy(M.getContext());
@@ -597,19 +597,19 @@ static FunctionCallee getOrInsertValueProfilingCall(
if (auto AK = TLI.getExtAttrForI32Param(false))
AL = AL.addParamAttribute(M.getContext(), 2, AK);
- assert((CallType == ValueProfilingCallType::Default ||
- CallType == ValueProfilingCallType::MemOp) &&
- "Must be Default or MemOp");
- Type *ParamTypes[] = {
+ assert((CallType == ValueProfilingCallType::Default ||
+ CallType == ValueProfilingCallType::MemOp) &&
+ "Must be Default or MemOp");
+ Type *ParamTypes[] = {
#define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
#include "llvm/ProfileData/InstrProfData.inc"
- };
- auto *ValueProfilingCallTy =
- FunctionType::get(ReturnTy, makeArrayRef(ParamTypes), false);
- StringRef FuncName = CallType == ValueProfilingCallType::Default
- ? getInstrProfValueProfFuncName()
- : getInstrProfValueProfMemOpFuncName();
- return M.getOrInsertFunction(FuncName, ValueProfilingCallTy, AL);
+ };
+ auto *ValueProfilingCallTy =
+ FunctionType::get(ReturnTy, makeArrayRef(ParamTypes), false);
+ StringRef FuncName = CallType == ValueProfilingCallType::Default
+ ? getInstrProfValueProfFuncName()
+ : getInstrProfValueProfMemOpFuncName();
+ return M.getOrInsertFunction(FuncName, ValueProfilingCallTy, AL);
}
void InstrProfiling::computeNumValueSiteCounts(InstrProfValueProfileInst *Ind) {
@@ -638,8 +638,8 @@ void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
Index += It->second.NumValueSites[Kind];
IRBuilder<> Builder(Ind);
- bool IsMemOpSize = (Ind->getValueKind()->getZExtValue() ==
- llvm::InstrProfValueKind::IPVK_MemOPSize);
+ bool IsMemOpSize = (Ind->getValueKind()->getZExtValue() ==
+ llvm::InstrProfValueKind::IPVK_MemOPSize);
CallInst *Call = nullptr;
auto *TLI = &GetTLI(*Ind->getFunction());
@@ -649,19 +649,19 @@ void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
// WinEHPrepare pass.
SmallVector<OperandBundleDef, 1> OpBundles;
Ind->getOperandBundlesAsDefs(OpBundles);
- if (!IsMemOpSize) {
+ if (!IsMemOpSize) {
Value *Args[3] = {Ind->getTargetValue(),
Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
Builder.getInt32(Index)};
Call = Builder.CreateCall(getOrInsertValueProfilingCall(*M, *TLI), Args,
OpBundles);
} else {
- Value *Args[3] = {Ind->getTargetValue(),
- Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
- Builder.getInt32(Index)};
- Call = Builder.CreateCall(
- getOrInsertValueProfilingCall(*M, *TLI, ValueProfilingCallType::MemOp),
- Args, OpBundles);
+ Value *Args[3] = {Ind->getTargetValue(),
+ Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
+ Builder.getInt32(Index)};
+ Call = Builder.CreateCall(
+ getOrInsertValueProfilingCall(*M, *TLI, ValueProfilingCallType::MemOp),
+ Args, OpBundles);
}
if (auto AK = TLI->getExtAttrForI32Param(false))
Call->addParamAttr(2, AK);
@@ -828,11 +828,11 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
Visibility = GlobalValue::HiddenVisibility;
}
}
- std::string DataVarName = getVarName(Inc, getInstrProfDataVarPrefix());
+ std::string DataVarName = getVarName(Inc, getInstrProfDataVarPrefix());
auto MaybeSetComdat = [=](GlobalVariable *GV) {
if (NeedComdat)
- GV->setComdat(M->getOrInsertComdat(TT.isOSBinFormatCOFF() ? GV->getName()
- : DataVarName));
+ GV->setComdat(M->getOrInsertComdat(TT.isOSBinFormatCOFF() ? GV->getName()
+ : DataVarName));
};
uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
@@ -897,9 +897,9 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init,
#include "llvm/ProfileData/InstrProfData.inc"
};
- auto *Data =
- new GlobalVariable(*M, DataTy, false, Linkage,
- ConstantStruct::get(DataTy, DataVals), DataVarName);
+ auto *Data =
+ new GlobalVariable(*M, DataTy, false, Linkage,
+ ConstantStruct::get(DataTy, DataVals), DataVarName);
Data->setVisibility(Visibility);
Data->setSection(getInstrProfSectionName(IPSK_data, TT.getObjectFormat()));
Data->setAlignment(Align(INSTR_PROF_DATA_ALIGNMENT));
diff --git a/contrib/libs/llvm12/lib/Transforms/Instrumentation/Instrumentation.cpp b/contrib/libs/llvm12/lib/Transforms/Instrumentation/Instrumentation.cpp
index ff4a00fcb7..cfdf3cad97 100644
--- a/contrib/libs/llvm12/lib/Transforms/Instrumentation/Instrumentation.cpp
+++ b/contrib/libs/llvm12/lib/Transforms/Instrumentation/Instrumentation.cpp
@@ -105,8 +105,8 @@ Comdat *llvm::GetOrCreateFunctionComdat(Function &F, Triple &T,
void llvm::initializeInstrumentation(PassRegistry &Registry) {
initializeAddressSanitizerLegacyPassPass(Registry);
initializeModuleAddressSanitizerLegacyPassPass(Registry);
- initializeMemProfilerLegacyPassPass(Registry);
- initializeModuleMemProfilerLegacyPassPass(Registry);
+ initializeMemProfilerLegacyPassPass(Registry);
+ initializeModuleMemProfilerLegacyPassPass(Registry);
initializeBoundsCheckingLegacyPassPass(Registry);
initializeControlHeightReductionLegacyPassPass(Registry);
initializeGCOVProfilerLegacyPassPass(Registry);
@@ -121,7 +121,7 @@ void llvm::initializeInstrumentation(PassRegistry &Registry) {
initializeHWAddressSanitizerLegacyPassPass(Registry);
initializeThreadSanitizerLegacyPassPass(Registry);
initializeModuleSanitizerCoverageLegacyPassPass(Registry);
- initializeDataFlowSanitizerLegacyPassPass(Registry);
+ initializeDataFlowSanitizerLegacyPassPass(Registry);
}
/// LLVMInitializeInstrumentation - C binding for
diff --git a/contrib/libs/llvm12/lib/Transforms/Instrumentation/MemProfiler.cpp b/contrib/libs/llvm12/lib/Transforms/Instrumentation/MemProfiler.cpp
index fa2edf52a2..0e6a404a9e 100644
--- a/contrib/libs/llvm12/lib/Transforms/Instrumentation/MemProfiler.cpp
+++ b/contrib/libs/llvm12/lib/Transforms/Instrumentation/MemProfiler.cpp
@@ -1,638 +1,638 @@
-//===- MemProfiler.cpp - memory allocation and access profiler ------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file is a part of MemProfiler. Memory accesses are instrumented
-// to increment the access count held in a shadow memory location, or
-// alternatively to call into the runtime. Memory intrinsic calls (memmove,
-// memcpy, memset) are changed to call the memory profiling runtime version
-// instead.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Transforms/Instrumentation/MemProfiler.h"
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/Statistic.h"
-#include "llvm/ADT/StringRef.h"
-#include "llvm/ADT/Triple.h"
-#include "llvm/IR/Constant.h"
-#include "llvm/IR/DataLayout.h"
-#include "llvm/IR/Function.h"
-#include "llvm/IR/GlobalValue.h"
-#include "llvm/IR/IRBuilder.h"
-#include "llvm/IR/Instruction.h"
-#include "llvm/IR/LLVMContext.h"
-#include "llvm/IR/Module.h"
-#include "llvm/IR/Type.h"
-#include "llvm/IR/Value.h"
-#include "llvm/InitializePasses.h"
-#include "llvm/Pass.h"
-#include "llvm/Support/CommandLine.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Transforms/Instrumentation.h"
-#include "llvm/Transforms/Utils/BasicBlockUtils.h"
-#include "llvm/Transforms/Utils/ModuleUtils.h"
-
-using namespace llvm;
-
-#define DEBUG_TYPE "memprof"
-
-constexpr int LLVM_MEM_PROFILER_VERSION = 1;
-
-// Size of memory mapped to a single shadow location.
-constexpr uint64_t DefaultShadowGranularity = 64;
-
-// Scale from granularity down to shadow size.
-constexpr uint64_t DefaultShadowScale = 3;
-
-constexpr char MemProfModuleCtorName[] = "memprof.module_ctor";
-constexpr uint64_t MemProfCtorAndDtorPriority = 1;
-// On Emscripten, the system needs more than one priorities for constructors.
-constexpr uint64_t MemProfEmscriptenCtorAndDtorPriority = 50;
-constexpr char MemProfInitName[] = "__memprof_init";
-constexpr char MemProfVersionCheckNamePrefix[] =
- "__memprof_version_mismatch_check_v";
-
-constexpr char MemProfShadowMemoryDynamicAddress[] =
- "__memprof_shadow_memory_dynamic_address";
-
-constexpr char MemProfFilenameVar[] = "__memprof_profile_filename";
-
-// Command-line flags.
-
-static cl::opt<bool> ClInsertVersionCheck(
- "memprof-guard-against-version-mismatch",
- cl::desc("Guard against compiler/runtime version mismatch."), cl::Hidden,
- cl::init(true));
-
-// This flag may need to be replaced with -f[no-]memprof-reads.
-static cl::opt<bool> ClInstrumentReads("memprof-instrument-reads",
- cl::desc("instrument read instructions"),
- cl::Hidden, cl::init(true));
-
-static cl::opt<bool>
- ClInstrumentWrites("memprof-instrument-writes",
- cl::desc("instrument write instructions"), cl::Hidden,
- cl::init(true));
-
-static cl::opt<bool> ClInstrumentAtomics(
- "memprof-instrument-atomics",
- cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden,
- cl::init(true));
-
-static cl::opt<bool> ClUseCalls(
- "memprof-use-callbacks",
- cl::desc("Use callbacks instead of inline instrumentation sequences."),
- cl::Hidden, cl::init(false));
-
-static cl::opt<std::string>
- ClMemoryAccessCallbackPrefix("memprof-memory-access-callback-prefix",
- cl::desc("Prefix for memory access callbacks"),
- cl::Hidden, cl::init("__memprof_"));
-
-// These flags allow to change the shadow mapping.
-// The shadow mapping looks like
-// Shadow = ((Mem & mask) >> scale) + offset
-
-static cl::opt<int> ClMappingScale("memprof-mapping-scale",
- cl::desc("scale of memprof shadow mapping"),
- cl::Hidden, cl::init(DefaultShadowScale));
-
-static cl::opt<int>
- ClMappingGranularity("memprof-mapping-granularity",
- cl::desc("granularity of memprof shadow mapping"),
- cl::Hidden, cl::init(DefaultShadowGranularity));
-
-// Debug flags.
-
-static cl::opt<int> ClDebug("memprof-debug", cl::desc("debug"), cl::Hidden,
- cl::init(0));
-
-static cl::opt<std::string> ClDebugFunc("memprof-debug-func", cl::Hidden,
- cl::desc("Debug func"));
-
-static cl::opt<int> ClDebugMin("memprof-debug-min", cl::desc("Debug min inst"),
- cl::Hidden, cl::init(-1));
-
-static cl::opt<int> ClDebugMax("memprof-debug-max", cl::desc("Debug max inst"),
- cl::Hidden, cl::init(-1));
-
-STATISTIC(NumInstrumentedReads, "Number of instrumented reads");
-STATISTIC(NumInstrumentedWrites, "Number of instrumented writes");
-
-namespace {
-
-/// This struct defines the shadow mapping using the rule:
-/// shadow = ((mem & mask) >> Scale) ADD DynamicShadowOffset.
-struct ShadowMapping {
- ShadowMapping() {
- Scale = ClMappingScale;
- Granularity = ClMappingGranularity;
- Mask = ~(Granularity - 1);
- }
-
- int Scale;
- int Granularity;
- uint64_t Mask; // Computed as ~(Granularity-1)
-};
-
-static uint64_t getCtorAndDtorPriority(Triple &TargetTriple) {
- return TargetTriple.isOSEmscripten() ? MemProfEmscriptenCtorAndDtorPriority
- : MemProfCtorAndDtorPriority;
-}
-
-struct InterestingMemoryAccess {
- Value *Addr = nullptr;
- bool IsWrite;
- unsigned Alignment;
- uint64_t TypeSize;
- Value *MaybeMask = nullptr;
-};
-
-/// Instrument the code in module to profile memory accesses.
-class MemProfiler {
-public:
- MemProfiler(Module &M) {
- C = &(M.getContext());
- LongSize = M.getDataLayout().getPointerSizeInBits();
- IntptrTy = Type::getIntNTy(*C, LongSize);
- }
-
- /// If it is an interesting memory access, populate information
- /// about the access and return a InterestingMemoryAccess struct.
- /// Otherwise return None.
- Optional<InterestingMemoryAccess>
- isInterestingMemoryAccess(Instruction *I) const;
-
- void instrumentMop(Instruction *I, const DataLayout &DL,
- InterestingMemoryAccess &Access);
- void instrumentAddress(Instruction *OrigIns, Instruction *InsertBefore,
- Value *Addr, uint32_t TypeSize, bool IsWrite);
- void instrumentMaskedLoadOrStore(const DataLayout &DL, Value *Mask,
- Instruction *I, Value *Addr,
- unsigned Alignment, uint32_t TypeSize,
- bool IsWrite);
- void instrumentMemIntrinsic(MemIntrinsic *MI);
- Value *memToShadow(Value *Shadow, IRBuilder<> &IRB);
- bool instrumentFunction(Function &F);
- bool maybeInsertMemProfInitAtFunctionEntry(Function &F);
- bool insertDynamicShadowAtFunctionEntry(Function &F);
-
-private:
- void initializeCallbacks(Module &M);
-
- LLVMContext *C;
- int LongSize;
- Type *IntptrTy;
- ShadowMapping Mapping;
-
- // These arrays is indexed by AccessIsWrite
- FunctionCallee MemProfMemoryAccessCallback[2];
- FunctionCallee MemProfMemoryAccessCallbackSized[2];
-
- FunctionCallee MemProfMemmove, MemProfMemcpy, MemProfMemset;
- Value *DynamicShadowOffset = nullptr;
-};
-
-class MemProfilerLegacyPass : public FunctionPass {
-public:
- static char ID;
-
- explicit MemProfilerLegacyPass() : FunctionPass(ID) {
- initializeMemProfilerLegacyPassPass(*PassRegistry::getPassRegistry());
- }
-
- StringRef getPassName() const override { return "MemProfilerFunctionPass"; }
-
- bool runOnFunction(Function &F) override {
- MemProfiler Profiler(*F.getParent());
- return Profiler.instrumentFunction(F);
- }
-};
-
-class ModuleMemProfiler {
-public:
- ModuleMemProfiler(Module &M) { TargetTriple = Triple(M.getTargetTriple()); }
-
- bool instrumentModule(Module &);
-
-private:
- Triple TargetTriple;
- ShadowMapping Mapping;
- Function *MemProfCtorFunction = nullptr;
-};
-
-class ModuleMemProfilerLegacyPass : public ModulePass {
-public:
- static char ID;
-
- explicit ModuleMemProfilerLegacyPass() : ModulePass(ID) {
- initializeModuleMemProfilerLegacyPassPass(*PassRegistry::getPassRegistry());
- }
-
- StringRef getPassName() const override { return "ModuleMemProfiler"; }
-
- void getAnalysisUsage(AnalysisUsage &AU) const override {}
-
- bool runOnModule(Module &M) override {
- ModuleMemProfiler MemProfiler(M);
- return MemProfiler.instrumentModule(M);
- }
-};
-
-} // end anonymous namespace
-
-MemProfilerPass::MemProfilerPass() {}
-
-PreservedAnalyses MemProfilerPass::run(Function &F,
- AnalysisManager<Function> &AM) {
- Module &M = *F.getParent();
- MemProfiler Profiler(M);
- if (Profiler.instrumentFunction(F))
- return PreservedAnalyses::none();
- return PreservedAnalyses::all();
-
- return PreservedAnalyses::all();
-}
-
-ModuleMemProfilerPass::ModuleMemProfilerPass() {}
-
-PreservedAnalyses ModuleMemProfilerPass::run(Module &M,
- AnalysisManager<Module> &AM) {
- ModuleMemProfiler Profiler(M);
- if (Profiler.instrumentModule(M))
- return PreservedAnalyses::none();
- return PreservedAnalyses::all();
-}
-
-char MemProfilerLegacyPass::ID = 0;
-
-INITIALIZE_PASS_BEGIN(MemProfilerLegacyPass, "memprof",
- "MemProfiler: profile memory allocations and accesses.",
- false, false)
-INITIALIZE_PASS_END(MemProfilerLegacyPass, "memprof",
- "MemProfiler: profile memory allocations and accesses.",
- false, false)
-
-FunctionPass *llvm::createMemProfilerFunctionPass() {
- return new MemProfilerLegacyPass();
-}
-
-char ModuleMemProfilerLegacyPass::ID = 0;
-
-INITIALIZE_PASS(ModuleMemProfilerLegacyPass, "memprof-module",
- "MemProfiler: profile memory allocations and accesses."
- "ModulePass",
- false, false)
-
-ModulePass *llvm::createModuleMemProfilerLegacyPassPass() {
- return new ModuleMemProfilerLegacyPass();
-}
-
-Value *MemProfiler::memToShadow(Value *Shadow, IRBuilder<> &IRB) {
- // (Shadow & mask) >> scale
- Shadow = IRB.CreateAnd(Shadow, Mapping.Mask);
- Shadow = IRB.CreateLShr(Shadow, Mapping.Scale);
- // (Shadow >> scale) | offset
- assert(DynamicShadowOffset);
- return IRB.CreateAdd(Shadow, DynamicShadowOffset);
-}
-
-// Instrument memset/memmove/memcpy
-void MemProfiler::instrumentMemIntrinsic(MemIntrinsic *MI) {
- IRBuilder<> IRB(MI);
- if (isa<MemTransferInst>(MI)) {
- IRB.CreateCall(
- isa<MemMoveInst>(MI) ? MemProfMemmove : MemProfMemcpy,
- {IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()),
- IRB.CreatePointerCast(MI->getOperand(1), IRB.getInt8PtrTy()),
- IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
- } else if (isa<MemSetInst>(MI)) {
- IRB.CreateCall(
- MemProfMemset,
- {IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()),
- IRB.CreateIntCast(MI->getOperand(1), IRB.getInt32Ty(), false),
- IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
- }
- MI->eraseFromParent();
-}
-
-Optional<InterestingMemoryAccess>
-MemProfiler::isInterestingMemoryAccess(Instruction *I) const {
- // Do not instrument the load fetching the dynamic shadow address.
- if (DynamicShadowOffset == I)
- return None;
-
- InterestingMemoryAccess Access;
-
- const DataLayout &DL = I->getModule()->getDataLayout();
- if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
- if (!ClInstrumentReads)
- return None;
- Access.IsWrite = false;
- Access.TypeSize = DL.getTypeStoreSizeInBits(LI->getType());
- Access.Alignment = LI->getAlignment();
- Access.Addr = LI->getPointerOperand();
- } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
- if (!ClInstrumentWrites)
- return None;
- Access.IsWrite = true;
- Access.TypeSize =
- DL.getTypeStoreSizeInBits(SI->getValueOperand()->getType());
- Access.Alignment = SI->getAlignment();
- Access.Addr = SI->getPointerOperand();
- } else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I)) {
- if (!ClInstrumentAtomics)
- return None;
- Access.IsWrite = true;
- Access.TypeSize =
- DL.getTypeStoreSizeInBits(RMW->getValOperand()->getType());
- Access.Alignment = 0;
- Access.Addr = RMW->getPointerOperand();
- } else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I)) {
- if (!ClInstrumentAtomics)
- return None;
- Access.IsWrite = true;
- Access.TypeSize =
- DL.getTypeStoreSizeInBits(XCHG->getCompareOperand()->getType());
- Access.Alignment = 0;
- Access.Addr = XCHG->getPointerOperand();
- } else if (auto *CI = dyn_cast<CallInst>(I)) {
- auto *F = CI->getCalledFunction();
- if (F && (F->getIntrinsicID() == Intrinsic::masked_load ||
- F->getIntrinsicID() == Intrinsic::masked_store)) {
- unsigned OpOffset = 0;
- if (F->getIntrinsicID() == Intrinsic::masked_store) {
- if (!ClInstrumentWrites)
- return None;
- // Masked store has an initial operand for the value.
- OpOffset = 1;
- Access.IsWrite = true;
- } else {
- if (!ClInstrumentReads)
- return None;
- Access.IsWrite = false;
- }
-
- auto *BasePtr = CI->getOperand(0 + OpOffset);
- auto *Ty = cast<PointerType>(BasePtr->getType())->getElementType();
- Access.TypeSize = DL.getTypeStoreSizeInBits(Ty);
- if (auto *AlignmentConstant =
- dyn_cast<ConstantInt>(CI->getOperand(1 + OpOffset)))
- Access.Alignment = (unsigned)AlignmentConstant->getZExtValue();
- else
- Access.Alignment = 1; // No alignment guarantees. We probably got Undef
- Access.MaybeMask = CI->getOperand(2 + OpOffset);
- Access.Addr = BasePtr;
- }
- }
-
- if (!Access.Addr)
- return None;
-
- // Do not instrument acesses from different address spaces; we cannot deal
- // with them.
- Type *PtrTy = cast<PointerType>(Access.Addr->getType()->getScalarType());
- if (PtrTy->getPointerAddressSpace() != 0)
- return None;
-
- // Ignore swifterror addresses.
- // swifterror memory addresses are mem2reg promoted by instruction
- // selection. As such they cannot have regular uses like an instrumentation
- // function and it makes no sense to track them as memory.
- if (Access.Addr->isSwiftError())
- return None;
-
- return Access;
-}
-
-void MemProfiler::instrumentMaskedLoadOrStore(const DataLayout &DL, Value *Mask,
- Instruction *I, Value *Addr,
- unsigned Alignment,
- uint32_t TypeSize, bool IsWrite) {
- auto *VTy = cast<FixedVectorType>(
- cast<PointerType>(Addr->getType())->getElementType());
- uint64_t ElemTypeSize = DL.getTypeStoreSizeInBits(VTy->getScalarType());
- unsigned Num = VTy->getNumElements();
- auto *Zero = ConstantInt::get(IntptrTy, 0);
- for (unsigned Idx = 0; Idx < Num; ++Idx) {
- Value *InstrumentedAddress = nullptr;
- Instruction *InsertBefore = I;
- if (auto *Vector = dyn_cast<ConstantVector>(Mask)) {
- // dyn_cast as we might get UndefValue
- if (auto *Masked = dyn_cast<ConstantInt>(Vector->getOperand(Idx))) {
- if (Masked->isZero())
- // Mask is constant false, so no instrumentation needed.
- continue;
- // If we have a true or undef value, fall through to instrumentAddress.
- // with InsertBefore == I
- }
- } else {
- IRBuilder<> IRB(I);
- Value *MaskElem = IRB.CreateExtractElement(Mask, Idx);
- Instruction *ThenTerm = SplitBlockAndInsertIfThen(MaskElem, I, false);
- InsertBefore = ThenTerm;
- }
-
- IRBuilder<> IRB(InsertBefore);
- InstrumentedAddress =
- IRB.CreateGEP(VTy, Addr, {Zero, ConstantInt::get(IntptrTy, Idx)});
- instrumentAddress(I, InsertBefore, InstrumentedAddress, ElemTypeSize,
- IsWrite);
- }
-}
-
-void MemProfiler::instrumentMop(Instruction *I, const DataLayout &DL,
- InterestingMemoryAccess &Access) {
- if (Access.IsWrite)
- NumInstrumentedWrites++;
- else
- NumInstrumentedReads++;
-
- if (Access.MaybeMask) {
- instrumentMaskedLoadOrStore(DL, Access.MaybeMask, I, Access.Addr,
- Access.Alignment, Access.TypeSize,
- Access.IsWrite);
- } else {
- // Since the access counts will be accumulated across the entire allocation,
- // we only update the shadow access count for the first location and thus
- // don't need to worry about alignment and type size.
- instrumentAddress(I, I, Access.Addr, Access.TypeSize, Access.IsWrite);
- }
-}
-
-void MemProfiler::instrumentAddress(Instruction *OrigIns,
- Instruction *InsertBefore, Value *Addr,
- uint32_t TypeSize, bool IsWrite) {
- IRBuilder<> IRB(InsertBefore);
- Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy);
-
- if (ClUseCalls) {
- IRB.CreateCall(MemProfMemoryAccessCallback[IsWrite], AddrLong);
- return;
- }
-
- // Create an inline sequence to compute shadow location, and increment the
- // value by one.
- Type *ShadowTy = Type::getInt64Ty(*C);
- Type *ShadowPtrTy = PointerType::get(ShadowTy, 0);
- Value *ShadowPtr = memToShadow(AddrLong, IRB);
- Value *ShadowAddr = IRB.CreateIntToPtr(ShadowPtr, ShadowPtrTy);
- Value *ShadowValue = IRB.CreateLoad(ShadowTy, ShadowAddr);
- Value *Inc = ConstantInt::get(Type::getInt64Ty(*C), 1);
- ShadowValue = IRB.CreateAdd(ShadowValue, Inc);
- IRB.CreateStore(ShadowValue, ShadowAddr);
-}
-
-// Create the variable for the profile file name.
-void createProfileFileNameVar(Module &M) {
- const MDString *MemProfFilename =
- dyn_cast_or_null<MDString>(M.getModuleFlag("MemProfProfileFilename"));
- if (!MemProfFilename)
- return;
- assert(!MemProfFilename->getString().empty() &&
- "Unexpected MemProfProfileFilename metadata with empty string");
- Constant *ProfileNameConst = ConstantDataArray::getString(
- M.getContext(), MemProfFilename->getString(), true);
- GlobalVariable *ProfileNameVar = new GlobalVariable(
- M, ProfileNameConst->getType(), /*isConstant=*/true,
- GlobalValue::WeakAnyLinkage, ProfileNameConst, MemProfFilenameVar);
- Triple TT(M.getTargetTriple());
- if (TT.supportsCOMDAT()) {
- ProfileNameVar->setLinkage(GlobalValue::ExternalLinkage);
- ProfileNameVar->setComdat(M.getOrInsertComdat(MemProfFilenameVar));
- }
-}
-
-bool ModuleMemProfiler::instrumentModule(Module &M) {
- // Create a module constructor.
- std::string MemProfVersion = std::to_string(LLVM_MEM_PROFILER_VERSION);
- std::string VersionCheckName =
- ClInsertVersionCheck ? (MemProfVersionCheckNamePrefix + MemProfVersion)
- : "";
- std::tie(MemProfCtorFunction, std::ignore) =
- createSanitizerCtorAndInitFunctions(M, MemProfModuleCtorName,
- MemProfInitName, /*InitArgTypes=*/{},
- /*InitArgs=*/{}, VersionCheckName);
-
- const uint64_t Priority = getCtorAndDtorPriority(TargetTriple);
- appendToGlobalCtors(M, MemProfCtorFunction, Priority);
-
- createProfileFileNameVar(M);
-
- return true;
-}
-
-void MemProfiler::initializeCallbacks(Module &M) {
- IRBuilder<> IRB(*C);
-
- for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) {
- const std::string TypeStr = AccessIsWrite ? "store" : "load";
-
- SmallVector<Type *, 3> Args2 = {IntptrTy, IntptrTy};
- SmallVector<Type *, 2> Args1{1, IntptrTy};
- MemProfMemoryAccessCallbackSized[AccessIsWrite] =
- M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + TypeStr + "N",
- FunctionType::get(IRB.getVoidTy(), Args2, false));
-
- MemProfMemoryAccessCallback[AccessIsWrite] =
- M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + TypeStr,
- FunctionType::get(IRB.getVoidTy(), Args1, false));
- }
- MemProfMemmove = M.getOrInsertFunction(
- ClMemoryAccessCallbackPrefix + "memmove", IRB.getInt8PtrTy(),
- IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy);
- MemProfMemcpy = M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + "memcpy",
- IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
- IRB.getInt8PtrTy(), IntptrTy);
- MemProfMemset = M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + "memset",
- IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
- IRB.getInt32Ty(), IntptrTy);
-}
-
-bool MemProfiler::maybeInsertMemProfInitAtFunctionEntry(Function &F) {
- // For each NSObject descendant having a +load method, this method is invoked
- // by the ObjC runtime before any of the static constructors is called.
- // Therefore we need to instrument such methods with a call to __memprof_init
- // at the beginning in order to initialize our runtime before any access to
- // the shadow memory.
- // We cannot just ignore these methods, because they may call other
- // instrumented functions.
- if (F.getName().find(" load]") != std::string::npos) {
- FunctionCallee MemProfInitFunction =
- declareSanitizerInitFunction(*F.getParent(), MemProfInitName, {});
- IRBuilder<> IRB(&F.front(), F.front().begin());
- IRB.CreateCall(MemProfInitFunction, {});
- return true;
- }
- return false;
-}
-
-bool MemProfiler::insertDynamicShadowAtFunctionEntry(Function &F) {
- IRBuilder<> IRB(&F.front().front());
- Value *GlobalDynamicAddress = F.getParent()->getOrInsertGlobal(
- MemProfShadowMemoryDynamicAddress, IntptrTy);
- if (F.getParent()->getPICLevel() == PICLevel::NotPIC)
- cast<GlobalVariable>(GlobalDynamicAddress)->setDSOLocal(true);
- DynamicShadowOffset = IRB.CreateLoad(IntptrTy, GlobalDynamicAddress);
- return true;
-}
-
-bool MemProfiler::instrumentFunction(Function &F) {
- if (F.getLinkage() == GlobalValue::AvailableExternallyLinkage)
- return false;
- if (ClDebugFunc == F.getName())
- return false;
- if (F.getName().startswith("__memprof_"))
- return false;
-
- bool FunctionModified = false;
-
- // If needed, insert __memprof_init.
- // This function needs to be called even if the function body is not
- // instrumented.
- if (maybeInsertMemProfInitAtFunctionEntry(F))
- FunctionModified = true;
-
- LLVM_DEBUG(dbgs() << "MEMPROF instrumenting:\n" << F << "\n");
-
- initializeCallbacks(*F.getParent());
-
- FunctionModified |= insertDynamicShadowAtFunctionEntry(F);
-
- SmallVector<Instruction *, 16> ToInstrument;
-
- // Fill the set of memory operations to instrument.
- for (auto &BB : F) {
- for (auto &Inst : BB) {
- if (isInterestingMemoryAccess(&Inst) || isa<MemIntrinsic>(Inst))
- ToInstrument.push_back(&Inst);
- }
- }
-
- int NumInstrumented = 0;
- for (auto *Inst : ToInstrument) {
- if (ClDebugMin < 0 || ClDebugMax < 0 ||
- (NumInstrumented >= ClDebugMin && NumInstrumented <= ClDebugMax)) {
- Optional<InterestingMemoryAccess> Access =
- isInterestingMemoryAccess(Inst);
- if (Access)
- instrumentMop(Inst, F.getParent()->getDataLayout(), *Access);
- else
- instrumentMemIntrinsic(cast<MemIntrinsic>(Inst));
- }
- NumInstrumented++;
- }
-
- if (NumInstrumented > 0)
- FunctionModified = true;
-
- LLVM_DEBUG(dbgs() << "MEMPROF done instrumenting: " << FunctionModified << " "
- << F << "\n");
-
- return FunctionModified;
-}
+//===- MemProfiler.cpp - memory allocation and access profiler ------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of MemProfiler. Memory accesses are instrumented
+// to increment the access count held in a shadow memory location, or
+// alternatively to call into the runtime. Memory intrinsic calls (memmove,
+// memcpy, memset) are changed to call the memory profiling runtime version
+// instead.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/Instrumentation/MemProfiler.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/Triple.h"
+#include "llvm/IR/Constant.h"
+#include "llvm/IR/DataLayout.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/GlobalValue.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/Instruction.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/Type.h"
+#include "llvm/IR/Value.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Pass.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Transforms/Instrumentation.h"
+#include "llvm/Transforms/Utils/BasicBlockUtils.h"
+#include "llvm/Transforms/Utils/ModuleUtils.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "memprof"
+
+constexpr int LLVM_MEM_PROFILER_VERSION = 1;
+
+// Size of memory mapped to a single shadow location.
+constexpr uint64_t DefaultShadowGranularity = 64;
+
+// Scale from granularity down to shadow size.
+constexpr uint64_t DefaultShadowScale = 3;
+
+constexpr char MemProfModuleCtorName[] = "memprof.module_ctor";
+constexpr uint64_t MemProfCtorAndDtorPriority = 1;
+// On Emscripten, the system needs more than one priorities for constructors.
+constexpr uint64_t MemProfEmscriptenCtorAndDtorPriority = 50;
+constexpr char MemProfInitName[] = "__memprof_init";
+constexpr char MemProfVersionCheckNamePrefix[] =
+ "__memprof_version_mismatch_check_v";
+
+constexpr char MemProfShadowMemoryDynamicAddress[] =
+ "__memprof_shadow_memory_dynamic_address";
+
+constexpr char MemProfFilenameVar[] = "__memprof_profile_filename";
+
+// Command-line flags.
+
+static cl::opt<bool> ClInsertVersionCheck(
+ "memprof-guard-against-version-mismatch",
+ cl::desc("Guard against compiler/runtime version mismatch."), cl::Hidden,
+ cl::init(true));
+
+// This flag may need to be replaced with -f[no-]memprof-reads.
+static cl::opt<bool> ClInstrumentReads("memprof-instrument-reads",
+ cl::desc("instrument read instructions"),
+ cl::Hidden, cl::init(true));
+
+static cl::opt<bool>
+ ClInstrumentWrites("memprof-instrument-writes",
+ cl::desc("instrument write instructions"), cl::Hidden,
+ cl::init(true));
+
+static cl::opt<bool> ClInstrumentAtomics(
+ "memprof-instrument-atomics",
+ cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden,
+ cl::init(true));
+
+static cl::opt<bool> ClUseCalls(
+ "memprof-use-callbacks",
+ cl::desc("Use callbacks instead of inline instrumentation sequences."),
+ cl::Hidden, cl::init(false));
+
+static cl::opt<std::string>
+ ClMemoryAccessCallbackPrefix("memprof-memory-access-callback-prefix",
+ cl::desc("Prefix for memory access callbacks"),
+ cl::Hidden, cl::init("__memprof_"));
+
+// These flags allow to change the shadow mapping.
+// The shadow mapping looks like
+// Shadow = ((Mem & mask) >> scale) + offset
+
+static cl::opt<int> ClMappingScale("memprof-mapping-scale",
+ cl::desc("scale of memprof shadow mapping"),
+ cl::Hidden, cl::init(DefaultShadowScale));
+
+static cl::opt<int>
+ ClMappingGranularity("memprof-mapping-granularity",
+ cl::desc("granularity of memprof shadow mapping"),
+ cl::Hidden, cl::init(DefaultShadowGranularity));
+
+// Debug flags.
+
+static cl::opt<int> ClDebug("memprof-debug", cl::desc("debug"), cl::Hidden,
+ cl::init(0));
+
+static cl::opt<std::string> ClDebugFunc("memprof-debug-func", cl::Hidden,
+ cl::desc("Debug func"));
+
+static cl::opt<int> ClDebugMin("memprof-debug-min", cl::desc("Debug min inst"),
+ cl::Hidden, cl::init(-1));
+
+static cl::opt<int> ClDebugMax("memprof-debug-max", cl::desc("Debug max inst"),
+ cl::Hidden, cl::init(-1));
+
+STATISTIC(NumInstrumentedReads, "Number of instrumented reads");
+STATISTIC(NumInstrumentedWrites, "Number of instrumented writes");
+
+namespace {
+
+/// This struct defines the shadow mapping using the rule:
+/// shadow = ((mem & mask) >> Scale) ADD DynamicShadowOffset.
+struct ShadowMapping {
+ ShadowMapping() {
+ Scale = ClMappingScale;
+ Granularity = ClMappingGranularity;
+ Mask = ~(Granularity - 1);
+ }
+
+ int Scale;
+ int Granularity;
+ uint64_t Mask; // Computed as ~(Granularity-1)
+};
+
+static uint64_t getCtorAndDtorPriority(Triple &TargetTriple) {
+ return TargetTriple.isOSEmscripten() ? MemProfEmscriptenCtorAndDtorPriority
+ : MemProfCtorAndDtorPriority;
+}
+
+struct InterestingMemoryAccess {
+ Value *Addr = nullptr;
+ bool IsWrite;
+ unsigned Alignment;
+ uint64_t TypeSize;
+ Value *MaybeMask = nullptr;
+};
+
+/// Instrument the code in module to profile memory accesses.
+class MemProfiler {
+public:
+ MemProfiler(Module &M) {
+ C = &(M.getContext());
+ LongSize = M.getDataLayout().getPointerSizeInBits();
+ IntptrTy = Type::getIntNTy(*C, LongSize);
+ }
+
+ /// If it is an interesting memory access, populate information
+ /// about the access and return a InterestingMemoryAccess struct.
+ /// Otherwise return None.
+ Optional<InterestingMemoryAccess>
+ isInterestingMemoryAccess(Instruction *I) const;
+
+ void instrumentMop(Instruction *I, const DataLayout &DL,
+ InterestingMemoryAccess &Access);
+ void instrumentAddress(Instruction *OrigIns, Instruction *InsertBefore,
+ Value *Addr, uint32_t TypeSize, bool IsWrite);
+ void instrumentMaskedLoadOrStore(const DataLayout &DL, Value *Mask,
+ Instruction *I, Value *Addr,
+ unsigned Alignment, uint32_t TypeSize,
+ bool IsWrite);
+ void instrumentMemIntrinsic(MemIntrinsic *MI);
+ Value *memToShadow(Value *Shadow, IRBuilder<> &IRB);
+ bool instrumentFunction(Function &F);
+ bool maybeInsertMemProfInitAtFunctionEntry(Function &F);
+ bool insertDynamicShadowAtFunctionEntry(Function &F);
+
+private:
+ void initializeCallbacks(Module &M);
+
+ LLVMContext *C;
+ int LongSize;
+ Type *IntptrTy;
+ ShadowMapping Mapping;
+
+ // These arrays is indexed by AccessIsWrite
+ FunctionCallee MemProfMemoryAccessCallback[2];
+ FunctionCallee MemProfMemoryAccessCallbackSized[2];
+
+ FunctionCallee MemProfMemmove, MemProfMemcpy, MemProfMemset;
+ Value *DynamicShadowOffset = nullptr;
+};
+
+class MemProfilerLegacyPass : public FunctionPass {
+public:
+ static char ID;
+
+ explicit MemProfilerLegacyPass() : FunctionPass(ID) {
+ initializeMemProfilerLegacyPassPass(*PassRegistry::getPassRegistry());
+ }
+
+ StringRef getPassName() const override { return "MemProfilerFunctionPass"; }
+
+ bool runOnFunction(Function &F) override {
+ MemProfiler Profiler(*F.getParent());
+ return Profiler.instrumentFunction(F);
+ }
+};
+
+class ModuleMemProfiler {
+public:
+ ModuleMemProfiler(Module &M) { TargetTriple = Triple(M.getTargetTriple()); }
+
+ bool instrumentModule(Module &);
+
+private:
+ Triple TargetTriple;
+ ShadowMapping Mapping;
+ Function *MemProfCtorFunction = nullptr;
+};
+
+class ModuleMemProfilerLegacyPass : public ModulePass {
+public:
+ static char ID;
+
+ explicit ModuleMemProfilerLegacyPass() : ModulePass(ID) {
+ initializeModuleMemProfilerLegacyPassPass(*PassRegistry::getPassRegistry());
+ }
+
+ StringRef getPassName() const override { return "ModuleMemProfiler"; }
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override {}
+
+ bool runOnModule(Module &M) override {
+ ModuleMemProfiler MemProfiler(M);
+ return MemProfiler.instrumentModule(M);
+ }
+};
+
+} // end anonymous namespace
+
+MemProfilerPass::MemProfilerPass() {}
+
+PreservedAnalyses MemProfilerPass::run(Function &F,
+ AnalysisManager<Function> &AM) {
+ Module &M = *F.getParent();
+ MemProfiler Profiler(M);
+ if (Profiler.instrumentFunction(F))
+ return PreservedAnalyses::none();
+ return PreservedAnalyses::all();
+
+ return PreservedAnalyses::all();
+}
+
+ModuleMemProfilerPass::ModuleMemProfilerPass() {}
+
+PreservedAnalyses ModuleMemProfilerPass::run(Module &M,
+ AnalysisManager<Module> &AM) {
+ ModuleMemProfiler Profiler(M);
+ if (Profiler.instrumentModule(M))
+ return PreservedAnalyses::none();
+ return PreservedAnalyses::all();
+}
+
+char MemProfilerLegacyPass::ID = 0;
+
+INITIALIZE_PASS_BEGIN(MemProfilerLegacyPass, "memprof",
+ "MemProfiler: profile memory allocations and accesses.",
+ false, false)
+INITIALIZE_PASS_END(MemProfilerLegacyPass, "memprof",
+ "MemProfiler: profile memory allocations and accesses.",
+ false, false)
+
+FunctionPass *llvm::createMemProfilerFunctionPass() {
+ return new MemProfilerLegacyPass();
+}
+
+char ModuleMemProfilerLegacyPass::ID = 0;
+
+INITIALIZE_PASS(ModuleMemProfilerLegacyPass, "memprof-module",
+ "MemProfiler: profile memory allocations and accesses."
+ "ModulePass",
+ false, false)
+
+ModulePass *llvm::createModuleMemProfilerLegacyPassPass() {
+ return new ModuleMemProfilerLegacyPass();
+}
+
+Value *MemProfiler::memToShadow(Value *Shadow, IRBuilder<> &IRB) {
+ // (Shadow & mask) >> scale
+ Shadow = IRB.CreateAnd(Shadow, Mapping.Mask);
+ Shadow = IRB.CreateLShr(Shadow, Mapping.Scale);
+ // (Shadow >> scale) | offset
+ assert(DynamicShadowOffset);
+ return IRB.CreateAdd(Shadow, DynamicShadowOffset);
+}
+
+// Instrument memset/memmove/memcpy
+void MemProfiler::instrumentMemIntrinsic(MemIntrinsic *MI) {
+ IRBuilder<> IRB(MI);
+ if (isa<MemTransferInst>(MI)) {
+ IRB.CreateCall(
+ isa<MemMoveInst>(MI) ? MemProfMemmove : MemProfMemcpy,
+ {IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()),
+ IRB.CreatePointerCast(MI->getOperand(1), IRB.getInt8PtrTy()),
+ IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
+ } else if (isa<MemSetInst>(MI)) {
+ IRB.CreateCall(
+ MemProfMemset,
+ {IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()),
+ IRB.CreateIntCast(MI->getOperand(1), IRB.getInt32Ty(), false),
+ IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
+ }
+ MI->eraseFromParent();
+}
+
+Optional<InterestingMemoryAccess>
+MemProfiler::isInterestingMemoryAccess(Instruction *I) const {
+ // Do not instrument the load fetching the dynamic shadow address.
+ if (DynamicShadowOffset == I)
+ return None;
+
+ InterestingMemoryAccess Access;
+
+ const DataLayout &DL = I->getModule()->getDataLayout();
+ if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
+ if (!ClInstrumentReads)
+ return None;
+ Access.IsWrite = false;
+ Access.TypeSize = DL.getTypeStoreSizeInBits(LI->getType());
+ Access.Alignment = LI->getAlignment();
+ Access.Addr = LI->getPointerOperand();
+ } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
+ if (!ClInstrumentWrites)
+ return None;
+ Access.IsWrite = true;
+ Access.TypeSize =
+ DL.getTypeStoreSizeInBits(SI->getValueOperand()->getType());
+ Access.Alignment = SI->getAlignment();
+ Access.Addr = SI->getPointerOperand();
+ } else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I)) {
+ if (!ClInstrumentAtomics)
+ return None;
+ Access.IsWrite = true;
+ Access.TypeSize =
+ DL.getTypeStoreSizeInBits(RMW->getValOperand()->getType());
+ Access.Alignment = 0;
+ Access.Addr = RMW->getPointerOperand();
+ } else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I)) {
+ if (!ClInstrumentAtomics)
+ return None;
+ Access.IsWrite = true;
+ Access.TypeSize =
+ DL.getTypeStoreSizeInBits(XCHG->getCompareOperand()->getType());
+ Access.Alignment = 0;
+ Access.Addr = XCHG->getPointerOperand();
+ } else if (auto *CI = dyn_cast<CallInst>(I)) {
+ auto *F = CI->getCalledFunction();
+ if (F && (F->getIntrinsicID() == Intrinsic::masked_load ||
+ F->getIntrinsicID() == Intrinsic::masked_store)) {
+ unsigned OpOffset = 0;
+ if (F->getIntrinsicID() == Intrinsic::masked_store) {
+ if (!ClInstrumentWrites)
+ return None;
+ // Masked store has an initial operand for the value.
+ OpOffset = 1;
+ Access.IsWrite = true;
+ } else {
+ if (!ClInstrumentReads)
+ return None;
+ Access.IsWrite = false;
+ }
+
+ auto *BasePtr = CI->getOperand(0 + OpOffset);
+ auto *Ty = cast<PointerType>(BasePtr->getType())->getElementType();
+ Access.TypeSize = DL.getTypeStoreSizeInBits(Ty);
+ if (auto *AlignmentConstant =
+ dyn_cast<ConstantInt>(CI->getOperand(1 + OpOffset)))
+ Access.Alignment = (unsigned)AlignmentConstant->getZExtValue();
+ else
+ Access.Alignment = 1; // No alignment guarantees. We probably got Undef
+ Access.MaybeMask = CI->getOperand(2 + OpOffset);
+ Access.Addr = BasePtr;
+ }
+ }
+
+ if (!Access.Addr)
+ return None;
+
+ // Do not instrument acesses from different address spaces; we cannot deal
+ // with them.
+ Type *PtrTy = cast<PointerType>(Access.Addr->getType()->getScalarType());
+ if (PtrTy->getPointerAddressSpace() != 0)
+ return None;
+
+ // Ignore swifterror addresses.
+ // swifterror memory addresses are mem2reg promoted by instruction
+ // selection. As such they cannot have regular uses like an instrumentation
+ // function and it makes no sense to track them as memory.
+ if (Access.Addr->isSwiftError())
+ return None;
+
+ return Access;
+}
+
+void MemProfiler::instrumentMaskedLoadOrStore(const DataLayout &DL, Value *Mask,
+ Instruction *I, Value *Addr,
+ unsigned Alignment,
+ uint32_t TypeSize, bool IsWrite) {
+ auto *VTy = cast<FixedVectorType>(
+ cast<PointerType>(Addr->getType())->getElementType());
+ uint64_t ElemTypeSize = DL.getTypeStoreSizeInBits(VTy->getScalarType());
+ unsigned Num = VTy->getNumElements();
+ auto *Zero = ConstantInt::get(IntptrTy, 0);
+ for (unsigned Idx = 0; Idx < Num; ++Idx) {
+ Value *InstrumentedAddress = nullptr;
+ Instruction *InsertBefore = I;
+ if (auto *Vector = dyn_cast<ConstantVector>(Mask)) {
+ // dyn_cast as we might get UndefValue
+ if (auto *Masked = dyn_cast<ConstantInt>(Vector->getOperand(Idx))) {
+ if (Masked->isZero())
+ // Mask is constant false, so no instrumentation needed.
+ continue;
+ // If we have a true or undef value, fall through to instrumentAddress.
+ // with InsertBefore == I
+ }
+ } else {
+ IRBuilder<> IRB(I);
+ Value *MaskElem = IRB.CreateExtractElement(Mask, Idx);
+ Instruction *ThenTerm = SplitBlockAndInsertIfThen(MaskElem, I, false);
+ InsertBefore = ThenTerm;
+ }
+
+ IRBuilder<> IRB(InsertBefore);
+ InstrumentedAddress =
+ IRB.CreateGEP(VTy, Addr, {Zero, ConstantInt::get(IntptrTy, Idx)});
+ instrumentAddress(I, InsertBefore, InstrumentedAddress, ElemTypeSize,
+ IsWrite);
+ }
+}
+
+void MemProfiler::instrumentMop(Instruction *I, const DataLayout &DL,
+ InterestingMemoryAccess &Access) {
+ if (Access.IsWrite)
+ NumInstrumentedWrites++;
+ else
+ NumInstrumentedReads++;
+
+ if (Access.MaybeMask) {
+ instrumentMaskedLoadOrStore(DL, Access.MaybeMask, I, Access.Addr,
+ Access.Alignment, Access.TypeSize,
+ Access.IsWrite);
+ } else {
+ // Since the access counts will be accumulated across the entire allocation,
+ // we only update the shadow access count for the first location and thus
+ // don't need to worry about alignment and type size.
+ instrumentAddress(I, I, Access.Addr, Access.TypeSize, Access.IsWrite);
+ }
+}
+
+void MemProfiler::instrumentAddress(Instruction *OrigIns,
+ Instruction *InsertBefore, Value *Addr,
+ uint32_t TypeSize, bool IsWrite) {
+ IRBuilder<> IRB(InsertBefore);
+ Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy);
+
+ if (ClUseCalls) {
+ IRB.CreateCall(MemProfMemoryAccessCallback[IsWrite], AddrLong);
+ return;
+ }
+
+ // Create an inline sequence to compute shadow location, and increment the
+ // value by one.
+ Type *ShadowTy = Type::getInt64Ty(*C);
+ Type *ShadowPtrTy = PointerType::get(ShadowTy, 0);
+ Value *ShadowPtr = memToShadow(AddrLong, IRB);
+ Value *ShadowAddr = IRB.CreateIntToPtr(ShadowPtr, ShadowPtrTy);
+ Value *ShadowValue = IRB.CreateLoad(ShadowTy, ShadowAddr);
+ Value *Inc = ConstantInt::get(Type::getInt64Ty(*C), 1);
+ ShadowValue = IRB.CreateAdd(ShadowValue, Inc);
+ IRB.CreateStore(ShadowValue, ShadowAddr);
+}
+
+// Create the variable for the profile file name.
+void createProfileFileNameVar(Module &M) {
+ const MDString *MemProfFilename =
+ dyn_cast_or_null<MDString>(M.getModuleFlag("MemProfProfileFilename"));
+ if (!MemProfFilename)
+ return;
+ assert(!MemProfFilename->getString().empty() &&
+ "Unexpected MemProfProfileFilename metadata with empty string");
+ Constant *ProfileNameConst = ConstantDataArray::getString(
+ M.getContext(), MemProfFilename->getString(), true);
+ GlobalVariable *ProfileNameVar = new GlobalVariable(
+ M, ProfileNameConst->getType(), /*isConstant=*/true,
+ GlobalValue::WeakAnyLinkage, ProfileNameConst, MemProfFilenameVar);
+ Triple TT(M.getTargetTriple());
+ if (TT.supportsCOMDAT()) {
+ ProfileNameVar->setLinkage(GlobalValue::ExternalLinkage);
+ ProfileNameVar->setComdat(M.getOrInsertComdat(MemProfFilenameVar));
+ }
+}
+
+bool ModuleMemProfiler::instrumentModule(Module &M) {
+ // Create a module constructor.
+ std::string MemProfVersion = std::to_string(LLVM_MEM_PROFILER_VERSION);
+ std::string VersionCheckName =
+ ClInsertVersionCheck ? (MemProfVersionCheckNamePrefix + MemProfVersion)
+ : "";
+ std::tie(MemProfCtorFunction, std::ignore) =
+ createSanitizerCtorAndInitFunctions(M, MemProfModuleCtorName,
+ MemProfInitName, /*InitArgTypes=*/{},
+ /*InitArgs=*/{}, VersionCheckName);
+
+ const uint64_t Priority = getCtorAndDtorPriority(TargetTriple);
+ appendToGlobalCtors(M, MemProfCtorFunction, Priority);
+
+ createProfileFileNameVar(M);
+
+ return true;
+}
+
+void MemProfiler::initializeCallbacks(Module &M) {
+ IRBuilder<> IRB(*C);
+
+ for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) {
+ const std::string TypeStr = AccessIsWrite ? "store" : "load";
+
+ SmallVector<Type *, 3> Args2 = {IntptrTy, IntptrTy};
+ SmallVector<Type *, 2> Args1{1, IntptrTy};
+ MemProfMemoryAccessCallbackSized[AccessIsWrite] =
+ M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + TypeStr + "N",
+ FunctionType::get(IRB.getVoidTy(), Args2, false));
+
+ MemProfMemoryAccessCallback[AccessIsWrite] =
+ M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + TypeStr,
+ FunctionType::get(IRB.getVoidTy(), Args1, false));
+ }
+ MemProfMemmove = M.getOrInsertFunction(
+ ClMemoryAccessCallbackPrefix + "memmove", IRB.getInt8PtrTy(),
+ IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy);
+ MemProfMemcpy = M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + "memcpy",
+ IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
+ IRB.getInt8PtrTy(), IntptrTy);
+ MemProfMemset = M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + "memset",
+ IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
+ IRB.getInt32Ty(), IntptrTy);
+}
+
+bool MemProfiler::maybeInsertMemProfInitAtFunctionEntry(Function &F) {
+ // For each NSObject descendant having a +load method, this method is invoked
+ // by the ObjC runtime before any of the static constructors is called.
+ // Therefore we need to instrument such methods with a call to __memprof_init
+ // at the beginning in order to initialize our runtime before any access to
+ // the shadow memory.
+ // We cannot just ignore these methods, because they may call other
+ // instrumented functions.
+ if (F.getName().find(" load]") != std::string::npos) {
+ FunctionCallee MemProfInitFunction =
+ declareSanitizerInitFunction(*F.getParent(), MemProfInitName, {});
+ IRBuilder<> IRB(&F.front(), F.front().begin());
+ IRB.CreateCall(MemProfInitFunction, {});
+ return true;
+ }
+ return false;
+}
+
+bool MemProfiler::insertDynamicShadowAtFunctionEntry(Function &F) {
+ IRBuilder<> IRB(&F.front().front());
+ Value *GlobalDynamicAddress = F.getParent()->getOrInsertGlobal(
+ MemProfShadowMemoryDynamicAddress, IntptrTy);
+ if (F.getParent()->getPICLevel() == PICLevel::NotPIC)
+ cast<GlobalVariable>(GlobalDynamicAddress)->setDSOLocal(true);
+ DynamicShadowOffset = IRB.CreateLoad(IntptrTy, GlobalDynamicAddress);
+ return true;
+}
+
+bool MemProfiler::instrumentFunction(Function &F) {
+ if (F.getLinkage() == GlobalValue::AvailableExternallyLinkage)
+ return false;
+ if (ClDebugFunc == F.getName())
+ return false;
+ if (F.getName().startswith("__memprof_"))
+ return false;
+
+ bool FunctionModified = false;
+
+ // If needed, insert __memprof_init.
+ // This function needs to be called even if the function body is not
+ // instrumented.
+ if (maybeInsertMemProfInitAtFunctionEntry(F))
+ FunctionModified = true;
+
+ LLVM_DEBUG(dbgs() << "MEMPROF instrumenting:\n" << F << "\n");
+
+ initializeCallbacks(*F.getParent());
+
+ FunctionModified |= insertDynamicShadowAtFunctionEntry(F);
+
+ SmallVector<Instruction *, 16> ToInstrument;
+
+ // Fill the set of memory operations to instrument.
+ for (auto &BB : F) {
+ for (auto &Inst : BB) {
+ if (isInterestingMemoryAccess(&Inst) || isa<MemIntrinsic>(Inst))
+ ToInstrument.push_back(&Inst);
+ }
+ }
+
+ int NumInstrumented = 0;
+ for (auto *Inst : ToInstrument) {
+ if (ClDebugMin < 0 || ClDebugMax < 0 ||
+ (NumInstrumented >= ClDebugMin && NumInstrumented <= ClDebugMax)) {
+ Optional<InterestingMemoryAccess> Access =
+ isInterestingMemoryAccess(Inst);
+ if (Access)
+ instrumentMop(Inst, F.getParent()->getDataLayout(), *Access);
+ else
+ instrumentMemIntrinsic(cast<MemIntrinsic>(Inst));
+ }
+ NumInstrumented++;
+ }
+
+ if (NumInstrumented > 0)
+ FunctionModified = true;
+
+ LLVM_DEBUG(dbgs() << "MEMPROF done instrumenting: " << FunctionModified << " "
+ << F << "\n");
+
+ return FunctionModified;
+}
diff --git a/contrib/libs/llvm12/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/contrib/libs/llvm12/lib/Transforms/Instrumentation/MemorySanitizer.cpp
index affc204924..7a6874584d 100644
--- a/contrib/libs/llvm12/lib/Transforms/Instrumentation/MemorySanitizer.cpp
+++ b/contrib/libs/llvm12/lib/Transforms/Instrumentation/MemorySanitizer.cpp
@@ -153,7 +153,7 @@
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/Triple.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
-#include "llvm/Analysis/ValueTracking.h"
+#include "llvm/Analysis/ValueTracking.h"
#include "llvm/IR/Argument.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/BasicBlock.h"
@@ -338,8 +338,8 @@ static cl::opt<uint64_t> ClOriginBase("msan-origin-base",
cl::desc("Define custom MSan OriginBase"),
cl::Hidden, cl::init(0));
-const char kMsanModuleCtorName[] = "msan.module_ctor";
-const char kMsanInitName[] = "__msan_init";
+const char kMsanModuleCtorName[] = "msan.module_ctor";
+const char kMsanInitName[] = "__msan_init";
namespace {
@@ -573,9 +573,9 @@ private:
/// uninitialized value and returns an updated origin id encoding this info.
FunctionCallee MsanChainOriginFn;
- /// Run-time helper that paints an origin over a region.
- FunctionCallee MsanSetOriginFn;
-
+ /// Run-time helper that paints an origin over a region.
+ FunctionCallee MsanSetOriginFn;
+
/// MSan runtime replacements for memmove, memcpy and memset.
FunctionCallee MemmoveFn, MemcpyFn, MemsetFn;
@@ -854,9 +854,9 @@ void MemorySanitizer::initializeCallbacks(Module &M) {
// instrumentation.
MsanChainOriginFn = M.getOrInsertFunction(
"__msan_chain_origin", IRB.getInt32Ty(), IRB.getInt32Ty());
- MsanSetOriginFn =
- M.getOrInsertFunction("__msan_set_origin", IRB.getVoidTy(),
- IRB.getInt8PtrTy(), IntptrTy, IRB.getInt32Ty());
+ MsanSetOriginFn =
+ M.getOrInsertFunction("__msan_set_origin", IRB.getVoidTy(),
+ IRB.getInt8PtrTy(), IntptrTy, IRB.getInt32Ty());
MemmoveFn = M.getOrInsertFunction(
"__msan_memmove", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
IRB.getInt8PtrTy(), IntptrTy);
@@ -1056,7 +1056,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
ValueMap<Value*, Value*> ShadowMap, OriginMap;
std::unique_ptr<VarArgHelper> VAHelper;
const TargetLibraryInfo *TLI;
- Instruction *FnPrologueEnd;
+ Instruction *FnPrologueEnd;
// The following flags disable parts of MSan instrumentation based on
// exclusion list contents and command-line options.
@@ -1088,31 +1088,31 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
PoisonStack = SanitizeFunction && ClPoisonStack;
PoisonUndef = SanitizeFunction && ClPoisonUndef;
- // In the presence of unreachable blocks, we may see Phi nodes with
- // incoming nodes from such blocks. Since InstVisitor skips unreachable
- // blocks, such nodes will not have any shadow value associated with them.
- // It's easier to remove unreachable blocks than deal with missing shadow.
- removeUnreachableBlocks(F);
-
+ // In the presence of unreachable blocks, we may see Phi nodes with
+ // incoming nodes from such blocks. Since InstVisitor skips unreachable
+ // blocks, such nodes will not have any shadow value associated with them.
+ // It's easier to remove unreachable blocks than deal with missing shadow.
+ removeUnreachableBlocks(F);
+
MS.initializeCallbacks(*F.getParent());
- FnPrologueEnd = IRBuilder<>(F.getEntryBlock().getFirstNonPHI())
- .CreateIntrinsic(Intrinsic::donothing, {}, {});
-
- if (MS.CompileKernel) {
- IRBuilder<> IRB(FnPrologueEnd);
- insertKmsanPrologue(IRB);
- }
-
+ FnPrologueEnd = IRBuilder<>(F.getEntryBlock().getFirstNonPHI())
+ .CreateIntrinsic(Intrinsic::donothing, {}, {});
+
+ if (MS.CompileKernel) {
+ IRBuilder<> IRB(FnPrologueEnd);
+ insertKmsanPrologue(IRB);
+ }
+
LLVM_DEBUG(if (!InsertChecks) dbgs()
<< "MemorySanitizer is not inserting checks into '"
<< F.getName() << "'\n");
}
- bool isInPrologue(Instruction &I) {
- return I.getParent() == FnPrologueEnd->getParent() &&
- (&I == FnPrologueEnd || I.comesBefore(FnPrologueEnd));
- }
-
+ bool isInPrologue(Instruction &I) {
+ return I.getParent() == FnPrologueEnd->getParent() &&
+ (&I == FnPrologueEnd || I.comesBefore(FnPrologueEnd));
+ }
+
Value *updateOrigin(Value *V, IRBuilder<> &IRB) {
if (MS.TrackOrigins <= 1) return V;
return IRB.CreateCall(MS.MsanChainOriginFn, V);
@@ -1164,31 +1164,31 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
const DataLayout &DL = F.getParent()->getDataLayout();
const Align OriginAlignment = std::max(kMinOriginAlignment, Alignment);
unsigned StoreSize = DL.getTypeStoreSize(Shadow->getType());
- Value *ConvertedShadow = convertShadowToScalar(Shadow, IRB);
- if (auto *ConstantShadow = dyn_cast<Constant>(ConvertedShadow)) {
- if (ClCheckConstantShadow && !ConstantShadow->isZeroValue())
- paintOrigin(IRB, updateOrigin(Origin, IRB), OriginPtr, StoreSize,
+ Value *ConvertedShadow = convertShadowToScalar(Shadow, IRB);
+ if (auto *ConstantShadow = dyn_cast<Constant>(ConvertedShadow)) {
+ if (ClCheckConstantShadow && !ConstantShadow->isZeroValue())
+ paintOrigin(IRB, updateOrigin(Origin, IRB), OriginPtr, StoreSize,
OriginAlignment);
- return;
+ return;
+ }
+
+ unsigned TypeSizeInBits = DL.getTypeSizeInBits(ConvertedShadow->getType());
+ unsigned SizeIndex = TypeSizeToSizeIndex(TypeSizeInBits);
+ if (AsCall && SizeIndex < kNumberOfAccessSizes && !MS.CompileKernel) {
+ FunctionCallee Fn = MS.MaybeStoreOriginFn[SizeIndex];
+ Value *ConvertedShadow2 =
+ IRB.CreateZExt(ConvertedShadow, IRB.getIntNTy(8 * (1 << SizeIndex)));
+ IRB.CreateCall(Fn,
+ {ConvertedShadow2,
+ IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy()), Origin});
+ } else {
+ Value *Cmp = convertToBool(ConvertedShadow, IRB, "_mscmp");
+ Instruction *CheckTerm = SplitBlockAndInsertIfThen(
+ Cmp, &*IRB.GetInsertPoint(), false, MS.OriginStoreWeights);
+ IRBuilder<> IRBNew(CheckTerm);
+ paintOrigin(IRBNew, updateOrigin(Origin, IRBNew), OriginPtr, StoreSize,
+ OriginAlignment);
}
-
- unsigned TypeSizeInBits = DL.getTypeSizeInBits(ConvertedShadow->getType());
- unsigned SizeIndex = TypeSizeToSizeIndex(TypeSizeInBits);
- if (AsCall && SizeIndex < kNumberOfAccessSizes && !MS.CompileKernel) {
- FunctionCallee Fn = MS.MaybeStoreOriginFn[SizeIndex];
- Value *ConvertedShadow2 =
- IRB.CreateZExt(ConvertedShadow, IRB.getIntNTy(8 * (1 << SizeIndex)));
- IRB.CreateCall(Fn,
- {ConvertedShadow2,
- IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy()), Origin});
- } else {
- Value *Cmp = convertToBool(ConvertedShadow, IRB, "_mscmp");
- Instruction *CheckTerm = SplitBlockAndInsertIfThen(
- Cmp, &*IRB.GetInsertPoint(), false, MS.OriginStoreWeights);
- IRBuilder<> IRBNew(CheckTerm);
- paintOrigin(IRBNew, updateOrigin(Origin, IRBNew), OriginPtr, StoreSize,
- OriginAlignment);
- }
}
void materializeStores(bool InstrumentWithCalls) {
@@ -1232,7 +1232,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
bool AsCall) {
IRBuilder<> IRB(OrigIns);
LLVM_DEBUG(dbgs() << " SHAD0 : " << *Shadow << "\n");
- Value *ConvertedShadow = convertShadowToScalar(Shadow, IRB);
+ Value *ConvertedShadow = convertShadowToScalar(Shadow, IRB);
LLVM_DEBUG(dbgs() << " SHAD1 : " << *ConvertedShadow << "\n");
if (auto *ConstantShadow = dyn_cast<Constant>(ConvertedShadow)) {
@@ -1254,7 +1254,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
? Origin
: (Value *)IRB.getInt32(0)});
} else {
- Value *Cmp = convertToBool(ConvertedShadow, IRB, "_mscmp");
+ Value *Cmp = convertToBool(ConvertedShadow, IRB, "_mscmp");
Instruction *CheckTerm = SplitBlockAndInsertIfThen(
Cmp, OrigIns,
/* Unreachable */ !MS.Recover, MS.ColdCallWeights);
@@ -1275,8 +1275,8 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
LLVM_DEBUG(dbgs() << "DONE:\n" << F);
}
- // Returns the last instruction in the new prologue
- void insertKmsanPrologue(IRBuilder<> &IRB) {
+ // Returns the last instruction in the new prologue
+ void insertKmsanPrologue(IRBuilder<> &IRB) {
Value *ContextState = IRB.CreateCall(MS.MsanGetContextStateFn, {});
Constant *Zero = IRB.getInt32(0);
MS.ParamTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
@@ -1302,7 +1302,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
// Iterate all BBs in depth-first order and create shadow instructions
// for all instructions (where applicable).
// For PHI nodes we create dummy shadow PHIs which will be finalized later.
- for (BasicBlock *BB : depth_first(FnPrologueEnd->getParent()))
+ for (BasicBlock *BB : depth_first(FnPrologueEnd->getParent()))
visit(*BB);
// Finalize PHI nodes.
@@ -1389,68 +1389,68 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
return ty;
}
- /// Extract combined shadow of struct elements as a bool
- Value *collapseStructShadow(StructType *Struct, Value *Shadow,
- IRBuilder<> &IRB) {
- Value *FalseVal = IRB.getIntN(/* width */ 1, /* value */ 0);
- Value *Aggregator = FalseVal;
-
- for (unsigned Idx = 0; Idx < Struct->getNumElements(); Idx++) {
- // Combine by ORing together each element's bool shadow
- Value *ShadowItem = IRB.CreateExtractValue(Shadow, Idx);
- Value *ShadowInner = convertShadowToScalar(ShadowItem, IRB);
- Value *ShadowBool = convertToBool(ShadowInner, IRB);
-
- if (Aggregator != FalseVal)
- Aggregator = IRB.CreateOr(Aggregator, ShadowBool);
- else
- Aggregator = ShadowBool;
- }
-
- return Aggregator;
- }
-
- // Extract combined shadow of array elements
- Value *collapseArrayShadow(ArrayType *Array, Value *Shadow,
- IRBuilder<> &IRB) {
- if (!Array->getNumElements())
- return IRB.getIntN(/* width */ 1, /* value */ 0);
-
- Value *FirstItem = IRB.CreateExtractValue(Shadow, 0);
- Value *Aggregator = convertShadowToScalar(FirstItem, IRB);
-
- for (unsigned Idx = 1; Idx < Array->getNumElements(); Idx++) {
- Value *ShadowItem = IRB.CreateExtractValue(Shadow, Idx);
- Value *ShadowInner = convertShadowToScalar(ShadowItem, IRB);
- Aggregator = IRB.CreateOr(Aggregator, ShadowInner);
- }
- return Aggregator;
- }
-
- /// Convert a shadow value to it's flattened variant. The resulting
- /// shadow may not necessarily have the same bit width as the input
- /// value, but it will always be comparable to zero.
- Value *convertShadowToScalar(Value *V, IRBuilder<> &IRB) {
- if (StructType *Struct = dyn_cast<StructType>(V->getType()))
- return collapseStructShadow(Struct, V, IRB);
- if (ArrayType *Array = dyn_cast<ArrayType>(V->getType()))
- return collapseArrayShadow(Array, V, IRB);
+ /// Extract combined shadow of struct elements as a bool
+ Value *collapseStructShadow(StructType *Struct, Value *Shadow,
+ IRBuilder<> &IRB) {
+ Value *FalseVal = IRB.getIntN(/* width */ 1, /* value */ 0);
+ Value *Aggregator = FalseVal;
+
+ for (unsigned Idx = 0; Idx < Struct->getNumElements(); Idx++) {
+ // Combine by ORing together each element's bool shadow
+ Value *ShadowItem = IRB.CreateExtractValue(Shadow, Idx);
+ Value *ShadowInner = convertShadowToScalar(ShadowItem, IRB);
+ Value *ShadowBool = convertToBool(ShadowInner, IRB);
+
+ if (Aggregator != FalseVal)
+ Aggregator = IRB.CreateOr(Aggregator, ShadowBool);
+ else
+ Aggregator = ShadowBool;
+ }
+
+ return Aggregator;
+ }
+
+ // Extract combined shadow of array elements
+ Value *collapseArrayShadow(ArrayType *Array, Value *Shadow,
+ IRBuilder<> &IRB) {
+ if (!Array->getNumElements())
+ return IRB.getIntN(/* width */ 1, /* value */ 0);
+
+ Value *FirstItem = IRB.CreateExtractValue(Shadow, 0);
+ Value *Aggregator = convertShadowToScalar(FirstItem, IRB);
+
+ for (unsigned Idx = 1; Idx < Array->getNumElements(); Idx++) {
+ Value *ShadowItem = IRB.CreateExtractValue(Shadow, Idx);
+ Value *ShadowInner = convertShadowToScalar(ShadowItem, IRB);
+ Aggregator = IRB.CreateOr(Aggregator, ShadowInner);
+ }
+ return Aggregator;
+ }
+
+ /// Convert a shadow value to it's flattened variant. The resulting
+ /// shadow may not necessarily have the same bit width as the input
+ /// value, but it will always be comparable to zero.
+ Value *convertShadowToScalar(Value *V, IRBuilder<> &IRB) {
+ if (StructType *Struct = dyn_cast<StructType>(V->getType()))
+ return collapseStructShadow(Struct, V, IRB);
+ if (ArrayType *Array = dyn_cast<ArrayType>(V->getType()))
+ return collapseArrayShadow(Array, V, IRB);
Type *Ty = V->getType();
Type *NoVecTy = getShadowTyNoVec(Ty);
if (Ty == NoVecTy) return V;
return IRB.CreateBitCast(V, NoVecTy);
}
- // Convert a scalar value to an i1 by comparing with 0
- Value *convertToBool(Value *V, IRBuilder<> &IRB, const Twine &name = "") {
- Type *VTy = V->getType();
- assert(VTy->isIntegerTy());
- if (VTy->getIntegerBitWidth() == 1)
- // Just converting a bool to a bool, so do nothing.
- return V;
- return IRB.CreateICmpNE(V, ConstantInt::get(VTy, 0), name);
- }
-
+ // Convert a scalar value to an i1 by comparing with 0
+ Value *convertToBool(Value *V, IRBuilder<> &IRB, const Twine &name = "") {
+ Type *VTy = V->getType();
+ assert(VTy->isIntegerTy());
+ if (VTy->getIntegerBitWidth() == 1)
+ // Just converting a bool to a bool, so do nothing.
+ return V;
+ return IRB.CreateICmpNE(V, ConstantInt::get(VTy, 0), name);
+ }
+
/// Compute the integer shadow offset that corresponds to a given
/// application address.
///
@@ -1669,7 +1669,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
if (*ShadowPtr)
return *ShadowPtr;
Function *F = A->getParent();
- IRBuilder<> EntryIRB(FnPrologueEnd);
+ IRBuilder<> EntryIRB(FnPrologueEnd);
unsigned ArgOffset = 0;
const DataLayout &DL = F->getParent()->getDataLayout();
for (auto &FArg : F->args()) {
@@ -1737,8 +1737,8 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
} else {
setOrigin(A, getCleanOrigin());
}
-
- break;
+
+ break;
}
if (!FArgEagerCheck)
@@ -1786,10 +1786,10 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
if (!InsertChecks) return;
#ifndef NDEBUG
Type *ShadowTy = Shadow->getType();
- assert((isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy) ||
- isa<StructType>(ShadowTy) || isa<ArrayType>(ShadowTy)) &&
- "Can only insert checks for integer, vector, and aggregate shadow "
- "types");
+ assert((isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy) ||
+ isa<StructType>(ShadowTy) || isa<ArrayType>(ShadowTy)) &&
+ "Can only insert checks for integer, vector, and aggregate shadow "
+ "types");
#endif
InstrumentationList.push_back(
ShadowOriginAndInsertPoint(Shadow, Origin, OrigIns));
@@ -1831,24 +1831,24 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
llvm_unreachable("Unknown ordering");
}
- Value *makeAddReleaseOrderingTable(IRBuilder<> &IRB) {
- constexpr int NumOrderings = (int)AtomicOrderingCABI::seq_cst + 1;
- uint32_t OrderingTable[NumOrderings] = {};
-
- OrderingTable[(int)AtomicOrderingCABI::relaxed] =
- OrderingTable[(int)AtomicOrderingCABI::release] =
- (int)AtomicOrderingCABI::release;
- OrderingTable[(int)AtomicOrderingCABI::consume] =
- OrderingTable[(int)AtomicOrderingCABI::acquire] =
- OrderingTable[(int)AtomicOrderingCABI::acq_rel] =
- (int)AtomicOrderingCABI::acq_rel;
- OrderingTable[(int)AtomicOrderingCABI::seq_cst] =
- (int)AtomicOrderingCABI::seq_cst;
-
- return ConstantDataVector::get(IRB.getContext(),
- makeArrayRef(OrderingTable, NumOrderings));
- }
-
+ Value *makeAddReleaseOrderingTable(IRBuilder<> &IRB) {
+ constexpr int NumOrderings = (int)AtomicOrderingCABI::seq_cst + 1;
+ uint32_t OrderingTable[NumOrderings] = {};
+
+ OrderingTable[(int)AtomicOrderingCABI::relaxed] =
+ OrderingTable[(int)AtomicOrderingCABI::release] =
+ (int)AtomicOrderingCABI::release;
+ OrderingTable[(int)AtomicOrderingCABI::consume] =
+ OrderingTable[(int)AtomicOrderingCABI::acquire] =
+ OrderingTable[(int)AtomicOrderingCABI::acq_rel] =
+ (int)AtomicOrderingCABI::acq_rel;
+ OrderingTable[(int)AtomicOrderingCABI::seq_cst] =
+ (int)AtomicOrderingCABI::seq_cst;
+
+ return ConstantDataVector::get(IRB.getContext(),
+ makeArrayRef(OrderingTable, NumOrderings));
+ }
+
AtomicOrdering addAcquireOrdering(AtomicOrdering a) {
switch (a) {
case AtomicOrdering::NotAtomic:
@@ -1866,33 +1866,33 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
llvm_unreachable("Unknown ordering");
}
- Value *makeAddAcquireOrderingTable(IRBuilder<> &IRB) {
- constexpr int NumOrderings = (int)AtomicOrderingCABI::seq_cst + 1;
- uint32_t OrderingTable[NumOrderings] = {};
-
- OrderingTable[(int)AtomicOrderingCABI::relaxed] =
- OrderingTable[(int)AtomicOrderingCABI::acquire] =
- OrderingTable[(int)AtomicOrderingCABI::consume] =
- (int)AtomicOrderingCABI::acquire;
- OrderingTable[(int)AtomicOrderingCABI::release] =
- OrderingTable[(int)AtomicOrderingCABI::acq_rel] =
- (int)AtomicOrderingCABI::acq_rel;
- OrderingTable[(int)AtomicOrderingCABI::seq_cst] =
- (int)AtomicOrderingCABI::seq_cst;
-
- return ConstantDataVector::get(IRB.getContext(),
- makeArrayRef(OrderingTable, NumOrderings));
- }
-
+ Value *makeAddAcquireOrderingTable(IRBuilder<> &IRB) {
+ constexpr int NumOrderings = (int)AtomicOrderingCABI::seq_cst + 1;
+ uint32_t OrderingTable[NumOrderings] = {};
+
+ OrderingTable[(int)AtomicOrderingCABI::relaxed] =
+ OrderingTable[(int)AtomicOrderingCABI::acquire] =
+ OrderingTable[(int)AtomicOrderingCABI::consume] =
+ (int)AtomicOrderingCABI::acquire;
+ OrderingTable[(int)AtomicOrderingCABI::release] =
+ OrderingTable[(int)AtomicOrderingCABI::acq_rel] =
+ (int)AtomicOrderingCABI::acq_rel;
+ OrderingTable[(int)AtomicOrderingCABI::seq_cst] =
+ (int)AtomicOrderingCABI::seq_cst;
+
+ return ConstantDataVector::get(IRB.getContext(),
+ makeArrayRef(OrderingTable, NumOrderings));
+ }
+
// ------------------- Visitors.
using InstVisitor<MemorySanitizerVisitor>::visit;
void visit(Instruction &I) {
- if (I.getMetadata("nosanitize"))
- return;
- // Don't want to visit if we're in the prologue
- if (isInPrologue(I))
- return;
- InstVisitor<MemorySanitizerVisitor>::visit(I);
+ if (I.getMetadata("nosanitize"))
+ return;
+ // Don't want to visit if we're in the prologue
+ if (isInPrologue(I))
+ return;
+ InstVisitor<MemorySanitizerVisitor>::visit(I);
}
/// Instrument LoadInst
@@ -2148,7 +2148,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
Constant *ConstOrigin = dyn_cast<Constant>(OpOrigin);
// No point in adding something that might result in 0 origin value.
if (!ConstOrigin || !ConstOrigin->isNullValue()) {
- Value *FlatShadow = MSV->convertShadowToScalar(OpShadow, IRB);
+ Value *FlatShadow = MSV->convertShadowToScalar(OpShadow, IRB);
Value *Cond =
IRB.CreateICmpNE(FlatShadow, MSV->getCleanShadow(FlatShadow));
Origin = IRB.CreateSelect(Cond, OpOrigin, Origin);
@@ -2703,7 +2703,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
void handleLifetimeStart(IntrinsicInst &I) {
if (!PoisonStack)
return;
- AllocaInst *AI = llvm::findAllocaForValue(I.getArgOperand(1));
+ AllocaInst *AI = llvm::findAllocaForValue(I.getArgOperand(1));
if (!AI)
InstrumentLifetimeStart = false;
LifetimeStartList.push_back(std::make_pair(&I, AI));
@@ -2734,16 +2734,16 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
// We copy the shadow of \p CopyOp[NumUsedElements:] to \p
// Out[NumUsedElements:]. This means that intrinsics without \p CopyOp always
// return a fully initialized value.
- void handleVectorConvertIntrinsic(IntrinsicInst &I, int NumUsedElements,
- bool HasRoundingMode = false) {
+ void handleVectorConvertIntrinsic(IntrinsicInst &I, int NumUsedElements,
+ bool HasRoundingMode = false) {
IRBuilder<> IRB(&I);
Value *CopyOp, *ConvertOp;
- assert((!HasRoundingMode ||
- isa<ConstantInt>(I.getArgOperand(I.getNumArgOperands() - 1))) &&
- "Invalid rounding mode");
-
- switch (I.getNumArgOperands() - HasRoundingMode) {
+ assert((!HasRoundingMode ||
+ isa<ConstantInt>(I.getArgOperand(I.getNumArgOperands() - 1))) &&
+ "Invalid rounding mode");
+
+ switch (I.getNumArgOperands() - HasRoundingMode) {
case 2:
CopyOp = I.getArgOperand(0);
ConvertOp = I.getArgOperand(1);
@@ -2999,7 +2999,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
setOrigin(&I, getOrigin(&I, 0));
}
- // Instrument vector.reduce.or intrinsic.
+ // Instrument vector.reduce.or intrinsic.
// Valid (non-poisoned) set bits in the operand pull low the
// corresponding shadow bits.
void handleVectorReduceOrIntrinsic(IntrinsicInst &I) {
@@ -3017,7 +3017,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
setOrigin(&I, getOrigin(&I, 0));
}
- // Instrument vector.reduce.and intrinsic.
+ // Instrument vector.reduce.and intrinsic.
// Valid (non-poisoned) unset bits in the operand pull down the
// corresponding shadow bits.
void handleVectorReduceAndIntrinsic(IntrinsicInst &I) {
@@ -3195,10 +3195,10 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
assert(isa<ConstantInt>(I.getArgOperand(2)) &&
"pclmul 3rd operand must be a constant");
unsigned Imm = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue();
- Value *Shuf0 = IRB.CreateShuffleVector(getShadow(&I, 0),
- getPclmulMask(Width, Imm & 0x01));
- Value *Shuf1 = IRB.CreateShuffleVector(getShadow(&I, 1),
- getPclmulMask(Width, Imm & 0x10));
+ Value *Shuf0 = IRB.CreateShuffleVector(getShadow(&I, 0),
+ getPclmulMask(Width, Imm & 0x01));
+ Value *Shuf1 = IRB.CreateShuffleVector(getShadow(&I, 1),
+ getPclmulMask(Width, Imm & 0x10));
ShadowAndOriginCombiner SOC(this, IRB);
SOC.Add(Shuf0, getOrigin(&I, 0));
SOC.Add(Shuf1, getOrigin(&I, 1));
@@ -3231,24 +3231,24 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
setOriginForNaryOp(I);
}
- // Instrument abs intrinsic.
- // handleUnknownIntrinsic can't handle it because of the last
- // is_int_min_poison argument which does not match the result type.
- void handleAbsIntrinsic(IntrinsicInst &I) {
- assert(I.getType()->isIntOrIntVectorTy());
- assert(I.getArgOperand(0)->getType() == I.getType());
-
- // FIXME: Handle is_int_min_poison.
- IRBuilder<> IRB(&I);
- setShadow(&I, getShadow(&I, 0));
- setOrigin(&I, getOrigin(&I, 0));
- }
-
+ // Instrument abs intrinsic.
+ // handleUnknownIntrinsic can't handle it because of the last
+ // is_int_min_poison argument which does not match the result type.
+ void handleAbsIntrinsic(IntrinsicInst &I) {
+ assert(I.getType()->isIntOrIntVectorTy());
+ assert(I.getArgOperand(0)->getType() == I.getType());
+
+ // FIXME: Handle is_int_min_poison.
+ IRBuilder<> IRB(&I);
+ setShadow(&I, getShadow(&I, 0));
+ setOrigin(&I, getOrigin(&I, 0));
+ }
+
void visitIntrinsicInst(IntrinsicInst &I) {
switch (I.getIntrinsicID()) {
- case Intrinsic::abs:
- handleAbsIntrinsic(I);
- break;
+ case Intrinsic::abs:
+ handleAbsIntrinsic(I);
+ break;
case Intrinsic::lifetime_start:
handleLifetimeStart(I);
break;
@@ -3265,15 +3265,15 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
case Intrinsic::masked_load:
handleMaskedLoad(I);
break;
- case Intrinsic::vector_reduce_and:
+ case Intrinsic::vector_reduce_and:
handleVectorReduceAndIntrinsic(I);
break;
- case Intrinsic::vector_reduce_or:
+ case Intrinsic::vector_reduce_or:
handleVectorReduceOrIntrinsic(I);
break;
- case Intrinsic::vector_reduce_add:
- case Intrinsic::vector_reduce_xor:
- case Intrinsic::vector_reduce_mul:
+ case Intrinsic::vector_reduce_add:
+ case Intrinsic::vector_reduce_xor:
+ case Intrinsic::vector_reduce_mul:
handleVectorReduceIntrinsic(I);
break;
case Intrinsic::x86_sse_stmxcsr:
@@ -3293,8 +3293,8 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
case Intrinsic::x86_avx512_cvtusi2ss:
case Intrinsic::x86_avx512_cvtusi642sd:
case Intrinsic::x86_avx512_cvtusi642ss:
- handleVectorConvertIntrinsic(I, 1, true);
- break;
+ handleVectorConvertIntrinsic(I, 1, true);
+ break;
case Intrinsic::x86_sse2_cvtsd2si64:
case Intrinsic::x86_sse2_cvtsd2si:
case Intrinsic::x86_sse2_cvtsd2ss:
@@ -3520,63 +3520,63 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
}
}
- void visitLibAtomicLoad(CallBase &CB) {
- // Since we use getNextNode here, we can't have CB terminate the BB.
- assert(isa<CallInst>(CB));
-
- IRBuilder<> IRB(&CB);
- Value *Size = CB.getArgOperand(0);
- Value *SrcPtr = CB.getArgOperand(1);
- Value *DstPtr = CB.getArgOperand(2);
- Value *Ordering = CB.getArgOperand(3);
- // Convert the call to have at least Acquire ordering to make sure
- // the shadow operations aren't reordered before it.
- Value *NewOrdering =
- IRB.CreateExtractElement(makeAddAcquireOrderingTable(IRB), Ordering);
- CB.setArgOperand(3, NewOrdering);
-
- IRBuilder<> NextIRB(CB.getNextNode());
- NextIRB.SetCurrentDebugLocation(CB.getDebugLoc());
-
- Value *SrcShadowPtr, *SrcOriginPtr;
- std::tie(SrcShadowPtr, SrcOriginPtr) =
- getShadowOriginPtr(SrcPtr, NextIRB, NextIRB.getInt8Ty(), Align(1),
- /*isStore*/ false);
- Value *DstShadowPtr =
- getShadowOriginPtr(DstPtr, NextIRB, NextIRB.getInt8Ty(), Align(1),
- /*isStore*/ true)
- .first;
-
- NextIRB.CreateMemCpy(DstShadowPtr, Align(1), SrcShadowPtr, Align(1), Size);
- if (MS.TrackOrigins) {
- Value *SrcOrigin = NextIRB.CreateAlignedLoad(MS.OriginTy, SrcOriginPtr,
- kMinOriginAlignment);
- Value *NewOrigin = updateOrigin(SrcOrigin, NextIRB);
- NextIRB.CreateCall(MS.MsanSetOriginFn, {DstPtr, Size, NewOrigin});
- }
- }
-
- void visitLibAtomicStore(CallBase &CB) {
- IRBuilder<> IRB(&CB);
- Value *Size = CB.getArgOperand(0);
- Value *DstPtr = CB.getArgOperand(2);
- Value *Ordering = CB.getArgOperand(3);
- // Convert the call to have at least Release ordering to make sure
- // the shadow operations aren't reordered after it.
- Value *NewOrdering =
- IRB.CreateExtractElement(makeAddReleaseOrderingTable(IRB), Ordering);
- CB.setArgOperand(3, NewOrdering);
-
- Value *DstShadowPtr =
- getShadowOriginPtr(DstPtr, IRB, IRB.getInt8Ty(), Align(1),
- /*isStore*/ true)
- .first;
-
- // Atomic store always paints clean shadow/origin. See file header.
- IRB.CreateMemSet(DstShadowPtr, getCleanShadow(IRB.getInt8Ty()), Size,
- Align(1));
- }
-
+ void visitLibAtomicLoad(CallBase &CB) {
+ // Since we use getNextNode here, we can't have CB terminate the BB.
+ assert(isa<CallInst>(CB));
+
+ IRBuilder<> IRB(&CB);
+ Value *Size = CB.getArgOperand(0);
+ Value *SrcPtr = CB.getArgOperand(1);
+ Value *DstPtr = CB.getArgOperand(2);
+ Value *Ordering = CB.getArgOperand(3);
+ // Convert the call to have at least Acquire ordering to make sure
+ // the shadow operations aren't reordered before it.
+ Value *NewOrdering =
+ IRB.CreateExtractElement(makeAddAcquireOrderingTable(IRB), Ordering);
+ CB.setArgOperand(3, NewOrdering);
+
+ IRBuilder<> NextIRB(CB.getNextNode());
+ NextIRB.SetCurrentDebugLocation(CB.getDebugLoc());
+
+ Value *SrcShadowPtr, *SrcOriginPtr;
+ std::tie(SrcShadowPtr, SrcOriginPtr) =
+ getShadowOriginPtr(SrcPtr, NextIRB, NextIRB.getInt8Ty(), Align(1),
+ /*isStore*/ false);
+ Value *DstShadowPtr =
+ getShadowOriginPtr(DstPtr, NextIRB, NextIRB.getInt8Ty(), Align(1),
+ /*isStore*/ true)
+ .first;
+
+ NextIRB.CreateMemCpy(DstShadowPtr, Align(1), SrcShadowPtr, Align(1), Size);
+ if (MS.TrackOrigins) {
+ Value *SrcOrigin = NextIRB.CreateAlignedLoad(MS.OriginTy, SrcOriginPtr,
+ kMinOriginAlignment);
+ Value *NewOrigin = updateOrigin(SrcOrigin, NextIRB);
+ NextIRB.CreateCall(MS.MsanSetOriginFn, {DstPtr, Size, NewOrigin});
+ }
+ }
+
+ void visitLibAtomicStore(CallBase &CB) {
+ IRBuilder<> IRB(&CB);
+ Value *Size = CB.getArgOperand(0);
+ Value *DstPtr = CB.getArgOperand(2);
+ Value *Ordering = CB.getArgOperand(3);
+ // Convert the call to have at least Release ordering to make sure
+ // the shadow operations aren't reordered after it.
+ Value *NewOrdering =
+ IRB.CreateExtractElement(makeAddReleaseOrderingTable(IRB), Ordering);
+ CB.setArgOperand(3, NewOrdering);
+
+ Value *DstShadowPtr =
+ getShadowOriginPtr(DstPtr, IRB, IRB.getInt8Ty(), Align(1),
+ /*isStore*/ true)
+ .first;
+
+ // Atomic store always paints clean shadow/origin. See file header.
+ IRB.CreateMemSet(DstShadowPtr, getCleanShadow(IRB.getInt8Ty()), Size,
+ Align(1));
+ }
+
void visitCallBase(CallBase &CB) {
assert(!CB.getMetadata("nosanitize"));
if (CB.isInlineAsm()) {
@@ -3590,28 +3590,28 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
visitInstruction(CB);
return;
}
- LibFunc LF;
- if (TLI->getLibFunc(CB, LF)) {
- // libatomic.a functions need to have special handling because there isn't
- // a good way to intercept them or compile the library with
- // instrumentation.
- switch (LF) {
- case LibFunc_atomic_load:
- if (!isa<CallInst>(CB)) {
- llvm::errs() << "MSAN -- cannot instrument invoke of libatomic load."
- "Ignoring!\n";
- break;
- }
- visitLibAtomicLoad(CB);
- return;
- case LibFunc_atomic_store:
- visitLibAtomicStore(CB);
- return;
- default:
- break;
- }
- }
-
+ LibFunc LF;
+ if (TLI->getLibFunc(CB, LF)) {
+ // libatomic.a functions need to have special handling because there isn't
+ // a good way to intercept them or compile the library with
+ // instrumentation.
+ switch (LF) {
+ case LibFunc_atomic_load:
+ if (!isa<CallInst>(CB)) {
+ llvm::errs() << "MSAN -- cannot instrument invoke of libatomic load."
+ "Ignoring!\n";
+ break;
+ }
+ visitLibAtomicLoad(CB);
+ return;
+ case LibFunc_atomic_store:
+ visitLibAtomicStore(CB);
+ return;
+ default:
+ break;
+ }
+ }
+
if (auto *Call = dyn_cast<CallInst>(&CB)) {
assert(!isa<IntrinsicInst>(Call) && "intrinsics are handled elsewhere");
@@ -3619,14 +3619,14 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
// will become a non-readonly function after it is instrumented by us. To
// prevent this code from being optimized out, mark that function
// non-readonly in advance.
- AttrBuilder B;
- B.addAttribute(Attribute::ReadOnly)
- .addAttribute(Attribute::ReadNone)
- .addAttribute(Attribute::WriteOnly)
- .addAttribute(Attribute::ArgMemOnly)
- .addAttribute(Attribute::Speculatable);
-
- Call->removeAttributes(AttributeList::FunctionIndex, B);
+ AttrBuilder B;
+ B.addAttribute(Attribute::ReadOnly)
+ .addAttribute(Attribute::ReadNone)
+ .addAttribute(Attribute::WriteOnly)
+ .addAttribute(Attribute::ArgMemOnly)
+ .addAttribute(Attribute::Speculatable);
+
+ Call->removeAttributes(AttributeList::FunctionIndex, B);
if (Function *Func = Call->getCalledFunction()) {
Func->removeAttributes(AttributeList::FunctionIndex, B);
}
@@ -3634,12 +3634,12 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
maybeMarkSanitizerLibraryCallNoBuiltin(Call, TLI);
}
IRBuilder<> IRB(&CB);
- bool MayCheckCall = ClEagerChecks;
- if (Function *Func = CB.getCalledFunction()) {
- // __sanitizer_unaligned_{load,store} functions may be called by users
- // and always expects shadows in the TLS. So don't check them.
- MayCheckCall &= !Func->getName().startswith("__sanitizer_unaligned_");
- }
+ bool MayCheckCall = ClEagerChecks;
+ if (Function *Func = CB.getCalledFunction()) {
+ // __sanitizer_unaligned_{load,store} functions may be called by users
+ // and always expects shadows in the TLS. So don't check them.
+ MayCheckCall &= !Func->getName().startswith("__sanitizer_unaligned_");
+ }
unsigned ArgOffset = 0;
LLVM_DEBUG(dbgs() << " CallSite: " << CB << "\n");
@@ -3665,7 +3665,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
bool ByVal = CB.paramHasAttr(i, Attribute::ByVal);
bool NoUndef = CB.paramHasAttr(i, Attribute::NoUndef);
- bool EagerCheck = MayCheckCall && !ByVal && NoUndef;
+ bool EagerCheck = MayCheckCall && !ByVal && NoUndef;
if (EagerCheck) {
insertShadowCheck(A, &CB);
@@ -3705,7 +3705,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
(void)Store;
assert(Size != 0 && Store != nullptr);
LLVM_DEBUG(dbgs() << " Param:" << *Store << "\n");
- ArgOffset += alignTo(Size, kShadowTLSAlignment);
+ ArgOffset += alignTo(Size, kShadowTLSAlignment);
}
LLVM_DEBUG(dbgs() << " done with call args\n");
@@ -3721,7 +3721,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
if (isa<CallInst>(CB) && cast<CallInst>(CB).isMustTailCall())
return;
- if (MayCheckCall && CB.hasRetAttr(Attribute::NoUndef)) {
+ if (MayCheckCall && CB.hasRetAttr(Attribute::NoUndef)) {
setShadow(&CB, getCleanShadow(&CB));
setOrigin(&CB, getCleanOrigin());
return;
@@ -4104,12 +4104,12 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
setOrigin(&I, getCleanOrigin());
}
- void visitFreezeInst(FreezeInst &I) {
- // Freeze always returns a fully defined value.
- setShadow(&I, getCleanShadow(&I));
- setOrigin(&I, getCleanOrigin());
- }
-
+ void visitFreezeInst(FreezeInst &I) {
+ // Freeze always returns a fully defined value.
+ setShadow(&I, getCleanShadow(&I));
+ setOrigin(&I, getCleanOrigin());
+ }
+
void visitInstruction(Instruction &I) {
// Everything else: stop propagating and check for poisoned shadow.
if (ClDumpStrictInstructions)
@@ -4333,7 +4333,7 @@ struct VarArgAMD64Helper : public VarArgHelper {
if (!VAStartInstrumentationList.empty()) {
// If there is a va_start in this function, make a backup copy of
// va_arg_tls somewhere in the function entry block.
- IRBuilder<> IRB(MSV.FnPrologueEnd);
+ IRBuilder<> IRB(MSV.FnPrologueEnd);
VAArgOverflowSize =
IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
Value *CopySize =
@@ -4479,7 +4479,7 @@ struct VarArgMIPS64Helper : public VarArgHelper {
void finalizeInstrumentation() override {
assert(!VAArgSize && !VAArgTLSCopy &&
"finalizeInstrumentation called twice");
- IRBuilder<> IRB(MSV.FnPrologueEnd);
+ IRBuilder<> IRB(MSV.FnPrologueEnd);
VAArgSize = IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
Value *CopySize = IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, 0),
VAArgSize);
@@ -4672,7 +4672,7 @@ struct VarArgAArch64Helper : public VarArgHelper {
if (!VAStartInstrumentationList.empty()) {
// If there is a va_start in this function, make a backup copy of
// va_arg_tls somewhere in the function entry block.
- IRBuilder<> IRB(MSV.FnPrologueEnd);
+ IRBuilder<> IRB(MSV.FnPrologueEnd);
VAArgOverflowSize =
IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
Value *CopySize =
@@ -4792,9 +4792,9 @@ struct VarArgPowerPC64Helper : public VarArgHelper {
// For PowerPC, we need to deal with alignment of stack arguments -
// they are mostly aligned to 8 bytes, but vectors and i128 arrays
// are aligned to 16 bytes, byvals can be aligned to 8 or 16 bytes,
- // For that reason, we compute current offset from stack pointer (which is
- // always properly aligned), and offset for the first vararg, then subtract
- // them.
+ // For that reason, we compute current offset from stack pointer (which is
+ // always properly aligned), and offset for the first vararg, then subtract
+ // them.
unsigned VAArgBase;
Triple TargetTriple(F.getParent()->getTargetTriple());
// Parameter save area starts at 48 bytes from frame pointer for ABIv1,
@@ -4917,7 +4917,7 @@ struct VarArgPowerPC64Helper : public VarArgHelper {
void finalizeInstrumentation() override {
assert(!VAArgSize && !VAArgTLSCopy &&
"finalizeInstrumentation called twice");
- IRBuilder<> IRB(MSV.FnPrologueEnd);
+ IRBuilder<> IRB(MSV.FnPrologueEnd);
VAArgSize = IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
Value *CopySize = IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, 0),
VAArgSize);
@@ -5236,7 +5236,7 @@ struct VarArgSystemZHelper : public VarArgHelper {
if (!VAStartInstrumentationList.empty()) {
// If there is a va_start in this function, make a backup copy of
// va_arg_tls somewhere in the function entry block.
- IRBuilder<> IRB(MSV.FnPrologueEnd);
+ IRBuilder<> IRB(MSV.FnPrologueEnd);
VAArgOverflowSize =
IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
Value *CopySize =
diff --git a/contrib/libs/llvm12/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/contrib/libs/llvm12/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
index 763fe7c656..be6c8c6310 100644
--- a/contrib/libs/llvm12/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/contrib/libs/llvm12/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -248,38 +248,38 @@ static cl::opt<bool>
"optimization remarks: -{Rpass|"
"pass-remarks}=pgo-instrumentation"));
-static cl::opt<bool> PGOInstrumentEntry(
- "pgo-instrument-entry", cl::init(false), cl::Hidden,
- cl::desc("Force to instrument function entry basicblock."));
-
-static cl::opt<bool>
- PGOFixEntryCount("pgo-fix-entry-count", cl::init(true), cl::Hidden,
- cl::desc("Fix function entry count in profile use."));
-
-static cl::opt<bool> PGOVerifyHotBFI(
- "pgo-verify-hot-bfi", cl::init(false), cl::Hidden,
- cl::desc("Print out the non-match BFI count if a hot raw profile count "
- "becomes non-hot, or a cold raw profile count becomes hot. "
- "The print is enabled under -Rpass-analysis=pgo, or "
- "internal option -pass-remakrs-analysis=pgo."));
-
-static cl::opt<bool> PGOVerifyBFI(
- "pgo-verify-bfi", cl::init(false), cl::Hidden,
- cl::desc("Print out mismatched BFI counts after setting profile metadata "
- "The print is enabled under -Rpass-analysis=pgo, or "
- "internal option -pass-remakrs-analysis=pgo."));
-
-static cl::opt<unsigned> PGOVerifyBFIRatio(
- "pgo-verify-bfi-ratio", cl::init(5), cl::Hidden,
- cl::desc("Set the threshold for pgo-verify-big -- only print out "
- "mismatched BFI if the difference percentage is greater than "
- "this value (in percentage)."));
-
-static cl::opt<unsigned> PGOVerifyBFICutoff(
- "pgo-verify-bfi-cutoff", cl::init(1), cl::Hidden,
- cl::desc("Set the threshold for pgo-verify-bfi -- skip the counts whose "
- "profile count value is below."));
-
+static cl::opt<bool> PGOInstrumentEntry(
+ "pgo-instrument-entry", cl::init(false), cl::Hidden,
+ cl::desc("Force to instrument function entry basicblock."));
+
+static cl::opt<bool>
+ PGOFixEntryCount("pgo-fix-entry-count", cl::init(true), cl::Hidden,
+ cl::desc("Fix function entry count in profile use."));
+
+static cl::opt<bool> PGOVerifyHotBFI(
+ "pgo-verify-hot-bfi", cl::init(false), cl::Hidden,
+ cl::desc("Print out the non-match BFI count if a hot raw profile count "
+ "becomes non-hot, or a cold raw profile count becomes hot. "
+ "The print is enabled under -Rpass-analysis=pgo, or "
+ "internal option -pass-remakrs-analysis=pgo."));
+
+static cl::opt<bool> PGOVerifyBFI(
+ "pgo-verify-bfi", cl::init(false), cl::Hidden,
+ cl::desc("Print out mismatched BFI counts after setting profile metadata "
+ "The print is enabled under -Rpass-analysis=pgo, or "
+ "internal option -pass-remakrs-analysis=pgo."));
+
+static cl::opt<unsigned> PGOVerifyBFIRatio(
+ "pgo-verify-bfi-ratio", cl::init(5), cl::Hidden,
+ cl::desc("Set the threshold for pgo-verify-big -- only print out "
+ "mismatched BFI if the difference percentage is greater than "
+ "this value (in percentage)."));
+
+static cl::opt<unsigned> PGOVerifyBFICutoff(
+ "pgo-verify-bfi-cutoff", cl::init(1), cl::Hidden,
+ cl::desc("Set the threshold for pgo-verify-bfi -- skip the counts whose "
+ "profile count value is below."));
+
// Command line option to turn on CFG dot dump after profile annotation.
// Defined in Analysis/BlockFrequencyInfo.cpp: -pgo-view-counts
extern cl::opt<PGOViewCountsType> PGOViewCounts;
@@ -288,10 +288,10 @@ extern cl::opt<PGOViewCountsType> PGOViewCounts;
// Defined in Analysis/BlockFrequencyInfo.cpp: -view-bfi-func-name=
extern cl::opt<std::string> ViewBlockFreqFuncName;
-static cl::opt<bool>
- PGOOldCFGHashing("pgo-instr-old-cfg-hashing", cl::init(false), cl::Hidden,
- cl::desc("Use the old CFG function hashing"));
-
+static cl::opt<bool>
+ PGOOldCFGHashing("pgo-instr-old-cfg-hashing", cl::init(false), cl::Hidden,
+ cl::desc("Use the old CFG function hashing"));
+
// Return a string describing the branch condition that can be
// used in static branch probability heuristics:
static std::string getBranchCondString(Instruction *TI) {
@@ -460,7 +460,7 @@ public:
private:
bool runOnModule(Module &M) override {
createProfileFileNameVar(M, InstrProfileOutput);
- createIRLevelProfileFlagVar(M, /* IsCS */ true, PGOInstrumentEntry);
+ createIRLevelProfileFlagVar(M, /* IsCS */ true, PGOInstrumentEntry);
return false;
}
std::string InstrProfileOutput;
@@ -607,11 +607,11 @@ public:
Function &Func, TargetLibraryInfo &TLI,
std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers,
bool CreateGlobalVar = false, BranchProbabilityInfo *BPI = nullptr,
- BlockFrequencyInfo *BFI = nullptr, bool IsCS = false,
- bool InstrumentFuncEntry = true)
+ BlockFrequencyInfo *BFI = nullptr, bool IsCS = false,
+ bool InstrumentFuncEntry = true)
: F(Func), IsCS(IsCS), ComdatMembers(ComdatMembers), VPC(Func, TLI),
- ValueSites(IPVK_Last + 1), SIVisitor(Func),
- MST(F, InstrumentFuncEntry, BPI, BFI) {
+ ValueSites(IPVK_Last + 1), SIVisitor(Func),
+ MST(F, InstrumentFuncEntry, BPI, BFI) {
// This should be done before CFG hash computation.
SIVisitor.countSelects(Func);
ValueSites[IPVK_MemOPSize] = VPC.get(IPVK_MemOPSize);
@@ -648,8 +648,8 @@ public:
} // end anonymous namespace
// Compute Hash value for the CFG: the lower 32 bits are CRC32 of the index
-// value of each BB in the CFG. The higher 32 bits are the CRC32 of the numbers
-// of selects, indirect calls, mem ops and edges.
+// value of each BB in the CFG. The higher 32 bits are the CRC32 of the numbers
+// of selects, indirect calls, mem ops and edges.
template <class Edge, class BBInfo>
void FuncPGOInstrumentation<Edge, BBInfo>::computeCFGHash() {
std::vector<uint8_t> Indexes;
@@ -668,31 +668,31 @@ void FuncPGOInstrumentation<Edge, BBInfo>::computeCFGHash() {
}
JC.update(Indexes);
- JamCRC JCH;
- if (PGOOldCFGHashing) {
- // Hash format for context sensitive profile. Reserve 4 bits for other
- // information.
- FunctionHash = (uint64_t)SIVisitor.getNumOfSelectInsts() << 56 |
- (uint64_t)ValueSites[IPVK_IndirectCallTarget].size() << 48 |
- //(uint64_t)ValueSites[IPVK_MemOPSize].size() << 40 |
- (uint64_t)MST.AllEdges.size() << 32 | JC.getCRC();
- } else {
- // The higher 32 bits.
- auto updateJCH = [&JCH](uint64_t Num) {
- uint8_t Data[8];
- support::endian::write64le(Data, Num);
- JCH.update(Data);
- };
- updateJCH((uint64_t)SIVisitor.getNumOfSelectInsts());
- updateJCH((uint64_t)ValueSites[IPVK_IndirectCallTarget].size());
- updateJCH((uint64_t)ValueSites[IPVK_MemOPSize].size());
- updateJCH((uint64_t)MST.AllEdges.size());
-
- // Hash format for context sensitive profile. Reserve 4 bits for other
- // information.
- FunctionHash = (((uint64_t)JCH.getCRC()) << 28) + JC.getCRC();
- }
-
+ JamCRC JCH;
+ if (PGOOldCFGHashing) {
+ // Hash format for context sensitive profile. Reserve 4 bits for other
+ // information.
+ FunctionHash = (uint64_t)SIVisitor.getNumOfSelectInsts() << 56 |
+ (uint64_t)ValueSites[IPVK_IndirectCallTarget].size() << 48 |
+ //(uint64_t)ValueSites[IPVK_MemOPSize].size() << 40 |
+ (uint64_t)MST.AllEdges.size() << 32 | JC.getCRC();
+ } else {
+ // The higher 32 bits.
+ auto updateJCH = [&JCH](uint64_t Num) {
+ uint8_t Data[8];
+ support::endian::write64le(Data, Num);
+ JCH.update(Data);
+ };
+ updateJCH((uint64_t)SIVisitor.getNumOfSelectInsts());
+ updateJCH((uint64_t)ValueSites[IPVK_IndirectCallTarget].size());
+ updateJCH((uint64_t)ValueSites[IPVK_MemOPSize].size());
+ updateJCH((uint64_t)MST.AllEdges.size());
+
+ // Hash format for context sensitive profile. Reserve 4 bits for other
+ // information.
+ FunctionHash = (((uint64_t)JCH.getCRC()) << 28) + JC.getCRC();
+ }
+
// Reserve bit 60-63 for other information purpose.
FunctionHash &= 0x0FFFFFFFFFFFFFFF;
if (IsCS)
@@ -701,12 +701,12 @@ void FuncPGOInstrumentation<Edge, BBInfo>::computeCFGHash() {
<< " CRC = " << JC.getCRC()
<< ", Selects = " << SIVisitor.getNumOfSelectInsts()
<< ", Edges = " << MST.AllEdges.size() << ", ICSites = "
- << ValueSites[IPVK_IndirectCallTarget].size());
- if (!PGOOldCFGHashing) {
- LLVM_DEBUG(dbgs() << ", Memops = " << ValueSites[IPVK_MemOPSize].size()
- << ", High32 CRC = " << JCH.getCRC());
- }
- LLVM_DEBUG(dbgs() << ", Hash = " << FunctionHash << "\n";);
+ << ValueSites[IPVK_IndirectCallTarget].size());
+ if (!PGOOldCFGHashing) {
+ LLVM_DEBUG(dbgs() << ", Memops = " << ValueSites[IPVK_MemOPSize].size()
+ << ", High32 CRC = " << JCH.getCRC());
+ }
+ LLVM_DEBUG(dbgs() << ", Hash = " << FunctionHash << "\n";);
}
// Check if we can safely rename this Comdat function.
@@ -717,7 +717,7 @@ static bool canRenameComdat(
return false;
// FIXME: Current only handle those Comdat groups that only containing one
- // function.
+ // function.
// (1) For a Comdat group containing multiple functions, we need to have a
// unique postfix based on the hashes for each function. There is a
// non-trivial code refactoring to do this efficiently.
@@ -725,7 +725,7 @@ static bool canRenameComdat(
// group including global vars.
Comdat *C = F.getComdat();
for (auto &&CM : make_range(ComdatMembers.equal_range(C))) {
- assert(!isa<GlobalAlias>(CM.second));
+ assert(!isa<GlobalAlias>(CM.second));
Function *FM = dyn_cast<Function>(CM.second);
if (FM != &F)
return false;
@@ -766,7 +766,7 @@ void FuncPGOInstrumentation<Edge, BBInfo>::renameComdatFunction() {
for (auto &&CM : make_range(ComdatMembers.equal_range(OrigComdat))) {
// Must be a function.
- cast<Function>(CM.second)->setComdat(NewComdat);
+ cast<Function>(CM.second)->setComdat(NewComdat);
}
}
@@ -831,11 +831,11 @@ BasicBlock *FuncPGOInstrumentation<Edge, BBInfo>::getInstrBB(Edge *E) {
if (!E->IsCritical)
return canInstrument(DestBB);
- // Some IndirectBr critical edges cannot be split by the previous
- // SplitIndirectBrCriticalEdges call. Bail out.
+ // Some IndirectBr critical edges cannot be split by the previous
+ // SplitIndirectBrCriticalEdges call. Bail out.
unsigned SuccNum = GetSuccessorNumber(SrcBB, DestBB);
- BasicBlock *InstrBB =
- isa<IndirectBrInst>(TI) ? nullptr : SplitCriticalEdge(TI, SuccNum);
+ BasicBlock *InstrBB =
+ isa<IndirectBrInst>(TI) ? nullptr : SplitCriticalEdge(TI, SuccNum);
if (!InstrBB) {
LLVM_DEBUG(
dbgs() << "Fail to split critical edge: not instrument this edge.\n");
@@ -898,8 +898,8 @@ static void instrumentOneFunc(
// later in getInstrBB() to avoid invalidating it.
SplitIndirectBrCriticalEdges(F, BPI, BFI);
- FuncPGOInstrumentation<PGOEdge, BBInfo> FuncInfo(
- F, TLI, ComdatMembers, true, BPI, BFI, IsCS, PGOInstrumentEntry);
+ FuncPGOInstrumentation<PGOEdge, BBInfo> FuncInfo(
+ F, TLI, ComdatMembers, true, BPI, BFI, IsCS, PGOInstrumentEntry);
std::vector<BasicBlock *> InstrumentBBs;
FuncInfo.getInstrumentBBs(InstrumentBBs);
unsigned NumCounters =
@@ -1057,15 +1057,15 @@ public:
PGOUseFunc(Function &Func, Module *Modu, TargetLibraryInfo &TLI,
std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers,
BranchProbabilityInfo *BPI, BlockFrequencyInfo *BFIin,
- ProfileSummaryInfo *PSI, bool IsCS, bool InstrumentFuncEntry)
+ ProfileSummaryInfo *PSI, bool IsCS, bool InstrumentFuncEntry)
: F(Func), M(Modu), BFI(BFIin), PSI(PSI),
- FuncInfo(Func, TLI, ComdatMembers, false, BPI, BFIin, IsCS,
- InstrumentFuncEntry),
+ FuncInfo(Func, TLI, ComdatMembers, false, BPI, BFIin, IsCS,
+ InstrumentFuncEntry),
FreqAttr(FFA_Normal), IsCS(IsCS) {}
// Read counts for the instrumented BB from profile.
- bool readCounters(IndexedInstrProfReader *PGOReader, bool &AllZeros,
- bool &AllMinusOnes);
+ bool readCounters(IndexedInstrProfReader *PGOReader, bool &AllZeros,
+ bool &AllMinusOnes);
// Populate the counts for all BBs.
void populateCounters();
@@ -1176,18 +1176,18 @@ bool PGOUseFunc::setInstrumentedCounts(
if (NumCounters != CountFromProfile.size()) {
return false;
}
- auto *FuncEntry = &*F.begin();
-
+ auto *FuncEntry = &*F.begin();
+
// Set the profile count to the Instrumented BBs.
uint32_t I = 0;
for (BasicBlock *InstrBB : InstrumentBBs) {
uint64_t CountValue = CountFromProfile[I++];
UseBBInfo &Info = getBBInfo(InstrBB);
- // If we reach here, we know that we have some nonzero count
- // values in this function. The entry count should not be 0.
- // Fix it if necessary.
- if (InstrBB == FuncEntry && CountValue == 0)
- CountValue = 1;
+ // If we reach here, we know that we have some nonzero count
+ // values in this function. The entry count should not be 0.
+ // Fix it if necessary.
+ if (InstrBB == FuncEntry && CountValue == 0)
+ CountValue = 1;
Info.setBBInfoCount(CountValue);
}
ProfileCountSize = CountFromProfile.size();
@@ -1248,8 +1248,8 @@ void PGOUseFunc::setEdgeCount(DirectEdges &Edges, uint64_t Value) {
// Read the profile from ProfileFileName and assign the value to the
// instrumented BB and the edges. This function also updates ProgramMaxCount.
// Return true if the profile are successfully read, and false on errors.
-bool PGOUseFunc::readCounters(IndexedInstrProfReader *PGOReader, bool &AllZeros,
- bool &AllMinusOnes) {
+bool PGOUseFunc::readCounters(IndexedInstrProfReader *PGOReader, bool &AllZeros,
+ bool &AllMinusOnes) {
auto &Ctx = M->getContext();
Expected<InstrProfRecord> Result =
PGOReader->getInstrProfRecord(FuncInfo.FuncName, FuncInfo.FunctionHash);
@@ -1292,13 +1292,13 @@ bool PGOUseFunc::readCounters(IndexedInstrProfReader *PGOReader, bool &AllZeros,
IsCS ? NumOfCSPGOFunc++ : NumOfPGOFunc++;
LLVM_DEBUG(dbgs() << CountFromProfile.size() << " counts\n");
- AllMinusOnes = (CountFromProfile.size() > 0);
+ AllMinusOnes = (CountFromProfile.size() > 0);
uint64_t ValueSum = 0;
for (unsigned I = 0, S = CountFromProfile.size(); I < S; I++) {
LLVM_DEBUG(dbgs() << " " << I << ": " << CountFromProfile[I] << "\n");
ValueSum += CountFromProfile[I];
- if (CountFromProfile[I] != (uint64_t)-1)
- AllMinusOnes = false;
+ if (CountFromProfile[I] != (uint64_t)-1)
+ AllMinusOnes = false;
}
AllZeros = (ValueSum == 0);
@@ -1389,11 +1389,11 @@ void PGOUseFunc::populateCounters() {
continue;
FuncMaxCount = std::max(FuncMaxCount, BI->CountValue);
}
-
- // Fix the obviously inconsistent entry count.
- if (FuncMaxCount > 0 && FuncEntryCount == 0)
- FuncEntryCount = 1;
- F.setEntryCount(ProfileCount(FuncEntryCount, Function::PCT_Real));
+
+ // Fix the obviously inconsistent entry count.
+ if (FuncMaxCount > 0 && FuncEntryCount == 0)
+ FuncEntryCount = 1;
+ F.setEntryCount(ProfileCount(FuncEntryCount, Function::PCT_Real));
markFunctionAttributes(FuncEntryCount, FuncMaxCount);
// Now annotate select instructions
@@ -1584,15 +1584,15 @@ static bool InstrumentAllFunctions(
// For the context-sensitve instrumentation, we should have a separated pass
// (before LTO/ThinLTO linking) to create these variables.
if (!IsCS)
- createIRLevelProfileFlagVar(M, /* IsCS */ false, PGOInstrumentEntry);
+ createIRLevelProfileFlagVar(M, /* IsCS */ false, PGOInstrumentEntry);
std::unordered_multimap<Comdat *, GlobalValue *> ComdatMembers;
collectComdatMembers(M, ComdatMembers);
for (auto &F : M) {
if (F.isDeclaration())
continue;
- if (F.hasFnAttribute(llvm::Attribute::NoProfile))
- continue;
+ if (F.hasFnAttribute(llvm::Attribute::NoProfile))
+ continue;
auto &TLI = LookupTLI(F);
auto *BPI = LookupBPI(F);
auto *BFI = LookupBFI(F);
@@ -1604,7 +1604,7 @@ static bool InstrumentAllFunctions(
PreservedAnalyses
PGOInstrumentationGenCreateVar::run(Module &M, ModuleAnalysisManager &AM) {
createProfileFileNameVar(M, CSInstrName);
- createIRLevelProfileFlagVar(M, /* IsCS */ true, PGOInstrumentEntry);
+ createIRLevelProfileFlagVar(M, /* IsCS */ true, PGOInstrumentEntry);
return PreservedAnalyses::all();
}
@@ -1643,129 +1643,129 @@ PreservedAnalyses PGOInstrumentationGen::run(Module &M,
return PreservedAnalyses::none();
}
-// Using the ratio b/w sums of profile count values and BFI count values to
-// adjust the func entry count.
-static void fixFuncEntryCount(PGOUseFunc &Func, LoopInfo &LI,
- BranchProbabilityInfo &NBPI) {
- Function &F = Func.getFunc();
- BlockFrequencyInfo NBFI(F, NBPI, LI);
-#ifndef NDEBUG
- auto BFIEntryCount = F.getEntryCount();
- assert(BFIEntryCount.hasValue() && (BFIEntryCount.getCount() > 0) &&
- "Invalid BFI Entrycount");
-#endif
- auto SumCount = APFloat::getZero(APFloat::IEEEdouble());
- auto SumBFICount = APFloat::getZero(APFloat::IEEEdouble());
- for (auto &BBI : F) {
- uint64_t CountValue = 0;
- uint64_t BFICountValue = 0;
- if (!Func.findBBInfo(&BBI))
- continue;
- auto BFICount = NBFI.getBlockProfileCount(&BBI);
- CountValue = Func.getBBInfo(&BBI).CountValue;
- BFICountValue = BFICount.getValue();
- SumCount.add(APFloat(CountValue * 1.0), APFloat::rmNearestTiesToEven);
- SumBFICount.add(APFloat(BFICountValue * 1.0), APFloat::rmNearestTiesToEven);
- }
- if (SumCount.isZero())
- return;
-
- assert(SumBFICount.compare(APFloat(0.0)) == APFloat::cmpGreaterThan &&
- "Incorrect sum of BFI counts");
- if (SumBFICount.compare(SumCount) == APFloat::cmpEqual)
- return;
- double Scale = (SumCount / SumBFICount).convertToDouble();
- if (Scale < 1.001 && Scale > 0.999)
- return;
-
- uint64_t FuncEntryCount = Func.getBBInfo(&*F.begin()).CountValue;
- uint64_t NewEntryCount = 0.5 + FuncEntryCount * Scale;
- if (NewEntryCount == 0)
- NewEntryCount = 1;
- if (NewEntryCount != FuncEntryCount) {
- F.setEntryCount(ProfileCount(NewEntryCount, Function::PCT_Real));
- LLVM_DEBUG(dbgs() << "FixFuncEntryCount: in " << F.getName()
- << ", entry_count " << FuncEntryCount << " --> "
- << NewEntryCount << "\n");
- }
-}
-
-// Compare the profile count values with BFI count values, and print out
-// the non-matching ones.
-static void verifyFuncBFI(PGOUseFunc &Func, LoopInfo &LI,
- BranchProbabilityInfo &NBPI,
- uint64_t HotCountThreshold,
- uint64_t ColdCountThreshold) {
- Function &F = Func.getFunc();
- BlockFrequencyInfo NBFI(F, NBPI, LI);
- // bool PrintFunc = false;
- bool HotBBOnly = PGOVerifyHotBFI;
- std::string Msg;
- OptimizationRemarkEmitter ORE(&F);
-
- unsigned BBNum = 0, BBMisMatchNum = 0, NonZeroBBNum = 0;
- for (auto &BBI : F) {
- uint64_t CountValue = 0;
- uint64_t BFICountValue = 0;
-
- if (Func.getBBInfo(&BBI).CountValid)
- CountValue = Func.getBBInfo(&BBI).CountValue;
-
- BBNum++;
- if (CountValue)
- NonZeroBBNum++;
- auto BFICount = NBFI.getBlockProfileCount(&BBI);
- if (BFICount)
- BFICountValue = BFICount.getValue();
-
- if (HotBBOnly) {
- bool rawIsHot = CountValue >= HotCountThreshold;
- bool BFIIsHot = BFICountValue >= HotCountThreshold;
- bool rawIsCold = CountValue <= ColdCountThreshold;
- bool ShowCount = false;
- if (rawIsHot && !BFIIsHot) {
- Msg = "raw-Hot to BFI-nonHot";
- ShowCount = true;
- } else if (rawIsCold && BFIIsHot) {
- Msg = "raw-Cold to BFI-Hot";
- ShowCount = true;
- }
- if (!ShowCount)
- continue;
- } else {
- if ((CountValue < PGOVerifyBFICutoff) &&
- (BFICountValue < PGOVerifyBFICutoff))
- continue;
- uint64_t Diff = (BFICountValue >= CountValue)
- ? BFICountValue - CountValue
- : CountValue - BFICountValue;
- if (Diff < CountValue / 100 * PGOVerifyBFIRatio)
- continue;
- }
- BBMisMatchNum++;
-
- ORE.emit([&]() {
- OptimizationRemarkAnalysis Remark(DEBUG_TYPE, "bfi-verify",
- F.getSubprogram(), &BBI);
- Remark << "BB " << ore::NV("Block", BBI.getName())
- << " Count=" << ore::NV("Count", CountValue)
- << " BFI_Count=" << ore::NV("Count", BFICountValue);
- if (!Msg.empty())
- Remark << " (" << Msg << ")";
- return Remark;
- });
- }
- if (BBMisMatchNum)
- ORE.emit([&]() {
- return OptimizationRemarkAnalysis(DEBUG_TYPE, "bfi-verify",
- F.getSubprogram(), &F.getEntryBlock())
- << "In Func " << ore::NV("Function", F.getName())
- << ": Num_of_BB=" << ore::NV("Count", BBNum)
- << ", Num_of_non_zerovalue_BB=" << ore::NV("Count", NonZeroBBNum)
- << ", Num_of_mis_matching_BB=" << ore::NV("Count", BBMisMatchNum);
- });
-}
-
+// Using the ratio b/w sums of profile count values and BFI count values to
+// adjust the func entry count.
+static void fixFuncEntryCount(PGOUseFunc &Func, LoopInfo &LI,
+ BranchProbabilityInfo &NBPI) {
+ Function &F = Func.getFunc();
+ BlockFrequencyInfo NBFI(F, NBPI, LI);
+#ifndef NDEBUG
+ auto BFIEntryCount = F.getEntryCount();
+ assert(BFIEntryCount.hasValue() && (BFIEntryCount.getCount() > 0) &&
+ "Invalid BFI Entrycount");
+#endif
+ auto SumCount = APFloat::getZero(APFloat::IEEEdouble());
+ auto SumBFICount = APFloat::getZero(APFloat::IEEEdouble());
+ for (auto &BBI : F) {
+ uint64_t CountValue = 0;
+ uint64_t BFICountValue = 0;
+ if (!Func.findBBInfo(&BBI))
+ continue;
+ auto BFICount = NBFI.getBlockProfileCount(&BBI);
+ CountValue = Func.getBBInfo(&BBI).CountValue;
+ BFICountValue = BFICount.getValue();
+ SumCount.add(APFloat(CountValue * 1.0), APFloat::rmNearestTiesToEven);
+ SumBFICount.add(APFloat(BFICountValue * 1.0), APFloat::rmNearestTiesToEven);
+ }
+ if (SumCount.isZero())
+ return;
+
+ assert(SumBFICount.compare(APFloat(0.0)) == APFloat::cmpGreaterThan &&
+ "Incorrect sum of BFI counts");
+ if (SumBFICount.compare(SumCount) == APFloat::cmpEqual)
+ return;
+ double Scale = (SumCount / SumBFICount).convertToDouble();
+ if (Scale < 1.001 && Scale > 0.999)
+ return;
+
+ uint64_t FuncEntryCount = Func.getBBInfo(&*F.begin()).CountValue;
+ uint64_t NewEntryCount = 0.5 + FuncEntryCount * Scale;
+ if (NewEntryCount == 0)
+ NewEntryCount = 1;
+ if (NewEntryCount != FuncEntryCount) {
+ F.setEntryCount(ProfileCount(NewEntryCount, Function::PCT_Real));
+ LLVM_DEBUG(dbgs() << "FixFuncEntryCount: in " << F.getName()
+ << ", entry_count " << FuncEntryCount << " --> "
+ << NewEntryCount << "\n");
+ }
+}
+
+// Compare the profile count values with BFI count values, and print out
+// the non-matching ones.
+static void verifyFuncBFI(PGOUseFunc &Func, LoopInfo &LI,
+ BranchProbabilityInfo &NBPI,
+ uint64_t HotCountThreshold,
+ uint64_t ColdCountThreshold) {
+ Function &F = Func.getFunc();
+ BlockFrequencyInfo NBFI(F, NBPI, LI);
+ // bool PrintFunc = false;
+ bool HotBBOnly = PGOVerifyHotBFI;
+ std::string Msg;
+ OptimizationRemarkEmitter ORE(&F);
+
+ unsigned BBNum = 0, BBMisMatchNum = 0, NonZeroBBNum = 0;
+ for (auto &BBI : F) {
+ uint64_t CountValue = 0;
+ uint64_t BFICountValue = 0;
+
+ if (Func.getBBInfo(&BBI).CountValid)
+ CountValue = Func.getBBInfo(&BBI).CountValue;
+
+ BBNum++;
+ if (CountValue)
+ NonZeroBBNum++;
+ auto BFICount = NBFI.getBlockProfileCount(&BBI);
+ if (BFICount)
+ BFICountValue = BFICount.getValue();
+
+ if (HotBBOnly) {
+ bool rawIsHot = CountValue >= HotCountThreshold;
+ bool BFIIsHot = BFICountValue >= HotCountThreshold;
+ bool rawIsCold = CountValue <= ColdCountThreshold;
+ bool ShowCount = false;
+ if (rawIsHot && !BFIIsHot) {
+ Msg = "raw-Hot to BFI-nonHot";
+ ShowCount = true;
+ } else if (rawIsCold && BFIIsHot) {
+ Msg = "raw-Cold to BFI-Hot";
+ ShowCount = true;
+ }
+ if (!ShowCount)
+ continue;
+ } else {
+ if ((CountValue < PGOVerifyBFICutoff) &&
+ (BFICountValue < PGOVerifyBFICutoff))
+ continue;
+ uint64_t Diff = (BFICountValue >= CountValue)
+ ? BFICountValue - CountValue
+ : CountValue - BFICountValue;
+ if (Diff < CountValue / 100 * PGOVerifyBFIRatio)
+ continue;
+ }
+ BBMisMatchNum++;
+
+ ORE.emit([&]() {
+ OptimizationRemarkAnalysis Remark(DEBUG_TYPE, "bfi-verify",
+ F.getSubprogram(), &BBI);
+ Remark << "BB " << ore::NV("Block", BBI.getName())
+ << " Count=" << ore::NV("Count", CountValue)
+ << " BFI_Count=" << ore::NV("Count", BFICountValue);
+ if (!Msg.empty())
+ Remark << " (" << Msg << ")";
+ return Remark;
+ });
+ }
+ if (BBMisMatchNum)
+ ORE.emit([&]() {
+ return OptimizationRemarkAnalysis(DEBUG_TYPE, "bfi-verify",
+ F.getSubprogram(), &F.getEntryBlock())
+ << "In Func " << ore::NV("Function", F.getName())
+ << ": Num_of_BB=" << ore::NV("Count", BBNum)
+ << ", Num_of_non_zerovalue_BB=" << ore::NV("Count", NonZeroBBNum)
+ << ", Num_of_mis_matching_BB=" << ore::NV("Count", BBMisMatchNum);
+ });
+}
+
static bool annotateAllFunctions(
Module &M, StringRef ProfileFileName, StringRef ProfileRemappingFileName,
function_ref<TargetLibraryInfo &(Function &)> LookupTLI,
@@ -1814,12 +1814,12 @@ static bool annotateAllFunctions(
collectComdatMembers(M, ComdatMembers);
std::vector<Function *> HotFunctions;
std::vector<Function *> ColdFunctions;
-
- // If the profile marked as always instrument the entry BB, do the
- // same. Note this can be overwritten by the internal option in CFGMST.h
- bool InstrumentFuncEntry = PGOReader->instrEntryBBEnabled();
- if (PGOInstrumentEntry.getNumOccurrences() > 0)
- InstrumentFuncEntry = PGOInstrumentEntry;
+
+ // If the profile marked as always instrument the entry BB, do the
+ // same. Note this can be overwritten by the internal option in CFGMST.h
+ bool InstrumentFuncEntry = PGOReader->instrEntryBBEnabled();
+ if (PGOInstrumentEntry.getNumOccurrences() > 0)
+ InstrumentFuncEntry = PGOInstrumentEntry;
for (auto &F : M) {
if (F.isDeclaration())
continue;
@@ -1829,15 +1829,15 @@ static bool annotateAllFunctions(
// Split indirectbr critical edges here before computing the MST rather than
// later in getInstrBB() to avoid invalidating it.
SplitIndirectBrCriticalEdges(F, BPI, BFI);
- PGOUseFunc Func(F, &M, TLI, ComdatMembers, BPI, BFI, PSI, IsCS,
- InstrumentFuncEntry);
- // When AllMinusOnes is true, it means the profile for the function
- // is unrepresentative and this function is actually hot. Set the
- // entry count of the function to be multiple times of hot threshold
- // and drop all its internal counters.
- bool AllMinusOnes = false;
+ PGOUseFunc Func(F, &M, TLI, ComdatMembers, BPI, BFI, PSI, IsCS,
+ InstrumentFuncEntry);
+ // When AllMinusOnes is true, it means the profile for the function
+ // is unrepresentative and this function is actually hot. Set the
+ // entry count of the function to be multiple times of hot threshold
+ // and drop all its internal counters.
+ bool AllMinusOnes = false;
bool AllZeros = false;
- if (!Func.readCounters(PGOReader.get(), AllZeros, AllMinusOnes))
+ if (!Func.readCounters(PGOReader.get(), AllZeros, AllMinusOnes))
continue;
if (AllZeros) {
F.setEntryCount(ProfileCount(0, Function::PCT_Real));
@@ -1845,15 +1845,15 @@ static bool annotateAllFunctions(
ColdFunctions.push_back(&F);
continue;
}
- const unsigned MultiplyFactor = 3;
- if (AllMinusOnes) {
- uint64_t HotThreshold = PSI->getHotCountThreshold();
- if (HotThreshold)
- F.setEntryCount(
- ProfileCount(HotThreshold * MultiplyFactor, Function::PCT_Real));
- HotFunctions.push_back(&F);
- continue;
- }
+ const unsigned MultiplyFactor = 3;
+ if (AllMinusOnes) {
+ uint64_t HotThreshold = PSI->getHotCountThreshold();
+ if (HotThreshold)
+ F.setEntryCount(
+ ProfileCount(HotThreshold * MultiplyFactor, Function::PCT_Real));
+ HotFunctions.push_back(&F);
+ continue;
+ }
Func.populateCounters();
Func.setBranchWeights();
Func.annotateValueSites();
@@ -1891,23 +1891,23 @@ static bool annotateAllFunctions(
Func.dumpInfo();
}
}
-
- if (PGOVerifyBFI || PGOVerifyHotBFI || PGOFixEntryCount) {
- LoopInfo LI{DominatorTree(F)};
- BranchProbabilityInfo NBPI(F, LI);
-
- // Fix func entry count.
- if (PGOFixEntryCount)
- fixFuncEntryCount(Func, LI, NBPI);
-
- // Verify BlockFrequency information.
- uint64_t HotCountThreshold = 0, ColdCountThreshold = 0;
- if (PGOVerifyHotBFI) {
- HotCountThreshold = PSI->getOrCompHotCountThreshold();
- ColdCountThreshold = PSI->getOrCompColdCountThreshold();
- }
- verifyFuncBFI(Func, LI, NBPI, HotCountThreshold, ColdCountThreshold);
- }
+
+ if (PGOVerifyBFI || PGOVerifyHotBFI || PGOFixEntryCount) {
+ LoopInfo LI{DominatorTree(F)};
+ BranchProbabilityInfo NBPI(F, LI);
+
+ // Fix func entry count.
+ if (PGOFixEntryCount)
+ fixFuncEntryCount(Func, LI, NBPI);
+
+ // Verify BlockFrequency information.
+ uint64_t HotCountThreshold = 0, ColdCountThreshold = 0;
+ if (PGOVerifyHotBFI) {
+ HotCountThreshold = PSI->getOrCompHotCountThreshold();
+ ColdCountThreshold = PSI->getOrCompColdCountThreshold();
+ }
+ verifyFuncBFI(Func, LI, NBPI, HotCountThreshold, ColdCountThreshold);
+ }
}
// Set function hotness attribute from the profile.
@@ -1920,17 +1920,17 @@ static bool annotateAllFunctions(
<< "\n");
}
for (auto &F : ColdFunctions) {
- // Only set when there is no Attribute::Hot set by the user. For Hot
- // attribute, user's annotation has the precedence over the profile.
- if (F->hasFnAttribute(Attribute::Hot)) {
- auto &Ctx = M.getContext();
- std::string Msg = std::string("Function ") + F->getName().str() +
- std::string(" is annotated as a hot function but"
- " the profile is cold");
- Ctx.diagnose(
- DiagnosticInfoPGOProfile(M.getName().data(), Msg, DS_Warning));
- continue;
- }
+ // Only set when there is no Attribute::Hot set by the user. For Hot
+ // attribute, user's annotation has the precedence over the profile.
+ if (F->hasFnAttribute(Attribute::Hot)) {
+ auto &Ctx = M.getContext();
+ std::string Msg = std::string("Function ") + F->getName().str() +
+ std::string(" is annotated as a hot function but"
+ " the profile is cold");
+ Ctx.diagnose(
+ DiagnosticInfoPGOProfile(M.getName().data(), Msg, DS_Warning));
+ continue;
+ }
F->addFnAttr(Attribute::Cold);
LLVM_DEBUG(dbgs() << "Set cold attribute to function: " << F->getName()
<< "\n");
diff --git a/contrib/libs/llvm12/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp b/contrib/libs/llvm12/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp
index 3f5db5380e..55a93b6152 100644
--- a/contrib/libs/llvm12/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp
+++ b/contrib/libs/llvm12/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp
@@ -22,7 +22,7 @@
#include "llvm/Analysis/DomTreeUpdater.h"
#include "llvm/Analysis/GlobalsModRef.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
-#include "llvm/Analysis/TargetLibraryInfo.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Dominators.h"
@@ -39,8 +39,8 @@
#include "llvm/Pass.h"
#include "llvm/PassRegistry.h"
#include "llvm/ProfileData/InstrProf.h"
-#define INSTR_PROF_VALUE_PROF_MEMOP_API
-#include "llvm/ProfileData/InstrProfData.inc"
+#define INSTR_PROF_VALUE_PROF_MEMOP_API
+#include "llvm/ProfileData/InstrProfData.inc"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
@@ -97,10 +97,10 @@ cl::opt<bool>
cl::Hidden,
cl::desc("Size-specialize memcmp and bcmp calls"));
-static cl::opt<unsigned>
- MemOpMaxOptSize("memop-value-prof-max-opt-size", cl::Hidden, cl::init(128),
- cl::desc("Optimize the memop size <= this value"));
-
+static cl::opt<unsigned>
+ MemOpMaxOptSize("memop-value-prof-max-opt-size", cl::Hidden, cl::init(128),
+ cl::desc("Optimize the memop size <= this value"));
+
namespace {
class PGOMemOPSizeOptLegacyPass : public FunctionPass {
public:
@@ -254,7 +254,7 @@ public:
LibFunc Func;
if (TLI.getLibFunc(CI, Func) &&
(Func == LibFunc_memcmp || Func == LibFunc_bcmp) &&
- !isa<ConstantInt>(CI.getArgOperand(2))) {
+ !isa<ConstantInt>(CI.getArgOperand(2))) {
WorkList.push_back(MemOp(&CI));
}
}
@@ -346,7 +346,7 @@ bool MemOPSizeOpt::perform(MemOp MO) {
if (MemOPScaleCount)
C = getScaledCount(C, ActualCount, SavedTotalCount);
- if (!InstrProfIsSingleValRange(V) || V > MemOpMaxOptSize)
+ if (!InstrProfIsSingleValRange(V) || V > MemOpMaxOptSize)
continue;
// ValueCounts are sorted on the count. Break at the first un-profitable
diff --git a/contrib/libs/llvm12/lib/Transforms/Instrumentation/PoisonChecking.cpp b/contrib/libs/llvm12/lib/Transforms/Instrumentation/PoisonChecking.cpp
index 1cd36629b5..fc52672618 100644
--- a/contrib/libs/llvm12/lib/Transforms/Instrumentation/PoisonChecking.cpp
+++ b/contrib/libs/llvm12/lib/Transforms/Instrumentation/PoisonChecking.cpp
@@ -282,10 +282,10 @@ static bool rewrite(Function &F) {
// Note: There are many more sources of documented UB, but this pass only
// attempts to find UB triggered by propagation of poison.
- SmallPtrSet<const Value *, 4> NonPoisonOps;
- getGuaranteedNonPoisonOps(&I, NonPoisonOps);
- for (const Value *Op : NonPoisonOps)
- CreateAssertNot(B, getPoisonFor(ValToPoison, const_cast<Value *>(Op)));
+ SmallPtrSet<const Value *, 4> NonPoisonOps;
+ getGuaranteedNonPoisonOps(&I, NonPoisonOps);
+ for (const Value *Op : NonPoisonOps)
+ CreateAssertNot(B, getPoisonFor(ValToPoison, const_cast<Value *>(Op)));
if (LocalCheck)
if (auto *RI = dyn_cast<ReturnInst>(&I))
@@ -295,11 +295,11 @@ static bool rewrite(Function &F) {
}
SmallVector<Value*, 4> Checks;
- if (propagatesPoison(cast<Operator>(&I)))
+ if (propagatesPoison(cast<Operator>(&I)))
for (Value *V : I.operands())
Checks.push_back(getPoisonFor(ValToPoison, V));
- if (canCreatePoison(cast<Operator>(&I)))
+ if (canCreatePoison(cast<Operator>(&I)))
generateCreationChecks(I, Checks);
ValToPoison[&I] = buildOrChain(B, Checks);
}
diff --git a/contrib/libs/llvm12/lib/Transforms/Instrumentation/SanitizerCoverage.cpp b/contrib/libs/llvm12/lib/Transforms/Instrumentation/SanitizerCoverage.cpp
index b840aafde7..2d4b079394 100644
--- a/contrib/libs/llvm12/lib/Transforms/Instrumentation/SanitizerCoverage.cpp
+++ b/contrib/libs/llvm12/lib/Transforms/Instrumentation/SanitizerCoverage.cpp
@@ -45,39 +45,39 @@ using namespace llvm;
#define DEBUG_TYPE "sancov"
-const char SanCovTracePCIndirName[] = "__sanitizer_cov_trace_pc_indir";
-const char SanCovTracePCName[] = "__sanitizer_cov_trace_pc";
-const char SanCovTraceCmp1[] = "__sanitizer_cov_trace_cmp1";
-const char SanCovTraceCmp2[] = "__sanitizer_cov_trace_cmp2";
-const char SanCovTraceCmp4[] = "__sanitizer_cov_trace_cmp4";
-const char SanCovTraceCmp8[] = "__sanitizer_cov_trace_cmp8";
-const char SanCovTraceConstCmp1[] = "__sanitizer_cov_trace_const_cmp1";
-const char SanCovTraceConstCmp2[] = "__sanitizer_cov_trace_const_cmp2";
-const char SanCovTraceConstCmp4[] = "__sanitizer_cov_trace_const_cmp4";
-const char SanCovTraceConstCmp8[] = "__sanitizer_cov_trace_const_cmp8";
-const char SanCovTraceDiv4[] = "__sanitizer_cov_trace_div4";
-const char SanCovTraceDiv8[] = "__sanitizer_cov_trace_div8";
-const char SanCovTraceGep[] = "__sanitizer_cov_trace_gep";
-const char SanCovTraceSwitchName[] = "__sanitizer_cov_trace_switch";
-const char SanCovModuleCtorTracePcGuardName[] =
+const char SanCovTracePCIndirName[] = "__sanitizer_cov_trace_pc_indir";
+const char SanCovTracePCName[] = "__sanitizer_cov_trace_pc";
+const char SanCovTraceCmp1[] = "__sanitizer_cov_trace_cmp1";
+const char SanCovTraceCmp2[] = "__sanitizer_cov_trace_cmp2";
+const char SanCovTraceCmp4[] = "__sanitizer_cov_trace_cmp4";
+const char SanCovTraceCmp8[] = "__sanitizer_cov_trace_cmp8";
+const char SanCovTraceConstCmp1[] = "__sanitizer_cov_trace_const_cmp1";
+const char SanCovTraceConstCmp2[] = "__sanitizer_cov_trace_const_cmp2";
+const char SanCovTraceConstCmp4[] = "__sanitizer_cov_trace_const_cmp4";
+const char SanCovTraceConstCmp8[] = "__sanitizer_cov_trace_const_cmp8";
+const char SanCovTraceDiv4[] = "__sanitizer_cov_trace_div4";
+const char SanCovTraceDiv8[] = "__sanitizer_cov_trace_div8";
+const char SanCovTraceGep[] = "__sanitizer_cov_trace_gep";
+const char SanCovTraceSwitchName[] = "__sanitizer_cov_trace_switch";
+const char SanCovModuleCtorTracePcGuardName[] =
"sancov.module_ctor_trace_pc_guard";
-const char SanCovModuleCtor8bitCountersName[] =
+const char SanCovModuleCtor8bitCountersName[] =
"sancov.module_ctor_8bit_counters";
-const char SanCovModuleCtorBoolFlagName[] = "sancov.module_ctor_bool_flag";
+const char SanCovModuleCtorBoolFlagName[] = "sancov.module_ctor_bool_flag";
static const uint64_t SanCtorAndDtorPriority = 2;
-const char SanCovTracePCGuardName[] = "__sanitizer_cov_trace_pc_guard";
-const char SanCovTracePCGuardInitName[] = "__sanitizer_cov_trace_pc_guard_init";
-const char SanCov8bitCountersInitName[] = "__sanitizer_cov_8bit_counters_init";
-const char SanCovBoolFlagInitName[] = "__sanitizer_cov_bool_flag_init";
-const char SanCovPCsInitName[] = "__sanitizer_cov_pcs_init";
+const char SanCovTracePCGuardName[] = "__sanitizer_cov_trace_pc_guard";
+const char SanCovTracePCGuardInitName[] = "__sanitizer_cov_trace_pc_guard_init";
+const char SanCov8bitCountersInitName[] = "__sanitizer_cov_8bit_counters_init";
+const char SanCovBoolFlagInitName[] = "__sanitizer_cov_bool_flag_init";
+const char SanCovPCsInitName[] = "__sanitizer_cov_pcs_init";
-const char SanCovGuardsSectionName[] = "sancov_guards";
-const char SanCovCountersSectionName[] = "sancov_cntrs";
-const char SanCovBoolFlagSectionName[] = "sancov_bools";
-const char SanCovPCsSectionName[] = "sancov_pcs";
+const char SanCovGuardsSectionName[] = "sancov_guards";
+const char SanCovCountersSectionName[] = "sancov_cntrs";
+const char SanCovBoolFlagSectionName[] = "sancov_bools";
+const char SanCovPCsSectionName[] = "sancov_pcs";
-const char SanCovLowestStackName[] = "__sancov_lowest_stack";
+const char SanCovLowestStackName[] = "__sancov_lowest_stack";
static cl::opt<int> ClCoverageLevel(
"sanitizer-coverage-level",
@@ -328,24 +328,24 @@ PreservedAnalyses ModuleSanitizerCoveragePass::run(Module &M,
std::pair<Value *, Value *>
ModuleSanitizerCoverage::CreateSecStartEnd(Module &M, const char *Section,
Type *Ty) {
- GlobalVariable *SecStart = new GlobalVariable(
- M, Ty->getPointerElementType(), false, GlobalVariable::ExternalLinkage,
- nullptr, getSectionStart(Section));
+ GlobalVariable *SecStart = new GlobalVariable(
+ M, Ty->getPointerElementType(), false, GlobalVariable::ExternalLinkage,
+ nullptr, getSectionStart(Section));
SecStart->setVisibility(GlobalValue::HiddenVisibility);
- GlobalVariable *SecEnd = new GlobalVariable(
- M, Ty->getPointerElementType(), false, GlobalVariable::ExternalLinkage,
- nullptr, getSectionEnd(Section));
+ GlobalVariable *SecEnd = new GlobalVariable(
+ M, Ty->getPointerElementType(), false, GlobalVariable::ExternalLinkage,
+ nullptr, getSectionEnd(Section));
SecEnd->setVisibility(GlobalValue::HiddenVisibility);
IRBuilder<> IRB(M.getContext());
if (!TargetTriple.isOSBinFormatCOFF())
- return std::make_pair(SecStart, SecEnd);
+ return std::make_pair(SecStart, SecEnd);
// Account for the fact that on windows-msvc __start_* symbols actually
// point to a uint64_t before the start of the array.
auto SecStartI8Ptr = IRB.CreatePointerCast(SecStart, Int8PtrTy);
auto GEP = IRB.CreateGEP(Int8Ty, SecStartI8Ptr,
ConstantInt::get(IntptrTy, sizeof(uint64_t)));
- return std::make_pair(IRB.CreatePointerCast(GEP, Ty), SecEnd);
+ return std::make_pair(IRB.CreatePointerCast(GEP, Ty), SecEnd);
}
Function *ModuleSanitizerCoverage::CreateInitCallsForSections(
@@ -415,13 +415,13 @@ bool ModuleSanitizerCoverage::instrumentModule(
SanCovTracePCIndir =
M.getOrInsertFunction(SanCovTracePCIndirName, VoidTy, IntptrTy);
- // Make sure smaller parameters are zero-extended to i64 if required by the
- // target ABI.
+ // Make sure smaller parameters are zero-extended to i64 if required by the
+ // target ABI.
AttributeList SanCovTraceCmpZeroExtAL;
- SanCovTraceCmpZeroExtAL =
- SanCovTraceCmpZeroExtAL.addParamAttribute(*C, 0, Attribute::ZExt);
- SanCovTraceCmpZeroExtAL =
- SanCovTraceCmpZeroExtAL.addParamAttribute(*C, 1, Attribute::ZExt);
+ SanCovTraceCmpZeroExtAL =
+ SanCovTraceCmpZeroExtAL.addParamAttribute(*C, 0, Attribute::ZExt);
+ SanCovTraceCmpZeroExtAL =
+ SanCovTraceCmpZeroExtAL.addParamAttribute(*C, 1, Attribute::ZExt);
SanCovTraceCmpFunction[0] =
M.getOrInsertFunction(SanCovTraceCmp1, SanCovTraceCmpZeroExtAL, VoidTy,
@@ -446,7 +446,7 @@ bool ModuleSanitizerCoverage::instrumentModule(
{
AttributeList AL;
- AL = AL.addParamAttribute(*C, 0, Attribute::ZExt);
+ AL = AL.addParamAttribute(*C, 0, Attribute::ZExt);
SanCovTraceDivFunction[0] =
M.getOrInsertFunction(SanCovTraceDiv4, AL, VoidTy, IRB.getInt32Ty());
}
@@ -509,23 +509,23 @@ bool ModuleSanitizerCoverage::instrumentModule(
// True if block has successors and it dominates all of them.
static bool isFullDominator(const BasicBlock *BB, const DominatorTree *DT) {
- if (succ_empty(BB))
+ if (succ_empty(BB))
return false;
- return llvm::all_of(successors(BB), [&](const BasicBlock *SUCC) {
- return DT->dominates(BB, SUCC);
- });
+ return llvm::all_of(successors(BB), [&](const BasicBlock *SUCC) {
+ return DT->dominates(BB, SUCC);
+ });
}
// True if block has predecessors and it postdominates all of them.
static bool isFullPostDominator(const BasicBlock *BB,
const PostDominatorTree *PDT) {
- if (pred_empty(BB))
+ if (pred_empty(BB))
return false;
- return llvm::all_of(predecessors(BB), [&](const BasicBlock *PRED) {
- return PDT->dominates(BB, PRED);
- });
+ return llvm::all_of(predecessors(BB), [&](const BasicBlock *PRED) {
+ return PDT->dominates(BB, PRED);
+ });
}
static bool shouldInstrumentBlock(const Function &F, const BasicBlock *BB,
@@ -795,7 +795,7 @@ void ModuleSanitizerCoverage::InjectTraceForSwitch(
C = ConstantExpr::getCast(CastInst::ZExt, It.getCaseValue(), Int64Ty);
Initializers.push_back(C);
}
- llvm::sort(drop_begin(Initializers, 2),
+ llvm::sort(drop_begin(Initializers, 2),
[](const Constant *A, const Constant *B) {
return cast<ConstantInt>(A)->getLimitedValue() <
cast<ConstantInt>(B)->getLimitedValue();
@@ -883,7 +883,7 @@ void ModuleSanitizerCoverage::InjectCoverageAtBlock(Function &F, BasicBlock &BB,
DebugLoc EntryLoc;
if (IsEntryBB) {
if (auto SP = F.getSubprogram())
- EntryLoc = DILocation::get(SP->getContext(), SP->getScopeLine(), 0, SP);
+ EntryLoc = DILocation::get(SP->getContext(), SP->getScopeLine(), 0, SP);
// Keep static allocas and llvm.localescape calls in the entry block. Even
// if we aren't splitting the block, it's nice for allocas to be before
// calls.
diff --git a/contrib/libs/llvm12/lib/Transforms/Instrumentation/ThreadSanitizer.cpp b/contrib/libs/llvm12/lib/Transforms/Instrumentation/ThreadSanitizer.cpp
index 85d36a0d4f..783878cf1e 100644
--- a/contrib/libs/llvm12/lib/Transforms/Instrumentation/ThreadSanitizer.cpp
+++ b/contrib/libs/llvm12/lib/Transforms/Instrumentation/ThreadSanitizer.cpp
@@ -19,8 +19,8 @@
//===----------------------------------------------------------------------===//
#include "llvm/Transforms/Instrumentation/ThreadSanitizer.h"
-#include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/Optional.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/Optional.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
@@ -53,36 +53,36 @@ using namespace llvm;
#define DEBUG_TYPE "tsan"
-static cl::opt<bool> ClInstrumentMemoryAccesses(
+static cl::opt<bool> ClInstrumentMemoryAccesses(
"tsan-instrument-memory-accesses", cl::init(true),
cl::desc("Instrument memory accesses"), cl::Hidden);
-static cl::opt<bool>
- ClInstrumentFuncEntryExit("tsan-instrument-func-entry-exit", cl::init(true),
- cl::desc("Instrument function entry and exit"),
- cl::Hidden);
-static cl::opt<bool> ClHandleCxxExceptions(
+static cl::opt<bool>
+ ClInstrumentFuncEntryExit("tsan-instrument-func-entry-exit", cl::init(true),
+ cl::desc("Instrument function entry and exit"),
+ cl::Hidden);
+static cl::opt<bool> ClHandleCxxExceptions(
"tsan-handle-cxx-exceptions", cl::init(true),
cl::desc("Handle C++ exceptions (insert cleanup blocks for unwinding)"),
cl::Hidden);
-static cl::opt<bool> ClInstrumentAtomics("tsan-instrument-atomics",
- cl::init(true),
- cl::desc("Instrument atomics"),
- cl::Hidden);
-static cl::opt<bool> ClInstrumentMemIntrinsics(
+static cl::opt<bool> ClInstrumentAtomics("tsan-instrument-atomics",
+ cl::init(true),
+ cl::desc("Instrument atomics"),
+ cl::Hidden);
+static cl::opt<bool> ClInstrumentMemIntrinsics(
"tsan-instrument-memintrinsics", cl::init(true),
cl::desc("Instrument memintrinsics (memset/memcpy/memmove)"), cl::Hidden);
-static cl::opt<bool> ClDistinguishVolatile(
+static cl::opt<bool> ClDistinguishVolatile(
"tsan-distinguish-volatile", cl::init(false),
cl::desc("Emit special instrumentation for accesses to volatiles"),
cl::Hidden);
-static cl::opt<bool> ClInstrumentReadBeforeWrite(
+static cl::opt<bool> ClInstrumentReadBeforeWrite(
"tsan-instrument-read-before-write", cl::init(false),
cl::desc("Do not eliminate read instrumentation for read-before-writes"),
cl::Hidden);
-static cl::opt<bool> ClCompoundReadBeforeWrite(
- "tsan-compound-read-before-write", cl::init(false),
- cl::desc("Emit special compound instrumentation for reads-before-writes"),
- cl::Hidden);
+static cl::opt<bool> ClCompoundReadBeforeWrite(
+ "tsan-compound-read-before-write", cl::init(false),
+ cl::desc("Emit special compound instrumentation for reads-before-writes"),
+ cl::Hidden);
STATISTIC(NumInstrumentedReads, "Number of instrumented reads");
STATISTIC(NumInstrumentedWrites, "Number of instrumented writes");
@@ -96,8 +96,8 @@ STATISTIC(NumOmittedReadsFromConstantGlobals,
STATISTIC(NumOmittedReadsFromVtable, "Number of vtable reads");
STATISTIC(NumOmittedNonCaptured, "Number of accesses ignored due to capturing");
-const char kTsanModuleCtorName[] = "tsan.module_ctor";
-const char kTsanInitName[] = "__tsan_init";
+const char kTsanModuleCtorName[] = "tsan.module_ctor";
+const char kTsanInitName[] = "__tsan_init";
namespace {
@@ -108,37 +108,37 @@ namespace {
/// ensures the __tsan_init function is in the list of global constructors for
/// the module.
struct ThreadSanitizer {
- ThreadSanitizer() {
- // Sanity check options and warn user.
- if (ClInstrumentReadBeforeWrite && ClCompoundReadBeforeWrite) {
- errs()
- << "warning: Option -tsan-compound-read-before-write has no effect "
- "when -tsan-instrument-read-before-write is set.\n";
- }
- }
-
+ ThreadSanitizer() {
+ // Sanity check options and warn user.
+ if (ClInstrumentReadBeforeWrite && ClCompoundReadBeforeWrite) {
+ errs()
+ << "warning: Option -tsan-compound-read-before-write has no effect "
+ "when -tsan-instrument-read-before-write is set.\n";
+ }
+ }
+
bool sanitizeFunction(Function &F, const TargetLibraryInfo &TLI);
private:
- // Internal Instruction wrapper that contains more information about the
- // Instruction from prior analysis.
- struct InstructionInfo {
- // Instrumentation emitted for this instruction is for a compounded set of
- // read and write operations in the same basic block.
- static constexpr unsigned kCompoundRW = (1U << 0);
-
- explicit InstructionInfo(Instruction *Inst) : Inst(Inst) {}
-
- Instruction *Inst;
- unsigned Flags = 0;
- };
-
+ // Internal Instruction wrapper that contains more information about the
+ // Instruction from prior analysis.
+ struct InstructionInfo {
+ // Instrumentation emitted for this instruction is for a compounded set of
+ // read and write operations in the same basic block.
+ static constexpr unsigned kCompoundRW = (1U << 0);
+
+ explicit InstructionInfo(Instruction *Inst) : Inst(Inst) {}
+
+ Instruction *Inst;
+ unsigned Flags = 0;
+ };
+
void initialize(Module &M);
- bool instrumentLoadOrStore(const InstructionInfo &II, const DataLayout &DL);
+ bool instrumentLoadOrStore(const InstructionInfo &II, const DataLayout &DL);
bool instrumentAtomic(Instruction *I, const DataLayout &DL);
bool instrumentMemIntrinsic(Instruction *I);
void chooseInstructionsToInstrument(SmallVectorImpl<Instruction *> &Local,
- SmallVectorImpl<InstructionInfo> &All,
+ SmallVectorImpl<InstructionInfo> &All,
const DataLayout &DL);
bool addrPointsToConstantData(Value *Addr);
int getMemoryAccessFuncIndex(Value *Addr, const DataLayout &DL);
@@ -159,8 +159,8 @@ private:
FunctionCallee TsanVolatileWrite[kNumberOfAccessSizes];
FunctionCallee TsanUnalignedVolatileRead[kNumberOfAccessSizes];
FunctionCallee TsanUnalignedVolatileWrite[kNumberOfAccessSizes];
- FunctionCallee TsanCompoundRW[kNumberOfAccessSizes];
- FunctionCallee TsanUnalignedCompoundRW[kNumberOfAccessSizes];
+ FunctionCallee TsanCompoundRW[kNumberOfAccessSizes];
+ FunctionCallee TsanUnalignedCompoundRW[kNumberOfAccessSizes];
FunctionCallee TsanAtomicLoad[kNumberOfAccessSizes];
FunctionCallee TsanAtomicStore[kNumberOfAccessSizes];
FunctionCallee TsanAtomicRMW[AtomicRMWInst::LAST_BINOP + 1]
@@ -299,15 +299,15 @@ void ThreadSanitizer::initialize(Module &M) {
TsanUnalignedVolatileWrite[i] = M.getOrInsertFunction(
UnalignedVolatileWriteName, Attr, IRB.getVoidTy(), IRB.getInt8PtrTy());
- SmallString<64> CompoundRWName("__tsan_read_write" + ByteSizeStr);
- TsanCompoundRW[i] = M.getOrInsertFunction(
- CompoundRWName, Attr, IRB.getVoidTy(), IRB.getInt8PtrTy());
-
- SmallString<64> UnalignedCompoundRWName("__tsan_unaligned_read_write" +
- ByteSizeStr);
- TsanUnalignedCompoundRW[i] = M.getOrInsertFunction(
- UnalignedCompoundRWName, Attr, IRB.getVoidTy(), IRB.getInt8PtrTy());
-
+ SmallString<64> CompoundRWName("__tsan_read_write" + ByteSizeStr);
+ TsanCompoundRW[i] = M.getOrInsertFunction(
+ CompoundRWName, Attr, IRB.getVoidTy(), IRB.getInt8PtrTy());
+
+ SmallString<64> UnalignedCompoundRWName("__tsan_unaligned_read_write" +
+ ByteSizeStr);
+ TsanUnalignedCompoundRW[i] = M.getOrInsertFunction(
+ UnalignedCompoundRWName, Attr, IRB.getVoidTy(), IRB.getInt8PtrTy());
+
Type *Ty = Type::getIntNTy(M.getContext(), BitSize);
Type *PtrTy = Ty->getPointerTo();
SmallString<32> AtomicLoadName("__tsan_atomic" + BitSizeStr + "_load");
@@ -442,43 +442,43 @@ bool ThreadSanitizer::addrPointsToConstantData(Value *Addr) {
// 'Local' is a vector of insns within the same BB (no calls between).
// 'All' is a vector of insns that will be instrumented.
void ThreadSanitizer::chooseInstructionsToInstrument(
- SmallVectorImpl<Instruction *> &Local,
- SmallVectorImpl<InstructionInfo> &All, const DataLayout &DL) {
- DenseMap<Value *, size_t> WriteTargets; // Map of addresses to index in All
+ SmallVectorImpl<Instruction *> &Local,
+ SmallVectorImpl<InstructionInfo> &All, const DataLayout &DL) {
+ DenseMap<Value *, size_t> WriteTargets; // Map of addresses to index in All
// Iterate from the end.
for (Instruction *I : reverse(Local)) {
- const bool IsWrite = isa<StoreInst>(*I);
- Value *Addr = IsWrite ? cast<StoreInst>(I)->getPointerOperand()
- : cast<LoadInst>(I)->getPointerOperand();
-
- if (!shouldInstrumentReadWriteFromAddress(I->getModule(), Addr))
- continue;
-
- if (!IsWrite) {
- const auto WriteEntry = WriteTargets.find(Addr);
- if (!ClInstrumentReadBeforeWrite && WriteEntry != WriteTargets.end()) {
- auto &WI = All[WriteEntry->second];
- // If we distinguish volatile accesses and if either the read or write
- // is volatile, do not omit any instrumentation.
- const bool AnyVolatile =
- ClDistinguishVolatile && (cast<LoadInst>(I)->isVolatile() ||
- cast<StoreInst>(WI.Inst)->isVolatile());
- if (!AnyVolatile) {
- // We will write to this temp, so no reason to analyze the read.
- // Mark the write instruction as compound.
- WI.Flags |= InstructionInfo::kCompoundRW;
- NumOmittedReadsBeforeWrite++;
- continue;
- }
+ const bool IsWrite = isa<StoreInst>(*I);
+ Value *Addr = IsWrite ? cast<StoreInst>(I)->getPointerOperand()
+ : cast<LoadInst>(I)->getPointerOperand();
+
+ if (!shouldInstrumentReadWriteFromAddress(I->getModule(), Addr))
+ continue;
+
+ if (!IsWrite) {
+ const auto WriteEntry = WriteTargets.find(Addr);
+ if (!ClInstrumentReadBeforeWrite && WriteEntry != WriteTargets.end()) {
+ auto &WI = All[WriteEntry->second];
+ // If we distinguish volatile accesses and if either the read or write
+ // is volatile, do not omit any instrumentation.
+ const bool AnyVolatile =
+ ClDistinguishVolatile && (cast<LoadInst>(I)->isVolatile() ||
+ cast<StoreInst>(WI.Inst)->isVolatile());
+ if (!AnyVolatile) {
+ // We will write to this temp, so no reason to analyze the read.
+ // Mark the write instruction as compound.
+ WI.Flags |= InstructionInfo::kCompoundRW;
+ NumOmittedReadsBeforeWrite++;
+ continue;
+ }
}
-
+
if (addrPointsToConstantData(Addr)) {
// Addr points to some constant data -- it can not race with any writes.
continue;
}
}
-
- if (isa<AllocaInst>(getUnderlyingObject(Addr)) &&
+
+ if (isa<AllocaInst>(getUnderlyingObject(Addr)) &&
!PointerMayBeCaptured(Addr, true, true)) {
// The variable is addressable but not captured, so it cannot be
// referenced from a different thread and participate in a data race
@@ -486,14 +486,14 @@ void ThreadSanitizer::chooseInstructionsToInstrument(
NumOmittedNonCaptured++;
continue;
}
-
- // Instrument this instruction.
- All.emplace_back(I);
- if (IsWrite) {
- // For read-before-write and compound instrumentation we only need one
- // write target, and we can override any previous entry if it exists.
- WriteTargets[Addr] = All.size() - 1;
- }
+
+ // Instrument this instruction.
+ All.emplace_back(I);
+ if (IsWrite) {
+ // For read-before-write and compound instrumentation we only need one
+ // write target, and we can override any previous entry if it exists.
+ WriteTargets[Addr] = All.size() - 1;
+ }
}
Local.clear();
}
@@ -534,7 +534,7 @@ bool ThreadSanitizer::sanitizeFunction(Function &F,
if (F.hasFnAttribute(Attribute::Naked))
return false;
initialize(*F.getParent());
- SmallVector<InstructionInfo, 8> AllLoadsAndStores;
+ SmallVector<InstructionInfo, 8> AllLoadsAndStores;
SmallVector<Instruction*, 8> LocalLoadsAndStores;
SmallVector<Instruction*, 8> AtomicAccesses;
SmallVector<Instruction*, 8> MemIntrinCalls;
@@ -569,8 +569,8 @@ bool ThreadSanitizer::sanitizeFunction(Function &F,
// Instrument memory accesses only if we want to report bugs in the function.
if (ClInstrumentMemoryAccesses && SanitizeFunction)
- for (const auto &II : AllLoadsAndStores) {
- Res |= instrumentLoadOrStore(II, DL);
+ for (const auto &II : AllLoadsAndStores) {
+ Res |= instrumentLoadOrStore(II, DL);
}
// Instrument atomic memory accesses in any case (they can be used to
@@ -608,12 +608,12 @@ bool ThreadSanitizer::sanitizeFunction(Function &F,
return Res;
}
-bool ThreadSanitizer::instrumentLoadOrStore(const InstructionInfo &II,
+bool ThreadSanitizer::instrumentLoadOrStore(const InstructionInfo &II,
const DataLayout &DL) {
- IRBuilder<> IRB(II.Inst);
- const bool IsWrite = isa<StoreInst>(*II.Inst);
- Value *Addr = IsWrite ? cast<StoreInst>(II.Inst)->getPointerOperand()
- : cast<LoadInst>(II.Inst)->getPointerOperand();
+ IRBuilder<> IRB(II.Inst);
+ const bool IsWrite = isa<StoreInst>(*II.Inst);
+ Value *Addr = IsWrite ? cast<StoreInst>(II.Inst)->getPointerOperand()
+ : cast<LoadInst>(II.Inst)->getPointerOperand();
// swifterror memory addresses are mem2reg promoted by instruction selection.
// As such they cannot have regular uses like an instrumentation function and
@@ -624,9 +624,9 @@ bool ThreadSanitizer::instrumentLoadOrStore(const InstructionInfo &II,
int Idx = getMemoryAccessFuncIndex(Addr, DL);
if (Idx < 0)
return false;
- if (IsWrite && isVtableAccess(II.Inst)) {
- LLVM_DEBUG(dbgs() << " VPTR : " << *II.Inst << "\n");
- Value *StoredValue = cast<StoreInst>(II.Inst)->getValueOperand();
+ if (IsWrite && isVtableAccess(II.Inst)) {
+ LLVM_DEBUG(dbgs() << " VPTR : " << *II.Inst << "\n");
+ Value *StoredValue = cast<StoreInst>(II.Inst)->getValueOperand();
// StoredValue may be a vector type if we are storing several vptrs at once.
// In this case, just take the first element of the vector since this is
// enough to find vptr races.
@@ -642,46 +642,46 @@ bool ThreadSanitizer::instrumentLoadOrStore(const InstructionInfo &II,
NumInstrumentedVtableWrites++;
return true;
}
- if (!IsWrite && isVtableAccess(II.Inst)) {
+ if (!IsWrite && isVtableAccess(II.Inst)) {
IRB.CreateCall(TsanVptrLoad,
IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy()));
NumInstrumentedVtableReads++;
return true;
}
-
- const unsigned Alignment = IsWrite ? cast<StoreInst>(II.Inst)->getAlignment()
- : cast<LoadInst>(II.Inst)->getAlignment();
- const bool IsCompoundRW =
- ClCompoundReadBeforeWrite && (II.Flags & InstructionInfo::kCompoundRW);
- const bool IsVolatile = ClDistinguishVolatile &&
- (IsWrite ? cast<StoreInst>(II.Inst)->isVolatile()
- : cast<LoadInst>(II.Inst)->isVolatile());
- assert((!IsVolatile || !IsCompoundRW) && "Compound volatile invalid!");
-
+
+ const unsigned Alignment = IsWrite ? cast<StoreInst>(II.Inst)->getAlignment()
+ : cast<LoadInst>(II.Inst)->getAlignment();
+ const bool IsCompoundRW =
+ ClCompoundReadBeforeWrite && (II.Flags & InstructionInfo::kCompoundRW);
+ const bool IsVolatile = ClDistinguishVolatile &&
+ (IsWrite ? cast<StoreInst>(II.Inst)->isVolatile()
+ : cast<LoadInst>(II.Inst)->isVolatile());
+ assert((!IsVolatile || !IsCompoundRW) && "Compound volatile invalid!");
+
Type *OrigTy = cast<PointerType>(Addr->getType())->getElementType();
const uint32_t TypeSize = DL.getTypeStoreSizeInBits(OrigTy);
FunctionCallee OnAccessFunc = nullptr;
if (Alignment == 0 || Alignment >= 8 || (Alignment % (TypeSize / 8)) == 0) {
- if (IsCompoundRW)
- OnAccessFunc = TsanCompoundRW[Idx];
- else if (IsVolatile)
+ if (IsCompoundRW)
+ OnAccessFunc = TsanCompoundRW[Idx];
+ else if (IsVolatile)
OnAccessFunc = IsWrite ? TsanVolatileWrite[Idx] : TsanVolatileRead[Idx];
else
OnAccessFunc = IsWrite ? TsanWrite[Idx] : TsanRead[Idx];
} else {
- if (IsCompoundRW)
- OnAccessFunc = TsanUnalignedCompoundRW[Idx];
- else if (IsVolatile)
+ if (IsCompoundRW)
+ OnAccessFunc = TsanUnalignedCompoundRW[Idx];
+ else if (IsVolatile)
OnAccessFunc = IsWrite ? TsanUnalignedVolatileWrite[Idx]
: TsanUnalignedVolatileRead[Idx];
else
OnAccessFunc = IsWrite ? TsanUnalignedWrite[Idx] : TsanUnalignedRead[Idx];
}
IRB.CreateCall(OnAccessFunc, IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy()));
- if (IsCompoundRW || IsWrite)
- NumInstrumentedWrites++;
- if (IsCompoundRW || !IsWrite)
- NumInstrumentedReads++;
+ if (IsCompoundRW || IsWrite)
+ NumInstrumentedWrites++;
+ if (IsCompoundRW || !IsWrite)
+ NumInstrumentedReads++;
return true;
}
diff --git a/contrib/libs/llvm12/lib/Transforms/Instrumentation/ValueProfileCollector.cpp b/contrib/libs/llvm12/lib/Transforms/Instrumentation/ValueProfileCollector.cpp
index e4c06b9466..fb6216bb21 100644
--- a/contrib/libs/llvm12/lib/Transforms/Instrumentation/ValueProfileCollector.cpp
+++ b/contrib/libs/llvm12/lib/Transforms/Instrumentation/ValueProfileCollector.cpp
@@ -11,7 +11,7 @@
//===----------------------------------------------------------------------===//
#include "ValueProfilePlugins.inc"
-#include "llvm/IR/Function.h"
+#include "llvm/IR/Function.h"
#include "llvm/IR/InstIterator.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/InitializePasses.h"
diff --git a/contrib/libs/llvm12/lib/Transforms/Instrumentation/ValueProfileCollector.h b/contrib/libs/llvm12/lib/Transforms/Instrumentation/ValueProfileCollector.h
index b3840d7a69..584a60ab45 100644
--- a/contrib/libs/llvm12/lib/Transforms/Instrumentation/ValueProfileCollector.h
+++ b/contrib/libs/llvm12/lib/Transforms/Instrumentation/ValueProfileCollector.h
@@ -18,15 +18,15 @@
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/ProfileData/InstrProf.h"
-#include <memory>
-#include <vector>
+#include <memory>
+#include <vector>
namespace llvm {
-class Function;
-class Instruction;
-class Value;
-
+class Function;
+class Instruction;
+class Value;
+
/// Utility analysis that determines what values are worth profiling.
/// The actual logic is inside the ValueProfileCollectorImpl, whose job is to
/// populate the Candidates vector.
diff --git a/contrib/libs/llvm12/lib/Transforms/Instrumentation/ya.make b/contrib/libs/llvm12/lib/Transforms/Instrumentation/ya.make
index 82f8c92dc5..39dab1eb7d 100644
--- a/contrib/libs/llvm12/lib/Transforms/Instrumentation/ya.make
+++ b/contrib/libs/llvm12/lib/Transforms/Instrumentation/ya.make
@@ -15,14 +15,14 @@ LICENSE(
LICENSE_TEXTS(.yandex_meta/licenses.list.txt)
PEERDIR(
- contrib/libs/llvm12
- contrib/libs/llvm12/include
- contrib/libs/llvm12/lib/Analysis
- contrib/libs/llvm12/lib/IR
- contrib/libs/llvm12/lib/MC
- contrib/libs/llvm12/lib/ProfileData
- contrib/libs/llvm12/lib/Support
- contrib/libs/llvm12/lib/Transforms/Utils
+ contrib/libs/llvm12
+ contrib/libs/llvm12/include
+ contrib/libs/llvm12/lib/Analysis
+ contrib/libs/llvm12/lib/IR
+ contrib/libs/llvm12/lib/MC
+ contrib/libs/llvm12/lib/ProfileData
+ contrib/libs/llvm12/lib/Support
+ contrib/libs/llvm12/lib/Transforms/Utils
)
ADDINCL(
@@ -45,7 +45,7 @@ SRCS(
InstrOrderFile.cpp
InstrProfiling.cpp
Instrumentation.cpp
- MemProfiler.cpp
+ MemProfiler.cpp
MemorySanitizer.cpp
PGOInstrumentation.cpp
PGOMemOPSizeOpt.cpp
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-24 22:36:16 +0000