aboutsummaryrefslogtreecommitdiffstats
path: root/contrib/libs/llvm12/lib/Transforms/Scalar/LoopSink.cpp
diff options
context:
space:
mode:
authorshadchin <shadchin@yandex-team.ru>2022-02-10 16:44:39 +0300
committerDaniil Cherednik <dcherednik@yandex-team.ru>2022-02-10 16:44:39 +0300
commite9656aae26e0358d5378e5b63dcac5c8dbe0e4d0 (patch)
tree64175d5cadab313b3e7039ebaa06c5bc3295e274 /contrib/libs/llvm12/lib/Transforms/Scalar/LoopSink.cpp
parent2598ef1d0aee359b4b6d5fdd1758916d5907d04f (diff)
downloadydb-e9656aae26e0358d5378e5b63dcac5c8dbe0e4d0.tar.gz
Restoring authorship annotation for <shadchin@yandex-team.ru>. Commit 2 of 2.
Diffstat (limited to 'contrib/libs/llvm12/lib/Transforms/Scalar/LoopSink.cpp')
-rw-r--r--contrib/libs/llvm12/lib/Transforms/Scalar/LoopSink.cpp242
1 files changed, 121 insertions, 121 deletions
diff --git a/contrib/libs/llvm12/lib/Transforms/Scalar/LoopSink.cpp b/contrib/libs/llvm12/lib/Transforms/Scalar/LoopSink.cpp
index 0296b12878..47698fdde6 100644
--- a/contrib/libs/llvm12/lib/Transforms/Scalar/LoopSink.cpp
+++ b/contrib/libs/llvm12/lib/Transforms/Scalar/LoopSink.cpp
@@ -39,8 +39,8 @@
#include "llvm/Analysis/Loads.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/LoopPass.h"
-#include "llvm/Analysis/MemorySSA.h"
-#include "llvm/Analysis/MemorySSAUpdater.h"
+#include "llvm/Analysis/MemorySSA.h"
+#include "llvm/Analysis/MemorySSAUpdater.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h"
#include "llvm/IR/Dominators.h"
@@ -69,14 +69,14 @@ static cl::opt<unsigned> MaxNumberOfUseBBsForSinking(
"max-uses-for-sinking", cl::Hidden, cl::init(30),
cl::desc("Do not sink instructions that have too many uses."));
-static cl::opt<bool> EnableMSSAInLoopSink(
- "enable-mssa-in-loop-sink", cl::Hidden, cl::init(true),
- cl::desc("Enable MemorySSA for LoopSink in new pass manager"));
-
-static cl::opt<bool> EnableMSSAInLegacyLoopSink(
- "enable-mssa-in-legacy-loop-sink", cl::Hidden, cl::init(false),
- cl::desc("Enable MemorySSA for LoopSink in legacy pass manager"));
-
+static cl::opt<bool> EnableMSSAInLoopSink(
+ "enable-mssa-in-loop-sink", cl::Hidden, cl::init(true),
+ cl::desc("Enable MemorySSA for LoopSink in new pass manager"));
+
+static cl::opt<bool> EnableMSSAInLegacyLoopSink(
+ "enable-mssa-in-legacy-loop-sink", cl::Hidden, cl::init(false),
+ cl::desc("Enable MemorySSA for LoopSink in legacy pass manager"));
+
/// Return adjusted total frequency of \p BBs.
///
/// * If there is only one BB, sinking instruction will not introduce code
@@ -182,10 +182,10 @@ findBBsToSinkInto(const Loop &L, const SmallPtrSetImpl<BasicBlock *> &UseBBs,
// sinking is successful.
// \p LoopBlockNumber is used to sort the insertion blocks to ensure
// determinism.
-static bool sinkInstruction(
- Loop &L, Instruction &I, const SmallVectorImpl<BasicBlock *> &ColdLoopBBs,
- const SmallDenseMap<BasicBlock *, int, 16> &LoopBlockNumber, LoopInfo &LI,
- DominatorTree &DT, BlockFrequencyInfo &BFI, MemorySSAUpdater *MSSAU) {
+static bool sinkInstruction(
+ Loop &L, Instruction &I, const SmallVectorImpl<BasicBlock *> &ColdLoopBBs,
+ const SmallDenseMap<BasicBlock *, int, 16> &LoopBlockNumber, LoopInfo &LI,
+ DominatorTree &DT, BlockFrequencyInfo &BFI, MemorySSAUpdater *MSSAU) {
// Compute the set of blocks in loop L which contain a use of I.
SmallPtrSet<BasicBlock *, 2> BBs;
for (auto &U : I.uses()) {
@@ -222,7 +222,7 @@ static bool sinkInstruction(
// of the loop block numbers as iterating the set doesn't give a useful
// order. No need to stable sort as the block numbers are a total ordering.
SmallVector<BasicBlock *, 2> SortedBBsToSinkInto;
- llvm::append_range(SortedBBsToSinkInto, BBsToSinkInto);
+ llvm::append_range(SortedBBsToSinkInto, BBsToSinkInto);
llvm::sort(SortedBBsToSinkInto, [&](BasicBlock *A, BasicBlock *B) {
return LoopBlockNumber.find(A)->second < LoopBlockNumber.find(B)->second;
});
@@ -238,21 +238,21 @@ static bool sinkInstruction(
Instruction *IC = I.clone();
IC->setName(I.getName());
IC->insertBefore(&*N->getFirstInsertionPt());
-
- if (MSSAU && MSSAU->getMemorySSA()->getMemoryAccess(&I)) {
- // Create a new MemoryAccess and let MemorySSA set its defining access.
- MemoryAccess *NewMemAcc =
- MSSAU->createMemoryAccessInBB(IC, nullptr, N, MemorySSA::Beginning);
- if (NewMemAcc) {
- if (auto *MemDef = dyn_cast<MemoryDef>(NewMemAcc))
- MSSAU->insertDef(MemDef, /*RenameUses=*/true);
- else {
- auto *MemUse = cast<MemoryUse>(NewMemAcc);
- MSSAU->insertUse(MemUse, /*RenameUses=*/true);
- }
- }
- }
-
+
+ if (MSSAU && MSSAU->getMemorySSA()->getMemoryAccess(&I)) {
+ // Create a new MemoryAccess and let MemorySSA set its defining access.
+ MemoryAccess *NewMemAcc =
+ MSSAU->createMemoryAccessInBB(IC, nullptr, N, MemorySSA::Beginning);
+ if (NewMemAcc) {
+ if (auto *MemDef = dyn_cast<MemoryDef>(NewMemAcc))
+ MSSAU->insertDef(MemDef, /*RenameUses=*/true);
+ else {
+ auto *MemUse = cast<MemoryUse>(NewMemAcc);
+ MSSAU->insertUse(MemUse, /*RenameUses=*/true);
+ }
+ }
+ }
+
// Replaces uses of I with IC in N
I.replaceUsesWithIf(IC, [N](Use &U) {
return cast<Instruction>(U.getUser())->getParent() == N;
@@ -267,11 +267,11 @@ static bool sinkInstruction(
NumLoopSunk++;
I.moveBefore(&*MoveBB->getFirstInsertionPt());
- if (MSSAU)
- if (MemoryUseOrDef *OldMemAcc = cast_or_null<MemoryUseOrDef>(
- MSSAU->getMemorySSA()->getMemoryAccess(&I)))
- MSSAU->moveToPlace(OldMemAcc, MoveBB, MemorySSA::Beginning);
-
+ if (MSSAU)
+ if (MemoryUseOrDef *OldMemAcc = cast_or_null<MemoryUseOrDef>(
+ MSSAU->getMemorySSA()->getMemoryAccess(&I)))
+ MSSAU->moveToPlace(OldMemAcc, MoveBB, MemorySSA::Beginning);
+
return true;
}
@@ -280,14 +280,14 @@ static bool sinkInstruction(
static bool sinkLoopInvariantInstructions(Loop &L, AAResults &AA, LoopInfo &LI,
DominatorTree &DT,
BlockFrequencyInfo &BFI,
- ScalarEvolution *SE,
- AliasSetTracker *CurAST,
- MemorySSA *MSSA) {
+ ScalarEvolution *SE,
+ AliasSetTracker *CurAST,
+ MemorySSA *MSSA) {
BasicBlock *Preheader = L.getLoopPreheader();
- assert(Preheader && "Expected loop to have preheader");
+ assert(Preheader && "Expected loop to have preheader");
- assert(Preheader->getParent()->hasProfileData() &&
- "Unexpected call when profile data unavailable.");
+ assert(Preheader->getParent()->hasProfileData() &&
+ "Unexpected call when profile data unavailable.");
const BlockFrequency PreheaderFreq = BFI.getBlockFreq(Preheader);
// If there are no basic blocks with lower frequency than the preheader then
@@ -298,14 +298,14 @@ static bool sinkLoopInvariantInstructions(Loop &L, AAResults &AA, LoopInfo &LI,
}))
return false;
- std::unique_ptr<MemorySSAUpdater> MSSAU;
- std::unique_ptr<SinkAndHoistLICMFlags> LICMFlags;
- if (MSSA) {
- MSSAU = std::make_unique<MemorySSAUpdater>(MSSA);
- LICMFlags =
- std::make_unique<SinkAndHoistLICMFlags>(/*IsSink=*/true, &L, MSSA);
- }
-
+ std::unique_ptr<MemorySSAUpdater> MSSAU;
+ std::unique_ptr<SinkAndHoistLICMFlags> LICMFlags;
+ if (MSSA) {
+ MSSAU = std::make_unique<MemorySSAUpdater>(MSSA);
+ LICMFlags =
+ std::make_unique<SinkAndHoistLICMFlags>(/*IsSink=*/true, &L, MSSA);
+ }
+
bool Changed = false;
// Sort loop's basic blocks by frequency
@@ -329,11 +329,11 @@ static bool sinkLoopInvariantInstructions(Loop &L, AAResults &AA, LoopInfo &LI,
// No need to check for instruction's operands are loop invariant.
assert(L.hasLoopInvariantOperands(I) &&
"Insts in a loop's preheader should have loop invariant operands!");
- if (!canSinkOrHoistInst(*I, &AA, &DT, &L, CurAST, MSSAU.get(), false,
- LICMFlags.get()))
+ if (!canSinkOrHoistInst(*I, &AA, &DT, &L, CurAST, MSSAU.get(), false,
+ LICMFlags.get()))
continue;
- if (sinkInstruction(L, *I, ColdLoopBBs, LoopBlockNumber, LI, DT, BFI,
- MSSAU.get()))
+ if (sinkInstruction(L, *I, ColdLoopBBs, LoopBlockNumber, LI, DT, BFI,
+ MSSAU.get()))
Changed = true;
}
@@ -342,13 +342,13 @@ static bool sinkLoopInvariantInstructions(Loop &L, AAResults &AA, LoopInfo &LI,
return Changed;
}
-static void computeAliasSet(Loop &L, BasicBlock &Preheader,
- AliasSetTracker &CurAST) {
- for (BasicBlock *BB : L.blocks())
- CurAST.add(*BB);
- CurAST.add(Preheader);
-}
-
+static void computeAliasSet(Loop &L, BasicBlock &Preheader,
+ AliasSetTracker &CurAST) {
+ for (BasicBlock *BB : L.blocks())
+ CurAST.add(*BB);
+ CurAST.add(Preheader);
+}
+
PreservedAnalyses LoopSinkPass::run(Function &F, FunctionAnalysisManager &FAM) {
LoopInfo &LI = FAM.getResult<LoopAnalysis>(F);
// Nothing to do if there are no loops.
@@ -359,10 +359,10 @@ PreservedAnalyses LoopSinkPass::run(Function &F, FunctionAnalysisManager &FAM) {
DominatorTree &DT = FAM.getResult<DominatorTreeAnalysis>(F);
BlockFrequencyInfo &BFI = FAM.getResult<BlockFrequencyAnalysis>(F);
- MemorySSA *MSSA = EnableMSSAInLoopSink
- ? &FAM.getResult<MemorySSAAnalysis>(F).getMSSA()
- : nullptr;
-
+ MemorySSA *MSSA = EnableMSSAInLoopSink
+ ? &FAM.getResult<MemorySSAAnalysis>(F).getMSSA()
+ : nullptr;
+
// We want to do a postorder walk over the loops. Since loops are a tree this
// is equivalent to a reversed preorder walk and preorder is easy to compute
// without recursion. Since we reverse the preorder, we will visit siblings
@@ -374,27 +374,27 @@ PreservedAnalyses LoopSinkPass::run(Function &F, FunctionAnalysisManager &FAM) {
do {
Loop &L = *PreorderLoops.pop_back_val();
- BasicBlock *Preheader = L.getLoopPreheader();
- if (!Preheader)
- continue;
-
- // Enable LoopSink only when runtime profile is available.
- // With static profile, the sinking decision may be sub-optimal.
- if (!Preheader->getParent()->hasProfileData())
- continue;
-
- std::unique_ptr<AliasSetTracker> CurAST;
- if (!EnableMSSAInLoopSink) {
- CurAST = std::make_unique<AliasSetTracker>(AA);
- computeAliasSet(L, *Preheader, *CurAST.get());
- }
-
+ BasicBlock *Preheader = L.getLoopPreheader();
+ if (!Preheader)
+ continue;
+
+ // Enable LoopSink only when runtime profile is available.
+ // With static profile, the sinking decision may be sub-optimal.
+ if (!Preheader->getParent()->hasProfileData())
+ continue;
+
+ std::unique_ptr<AliasSetTracker> CurAST;
+ if (!EnableMSSAInLoopSink) {
+ CurAST = std::make_unique<AliasSetTracker>(AA);
+ computeAliasSet(L, *Preheader, *CurAST.get());
+ }
+
// Note that we don't pass SCEV here because it is only used to invalidate
// loops in SCEV and we don't preserve (or request) SCEV at all making that
// unnecessary.
Changed |= sinkLoopInvariantInstructions(L, AA, LI, DT, BFI,
- /*ScalarEvolution*/ nullptr,
- CurAST.get(), MSSA);
+ /*ScalarEvolution*/ nullptr,
+ CurAST.get(), MSSA);
} while (!PreorderLoops.empty());
if (!Changed)
@@ -402,14 +402,14 @@ PreservedAnalyses LoopSinkPass::run(Function &F, FunctionAnalysisManager &FAM) {
PreservedAnalyses PA;
PA.preserveSet<CFGAnalyses>();
-
- if (MSSA) {
- PA.preserve<MemorySSAAnalysis>();
-
- if (VerifyMemorySSA)
- MSSA->verifyMemorySSA();
- }
-
+
+ if (MSSA) {
+ PA.preserve<MemorySSAAnalysis>();
+
+ if (VerifyMemorySSA)
+ MSSA->verifyMemorySSA();
+ }
+
return PA;
}
@@ -424,46 +424,46 @@ struct LegacyLoopSinkPass : public LoopPass {
if (skipLoop(L))
return false;
- BasicBlock *Preheader = L->getLoopPreheader();
- if (!Preheader)
- return false;
-
- // Enable LoopSink only when runtime profile is available.
- // With static profile, the sinking decision may be sub-optimal.
- if (!Preheader->getParent()->hasProfileData())
- return false;
-
- AAResults &AA = getAnalysis<AAResultsWrapperPass>().getAAResults();
+ BasicBlock *Preheader = L->getLoopPreheader();
+ if (!Preheader)
+ return false;
+
+ // Enable LoopSink only when runtime profile is available.
+ // With static profile, the sinking decision may be sub-optimal.
+ if (!Preheader->getParent()->hasProfileData())
+ return false;
+
+ AAResults &AA = getAnalysis<AAResultsWrapperPass>().getAAResults();
auto *SE = getAnalysisIfAvailable<ScalarEvolutionWrapperPass>();
- std::unique_ptr<AliasSetTracker> CurAST;
- MemorySSA *MSSA = nullptr;
- if (EnableMSSAInLegacyLoopSink)
- MSSA = &getAnalysis<MemorySSAWrapperPass>().getMSSA();
- else {
- CurAST = std::make_unique<AliasSetTracker>(AA);
- computeAliasSet(*L, *Preheader, *CurAST.get());
- }
-
- bool Changed = sinkLoopInvariantInstructions(
- *L, AA, getAnalysis<LoopInfoWrapperPass>().getLoopInfo(),
+ std::unique_ptr<AliasSetTracker> CurAST;
+ MemorySSA *MSSA = nullptr;
+ if (EnableMSSAInLegacyLoopSink)
+ MSSA = &getAnalysis<MemorySSAWrapperPass>().getMSSA();
+ else {
+ CurAST = std::make_unique<AliasSetTracker>(AA);
+ computeAliasSet(*L, *Preheader, *CurAST.get());
+ }
+
+ bool Changed = sinkLoopInvariantInstructions(
+ *L, AA, getAnalysis<LoopInfoWrapperPass>().getLoopInfo(),
getAnalysis<DominatorTreeWrapperPass>().getDomTree(),
getAnalysis<BlockFrequencyInfoWrapperPass>().getBFI(),
- SE ? &SE->getSE() : nullptr, CurAST.get(), MSSA);
-
- if (MSSA && VerifyMemorySSA)
- MSSA->verifyMemorySSA();
-
- return Changed;
+ SE ? &SE->getSE() : nullptr, CurAST.get(), MSSA);
+
+ if (MSSA && VerifyMemorySSA)
+ MSSA->verifyMemorySSA();
+
+ return Changed;
}
void getAnalysisUsage(AnalysisUsage &AU) const override {
AU.setPreservesCFG();
AU.addRequired<BlockFrequencyInfoWrapperPass>();
getLoopAnalysisUsage(AU);
- if (EnableMSSAInLegacyLoopSink) {
- AU.addRequired<MemorySSAWrapperPass>();
- AU.addPreserved<MemorySSAWrapperPass>();
- }
+ if (EnableMSSAInLegacyLoopSink) {
+ AU.addRequired<MemorySSAWrapperPass>();
+ AU.addPreserved<MemorySSAWrapperPass>();
+ }
}
};
}
@@ -473,7 +473,7 @@ INITIALIZE_PASS_BEGIN(LegacyLoopSinkPass, "loop-sink", "Loop Sink", false,
false)
INITIALIZE_PASS_DEPENDENCY(LoopPass)
INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass)
-INITIALIZE_PASS_DEPENDENCY(MemorySSAWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(MemorySSAWrapperPass)
INITIALIZE_PASS_END(LegacyLoopSinkPass, "loop-sink", "Loop Sink", false, false)
Pass *llvm::createLoopSinkPass() { return new LegacyLoopSinkPass(); }