diff options
author | shadchin <shadchin@yandex-team.ru> | 2022-02-10 16:44:39 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:44:39 +0300 |
commit | e9656aae26e0358d5378e5b63dcac5c8dbe0e4d0 (patch) | |
tree | 64175d5cadab313b3e7039ebaa06c5bc3295e274 /contrib/libs/llvm12/lib/Transforms/Scalar/LoopSink.cpp | |
parent | 2598ef1d0aee359b4b6d5fdd1758916d5907d04f (diff) | |
download | ydb-e9656aae26e0358d5378e5b63dcac5c8dbe0e4d0.tar.gz |
Restoring authorship annotation for <shadchin@yandex-team.ru>. Commit 2 of 2.
Diffstat (limited to 'contrib/libs/llvm12/lib/Transforms/Scalar/LoopSink.cpp')
-rw-r--r-- | contrib/libs/llvm12/lib/Transforms/Scalar/LoopSink.cpp | 242 |
1 files changed, 121 insertions, 121 deletions
diff --git a/contrib/libs/llvm12/lib/Transforms/Scalar/LoopSink.cpp b/contrib/libs/llvm12/lib/Transforms/Scalar/LoopSink.cpp index 0296b12878..47698fdde6 100644 --- a/contrib/libs/llvm12/lib/Transforms/Scalar/LoopSink.cpp +++ b/contrib/libs/llvm12/lib/Transforms/Scalar/LoopSink.cpp @@ -39,8 +39,8 @@ #include "llvm/Analysis/Loads.h" #include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/LoopPass.h" -#include "llvm/Analysis/MemorySSA.h" -#include "llvm/Analysis/MemorySSAUpdater.h" +#include "llvm/Analysis/MemorySSA.h" +#include "llvm/Analysis/MemorySSAUpdater.h" #include "llvm/Analysis/ScalarEvolution.h" #include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h" #include "llvm/IR/Dominators.h" @@ -69,14 +69,14 @@ static cl::opt<unsigned> MaxNumberOfUseBBsForSinking( "max-uses-for-sinking", cl::Hidden, cl::init(30), cl::desc("Do not sink instructions that have too many uses.")); -static cl::opt<bool> EnableMSSAInLoopSink( - "enable-mssa-in-loop-sink", cl::Hidden, cl::init(true), - cl::desc("Enable MemorySSA for LoopSink in new pass manager")); - -static cl::opt<bool> EnableMSSAInLegacyLoopSink( - "enable-mssa-in-legacy-loop-sink", cl::Hidden, cl::init(false), - cl::desc("Enable MemorySSA for LoopSink in legacy pass manager")); - +static cl::opt<bool> EnableMSSAInLoopSink( + "enable-mssa-in-loop-sink", cl::Hidden, cl::init(true), + cl::desc("Enable MemorySSA for LoopSink in new pass manager")); + +static cl::opt<bool> EnableMSSAInLegacyLoopSink( + "enable-mssa-in-legacy-loop-sink", cl::Hidden, cl::init(false), + cl::desc("Enable MemorySSA for LoopSink in legacy pass manager")); + /// Return adjusted total frequency of \p BBs. /// /// * If there is only one BB, sinking instruction will not introduce code @@ -182,10 +182,10 @@ findBBsToSinkInto(const Loop &L, const SmallPtrSetImpl<BasicBlock *> &UseBBs, // sinking is successful. // \p LoopBlockNumber is used to sort the insertion blocks to ensure // determinism. -static bool sinkInstruction( - Loop &L, Instruction &I, const SmallVectorImpl<BasicBlock *> &ColdLoopBBs, - const SmallDenseMap<BasicBlock *, int, 16> &LoopBlockNumber, LoopInfo &LI, - DominatorTree &DT, BlockFrequencyInfo &BFI, MemorySSAUpdater *MSSAU) { +static bool sinkInstruction( + Loop &L, Instruction &I, const SmallVectorImpl<BasicBlock *> &ColdLoopBBs, + const SmallDenseMap<BasicBlock *, int, 16> &LoopBlockNumber, LoopInfo &LI, + DominatorTree &DT, BlockFrequencyInfo &BFI, MemorySSAUpdater *MSSAU) { // Compute the set of blocks in loop L which contain a use of I. SmallPtrSet<BasicBlock *, 2> BBs; for (auto &U : I.uses()) { @@ -222,7 +222,7 @@ static bool sinkInstruction( // of the loop block numbers as iterating the set doesn't give a useful // order. No need to stable sort as the block numbers are a total ordering. SmallVector<BasicBlock *, 2> SortedBBsToSinkInto; - llvm::append_range(SortedBBsToSinkInto, BBsToSinkInto); + llvm::append_range(SortedBBsToSinkInto, BBsToSinkInto); llvm::sort(SortedBBsToSinkInto, [&](BasicBlock *A, BasicBlock *B) { return LoopBlockNumber.find(A)->second < LoopBlockNumber.find(B)->second; }); @@ -238,21 +238,21 @@ static bool sinkInstruction( Instruction *IC = I.clone(); IC->setName(I.getName()); IC->insertBefore(&*N->getFirstInsertionPt()); - - if (MSSAU && MSSAU->getMemorySSA()->getMemoryAccess(&I)) { - // Create a new MemoryAccess and let MemorySSA set its defining access. - MemoryAccess *NewMemAcc = - MSSAU->createMemoryAccessInBB(IC, nullptr, N, MemorySSA::Beginning); - if (NewMemAcc) { - if (auto *MemDef = dyn_cast<MemoryDef>(NewMemAcc)) - MSSAU->insertDef(MemDef, /*RenameUses=*/true); - else { - auto *MemUse = cast<MemoryUse>(NewMemAcc); - MSSAU->insertUse(MemUse, /*RenameUses=*/true); - } - } - } - + + if (MSSAU && MSSAU->getMemorySSA()->getMemoryAccess(&I)) { + // Create a new MemoryAccess and let MemorySSA set its defining access. + MemoryAccess *NewMemAcc = + MSSAU->createMemoryAccessInBB(IC, nullptr, N, MemorySSA::Beginning); + if (NewMemAcc) { + if (auto *MemDef = dyn_cast<MemoryDef>(NewMemAcc)) + MSSAU->insertDef(MemDef, /*RenameUses=*/true); + else { + auto *MemUse = cast<MemoryUse>(NewMemAcc); + MSSAU->insertUse(MemUse, /*RenameUses=*/true); + } + } + } + // Replaces uses of I with IC in N I.replaceUsesWithIf(IC, [N](Use &U) { return cast<Instruction>(U.getUser())->getParent() == N; @@ -267,11 +267,11 @@ static bool sinkInstruction( NumLoopSunk++; I.moveBefore(&*MoveBB->getFirstInsertionPt()); - if (MSSAU) - if (MemoryUseOrDef *OldMemAcc = cast_or_null<MemoryUseOrDef>( - MSSAU->getMemorySSA()->getMemoryAccess(&I))) - MSSAU->moveToPlace(OldMemAcc, MoveBB, MemorySSA::Beginning); - + if (MSSAU) + if (MemoryUseOrDef *OldMemAcc = cast_or_null<MemoryUseOrDef>( + MSSAU->getMemorySSA()->getMemoryAccess(&I))) + MSSAU->moveToPlace(OldMemAcc, MoveBB, MemorySSA::Beginning); + return true; } @@ -280,14 +280,14 @@ static bool sinkInstruction( static bool sinkLoopInvariantInstructions(Loop &L, AAResults &AA, LoopInfo &LI, DominatorTree &DT, BlockFrequencyInfo &BFI, - ScalarEvolution *SE, - AliasSetTracker *CurAST, - MemorySSA *MSSA) { + ScalarEvolution *SE, + AliasSetTracker *CurAST, + MemorySSA *MSSA) { BasicBlock *Preheader = L.getLoopPreheader(); - assert(Preheader && "Expected loop to have preheader"); + assert(Preheader && "Expected loop to have preheader"); - assert(Preheader->getParent()->hasProfileData() && - "Unexpected call when profile data unavailable."); + assert(Preheader->getParent()->hasProfileData() && + "Unexpected call when profile data unavailable."); const BlockFrequency PreheaderFreq = BFI.getBlockFreq(Preheader); // If there are no basic blocks with lower frequency than the preheader then @@ -298,14 +298,14 @@ static bool sinkLoopInvariantInstructions(Loop &L, AAResults &AA, LoopInfo &LI, })) return false; - std::unique_ptr<MemorySSAUpdater> MSSAU; - std::unique_ptr<SinkAndHoistLICMFlags> LICMFlags; - if (MSSA) { - MSSAU = std::make_unique<MemorySSAUpdater>(MSSA); - LICMFlags = - std::make_unique<SinkAndHoistLICMFlags>(/*IsSink=*/true, &L, MSSA); - } - + std::unique_ptr<MemorySSAUpdater> MSSAU; + std::unique_ptr<SinkAndHoistLICMFlags> LICMFlags; + if (MSSA) { + MSSAU = std::make_unique<MemorySSAUpdater>(MSSA); + LICMFlags = + std::make_unique<SinkAndHoistLICMFlags>(/*IsSink=*/true, &L, MSSA); + } + bool Changed = false; // Sort loop's basic blocks by frequency @@ -329,11 +329,11 @@ static bool sinkLoopInvariantInstructions(Loop &L, AAResults &AA, LoopInfo &LI, // No need to check for instruction's operands are loop invariant. assert(L.hasLoopInvariantOperands(I) && "Insts in a loop's preheader should have loop invariant operands!"); - if (!canSinkOrHoistInst(*I, &AA, &DT, &L, CurAST, MSSAU.get(), false, - LICMFlags.get())) + if (!canSinkOrHoistInst(*I, &AA, &DT, &L, CurAST, MSSAU.get(), false, + LICMFlags.get())) continue; - if (sinkInstruction(L, *I, ColdLoopBBs, LoopBlockNumber, LI, DT, BFI, - MSSAU.get())) + if (sinkInstruction(L, *I, ColdLoopBBs, LoopBlockNumber, LI, DT, BFI, + MSSAU.get())) Changed = true; } @@ -342,13 +342,13 @@ static bool sinkLoopInvariantInstructions(Loop &L, AAResults &AA, LoopInfo &LI, return Changed; } -static void computeAliasSet(Loop &L, BasicBlock &Preheader, - AliasSetTracker &CurAST) { - for (BasicBlock *BB : L.blocks()) - CurAST.add(*BB); - CurAST.add(Preheader); -} - +static void computeAliasSet(Loop &L, BasicBlock &Preheader, + AliasSetTracker &CurAST) { + for (BasicBlock *BB : L.blocks()) + CurAST.add(*BB); + CurAST.add(Preheader); +} + PreservedAnalyses LoopSinkPass::run(Function &F, FunctionAnalysisManager &FAM) { LoopInfo &LI = FAM.getResult<LoopAnalysis>(F); // Nothing to do if there are no loops. @@ -359,10 +359,10 @@ PreservedAnalyses LoopSinkPass::run(Function &F, FunctionAnalysisManager &FAM) { DominatorTree &DT = FAM.getResult<DominatorTreeAnalysis>(F); BlockFrequencyInfo &BFI = FAM.getResult<BlockFrequencyAnalysis>(F); - MemorySSA *MSSA = EnableMSSAInLoopSink - ? &FAM.getResult<MemorySSAAnalysis>(F).getMSSA() - : nullptr; - + MemorySSA *MSSA = EnableMSSAInLoopSink + ? &FAM.getResult<MemorySSAAnalysis>(F).getMSSA() + : nullptr; + // We want to do a postorder walk over the loops. Since loops are a tree this // is equivalent to a reversed preorder walk and preorder is easy to compute // without recursion. Since we reverse the preorder, we will visit siblings @@ -374,27 +374,27 @@ PreservedAnalyses LoopSinkPass::run(Function &F, FunctionAnalysisManager &FAM) { do { Loop &L = *PreorderLoops.pop_back_val(); - BasicBlock *Preheader = L.getLoopPreheader(); - if (!Preheader) - continue; - - // Enable LoopSink only when runtime profile is available. - // With static profile, the sinking decision may be sub-optimal. - if (!Preheader->getParent()->hasProfileData()) - continue; - - std::unique_ptr<AliasSetTracker> CurAST; - if (!EnableMSSAInLoopSink) { - CurAST = std::make_unique<AliasSetTracker>(AA); - computeAliasSet(L, *Preheader, *CurAST.get()); - } - + BasicBlock *Preheader = L.getLoopPreheader(); + if (!Preheader) + continue; + + // Enable LoopSink only when runtime profile is available. + // With static profile, the sinking decision may be sub-optimal. + if (!Preheader->getParent()->hasProfileData()) + continue; + + std::unique_ptr<AliasSetTracker> CurAST; + if (!EnableMSSAInLoopSink) { + CurAST = std::make_unique<AliasSetTracker>(AA); + computeAliasSet(L, *Preheader, *CurAST.get()); + } + // Note that we don't pass SCEV here because it is only used to invalidate // loops in SCEV and we don't preserve (or request) SCEV at all making that // unnecessary. Changed |= sinkLoopInvariantInstructions(L, AA, LI, DT, BFI, - /*ScalarEvolution*/ nullptr, - CurAST.get(), MSSA); + /*ScalarEvolution*/ nullptr, + CurAST.get(), MSSA); } while (!PreorderLoops.empty()); if (!Changed) @@ -402,14 +402,14 @@ PreservedAnalyses LoopSinkPass::run(Function &F, FunctionAnalysisManager &FAM) { PreservedAnalyses PA; PA.preserveSet<CFGAnalyses>(); - - if (MSSA) { - PA.preserve<MemorySSAAnalysis>(); - - if (VerifyMemorySSA) - MSSA->verifyMemorySSA(); - } - + + if (MSSA) { + PA.preserve<MemorySSAAnalysis>(); + + if (VerifyMemorySSA) + MSSA->verifyMemorySSA(); + } + return PA; } @@ -424,46 +424,46 @@ struct LegacyLoopSinkPass : public LoopPass { if (skipLoop(L)) return false; - BasicBlock *Preheader = L->getLoopPreheader(); - if (!Preheader) - return false; - - // Enable LoopSink only when runtime profile is available. - // With static profile, the sinking decision may be sub-optimal. - if (!Preheader->getParent()->hasProfileData()) - return false; - - AAResults &AA = getAnalysis<AAResultsWrapperPass>().getAAResults(); + BasicBlock *Preheader = L->getLoopPreheader(); + if (!Preheader) + return false; + + // Enable LoopSink only when runtime profile is available. + // With static profile, the sinking decision may be sub-optimal. + if (!Preheader->getParent()->hasProfileData()) + return false; + + AAResults &AA = getAnalysis<AAResultsWrapperPass>().getAAResults(); auto *SE = getAnalysisIfAvailable<ScalarEvolutionWrapperPass>(); - std::unique_ptr<AliasSetTracker> CurAST; - MemorySSA *MSSA = nullptr; - if (EnableMSSAInLegacyLoopSink) - MSSA = &getAnalysis<MemorySSAWrapperPass>().getMSSA(); - else { - CurAST = std::make_unique<AliasSetTracker>(AA); - computeAliasSet(*L, *Preheader, *CurAST.get()); - } - - bool Changed = sinkLoopInvariantInstructions( - *L, AA, getAnalysis<LoopInfoWrapperPass>().getLoopInfo(), + std::unique_ptr<AliasSetTracker> CurAST; + MemorySSA *MSSA = nullptr; + if (EnableMSSAInLegacyLoopSink) + MSSA = &getAnalysis<MemorySSAWrapperPass>().getMSSA(); + else { + CurAST = std::make_unique<AliasSetTracker>(AA); + computeAliasSet(*L, *Preheader, *CurAST.get()); + } + + bool Changed = sinkLoopInvariantInstructions( + *L, AA, getAnalysis<LoopInfoWrapperPass>().getLoopInfo(), getAnalysis<DominatorTreeWrapperPass>().getDomTree(), getAnalysis<BlockFrequencyInfoWrapperPass>().getBFI(), - SE ? &SE->getSE() : nullptr, CurAST.get(), MSSA); - - if (MSSA && VerifyMemorySSA) - MSSA->verifyMemorySSA(); - - return Changed; + SE ? &SE->getSE() : nullptr, CurAST.get(), MSSA); + + if (MSSA && VerifyMemorySSA) + MSSA->verifyMemorySSA(); + + return Changed; } void getAnalysisUsage(AnalysisUsage &AU) const override { AU.setPreservesCFG(); AU.addRequired<BlockFrequencyInfoWrapperPass>(); getLoopAnalysisUsage(AU); - if (EnableMSSAInLegacyLoopSink) { - AU.addRequired<MemorySSAWrapperPass>(); - AU.addPreserved<MemorySSAWrapperPass>(); - } + if (EnableMSSAInLegacyLoopSink) { + AU.addRequired<MemorySSAWrapperPass>(); + AU.addPreserved<MemorySSAWrapperPass>(); + } } }; } @@ -473,7 +473,7 @@ INITIALIZE_PASS_BEGIN(LegacyLoopSinkPass, "loop-sink", "Loop Sink", false, false) INITIALIZE_PASS_DEPENDENCY(LoopPass) INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass) -INITIALIZE_PASS_DEPENDENCY(MemorySSAWrapperPass) +INITIALIZE_PASS_DEPENDENCY(MemorySSAWrapperPass) INITIALIZE_PASS_END(LegacyLoopSinkPass, "loop-sink", "Loop Sink", false, false) Pass *llvm::createLoopSinkPass() { return new LegacyLoopSinkPass(); } |