diff options
author | shadchin <shadchin@yandex-team.ru> | 2022-02-10 16:44:30 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:44:30 +0300 |
commit | 2598ef1d0aee359b4b6d5fdd1758916d5907d04f (patch) | |
tree | 012bb94d777798f1f56ac1cec429509766d05181 /contrib/libs/llvm12/lib/CodeGen/InterleavedAccessPass.cpp | |
parent | 6751af0b0c1b952fede40b19b71da8025b5d8bcf (diff) | |
download | ydb-2598ef1d0aee359b4b6d5fdd1758916d5907d04f.tar.gz |
Restoring authorship annotation for <shadchin@yandex-team.ru>. Commit 1 of 2.
Diffstat (limited to 'contrib/libs/llvm12/lib/CodeGen/InterleavedAccessPass.cpp')
-rw-r--r-- | contrib/libs/llvm12/lib/CodeGen/InterleavedAccessPass.cpp | 194 |
1 files changed, 97 insertions, 97 deletions
diff --git a/contrib/libs/llvm12/lib/CodeGen/InterleavedAccessPass.cpp b/contrib/libs/llvm12/lib/CodeGen/InterleavedAccessPass.cpp index b22e6faeb9..feb71e2221 100644 --- a/contrib/libs/llvm12/lib/CodeGen/InterleavedAccessPass.cpp +++ b/contrib/libs/llvm12/lib/CodeGen/InterleavedAccessPass.cpp @@ -22,8 +22,8 @@ // // E.g. An interleaved load (Factor = 2): // %wide.vec = load <8 x i32>, <8 x i32>* %ptr -// %v0 = shuffle <8 x i32> %wide.vec, <8 x i32> poison, <0, 2, 4, 6> -// %v1 = shuffle <8 x i32> %wide.vec, <8 x i32> poison, <1, 3, 5, 7> +// %v0 = shuffle <8 x i32> %wide.vec, <8 x i32> poison, <0, 2, 4, 6> +// %v1 = shuffle <8 x i32> %wide.vec, <8 x i32> poison, <1, 3, 5, 7> // // It could be transformed into a ld2 intrinsic in AArch64 backend or a vld2 // intrinsic in ARM backend. @@ -66,7 +66,7 @@ #include "llvm/Support/MathExtras.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetMachine.h" -#include "llvm/Transforms/Utils/Local.h" +#include "llvm/Transforms/Utils/Local.h" #include <cassert> #include <utility> @@ -119,15 +119,15 @@ private: /// replacements are also performed. bool tryReplaceExtracts(ArrayRef<ExtractElementInst *> Extracts, ArrayRef<ShuffleVectorInst *> Shuffles); - - /// Given a number of shuffles of the form shuffle(binop(x,y)), convert them - /// to binop(shuffle(x), shuffle(y)) to allow the formation of an - /// interleaving load. Any newly created shuffles that operate on \p LI will - /// be added to \p Shuffles. Returns true, if any changes to the IR have been - /// made. - bool replaceBinOpShuffles(ArrayRef<ShuffleVectorInst *> BinOpShuffles, - SmallVectorImpl<ShuffleVectorInst *> &Shuffles, - LoadInst *LI); + + /// Given a number of shuffles of the form shuffle(binop(x,y)), convert them + /// to binop(shuffle(x), shuffle(y)) to allow the formation of an + /// interleaving load. Any newly created shuffles that operate on \p LI will + /// be added to \p Shuffles. Returns true, if any changes to the IR have been + /// made. + bool replaceBinOpShuffles(ArrayRef<ShuffleVectorInst *> BinOpShuffles, + SmallVectorImpl<ShuffleVectorInst *> &Shuffles, + LoadInst *LI); }; } // end anonymous namespace. @@ -293,97 +293,97 @@ bool InterleavedAccess::lowerInterleavedLoad( if (!LI->isSimple() || isa<ScalableVectorType>(LI->getType())) return false; - // Check if all users of this load are shufflevectors. If we encounter any - // users that are extractelement instructions or binary operators, we save - // them to later check if they can be modified to extract from one of the - // shufflevectors instead of the load. - + // Check if all users of this load are shufflevectors. If we encounter any + // users that are extractelement instructions or binary operators, we save + // them to later check if they can be modified to extract from one of the + // shufflevectors instead of the load. + SmallVector<ShuffleVectorInst *, 4> Shuffles; SmallVector<ExtractElementInst *, 4> Extracts; - // BinOpShuffles need to be handled a single time in case both operands of the - // binop are the same load. - SmallSetVector<ShuffleVectorInst *, 4> BinOpShuffles; + // BinOpShuffles need to be handled a single time in case both operands of the + // binop are the same load. + SmallSetVector<ShuffleVectorInst *, 4> BinOpShuffles; - for (auto *User : LI->users()) { - auto *Extract = dyn_cast<ExtractElementInst>(User); + for (auto *User : LI->users()) { + auto *Extract = dyn_cast<ExtractElementInst>(User); if (Extract && isa<ConstantInt>(Extract->getIndexOperand())) { Extracts.push_back(Extract); continue; } - auto *BI = dyn_cast<BinaryOperator>(User); - if (BI && BI->hasOneUse()) { - if (auto *SVI = dyn_cast<ShuffleVectorInst>(*BI->user_begin())) { - BinOpShuffles.insert(SVI); - continue; - } - } - auto *SVI = dyn_cast<ShuffleVectorInst>(User); + auto *BI = dyn_cast<BinaryOperator>(User); + if (BI && BI->hasOneUse()) { + if (auto *SVI = dyn_cast<ShuffleVectorInst>(*BI->user_begin())) { + BinOpShuffles.insert(SVI); + continue; + } + } + auto *SVI = dyn_cast<ShuffleVectorInst>(User); if (!SVI || !isa<UndefValue>(SVI->getOperand(1))) return false; Shuffles.push_back(SVI); } - if (Shuffles.empty() && BinOpShuffles.empty()) + if (Shuffles.empty() && BinOpShuffles.empty()) return false; unsigned Factor, Index; unsigned NumLoadElements = cast<FixedVectorType>(LI->getType())->getNumElements(); - auto *FirstSVI = Shuffles.size() > 0 ? Shuffles[0] : BinOpShuffles[0]; + auto *FirstSVI = Shuffles.size() > 0 ? Shuffles[0] : BinOpShuffles[0]; // Check if the first shufflevector is DE-interleave shuffle. - if (!isDeInterleaveMask(FirstSVI->getShuffleMask(), Factor, Index, MaxFactor, - NumLoadElements)) + if (!isDeInterleaveMask(FirstSVI->getShuffleMask(), Factor, Index, MaxFactor, + NumLoadElements)) return false; // Holds the corresponding index for each DE-interleave shuffle. SmallVector<unsigned, 4> Indices; - Type *VecTy = FirstSVI->getType(); + Type *VecTy = FirstSVI->getType(); // Check if other shufflevectors are also DE-interleaved of the same type // and factor as the first shufflevector. - for (auto *Shuffle : Shuffles) { - if (Shuffle->getType() != VecTy) + for (auto *Shuffle : Shuffles) { + if (Shuffle->getType() != VecTy) return false; - if (!isDeInterleaveMaskOfFactor(Shuffle->getShuffleMask(), Factor, + if (!isDeInterleaveMaskOfFactor(Shuffle->getShuffleMask(), Factor, Index)) return false; - assert(Shuffle->getShuffleMask().size() <= NumLoadElements); + assert(Shuffle->getShuffleMask().size() <= NumLoadElements); Indices.push_back(Index); } - for (auto *Shuffle : BinOpShuffles) { - if (Shuffle->getType() != VecTy) - return false; - if (!isDeInterleaveMaskOfFactor(Shuffle->getShuffleMask(), Factor, - Index)) - return false; - - assert(Shuffle->getShuffleMask().size() <= NumLoadElements); - - if (cast<Instruction>(Shuffle->getOperand(0))->getOperand(0) == LI) - Indices.push_back(Index); - if (cast<Instruction>(Shuffle->getOperand(0))->getOperand(1) == LI) - Indices.push_back(Index); - } - + for (auto *Shuffle : BinOpShuffles) { + if (Shuffle->getType() != VecTy) + return false; + if (!isDeInterleaveMaskOfFactor(Shuffle->getShuffleMask(), Factor, + Index)) + return false; + + assert(Shuffle->getShuffleMask().size() <= NumLoadElements); + + if (cast<Instruction>(Shuffle->getOperand(0))->getOperand(0) == LI) + Indices.push_back(Index); + if (cast<Instruction>(Shuffle->getOperand(0))->getOperand(1) == LI) + Indices.push_back(Index); + } + // Try and modify users of the load that are extractelement instructions to // use the shufflevector instructions instead of the load. if (!tryReplaceExtracts(Extracts, Shuffles)) return false; - bool BinOpShuffleChanged = - replaceBinOpShuffles(BinOpShuffles.getArrayRef(), Shuffles, LI); - + bool BinOpShuffleChanged = + replaceBinOpShuffles(BinOpShuffles.getArrayRef(), Shuffles, LI); + LLVM_DEBUG(dbgs() << "IA: Found an interleaved load: " << *LI << "\n"); // Try to create target specific intrinsics to replace the load and shuffles. - if (!TLI->lowerInterleavedLoad(LI, Shuffles, Indices, Factor)) { - // If Extracts is not empty, tryReplaceExtracts made changes earlier. - return !Extracts.empty() || BinOpShuffleChanged; - } + if (!TLI->lowerInterleavedLoad(LI, Shuffles, Indices, Factor)) { + // If Extracts is not empty, tryReplaceExtracts made changes earlier. + return !Extracts.empty() || BinOpShuffleChanged; + } for (auto SVI : Shuffles) DeadInsts.push_back(SVI); @@ -392,39 +392,39 @@ bool InterleavedAccess::lowerInterleavedLoad( return true; } -bool InterleavedAccess::replaceBinOpShuffles( - ArrayRef<ShuffleVectorInst *> BinOpShuffles, - SmallVectorImpl<ShuffleVectorInst *> &Shuffles, LoadInst *LI) { - for (auto *SVI : BinOpShuffles) { - BinaryOperator *BI = cast<BinaryOperator>(SVI->getOperand(0)); - Type *BIOp0Ty = BI->getOperand(0)->getType(); - ArrayRef<int> Mask = SVI->getShuffleMask(); - assert(all_of(Mask, [&](int Idx) { - return Idx < (int)cast<FixedVectorType>(BIOp0Ty)->getNumElements(); - })); - - auto *NewSVI1 = - new ShuffleVectorInst(BI->getOperand(0), PoisonValue::get(BIOp0Ty), - Mask, SVI->getName(), SVI); - auto *NewSVI2 = new ShuffleVectorInst( - BI->getOperand(1), PoisonValue::get(BI->getOperand(1)->getType()), Mask, - SVI->getName(), SVI); - Value *NewBI = BinaryOperator::Create(BI->getOpcode(), NewSVI1, NewSVI2, - BI->getName(), SVI); - SVI->replaceAllUsesWith(NewBI); - LLVM_DEBUG(dbgs() << " Replaced: " << *BI << "\n And : " << *SVI - << "\n With : " << *NewSVI1 << "\n And : " - << *NewSVI2 << "\n And : " << *NewBI << "\n"); - RecursivelyDeleteTriviallyDeadInstructions(SVI); - if (NewSVI1->getOperand(0) == LI) - Shuffles.push_back(NewSVI1); - if (NewSVI2->getOperand(0) == LI) - Shuffles.push_back(NewSVI2); - } - - return !BinOpShuffles.empty(); -} - +bool InterleavedAccess::replaceBinOpShuffles( + ArrayRef<ShuffleVectorInst *> BinOpShuffles, + SmallVectorImpl<ShuffleVectorInst *> &Shuffles, LoadInst *LI) { + for (auto *SVI : BinOpShuffles) { + BinaryOperator *BI = cast<BinaryOperator>(SVI->getOperand(0)); + Type *BIOp0Ty = BI->getOperand(0)->getType(); + ArrayRef<int> Mask = SVI->getShuffleMask(); + assert(all_of(Mask, [&](int Idx) { + return Idx < (int)cast<FixedVectorType>(BIOp0Ty)->getNumElements(); + })); + + auto *NewSVI1 = + new ShuffleVectorInst(BI->getOperand(0), PoisonValue::get(BIOp0Ty), + Mask, SVI->getName(), SVI); + auto *NewSVI2 = new ShuffleVectorInst( + BI->getOperand(1), PoisonValue::get(BI->getOperand(1)->getType()), Mask, + SVI->getName(), SVI); + Value *NewBI = BinaryOperator::Create(BI->getOpcode(), NewSVI1, NewSVI2, + BI->getName(), SVI); + SVI->replaceAllUsesWith(NewBI); + LLVM_DEBUG(dbgs() << " Replaced: " << *BI << "\n And : " << *SVI + << "\n With : " << *NewSVI1 << "\n And : " + << *NewSVI2 << "\n And : " << *NewBI << "\n"); + RecursivelyDeleteTriviallyDeadInstructions(SVI); + if (NewSVI1->getOperand(0) == LI) + Shuffles.push_back(NewSVI1); + if (NewSVI2->getOperand(0) == LI) + Shuffles.push_back(NewSVI2); + } + + return !BinOpShuffles.empty(); +} + bool InterleavedAccess::tryReplaceExtracts( ArrayRef<ExtractElementInst *> Extracts, ArrayRef<ShuffleVectorInst *> Shuffles) { @@ -494,7 +494,7 @@ bool InterleavedAccess::lowerInterleavedStore( if (!SI->isSimple()) return false; - auto *SVI = dyn_cast<ShuffleVectorInst>(SI->getValueOperand()); + auto *SVI = dyn_cast<ShuffleVectorInst>(SI->getValueOperand()); if (!SVI || !SVI->hasOneUse() || isa<ScalableVectorType>(SVI->getType())) return false; @@ -534,10 +534,10 @@ bool InterleavedAccess::runOnFunction(Function &F) { bool Changed = false; for (auto &I : instructions(F)) { - if (auto *LI = dyn_cast<LoadInst>(&I)) + if (auto *LI = dyn_cast<LoadInst>(&I)) Changed |= lowerInterleavedLoad(LI, DeadInsts); - if (auto *SI = dyn_cast<StoreInst>(&I)) + if (auto *SI = dyn_cast<StoreInst>(&I)) Changed |= lowerInterleavedStore(SI, DeadInsts); } |