aboutsummaryrefslogtreecommitdiffstats
path: root/contrib/libs/llvm12/lib/CodeGen/InterleavedAccessPass.cpp
diff options
context:
space:
mode:
authorshadchin <shadchin@yandex-team.ru>2022-02-10 16:44:30 +0300
committerDaniil Cherednik <dcherednik@yandex-team.ru>2022-02-10 16:44:30 +0300
commit2598ef1d0aee359b4b6d5fdd1758916d5907d04f (patch)
tree012bb94d777798f1f56ac1cec429509766d05181 /contrib/libs/llvm12/lib/CodeGen/InterleavedAccessPass.cpp
parent6751af0b0c1b952fede40b19b71da8025b5d8bcf (diff)
downloadydb-2598ef1d0aee359b4b6d5fdd1758916d5907d04f.tar.gz
Restoring authorship annotation for <shadchin@yandex-team.ru>. Commit 1 of 2.
Diffstat (limited to 'contrib/libs/llvm12/lib/CodeGen/InterleavedAccessPass.cpp')
-rw-r--r--contrib/libs/llvm12/lib/CodeGen/InterleavedAccessPass.cpp194
1 files changed, 97 insertions, 97 deletions
diff --git a/contrib/libs/llvm12/lib/CodeGen/InterleavedAccessPass.cpp b/contrib/libs/llvm12/lib/CodeGen/InterleavedAccessPass.cpp
index b22e6faeb9..feb71e2221 100644
--- a/contrib/libs/llvm12/lib/CodeGen/InterleavedAccessPass.cpp
+++ b/contrib/libs/llvm12/lib/CodeGen/InterleavedAccessPass.cpp
@@ -22,8 +22,8 @@
//
// E.g. An interleaved load (Factor = 2):
// %wide.vec = load <8 x i32>, <8 x i32>* %ptr
-// %v0 = shuffle <8 x i32> %wide.vec, <8 x i32> poison, <0, 2, 4, 6>
-// %v1 = shuffle <8 x i32> %wide.vec, <8 x i32> poison, <1, 3, 5, 7>
+// %v0 = shuffle <8 x i32> %wide.vec, <8 x i32> poison, <0, 2, 4, 6>
+// %v1 = shuffle <8 x i32> %wide.vec, <8 x i32> poison, <1, 3, 5, 7>
//
// It could be transformed into a ld2 intrinsic in AArch64 backend or a vld2
// intrinsic in ARM backend.
@@ -66,7 +66,7 @@
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetMachine.h"
-#include "llvm/Transforms/Utils/Local.h"
+#include "llvm/Transforms/Utils/Local.h"
#include <cassert>
#include <utility>
@@ -119,15 +119,15 @@ private:
/// replacements are also performed.
bool tryReplaceExtracts(ArrayRef<ExtractElementInst *> Extracts,
ArrayRef<ShuffleVectorInst *> Shuffles);
-
- /// Given a number of shuffles of the form shuffle(binop(x,y)), convert them
- /// to binop(shuffle(x), shuffle(y)) to allow the formation of an
- /// interleaving load. Any newly created shuffles that operate on \p LI will
- /// be added to \p Shuffles. Returns true, if any changes to the IR have been
- /// made.
- bool replaceBinOpShuffles(ArrayRef<ShuffleVectorInst *> BinOpShuffles,
- SmallVectorImpl<ShuffleVectorInst *> &Shuffles,
- LoadInst *LI);
+
+ /// Given a number of shuffles of the form shuffle(binop(x,y)), convert them
+ /// to binop(shuffle(x), shuffle(y)) to allow the formation of an
+ /// interleaving load. Any newly created shuffles that operate on \p LI will
+ /// be added to \p Shuffles. Returns true, if any changes to the IR have been
+ /// made.
+ bool replaceBinOpShuffles(ArrayRef<ShuffleVectorInst *> BinOpShuffles,
+ SmallVectorImpl<ShuffleVectorInst *> &Shuffles,
+ LoadInst *LI);
};
} // end anonymous namespace.
@@ -293,97 +293,97 @@ bool InterleavedAccess::lowerInterleavedLoad(
if (!LI->isSimple() || isa<ScalableVectorType>(LI->getType()))
return false;
- // Check if all users of this load are shufflevectors. If we encounter any
- // users that are extractelement instructions or binary operators, we save
- // them to later check if they can be modified to extract from one of the
- // shufflevectors instead of the load.
-
+ // Check if all users of this load are shufflevectors. If we encounter any
+ // users that are extractelement instructions or binary operators, we save
+ // them to later check if they can be modified to extract from one of the
+ // shufflevectors instead of the load.
+
SmallVector<ShuffleVectorInst *, 4> Shuffles;
SmallVector<ExtractElementInst *, 4> Extracts;
- // BinOpShuffles need to be handled a single time in case both operands of the
- // binop are the same load.
- SmallSetVector<ShuffleVectorInst *, 4> BinOpShuffles;
+ // BinOpShuffles need to be handled a single time in case both operands of the
+ // binop are the same load.
+ SmallSetVector<ShuffleVectorInst *, 4> BinOpShuffles;
- for (auto *User : LI->users()) {
- auto *Extract = dyn_cast<ExtractElementInst>(User);
+ for (auto *User : LI->users()) {
+ auto *Extract = dyn_cast<ExtractElementInst>(User);
if (Extract && isa<ConstantInt>(Extract->getIndexOperand())) {
Extracts.push_back(Extract);
continue;
}
- auto *BI = dyn_cast<BinaryOperator>(User);
- if (BI && BI->hasOneUse()) {
- if (auto *SVI = dyn_cast<ShuffleVectorInst>(*BI->user_begin())) {
- BinOpShuffles.insert(SVI);
- continue;
- }
- }
- auto *SVI = dyn_cast<ShuffleVectorInst>(User);
+ auto *BI = dyn_cast<BinaryOperator>(User);
+ if (BI && BI->hasOneUse()) {
+ if (auto *SVI = dyn_cast<ShuffleVectorInst>(*BI->user_begin())) {
+ BinOpShuffles.insert(SVI);
+ continue;
+ }
+ }
+ auto *SVI = dyn_cast<ShuffleVectorInst>(User);
if (!SVI || !isa<UndefValue>(SVI->getOperand(1)))
return false;
Shuffles.push_back(SVI);
}
- if (Shuffles.empty() && BinOpShuffles.empty())
+ if (Shuffles.empty() && BinOpShuffles.empty())
return false;
unsigned Factor, Index;
unsigned NumLoadElements =
cast<FixedVectorType>(LI->getType())->getNumElements();
- auto *FirstSVI = Shuffles.size() > 0 ? Shuffles[0] : BinOpShuffles[0];
+ auto *FirstSVI = Shuffles.size() > 0 ? Shuffles[0] : BinOpShuffles[0];
// Check if the first shufflevector is DE-interleave shuffle.
- if (!isDeInterleaveMask(FirstSVI->getShuffleMask(), Factor, Index, MaxFactor,
- NumLoadElements))
+ if (!isDeInterleaveMask(FirstSVI->getShuffleMask(), Factor, Index, MaxFactor,
+ NumLoadElements))
return false;
// Holds the corresponding index for each DE-interleave shuffle.
SmallVector<unsigned, 4> Indices;
- Type *VecTy = FirstSVI->getType();
+ Type *VecTy = FirstSVI->getType();
// Check if other shufflevectors are also DE-interleaved of the same type
// and factor as the first shufflevector.
- for (auto *Shuffle : Shuffles) {
- if (Shuffle->getType() != VecTy)
+ for (auto *Shuffle : Shuffles) {
+ if (Shuffle->getType() != VecTy)
return false;
- if (!isDeInterleaveMaskOfFactor(Shuffle->getShuffleMask(), Factor,
+ if (!isDeInterleaveMaskOfFactor(Shuffle->getShuffleMask(), Factor,
Index))
return false;
- assert(Shuffle->getShuffleMask().size() <= NumLoadElements);
+ assert(Shuffle->getShuffleMask().size() <= NumLoadElements);
Indices.push_back(Index);
}
- for (auto *Shuffle : BinOpShuffles) {
- if (Shuffle->getType() != VecTy)
- return false;
- if (!isDeInterleaveMaskOfFactor(Shuffle->getShuffleMask(), Factor,
- Index))
- return false;
-
- assert(Shuffle->getShuffleMask().size() <= NumLoadElements);
-
- if (cast<Instruction>(Shuffle->getOperand(0))->getOperand(0) == LI)
- Indices.push_back(Index);
- if (cast<Instruction>(Shuffle->getOperand(0))->getOperand(1) == LI)
- Indices.push_back(Index);
- }
-
+ for (auto *Shuffle : BinOpShuffles) {
+ if (Shuffle->getType() != VecTy)
+ return false;
+ if (!isDeInterleaveMaskOfFactor(Shuffle->getShuffleMask(), Factor,
+ Index))
+ return false;
+
+ assert(Shuffle->getShuffleMask().size() <= NumLoadElements);
+
+ if (cast<Instruction>(Shuffle->getOperand(0))->getOperand(0) == LI)
+ Indices.push_back(Index);
+ if (cast<Instruction>(Shuffle->getOperand(0))->getOperand(1) == LI)
+ Indices.push_back(Index);
+ }
+
// Try and modify users of the load that are extractelement instructions to
// use the shufflevector instructions instead of the load.
if (!tryReplaceExtracts(Extracts, Shuffles))
return false;
- bool BinOpShuffleChanged =
- replaceBinOpShuffles(BinOpShuffles.getArrayRef(), Shuffles, LI);
-
+ bool BinOpShuffleChanged =
+ replaceBinOpShuffles(BinOpShuffles.getArrayRef(), Shuffles, LI);
+
LLVM_DEBUG(dbgs() << "IA: Found an interleaved load: " << *LI << "\n");
// Try to create target specific intrinsics to replace the load and shuffles.
- if (!TLI->lowerInterleavedLoad(LI, Shuffles, Indices, Factor)) {
- // If Extracts is not empty, tryReplaceExtracts made changes earlier.
- return !Extracts.empty() || BinOpShuffleChanged;
- }
+ if (!TLI->lowerInterleavedLoad(LI, Shuffles, Indices, Factor)) {
+ // If Extracts is not empty, tryReplaceExtracts made changes earlier.
+ return !Extracts.empty() || BinOpShuffleChanged;
+ }
for (auto SVI : Shuffles)
DeadInsts.push_back(SVI);
@@ -392,39 +392,39 @@ bool InterleavedAccess::lowerInterleavedLoad(
return true;
}
-bool InterleavedAccess::replaceBinOpShuffles(
- ArrayRef<ShuffleVectorInst *> BinOpShuffles,
- SmallVectorImpl<ShuffleVectorInst *> &Shuffles, LoadInst *LI) {
- for (auto *SVI : BinOpShuffles) {
- BinaryOperator *BI = cast<BinaryOperator>(SVI->getOperand(0));
- Type *BIOp0Ty = BI->getOperand(0)->getType();
- ArrayRef<int> Mask = SVI->getShuffleMask();
- assert(all_of(Mask, [&](int Idx) {
- return Idx < (int)cast<FixedVectorType>(BIOp0Ty)->getNumElements();
- }));
-
- auto *NewSVI1 =
- new ShuffleVectorInst(BI->getOperand(0), PoisonValue::get(BIOp0Ty),
- Mask, SVI->getName(), SVI);
- auto *NewSVI2 = new ShuffleVectorInst(
- BI->getOperand(1), PoisonValue::get(BI->getOperand(1)->getType()), Mask,
- SVI->getName(), SVI);
- Value *NewBI = BinaryOperator::Create(BI->getOpcode(), NewSVI1, NewSVI2,
- BI->getName(), SVI);
- SVI->replaceAllUsesWith(NewBI);
- LLVM_DEBUG(dbgs() << " Replaced: " << *BI << "\n And : " << *SVI
- << "\n With : " << *NewSVI1 << "\n And : "
- << *NewSVI2 << "\n And : " << *NewBI << "\n");
- RecursivelyDeleteTriviallyDeadInstructions(SVI);
- if (NewSVI1->getOperand(0) == LI)
- Shuffles.push_back(NewSVI1);
- if (NewSVI2->getOperand(0) == LI)
- Shuffles.push_back(NewSVI2);
- }
-
- return !BinOpShuffles.empty();
-}
-
+bool InterleavedAccess::replaceBinOpShuffles(
+ ArrayRef<ShuffleVectorInst *> BinOpShuffles,
+ SmallVectorImpl<ShuffleVectorInst *> &Shuffles, LoadInst *LI) {
+ for (auto *SVI : BinOpShuffles) {
+ BinaryOperator *BI = cast<BinaryOperator>(SVI->getOperand(0));
+ Type *BIOp0Ty = BI->getOperand(0)->getType();
+ ArrayRef<int> Mask = SVI->getShuffleMask();
+ assert(all_of(Mask, [&](int Idx) {
+ return Idx < (int)cast<FixedVectorType>(BIOp0Ty)->getNumElements();
+ }));
+
+ auto *NewSVI1 =
+ new ShuffleVectorInst(BI->getOperand(0), PoisonValue::get(BIOp0Ty),
+ Mask, SVI->getName(), SVI);
+ auto *NewSVI2 = new ShuffleVectorInst(
+ BI->getOperand(1), PoisonValue::get(BI->getOperand(1)->getType()), Mask,
+ SVI->getName(), SVI);
+ Value *NewBI = BinaryOperator::Create(BI->getOpcode(), NewSVI1, NewSVI2,
+ BI->getName(), SVI);
+ SVI->replaceAllUsesWith(NewBI);
+ LLVM_DEBUG(dbgs() << " Replaced: " << *BI << "\n And : " << *SVI
+ << "\n With : " << *NewSVI1 << "\n And : "
+ << *NewSVI2 << "\n And : " << *NewBI << "\n");
+ RecursivelyDeleteTriviallyDeadInstructions(SVI);
+ if (NewSVI1->getOperand(0) == LI)
+ Shuffles.push_back(NewSVI1);
+ if (NewSVI2->getOperand(0) == LI)
+ Shuffles.push_back(NewSVI2);
+ }
+
+ return !BinOpShuffles.empty();
+}
+
bool InterleavedAccess::tryReplaceExtracts(
ArrayRef<ExtractElementInst *> Extracts,
ArrayRef<ShuffleVectorInst *> Shuffles) {
@@ -494,7 +494,7 @@ bool InterleavedAccess::lowerInterleavedStore(
if (!SI->isSimple())
return false;
- auto *SVI = dyn_cast<ShuffleVectorInst>(SI->getValueOperand());
+ auto *SVI = dyn_cast<ShuffleVectorInst>(SI->getValueOperand());
if (!SVI || !SVI->hasOneUse() || isa<ScalableVectorType>(SVI->getType()))
return false;
@@ -534,10 +534,10 @@ bool InterleavedAccess::runOnFunction(Function &F) {
bool Changed = false;
for (auto &I : instructions(F)) {
- if (auto *LI = dyn_cast<LoadInst>(&I))
+ if (auto *LI = dyn_cast<LoadInst>(&I))
Changed |= lowerInterleavedLoad(LI, DeadInsts);
- if (auto *SI = dyn_cast<StoreInst>(&I))
+ if (auto *SI = dyn_cast<StoreInst>(&I))
Changed |= lowerInterleavedStore(SI, DeadInsts);
}