diff options
author | shadchin <shadchin@yandex-team.ru> | 2022-02-10 16:44:30 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:44:30 +0300 |
commit | 2598ef1d0aee359b4b6d5fdd1758916d5907d04f (patch) | |
tree | 012bb94d777798f1f56ac1cec429509766d05181 /contrib/libs/llvm12/lib/CodeGen/SelectionDAG/SelectionDAG.cpp | |
parent | 6751af0b0c1b952fede40b19b71da8025b5d8bcf (diff) | |
download | ydb-2598ef1d0aee359b4b6d5fdd1758916d5907d04f.tar.gz |
Restoring authorship annotation for <shadchin@yandex-team.ru>. Commit 1 of 2.
Diffstat (limited to 'contrib/libs/llvm12/lib/CodeGen/SelectionDAG/SelectionDAG.cpp')
-rw-r--r-- | contrib/libs/llvm12/lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 1186 |
1 files changed, 593 insertions, 593 deletions
diff --git a/contrib/libs/llvm12/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/contrib/libs/llvm12/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 2090762e2f..b0f50ffafd 100644 --- a/contrib/libs/llvm12/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/contrib/libs/llvm12/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -28,7 +28,7 @@ #include "llvm/Analysis/MemoryLocation.h" #include "llvm/Analysis/ProfileSummaryInfo.h" #include "llvm/Analysis/ValueTracking.h" -#include "llvm/CodeGen/FunctionLoweringInfo.h" +#include "llvm/CodeGen/FunctionLoweringInfo.h" #include "llvm/CodeGen/ISDOpcodes.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineConstantPool.h" @@ -139,15 +139,15 @@ bool ConstantFPSDNode::isValueValidForType(EVT VT, //===----------------------------------------------------------------------===// bool ISD::isConstantSplatVector(const SDNode *N, APInt &SplatVal) { - if (N->getOpcode() == ISD::SPLAT_VECTOR) { - unsigned EltSize = - N->getValueType(0).getVectorElementType().getSizeInBits(); - if (auto *Op0 = dyn_cast<ConstantSDNode>(N->getOperand(0))) { - SplatVal = Op0->getAPIntValue().truncOrSelf(EltSize); - return true; - } - } - + if (N->getOpcode() == ISD::SPLAT_VECTOR) { + unsigned EltSize = + N->getValueType(0).getVectorElementType().getSizeInBits(); + if (auto *Op0 = dyn_cast<ConstantSDNode>(N->getOperand(0))) { + SplatVal = Op0->getAPIntValue().truncOrSelf(EltSize); + return true; + } + } + auto *BV = dyn_cast<BuildVectorSDNode>(N); if (!BV) return false; @@ -164,16 +164,16 @@ bool ISD::isConstantSplatVector(const SDNode *N, APInt &SplatVal) { // FIXME: AllOnes and AllZeros duplicate a lot of code. Could these be // specializations of the more general isConstantSplatVector()? -bool ISD::isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly) { +bool ISD::isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly) { // Look through a bit convert. while (N->getOpcode() == ISD::BITCAST) N = N->getOperand(0).getNode(); - if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) { - APInt SplatVal; - return isConstantSplatVector(N, SplatVal) && SplatVal.isAllOnesValue(); - } - + if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) { + APInt SplatVal; + return isConstantSplatVector(N, SplatVal) && SplatVal.isAllOnesValue(); + } + if (N->getOpcode() != ISD::BUILD_VECTOR) return false; unsigned i = 0, e = N->getNumOperands(); @@ -213,16 +213,16 @@ bool ISD::isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly) { return true; } -bool ISD::isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly) { +bool ISD::isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly) { // Look through a bit convert. while (N->getOpcode() == ISD::BITCAST) N = N->getOperand(0).getNode(); - if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) { - APInt SplatVal; - return isConstantSplatVector(N, SplatVal) && SplatVal.isNullValue(); - } - + if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) { + APInt SplatVal; + return isConstantSplatVector(N, SplatVal) && SplatVal.isNullValue(); + } + if (N->getOpcode() != ISD::BUILD_VECTOR) return false; bool IsAllUndef = true; @@ -255,14 +255,14 @@ bool ISD::isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly) { return true; } -bool ISD::isBuildVectorAllOnes(const SDNode *N) { - return isConstantSplatVectorAllOnes(N, /*BuildVectorOnly*/ true); -} - -bool ISD::isBuildVectorAllZeros(const SDNode *N) { - return isConstantSplatVectorAllZeros(N, /*BuildVectorOnly*/ true); -} - +bool ISD::isBuildVectorAllOnes(const SDNode *N) { + return isConstantSplatVectorAllOnes(N, /*BuildVectorOnly*/ true); +} + +bool ISD::isBuildVectorAllZeros(const SDNode *N) { + return isConstantSplatVectorAllZeros(N, /*BuildVectorOnly*/ true); +} + bool ISD::isBuildVectorOfConstantSDNodes(const SDNode *N) { if (N->getOpcode() != ISD::BUILD_VECTOR) return false; @@ -306,8 +306,8 @@ bool ISD::matchUnaryPredicate(SDValue Op, return Match(Cst); // FIXME: Add support for vector UNDEF cases? - if (ISD::BUILD_VECTOR != Op.getOpcode() && - ISD::SPLAT_VECTOR != Op.getOpcode()) + if (ISD::BUILD_VECTOR != Op.getOpcode() && + ISD::SPLAT_VECTOR != Op.getOpcode()) return false; EVT SVT = Op.getValueType().getScalarType(); @@ -361,76 +361,76 @@ bool ISD::matchBinaryPredicate( return true; } -ISD::NodeType ISD::getVecReduceBaseOpcode(unsigned VecReduceOpcode) { - switch (VecReduceOpcode) { - default: - llvm_unreachable("Expected VECREDUCE opcode"); - case ISD::VECREDUCE_FADD: - case ISD::VECREDUCE_SEQ_FADD: - return ISD::FADD; - case ISD::VECREDUCE_FMUL: - case ISD::VECREDUCE_SEQ_FMUL: - return ISD::FMUL; - case ISD::VECREDUCE_ADD: - return ISD::ADD; - case ISD::VECREDUCE_MUL: - return ISD::MUL; - case ISD::VECREDUCE_AND: - return ISD::AND; - case ISD::VECREDUCE_OR: - return ISD::OR; - case ISD::VECREDUCE_XOR: - return ISD::XOR; - case ISD::VECREDUCE_SMAX: - return ISD::SMAX; - case ISD::VECREDUCE_SMIN: - return ISD::SMIN; - case ISD::VECREDUCE_UMAX: - return ISD::UMAX; - case ISD::VECREDUCE_UMIN: - return ISD::UMIN; - case ISD::VECREDUCE_FMAX: - return ISD::FMAXNUM; - case ISD::VECREDUCE_FMIN: - return ISD::FMINNUM; - } -} - -bool ISD::isVPOpcode(unsigned Opcode) { - switch (Opcode) { - default: - return false; -#define BEGIN_REGISTER_VP_SDNODE(SDOPC, ...) \ - case ISD::SDOPC: \ - return true; -#include "llvm/IR/VPIntrinsics.def" - } -} - -/// The operand position of the vector mask. -Optional<unsigned> ISD::getVPMaskIdx(unsigned Opcode) { - switch (Opcode) { - default: - return None; -#define BEGIN_REGISTER_VP_SDNODE(SDOPC, LEGALPOS, TDNAME, MASKPOS, ...) \ - case ISD::SDOPC: \ - return MASKPOS; -#include "llvm/IR/VPIntrinsics.def" - } -} - -/// The operand position of the explicit vector length parameter. -Optional<unsigned> ISD::getVPExplicitVectorLengthIdx(unsigned Opcode) { - switch (Opcode) { - default: - return None; -#define BEGIN_REGISTER_VP_SDNODE(SDOPC, LEGALPOS, TDNAME, MASKPOS, EVLPOS) \ - case ISD::SDOPC: \ - return EVLPOS; -#include "llvm/IR/VPIntrinsics.def" - } -} - +ISD::NodeType ISD::getVecReduceBaseOpcode(unsigned VecReduceOpcode) { + switch (VecReduceOpcode) { + default: + llvm_unreachable("Expected VECREDUCE opcode"); + case ISD::VECREDUCE_FADD: + case ISD::VECREDUCE_SEQ_FADD: + return ISD::FADD; + case ISD::VECREDUCE_FMUL: + case ISD::VECREDUCE_SEQ_FMUL: + return ISD::FMUL; + case ISD::VECREDUCE_ADD: + return ISD::ADD; + case ISD::VECREDUCE_MUL: + return ISD::MUL; + case ISD::VECREDUCE_AND: + return ISD::AND; + case ISD::VECREDUCE_OR: + return ISD::OR; + case ISD::VECREDUCE_XOR: + return ISD::XOR; + case ISD::VECREDUCE_SMAX: + return ISD::SMAX; + case ISD::VECREDUCE_SMIN: + return ISD::SMIN; + case ISD::VECREDUCE_UMAX: + return ISD::UMAX; + case ISD::VECREDUCE_UMIN: + return ISD::UMIN; + case ISD::VECREDUCE_FMAX: + return ISD::FMAXNUM; + case ISD::VECREDUCE_FMIN: + return ISD::FMINNUM; + } +} + +bool ISD::isVPOpcode(unsigned Opcode) { + switch (Opcode) { + default: + return false; +#define BEGIN_REGISTER_VP_SDNODE(SDOPC, ...) \ + case ISD::SDOPC: \ + return true; +#include "llvm/IR/VPIntrinsics.def" + } +} + +/// The operand position of the vector mask. +Optional<unsigned> ISD::getVPMaskIdx(unsigned Opcode) { + switch (Opcode) { + default: + return None; +#define BEGIN_REGISTER_VP_SDNODE(SDOPC, LEGALPOS, TDNAME, MASKPOS, ...) \ + case ISD::SDOPC: \ + return MASKPOS; +#include "llvm/IR/VPIntrinsics.def" + } +} + +/// The operand position of the explicit vector length parameter. +Optional<unsigned> ISD::getVPExplicitVectorLengthIdx(unsigned Opcode) { + switch (Opcode) { + default: + return None; +#define BEGIN_REGISTER_VP_SDNODE(SDOPC, LEGALPOS, TDNAME, MASKPOS, EVLPOS) \ + case ISD::SDOPC: \ + return EVLPOS; +#include "llvm/IR/VPIntrinsics.def" + } +} + ISD::NodeType ISD::getExtForLoadExtType(bool IsFP, ISD::LoadExtType ExtType) { switch (ExtType) { case ISD::EXTLOAD: @@ -635,11 +635,11 @@ static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) { ID.AddInteger(cast<LifetimeSDNode>(N)->getOffset()); } break; - case ISD::PSEUDO_PROBE: - ID.AddInteger(cast<PseudoProbeSDNode>(N)->getGuid()); - ID.AddInteger(cast<PseudoProbeSDNode>(N)->getIndex()); - ID.AddInteger(cast<PseudoProbeSDNode>(N)->getAttributes()); - break; + case ISD::PSEUDO_PROBE: + ID.AddInteger(cast<PseudoProbeSDNode>(N)->getGuid()); + ID.AddInteger(cast<PseudoProbeSDNode>(N)->getIndex()); + ID.AddInteger(cast<PseudoProbeSDNode>(N)->getAttributes()); + break; case ISD::JumpTable: case ISD::TargetJumpTable: ID.AddInteger(cast<JumpTableSDNode>(N)->getIndex()); @@ -1333,7 +1333,7 @@ SDValue SelectionDAG::getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isT, bool isO) { EVT EltVT = VT.getScalarType(); assert((EltVT.getSizeInBits() >= 64 || - (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) && + (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) && "getConstant with a uint64_t value that doesn't fit in the type!"); return getConstant(APInt(EltVT.getSizeInBits(), Val), DL, VT, isT, isO); } @@ -1355,10 +1355,10 @@ SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL, // inserted value (the type does not need to match the vector element type). // Any extra bits introduced will be truncated away. if (VT.isVector() && TLI->getTypeAction(*getContext(), EltVT) == - TargetLowering::TypePromoteInteger) { - EltVT = TLI->getTypeToTransformTo(*getContext(), EltVT); - APInt NewVal = Elt->getValue().zextOrTrunc(EltVT.getSizeInBits()); - Elt = ConstantInt::get(*getContext(), NewVal); + TargetLowering::TypePromoteInteger) { + EltVT = TLI->getTypeToTransformTo(*getContext(), EltVT); + APInt NewVal = Elt->getValue().zextOrTrunc(EltVT.getSizeInBits()); + Elt = ConstantInt::get(*getContext(), NewVal); } // In other cases the element type is illegal and needs to be expanded, for // example v2i64 on MIPS32. In this case, find the nearest legal type, split @@ -1368,7 +1368,7 @@ SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL, // only legalize if the DAG tells us we must produce legal types. else if (NewNodesMustHaveLegalTypes && VT.isVector() && TLI->getTypeAction(*getContext(), EltVT) == - TargetLowering::TypeExpandInteger) { + TargetLowering::TypeExpandInteger) { const APInt &NewVal = Elt->getValue(); EVT ViaEltVT = TLI->getTypeToTransformTo(*getContext(), EltVT); unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits(); @@ -1382,9 +1382,9 @@ SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL, SmallVector<SDValue, 2> EltParts; for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i) { - EltParts.push_back(getConstant( - NewVal.lshr(i * ViaEltSizeInBits).zextOrTrunc(ViaEltSizeInBits), DL, - ViaEltVT, isT, isO)); + EltParts.push_back(getConstant( + NewVal.lshr(i * ViaEltSizeInBits).zextOrTrunc(ViaEltSizeInBits), DL, + ViaEltVT, isT, isO)); } // EltParts is currently in little endian order. If we actually want @@ -1401,10 +1401,10 @@ SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL, SmallVector<SDValue, 8> Ops; for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) - llvm::append_range(Ops, EltParts); + llvm::append_range(Ops, EltParts); - SDValue V = - getNode(ISD::BITCAST, DL, VT, getBuildVector(ViaVecVT, DL, Ops)); + SDValue V = + getNode(ISD::BITCAST, DL, VT, getBuildVector(ViaVecVT, DL, Ops)); return V; } @@ -1485,9 +1485,9 @@ SDValue SelectionDAG::getConstantFP(const ConstantFP &V, const SDLoc &DL, } SDValue Result(N, 0); - if (VT.isScalableVector()) - Result = getSplatVector(VT, DL, Result); - else if (VT.isVector()) + if (VT.isScalableVector()) + Result = getSplatVector(VT, DL, Result); + else if (VT.isVector()) Result = getSplatBuildVector(VT, DL, Result); NewSDValueDbgMsg(Result, "Creating fp constant: ", this); return Result; @@ -2130,14 +2130,14 @@ Align SelectionDAG::getReducedAlign(EVT VT, bool UseABI) { SDValue SelectionDAG::CreateStackTemporary(TypeSize Bytes, Align Alignment) { MachineFrameInfo &MFI = MF->getFrameInfo(); - const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering(); - int StackID = 0; - if (Bytes.isScalable()) - StackID = TFI->getStackIDForScalableVectors(); - // The stack id gives an indication of whether the object is scalable or - // not, so it's safe to pass in the minimum size here. - int FrameIdx = MFI.CreateStackObject(Bytes.getKnownMinSize(), Alignment, - false, nullptr, StackID); + const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering(); + int StackID = 0; + if (Bytes.isScalable()) + StackID = TFI->getStackIDForScalableVectors(); + // The stack id gives an indication of whether the object is scalable or + // not, so it's safe to pass in the minimum size here. + int FrameIdx = MFI.CreateStackObject(Bytes.getKnownMinSize(), Alignment, + false, nullptr, StackID); return getFrameIndex(FrameIdx, TLI->getFrameIndexTy(getDataLayout())); } @@ -2149,14 +2149,14 @@ SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) { } SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) { - TypeSize VT1Size = VT1.getStoreSize(); - TypeSize VT2Size = VT2.getStoreSize(); - assert(VT1Size.isScalable() == VT2Size.isScalable() && - "Don't know how to choose the maximum size when creating a stack " - "temporary"); - TypeSize Bytes = - VT1Size.getKnownMinSize() > VT2Size.getKnownMinSize() ? VT1Size : VT2Size; - + TypeSize VT1Size = VT1.getStoreSize(); + TypeSize VT2Size = VT2.getStoreSize(); + assert(VT1Size.isScalable() == VT2Size.isScalable() && + "Don't know how to choose the maximum size when creating a stack " + "temporary"); + TypeSize Bytes = + VT1Size.getKnownMinSize() > VT2Size.getKnownMinSize() ? VT1Size : VT2Size; + Type *Ty1 = VT1.getTypeForEVT(*getContext()); Type *Ty2 = VT2.getTypeForEVT(*getContext()); const DataLayout &DL = getDataLayout(); @@ -2325,10 +2325,10 @@ SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, SDValue N2, /// SimplifyMultipleUseDemandedBits and not generate any new nodes. SDValue SelectionDAG::GetDemandedBits(SDValue V, const APInt &DemandedBits) { EVT VT = V.getValueType(); - - if (VT.isScalableVector()) - return SDValue(); - + + if (VT.isScalableVector()) + return SDValue(); + APInt DemandedElts = VT.isVector() ? APInt::getAllOnesValue(VT.getVectorNumElements()) : APInt(1, 1); @@ -2410,23 +2410,23 @@ bool SelectionDAG::MaskedValueIsAllOnes(SDValue V, const APInt &Mask, /// sense to specify which elements are demanded or undefined, therefore /// they are simply ignored. bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts, - APInt &UndefElts, unsigned Depth) { + APInt &UndefElts, unsigned Depth) { EVT VT = V.getValueType(); assert(VT.isVector() && "Vector type expected"); if (!VT.isScalableVector() && !DemandedElts) return false; // No demanded elts, better to assume we don't know anything. - if (Depth >= MaxRecursionDepth) - return false; // Limit search depth. - + if (Depth >= MaxRecursionDepth) + return false; // Limit search depth. + // Deal with some common cases here that work for both fixed and scalable // vector types. switch (V.getOpcode()) { case ISD::SPLAT_VECTOR: - UndefElts = V.getOperand(0).isUndef() - ? APInt::getAllOnesValue(DemandedElts.getBitWidth()) - : APInt(DemandedElts.getBitWidth(), 0); + UndefElts = V.getOperand(0).isUndef() + ? APInt::getAllOnesValue(DemandedElts.getBitWidth()) + : APInt(DemandedElts.getBitWidth(), 0); return true; case ISD::ADD: case ISD::SUB: @@ -2434,17 +2434,17 @@ bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts, APInt UndefLHS, UndefRHS; SDValue LHS = V.getOperand(0); SDValue RHS = V.getOperand(1); - if (isSplatValue(LHS, DemandedElts, UndefLHS, Depth + 1) && - isSplatValue(RHS, DemandedElts, UndefRHS, Depth + 1)) { + if (isSplatValue(LHS, DemandedElts, UndefLHS, Depth + 1) && + isSplatValue(RHS, DemandedElts, UndefRHS, Depth + 1)) { UndefElts = UndefLHS | UndefRHS; return true; } break; } - case ISD::TRUNCATE: - case ISD::SIGN_EXTEND: - case ISD::ZERO_EXTEND: - return isSplatValue(V.getOperand(0), DemandedElts, UndefElts, Depth + 1); + case ISD::TRUNCATE: + case ISD::SIGN_EXTEND: + case ISD::ZERO_EXTEND: + return isSplatValue(V.getOperand(0), DemandedElts, UndefElts, Depth + 1); } // We don't support other cases than those above for scalable vectors at @@ -2499,7 +2499,7 @@ bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts, unsigned NumSrcElts = Src.getValueType().getVectorNumElements(); APInt UndefSrcElts; APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx); - if (isSplatValue(Src, DemandedSrcElts, UndefSrcElts, Depth + 1)) { + if (isSplatValue(Src, DemandedSrcElts, UndefSrcElts, Depth + 1)) { UndefElts = UndefSrcElts.extractBits(NumElts, Idx); return true; } @@ -2696,11 +2696,11 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, if (auto *C = dyn_cast<ConstantSDNode>(Op)) { // We know all of the bits for a constant! - return KnownBits::makeConstant(C->getAPIntValue()); + return KnownBits::makeConstant(C->getAPIntValue()); } if (auto *C = dyn_cast<ConstantFPSDNode>(Op)) { // We know all of the bits for a constant fp! - return KnownBits::makeConstant(C->getValueAPF().bitcastToAPInt()); + return KnownBits::makeConstant(C->getValueAPF().bitcastToAPInt()); } if (Depth >= MaxRecursionDepth) @@ -2735,7 +2735,7 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, } // Known bits are the values that are shared by every demanded element. - Known = KnownBits::commonBits(Known, Known2); + Known = KnownBits::commonBits(Known, Known2); // If we don't know any bits, early out. if (Known.isUnknown()) @@ -2772,7 +2772,7 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, if (!!DemandedLHS) { SDValue LHS = Op.getOperand(0); Known2 = computeKnownBits(LHS, DemandedLHS, Depth + 1); - Known = KnownBits::commonBits(Known, Known2); + Known = KnownBits::commonBits(Known, Known2); } // If we don't know any bits, early out. if (Known.isUnknown()) @@ -2780,7 +2780,7 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, if (!!DemandedRHS) { SDValue RHS = Op.getOperand(1); Known2 = computeKnownBits(RHS, DemandedRHS, Depth + 1); - Known = KnownBits::commonBits(Known, Known2); + Known = KnownBits::commonBits(Known, Known2); } break; } @@ -2796,7 +2796,7 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, if (!!DemandedSub) { SDValue Sub = Op.getOperand(i); Known2 = computeKnownBits(Sub, DemandedSub, Depth + 1); - Known = KnownBits::commonBits(Known, Known2); + Known = KnownBits::commonBits(Known, Known2); } // If we don't know any bits, early out. if (Known.isUnknown()) @@ -2824,7 +2824,7 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, } if (!!DemandedSrcElts) { Known2 = computeKnownBits(Src, DemandedSrcElts, Depth + 1); - Known = KnownBits::commonBits(Known, Known2); + Known = KnownBits::commonBits(Known, Known2); } break; } @@ -2943,13 +2943,13 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, case ISD::MUL: { Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); - Known = KnownBits::computeForMul(Known, Known2); + Known = KnownBits::computeForMul(Known, Known2); break; } case ISD::UDIV: { - Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); + Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); - Known = KnownBits::udiv(Known, Known2); + Known = KnownBits::udiv(Known, Known2); break; } case ISD::SELECT: @@ -2961,7 +2961,7 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth+1); // Only known if known in both the LHS and RHS. - Known = KnownBits::commonBits(Known, Known2); + Known = KnownBits::commonBits(Known, Known2); break; case ISD::SELECT_CC: Known = computeKnownBits(Op.getOperand(3), DemandedElts, Depth+1); @@ -2971,7 +2971,7 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, Known2 = computeKnownBits(Op.getOperand(2), DemandedElts, Depth+1); // Only known if known in both the LHS and RHS. - Known = KnownBits::commonBits(Known, Known2); + Known = KnownBits::commonBits(Known, Known2); break; case ISD::SMULO: case ISD::UMULO: @@ -3000,8 +3000,8 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, } case ISD::SHL: Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); - Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); - Known = KnownBits::shl(Known, Known2); + Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); + Known = KnownBits::shl(Known, Known2); // Minimum shift low bits are known zero. if (const APInt *ShMinAmt = @@ -3010,8 +3010,8 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, break; case ISD::SRL: Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); - Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); - Known = KnownBits::lshr(Known, Known2); + Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); + Known = KnownBits::lshr(Known, Known2); // Minimum shift high bits are known zero. if (const APInt *ShMinAmt = @@ -3019,10 +3019,10 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, Known.Zero.setHighBits(ShMinAmt->getZExtValue()); break; case ISD::SRA: - Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); - Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); - Known = KnownBits::ashr(Known, Known2); - // TODO: Add minimum shift high known sign bits. + Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); + Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); + Known = KnownBits::ashr(Known, Known2); + // TODO: Add minimum shift high known sign bits. break; case ISD::FSHL: case ISD::FSHR: @@ -3057,9 +3057,9 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, } break; case ISD::SIGN_EXTEND_INREG: { - Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); + Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); EVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT(); - Known = Known.sextInReg(EVT.getScalarSizeInBits()); + Known = Known.sextInReg(EVT.getScalarSizeInBits()); break; } case ISD::CTTZ: @@ -3087,11 +3087,11 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, Known.Zero.setBitsFrom(Log2_32(PossibleOnes) + 1); break; } - case ISD::PARITY: { - // Parity returns 0 everywhere but the LSB. - Known.Zero.setBitsFrom(1); - break; - } + case ISD::PARITY: { + // Parity returns 0 everywhere but the LSB. + Known.Zero.setBitsFrom(1); + break; + } case ISD::LOAD: { LoadSDNode *LD = cast<LoadSDNode>(Op); const Constant *Cst = TLI->getTargetConstantFromLoad(LD); @@ -3135,10 +3135,10 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, } } else if (BitWidth == CstTy->getPrimitiveSizeInBits()) { if (auto *CInt = dyn_cast<ConstantInt>(Cst)) { - Known = KnownBits::makeConstant(CInt->getValue()); + Known = KnownBits::makeConstant(CInt->getValue()); } else if (auto *CFP = dyn_cast<ConstantFP>(Cst)) { - Known = - KnownBits::makeConstant(CFP->getValueAPF().bitcastToAPInt()); + Known = + KnownBits::makeConstant(CFP->getValueAPF().bitcastToAPInt()); } } } @@ -3278,16 +3278,16 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, Known = KnownBits::computeForAddCarry(Known, Known2, Carry); break; } - case ISD::SREM: { - Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); - Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); - Known = KnownBits::srem(Known, Known2); + case ISD::SREM: { + Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); + Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); + Known = KnownBits::srem(Known, Known2); break; - } + } case ISD::UREM: { Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); - Known = KnownBits::urem(Known, Known2); + Known = KnownBits::urem(Known, Known2); break; } case ISD::EXTRACT_ELEMENT: { @@ -3307,9 +3307,9 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, SDValue InVec = Op.getOperand(0); SDValue EltNo = Op.getOperand(1); EVT VecVT = InVec.getValueType(); - // computeKnownBits not yet implemented for scalable vectors. - if (VecVT.isScalableVector()) - break; + // computeKnownBits not yet implemented for scalable vectors. + if (VecVT.isScalableVector()) + break; const unsigned EltBitWidth = VecVT.getScalarSizeInBits(); const unsigned NumSrcElts = VecVT.getVectorNumElements(); @@ -3350,39 +3350,39 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, Known.Zero.setAllBits(); if (DemandedVal) { Known2 = computeKnownBits(InVal, Depth + 1); - Known = KnownBits::commonBits(Known, Known2.zextOrTrunc(BitWidth)); + Known = KnownBits::commonBits(Known, Known2.zextOrTrunc(BitWidth)); } if (!!DemandedVecElts) { Known2 = computeKnownBits(InVec, DemandedVecElts, Depth + 1); - Known = KnownBits::commonBits(Known, Known2); + Known = KnownBits::commonBits(Known, Known2); } break; } case ISD::BITREVERSE: { Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); - Known = Known2.reverseBits(); + Known = Known2.reverseBits(); break; } case ISD::BSWAP: { Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); - Known = Known2.byteSwap(); + Known = Known2.byteSwap(); break; } case ISD::ABS: { Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); - Known = Known2.abs(); + Known = Known2.abs(); break; } case ISD::UMIN: { Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); - Known = KnownBits::umin(Known, Known2); + Known = KnownBits::umin(Known, Known2); break; } case ISD::UMAX: { Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); - Known = KnownBits::umax(Known, Known2); + Known = KnownBits::umax(Known, Known2); break; } case ISD::SMIN: @@ -3418,10 +3418,10 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); - if (IsMax) - Known = KnownBits::smax(Known, Known2); - else - Known = KnownBits::smin(Known, Known2); + if (IsMax) + Known = KnownBits::smax(Known, Known2); + else + Known = KnownBits::smin(Known, Known2); break; } case ISD::FrameIndex: @@ -4364,16 +4364,16 @@ static SDValue foldCONCAT_VECTORS(const SDLoc &DL, EVT VT, for (SDValue Op : Elts) SVT = (SVT.bitsLT(Op.getValueType()) ? Op.getValueType() : SVT); - if (SVT.bitsGT(VT.getScalarType())) { - for (SDValue &Op : Elts) { - if (Op.isUndef()) - Op = DAG.getUNDEF(SVT); - else - Op = DAG.getTargetLoweringInfo().isZExtFree(Op.getValueType(), SVT) - ? DAG.getZExtOrTrunc(Op, DL, SVT) - : DAG.getSExtOrTrunc(Op, DL, SVT); - } - } + if (SVT.bitsGT(VT.getScalarType())) { + for (SDValue &Op : Elts) { + if (Op.isUndef()) + Op = DAG.getUNDEF(SVT); + else + Op = DAG.getTargetLoweringInfo().isZExtFree(Op.getValueType(), SVT) + ? DAG.getZExtOrTrunc(Op, DL, SVT) + : DAG.getSExtOrTrunc(Op, DL, SVT); + } + } SDValue V = DAG.getBuildVector(VT, DL, Elts); NewSDValueDbgMsg(V, "New node fold concat vectors: ", &DAG); @@ -4399,14 +4399,14 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT) { } SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, - SDValue Operand) { - SDNodeFlags Flags; - if (Inserter) - Flags = Inserter->getFlags(); - return getNode(Opcode, DL, VT, Operand, Flags); -} - -SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, + SDValue Operand) { + SDNodeFlags Flags; + if (Inserter) + Flags = Inserter->getFlags(); + return getNode(Opcode, DL, VT, Operand, Flags); +} + +SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue Operand, const SDNodeFlags Flags) { // Constant fold unary operations with an integer constant operand. Even // opaque constant will be folded, because the folding of unary operations @@ -4607,8 +4607,8 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, Operand.getValueType().isFloatingPoint() && "Invalid FP cast!"); if (Operand.getValueType() == VT) return Operand; // noop conversion. assert((!VT.isVector() || - VT.getVectorElementCount() == - Operand.getValueType().getVectorElementCount()) && + VT.getVectorElementCount() == + Operand.getValueType().getVectorElementCount()) && "Vector element count mismatch!"); assert(Operand.getValueType().bitsLT(VT) && "Invalid fpext node, dst < src!"); @@ -4793,25 +4793,25 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, case ISD::VSCALE: assert(VT == Operand.getValueType() && "Unexpected VT!"); break; - case ISD::CTPOP: - if (Operand.getValueType().getScalarType() == MVT::i1) - return Operand; - break; - case ISD::CTLZ: - case ISD::CTTZ: - if (Operand.getValueType().getScalarType() == MVT::i1) - return getNOT(DL, Operand, Operand.getValueType()); - break; - case ISD::VECREDUCE_SMIN: - case ISD::VECREDUCE_UMAX: - if (Operand.getValueType().getScalarType() == MVT::i1) - return getNode(ISD::VECREDUCE_OR, DL, VT, Operand); - break; - case ISD::VECREDUCE_SMAX: - case ISD::VECREDUCE_UMIN: - if (Operand.getValueType().getScalarType() == MVT::i1) - return getNode(ISD::VECREDUCE_AND, DL, VT, Operand); - break; + case ISD::CTPOP: + if (Operand.getValueType().getScalarType() == MVT::i1) + return Operand; + break; + case ISD::CTLZ: + case ISD::CTTZ: + if (Operand.getValueType().getScalarType() == MVT::i1) + return getNOT(DL, Operand, Operand.getValueType()); + break; + case ISD::VECREDUCE_SMIN: + case ISD::VECREDUCE_UMAX: + if (Operand.getValueType().getScalarType() == MVT::i1) + return getNode(ISD::VECREDUCE_OR, DL, VT, Operand); + break; + case ISD::VECREDUCE_SMAX: + case ISD::VECREDUCE_UMIN: + if (Operand.getValueType().getScalarType() == MVT::i1) + return getNode(ISD::VECREDUCE_AND, DL, VT, Operand); + break; } SDNode *N; @@ -5234,14 +5234,14 @@ SDValue SelectionDAG::getAssertAlign(const SDLoc &DL, SDValue Val, Align A) { } SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, - SDValue N1, SDValue N2) { - SDNodeFlags Flags; - if (Inserter) - Flags = Inserter->getFlags(); - return getNode(Opcode, DL, VT, N1, N2, Flags); -} - -SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, + SDValue N1, SDValue N2) { + SDNodeFlags Flags; + if (Inserter) + Flags = Inserter->getFlags(); + return getNode(Opcode, DL, VT, N1, N2, Flags); +} + +SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1, SDValue N2, const SDNodeFlags Flags) { ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1); ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2); @@ -5329,22 +5329,22 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, assert(N1.getValueType() == N2.getValueType() && N1.getValueType() == VT && "Binary operator types must match!"); break; - case ISD::SMIN: - case ISD::UMAX: - assert(VT.isInteger() && "This operator does not apply to FP types!"); - assert(N1.getValueType() == N2.getValueType() && - N1.getValueType() == VT && "Binary operator types must match!"); - if (VT.isVector() && VT.getVectorElementType() == MVT::i1) - return getNode(ISD::OR, DL, VT, N1, N2); - break; - case ISD::SMAX: - case ISD::UMIN: - assert(VT.isInteger() && "This operator does not apply to FP types!"); - assert(N1.getValueType() == N2.getValueType() && - N1.getValueType() == VT && "Binary operator types must match!"); - if (VT.isVector() && VT.getVectorElementType() == MVT::i1) - return getNode(ISD::AND, DL, VT, N1, N2); - break; + case ISD::SMIN: + case ISD::UMAX: + assert(VT.isInteger() && "This operator does not apply to FP types!"); + assert(N1.getValueType() == N2.getValueType() && + N1.getValueType() == VT && "Binary operator types must match!"); + if (VT.isVector() && VT.getVectorElementType() == MVT::i1) + return getNode(ISD::OR, DL, VT, N1, N2); + break; + case ISD::SMAX: + case ISD::UMIN: + assert(VT.isInteger() && "This operator does not apply to FP types!"); + assert(N1.getValueType() == N2.getValueType() && + N1.getValueType() == VT && "Binary operator types must match!"); + if (VT.isVector() && VT.getVectorElementType() == MVT::i1) + return getNode(ISD::AND, DL, VT, N1, N2); + break; case ISD::FADD: case ISD::FSUB: case ISD::FMUL: @@ -5386,8 +5386,8 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, // amounts. This catches things like trying to shift an i1024 value by an // i8, which is easy to fall into in generic code that uses // TLI.getShiftAmount(). - assert(N2.getValueType().getScalarSizeInBits() >= - Log2_32_Ceil(VT.getScalarSizeInBits()) && + assert(N2.getValueType().getScalarSizeInBits() >= + Log2_32_Ceil(VT.getScalarSizeInBits()) && "Invalid use of small shift amount with oversized value!"); // Always fold shifts of i1 values so the code generator doesn't need to @@ -5583,11 +5583,11 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, (VT.getVectorMinNumElements() + N2C->getZExtValue()) <= N1VT.getVectorMinNumElements()) && "Extract subvector overflow!"); - assert(N2C->getAPIntValue().getBitWidth() == - TLI->getVectorIdxTy(getDataLayout()) - .getSizeInBits() - .getFixedSize() && - "Constant index for EXTRACT_SUBVECTOR has an invalid size"); + assert(N2C->getAPIntValue().getBitWidth() == + TLI->getVectorIdxTy(getDataLayout()) + .getSizeInBits() + .getFixedSize() && + "Constant index for EXTRACT_SUBVECTOR has an invalid size"); // Trivial extraction. if (VT == N1VT) @@ -5599,8 +5599,8 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, // EXTRACT_SUBVECTOR of CONCAT_VECTOR can be simplified if the pieces of // the concat have the same type as the extract. - if (N1.getOpcode() == ISD::CONCAT_VECTORS && N1.getNumOperands() > 0 && - VT == N1.getOperand(0).getValueType()) { + if (N1.getOpcode() == ISD::CONCAT_VECTORS && N1.getNumOperands() > 0 && + VT == N1.getOperand(0).getValueType()) { unsigned Factor = VT.getVectorMinNumElements(); return N1.getOperand(N2C->getZExtValue() / Factor); } @@ -5697,14 +5697,14 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, } SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, - SDValue N1, SDValue N2, SDValue N3) { - SDNodeFlags Flags; - if (Inserter) - Flags = Inserter->getFlags(); - return getNode(Opcode, DL, VT, N1, N2, N3, Flags); -} - -SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, + SDValue N1, SDValue N2, SDValue N3) { + SDNodeFlags Flags; + if (Inserter) + Flags = Inserter->getFlags(); + return getNode(Opcode, DL, VT, N1, N2, N3, Flags); +} + +SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1, SDValue N2, SDValue N3, const SDNodeFlags Flags) { // Perform various simplifications. @@ -5974,20 +5974,20 @@ static SDValue getMemsetStringVal(EVT VT, const SDLoc &dl, SelectionDAG &DAG, return SDValue(nullptr, 0); } -SDValue SelectionDAG::getMemBasePlusOffset(SDValue Base, TypeSize Offset, +SDValue SelectionDAG::getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags) { EVT VT = Base.getValueType(); - SDValue Index; - - if (Offset.isScalable()) - Index = getVScale(DL, Base.getValueType(), - APInt(Base.getValueSizeInBits().getFixedSize(), - Offset.getKnownMinSize())); - else - Index = getConstant(Offset.getFixedSize(), DL, VT); - - return getMemBasePlusOffset(Base, Index, DL, Flags); + SDValue Index; + + if (Offset.isScalable()) + Index = getVScale(DL, Base.getValueType(), + APInt(Base.getValueSizeInBits().getFixedSize(), + Offset.getKnownMinSize())); + else + Index = getConstant(Offset.getFixedSize(), DL, VT); + + return getMemBasePlusOffset(Base, Index, DL, Flags); } SDValue SelectionDAG::getMemBasePlusOffset(SDValue Ptr, SDValue Offset, @@ -6082,8 +6082,8 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, SrcAlign = Alignment; assert(SrcAlign && "SrcAlign must be set"); ConstantDataArraySlice Slice; - // If marked as volatile, perform a copy even when marked as constant. - bool CopyFromConstant = !isVol && isMemSrcFromConstant(Src, Slice); + // If marked as volatile, perform a copy even when marked as constant. + bool CopyFromConstant = !isVol && isMemSrcFromConstant(Src, Slice); bool isZeroConstant = CopyFromConstant && Slice.Array == nullptr; unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize); const MemOp Op = isZeroConstant @@ -6155,9 +6155,9 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, Value = getMemsetStringVal(VT, dl, DAG, TLI, SubSlice); if (Value.getNode()) { Store = DAG.getStore( - Chain, dl, Value, - DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl), - DstPtrInfo.getWithOffset(DstOff), Alignment, MMOFlags); + Chain, dl, Value, + DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl), + DstPtrInfo.getWithOffset(DstOff), Alignment, MMOFlags); OutChains.push_back(Store); } } @@ -6177,17 +6177,17 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, if (isDereferenceable) SrcMMOFlags |= MachineMemOperand::MODereferenceable; - Value = DAG.getExtLoad( - ISD::EXTLOAD, dl, NVT, Chain, - DAG.getMemBasePlusOffset(Src, TypeSize::Fixed(SrcOff), dl), - SrcPtrInfo.getWithOffset(SrcOff), VT, - commonAlignment(*SrcAlign, SrcOff), SrcMMOFlags); + Value = DAG.getExtLoad( + ISD::EXTLOAD, dl, NVT, Chain, + DAG.getMemBasePlusOffset(Src, TypeSize::Fixed(SrcOff), dl), + SrcPtrInfo.getWithOffset(SrcOff), VT, + commonAlignment(*SrcAlign, SrcOff), SrcMMOFlags); OutLoadChains.push_back(Value.getValue(1)); Store = DAG.getTruncStore( - Chain, dl, Value, - DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl), - DstPtrInfo.getWithOffset(DstOff), VT, Alignment, MMOFlags); + Chain, dl, Value, + DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl), + DstPtrInfo.getWithOffset(DstOff), VT, Alignment, MMOFlags); OutStoreChains.push_back(Store); } SrcOff += VTSize; @@ -6307,10 +6307,10 @@ static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, if (isDereferenceable) SrcMMOFlags |= MachineMemOperand::MODereferenceable; - Value = - DAG.getLoad(VT, dl, Chain, - DAG.getMemBasePlusOffset(Src, TypeSize::Fixed(SrcOff), dl), - SrcPtrInfo.getWithOffset(SrcOff), *SrcAlign, SrcMMOFlags); + Value = + DAG.getLoad(VT, dl, Chain, + DAG.getMemBasePlusOffset(Src, TypeSize::Fixed(SrcOff), dl), + SrcPtrInfo.getWithOffset(SrcOff), *SrcAlign, SrcMMOFlags); LoadValues.push_back(Value); LoadChains.push_back(Value.getValue(1)); SrcOff += VTSize; @@ -6322,10 +6322,10 @@ static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, unsigned VTSize = VT.getSizeInBits() / 8; SDValue Store; - Store = - DAG.getStore(Chain, dl, LoadValues[i], - DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl), - DstPtrInfo.getWithOffset(DstOff), Alignment, MMOFlags); + Store = + DAG.getStore(Chain, dl, LoadValues[i], + DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl), + DstPtrInfo.getWithOffset(DstOff), Alignment, MMOFlags); OutChains.push_back(Store); DstOff += VTSize; } @@ -6423,9 +6423,9 @@ static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl, } assert(Value.getValueType() == VT && "Value with wrong type."); SDValue Store = DAG.getStore( - Chain, dl, Value, - DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl), - DstPtrInfo.getWithOffset(DstOff), Alignment, + Chain, dl, Value, + DAG.getMemBasePlusOffset(Dst, TypeSize::Fixed(DstOff), dl), + DstPtrInfo.getWithOffset(DstOff), Alignment, isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone); OutChains.push_back(Store); DstOff += VT.getSizeInBits() / 8; @@ -6439,7 +6439,7 @@ static void checkAddrSpaceIsValidForLibcall(const TargetLowering *TLI, unsigned AS) { // Lowering memcpy / memset / memmove intrinsics to calls is only valid if all // pointer operands can be losslessly bitcasted to pointers of address space 0 - if (AS != 0 && !TLI->getTargetMachine().isNoopAddrSpaceCast(AS, 0)) { + if (AS != 0 && !TLI->getTargetMachine().isNoopAddrSpaceCast(AS, 0)) { report_fatal_error("cannot lower memory intrinsic in address space " + Twine(AS)); } @@ -6931,30 +6931,30 @@ SDValue SelectionDAG::getLifetimeNode(bool IsStart, const SDLoc &dl, return V; } -SDValue SelectionDAG::getPseudoProbeNode(const SDLoc &Dl, SDValue Chain, - uint64_t Guid, uint64_t Index, - uint32_t Attr) { - const unsigned Opcode = ISD::PSEUDO_PROBE; - const auto VTs = getVTList(MVT::Other); - SDValue Ops[] = {Chain}; - FoldingSetNodeID ID; - AddNodeIDNode(ID, Opcode, VTs, Ops); - ID.AddInteger(Guid); - ID.AddInteger(Index); - void *IP = nullptr; - if (SDNode *E = FindNodeOrInsertPos(ID, Dl, IP)) - return SDValue(E, 0); - - auto *N = newSDNode<PseudoProbeSDNode>( - Opcode, Dl.getIROrder(), Dl.getDebugLoc(), VTs, Guid, Index, Attr); - createOperands(N, Ops); - CSEMap.InsertNode(N, IP); - InsertNode(N); - SDValue V(N, 0); - NewSDValueDbgMsg(V, "Creating new node: ", this); - return V; -} - +SDValue SelectionDAG::getPseudoProbeNode(const SDLoc &Dl, SDValue Chain, + uint64_t Guid, uint64_t Index, + uint32_t Attr) { + const unsigned Opcode = ISD::PSEUDO_PROBE; + const auto VTs = getVTList(MVT::Other); + SDValue Ops[] = {Chain}; + FoldingSetNodeID ID; + AddNodeIDNode(ID, Opcode, VTs, Ops); + ID.AddInteger(Guid); + ID.AddInteger(Index); + void *IP = nullptr; + if (SDNode *E = FindNodeOrInsertPos(ID, Dl, IP)) + return SDValue(E, 0); + + auto *N = newSDNode<PseudoProbeSDNode>( + Opcode, Dl.getIROrder(), Dl.getDebugLoc(), VTs, Guid, Index, Attr); + createOperands(N, Ops); + CSEMap.InsertNode(N, IP); + InsertNode(N); + SDValue V(N, 0); + NewSDValueDbgMsg(V, "Creating new node: ", this); + return V; +} + /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a /// MachinePointerInfo record from it. This is particularly useful because the /// code generator has many cases where it doesn't bother passing in a @@ -7035,7 +7035,7 @@ SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, assert(VT.isVector() == MemVT.isVector() && "Cannot use an ext load to convert to or from a vector!"); assert((!VT.isVector() || - VT.getVectorElementCount() == MemVT.getVectorElementCount()) && + VT.getVectorElementCount() == MemVT.getVectorElementCount()) && "Cannot use an ext load to change the number of vector elements!"); } @@ -7114,7 +7114,7 @@ SDValue SelectionDAG::getIndexedLoad(SDValue OrigLoad, const SDLoc &dl, ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable); return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl, LD->getChain(), Base, Offset, LD->getPointerInfo(), - LD->getMemoryVT(), LD->getAlign(), MMOFlags, LD->getAAInfo()); + LD->getMemoryVT(), LD->getAlign(), MMOFlags, LD->getAAInfo()); } SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val, @@ -7184,8 +7184,8 @@ SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, MachineFunction &MF = getMachineFunction(); MachineMemOperand *MMO = MF.getMachineMemOperand( - PtrInfo, MMOFlags, MemoryLocation::getSizeOrUnknown(SVT.getStoreSize()), - Alignment, AAInfo); + PtrInfo, MMOFlags, MemoryLocation::getSizeOrUnknown(SVT.getStoreSize()), + Alignment, AAInfo); return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO); } @@ -7206,7 +7206,7 @@ SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, assert(VT.isVector() == SVT.isVector() && "Cannot use trunc store to convert to or from a vector!"); assert((!VT.isVector() || - VT.getVectorElementCount() == SVT.getVectorElementCount()) && + VT.getVectorElementCount() == SVT.getVectorElementCount()) && "Cannot use trunc store to change the number of vector elements!"); SDVTList VTs = getVTList(MVT::Other); @@ -7358,15 +7358,15 @@ SDValue SelectionDAG::getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl, SDValue SelectionDAG::getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef<SDValue> Ops, MachineMemOperand *MMO, - ISD::MemIndexType IndexType, - ISD::LoadExtType ExtTy) { + ISD::MemIndexType IndexType, + ISD::LoadExtType ExtTy) { assert(Ops.size() == 6 && "Incompatible number of operands"); FoldingSetNodeID ID; AddNodeIDNode(ID, ISD::MGATHER, VTs, Ops); ID.AddInteger(VT.getRawBits()); ID.AddInteger(getSyntheticNodeSubclassData<MaskedGatherSDNode>( - dl.getIROrder(), VTs, VT, MMO, IndexType, ExtTy)); + dl.getIROrder(), VTs, VT, MMO, IndexType, ExtTy)); ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); void *IP = nullptr; if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { @@ -7374,22 +7374,22 @@ SDValue SelectionDAG::getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl, return SDValue(E, 0); } - IndexType = TLI->getCanonicalIndexType(IndexType, VT, Ops[4]); + IndexType = TLI->getCanonicalIndexType(IndexType, VT, Ops[4]); auto *N = newSDNode<MaskedGatherSDNode>(dl.getIROrder(), dl.getDebugLoc(), - VTs, VT, MMO, IndexType, ExtTy); + VTs, VT, MMO, IndexType, ExtTy); createOperands(N, Ops); assert(N->getPassThru().getValueType() == N->getValueType(0) && "Incompatible type of the PassThru value in MaskedGatherSDNode"); - assert(N->getMask().getValueType().getVectorElementCount() == - N->getValueType(0).getVectorElementCount() && + assert(N->getMask().getValueType().getVectorElementCount() == + N->getValueType(0).getVectorElementCount() && "Vector width mismatch between mask and data"); - assert(N->getIndex().getValueType().getVectorElementCount().isScalable() == - N->getValueType(0).getVectorElementCount().isScalable() && - "Scalable flags of index and data do not match"); - assert(ElementCount::isKnownGE( - N->getIndex().getValueType().getVectorElementCount(), - N->getValueType(0).getVectorElementCount()) && + assert(N->getIndex().getValueType().getVectorElementCount().isScalable() == + N->getValueType(0).getVectorElementCount().isScalable() && + "Scalable flags of index and data do not match"); + assert(ElementCount::isKnownGE( + N->getIndex().getValueType().getVectorElementCount(), + N->getValueType(0).getVectorElementCount()) && "Vector width mismatch between index and data"); assert(isa<ConstantSDNode>(N->getScale()) && cast<ConstantSDNode>(N->getScale())->getAPIntValue().isPowerOf2() && @@ -7405,37 +7405,37 @@ SDValue SelectionDAG::getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl, SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef<SDValue> Ops, MachineMemOperand *MMO, - ISD::MemIndexType IndexType, - bool IsTrunc) { + ISD::MemIndexType IndexType, + bool IsTrunc) { assert(Ops.size() == 6 && "Incompatible number of operands"); FoldingSetNodeID ID; AddNodeIDNode(ID, ISD::MSCATTER, VTs, Ops); ID.AddInteger(VT.getRawBits()); ID.AddInteger(getSyntheticNodeSubclassData<MaskedScatterSDNode>( - dl.getIROrder(), VTs, VT, MMO, IndexType, IsTrunc)); + dl.getIROrder(), VTs, VT, MMO, IndexType, IsTrunc)); ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); void *IP = nullptr; if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) { cast<MaskedScatterSDNode>(E)->refineAlignment(MMO); return SDValue(E, 0); } - - IndexType = TLI->getCanonicalIndexType(IndexType, VT, Ops[4]); + + IndexType = TLI->getCanonicalIndexType(IndexType, VT, Ops[4]); auto *N = newSDNode<MaskedScatterSDNode>(dl.getIROrder(), dl.getDebugLoc(), - VTs, VT, MMO, IndexType, IsTrunc); + VTs, VT, MMO, IndexType, IsTrunc); createOperands(N, Ops); - assert(N->getMask().getValueType().getVectorElementCount() == - N->getValue().getValueType().getVectorElementCount() && + assert(N->getMask().getValueType().getVectorElementCount() == + N->getValue().getValueType().getVectorElementCount() && "Vector width mismatch between mask and data"); - assert( - N->getIndex().getValueType().getVectorElementCount().isScalable() == - N->getValue().getValueType().getVectorElementCount().isScalable() && - "Scalable flags of index and data do not match"); - assert(ElementCount::isKnownGE( - N->getIndex().getValueType().getVectorElementCount(), - N->getValue().getValueType().getVectorElementCount()) && + assert( + N->getIndex().getValueType().getVectorElementCount().isScalable() == + N->getValue().getValueType().getVectorElementCount().isScalable() && + "Scalable flags of index and data do not match"); + assert(ElementCount::isKnownGE( + N->getIndex().getValueType().getVectorElementCount(), + N->getValue().getValueType().getVectorElementCount()) && "Vector width mismatch between index and data"); assert(isa<ConstantSDNode>(N->getScale()) && cast<ConstantSDNode>(N->getScale())->getAPIntValue().isPowerOf2() && @@ -7539,11 +7539,11 @@ SDValue SelectionDAG::simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y, if (YC->getValueAPF().isExactlyValue(1.0)) return X; - // X * 0.0 --> 0.0 - if (Opcode == ISD::FMUL && Flags.hasNoNaNs() && Flags.hasNoSignedZeros()) - if (YC->getValueAPF().isZero()) - return getConstantFP(0.0, SDLoc(Y), Y.getValueType()); - + // X * 0.0 --> 0.0 + if (Opcode == ISD::FMUL && Flags.hasNoNaNs() && Flags.hasNoSignedZeros()) + if (YC->getValueAPF().isZero()) + return getConstantFP(0.0, SDLoc(Y), Y.getValueType()); + return SDValue(); } @@ -7570,14 +7570,14 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, } SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, - ArrayRef<SDValue> Ops) { - SDNodeFlags Flags; - if (Inserter) - Flags = Inserter->getFlags(); - return getNode(Opcode, DL, VT, Ops, Flags); -} - -SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, + ArrayRef<SDValue> Ops) { + SDNodeFlags Flags; + if (Inserter) + Flags = Inserter->getFlags(); + return getNode(Opcode, DL, VT, Ops, Flags); +} + +SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef<SDValue> Ops, const SDNodeFlags Flags) { unsigned NumOps = Ops.size(); switch (NumOps) { @@ -7649,14 +7649,14 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, } SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, - ArrayRef<SDValue> Ops) { - SDNodeFlags Flags; - if (Inserter) - Flags = Inserter->getFlags(); - return getNode(Opcode, DL, VTList, Ops, Flags); -} - -SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, + ArrayRef<SDValue> Ops) { + SDNodeFlags Flags; + if (Inserter) + Flags = Inserter->getFlags(); + return getNode(Opcode, DL, VTList, Ops, Flags); +} + +SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, ArrayRef<SDValue> Ops, const SDNodeFlags Flags) { if (VTList.NumVTs == 1) return getNode(Opcode, DL, VTList.VTs[0], Ops); @@ -8353,14 +8353,14 @@ SDValue SelectionDAG::getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT, /// getNodeIfExists - Get the specified node if it's already available, or /// else return NULL. SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList, - ArrayRef<SDValue> Ops) { - SDNodeFlags Flags; - if (Inserter) - Flags = Inserter->getFlags(); - return getNodeIfExists(Opcode, VTList, Ops, Flags); -} - -SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList, + ArrayRef<SDValue> Ops) { + SDNodeFlags Flags; + if (Inserter) + Flags = Inserter->getFlags(); + return getNodeIfExists(Opcode, VTList, Ops, Flags); +} + +SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList, ArrayRef<SDValue> Ops, const SDNodeFlags Flags) { if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) { @@ -8375,19 +8375,19 @@ SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList, return nullptr; } -/// doesNodeExist - Check if a node exists without modifying its flags. -bool SelectionDAG::doesNodeExist(unsigned Opcode, SDVTList VTList, - ArrayRef<SDValue> Ops) { - if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) { - FoldingSetNodeID ID; - AddNodeIDNode(ID, Opcode, VTList, Ops); - void *IP = nullptr; - if (FindNodeOrInsertPos(ID, SDLoc(), IP)) - return true; - } - return false; -} - +/// doesNodeExist - Check if a node exists without modifying its flags. +bool SelectionDAG::doesNodeExist(unsigned Opcode, SDVTList VTList, + ArrayRef<SDValue> Ops) { + if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) { + FoldingSetNodeID ID; + AddNodeIDNode(ID, Opcode, VTList, Ops); + void *IP = nullptr; + if (FindNodeOrInsertPos(ID, SDLoc(), IP)) + return true; + } + return false; +} + /// getDbgValue - Creates a SDDbgValue node. /// /// SDNode @@ -8805,31 +8805,31 @@ namespace { } // end anonymous namespace -bool SelectionDAG::calculateDivergence(SDNode *N) { - if (TLI->isSDNodeAlwaysUniform(N)) { - assert(!TLI->isSDNodeSourceOfDivergence(N, FLI, DA) && - "Conflicting divergence information!"); - return false; - } - if (TLI->isSDNodeSourceOfDivergence(N, FLI, DA)) - return true; +bool SelectionDAG::calculateDivergence(SDNode *N) { + if (TLI->isSDNodeAlwaysUniform(N)) { + assert(!TLI->isSDNodeSourceOfDivergence(N, FLI, DA) && + "Conflicting divergence information!"); + return false; + } + if (TLI->isSDNodeSourceOfDivergence(N, FLI, DA)) + return true; for (auto &Op : N->ops()) { - if (Op.Val.getValueType() != MVT::Other && Op.getNode()->isDivergent()) - return true; + if (Op.Val.getValueType() != MVT::Other && Op.getNode()->isDivergent()) + return true; } - return false; -} - -void SelectionDAG::updateDivergence(SDNode *N) { - SmallVector<SDNode *, 16> Worklist(1, N); - do { - N = Worklist.pop_back_val(); - bool IsDivergent = calculateDivergence(N); - if (N->SDNodeBits.IsDivergent != IsDivergent) { - N->SDNodeBits.IsDivergent = IsDivergent; - llvm::append_range(Worklist, N->uses()); + return false; +} + +void SelectionDAG::updateDivergence(SDNode *N) { + SmallVector<SDNode *, 16> Worklist(1, N); + do { + N = Worklist.pop_back_val(); + bool IsDivergent = calculateDivergence(N); + if (N->SDNodeBits.IsDivergent != IsDivergent) { + N->SDNodeBits.IsDivergent = IsDivergent; + llvm::append_range(Worklist, N->uses()); } - } while (!Worklist.empty()); + } while (!Worklist.empty()); } void SelectionDAG::CreateTopologicalOrder(std::vector<SDNode *> &Order) { @@ -8855,9 +8855,9 @@ void SelectionDAG::CreateTopologicalOrder(std::vector<SDNode *> &Order) { void SelectionDAG::VerifyDAGDiverence() { std::vector<SDNode *> TopoOrder; CreateTopologicalOrder(TopoOrder); - for (auto *N : TopoOrder) { - assert(calculateDivergence(N) == N->isDivergent() && - "Divergence bit inconsistency detected"); + for (auto *N : TopoOrder) { + assert(calculateDivergence(N) == N->isDivergent() && + "Divergence bit inconsistency detected"); } } #endif @@ -9026,32 +9026,32 @@ void SelectionDAG::AddDbgLabel(SDDbgLabel *DB) { DbgInfo->add(DB); } -SDValue SelectionDAG::makeEquivalentMemoryOrdering(SDValue OldChain, - SDValue NewMemOpChain) { - assert(isa<MemSDNode>(NewMemOpChain) && "Expected a memop node"); - assert(NewMemOpChain.getValueType() == MVT::Other && "Expected a token VT"); +SDValue SelectionDAG::makeEquivalentMemoryOrdering(SDValue OldChain, + SDValue NewMemOpChain) { + assert(isa<MemSDNode>(NewMemOpChain) && "Expected a memop node"); + assert(NewMemOpChain.getValueType() == MVT::Other && "Expected a token VT"); // The new memory operation must have the same position as the old load in // terms of memory dependency. Create a TokenFactor for the old load and new // memory operation and update uses of the old load's output chain to use that // TokenFactor. - if (OldChain == NewMemOpChain || OldChain.use_empty()) - return NewMemOpChain; + if (OldChain == NewMemOpChain || OldChain.use_empty()) + return NewMemOpChain; - SDValue TokenFactor = getNode(ISD::TokenFactor, SDLoc(OldChain), MVT::Other, - OldChain, NewMemOpChain); + SDValue TokenFactor = getNode(ISD::TokenFactor, SDLoc(OldChain), MVT::Other, + OldChain, NewMemOpChain); ReplaceAllUsesOfValueWith(OldChain, TokenFactor); - UpdateNodeOperands(TokenFactor.getNode(), OldChain, NewMemOpChain); + UpdateNodeOperands(TokenFactor.getNode(), OldChain, NewMemOpChain); return TokenFactor; } -SDValue SelectionDAG::makeEquivalentMemoryOrdering(LoadSDNode *OldLoad, - SDValue NewMemOp) { - assert(isa<MemSDNode>(NewMemOp.getNode()) && "Expected a memop node"); - SDValue OldChain = SDValue(OldLoad, 1); - SDValue NewMemOpChain = NewMemOp.getValue(1); - return makeEquivalentMemoryOrdering(OldChain, NewMemOpChain); -} - +SDValue SelectionDAG::makeEquivalentMemoryOrdering(LoadSDNode *OldLoad, + SDValue NewMemOp) { + assert(isa<MemSDNode>(NewMemOp.getNode()) && "Expected a memop node"); + SDValue OldChain = SDValue(OldLoad, 1); + SDValue NewMemOpChain = NewMemOp.getValue(1); + return makeEquivalentMemoryOrdering(OldChain, NewMemOpChain); +} + SDValue SelectionDAG::getSymbolFunctionGlobalAddress(SDValue Op, Function **OutFunction) { assert(isa<ExternalSymbolSDNode>(Op) && "Node should be an ExternalSymbol"); @@ -9135,18 +9135,18 @@ ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, bool AllowUndefs, if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N)) return CN; - // SplatVectors can truncate their operands. Ignore that case here unless - // AllowTruncation is set. - if (N->getOpcode() == ISD::SPLAT_VECTOR) { - EVT VecEltVT = N->getValueType(0).getVectorElementType(); - if (auto *CN = dyn_cast<ConstantSDNode>(N->getOperand(0))) { - EVT CVT = CN->getValueType(0); - assert(CVT.bitsGE(VecEltVT) && "Illegal splat_vector element extension"); - if (AllowTruncation || CVT == VecEltVT) - return CN; - } - } - + // SplatVectors can truncate their operands. Ignore that case here unless + // AllowTruncation is set. + if (N->getOpcode() == ISD::SPLAT_VECTOR) { + EVT VecEltVT = N->getValueType(0).getVectorElementType(); + if (auto *CN = dyn_cast<ConstantSDNode>(N->getOperand(0))) { + EVT CVT = CN->getValueType(0); + assert(CVT.bitsGE(VecEltVT) && "Illegal splat_vector element extension"); + if (AllowTruncation || CVT == VecEltVT) + return CN; + } + } + if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N)) { BitVector UndefElements; ConstantSDNode *CN = BV->getConstantSplatNode(&UndefElements); @@ -9200,10 +9200,10 @@ ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N, bool AllowUndefs) { return CN; } - if (N.getOpcode() == ISD::SPLAT_VECTOR) - if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(N.getOperand(0))) - return CN; - + if (N.getOpcode() == ISD::SPLAT_VECTOR) + if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(N.getOperand(0))) + return CN; + return nullptr; } @@ -9365,7 +9365,7 @@ bool SDNode::areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N) { bool Seen = false; for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) { SDNode *User = *I; - if (llvm::is_contained(Nodes, User)) + if (llvm::is_contained(Nodes, User)) Seen = true; else return false; @@ -9376,7 +9376,7 @@ bool SDNode::areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N) { /// isOperand - Return true if this node is an operand of N. bool SDValue::isOperandOf(const SDNode *N) const { - return is_contained(N->op_values(), *this); + return is_contained(N->op_values(), *this); } bool SDNode::isOperandOf(const SDNode *N) const { @@ -9765,19 +9765,19 @@ SelectionDAG::GetDependentSplitDestVTs(const EVT &VT, const EVT &EnvVT, // custom VL=9 with enveloping VL=8/8 yields 8/1 // custom VL=10 with enveloping VL=8/8 yields 8/2 // etc. - ElementCount VTNumElts = VT.getVectorElementCount(); - ElementCount EnvNumElts = EnvVT.getVectorElementCount(); - assert(VTNumElts.isScalable() == EnvNumElts.isScalable() && - "Mixing fixed width and scalable vectors when enveloping a type"); + ElementCount VTNumElts = VT.getVectorElementCount(); + ElementCount EnvNumElts = EnvVT.getVectorElementCount(); + assert(VTNumElts.isScalable() == EnvNumElts.isScalable() && + "Mixing fixed width and scalable vectors when enveloping a type"); EVT LoVT, HiVT; - if (VTNumElts.getKnownMinValue() > EnvNumElts.getKnownMinValue()) { + if (VTNumElts.getKnownMinValue() > EnvNumElts.getKnownMinValue()) { LoVT = EnvVT; - HiVT = EVT::getVectorVT(*getContext(), EltTp, VTNumElts - EnvNumElts); + HiVT = EVT::getVectorVT(*getContext(), EltTp, VTNumElts - EnvNumElts); *HiIsEmpty = false; } else { // Flag that hi type has zero storage size, but return split envelop type // (this would be easier if vector types with zero elements were allowed). - LoVT = EVT::getVectorVT(*getContext(), EltTp, VTNumElts); + LoVT = EVT::getVectorVT(*getContext(), EltTp, VTNumElts); HiVT = EnvVT; *HiIsEmpty = true; } @@ -9912,16 +9912,16 @@ bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, APInt &SplatUndef, SDValue BuildVectorSDNode::getSplatValue(const APInt &DemandedElts, BitVector *UndefElements) const { - unsigned NumOps = getNumOperands(); + unsigned NumOps = getNumOperands(); if (UndefElements) { UndefElements->clear(); - UndefElements->resize(NumOps); + UndefElements->resize(NumOps); } - assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size"); + assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size"); if (!DemandedElts) return SDValue(); SDValue Splatted; - for (unsigned i = 0; i != NumOps; ++i) { + for (unsigned i = 0; i != NumOps; ++i) { if (!DemandedElts[i]) continue; SDValue Op = getOperand(i); @@ -9950,58 +9950,58 @@ SDValue BuildVectorSDNode::getSplatValue(BitVector *UndefElements) const { return getSplatValue(DemandedElts, UndefElements); } -bool BuildVectorSDNode::getRepeatedSequence(const APInt &DemandedElts, - SmallVectorImpl<SDValue> &Sequence, - BitVector *UndefElements) const { - unsigned NumOps = getNumOperands(); - Sequence.clear(); - if (UndefElements) { - UndefElements->clear(); - UndefElements->resize(NumOps); - } - assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size"); - if (!DemandedElts || NumOps < 2 || !isPowerOf2_32(NumOps)) - return false; - - // Set the undefs even if we don't find a sequence (like getSplatValue). - if (UndefElements) - for (unsigned I = 0; I != NumOps; ++I) - if (DemandedElts[I] && getOperand(I).isUndef()) - (*UndefElements)[I] = true; - - // Iteratively widen the sequence length looking for repetitions. - for (unsigned SeqLen = 1; SeqLen < NumOps; SeqLen *= 2) { - Sequence.append(SeqLen, SDValue()); - for (unsigned I = 0; I != NumOps; ++I) { - if (!DemandedElts[I]) - continue; - SDValue &SeqOp = Sequence[I % SeqLen]; - SDValue Op = getOperand(I); - if (Op.isUndef()) { - if (!SeqOp) - SeqOp = Op; - continue; - } - if (SeqOp && !SeqOp.isUndef() && SeqOp != Op) { - Sequence.clear(); - break; - } - SeqOp = Op; - } - if (!Sequence.empty()) - return true; - } - - assert(Sequence.empty() && "Failed to empty non-repeating sequence pattern"); - return false; -} - -bool BuildVectorSDNode::getRepeatedSequence(SmallVectorImpl<SDValue> &Sequence, - BitVector *UndefElements) const { - APInt DemandedElts = APInt::getAllOnesValue(getNumOperands()); - return getRepeatedSequence(DemandedElts, Sequence, UndefElements); -} - +bool BuildVectorSDNode::getRepeatedSequence(const APInt &DemandedElts, + SmallVectorImpl<SDValue> &Sequence, + BitVector *UndefElements) const { + unsigned NumOps = getNumOperands(); + Sequence.clear(); + if (UndefElements) { + UndefElements->clear(); + UndefElements->resize(NumOps); + } + assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size"); + if (!DemandedElts || NumOps < 2 || !isPowerOf2_32(NumOps)) + return false; + + // Set the undefs even if we don't find a sequence (like getSplatValue). + if (UndefElements) + for (unsigned I = 0; I != NumOps; ++I) + if (DemandedElts[I] && getOperand(I).isUndef()) + (*UndefElements)[I] = true; + + // Iteratively widen the sequence length looking for repetitions. + for (unsigned SeqLen = 1; SeqLen < NumOps; SeqLen *= 2) { + Sequence.append(SeqLen, SDValue()); + for (unsigned I = 0; I != NumOps; ++I) { + if (!DemandedElts[I]) + continue; + SDValue &SeqOp = Sequence[I % SeqLen]; + SDValue Op = getOperand(I); + if (Op.isUndef()) { + if (!SeqOp) + SeqOp = Op; + continue; + } + if (SeqOp && !SeqOp.isUndef() && SeqOp != Op) { + Sequence.clear(); + break; + } + SeqOp = Op; + } + if (!Sequence.empty()) + return true; + } + + assert(Sequence.empty() && "Failed to empty non-repeating sequence pattern"); + return false; +} + +bool BuildVectorSDNode::getRepeatedSequence(SmallVectorImpl<SDValue> &Sequence, + BitVector *UndefElements) const { + APInt DemandedElts = APInt::getAllOnesValue(getNumOperands()); + return getRepeatedSequence(DemandedElts, Sequence, UndefElements); +} + ConstantSDNode * BuildVectorSDNode::getConstantSplatNode(const APInt &DemandedElts, BitVector *UndefElements) const { @@ -10074,7 +10074,7 @@ bool ShuffleVectorSDNode::isSplatMask(const int *Mask, EVT VT) { // Returns the SDNode if it is a constant integer BuildVector // or constant integer. -SDNode *SelectionDAG::isConstantIntBuildVectorOrConstantInt(SDValue N) const { +SDNode *SelectionDAG::isConstantIntBuildVectorOrConstantInt(SDValue N) const { if (isa<ConstantSDNode>(N)) return N.getNode(); if (ISD::isBuildVectorOfConstantSDNodes(N.getNode())) @@ -10085,15 +10085,15 @@ SDNode *SelectionDAG::isConstantIntBuildVectorOrConstantInt(SDValue N) const { if (GA->getOpcode() == ISD::GlobalAddress && TLI->isOffsetFoldingLegal(GA)) return GA; - if ((N.getOpcode() == ISD::SPLAT_VECTOR) && - isa<ConstantSDNode>(N.getOperand(0))) - return N.getNode(); + if ((N.getOpcode() == ISD::SPLAT_VECTOR) && + isa<ConstantSDNode>(N.getOperand(0))) + return N.getNode(); return nullptr; } -// Returns the SDNode if it is a constant float BuildVector -// or constant float. -SDNode *SelectionDAG::isConstantFPBuildVectorOrConstantFP(SDValue N) const { +// Returns the SDNode if it is a constant float BuildVector +// or constant float. +SDNode *SelectionDAG::isConstantFPBuildVectorOrConstantFP(SDValue N) const { if (isa<ConstantFPSDNode>(N)) return N.getNode(); @@ -10115,14 +10115,14 @@ void SelectionDAG::createOperands(SDNode *Node, ArrayRef<SDValue> Vals) { Ops[I].setUser(Node); Ops[I].setInitial(Vals[I]); if (Ops[I].Val.getValueType() != MVT::Other) // Skip Chain. It does not carry divergence. - IsDivergent |= Ops[I].getNode()->isDivergent(); + IsDivergent |= Ops[I].getNode()->isDivergent(); } Node->NumOperands = Vals.size(); Node->OperandList = Ops; - if (!TLI->isSDNodeAlwaysUniform(Node)) { - IsDivergent |= TLI->isSDNodeSourceOfDivergence(Node, FLI, DA); + if (!TLI->isSDNodeAlwaysUniform(Node)) { + IsDivergent |= TLI->isSDNodeSourceOfDivergence(Node, FLI, DA); Node->SDNodeBits.IsDivergent = IsDivergent; - } + } checkForCycles(Node); } @@ -10139,44 +10139,44 @@ SDValue SelectionDAG::getTokenFactor(const SDLoc &DL, return getNode(ISD::TokenFactor, DL, MVT::Other, Vals); } -SDValue SelectionDAG::getNeutralElement(unsigned Opcode, const SDLoc &DL, - EVT VT, SDNodeFlags Flags) { - switch (Opcode) { - default: - return SDValue(); - case ISD::ADD: - case ISD::OR: - case ISD::XOR: - case ISD::UMAX: - return getConstant(0, DL, VT); - case ISD::MUL: - return getConstant(1, DL, VT); - case ISD::AND: - case ISD::UMIN: - return getAllOnesConstant(DL, VT); - case ISD::SMAX: - return getConstant(APInt::getSignedMinValue(VT.getSizeInBits()), DL, VT); - case ISD::SMIN: - return getConstant(APInt::getSignedMaxValue(VT.getSizeInBits()), DL, VT); - case ISD::FADD: - return getConstantFP(-0.0, DL, VT); - case ISD::FMUL: - return getConstantFP(1.0, DL, VT); - case ISD::FMINNUM: - case ISD::FMAXNUM: { - // Neutral element for fminnum is NaN, Inf or FLT_MAX, depending on FMF. - const fltSemantics &Semantics = EVTToAPFloatSemantics(VT); - APFloat NeutralAF = !Flags.hasNoNaNs() ? APFloat::getQNaN(Semantics) : - !Flags.hasNoInfs() ? APFloat::getInf(Semantics) : - APFloat::getLargest(Semantics); - if (Opcode == ISD::FMAXNUM) - NeutralAF.changeSign(); - - return getConstantFP(NeutralAF, DL, VT); - } - } -} - +SDValue SelectionDAG::getNeutralElement(unsigned Opcode, const SDLoc &DL, + EVT VT, SDNodeFlags Flags) { + switch (Opcode) { + default: + return SDValue(); + case ISD::ADD: + case ISD::OR: + case ISD::XOR: + case ISD::UMAX: + return getConstant(0, DL, VT); + case ISD::MUL: + return getConstant(1, DL, VT); + case ISD::AND: + case ISD::UMIN: + return getAllOnesConstant(DL, VT); + case ISD::SMAX: + return getConstant(APInt::getSignedMinValue(VT.getSizeInBits()), DL, VT); + case ISD::SMIN: + return getConstant(APInt::getSignedMaxValue(VT.getSizeInBits()), DL, VT); + case ISD::FADD: + return getConstantFP(-0.0, DL, VT); + case ISD::FMUL: + return getConstantFP(1.0, DL, VT); + case ISD::FMINNUM: + case ISD::FMAXNUM: { + // Neutral element for fminnum is NaN, Inf or FLT_MAX, depending on FMF. + const fltSemantics &Semantics = EVTToAPFloatSemantics(VT); + APFloat NeutralAF = !Flags.hasNoNaNs() ? APFloat::getQNaN(Semantics) : + !Flags.hasNoInfs() ? APFloat::getInf(Semantics) : + APFloat::getLargest(Semantics); + if (Opcode == ISD::FMAXNUM) + NeutralAF.changeSign(); + + return getConstantFP(NeutralAF, DL, VT); + } + } +} + #ifndef NDEBUG static void checkForCyclesHelper(const SDNode *N, SmallPtrSetImpl<const SDNode*> &Visited, |