diff options
author | orivej <orivej@yandex-team.ru> | 2022-02-10 16:44:49 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:44:49 +0300 |
commit | 718c552901d703c502ccbefdfc3c9028d608b947 (patch) | |
tree | 46534a98bbefcd7b1f3faa5b52c138ab27db75b7 /contrib/libs/llvm12/lib/CodeGen/TargetInstrInfo.cpp | |
parent | e9656aae26e0358d5378e5b63dcac5c8dbe0e4d0 (diff) | |
download | ydb-718c552901d703c502ccbefdfc3c9028d608b947.tar.gz |
Restoring authorship annotation for <orivej@yandex-team.ru>. Commit 1 of 2.
Diffstat (limited to 'contrib/libs/llvm12/lib/CodeGen/TargetInstrInfo.cpp')
-rw-r--r-- | contrib/libs/llvm12/lib/CodeGen/TargetInstrInfo.cpp | 2770 |
1 files changed, 1385 insertions, 1385 deletions
diff --git a/contrib/libs/llvm12/lib/CodeGen/TargetInstrInfo.cpp b/contrib/libs/llvm12/lib/CodeGen/TargetInstrInfo.cpp index 165860ef1a..6332017e66 100644 --- a/contrib/libs/llvm12/lib/CodeGen/TargetInstrInfo.cpp +++ b/contrib/libs/llvm12/lib/CodeGen/TargetInstrInfo.cpp @@ -1,74 +1,74 @@ -//===-- TargetInstrInfo.cpp - Target Instruction Information --------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// This file implements the TargetInstrInfo class. -// -//===----------------------------------------------------------------------===// - -#include "llvm/CodeGen/TargetInstrInfo.h" -#include "llvm/ADT/StringExtras.h" -#include "llvm/CodeGen/MachineFrameInfo.h" -#include "llvm/CodeGen/MachineInstrBuilder.h" -#include "llvm/CodeGen/MachineMemOperand.h" -#include "llvm/CodeGen/MachineRegisterInfo.h" -#include "llvm/CodeGen/MachineScheduler.h" -#include "llvm/CodeGen/PseudoSourceValue.h" -#include "llvm/CodeGen/ScoreboardHazardRecognizer.h" -#include "llvm/CodeGen/StackMaps.h" -#include "llvm/CodeGen/TargetFrameLowering.h" -#include "llvm/CodeGen/TargetLowering.h" -#include "llvm/CodeGen/TargetRegisterInfo.h" -#include "llvm/CodeGen/TargetSchedule.h" -#include "llvm/IR/DataLayout.h" -#include "llvm/IR/DebugInfoMetadata.h" -#include "llvm/MC/MCAsmInfo.h" -#include "llvm/MC/MCInstrItineraries.h" -#include "llvm/Support/CommandLine.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/raw_ostream.h" -#include "llvm/Target/TargetMachine.h" -#include <cctype> - -using namespace llvm; - -static cl::opt<bool> DisableHazardRecognizer( - "disable-sched-hazard", cl::Hidden, cl::init(false), - cl::desc("Disable hazard detection during preRA scheduling")); - -TargetInstrInfo::~TargetInstrInfo() { -} - -const TargetRegisterClass* -TargetInstrInfo::getRegClass(const MCInstrDesc &MCID, unsigned OpNum, - const TargetRegisterInfo *TRI, - const MachineFunction &MF) const { - if (OpNum >= MCID.getNumOperands()) - return nullptr; - - short RegClass = MCID.OpInfo[OpNum].RegClass; - if (MCID.OpInfo[OpNum].isLookupPtrRegClass()) - return TRI->getPointerRegClass(MF, RegClass); - - // Instructions like INSERT_SUBREG do not have fixed register classes. - if (RegClass < 0) - return nullptr; - - // Otherwise just look it up normally. - return TRI->getRegClass(RegClass); -} - -/// insertNoop - Insert a noop into the instruction stream at the specified -/// point. -void TargetInstrInfo::insertNoop(MachineBasicBlock &MBB, - MachineBasicBlock::iterator MI) const { - llvm_unreachable("Target didn't implement insertNoop!"); -} - +//===-- TargetInstrInfo.cpp - Target Instruction Information --------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the TargetInstrInfo class. +// +//===----------------------------------------------------------------------===// + +#include "llvm/CodeGen/TargetInstrInfo.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/CodeGen/MachineFrameInfo.h" +#include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/CodeGen/MachineMemOperand.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/CodeGen/MachineScheduler.h" +#include "llvm/CodeGen/PseudoSourceValue.h" +#include "llvm/CodeGen/ScoreboardHazardRecognizer.h" +#include "llvm/CodeGen/StackMaps.h" +#include "llvm/CodeGen/TargetFrameLowering.h" +#include "llvm/CodeGen/TargetLowering.h" +#include "llvm/CodeGen/TargetRegisterInfo.h" +#include "llvm/CodeGen/TargetSchedule.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/DebugInfoMetadata.h" +#include "llvm/MC/MCAsmInfo.h" +#include "llvm/MC/MCInstrItineraries.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/Target/TargetMachine.h" +#include <cctype> + +using namespace llvm; + +static cl::opt<bool> DisableHazardRecognizer( + "disable-sched-hazard", cl::Hidden, cl::init(false), + cl::desc("Disable hazard detection during preRA scheduling")); + +TargetInstrInfo::~TargetInstrInfo() { +} + +const TargetRegisterClass* +TargetInstrInfo::getRegClass(const MCInstrDesc &MCID, unsigned OpNum, + const TargetRegisterInfo *TRI, + const MachineFunction &MF) const { + if (OpNum >= MCID.getNumOperands()) + return nullptr; + + short RegClass = MCID.OpInfo[OpNum].RegClass; + if (MCID.OpInfo[OpNum].isLookupPtrRegClass()) + return TRI->getPointerRegClass(MF, RegClass); + + // Instructions like INSERT_SUBREG do not have fixed register classes. + if (RegClass < 0) + return nullptr; + + // Otherwise just look it up normally. + return TRI->getRegClass(RegClass); +} + +/// insertNoop - Insert a noop into the instruction stream at the specified +/// point. +void TargetInstrInfo::insertNoop(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MI) const { + llvm_unreachable("Target didn't implement insertNoop!"); +} + /// insertNoops - Insert noops into the instruction stream at the specified /// point. void TargetInstrInfo::insertNoops(MachineBasicBlock &MBB, @@ -78,475 +78,475 @@ void TargetInstrInfo::insertNoops(MachineBasicBlock &MBB, insertNoop(MBB, MI); } -static bool isAsmComment(const char *Str, const MCAsmInfo &MAI) { - return strncmp(Str, MAI.getCommentString().data(), - MAI.getCommentString().size()) == 0; -} - -/// Measure the specified inline asm to determine an approximation of its -/// length. -/// Comments (which run till the next SeparatorString or newline) do not -/// count as an instruction. -/// Any other non-whitespace text is considered an instruction, with -/// multiple instructions separated by SeparatorString or newlines. -/// Variable-length instructions are not handled here; this function -/// may be overloaded in the target code to do that. -/// We implement a special case of the .space directive which takes only a -/// single integer argument in base 10 that is the size in bytes. This is a -/// restricted form of the GAS directive in that we only interpret -/// simple--i.e. not a logical or arithmetic expression--size values without -/// the optional fill value. This is primarily used for creating arbitrary -/// sized inline asm blocks for testing purposes. -unsigned TargetInstrInfo::getInlineAsmLength( - const char *Str, - const MCAsmInfo &MAI, const TargetSubtargetInfo *STI) const { - // Count the number of instructions in the asm. - bool AtInsnStart = true; - unsigned Length = 0; - const unsigned MaxInstLength = MAI.getMaxInstLength(STI); - for (; *Str; ++Str) { - if (*Str == '\n' || strncmp(Str, MAI.getSeparatorString(), - strlen(MAI.getSeparatorString())) == 0) { - AtInsnStart = true; - } else if (isAsmComment(Str, MAI)) { - // Stop counting as an instruction after a comment until the next - // separator. - AtInsnStart = false; - } - - if (AtInsnStart && !isSpace(static_cast<unsigned char>(*Str))) { - unsigned AddLength = MaxInstLength; - if (strncmp(Str, ".space", 6) == 0) { - char *EStr; - int SpaceSize; - SpaceSize = strtol(Str + 6, &EStr, 10); - SpaceSize = SpaceSize < 0 ? 0 : SpaceSize; - while (*EStr != '\n' && isSpace(static_cast<unsigned char>(*EStr))) - ++EStr; - if (*EStr == '\0' || *EStr == '\n' || - isAsmComment(EStr, MAI)) // Successfully parsed .space argument - AddLength = SpaceSize; - } - Length += AddLength; - AtInsnStart = false; - } - } - - return Length; -} - -/// ReplaceTailWithBranchTo - Delete the instruction OldInst and everything -/// after it, replacing it with an unconditional branch to NewDest. -void -TargetInstrInfo::ReplaceTailWithBranchTo(MachineBasicBlock::iterator Tail, - MachineBasicBlock *NewDest) const { - MachineBasicBlock *MBB = Tail->getParent(); - - // Remove all the old successors of MBB from the CFG. - while (!MBB->succ_empty()) - MBB->removeSuccessor(MBB->succ_begin()); - - // Save off the debug loc before erasing the instruction. - DebugLoc DL = Tail->getDebugLoc(); - - // Update call site info and remove all the dead instructions - // from the end of MBB. - while (Tail != MBB->end()) { - auto MI = Tail++; - if (MI->shouldUpdateCallSiteInfo()) - MBB->getParent()->eraseCallSiteInfo(&*MI); - MBB->erase(MI); - } - - // If MBB isn't immediately before MBB, insert a branch to it. - if (++MachineFunction::iterator(MBB) != MachineFunction::iterator(NewDest)) - insertBranch(*MBB, NewDest, nullptr, SmallVector<MachineOperand, 0>(), DL); - MBB->addSuccessor(NewDest); -} - -MachineInstr *TargetInstrInfo::commuteInstructionImpl(MachineInstr &MI, - bool NewMI, unsigned Idx1, - unsigned Idx2) const { - const MCInstrDesc &MCID = MI.getDesc(); - bool HasDef = MCID.getNumDefs(); - if (HasDef && !MI.getOperand(0).isReg()) - // No idea how to commute this instruction. Target should implement its own. - return nullptr; - - unsigned CommutableOpIdx1 = Idx1; (void)CommutableOpIdx1; - unsigned CommutableOpIdx2 = Idx2; (void)CommutableOpIdx2; - assert(findCommutedOpIndices(MI, CommutableOpIdx1, CommutableOpIdx2) && - CommutableOpIdx1 == Idx1 && CommutableOpIdx2 == Idx2 && - "TargetInstrInfo::CommuteInstructionImpl(): not commutable operands."); - assert(MI.getOperand(Idx1).isReg() && MI.getOperand(Idx2).isReg() && - "This only knows how to commute register operands so far"); - - Register Reg0 = HasDef ? MI.getOperand(0).getReg() : Register(); - Register Reg1 = MI.getOperand(Idx1).getReg(); - Register Reg2 = MI.getOperand(Idx2).getReg(); - unsigned SubReg0 = HasDef ? MI.getOperand(0).getSubReg() : 0; - unsigned SubReg1 = MI.getOperand(Idx1).getSubReg(); - unsigned SubReg2 = MI.getOperand(Idx2).getSubReg(); - bool Reg1IsKill = MI.getOperand(Idx1).isKill(); - bool Reg2IsKill = MI.getOperand(Idx2).isKill(); - bool Reg1IsUndef = MI.getOperand(Idx1).isUndef(); - bool Reg2IsUndef = MI.getOperand(Idx2).isUndef(); - bool Reg1IsInternal = MI.getOperand(Idx1).isInternalRead(); - bool Reg2IsInternal = MI.getOperand(Idx2).isInternalRead(); - // Avoid calling isRenamable for virtual registers since we assert that - // renamable property is only queried/set for physical registers. - bool Reg1IsRenamable = Register::isPhysicalRegister(Reg1) - ? MI.getOperand(Idx1).isRenamable() - : false; - bool Reg2IsRenamable = Register::isPhysicalRegister(Reg2) - ? MI.getOperand(Idx2).isRenamable() - : false; - // If destination is tied to either of the commuted source register, then - // it must be updated. - if (HasDef && Reg0 == Reg1 && - MI.getDesc().getOperandConstraint(Idx1, MCOI::TIED_TO) == 0) { - Reg2IsKill = false; - Reg0 = Reg2; - SubReg0 = SubReg2; - } else if (HasDef && Reg0 == Reg2 && - MI.getDesc().getOperandConstraint(Idx2, MCOI::TIED_TO) == 0) { - Reg1IsKill = false; - Reg0 = Reg1; - SubReg0 = SubReg1; - } - - MachineInstr *CommutedMI = nullptr; - if (NewMI) { - // Create a new instruction. - MachineFunction &MF = *MI.getMF(); - CommutedMI = MF.CloneMachineInstr(&MI); - } else { - CommutedMI = &MI; - } - - if (HasDef) { - CommutedMI->getOperand(0).setReg(Reg0); - CommutedMI->getOperand(0).setSubReg(SubReg0); - } - CommutedMI->getOperand(Idx2).setReg(Reg1); - CommutedMI->getOperand(Idx1).setReg(Reg2); - CommutedMI->getOperand(Idx2).setSubReg(SubReg1); - CommutedMI->getOperand(Idx1).setSubReg(SubReg2); - CommutedMI->getOperand(Idx2).setIsKill(Reg1IsKill); - CommutedMI->getOperand(Idx1).setIsKill(Reg2IsKill); - CommutedMI->getOperand(Idx2).setIsUndef(Reg1IsUndef); - CommutedMI->getOperand(Idx1).setIsUndef(Reg2IsUndef); - CommutedMI->getOperand(Idx2).setIsInternalRead(Reg1IsInternal); - CommutedMI->getOperand(Idx1).setIsInternalRead(Reg2IsInternal); - // Avoid calling setIsRenamable for virtual registers since we assert that - // renamable property is only queried/set for physical registers. - if (Register::isPhysicalRegister(Reg1)) - CommutedMI->getOperand(Idx2).setIsRenamable(Reg1IsRenamable); - if (Register::isPhysicalRegister(Reg2)) - CommutedMI->getOperand(Idx1).setIsRenamable(Reg2IsRenamable); - return CommutedMI; -} - -MachineInstr *TargetInstrInfo::commuteInstruction(MachineInstr &MI, bool NewMI, - unsigned OpIdx1, - unsigned OpIdx2) const { - // If OpIdx1 or OpIdx2 is not specified, then this method is free to choose - // any commutable operand, which is done in findCommutedOpIndices() method - // called below. - if ((OpIdx1 == CommuteAnyOperandIndex || OpIdx2 == CommuteAnyOperandIndex) && - !findCommutedOpIndices(MI, OpIdx1, OpIdx2)) { - assert(MI.isCommutable() && - "Precondition violation: MI must be commutable."); - return nullptr; - } - return commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2); -} - -bool TargetInstrInfo::fixCommutedOpIndices(unsigned &ResultIdx1, - unsigned &ResultIdx2, - unsigned CommutableOpIdx1, - unsigned CommutableOpIdx2) { - if (ResultIdx1 == CommuteAnyOperandIndex && - ResultIdx2 == CommuteAnyOperandIndex) { - ResultIdx1 = CommutableOpIdx1; - ResultIdx2 = CommutableOpIdx2; - } else if (ResultIdx1 == CommuteAnyOperandIndex) { - if (ResultIdx2 == CommutableOpIdx1) - ResultIdx1 = CommutableOpIdx2; - else if (ResultIdx2 == CommutableOpIdx2) - ResultIdx1 = CommutableOpIdx1; - else - return false; - } else if (ResultIdx2 == CommuteAnyOperandIndex) { - if (ResultIdx1 == CommutableOpIdx1) - ResultIdx2 = CommutableOpIdx2; - else if (ResultIdx1 == CommutableOpIdx2) - ResultIdx2 = CommutableOpIdx1; - else - return false; - } else - // Check that the result operand indices match the given commutable - // operand indices. - return (ResultIdx1 == CommutableOpIdx1 && ResultIdx2 == CommutableOpIdx2) || - (ResultIdx1 == CommutableOpIdx2 && ResultIdx2 == CommutableOpIdx1); - - return true; -} - -bool TargetInstrInfo::findCommutedOpIndices(const MachineInstr &MI, - unsigned &SrcOpIdx1, - unsigned &SrcOpIdx2) const { - assert(!MI.isBundle() && - "TargetInstrInfo::findCommutedOpIndices() can't handle bundles"); - - const MCInstrDesc &MCID = MI.getDesc(); - if (!MCID.isCommutable()) - return false; - - // This assumes v0 = op v1, v2 and commuting would swap v1 and v2. If this - // is not true, then the target must implement this. - unsigned CommutableOpIdx1 = MCID.getNumDefs(); - unsigned CommutableOpIdx2 = CommutableOpIdx1 + 1; - if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, - CommutableOpIdx1, CommutableOpIdx2)) - return false; - - if (!MI.getOperand(SrcOpIdx1).isReg() || !MI.getOperand(SrcOpIdx2).isReg()) - // No idea. - return false; - return true; -} - -bool TargetInstrInfo::isUnpredicatedTerminator(const MachineInstr &MI) const { - if (!MI.isTerminator()) return false; - - // Conditional branch is a special case. - if (MI.isBranch() && !MI.isBarrier()) - return true; - if (!MI.isPredicable()) - return true; - return !isPredicated(MI); -} - -bool TargetInstrInfo::PredicateInstruction( - MachineInstr &MI, ArrayRef<MachineOperand> Pred) const { - bool MadeChange = false; - - assert(!MI.isBundle() && - "TargetInstrInfo::PredicateInstruction() can't handle bundles"); - - const MCInstrDesc &MCID = MI.getDesc(); - if (!MI.isPredicable()) - return false; - - for (unsigned j = 0, i = 0, e = MI.getNumOperands(); i != e; ++i) { - if (MCID.OpInfo[i].isPredicate()) { - MachineOperand &MO = MI.getOperand(i); - if (MO.isReg()) { - MO.setReg(Pred[j].getReg()); - MadeChange = true; - } else if (MO.isImm()) { - MO.setImm(Pred[j].getImm()); - MadeChange = true; - } else if (MO.isMBB()) { - MO.setMBB(Pred[j].getMBB()); - MadeChange = true; - } - ++j; - } - } - return MadeChange; -} - -bool TargetInstrInfo::hasLoadFromStackSlot( - const MachineInstr &MI, - SmallVectorImpl<const MachineMemOperand *> &Accesses) const { - size_t StartSize = Accesses.size(); - for (MachineInstr::mmo_iterator o = MI.memoperands_begin(), - oe = MI.memoperands_end(); - o != oe; ++o) { - if ((*o)->isLoad() && - dyn_cast_or_null<FixedStackPseudoSourceValue>((*o)->getPseudoValue())) - Accesses.push_back(*o); - } - return Accesses.size() != StartSize; -} - -bool TargetInstrInfo::hasStoreToStackSlot( - const MachineInstr &MI, - SmallVectorImpl<const MachineMemOperand *> &Accesses) const { - size_t StartSize = Accesses.size(); - for (MachineInstr::mmo_iterator o = MI.memoperands_begin(), - oe = MI.memoperands_end(); - o != oe; ++o) { - if ((*o)->isStore() && - dyn_cast_or_null<FixedStackPseudoSourceValue>((*o)->getPseudoValue())) - Accesses.push_back(*o); - } - return Accesses.size() != StartSize; -} - -bool TargetInstrInfo::getStackSlotRange(const TargetRegisterClass *RC, - unsigned SubIdx, unsigned &Size, - unsigned &Offset, - const MachineFunction &MF) const { - const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); - if (!SubIdx) { - Size = TRI->getSpillSize(*RC); - Offset = 0; - return true; - } - unsigned BitSize = TRI->getSubRegIdxSize(SubIdx); - // Convert bit size to byte size. - if (BitSize % 8) - return false; - - int BitOffset = TRI->getSubRegIdxOffset(SubIdx); - if (BitOffset < 0 || BitOffset % 8) - return false; - - Size = BitSize / 8; - Offset = (unsigned)BitOffset / 8; - - assert(TRI->getSpillSize(*RC) >= (Offset + Size) && "bad subregister range"); - - if (!MF.getDataLayout().isLittleEndian()) { - Offset = TRI->getSpillSize(*RC) - (Offset + Size); - } - return true; -} - -void TargetInstrInfo::reMaterialize(MachineBasicBlock &MBB, - MachineBasicBlock::iterator I, - Register DestReg, unsigned SubIdx, - const MachineInstr &Orig, - const TargetRegisterInfo &TRI) const { - MachineInstr *MI = MBB.getParent()->CloneMachineInstr(&Orig); - MI->substituteRegister(MI->getOperand(0).getReg(), DestReg, SubIdx, TRI); - MBB.insert(I, MI); -} - -bool TargetInstrInfo::produceSameValue(const MachineInstr &MI0, - const MachineInstr &MI1, - const MachineRegisterInfo *MRI) const { - return MI0.isIdenticalTo(MI1, MachineInstr::IgnoreVRegDefs); -} - -MachineInstr &TargetInstrInfo::duplicate(MachineBasicBlock &MBB, - MachineBasicBlock::iterator InsertBefore, const MachineInstr &Orig) const { - assert(!Orig.isNotDuplicable() && "Instruction cannot be duplicated"); - MachineFunction &MF = *MBB.getParent(); - return MF.CloneMachineInstrBundle(MBB, InsertBefore, Orig); -} - -// If the COPY instruction in MI can be folded to a stack operation, return -// the register class to use. -static const TargetRegisterClass *canFoldCopy(const MachineInstr &MI, - unsigned FoldIdx) { - assert(MI.isCopy() && "MI must be a COPY instruction"); - if (MI.getNumOperands() != 2) - return nullptr; - assert(FoldIdx<2 && "FoldIdx refers no nonexistent operand"); - - const MachineOperand &FoldOp = MI.getOperand(FoldIdx); - const MachineOperand &LiveOp = MI.getOperand(1 - FoldIdx); - - if (FoldOp.getSubReg() || LiveOp.getSubReg()) - return nullptr; - - Register FoldReg = FoldOp.getReg(); - Register LiveReg = LiveOp.getReg(); - - assert(Register::isVirtualRegister(FoldReg) && "Cannot fold physregs"); - - const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo(); - const TargetRegisterClass *RC = MRI.getRegClass(FoldReg); - - if (Register::isPhysicalRegister(LiveOp.getReg())) - return RC->contains(LiveOp.getReg()) ? RC : nullptr; - - if (RC->hasSubClassEq(MRI.getRegClass(LiveReg))) - return RC; - - // FIXME: Allow folding when register classes are memory compatible. - return nullptr; -} - -void TargetInstrInfo::getNoop(MCInst &NopInst) const { - llvm_unreachable("Not implemented"); -} - -static MachineInstr *foldPatchpoint(MachineFunction &MF, MachineInstr &MI, - ArrayRef<unsigned> Ops, int FrameIndex, - const TargetInstrInfo &TII) { - unsigned StartIdx = 0; +static bool isAsmComment(const char *Str, const MCAsmInfo &MAI) { + return strncmp(Str, MAI.getCommentString().data(), + MAI.getCommentString().size()) == 0; +} + +/// Measure the specified inline asm to determine an approximation of its +/// length. +/// Comments (which run till the next SeparatorString or newline) do not +/// count as an instruction. +/// Any other non-whitespace text is considered an instruction, with +/// multiple instructions separated by SeparatorString or newlines. +/// Variable-length instructions are not handled here; this function +/// may be overloaded in the target code to do that. +/// We implement a special case of the .space directive which takes only a +/// single integer argument in base 10 that is the size in bytes. This is a +/// restricted form of the GAS directive in that we only interpret +/// simple--i.e. not a logical or arithmetic expression--size values without +/// the optional fill value. This is primarily used for creating arbitrary +/// sized inline asm blocks for testing purposes. +unsigned TargetInstrInfo::getInlineAsmLength( + const char *Str, + const MCAsmInfo &MAI, const TargetSubtargetInfo *STI) const { + // Count the number of instructions in the asm. + bool AtInsnStart = true; + unsigned Length = 0; + const unsigned MaxInstLength = MAI.getMaxInstLength(STI); + for (; *Str; ++Str) { + if (*Str == '\n' || strncmp(Str, MAI.getSeparatorString(), + strlen(MAI.getSeparatorString())) == 0) { + AtInsnStart = true; + } else if (isAsmComment(Str, MAI)) { + // Stop counting as an instruction after a comment until the next + // separator. + AtInsnStart = false; + } + + if (AtInsnStart && !isSpace(static_cast<unsigned char>(*Str))) { + unsigned AddLength = MaxInstLength; + if (strncmp(Str, ".space", 6) == 0) { + char *EStr; + int SpaceSize; + SpaceSize = strtol(Str + 6, &EStr, 10); + SpaceSize = SpaceSize < 0 ? 0 : SpaceSize; + while (*EStr != '\n' && isSpace(static_cast<unsigned char>(*EStr))) + ++EStr; + if (*EStr == '\0' || *EStr == '\n' || + isAsmComment(EStr, MAI)) // Successfully parsed .space argument + AddLength = SpaceSize; + } + Length += AddLength; + AtInsnStart = false; + } + } + + return Length; +} + +/// ReplaceTailWithBranchTo - Delete the instruction OldInst and everything +/// after it, replacing it with an unconditional branch to NewDest. +void +TargetInstrInfo::ReplaceTailWithBranchTo(MachineBasicBlock::iterator Tail, + MachineBasicBlock *NewDest) const { + MachineBasicBlock *MBB = Tail->getParent(); + + // Remove all the old successors of MBB from the CFG. + while (!MBB->succ_empty()) + MBB->removeSuccessor(MBB->succ_begin()); + + // Save off the debug loc before erasing the instruction. + DebugLoc DL = Tail->getDebugLoc(); + + // Update call site info and remove all the dead instructions + // from the end of MBB. + while (Tail != MBB->end()) { + auto MI = Tail++; + if (MI->shouldUpdateCallSiteInfo()) + MBB->getParent()->eraseCallSiteInfo(&*MI); + MBB->erase(MI); + } + + // If MBB isn't immediately before MBB, insert a branch to it. + if (++MachineFunction::iterator(MBB) != MachineFunction::iterator(NewDest)) + insertBranch(*MBB, NewDest, nullptr, SmallVector<MachineOperand, 0>(), DL); + MBB->addSuccessor(NewDest); +} + +MachineInstr *TargetInstrInfo::commuteInstructionImpl(MachineInstr &MI, + bool NewMI, unsigned Idx1, + unsigned Idx2) const { + const MCInstrDesc &MCID = MI.getDesc(); + bool HasDef = MCID.getNumDefs(); + if (HasDef && !MI.getOperand(0).isReg()) + // No idea how to commute this instruction. Target should implement its own. + return nullptr; + + unsigned CommutableOpIdx1 = Idx1; (void)CommutableOpIdx1; + unsigned CommutableOpIdx2 = Idx2; (void)CommutableOpIdx2; + assert(findCommutedOpIndices(MI, CommutableOpIdx1, CommutableOpIdx2) && + CommutableOpIdx1 == Idx1 && CommutableOpIdx2 == Idx2 && + "TargetInstrInfo::CommuteInstructionImpl(): not commutable operands."); + assert(MI.getOperand(Idx1).isReg() && MI.getOperand(Idx2).isReg() && + "This only knows how to commute register operands so far"); + + Register Reg0 = HasDef ? MI.getOperand(0).getReg() : Register(); + Register Reg1 = MI.getOperand(Idx1).getReg(); + Register Reg2 = MI.getOperand(Idx2).getReg(); + unsigned SubReg0 = HasDef ? MI.getOperand(0).getSubReg() : 0; + unsigned SubReg1 = MI.getOperand(Idx1).getSubReg(); + unsigned SubReg2 = MI.getOperand(Idx2).getSubReg(); + bool Reg1IsKill = MI.getOperand(Idx1).isKill(); + bool Reg2IsKill = MI.getOperand(Idx2).isKill(); + bool Reg1IsUndef = MI.getOperand(Idx1).isUndef(); + bool Reg2IsUndef = MI.getOperand(Idx2).isUndef(); + bool Reg1IsInternal = MI.getOperand(Idx1).isInternalRead(); + bool Reg2IsInternal = MI.getOperand(Idx2).isInternalRead(); + // Avoid calling isRenamable for virtual registers since we assert that + // renamable property is only queried/set for physical registers. + bool Reg1IsRenamable = Register::isPhysicalRegister(Reg1) + ? MI.getOperand(Idx1).isRenamable() + : false; + bool Reg2IsRenamable = Register::isPhysicalRegister(Reg2) + ? MI.getOperand(Idx2).isRenamable() + : false; + // If destination is tied to either of the commuted source register, then + // it must be updated. + if (HasDef && Reg0 == Reg1 && + MI.getDesc().getOperandConstraint(Idx1, MCOI::TIED_TO) == 0) { + Reg2IsKill = false; + Reg0 = Reg2; + SubReg0 = SubReg2; + } else if (HasDef && Reg0 == Reg2 && + MI.getDesc().getOperandConstraint(Idx2, MCOI::TIED_TO) == 0) { + Reg1IsKill = false; + Reg0 = Reg1; + SubReg0 = SubReg1; + } + + MachineInstr *CommutedMI = nullptr; + if (NewMI) { + // Create a new instruction. + MachineFunction &MF = *MI.getMF(); + CommutedMI = MF.CloneMachineInstr(&MI); + } else { + CommutedMI = &MI; + } + + if (HasDef) { + CommutedMI->getOperand(0).setReg(Reg0); + CommutedMI->getOperand(0).setSubReg(SubReg0); + } + CommutedMI->getOperand(Idx2).setReg(Reg1); + CommutedMI->getOperand(Idx1).setReg(Reg2); + CommutedMI->getOperand(Idx2).setSubReg(SubReg1); + CommutedMI->getOperand(Idx1).setSubReg(SubReg2); + CommutedMI->getOperand(Idx2).setIsKill(Reg1IsKill); + CommutedMI->getOperand(Idx1).setIsKill(Reg2IsKill); + CommutedMI->getOperand(Idx2).setIsUndef(Reg1IsUndef); + CommutedMI->getOperand(Idx1).setIsUndef(Reg2IsUndef); + CommutedMI->getOperand(Idx2).setIsInternalRead(Reg1IsInternal); + CommutedMI->getOperand(Idx1).setIsInternalRead(Reg2IsInternal); + // Avoid calling setIsRenamable for virtual registers since we assert that + // renamable property is only queried/set for physical registers. + if (Register::isPhysicalRegister(Reg1)) + CommutedMI->getOperand(Idx2).setIsRenamable(Reg1IsRenamable); + if (Register::isPhysicalRegister(Reg2)) + CommutedMI->getOperand(Idx1).setIsRenamable(Reg2IsRenamable); + return CommutedMI; +} + +MachineInstr *TargetInstrInfo::commuteInstruction(MachineInstr &MI, bool NewMI, + unsigned OpIdx1, + unsigned OpIdx2) const { + // If OpIdx1 or OpIdx2 is not specified, then this method is free to choose + // any commutable operand, which is done in findCommutedOpIndices() method + // called below. + if ((OpIdx1 == CommuteAnyOperandIndex || OpIdx2 == CommuteAnyOperandIndex) && + !findCommutedOpIndices(MI, OpIdx1, OpIdx2)) { + assert(MI.isCommutable() && + "Precondition violation: MI must be commutable."); + return nullptr; + } + return commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2); +} + +bool TargetInstrInfo::fixCommutedOpIndices(unsigned &ResultIdx1, + unsigned &ResultIdx2, + unsigned CommutableOpIdx1, + unsigned CommutableOpIdx2) { + if (ResultIdx1 == CommuteAnyOperandIndex && + ResultIdx2 == CommuteAnyOperandIndex) { + ResultIdx1 = CommutableOpIdx1; + ResultIdx2 = CommutableOpIdx2; + } else if (ResultIdx1 == CommuteAnyOperandIndex) { + if (ResultIdx2 == CommutableOpIdx1) + ResultIdx1 = CommutableOpIdx2; + else if (ResultIdx2 == CommutableOpIdx2) + ResultIdx1 = CommutableOpIdx1; + else + return false; + } else if (ResultIdx2 == CommuteAnyOperandIndex) { + if (ResultIdx1 == CommutableOpIdx1) + ResultIdx2 = CommutableOpIdx2; + else if (ResultIdx1 == CommutableOpIdx2) + ResultIdx2 = CommutableOpIdx1; + else + return false; + } else + // Check that the result operand indices match the given commutable + // operand indices. + return (ResultIdx1 == CommutableOpIdx1 && ResultIdx2 == CommutableOpIdx2) || + (ResultIdx1 == CommutableOpIdx2 && ResultIdx2 == CommutableOpIdx1); + + return true; +} + +bool TargetInstrInfo::findCommutedOpIndices(const MachineInstr &MI, + unsigned &SrcOpIdx1, + unsigned &SrcOpIdx2) const { + assert(!MI.isBundle() && + "TargetInstrInfo::findCommutedOpIndices() can't handle bundles"); + + const MCInstrDesc &MCID = MI.getDesc(); + if (!MCID.isCommutable()) + return false; + + // This assumes v0 = op v1, v2 and commuting would swap v1 and v2. If this + // is not true, then the target must implement this. + unsigned CommutableOpIdx1 = MCID.getNumDefs(); + unsigned CommutableOpIdx2 = CommutableOpIdx1 + 1; + if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, + CommutableOpIdx1, CommutableOpIdx2)) + return false; + + if (!MI.getOperand(SrcOpIdx1).isReg() || !MI.getOperand(SrcOpIdx2).isReg()) + // No idea. + return false; + return true; +} + +bool TargetInstrInfo::isUnpredicatedTerminator(const MachineInstr &MI) const { + if (!MI.isTerminator()) return false; + + // Conditional branch is a special case. + if (MI.isBranch() && !MI.isBarrier()) + return true; + if (!MI.isPredicable()) + return true; + return !isPredicated(MI); +} + +bool TargetInstrInfo::PredicateInstruction( + MachineInstr &MI, ArrayRef<MachineOperand> Pred) const { + bool MadeChange = false; + + assert(!MI.isBundle() && + "TargetInstrInfo::PredicateInstruction() can't handle bundles"); + + const MCInstrDesc &MCID = MI.getDesc(); + if (!MI.isPredicable()) + return false; + + for (unsigned j = 0, i = 0, e = MI.getNumOperands(); i != e; ++i) { + if (MCID.OpInfo[i].isPredicate()) { + MachineOperand &MO = MI.getOperand(i); + if (MO.isReg()) { + MO.setReg(Pred[j].getReg()); + MadeChange = true; + } else if (MO.isImm()) { + MO.setImm(Pred[j].getImm()); + MadeChange = true; + } else if (MO.isMBB()) { + MO.setMBB(Pred[j].getMBB()); + MadeChange = true; + } + ++j; + } + } + return MadeChange; +} + +bool TargetInstrInfo::hasLoadFromStackSlot( + const MachineInstr &MI, + SmallVectorImpl<const MachineMemOperand *> &Accesses) const { + size_t StartSize = Accesses.size(); + for (MachineInstr::mmo_iterator o = MI.memoperands_begin(), + oe = MI.memoperands_end(); + o != oe; ++o) { + if ((*o)->isLoad() && + dyn_cast_or_null<FixedStackPseudoSourceValue>((*o)->getPseudoValue())) + Accesses.push_back(*o); + } + return Accesses.size() != StartSize; +} + +bool TargetInstrInfo::hasStoreToStackSlot( + const MachineInstr &MI, + SmallVectorImpl<const MachineMemOperand *> &Accesses) const { + size_t StartSize = Accesses.size(); + for (MachineInstr::mmo_iterator o = MI.memoperands_begin(), + oe = MI.memoperands_end(); + o != oe; ++o) { + if ((*o)->isStore() && + dyn_cast_or_null<FixedStackPseudoSourceValue>((*o)->getPseudoValue())) + Accesses.push_back(*o); + } + return Accesses.size() != StartSize; +} + +bool TargetInstrInfo::getStackSlotRange(const TargetRegisterClass *RC, + unsigned SubIdx, unsigned &Size, + unsigned &Offset, + const MachineFunction &MF) const { + const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); + if (!SubIdx) { + Size = TRI->getSpillSize(*RC); + Offset = 0; + return true; + } + unsigned BitSize = TRI->getSubRegIdxSize(SubIdx); + // Convert bit size to byte size. + if (BitSize % 8) + return false; + + int BitOffset = TRI->getSubRegIdxOffset(SubIdx); + if (BitOffset < 0 || BitOffset % 8) + return false; + + Size = BitSize / 8; + Offset = (unsigned)BitOffset / 8; + + assert(TRI->getSpillSize(*RC) >= (Offset + Size) && "bad subregister range"); + + if (!MF.getDataLayout().isLittleEndian()) { + Offset = TRI->getSpillSize(*RC) - (Offset + Size); + } + return true; +} + +void TargetInstrInfo::reMaterialize(MachineBasicBlock &MBB, + MachineBasicBlock::iterator I, + Register DestReg, unsigned SubIdx, + const MachineInstr &Orig, + const TargetRegisterInfo &TRI) const { + MachineInstr *MI = MBB.getParent()->CloneMachineInstr(&Orig); + MI->substituteRegister(MI->getOperand(0).getReg(), DestReg, SubIdx, TRI); + MBB.insert(I, MI); +} + +bool TargetInstrInfo::produceSameValue(const MachineInstr &MI0, + const MachineInstr &MI1, + const MachineRegisterInfo *MRI) const { + return MI0.isIdenticalTo(MI1, MachineInstr::IgnoreVRegDefs); +} + +MachineInstr &TargetInstrInfo::duplicate(MachineBasicBlock &MBB, + MachineBasicBlock::iterator InsertBefore, const MachineInstr &Orig) const { + assert(!Orig.isNotDuplicable() && "Instruction cannot be duplicated"); + MachineFunction &MF = *MBB.getParent(); + return MF.CloneMachineInstrBundle(MBB, InsertBefore, Orig); +} + +// If the COPY instruction in MI can be folded to a stack operation, return +// the register class to use. +static const TargetRegisterClass *canFoldCopy(const MachineInstr &MI, + unsigned FoldIdx) { + assert(MI.isCopy() && "MI must be a COPY instruction"); + if (MI.getNumOperands() != 2) + return nullptr; + assert(FoldIdx<2 && "FoldIdx refers no nonexistent operand"); + + const MachineOperand &FoldOp = MI.getOperand(FoldIdx); + const MachineOperand &LiveOp = MI.getOperand(1 - FoldIdx); + + if (FoldOp.getSubReg() || LiveOp.getSubReg()) + return nullptr; + + Register FoldReg = FoldOp.getReg(); + Register LiveReg = LiveOp.getReg(); + + assert(Register::isVirtualRegister(FoldReg) && "Cannot fold physregs"); + + const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo(); + const TargetRegisterClass *RC = MRI.getRegClass(FoldReg); + + if (Register::isPhysicalRegister(LiveOp.getReg())) + return RC->contains(LiveOp.getReg()) ? RC : nullptr; + + if (RC->hasSubClassEq(MRI.getRegClass(LiveReg))) + return RC; + + // FIXME: Allow folding when register classes are memory compatible. + return nullptr; +} + +void TargetInstrInfo::getNoop(MCInst &NopInst) const { + llvm_unreachable("Not implemented"); +} + +static MachineInstr *foldPatchpoint(MachineFunction &MF, MachineInstr &MI, + ArrayRef<unsigned> Ops, int FrameIndex, + const TargetInstrInfo &TII) { + unsigned StartIdx = 0; unsigned NumDefs = 0; - switch (MI.getOpcode()) { - case TargetOpcode::STACKMAP: { - // StackMapLiveValues are foldable - StartIdx = StackMapOpers(&MI).getVarIdx(); - break; - } - case TargetOpcode::PATCHPOINT: { - // For PatchPoint, the call args are not foldable (even if reported in the - // stackmap e.g. via anyregcc). - StartIdx = PatchPointOpers(&MI).getVarIdx(); - break; - } - case TargetOpcode::STATEPOINT: { - // For statepoints, fold deopt and gc arguments, but not call arguments. - StartIdx = StatepointOpers(&MI).getVarIdx(); + switch (MI.getOpcode()) { + case TargetOpcode::STACKMAP: { + // StackMapLiveValues are foldable + StartIdx = StackMapOpers(&MI).getVarIdx(); + break; + } + case TargetOpcode::PATCHPOINT: { + // For PatchPoint, the call args are not foldable (even if reported in the + // stackmap e.g. via anyregcc). + StartIdx = PatchPointOpers(&MI).getVarIdx(); + break; + } + case TargetOpcode::STATEPOINT: { + // For statepoints, fold deopt and gc arguments, but not call arguments. + StartIdx = StatepointOpers(&MI).getVarIdx(); NumDefs = MI.getNumDefs(); - break; - } - default: - llvm_unreachable("unexpected stackmap opcode"); - } - + break; + } + default: + llvm_unreachable("unexpected stackmap opcode"); + } + unsigned DefToFoldIdx = MI.getNumOperands(); - // Return false if any operands requested for folding are not foldable (not - // part of the stackmap's live values). - for (unsigned Op : Ops) { + // Return false if any operands requested for folding are not foldable (not + // part of the stackmap's live values). + for (unsigned Op : Ops) { if (Op < NumDefs) { assert(DefToFoldIdx == MI.getNumOperands() && "Folding multiple defs"); DefToFoldIdx = Op; } else if (Op < StartIdx) { - return nullptr; + return nullptr; } if (MI.getOperand(Op).isTied()) return nullptr; - } - - MachineInstr *NewMI = - MF.CreateMachineInstr(TII.get(MI.getOpcode()), MI.getDebugLoc(), true); - MachineInstrBuilder MIB(MF, NewMI); - - // No need to fold return, the meta data, and function arguments - for (unsigned i = 0; i < StartIdx; ++i) + } + + MachineInstr *NewMI = + MF.CreateMachineInstr(TII.get(MI.getOpcode()), MI.getDebugLoc(), true); + MachineInstrBuilder MIB(MF, NewMI); + + // No need to fold return, the meta data, and function arguments + for (unsigned i = 0; i < StartIdx; ++i) if (i != DefToFoldIdx) MIB.add(MI.getOperand(i)); - + for (unsigned i = StartIdx, e = MI.getNumOperands(); i < e; ++i) { - MachineOperand &MO = MI.getOperand(i); + MachineOperand &MO = MI.getOperand(i); unsigned TiedTo = e; (void)MI.isRegTiedToDefOperand(i, &TiedTo); - if (is_contained(Ops, i)) { + if (is_contained(Ops, i)) { assert(TiedTo == e && "Cannot fold tied operands"); - unsigned SpillSize; - unsigned SpillOffset; - // Compute the spill slot size and offset. - const TargetRegisterClass *RC = - MF.getRegInfo().getRegClass(MO.getReg()); - bool Valid = - TII.getStackSlotRange(RC, MO.getSubReg(), SpillSize, SpillOffset, MF); - if (!Valid) - report_fatal_error("cannot spill patchpoint subregister operand"); - MIB.addImm(StackMaps::IndirectMemRefOp); - MIB.addImm(SpillSize); - MIB.addFrameIndex(FrameIndex); - MIB.addImm(SpillOffset); + unsigned SpillSize; + unsigned SpillOffset; + // Compute the spill slot size and offset. + const TargetRegisterClass *RC = + MF.getRegInfo().getRegClass(MO.getReg()); + bool Valid = + TII.getStackSlotRange(RC, MO.getSubReg(), SpillSize, SpillOffset, MF); + if (!Valid) + report_fatal_error("cannot spill patchpoint subregister operand"); + MIB.addImm(StackMaps::IndirectMemRefOp); + MIB.addImm(SpillSize); + MIB.addFrameIndex(FrameIndex); + MIB.addImm(SpillOffset); } else { MIB.add(MO); if (TiedTo < e) { @@ -555,868 +555,868 @@ static MachineInstr *foldPatchpoint(MachineFunction &MF, MachineInstr &MI, --TiedTo; NewMI->tieOperands(TiedTo, NewMI->getNumOperands() - 1); } - } - } - return NewMI; -} - -MachineInstr *TargetInstrInfo::foldMemoryOperand(MachineInstr &MI, - ArrayRef<unsigned> Ops, int FI, - LiveIntervals *LIS, - VirtRegMap *VRM) const { - auto Flags = MachineMemOperand::MONone; - for (unsigned OpIdx : Ops) - Flags |= MI.getOperand(OpIdx).isDef() ? MachineMemOperand::MOStore - : MachineMemOperand::MOLoad; - - MachineBasicBlock *MBB = MI.getParent(); - assert(MBB && "foldMemoryOperand needs an inserted instruction"); - MachineFunction &MF = *MBB->getParent(); - - // If we're not folding a load into a subreg, the size of the load is the - // size of the spill slot. But if we are, we need to figure out what the - // actual load size is. - int64_t MemSize = 0; - const MachineFrameInfo &MFI = MF.getFrameInfo(); - const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); - - if (Flags & MachineMemOperand::MOStore) { - MemSize = MFI.getObjectSize(FI); - } else { - for (unsigned OpIdx : Ops) { - int64_t OpSize = MFI.getObjectSize(FI); - - if (auto SubReg = MI.getOperand(OpIdx).getSubReg()) { - unsigned SubRegSize = TRI->getSubRegIdxSize(SubReg); - if (SubRegSize > 0 && !(SubRegSize % 8)) - OpSize = SubRegSize / 8; - } - - MemSize = std::max(MemSize, OpSize); - } - } - - assert(MemSize && "Did not expect a zero-sized stack slot"); - - MachineInstr *NewMI = nullptr; - - if (MI.getOpcode() == TargetOpcode::STACKMAP || - MI.getOpcode() == TargetOpcode::PATCHPOINT || - MI.getOpcode() == TargetOpcode::STATEPOINT) { - // Fold stackmap/patchpoint. - NewMI = foldPatchpoint(MF, MI, Ops, FI, *this); - if (NewMI) - MBB->insert(MI, NewMI); - } else { - // Ask the target to do the actual folding. - NewMI = foldMemoryOperandImpl(MF, MI, Ops, MI, FI, LIS, VRM); - } - - if (NewMI) { - NewMI->setMemRefs(MF, MI.memoperands()); - // Add a memory operand, foldMemoryOperandImpl doesn't do that. - assert((!(Flags & MachineMemOperand::MOStore) || - NewMI->mayStore()) && - "Folded a def to a non-store!"); - assert((!(Flags & MachineMemOperand::MOLoad) || - NewMI->mayLoad()) && - "Folded a use to a non-load!"); - assert(MFI.getObjectOffset(FI) != -1); - MachineMemOperand *MMO = - MF.getMachineMemOperand(MachinePointerInfo::getFixedStack(MF, FI), - Flags, MemSize, MFI.getObjectAlign(FI)); - NewMI->addMemOperand(MF, MMO); - - // The pass "x86 speculative load hardening" always attaches symbols to - // call instructions. We need copy it form old instruction. - NewMI->cloneInstrSymbols(MF, MI); - - return NewMI; - } - - // Straight COPY may fold as load/store. - if (!MI.isCopy() || Ops.size() != 1) - return nullptr; - - const TargetRegisterClass *RC = canFoldCopy(MI, Ops[0]); - if (!RC) - return nullptr; - - const MachineOperand &MO = MI.getOperand(1 - Ops[0]); - MachineBasicBlock::iterator Pos = MI; - - if (Flags == MachineMemOperand::MOStore) - storeRegToStackSlot(*MBB, Pos, MO.getReg(), MO.isKill(), FI, RC, TRI); - else - loadRegFromStackSlot(*MBB, Pos, MO.getReg(), FI, RC, TRI); - return &*--Pos; -} - -MachineInstr *TargetInstrInfo::foldMemoryOperand(MachineInstr &MI, - ArrayRef<unsigned> Ops, - MachineInstr &LoadMI, - LiveIntervals *LIS) const { - assert(LoadMI.canFoldAsLoad() && "LoadMI isn't foldable!"); -#ifndef NDEBUG - for (unsigned OpIdx : Ops) - assert(MI.getOperand(OpIdx).isUse() && "Folding load into def!"); -#endif - - MachineBasicBlock &MBB = *MI.getParent(); - MachineFunction &MF = *MBB.getParent(); - - // Ask the target to do the actual folding. - MachineInstr *NewMI = nullptr; - int FrameIndex = 0; - - if ((MI.getOpcode() == TargetOpcode::STACKMAP || - MI.getOpcode() == TargetOpcode::PATCHPOINT || - MI.getOpcode() == TargetOpcode::STATEPOINT) && - isLoadFromStackSlot(LoadMI, FrameIndex)) { - // Fold stackmap/patchpoint. - NewMI = foldPatchpoint(MF, MI, Ops, FrameIndex, *this); - if (NewMI) - NewMI = &*MBB.insert(MI, NewMI); - } else { - // Ask the target to do the actual folding. - NewMI = foldMemoryOperandImpl(MF, MI, Ops, MI, LoadMI, LIS); - } - - if (!NewMI) - return nullptr; - - // Copy the memoperands from the load to the folded instruction. - if (MI.memoperands_empty()) { - NewMI->setMemRefs(MF, LoadMI.memoperands()); - } else { - // Handle the rare case of folding multiple loads. - NewMI->setMemRefs(MF, MI.memoperands()); - for (MachineInstr::mmo_iterator I = LoadMI.memoperands_begin(), - E = LoadMI.memoperands_end(); - I != E; ++I) { - NewMI->addMemOperand(MF, *I); - } - } - return NewMI; -} - -bool TargetInstrInfo::hasReassociableOperands( - const MachineInstr &Inst, const MachineBasicBlock *MBB) const { - const MachineOperand &Op1 = Inst.getOperand(1); - const MachineOperand &Op2 = Inst.getOperand(2); - const MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo(); - - // We need virtual register definitions for the operands that we will - // reassociate. - MachineInstr *MI1 = nullptr; - MachineInstr *MI2 = nullptr; - if (Op1.isReg() && Register::isVirtualRegister(Op1.getReg())) - MI1 = MRI.getUniqueVRegDef(Op1.getReg()); - if (Op2.isReg() && Register::isVirtualRegister(Op2.getReg())) - MI2 = MRI.getUniqueVRegDef(Op2.getReg()); - - // And they need to be in the trace (otherwise, they won't have a depth). - return MI1 && MI2 && MI1->getParent() == MBB && MI2->getParent() == MBB; -} - -bool TargetInstrInfo::hasReassociableSibling(const MachineInstr &Inst, - bool &Commuted) const { - const MachineBasicBlock *MBB = Inst.getParent(); - const MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo(); - MachineInstr *MI1 = MRI.getUniqueVRegDef(Inst.getOperand(1).getReg()); - MachineInstr *MI2 = MRI.getUniqueVRegDef(Inst.getOperand(2).getReg()); - unsigned AssocOpcode = Inst.getOpcode(); - - // If only one operand has the same opcode and it's the second source operand, - // the operands must be commuted. - Commuted = MI1->getOpcode() != AssocOpcode && MI2->getOpcode() == AssocOpcode; - if (Commuted) - std::swap(MI1, MI2); - - // 1. The previous instruction must be the same type as Inst. - // 2. The previous instruction must also be associative/commutative (this can - // be different even for instructions with the same opcode if traits like - // fast-math-flags are included). - // 3. The previous instruction must have virtual register definitions for its - // operands in the same basic block as Inst. - // 4. The previous instruction's result must only be used by Inst. - return MI1->getOpcode() == AssocOpcode && isAssociativeAndCommutative(*MI1) && - hasReassociableOperands(*MI1, MBB) && - MRI.hasOneNonDBGUse(MI1->getOperand(0).getReg()); -} - -// 1. The operation must be associative and commutative. -// 2. The instruction must have virtual register definitions for its -// operands in the same basic block. -// 3. The instruction must have a reassociable sibling. -bool TargetInstrInfo::isReassociationCandidate(const MachineInstr &Inst, - bool &Commuted) const { - return isAssociativeAndCommutative(Inst) && - hasReassociableOperands(Inst, Inst.getParent()) && - hasReassociableSibling(Inst, Commuted); -} - -// The concept of the reassociation pass is that these operations can benefit -// from this kind of transformation: -// -// A = ? op ? -// B = A op X (Prev) -// C = B op Y (Root) -// --> -// A = ? op ? -// B = X op Y -// C = A op B -// -// breaking the dependency between A and B, allowing them to be executed in -// parallel (or back-to-back in a pipeline) instead of depending on each other. - -// FIXME: This has the potential to be expensive (compile time) while not -// improving the code at all. Some ways to limit the overhead: -// 1. Track successful transforms; bail out if hit rate gets too low. -// 2. Only enable at -O3 or some other non-default optimization level. -// 3. Pre-screen pattern candidates here: if an operand of the previous -// instruction is known to not increase the critical path, then don't match -// that pattern. -bool TargetInstrInfo::getMachineCombinerPatterns( + } + } + return NewMI; +} + +MachineInstr *TargetInstrInfo::foldMemoryOperand(MachineInstr &MI, + ArrayRef<unsigned> Ops, int FI, + LiveIntervals *LIS, + VirtRegMap *VRM) const { + auto Flags = MachineMemOperand::MONone; + for (unsigned OpIdx : Ops) + Flags |= MI.getOperand(OpIdx).isDef() ? MachineMemOperand::MOStore + : MachineMemOperand::MOLoad; + + MachineBasicBlock *MBB = MI.getParent(); + assert(MBB && "foldMemoryOperand needs an inserted instruction"); + MachineFunction &MF = *MBB->getParent(); + + // If we're not folding a load into a subreg, the size of the load is the + // size of the spill slot. But if we are, we need to figure out what the + // actual load size is. + int64_t MemSize = 0; + const MachineFrameInfo &MFI = MF.getFrameInfo(); + const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); + + if (Flags & MachineMemOperand::MOStore) { + MemSize = MFI.getObjectSize(FI); + } else { + for (unsigned OpIdx : Ops) { + int64_t OpSize = MFI.getObjectSize(FI); + + if (auto SubReg = MI.getOperand(OpIdx).getSubReg()) { + unsigned SubRegSize = TRI->getSubRegIdxSize(SubReg); + if (SubRegSize > 0 && !(SubRegSize % 8)) + OpSize = SubRegSize / 8; + } + + MemSize = std::max(MemSize, OpSize); + } + } + + assert(MemSize && "Did not expect a zero-sized stack slot"); + + MachineInstr *NewMI = nullptr; + + if (MI.getOpcode() == TargetOpcode::STACKMAP || + MI.getOpcode() == TargetOpcode::PATCHPOINT || + MI.getOpcode() == TargetOpcode::STATEPOINT) { + // Fold stackmap/patchpoint. + NewMI = foldPatchpoint(MF, MI, Ops, FI, *this); + if (NewMI) + MBB->insert(MI, NewMI); + } else { + // Ask the target to do the actual folding. + NewMI = foldMemoryOperandImpl(MF, MI, Ops, MI, FI, LIS, VRM); + } + + if (NewMI) { + NewMI->setMemRefs(MF, MI.memoperands()); + // Add a memory operand, foldMemoryOperandImpl doesn't do that. + assert((!(Flags & MachineMemOperand::MOStore) || + NewMI->mayStore()) && + "Folded a def to a non-store!"); + assert((!(Flags & MachineMemOperand::MOLoad) || + NewMI->mayLoad()) && + "Folded a use to a non-load!"); + assert(MFI.getObjectOffset(FI) != -1); + MachineMemOperand *MMO = + MF.getMachineMemOperand(MachinePointerInfo::getFixedStack(MF, FI), + Flags, MemSize, MFI.getObjectAlign(FI)); + NewMI->addMemOperand(MF, MMO); + + // The pass "x86 speculative load hardening" always attaches symbols to + // call instructions. We need copy it form old instruction. + NewMI->cloneInstrSymbols(MF, MI); + + return NewMI; + } + + // Straight COPY may fold as load/store. + if (!MI.isCopy() || Ops.size() != 1) + return nullptr; + + const TargetRegisterClass *RC = canFoldCopy(MI, Ops[0]); + if (!RC) + return nullptr; + + const MachineOperand &MO = MI.getOperand(1 - Ops[0]); + MachineBasicBlock::iterator Pos = MI; + + if (Flags == MachineMemOperand::MOStore) + storeRegToStackSlot(*MBB, Pos, MO.getReg(), MO.isKill(), FI, RC, TRI); + else + loadRegFromStackSlot(*MBB, Pos, MO.getReg(), FI, RC, TRI); + return &*--Pos; +} + +MachineInstr *TargetInstrInfo::foldMemoryOperand(MachineInstr &MI, + ArrayRef<unsigned> Ops, + MachineInstr &LoadMI, + LiveIntervals *LIS) const { + assert(LoadMI.canFoldAsLoad() && "LoadMI isn't foldable!"); +#ifndef NDEBUG + for (unsigned OpIdx : Ops) + assert(MI.getOperand(OpIdx).isUse() && "Folding load into def!"); +#endif + + MachineBasicBlock &MBB = *MI.getParent(); + MachineFunction &MF = *MBB.getParent(); + + // Ask the target to do the actual folding. + MachineInstr *NewMI = nullptr; + int FrameIndex = 0; + + if ((MI.getOpcode() == TargetOpcode::STACKMAP || + MI.getOpcode() == TargetOpcode::PATCHPOINT || + MI.getOpcode() == TargetOpcode::STATEPOINT) && + isLoadFromStackSlot(LoadMI, FrameIndex)) { + // Fold stackmap/patchpoint. + NewMI = foldPatchpoint(MF, MI, Ops, FrameIndex, *this); + if (NewMI) + NewMI = &*MBB.insert(MI, NewMI); + } else { + // Ask the target to do the actual folding. + NewMI = foldMemoryOperandImpl(MF, MI, Ops, MI, LoadMI, LIS); + } + + if (!NewMI) + return nullptr; + + // Copy the memoperands from the load to the folded instruction. + if (MI.memoperands_empty()) { + NewMI->setMemRefs(MF, LoadMI.memoperands()); + } else { + // Handle the rare case of folding multiple loads. + NewMI->setMemRefs(MF, MI.memoperands()); + for (MachineInstr::mmo_iterator I = LoadMI.memoperands_begin(), + E = LoadMI.memoperands_end(); + I != E; ++I) { + NewMI->addMemOperand(MF, *I); + } + } + return NewMI; +} + +bool TargetInstrInfo::hasReassociableOperands( + const MachineInstr &Inst, const MachineBasicBlock *MBB) const { + const MachineOperand &Op1 = Inst.getOperand(1); + const MachineOperand &Op2 = Inst.getOperand(2); + const MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo(); + + // We need virtual register definitions for the operands that we will + // reassociate. + MachineInstr *MI1 = nullptr; + MachineInstr *MI2 = nullptr; + if (Op1.isReg() && Register::isVirtualRegister(Op1.getReg())) + MI1 = MRI.getUniqueVRegDef(Op1.getReg()); + if (Op2.isReg() && Register::isVirtualRegister(Op2.getReg())) + MI2 = MRI.getUniqueVRegDef(Op2.getReg()); + + // And they need to be in the trace (otherwise, they won't have a depth). + return MI1 && MI2 && MI1->getParent() == MBB && MI2->getParent() == MBB; +} + +bool TargetInstrInfo::hasReassociableSibling(const MachineInstr &Inst, + bool &Commuted) const { + const MachineBasicBlock *MBB = Inst.getParent(); + const MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo(); + MachineInstr *MI1 = MRI.getUniqueVRegDef(Inst.getOperand(1).getReg()); + MachineInstr *MI2 = MRI.getUniqueVRegDef(Inst.getOperand(2).getReg()); + unsigned AssocOpcode = Inst.getOpcode(); + + // If only one operand has the same opcode and it's the second source operand, + // the operands must be commuted. + Commuted = MI1->getOpcode() != AssocOpcode && MI2->getOpcode() == AssocOpcode; + if (Commuted) + std::swap(MI1, MI2); + + // 1. The previous instruction must be the same type as Inst. + // 2. The previous instruction must also be associative/commutative (this can + // be different even for instructions with the same opcode if traits like + // fast-math-flags are included). + // 3. The previous instruction must have virtual register definitions for its + // operands in the same basic block as Inst. + // 4. The previous instruction's result must only be used by Inst. + return MI1->getOpcode() == AssocOpcode && isAssociativeAndCommutative(*MI1) && + hasReassociableOperands(*MI1, MBB) && + MRI.hasOneNonDBGUse(MI1->getOperand(0).getReg()); +} + +// 1. The operation must be associative and commutative. +// 2. The instruction must have virtual register definitions for its +// operands in the same basic block. +// 3. The instruction must have a reassociable sibling. +bool TargetInstrInfo::isReassociationCandidate(const MachineInstr &Inst, + bool &Commuted) const { + return isAssociativeAndCommutative(Inst) && + hasReassociableOperands(Inst, Inst.getParent()) && + hasReassociableSibling(Inst, Commuted); +} + +// The concept of the reassociation pass is that these operations can benefit +// from this kind of transformation: +// +// A = ? op ? +// B = A op X (Prev) +// C = B op Y (Root) +// --> +// A = ? op ? +// B = X op Y +// C = A op B +// +// breaking the dependency between A and B, allowing them to be executed in +// parallel (or back-to-back in a pipeline) instead of depending on each other. + +// FIXME: This has the potential to be expensive (compile time) while not +// improving the code at all. Some ways to limit the overhead: +// 1. Track successful transforms; bail out if hit rate gets too low. +// 2. Only enable at -O3 or some other non-default optimization level. +// 3. Pre-screen pattern candidates here: if an operand of the previous +// instruction is known to not increase the critical path, then don't match +// that pattern. +bool TargetInstrInfo::getMachineCombinerPatterns( MachineInstr &Root, SmallVectorImpl<MachineCombinerPattern> &Patterns, bool DoRegPressureReduce) const { - bool Commute; - if (isReassociationCandidate(Root, Commute)) { - // We found a sequence of instructions that may be suitable for a - // reassociation of operands to increase ILP. Specify each commutation - // possibility for the Prev instruction in the sequence and let the - // machine combiner decide if changing the operands is worthwhile. - if (Commute) { - Patterns.push_back(MachineCombinerPattern::REASSOC_AX_YB); - Patterns.push_back(MachineCombinerPattern::REASSOC_XA_YB); - } else { - Patterns.push_back(MachineCombinerPattern::REASSOC_AX_BY); - Patterns.push_back(MachineCombinerPattern::REASSOC_XA_BY); - } - return true; - } - - return false; -} - -/// Return true when a code sequence can improve loop throughput. -bool -TargetInstrInfo::isThroughputPattern(MachineCombinerPattern Pattern) const { - return false; -} - -/// Attempt the reassociation transformation to reduce critical path length. -/// See the above comments before getMachineCombinerPatterns(). -void TargetInstrInfo::reassociateOps( - MachineInstr &Root, MachineInstr &Prev, - MachineCombinerPattern Pattern, - SmallVectorImpl<MachineInstr *> &InsInstrs, - SmallVectorImpl<MachineInstr *> &DelInstrs, - DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const { - MachineFunction *MF = Root.getMF(); - MachineRegisterInfo &MRI = MF->getRegInfo(); - const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo(); - const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo(); - const TargetRegisterClass *RC = Root.getRegClassConstraint(0, TII, TRI); - - // This array encodes the operand index for each parameter because the - // operands may be commuted. Each row corresponds to a pattern value, - // and each column specifies the index of A, B, X, Y. - unsigned OpIdx[4][4] = { - { 1, 1, 2, 2 }, - { 1, 2, 2, 1 }, - { 2, 1, 1, 2 }, - { 2, 2, 1, 1 } - }; - - int Row; - switch (Pattern) { - case MachineCombinerPattern::REASSOC_AX_BY: Row = 0; break; - case MachineCombinerPattern::REASSOC_AX_YB: Row = 1; break; - case MachineCombinerPattern::REASSOC_XA_BY: Row = 2; break; - case MachineCombinerPattern::REASSOC_XA_YB: Row = 3; break; - default: llvm_unreachable("unexpected MachineCombinerPattern"); - } - - MachineOperand &OpA = Prev.getOperand(OpIdx[Row][0]); - MachineOperand &OpB = Root.getOperand(OpIdx[Row][1]); - MachineOperand &OpX = Prev.getOperand(OpIdx[Row][2]); - MachineOperand &OpY = Root.getOperand(OpIdx[Row][3]); - MachineOperand &OpC = Root.getOperand(0); - - Register RegA = OpA.getReg(); - Register RegB = OpB.getReg(); - Register RegX = OpX.getReg(); - Register RegY = OpY.getReg(); - Register RegC = OpC.getReg(); - - if (Register::isVirtualRegister(RegA)) - MRI.constrainRegClass(RegA, RC); - if (Register::isVirtualRegister(RegB)) - MRI.constrainRegClass(RegB, RC); - if (Register::isVirtualRegister(RegX)) - MRI.constrainRegClass(RegX, RC); - if (Register::isVirtualRegister(RegY)) - MRI.constrainRegClass(RegY, RC); - if (Register::isVirtualRegister(RegC)) - MRI.constrainRegClass(RegC, RC); - - // Create a new virtual register for the result of (X op Y) instead of - // recycling RegB because the MachineCombiner's computation of the critical - // path requires a new register definition rather than an existing one. - Register NewVR = MRI.createVirtualRegister(RC); - InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0)); - - unsigned Opcode = Root.getOpcode(); - bool KillA = OpA.isKill(); - bool KillX = OpX.isKill(); - bool KillY = OpY.isKill(); - - // Create new instructions for insertion. - MachineInstrBuilder MIB1 = - BuildMI(*MF, Prev.getDebugLoc(), TII->get(Opcode), NewVR) - .addReg(RegX, getKillRegState(KillX)) - .addReg(RegY, getKillRegState(KillY)); - MachineInstrBuilder MIB2 = - BuildMI(*MF, Root.getDebugLoc(), TII->get(Opcode), RegC) - .addReg(RegA, getKillRegState(KillA)) - .addReg(NewVR, getKillRegState(true)); - - setSpecialOperandAttr(Root, Prev, *MIB1, *MIB2); - - // Record new instructions for insertion and old instructions for deletion. - InsInstrs.push_back(MIB1); - InsInstrs.push_back(MIB2); - DelInstrs.push_back(&Prev); - DelInstrs.push_back(&Root); -} - -void TargetInstrInfo::genAlternativeCodeSequence( - MachineInstr &Root, MachineCombinerPattern Pattern, - SmallVectorImpl<MachineInstr *> &InsInstrs, - SmallVectorImpl<MachineInstr *> &DelInstrs, - DenseMap<unsigned, unsigned> &InstIdxForVirtReg) const { - MachineRegisterInfo &MRI = Root.getMF()->getRegInfo(); - - // Select the previous instruction in the sequence based on the input pattern. - MachineInstr *Prev = nullptr; - switch (Pattern) { - case MachineCombinerPattern::REASSOC_AX_BY: - case MachineCombinerPattern::REASSOC_XA_BY: - Prev = MRI.getUniqueVRegDef(Root.getOperand(1).getReg()); - break; - case MachineCombinerPattern::REASSOC_AX_YB: - case MachineCombinerPattern::REASSOC_XA_YB: - Prev = MRI.getUniqueVRegDef(Root.getOperand(2).getReg()); - break; - default: - break; - } - - assert(Prev && "Unknown pattern for machine combiner"); - - reassociateOps(Root, *Prev, Pattern, InsInstrs, DelInstrs, InstIdxForVirtReg); -} - -bool TargetInstrInfo::isReallyTriviallyReMaterializableGeneric( - const MachineInstr &MI, AAResults *AA) const { - const MachineFunction &MF = *MI.getMF(); - const MachineRegisterInfo &MRI = MF.getRegInfo(); - - // Remat clients assume operand 0 is the defined register. - if (!MI.getNumOperands() || !MI.getOperand(0).isReg()) - return false; - Register DefReg = MI.getOperand(0).getReg(); - - // A sub-register definition can only be rematerialized if the instruction - // doesn't read the other parts of the register. Otherwise it is really a - // read-modify-write operation on the full virtual register which cannot be - // moved safely. - if (Register::isVirtualRegister(DefReg) && MI.getOperand(0).getSubReg() && - MI.readsVirtualRegister(DefReg)) - return false; - - // A load from a fixed stack slot can be rematerialized. This may be - // redundant with subsequent checks, but it's target-independent, - // simple, and a common case. - int FrameIdx = 0; - if (isLoadFromStackSlot(MI, FrameIdx) && - MF.getFrameInfo().isImmutableObjectIndex(FrameIdx)) - return true; - - // Avoid instructions obviously unsafe for remat. - if (MI.isNotDuplicable() || MI.mayStore() || MI.mayRaiseFPException() || - MI.hasUnmodeledSideEffects()) - return false; - - // Don't remat inline asm. We have no idea how expensive it is - // even if it's side effect free. - if (MI.isInlineAsm()) - return false; - - // Avoid instructions which load from potentially varying memory. - if (MI.mayLoad() && !MI.isDereferenceableInvariantLoad(AA)) - return false; - - // If any of the registers accessed are non-constant, conservatively assume - // the instruction is not rematerializable. - for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { - const MachineOperand &MO = MI.getOperand(i); - if (!MO.isReg()) continue; - Register Reg = MO.getReg(); - if (Reg == 0) - continue; - - // Check for a well-behaved physical register. - if (Register::isPhysicalRegister(Reg)) { - if (MO.isUse()) { - // If the physreg has no defs anywhere, it's just an ambient register - // and we can freely move its uses. Alternatively, if it's allocatable, - // it could get allocated to something with a def during allocation. - if (!MRI.isConstantPhysReg(Reg)) - return false; - } else { - // A physreg def. We can't remat it. - return false; - } - continue; - } - - // Only allow one virtual-register def. There may be multiple defs of the - // same virtual register, though. - if (MO.isDef() && Reg != DefReg) - return false; - - // Don't allow any virtual-register uses. Rematting an instruction with - // virtual register uses would length the live ranges of the uses, which - // is not necessarily a good idea, certainly not "trivial". - if (MO.isUse()) - return false; - } - - // Everything checked out. - return true; -} - -int TargetInstrInfo::getSPAdjust(const MachineInstr &MI) const { - const MachineFunction *MF = MI.getMF(); - const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering(); - bool StackGrowsDown = - TFI->getStackGrowthDirection() == TargetFrameLowering::StackGrowsDown; - - unsigned FrameSetupOpcode = getCallFrameSetupOpcode(); - unsigned FrameDestroyOpcode = getCallFrameDestroyOpcode(); - - if (!isFrameInstr(MI)) - return 0; - - int SPAdj = TFI->alignSPAdjust(getFrameSize(MI)); - - if ((!StackGrowsDown && MI.getOpcode() == FrameSetupOpcode) || - (StackGrowsDown && MI.getOpcode() == FrameDestroyOpcode)) - SPAdj = -SPAdj; - - return SPAdj; -} - -/// isSchedulingBoundary - Test if the given instruction should be -/// considered a scheduling boundary. This primarily includes labels -/// and terminators. -bool TargetInstrInfo::isSchedulingBoundary(const MachineInstr &MI, - const MachineBasicBlock *MBB, - const MachineFunction &MF) const { - // Terminators and labels can't be scheduled around. - if (MI.isTerminator() || MI.isPosition()) - return true; - - // INLINEASM_BR can jump to another block - if (MI.getOpcode() == TargetOpcode::INLINEASM_BR) - return true; - - // Don't attempt to schedule around any instruction that defines - // a stack-oriented pointer, as it's unlikely to be profitable. This - // saves compile time, because it doesn't require every single - // stack slot reference to depend on the instruction that does the - // modification. - const TargetLowering &TLI = *MF.getSubtarget().getTargetLowering(); - const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); - return MI.modifiesRegister(TLI.getStackPointerRegisterToSaveRestore(), TRI); -} - -// Provide a global flag for disabling the PreRA hazard recognizer that targets -// may choose to honor. -bool TargetInstrInfo::usePreRAHazardRecognizer() const { - return !DisableHazardRecognizer; -} - -// Default implementation of CreateTargetRAHazardRecognizer. -ScheduleHazardRecognizer *TargetInstrInfo:: -CreateTargetHazardRecognizer(const TargetSubtargetInfo *STI, - const ScheduleDAG *DAG) const { - // Dummy hazard recognizer allows all instructions to issue. - return new ScheduleHazardRecognizer(); -} - -// Default implementation of CreateTargetMIHazardRecognizer. -ScheduleHazardRecognizer *TargetInstrInfo::CreateTargetMIHazardRecognizer( - const InstrItineraryData *II, const ScheduleDAGMI *DAG) const { - return new ScoreboardHazardRecognizer(II, DAG, "machine-scheduler"); -} - -// Default implementation of CreateTargetPostRAHazardRecognizer. -ScheduleHazardRecognizer *TargetInstrInfo:: -CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II, - const ScheduleDAG *DAG) const { - return new ScoreboardHazardRecognizer(II, DAG, "post-RA-sched"); -} - -// Default implementation of getMemOperandWithOffset. -bool TargetInstrInfo::getMemOperandWithOffset( - const MachineInstr &MI, const MachineOperand *&BaseOp, int64_t &Offset, - bool &OffsetIsScalable, const TargetRegisterInfo *TRI) const { - SmallVector<const MachineOperand *, 4> BaseOps; - unsigned Width; - if (!getMemOperandsWithOffsetWidth(MI, BaseOps, Offset, OffsetIsScalable, - Width, TRI) || - BaseOps.size() != 1) - return false; - BaseOp = BaseOps.front(); - return true; -} - -//===----------------------------------------------------------------------===// -// SelectionDAG latency interface. -//===----------------------------------------------------------------------===// - -int -TargetInstrInfo::getOperandLatency(const InstrItineraryData *ItinData, - SDNode *DefNode, unsigned DefIdx, - SDNode *UseNode, unsigned UseIdx) const { - if (!ItinData || ItinData->isEmpty()) - return -1; - - if (!DefNode->isMachineOpcode()) - return -1; - - unsigned DefClass = get(DefNode->getMachineOpcode()).getSchedClass(); - if (!UseNode->isMachineOpcode()) - return ItinData->getOperandCycle(DefClass, DefIdx); - unsigned UseClass = get(UseNode->getMachineOpcode()).getSchedClass(); - return ItinData->getOperandLatency(DefClass, DefIdx, UseClass, UseIdx); -} - -int TargetInstrInfo::getInstrLatency(const InstrItineraryData *ItinData, - SDNode *N) const { - if (!ItinData || ItinData->isEmpty()) - return 1; - - if (!N->isMachineOpcode()) - return 1; - - return ItinData->getStageLatency(get(N->getMachineOpcode()).getSchedClass()); -} - -//===----------------------------------------------------------------------===// -// MachineInstr latency interface. -//===----------------------------------------------------------------------===// - -unsigned TargetInstrInfo::getNumMicroOps(const InstrItineraryData *ItinData, - const MachineInstr &MI) const { - if (!ItinData || ItinData->isEmpty()) - return 1; - - unsigned Class = MI.getDesc().getSchedClass(); - int UOps = ItinData->Itineraries[Class].NumMicroOps; - if (UOps >= 0) - return UOps; - - // The # of u-ops is dynamically determined. The specific target should - // override this function to return the right number. - return 1; -} - -/// Return the default expected latency for a def based on it's opcode. -unsigned TargetInstrInfo::defaultDefLatency(const MCSchedModel &SchedModel, - const MachineInstr &DefMI) const { - if (DefMI.isTransient()) - return 0; - if (DefMI.mayLoad()) - return SchedModel.LoadLatency; - if (isHighLatencyDef(DefMI.getOpcode())) - return SchedModel.HighLatency; - return 1; -} - -unsigned TargetInstrInfo::getPredicationCost(const MachineInstr &) const { - return 0; -} - -unsigned TargetInstrInfo::getInstrLatency(const InstrItineraryData *ItinData, - const MachineInstr &MI, - unsigned *PredCost) const { - // Default to one cycle for no itinerary. However, an "empty" itinerary may - // still have a MinLatency property, which getStageLatency checks. - if (!ItinData) - return MI.mayLoad() ? 2 : 1; - - return ItinData->getStageLatency(MI.getDesc().getSchedClass()); -} - -bool TargetInstrInfo::hasLowDefLatency(const TargetSchedModel &SchedModel, - const MachineInstr &DefMI, - unsigned DefIdx) const { - const InstrItineraryData *ItinData = SchedModel.getInstrItineraries(); - if (!ItinData || ItinData->isEmpty()) - return false; - - unsigned DefClass = DefMI.getDesc().getSchedClass(); - int DefCycle = ItinData->getOperandCycle(DefClass, DefIdx); - return (DefCycle != -1 && DefCycle <= 1); -} - -Optional<ParamLoadedValue> -TargetInstrInfo::describeLoadedValue(const MachineInstr &MI, - Register Reg) const { - const MachineFunction *MF = MI.getMF(); - const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo(); - DIExpression *Expr = DIExpression::get(MF->getFunction().getContext(), {}); - int64_t Offset; - bool OffsetIsScalable; - - // To simplify the sub-register handling, verify that we only need to - // consider physical registers. - assert(MF->getProperties().hasProperty( - MachineFunctionProperties::Property::NoVRegs)); - - if (auto DestSrc = isCopyInstr(MI)) { - Register DestReg = DestSrc->Destination->getReg(); - - // If the copy destination is the forwarding reg, describe the forwarding - // reg using the copy source as the backup location. Example: - // - // x0 = MOV x7 - // call callee(x0) ; x0 described as x7 - if (Reg == DestReg) - return ParamLoadedValue(*DestSrc->Source, Expr); - - // Cases where super- or sub-registers needs to be described should - // be handled by the target's hook implementation. - assert(!TRI->isSuperOrSubRegisterEq(Reg, DestReg) && - "TargetInstrInfo::describeLoadedValue can't describe super- or " - "sub-regs for copy instructions"); - return None; - } else if (auto RegImm = isAddImmediate(MI, Reg)) { - Register SrcReg = RegImm->Reg; - Offset = RegImm->Imm; - Expr = DIExpression::prepend(Expr, DIExpression::ApplyOffset, Offset); - return ParamLoadedValue(MachineOperand::CreateReg(SrcReg, false), Expr); - } else if (MI.hasOneMemOperand()) { - // Only describe memory which provably does not escape the function. As - // described in llvm.org/PR43343, escaped memory may be clobbered by the - // callee (or by another thread). - const auto &TII = MF->getSubtarget().getInstrInfo(); - const MachineFrameInfo &MFI = MF->getFrameInfo(); - const MachineMemOperand *MMO = MI.memoperands()[0]; - const PseudoSourceValue *PSV = MMO->getPseudoValue(); - - // If the address points to "special" memory (e.g. a spill slot), it's - // sufficient to check that it isn't aliased by any high-level IR value. - if (!PSV || PSV->mayAlias(&MFI)) - return None; - - const MachineOperand *BaseOp; - if (!TII->getMemOperandWithOffset(MI, BaseOp, Offset, OffsetIsScalable, - TRI)) - return None; - - // FIXME: Scalable offsets are not yet handled in the offset code below. - if (OffsetIsScalable) - return None; - - // TODO: Can currently only handle mem instructions with a single define. - // An example from the x86 target: - // ... - // DIV64m $rsp, 1, $noreg, 24, $noreg, implicit-def dead $rax, implicit-def $rdx - // ... - // - if (MI.getNumExplicitDefs() != 1) - return None; - - // TODO: In what way do we need to take Reg into consideration here? - - SmallVector<uint64_t, 8> Ops; - DIExpression::appendOffset(Ops, Offset); - Ops.push_back(dwarf::DW_OP_deref_size); - Ops.push_back(MMO->getSize()); - Expr = DIExpression::prependOpcodes(Expr, Ops); - return ParamLoadedValue(*BaseOp, Expr); - } - - return None; -} - -/// Both DefMI and UseMI must be valid. By default, call directly to the -/// itinerary. This may be overriden by the target. -int TargetInstrInfo::getOperandLatency(const InstrItineraryData *ItinData, - const MachineInstr &DefMI, - unsigned DefIdx, - const MachineInstr &UseMI, - unsigned UseIdx) const { - unsigned DefClass = DefMI.getDesc().getSchedClass(); - unsigned UseClass = UseMI.getDesc().getSchedClass(); - return ItinData->getOperandLatency(DefClass, DefIdx, UseClass, UseIdx); -} - -/// If we can determine the operand latency from the def only, without itinerary -/// lookup, do so. Otherwise return -1. -int TargetInstrInfo::computeDefOperandLatency( - const InstrItineraryData *ItinData, const MachineInstr &DefMI) const { - - // Let the target hook getInstrLatency handle missing itineraries. - if (!ItinData) - return getInstrLatency(ItinData, DefMI); - - if(ItinData->isEmpty()) - return defaultDefLatency(ItinData->SchedModel, DefMI); - - // ...operand lookup required - return -1; -} - -bool TargetInstrInfo::getRegSequenceInputs( - const MachineInstr &MI, unsigned DefIdx, - SmallVectorImpl<RegSubRegPairAndIdx> &InputRegs) const { - assert((MI.isRegSequence() || - MI.isRegSequenceLike()) && "Instruction do not have the proper type"); - - if (!MI.isRegSequence()) - return getRegSequenceLikeInputs(MI, DefIdx, InputRegs); - - // We are looking at: - // Def = REG_SEQUENCE v0, sub0, v1, sub1, ... - assert(DefIdx == 0 && "REG_SEQUENCE only has one def"); - for (unsigned OpIdx = 1, EndOpIdx = MI.getNumOperands(); OpIdx != EndOpIdx; - OpIdx += 2) { - const MachineOperand &MOReg = MI.getOperand(OpIdx); - if (MOReg.isUndef()) - continue; - const MachineOperand &MOSubIdx = MI.getOperand(OpIdx + 1); - assert(MOSubIdx.isImm() && - "One of the subindex of the reg_sequence is not an immediate"); - // Record Reg:SubReg, SubIdx. - InputRegs.push_back(RegSubRegPairAndIdx(MOReg.getReg(), MOReg.getSubReg(), - (unsigned)MOSubIdx.getImm())); - } - return true; -} - -bool TargetInstrInfo::getExtractSubregInputs( - const MachineInstr &MI, unsigned DefIdx, - RegSubRegPairAndIdx &InputReg) const { - assert((MI.isExtractSubreg() || - MI.isExtractSubregLike()) && "Instruction do not have the proper type"); - - if (!MI.isExtractSubreg()) - return getExtractSubregLikeInputs(MI, DefIdx, InputReg); - - // We are looking at: - // Def = EXTRACT_SUBREG v0.sub1, sub0. - assert(DefIdx == 0 && "EXTRACT_SUBREG only has one def"); - const MachineOperand &MOReg = MI.getOperand(1); - if (MOReg.isUndef()) - return false; - const MachineOperand &MOSubIdx = MI.getOperand(2); - assert(MOSubIdx.isImm() && - "The subindex of the extract_subreg is not an immediate"); - - InputReg.Reg = MOReg.getReg(); - InputReg.SubReg = MOReg.getSubReg(); - InputReg.SubIdx = (unsigned)MOSubIdx.getImm(); - return true; -} - -bool TargetInstrInfo::getInsertSubregInputs( - const MachineInstr &MI, unsigned DefIdx, - RegSubRegPair &BaseReg, RegSubRegPairAndIdx &InsertedReg) const { - assert((MI.isInsertSubreg() || - MI.isInsertSubregLike()) && "Instruction do not have the proper type"); - - if (!MI.isInsertSubreg()) - return getInsertSubregLikeInputs(MI, DefIdx, BaseReg, InsertedReg); - - // We are looking at: - // Def = INSERT_SEQUENCE v0, v1, sub0. - assert(DefIdx == 0 && "INSERT_SUBREG only has one def"); - const MachineOperand &MOBaseReg = MI.getOperand(1); - const MachineOperand &MOInsertedReg = MI.getOperand(2); - if (MOInsertedReg.isUndef()) - return false; - const MachineOperand &MOSubIdx = MI.getOperand(3); - assert(MOSubIdx.isImm() && - "One of the subindex of the reg_sequence is not an immediate"); - BaseReg.Reg = MOBaseReg.getReg(); - BaseReg.SubReg = MOBaseReg.getSubReg(); - - InsertedReg.Reg = MOInsertedReg.getReg(); - InsertedReg.SubReg = MOInsertedReg.getSubReg(); - InsertedReg.SubIdx = (unsigned)MOSubIdx.getImm(); - return true; -} - -// Returns a MIRPrinter comment for this machine operand. -std::string TargetInstrInfo::createMIROperandComment( - const MachineInstr &MI, const MachineOperand &Op, unsigned OpIdx, - const TargetRegisterInfo *TRI) const { - - if (!MI.isInlineAsm()) - return ""; - - std::string Flags; - raw_string_ostream OS(Flags); - - if (OpIdx == InlineAsm::MIOp_ExtraInfo) { - // Print HasSideEffects, MayLoad, MayStore, IsAlignStack - unsigned ExtraInfo = Op.getImm(); - bool First = true; - for (StringRef Info : InlineAsm::getExtraInfoNames(ExtraInfo)) { - if (!First) - OS << " "; - First = false; - OS << Info; - } - - return OS.str(); - } - - int FlagIdx = MI.findInlineAsmFlagIdx(OpIdx); - if (FlagIdx < 0 || (unsigned)FlagIdx != OpIdx) - return ""; - - assert(Op.isImm() && "Expected flag operand to be an immediate"); - // Pretty print the inline asm operand descriptor. - unsigned Flag = Op.getImm(); - unsigned Kind = InlineAsm::getKind(Flag); - OS << InlineAsm::getKindName(Kind); - - unsigned RCID = 0; - if (!InlineAsm::isImmKind(Flag) && !InlineAsm::isMemKind(Flag) && - InlineAsm::hasRegClassConstraint(Flag, RCID)) { - if (TRI) { - OS << ':' << TRI->getRegClassName(TRI->getRegClass(RCID)); - } else - OS << ":RC" << RCID; - } - - if (InlineAsm::isMemKind(Flag)) { - unsigned MCID = InlineAsm::getMemoryConstraintID(Flag); - OS << ":" << InlineAsm::getMemConstraintName(MCID); - } - - unsigned TiedTo = 0; - if (InlineAsm::isUseOperandTiedToDef(Flag, TiedTo)) - OS << " tiedto:$" << TiedTo; - - return OS.str(); -} - -TargetInstrInfo::PipelinerLoopInfo::~PipelinerLoopInfo() {} + bool Commute; + if (isReassociationCandidate(Root, Commute)) { + // We found a sequence of instructions that may be suitable for a + // reassociation of operands to increase ILP. Specify each commutation + // possibility for the Prev instruction in the sequence and let the + // machine combiner decide if changing the operands is worthwhile. + if (Commute) { + Patterns.push_back(MachineCombinerPattern::REASSOC_AX_YB); + Patterns.push_back(MachineCombinerPattern::REASSOC_XA_YB); + } else { + Patterns.push_back(MachineCombinerPattern::REASSOC_AX_BY); + Patterns.push_back(MachineCombinerPattern::REASSOC_XA_BY); + } + return true; + } + + return false; +} + +/// Return true when a code sequence can improve loop throughput. +bool +TargetInstrInfo::isThroughputPattern(MachineCombinerPattern Pattern) const { + return false; +} + +/// Attempt the reassociation transformation to reduce critical path length. +/// See the above comments before getMachineCombinerPatterns(). +void TargetInstrInfo::reassociateOps( + MachineInstr &Root, MachineInstr &Prev, + MachineCombinerPattern Pattern, + SmallVectorImpl<MachineInstr *> &InsInstrs, + SmallVectorImpl<MachineInstr *> &DelInstrs, + DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const { + MachineFunction *MF = Root.getMF(); + MachineRegisterInfo &MRI = MF->getRegInfo(); + const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo(); + const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo(); + const TargetRegisterClass *RC = Root.getRegClassConstraint(0, TII, TRI); + + // This array encodes the operand index for each parameter because the + // operands may be commuted. Each row corresponds to a pattern value, + // and each column specifies the index of A, B, X, Y. + unsigned OpIdx[4][4] = { + { 1, 1, 2, 2 }, + { 1, 2, 2, 1 }, + { 2, 1, 1, 2 }, + { 2, 2, 1, 1 } + }; + + int Row; + switch (Pattern) { + case MachineCombinerPattern::REASSOC_AX_BY: Row = 0; break; + case MachineCombinerPattern::REASSOC_AX_YB: Row = 1; break; + case MachineCombinerPattern::REASSOC_XA_BY: Row = 2; break; + case MachineCombinerPattern::REASSOC_XA_YB: Row = 3; break; + default: llvm_unreachable("unexpected MachineCombinerPattern"); + } + + MachineOperand &OpA = Prev.getOperand(OpIdx[Row][0]); + MachineOperand &OpB = Root.getOperand(OpIdx[Row][1]); + MachineOperand &OpX = Prev.getOperand(OpIdx[Row][2]); + MachineOperand &OpY = Root.getOperand(OpIdx[Row][3]); + MachineOperand &OpC = Root.getOperand(0); + + Register RegA = OpA.getReg(); + Register RegB = OpB.getReg(); + Register RegX = OpX.getReg(); + Register RegY = OpY.getReg(); + Register RegC = OpC.getReg(); + + if (Register::isVirtualRegister(RegA)) + MRI.constrainRegClass(RegA, RC); + if (Register::isVirtualRegister(RegB)) + MRI.constrainRegClass(RegB, RC); + if (Register::isVirtualRegister(RegX)) + MRI.constrainRegClass(RegX, RC); + if (Register::isVirtualRegister(RegY)) + MRI.constrainRegClass(RegY, RC); + if (Register::isVirtualRegister(RegC)) + MRI.constrainRegClass(RegC, RC); + + // Create a new virtual register for the result of (X op Y) instead of + // recycling RegB because the MachineCombiner's computation of the critical + // path requires a new register definition rather than an existing one. + Register NewVR = MRI.createVirtualRegister(RC); + InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0)); + + unsigned Opcode = Root.getOpcode(); + bool KillA = OpA.isKill(); + bool KillX = OpX.isKill(); + bool KillY = OpY.isKill(); + + // Create new instructions for insertion. + MachineInstrBuilder MIB1 = + BuildMI(*MF, Prev.getDebugLoc(), TII->get(Opcode), NewVR) + .addReg(RegX, getKillRegState(KillX)) + .addReg(RegY, getKillRegState(KillY)); + MachineInstrBuilder MIB2 = + BuildMI(*MF, Root.getDebugLoc(), TII->get(Opcode), RegC) + .addReg(RegA, getKillRegState(KillA)) + .addReg(NewVR, getKillRegState(true)); + + setSpecialOperandAttr(Root, Prev, *MIB1, *MIB2); + + // Record new instructions for insertion and old instructions for deletion. + InsInstrs.push_back(MIB1); + InsInstrs.push_back(MIB2); + DelInstrs.push_back(&Prev); + DelInstrs.push_back(&Root); +} + +void TargetInstrInfo::genAlternativeCodeSequence( + MachineInstr &Root, MachineCombinerPattern Pattern, + SmallVectorImpl<MachineInstr *> &InsInstrs, + SmallVectorImpl<MachineInstr *> &DelInstrs, + DenseMap<unsigned, unsigned> &InstIdxForVirtReg) const { + MachineRegisterInfo &MRI = Root.getMF()->getRegInfo(); + + // Select the previous instruction in the sequence based on the input pattern. + MachineInstr *Prev = nullptr; + switch (Pattern) { + case MachineCombinerPattern::REASSOC_AX_BY: + case MachineCombinerPattern::REASSOC_XA_BY: + Prev = MRI.getUniqueVRegDef(Root.getOperand(1).getReg()); + break; + case MachineCombinerPattern::REASSOC_AX_YB: + case MachineCombinerPattern::REASSOC_XA_YB: + Prev = MRI.getUniqueVRegDef(Root.getOperand(2).getReg()); + break; + default: + break; + } + + assert(Prev && "Unknown pattern for machine combiner"); + + reassociateOps(Root, *Prev, Pattern, InsInstrs, DelInstrs, InstIdxForVirtReg); +} + +bool TargetInstrInfo::isReallyTriviallyReMaterializableGeneric( + const MachineInstr &MI, AAResults *AA) const { + const MachineFunction &MF = *MI.getMF(); + const MachineRegisterInfo &MRI = MF.getRegInfo(); + + // Remat clients assume operand 0 is the defined register. + if (!MI.getNumOperands() || !MI.getOperand(0).isReg()) + return false; + Register DefReg = MI.getOperand(0).getReg(); + + // A sub-register definition can only be rematerialized if the instruction + // doesn't read the other parts of the register. Otherwise it is really a + // read-modify-write operation on the full virtual register which cannot be + // moved safely. + if (Register::isVirtualRegister(DefReg) && MI.getOperand(0).getSubReg() && + MI.readsVirtualRegister(DefReg)) + return false; + + // A load from a fixed stack slot can be rematerialized. This may be + // redundant with subsequent checks, but it's target-independent, + // simple, and a common case. + int FrameIdx = 0; + if (isLoadFromStackSlot(MI, FrameIdx) && + MF.getFrameInfo().isImmutableObjectIndex(FrameIdx)) + return true; + + // Avoid instructions obviously unsafe for remat. + if (MI.isNotDuplicable() || MI.mayStore() || MI.mayRaiseFPException() || + MI.hasUnmodeledSideEffects()) + return false; + + // Don't remat inline asm. We have no idea how expensive it is + // even if it's side effect free. + if (MI.isInlineAsm()) + return false; + + // Avoid instructions which load from potentially varying memory. + if (MI.mayLoad() && !MI.isDereferenceableInvariantLoad(AA)) + return false; + + // If any of the registers accessed are non-constant, conservatively assume + // the instruction is not rematerializable. + for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { + const MachineOperand &MO = MI.getOperand(i); + if (!MO.isReg()) continue; + Register Reg = MO.getReg(); + if (Reg == 0) + continue; + + // Check for a well-behaved physical register. + if (Register::isPhysicalRegister(Reg)) { + if (MO.isUse()) { + // If the physreg has no defs anywhere, it's just an ambient register + // and we can freely move its uses. Alternatively, if it's allocatable, + // it could get allocated to something with a def during allocation. + if (!MRI.isConstantPhysReg(Reg)) + return false; + } else { + // A physreg def. We can't remat it. + return false; + } + continue; + } + + // Only allow one virtual-register def. There may be multiple defs of the + // same virtual register, though. + if (MO.isDef() && Reg != DefReg) + return false; + + // Don't allow any virtual-register uses. Rematting an instruction with + // virtual register uses would length the live ranges of the uses, which + // is not necessarily a good idea, certainly not "trivial". + if (MO.isUse()) + return false; + } + + // Everything checked out. + return true; +} + +int TargetInstrInfo::getSPAdjust(const MachineInstr &MI) const { + const MachineFunction *MF = MI.getMF(); + const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering(); + bool StackGrowsDown = + TFI->getStackGrowthDirection() == TargetFrameLowering::StackGrowsDown; + + unsigned FrameSetupOpcode = getCallFrameSetupOpcode(); + unsigned FrameDestroyOpcode = getCallFrameDestroyOpcode(); + + if (!isFrameInstr(MI)) + return 0; + + int SPAdj = TFI->alignSPAdjust(getFrameSize(MI)); + + if ((!StackGrowsDown && MI.getOpcode() == FrameSetupOpcode) || + (StackGrowsDown && MI.getOpcode() == FrameDestroyOpcode)) + SPAdj = -SPAdj; + + return SPAdj; +} + +/// isSchedulingBoundary - Test if the given instruction should be +/// considered a scheduling boundary. This primarily includes labels +/// and terminators. +bool TargetInstrInfo::isSchedulingBoundary(const MachineInstr &MI, + const MachineBasicBlock *MBB, + const MachineFunction &MF) const { + // Terminators and labels can't be scheduled around. + if (MI.isTerminator() || MI.isPosition()) + return true; + + // INLINEASM_BR can jump to another block + if (MI.getOpcode() == TargetOpcode::INLINEASM_BR) + return true; + + // Don't attempt to schedule around any instruction that defines + // a stack-oriented pointer, as it's unlikely to be profitable. This + // saves compile time, because it doesn't require every single + // stack slot reference to depend on the instruction that does the + // modification. + const TargetLowering &TLI = *MF.getSubtarget().getTargetLowering(); + const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); + return MI.modifiesRegister(TLI.getStackPointerRegisterToSaveRestore(), TRI); +} + +// Provide a global flag for disabling the PreRA hazard recognizer that targets +// may choose to honor. +bool TargetInstrInfo::usePreRAHazardRecognizer() const { + return !DisableHazardRecognizer; +} + +// Default implementation of CreateTargetRAHazardRecognizer. +ScheduleHazardRecognizer *TargetInstrInfo:: +CreateTargetHazardRecognizer(const TargetSubtargetInfo *STI, + const ScheduleDAG *DAG) const { + // Dummy hazard recognizer allows all instructions to issue. + return new ScheduleHazardRecognizer(); +} + +// Default implementation of CreateTargetMIHazardRecognizer. +ScheduleHazardRecognizer *TargetInstrInfo::CreateTargetMIHazardRecognizer( + const InstrItineraryData *II, const ScheduleDAGMI *DAG) const { + return new ScoreboardHazardRecognizer(II, DAG, "machine-scheduler"); +} + +// Default implementation of CreateTargetPostRAHazardRecognizer. +ScheduleHazardRecognizer *TargetInstrInfo:: +CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II, + const ScheduleDAG *DAG) const { + return new ScoreboardHazardRecognizer(II, DAG, "post-RA-sched"); +} + +// Default implementation of getMemOperandWithOffset. +bool TargetInstrInfo::getMemOperandWithOffset( + const MachineInstr &MI, const MachineOperand *&BaseOp, int64_t &Offset, + bool &OffsetIsScalable, const TargetRegisterInfo *TRI) const { + SmallVector<const MachineOperand *, 4> BaseOps; + unsigned Width; + if (!getMemOperandsWithOffsetWidth(MI, BaseOps, Offset, OffsetIsScalable, + Width, TRI) || + BaseOps.size() != 1) + return false; + BaseOp = BaseOps.front(); + return true; +} + +//===----------------------------------------------------------------------===// +// SelectionDAG latency interface. +//===----------------------------------------------------------------------===// + +int +TargetInstrInfo::getOperandLatency(const InstrItineraryData *ItinData, + SDNode *DefNode, unsigned DefIdx, + SDNode *UseNode, unsigned UseIdx) const { + if (!ItinData || ItinData->isEmpty()) + return -1; + + if (!DefNode->isMachineOpcode()) + return -1; + + unsigned DefClass = get(DefNode->getMachineOpcode()).getSchedClass(); + if (!UseNode->isMachineOpcode()) + return ItinData->getOperandCycle(DefClass, DefIdx); + unsigned UseClass = get(UseNode->getMachineOpcode()).getSchedClass(); + return ItinData->getOperandLatency(DefClass, DefIdx, UseClass, UseIdx); +} + +int TargetInstrInfo::getInstrLatency(const InstrItineraryData *ItinData, + SDNode *N) const { + if (!ItinData || ItinData->isEmpty()) + return 1; + + if (!N->isMachineOpcode()) + return 1; + + return ItinData->getStageLatency(get(N->getMachineOpcode()).getSchedClass()); +} + +//===----------------------------------------------------------------------===// +// MachineInstr latency interface. +//===----------------------------------------------------------------------===// + +unsigned TargetInstrInfo::getNumMicroOps(const InstrItineraryData *ItinData, + const MachineInstr &MI) const { + if (!ItinData || ItinData->isEmpty()) + return 1; + + unsigned Class = MI.getDesc().getSchedClass(); + int UOps = ItinData->Itineraries[Class].NumMicroOps; + if (UOps >= 0) + return UOps; + + // The # of u-ops is dynamically determined. The specific target should + // override this function to return the right number. + return 1; +} + +/// Return the default expected latency for a def based on it's opcode. +unsigned TargetInstrInfo::defaultDefLatency(const MCSchedModel &SchedModel, + const MachineInstr &DefMI) const { + if (DefMI.isTransient()) + return 0; + if (DefMI.mayLoad()) + return SchedModel.LoadLatency; + if (isHighLatencyDef(DefMI.getOpcode())) + return SchedModel.HighLatency; + return 1; +} + +unsigned TargetInstrInfo::getPredicationCost(const MachineInstr &) const { + return 0; +} + +unsigned TargetInstrInfo::getInstrLatency(const InstrItineraryData *ItinData, + const MachineInstr &MI, + unsigned *PredCost) const { + // Default to one cycle for no itinerary. However, an "empty" itinerary may + // still have a MinLatency property, which getStageLatency checks. + if (!ItinData) + return MI.mayLoad() ? 2 : 1; + + return ItinData->getStageLatency(MI.getDesc().getSchedClass()); +} + +bool TargetInstrInfo::hasLowDefLatency(const TargetSchedModel &SchedModel, + const MachineInstr &DefMI, + unsigned DefIdx) const { + const InstrItineraryData *ItinData = SchedModel.getInstrItineraries(); + if (!ItinData || ItinData->isEmpty()) + return false; + + unsigned DefClass = DefMI.getDesc().getSchedClass(); + int DefCycle = ItinData->getOperandCycle(DefClass, DefIdx); + return (DefCycle != -1 && DefCycle <= 1); +} + +Optional<ParamLoadedValue> +TargetInstrInfo::describeLoadedValue(const MachineInstr &MI, + Register Reg) const { + const MachineFunction *MF = MI.getMF(); + const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo(); + DIExpression *Expr = DIExpression::get(MF->getFunction().getContext(), {}); + int64_t Offset; + bool OffsetIsScalable; + + // To simplify the sub-register handling, verify that we only need to + // consider physical registers. + assert(MF->getProperties().hasProperty( + MachineFunctionProperties::Property::NoVRegs)); + + if (auto DestSrc = isCopyInstr(MI)) { + Register DestReg = DestSrc->Destination->getReg(); + + // If the copy destination is the forwarding reg, describe the forwarding + // reg using the copy source as the backup location. Example: + // + // x0 = MOV x7 + // call callee(x0) ; x0 described as x7 + if (Reg == DestReg) + return ParamLoadedValue(*DestSrc->Source, Expr); + + // Cases where super- or sub-registers needs to be described should + // be handled by the target's hook implementation. + assert(!TRI->isSuperOrSubRegisterEq(Reg, DestReg) && + "TargetInstrInfo::describeLoadedValue can't describe super- or " + "sub-regs for copy instructions"); + return None; + } else if (auto RegImm = isAddImmediate(MI, Reg)) { + Register SrcReg = RegImm->Reg; + Offset = RegImm->Imm; + Expr = DIExpression::prepend(Expr, DIExpression::ApplyOffset, Offset); + return ParamLoadedValue(MachineOperand::CreateReg(SrcReg, false), Expr); + } else if (MI.hasOneMemOperand()) { + // Only describe memory which provably does not escape the function. As + // described in llvm.org/PR43343, escaped memory may be clobbered by the + // callee (or by another thread). + const auto &TII = MF->getSubtarget().getInstrInfo(); + const MachineFrameInfo &MFI = MF->getFrameInfo(); + const MachineMemOperand *MMO = MI.memoperands()[0]; + const PseudoSourceValue *PSV = MMO->getPseudoValue(); + + // If the address points to "special" memory (e.g. a spill slot), it's + // sufficient to check that it isn't aliased by any high-level IR value. + if (!PSV || PSV->mayAlias(&MFI)) + return None; + + const MachineOperand *BaseOp; + if (!TII->getMemOperandWithOffset(MI, BaseOp, Offset, OffsetIsScalable, + TRI)) + return None; + + // FIXME: Scalable offsets are not yet handled in the offset code below. + if (OffsetIsScalable) + return None; + + // TODO: Can currently only handle mem instructions with a single define. + // An example from the x86 target: + // ... + // DIV64m $rsp, 1, $noreg, 24, $noreg, implicit-def dead $rax, implicit-def $rdx + // ... + // + if (MI.getNumExplicitDefs() != 1) + return None; + + // TODO: In what way do we need to take Reg into consideration here? + + SmallVector<uint64_t, 8> Ops; + DIExpression::appendOffset(Ops, Offset); + Ops.push_back(dwarf::DW_OP_deref_size); + Ops.push_back(MMO->getSize()); + Expr = DIExpression::prependOpcodes(Expr, Ops); + return ParamLoadedValue(*BaseOp, Expr); + } + + return None; +} + +/// Both DefMI and UseMI must be valid. By default, call directly to the +/// itinerary. This may be overriden by the target. +int TargetInstrInfo::getOperandLatency(const InstrItineraryData *ItinData, + const MachineInstr &DefMI, + unsigned DefIdx, + const MachineInstr &UseMI, + unsigned UseIdx) const { + unsigned DefClass = DefMI.getDesc().getSchedClass(); + unsigned UseClass = UseMI.getDesc().getSchedClass(); + return ItinData->getOperandLatency(DefClass, DefIdx, UseClass, UseIdx); +} + +/// If we can determine the operand latency from the def only, without itinerary +/// lookup, do so. Otherwise return -1. +int TargetInstrInfo::computeDefOperandLatency( + const InstrItineraryData *ItinData, const MachineInstr &DefMI) const { + + // Let the target hook getInstrLatency handle missing itineraries. + if (!ItinData) + return getInstrLatency(ItinData, DefMI); + + if(ItinData->isEmpty()) + return defaultDefLatency(ItinData->SchedModel, DefMI); + + // ...operand lookup required + return -1; +} + +bool TargetInstrInfo::getRegSequenceInputs( + const MachineInstr &MI, unsigned DefIdx, + SmallVectorImpl<RegSubRegPairAndIdx> &InputRegs) const { + assert((MI.isRegSequence() || + MI.isRegSequenceLike()) && "Instruction do not have the proper type"); + + if (!MI.isRegSequence()) + return getRegSequenceLikeInputs(MI, DefIdx, InputRegs); + + // We are looking at: + // Def = REG_SEQUENCE v0, sub0, v1, sub1, ... + assert(DefIdx == 0 && "REG_SEQUENCE only has one def"); + for (unsigned OpIdx = 1, EndOpIdx = MI.getNumOperands(); OpIdx != EndOpIdx; + OpIdx += 2) { + const MachineOperand &MOReg = MI.getOperand(OpIdx); + if (MOReg.isUndef()) + continue; + const MachineOperand &MOSubIdx = MI.getOperand(OpIdx + 1); + assert(MOSubIdx.isImm() && + "One of the subindex of the reg_sequence is not an immediate"); + // Record Reg:SubReg, SubIdx. + InputRegs.push_back(RegSubRegPairAndIdx(MOReg.getReg(), MOReg.getSubReg(), + (unsigned)MOSubIdx.getImm())); + } + return true; +} + +bool TargetInstrInfo::getExtractSubregInputs( + const MachineInstr &MI, unsigned DefIdx, + RegSubRegPairAndIdx &InputReg) const { + assert((MI.isExtractSubreg() || + MI.isExtractSubregLike()) && "Instruction do not have the proper type"); + + if (!MI.isExtractSubreg()) + return getExtractSubregLikeInputs(MI, DefIdx, InputReg); + + // We are looking at: + // Def = EXTRACT_SUBREG v0.sub1, sub0. + assert(DefIdx == 0 && "EXTRACT_SUBREG only has one def"); + const MachineOperand &MOReg = MI.getOperand(1); + if (MOReg.isUndef()) + return false; + const MachineOperand &MOSubIdx = MI.getOperand(2); + assert(MOSubIdx.isImm() && + "The subindex of the extract_subreg is not an immediate"); + + InputReg.Reg = MOReg.getReg(); + InputReg.SubReg = MOReg.getSubReg(); + InputReg.SubIdx = (unsigned)MOSubIdx.getImm(); + return true; +} + +bool TargetInstrInfo::getInsertSubregInputs( + const MachineInstr &MI, unsigned DefIdx, + RegSubRegPair &BaseReg, RegSubRegPairAndIdx &InsertedReg) const { + assert((MI.isInsertSubreg() || + MI.isInsertSubregLike()) && "Instruction do not have the proper type"); + + if (!MI.isInsertSubreg()) + return getInsertSubregLikeInputs(MI, DefIdx, BaseReg, InsertedReg); + + // We are looking at: + // Def = INSERT_SEQUENCE v0, v1, sub0. + assert(DefIdx == 0 && "INSERT_SUBREG only has one def"); + const MachineOperand &MOBaseReg = MI.getOperand(1); + const MachineOperand &MOInsertedReg = MI.getOperand(2); + if (MOInsertedReg.isUndef()) + return false; + const MachineOperand &MOSubIdx = MI.getOperand(3); + assert(MOSubIdx.isImm() && + "One of the subindex of the reg_sequence is not an immediate"); + BaseReg.Reg = MOBaseReg.getReg(); + BaseReg.SubReg = MOBaseReg.getSubReg(); + + InsertedReg.Reg = MOInsertedReg.getReg(); + InsertedReg.SubReg = MOInsertedReg.getSubReg(); + InsertedReg.SubIdx = (unsigned)MOSubIdx.getImm(); + return true; +} + +// Returns a MIRPrinter comment for this machine operand. +std::string TargetInstrInfo::createMIROperandComment( + const MachineInstr &MI, const MachineOperand &Op, unsigned OpIdx, + const TargetRegisterInfo *TRI) const { + + if (!MI.isInlineAsm()) + return ""; + + std::string Flags; + raw_string_ostream OS(Flags); + + if (OpIdx == InlineAsm::MIOp_ExtraInfo) { + // Print HasSideEffects, MayLoad, MayStore, IsAlignStack + unsigned ExtraInfo = Op.getImm(); + bool First = true; + for (StringRef Info : InlineAsm::getExtraInfoNames(ExtraInfo)) { + if (!First) + OS << " "; + First = false; + OS << Info; + } + + return OS.str(); + } + + int FlagIdx = MI.findInlineAsmFlagIdx(OpIdx); + if (FlagIdx < 0 || (unsigned)FlagIdx != OpIdx) + return ""; + + assert(Op.isImm() && "Expected flag operand to be an immediate"); + // Pretty print the inline asm operand descriptor. + unsigned Flag = Op.getImm(); + unsigned Kind = InlineAsm::getKind(Flag); + OS << InlineAsm::getKindName(Kind); + + unsigned RCID = 0; + if (!InlineAsm::isImmKind(Flag) && !InlineAsm::isMemKind(Flag) && + InlineAsm::hasRegClassConstraint(Flag, RCID)) { + if (TRI) { + OS << ':' << TRI->getRegClassName(TRI->getRegClass(RCID)); + } else + OS << ":RC" << RCID; + } + + if (InlineAsm::isMemKind(Flag)) { + unsigned MCID = InlineAsm::getMemoryConstraintID(Flag); + OS << ":" << InlineAsm::getMemConstraintName(MCID); + } + + unsigned TiedTo = 0; + if (InlineAsm::isUseOperandTiedToDef(Flag, TiedTo)) + OS << " tiedto:$" << TiedTo; + + return OS.str(); +} + +TargetInstrInfo::PipelinerLoopInfo::~PipelinerLoopInfo() {} |