diff options
author | shadchin <shadchin@yandex-team.ru> | 2022-02-10 16:44:30 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:44:30 +0300 |
commit | 2598ef1d0aee359b4b6d5fdd1758916d5907d04f (patch) | |
tree | 012bb94d777798f1f56ac1cec429509766d05181 /contrib/libs/llvm12/lib/Target/ARM/ARMBaseInstrInfo.cpp | |
parent | 6751af0b0c1b952fede40b19b71da8025b5d8bcf (diff) | |
download | ydb-2598ef1d0aee359b4b6d5fdd1758916d5907d04f.tar.gz |
Restoring authorship annotation for <shadchin@yandex-team.ru>. Commit 1 of 2.
Diffstat (limited to 'contrib/libs/llvm12/lib/Target/ARM/ARMBaseInstrInfo.cpp')
-rw-r--r-- | contrib/libs/llvm12/lib/Target/ARM/ARMBaseInstrInfo.cpp | 1210 |
1 files changed, 605 insertions, 605 deletions
diff --git a/contrib/libs/llvm12/lib/Target/ARM/ARMBaseInstrInfo.cpp b/contrib/libs/llvm12/lib/Target/ARM/ARMBaseInstrInfo.cpp index e418d53b56..d3047e1ae7 100644 --- a/contrib/libs/llvm12/lib/Target/ARM/ARMBaseInstrInfo.cpp +++ b/contrib/libs/llvm12/lib/Target/ARM/ARMBaseInstrInfo.cpp @@ -19,7 +19,7 @@ #include "ARMSubtarget.h" #include "MCTargetDesc/ARMAddressingModes.h" #include "MCTargetDesc/ARMBaseInfo.h" -#include "MVETailPredUtils.h" +#include "MVETailPredUtils.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallSet.h" @@ -36,8 +36,8 @@ #include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/MachineOperand.h" #include "llvm/CodeGen/MachineRegisterInfo.h" -#include "llvm/CodeGen/MachineScheduler.h" -#include "llvm/CodeGen/MultiHazardRecognizer.h" +#include "llvm/CodeGen/MachineScheduler.h" +#include "llvm/CodeGen/MultiHazardRecognizer.h" #include "llvm/CodeGen/ScoreboardHazardRecognizer.h" #include "llvm/CodeGen/SelectionDAGNodes.h" #include "llvm/CodeGen/TargetInstrInfo.h" @@ -134,43 +134,43 @@ ARMBaseInstrInfo::CreateTargetHazardRecognizer(const TargetSubtargetInfo *STI, return TargetInstrInfo::CreateTargetHazardRecognizer(STI, DAG); } -// Called during: -// - pre-RA scheduling -// - post-RA scheduling when FeatureUseMISched is set -ScheduleHazardRecognizer *ARMBaseInstrInfo::CreateTargetMIHazardRecognizer( - const InstrItineraryData *II, const ScheduleDAGMI *DAG) const { - MultiHazardRecognizer *MHR = new MultiHazardRecognizer(); - - // We would like to restrict this hazard recognizer to only - // post-RA scheduling; we can tell that we're post-RA because we don't - // track VRegLiveness. - // Cortex-M7: TRM indicates that there is a single ITCM bank and two DTCM - // banks banked on bit 2. Assume that TCMs are in use. - if (Subtarget.isCortexM7() && !DAG->hasVRegLiveness()) - MHR->AddHazardRecognizer( - std::make_unique<ARMBankConflictHazardRecognizer>(DAG, 0x4, true)); - - // Not inserting ARMHazardRecognizerFPMLx because that would change - // legacy behavior - - auto BHR = TargetInstrInfo::CreateTargetMIHazardRecognizer(II, DAG); - MHR->AddHazardRecognizer(std::unique_ptr<ScheduleHazardRecognizer>(BHR)); - return MHR; -} - -// Called during post-RA scheduling when FeatureUseMISched is not set +// Called during: +// - pre-RA scheduling +// - post-RA scheduling when FeatureUseMISched is set +ScheduleHazardRecognizer *ARMBaseInstrInfo::CreateTargetMIHazardRecognizer( + const InstrItineraryData *II, const ScheduleDAGMI *DAG) const { + MultiHazardRecognizer *MHR = new MultiHazardRecognizer(); + + // We would like to restrict this hazard recognizer to only + // post-RA scheduling; we can tell that we're post-RA because we don't + // track VRegLiveness. + // Cortex-M7: TRM indicates that there is a single ITCM bank and two DTCM + // banks banked on bit 2. Assume that TCMs are in use. + if (Subtarget.isCortexM7() && !DAG->hasVRegLiveness()) + MHR->AddHazardRecognizer( + std::make_unique<ARMBankConflictHazardRecognizer>(DAG, 0x4, true)); + + // Not inserting ARMHazardRecognizerFPMLx because that would change + // legacy behavior + + auto BHR = TargetInstrInfo::CreateTargetMIHazardRecognizer(II, DAG); + MHR->AddHazardRecognizer(std::unique_ptr<ScheduleHazardRecognizer>(BHR)); + return MHR; +} + +// Called during post-RA scheduling when FeatureUseMISched is not set ScheduleHazardRecognizer *ARMBaseInstrInfo:: CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II, const ScheduleDAG *DAG) const { - MultiHazardRecognizer *MHR = new MultiHazardRecognizer(); - + MultiHazardRecognizer *MHR = new MultiHazardRecognizer(); + if (Subtarget.isThumb2() || Subtarget.hasVFP2Base()) - MHR->AddHazardRecognizer(std::make_unique<ARMHazardRecognizerFPMLx>()); - - auto BHR = TargetInstrInfo::CreateTargetPostRAHazardRecognizer(II, DAG); - if (BHR) - MHR->AddHazardRecognizer(std::unique_ptr<ScheduleHazardRecognizer>(BHR)); - return MHR; + MHR->AddHazardRecognizer(std::make_unique<ARMHazardRecognizerFPMLx>()); + + auto BHR = TargetInstrInfo::CreateTargetPostRAHazardRecognizer(II, DAG); + if (BHR) + MHR->AddHazardRecognizer(std::unique_ptr<ScheduleHazardRecognizer>(BHR)); + return MHR; } MachineInstr *ARMBaseInstrInfo::convertToThreeAddress( @@ -351,8 +351,8 @@ bool ARMBaseInstrInfo::analyzeBranch(MachineBasicBlock &MBB, TBB = nullptr; FBB = nullptr; - MachineBasicBlock::instr_iterator I = MBB.instr_end(); - if (I == MBB.instr_begin()) + MachineBasicBlock::instr_iterator I = MBB.instr_end(); + if (I == MBB.instr_begin()) return false; // Empty blocks are easy. --I; @@ -364,12 +364,12 @@ bool ARMBaseInstrInfo::analyzeBranch(MachineBasicBlock &MBB, // out. bool CantAnalyze = false; - // Skip over DEBUG values, predicated nonterminators and speculation - // barrier terminators. - while (I->isDebugInstr() || !I->isTerminator() || - isSpeculationBarrierEndBBOpcode(I->getOpcode()) || - I->getOpcode() == ARM::t2DoLoopStartTP){ - if (I == MBB.instr_begin()) + // Skip over DEBUG values, predicated nonterminators and speculation + // barrier terminators. + while (I->isDebugInstr() || !I->isTerminator() || + isSpeculationBarrierEndBBOpcode(I->getOpcode()) || + I->getOpcode() == ARM::t2DoLoopStartTP){ + if (I == MBB.instr_begin()) return false; --I; } @@ -393,7 +393,7 @@ bool ARMBaseInstrInfo::analyzeBranch(MachineBasicBlock &MBB, Cond.push_back(I->getOperand(2)); } else if (I->isReturn()) { // Returns can't be analyzed, but we should run cleanup. - CantAnalyze = true; + CantAnalyze = true; } else { // We encountered other unrecognized terminator. Bail out immediately. return true; @@ -414,30 +414,30 @@ bool ARMBaseInstrInfo::analyzeBranch(MachineBasicBlock &MBB, // unconditional branch. if (AllowModify) { MachineBasicBlock::iterator DI = std::next(I); - while (DI != MBB.instr_end()) { + while (DI != MBB.instr_end()) { MachineInstr &InstToDelete = *DI; ++DI; - // Speculation barriers must not be deleted. - if (isSpeculationBarrierEndBBOpcode(InstToDelete.getOpcode())) - continue; + // Speculation barriers must not be deleted. + if (isSpeculationBarrierEndBBOpcode(InstToDelete.getOpcode())) + continue; InstToDelete.eraseFromParent(); } } } - if (CantAnalyze) { - // We may not be able to analyze the block, but we could still have - // an unconditional branch as the last instruction in the block, which - // just branches to layout successor. If this is the case, then just - // remove it if we're allowed to make modifications. - if (AllowModify && !isPredicated(MBB.back()) && - isUncondBranchOpcode(MBB.back().getOpcode()) && - TBB && MBB.isLayoutSuccessor(TBB)) - removeBranch(MBB); + if (CantAnalyze) { + // We may not be able to analyze the block, but we could still have + // an unconditional branch as the last instruction in the block, which + // just branches to layout successor. If this is the case, then just + // remove it if we're allowed to make modifications. + if (AllowModify && !isPredicated(MBB.back()) && + isUncondBranchOpcode(MBB.back().getOpcode()) && + TBB && MBB.isLayoutSuccessor(TBB)) + removeBranch(MBB); return true; - } + } - if (I == MBB.instr_begin()) + if (I == MBB.instr_begin()) return false; --I; @@ -586,18 +586,18 @@ bool ARMBaseInstrInfo::PredicateInstruction( MachineOperand &PMO = MI.getOperand(PIdx); PMO.setImm(Pred[0].getImm()); MI.getOperand(PIdx+1).setReg(Pred[1].getReg()); - - // Thumb 1 arithmetic instructions do not set CPSR when executed inside an - // IT block. This affects how they are printed. - const MCInstrDesc &MCID = MI.getDesc(); - if (MCID.TSFlags & ARMII::ThumbArithFlagSetting) { - assert(MCID.OpInfo[1].isOptionalDef() && "CPSR def isn't expected operand"); - assert((MI.getOperand(1).isDead() || - MI.getOperand(1).getReg() != ARM::CPSR) && - "if conversion tried to stop defining used CPSR"); - MI.getOperand(1).setReg(ARM::NoRegister); - } - + + // Thumb 1 arithmetic instructions do not set CPSR when executed inside an + // IT block. This affects how they are printed. + const MCInstrDesc &MCID = MI.getDesc(); + if (MCID.TSFlags & ARMII::ThumbArithFlagSetting) { + assert(MCID.OpInfo[1].isOptionalDef() && "CPSR def isn't expected operand"); + assert((MI.getOperand(1).isDead() || + MI.getOperand(1).getReg() != ARM::CPSR) && + "if conversion tried to stop defining used CPSR"); + MI.getOperand(1).setReg(ARM::NoRegister); + } + return true; } return false; @@ -629,23 +629,23 @@ bool ARMBaseInstrInfo::SubsumesPredicate(ArrayRef<MachineOperand> Pred1, } } -bool ARMBaseInstrInfo::ClobbersPredicate(MachineInstr &MI, - std::vector<MachineOperand> &Pred, - bool SkipDead) const { +bool ARMBaseInstrInfo::ClobbersPredicate(MachineInstr &MI, + std::vector<MachineOperand> &Pred, + bool SkipDead) const { bool Found = false; for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { const MachineOperand &MO = MI.getOperand(i); - bool ClobbersCPSR = MO.isRegMask() && MO.clobbersPhysReg(ARM::CPSR); - bool IsCPSR = MO.isReg() && MO.isDef() && MO.getReg() == ARM::CPSR; - if (ClobbersCPSR || IsCPSR) { - - // Filter out T1 instructions that have a dead CPSR, - // allowing IT blocks to be generated containing T1 instructions - const MCInstrDesc &MCID = MI.getDesc(); - if (MCID.TSFlags & ARMII::ThumbArithFlagSetting && MO.isDead() && - SkipDead) - continue; - + bool ClobbersCPSR = MO.isRegMask() && MO.clobbersPhysReg(ARM::CPSR); + bool IsCPSR = MO.isReg() && MO.isDef() && MO.getReg() == ARM::CPSR; + if (ClobbersCPSR || IsCPSR) { + + // Filter out T1 instructions that have a dead CPSR, + // allowing IT blocks to be generated containing T1 instructions + const MCInstrDesc &MCID = MI.getDesc(); + if (MCID.TSFlags & ARMII::ThumbArithFlagSetting && MO.isDead() && + SkipDead) + continue; + Pred.push_back(MO); Found = true; } @@ -703,23 +703,23 @@ bool ARMBaseInstrInfo::isPredicable(const MachineInstr &MI) const { if (!isEligibleForITBlock(&MI)) return false; - const MachineFunction *MF = MI.getParent()->getParent(); + const MachineFunction *MF = MI.getParent()->getParent(); const ARMFunctionInfo *AFI = - MF->getInfo<ARMFunctionInfo>(); + MF->getInfo<ARMFunctionInfo>(); // Neon instructions in Thumb2 IT blocks are deprecated, see ARMARM. // In their ARM encoding, they can't be encoded in a conditional form. if ((MI.getDesc().TSFlags & ARMII::DomainMask) == ARMII::DomainNEON) return false; - // Make indirect control flow changes unpredicable when SLS mitigation is - // enabled. - const ARMSubtarget &ST = MF->getSubtarget<ARMSubtarget>(); - if (ST.hardenSlsRetBr() && isIndirectControlFlowNotComingBack(MI)) - return false; - if (ST.hardenSlsBlr() && isIndirectCall(MI)) - return false; - + // Make indirect control flow changes unpredicable when SLS mitigation is + // enabled. + const ARMSubtarget &ST = MF->getSubtarget<ARMSubtarget>(); + if (ST.hardenSlsRetBr() && isIndirectControlFlowNotComingBack(MI)) + return false; + if (ST.hardenSlsBlr() && isIndirectCall(MI)) + return false; + if (AFI->isThumb2Function()) { if (getSubtarget().restrictIT()) return isV8EligibleForIT(&MI); @@ -802,14 +802,14 @@ unsigned ARMBaseInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const { Size = alignTo(Size, 4); return Size; } - case ARM::SpeculationBarrierISBDSBEndBB: - case ARM::t2SpeculationBarrierISBDSBEndBB: - // This gets lowered to 2 4-byte instructions. - return 8; - case ARM::SpeculationBarrierSBEndBB: - case ARM::t2SpeculationBarrierSBEndBB: - // This gets lowered to 1 4-byte instructions. - return 4; + case ARM::SpeculationBarrierISBDSBEndBB: + case ARM::t2SpeculationBarrierISBDSBEndBB: + // This gets lowered to 2 4-byte instructions. + return 8; + case ARM::SpeculationBarrierSBEndBB: + case ARM::t2SpeculationBarrierSBEndBB: + // This gets lowered to 1 4-byte instructions. + return 4; } } @@ -2175,12 +2175,12 @@ ARMBaseInstrInfo::extraSizeToPredicateInstructions(const MachineFunction &MF, // Thumb2 needs a 2-byte IT instruction to predicate up to 4 instructions. // ARM has a condition code field in every predicable instruction, using it // doesn't change code size. - if (!Subtarget.isThumb2()) - return 0; - - // It's possible that the size of the IT is restricted to a single block. - unsigned MaxInsts = Subtarget.restrictIT() ? 1 : 4; - return divideCeil(NumInsts, MaxInsts) * 2; + if (!Subtarget.isThumb2()) + return 0; + + // It's possible that the size of the IT is restricted to a single block. + unsigned MaxInsts = Subtarget.restrictIT() ? 1 : 4; + return divideCeil(NumInsts, MaxInsts) * 2; } unsigned @@ -3417,7 +3417,7 @@ bool ARMBaseInstrInfo::FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, case ARM::t2SUBspImm: case ARM::t2ADDri: case ARM::t2SUBri: - MRI->constrainRegClass(UseMI.getOperand(0).getReg(), TRC); + MRI->constrainRegClass(UseMI.getOperand(0).getReg(), TRC); } return true; } @@ -4838,14 +4838,14 @@ bool ARMBaseInstrInfo::verifyInstruction(const MachineInstr &MI, } } } - if (MI.getOpcode() == ARM::MVE_VMOV_q_rr) { - assert(MI.getOperand(4).isImm() && MI.getOperand(5).isImm()); - if ((MI.getOperand(4).getImm() != 2 && MI.getOperand(4).getImm() != 3) || - MI.getOperand(4).getImm() != MI.getOperand(5).getImm() + 2) { - ErrInfo = "Incorrect array index for MVE_VMOV_q_rr"; - return false; - } - } + if (MI.getOpcode() == ARM::MVE_VMOV_q_rr) { + assert(MI.getOperand(4).isImm() && MI.getOperand(5).isImm()); + if ((MI.getOperand(4).getImm() != 2 && MI.getOperand(4).getImm() != 3) || + MI.getOperand(4).getImm() != MI.getOperand(5).getImm() + 2) { + ErrInfo = "Incorrect array index for MVE_VMOV_q_rr"; + return false; + } + } return true; } @@ -5531,8 +5531,8 @@ unsigned llvm::ConstantMaterializationCost(unsigned Val, return ForCodesize ? 4 : 1; if (ARM_AM::isSOImmTwoPartVal(Val)) // two instrs return ForCodesize ? 8 : 2; - if (ARM_AM::isSOImmTwoPartValNeg(Val)) // two instrs - return ForCodesize ? 8 : 2; + if (ARM_AM::isSOImmTwoPartValNeg(Val)) // two instrs + return ForCodesize ? 8 : 2; } if (Subtarget->useMovt()) // MOVW + MOVT return ForCodesize ? 8 : 2; @@ -5637,32 +5637,32 @@ bool llvm::HasLowerConstantMaterializationCost(unsigned Val1, unsigned Val2, /// | Frame overhead in Bytes | 2 | 4 | /// | Stack fixup required | No | No | /// +-------------------------+--------+-----+ -/// -/// \p MachineOutlinerDefault implies that the function should be called with -/// a save and restore of LR to the stack. -/// -/// That is, -/// -/// I1 Save LR OUTLINED_FUNCTION: -/// I2 --> BL OUTLINED_FUNCTION I1 -/// I3 Restore LR I2 -/// I3 -/// BX LR -/// -/// +-------------------------+--------+-----+ -/// | | Thumb2 | ARM | -/// +-------------------------+--------+-----+ -/// | Call overhead in Bytes | 8 | 12 | -/// | Frame overhead in Bytes | 2 | 4 | -/// | Stack fixup required | Yes | Yes | -/// +-------------------------+--------+-----+ +/// +/// \p MachineOutlinerDefault implies that the function should be called with +/// a save and restore of LR to the stack. +/// +/// That is, +/// +/// I1 Save LR OUTLINED_FUNCTION: +/// I2 --> BL OUTLINED_FUNCTION I1 +/// I3 Restore LR I2 +/// I3 +/// BX LR +/// +/// +-------------------------+--------+-----+ +/// | | Thumb2 | ARM | +/// +-------------------------+--------+-----+ +/// | Call overhead in Bytes | 8 | 12 | +/// | Frame overhead in Bytes | 2 | 4 | +/// | Stack fixup required | Yes | Yes | +/// +-------------------------+--------+-----+ enum MachineOutlinerClass { MachineOutlinerTailCall, MachineOutlinerThunk, MachineOutlinerNoLRSave, - MachineOutlinerRegSave, - MachineOutlinerDefault + MachineOutlinerRegSave, + MachineOutlinerDefault }; enum MachineOutlinerMBBFlags { @@ -5680,9 +5680,9 @@ struct OutlinerCosts { const int FrameNoLRSave; const int CallRegSave; const int FrameRegSave; - const int CallDefault; - const int FrameDefault; - const int SaveRestoreLROnStack; + const int CallDefault; + const int FrameDefault; + const int SaveRestoreLROnStack; OutlinerCosts(const ARMSubtarget &target) : CallTailCall(target.isThumb() ? 4 : 4), @@ -5692,10 +5692,10 @@ struct OutlinerCosts { CallNoLRSave(target.isThumb() ? 4 : 4), FrameNoLRSave(target.isThumb() ? 4 : 4), CallRegSave(target.isThumb() ? 8 : 12), - FrameRegSave(target.isThumb() ? 2 : 4), - CallDefault(target.isThumb() ? 8 : 12), - FrameDefault(target.isThumb() ? 2 : 4), - SaveRestoreLROnStack(target.isThumb() ? 8 : 8) {} + FrameRegSave(target.isThumb() ? 2 : 4), + CallDefault(target.isThumb() ? 8 : 12), + FrameDefault(target.isThumb() ? 2 : 4), + SaveRestoreLROnStack(target.isThumb() ? 8 : 8) {} }; unsigned @@ -5720,37 +5720,37 @@ ARMBaseInstrInfo::findRegisterToSaveLRTo(const outliner::Candidate &C) const { return 0u; } -// Compute liveness of LR at the point after the interval [I, E), which -// denotes a *backward* iteration through instructions. Used only for return -// basic blocks, which do not end with a tail call. -static bool isLRAvailable(const TargetRegisterInfo &TRI, - MachineBasicBlock::reverse_iterator I, - MachineBasicBlock::reverse_iterator E) { - // At the end of the function LR dead. - bool Live = false; - for (; I != E; ++I) { - const MachineInstr &MI = *I; - - // Check defs of LR. - if (MI.modifiesRegister(ARM::LR, &TRI)) - Live = false; - - // Check uses of LR. - unsigned Opcode = MI.getOpcode(); - if (Opcode == ARM::BX_RET || Opcode == ARM::MOVPCLR || - Opcode == ARM::SUBS_PC_LR || Opcode == ARM::tBX_RET || - Opcode == ARM::tBXNS_RET) { - // These instructions use LR, but it's not an (explicit or implicit) - // operand. - Live = true; - continue; - } - if (MI.readsRegister(ARM::LR, &TRI)) - Live = true; - } - return !Live; -} - +// Compute liveness of LR at the point after the interval [I, E), which +// denotes a *backward* iteration through instructions. Used only for return +// basic blocks, which do not end with a tail call. +static bool isLRAvailable(const TargetRegisterInfo &TRI, + MachineBasicBlock::reverse_iterator I, + MachineBasicBlock::reverse_iterator E) { + // At the end of the function LR dead. + bool Live = false; + for (; I != E; ++I) { + const MachineInstr &MI = *I; + + // Check defs of LR. + if (MI.modifiesRegister(ARM::LR, &TRI)) + Live = false; + + // Check uses of LR. + unsigned Opcode = MI.getOpcode(); + if (Opcode == ARM::BX_RET || Opcode == ARM::MOVPCLR || + Opcode == ARM::SUBS_PC_LR || Opcode == ARM::tBX_RET || + Opcode == ARM::tBXNS_RET) { + // These instructions use LR, but it's not an (explicit or implicit) + // operand. + Live = true; + continue; + } + if (MI.readsRegister(ARM::LR, &TRI)) + Live = true; + } + return !Live; +} + outliner::OutlinedFunction ARMBaseInstrInfo::getOutliningCandidateInfo( std::vector<outliner::Candidate> &RepeatedSequenceLocs) const { outliner::Candidate &FirstCand = RepeatedSequenceLocs[0]; @@ -5796,7 +5796,7 @@ outliner::OutlinedFunction ARMBaseInstrInfo::getOutliningCandidateInfo( // Erase every candidate that violates the restrictions above. (It could be // true that we have viable candidates, so it's not worth bailing out in // the case that, say, 1 out of 20 candidates violate the restructions.) - llvm::erase_if(RepeatedSequenceLocs, CantGuaranteeValueAcrossCall); + llvm::erase_if(RepeatedSequenceLocs, CantGuaranteeValueAcrossCall); // If the sequence doesn't have enough candidates left, then we're done. if (RepeatedSequenceLocs.size() < 2) @@ -5816,8 +5816,8 @@ outliner::OutlinedFunction ARMBaseInstrInfo::getOutliningCandidateInfo( }; OutlinerCosts Costs(Subtarget); - unsigned FrameID = MachineOutlinerDefault; - unsigned NumBytesToCreateFrame = Costs.FrameDefault; + unsigned FrameID = MachineOutlinerDefault; + unsigned NumBytesToCreateFrame = Costs.FrameDefault; // If the last instruction in any candidate is a terminator, then we should // tail call all of the candidates. @@ -5826,31 +5826,31 @@ outliner::OutlinedFunction ARMBaseInstrInfo::getOutliningCandidateInfo( NumBytesToCreateFrame = Costs.FrameTailCall; SetCandidateCallInfo(MachineOutlinerTailCall, Costs.CallTailCall); } else if (LastInstrOpcode == ARM::BL || LastInstrOpcode == ARM::BLX || - LastInstrOpcode == ARM::BLX_noip || LastInstrOpcode == ARM::tBL || - LastInstrOpcode == ARM::tBLXr || - LastInstrOpcode == ARM::tBLXr_noip || + LastInstrOpcode == ARM::BLX_noip || LastInstrOpcode == ARM::tBL || + LastInstrOpcode == ARM::tBLXr || + LastInstrOpcode == ARM::tBLXr_noip || LastInstrOpcode == ARM::tBLXi) { FrameID = MachineOutlinerThunk; NumBytesToCreateFrame = Costs.FrameThunk; SetCandidateCallInfo(MachineOutlinerThunk, Costs.CallThunk); } else { // We need to decide how to emit calls + frames. We can always emit the same - // frame if we don't need to save to the stack. If we have to save to the - // stack, then we need a different frame. + // frame if we don't need to save to the stack. If we have to save to the + // stack, then we need a different frame. unsigned NumBytesNoStackCalls = 0; std::vector<outliner::Candidate> CandidatesWithoutStackFixups; for (outliner::Candidate &C : RepeatedSequenceLocs) { C.initLRU(TRI); - // LR liveness is overestimated in return blocks, unless they end with a - // tail call. - const auto Last = C.getMBB()->rbegin(); - const bool LRIsAvailable = - C.getMBB()->isReturnBlock() && !Last->isCall() - ? isLRAvailable(TRI, Last, - (MachineBasicBlock::reverse_iterator)C.front()) - : C.LRU.available(ARM::LR); - if (LRIsAvailable) { + // LR liveness is overestimated in return blocks, unless they end with a + // tail call. + const auto Last = C.getMBB()->rbegin(); + const bool LRIsAvailable = + C.getMBB()->isReturnBlock() && !Last->isCall() + ? isLRAvailable(TRI, Last, + (MachineBasicBlock::reverse_iterator)C.front()) + : C.LRU.available(ARM::LR); + if (LRIsAvailable) { FrameID = MachineOutlinerNoLRSave; NumBytesNoStackCalls += Costs.CallNoLRSave; C.setCallInfo(MachineOutlinerNoLRSave, Costs.CallNoLRSave); @@ -5865,157 +5865,157 @@ outliner::OutlinedFunction ARMBaseInstrInfo::getOutliningCandidateInfo( C.setCallInfo(MachineOutlinerRegSave, Costs.CallRegSave); CandidatesWithoutStackFixups.push_back(C); } - - // Is SP used in the sequence at all? If not, we don't have to modify - // the stack, so we are guaranteed to get the same frame. - else if (C.UsedInSequence.available(ARM::SP)) { - NumBytesNoStackCalls += Costs.CallDefault; - C.setCallInfo(MachineOutlinerDefault, Costs.CallDefault); - CandidatesWithoutStackFixups.push_back(C); - } - - // If we outline this, we need to modify the stack. Pretend we don't - // outline this by saving all of its bytes. - else - NumBytesNoStackCalls += SequenceSize; + + // Is SP used in the sequence at all? If not, we don't have to modify + // the stack, so we are guaranteed to get the same frame. + else if (C.UsedInSequence.available(ARM::SP)) { + NumBytesNoStackCalls += Costs.CallDefault; + C.setCallInfo(MachineOutlinerDefault, Costs.CallDefault); + CandidatesWithoutStackFixups.push_back(C); + } + + // If we outline this, we need to modify the stack. Pretend we don't + // outline this by saving all of its bytes. + else + NumBytesNoStackCalls += SequenceSize; } - // If there are no places where we have to save LR, then note that we don't - // have to update the stack. Otherwise, give every candidate the default - // call type - if (NumBytesNoStackCalls <= - RepeatedSequenceLocs.size() * Costs.CallDefault) { + // If there are no places where we have to save LR, then note that we don't + // have to update the stack. Otherwise, give every candidate the default + // call type + if (NumBytesNoStackCalls <= + RepeatedSequenceLocs.size() * Costs.CallDefault) { RepeatedSequenceLocs = CandidatesWithoutStackFixups; - FrameID = MachineOutlinerNoLRSave; + FrameID = MachineOutlinerNoLRSave; } else - SetCandidateCallInfo(MachineOutlinerDefault, Costs.CallDefault); - } - - // Does every candidate's MBB contain a call? If so, then we might have a - // call in the range. - if (FlagsSetInAll & MachineOutlinerMBBFlags::HasCalls) { - // check if the range contains a call. These require a save + restore of - // the link register. - if (std::any_of(FirstCand.front(), FirstCand.back(), - [](const MachineInstr &MI) { return MI.isCall(); })) - NumBytesToCreateFrame += Costs.SaveRestoreLROnStack; - - // Handle the last instruction separately. If it is tail call, then the - // last instruction is a call, we don't want to save + restore in this - // case. However, it could be possible that the last instruction is a - // call without it being valid to tail call this sequence. We should - // consider this as well. - else if (FrameID != MachineOutlinerThunk && - FrameID != MachineOutlinerTailCall && FirstCand.back()->isCall()) - NumBytesToCreateFrame += Costs.SaveRestoreLROnStack; - } - + SetCandidateCallInfo(MachineOutlinerDefault, Costs.CallDefault); + } + + // Does every candidate's MBB contain a call? If so, then we might have a + // call in the range. + if (FlagsSetInAll & MachineOutlinerMBBFlags::HasCalls) { + // check if the range contains a call. These require a save + restore of + // the link register. + if (std::any_of(FirstCand.front(), FirstCand.back(), + [](const MachineInstr &MI) { return MI.isCall(); })) + NumBytesToCreateFrame += Costs.SaveRestoreLROnStack; + + // Handle the last instruction separately. If it is tail call, then the + // last instruction is a call, we don't want to save + restore in this + // case. However, it could be possible that the last instruction is a + // call without it being valid to tail call this sequence. We should + // consider this as well. + else if (FrameID != MachineOutlinerThunk && + FrameID != MachineOutlinerTailCall && FirstCand.back()->isCall()) + NumBytesToCreateFrame += Costs.SaveRestoreLROnStack; + } + return outliner::OutlinedFunction(RepeatedSequenceLocs, SequenceSize, NumBytesToCreateFrame, FrameID); } -bool ARMBaseInstrInfo::checkAndUpdateStackOffset(MachineInstr *MI, - int64_t Fixup, - bool Updt) const { - int SPIdx = MI->findRegisterUseOperandIdx(ARM::SP); - unsigned AddrMode = (MI->getDesc().TSFlags & ARMII::AddrModeMask); - if (SPIdx < 0) - // No SP operand - return true; - else if (SPIdx != 1 && (AddrMode != ARMII::AddrModeT2_i8s4 || SPIdx != 2)) - // If SP is not the base register we can't do much - return false; - - // Stack might be involved but addressing mode doesn't handle any offset. - // Rq: AddrModeT1_[1|2|4] don't operate on SP - if (AddrMode == ARMII::AddrMode1 // Arithmetic instructions - || AddrMode == ARMII::AddrMode4 // Load/Store Multiple - || AddrMode == ARMII::AddrMode6 // Neon Load/Store Multiple - || AddrMode == ARMII::AddrModeT2_so // SP can't be used as based register - || AddrMode == ARMII::AddrModeT2_pc // PCrel access - || AddrMode == ARMII::AddrMode2 // Used by PRE and POST indexed LD/ST - || AddrMode == ARMII::AddrModeT2_i7 // v8.1-M MVE - || AddrMode == ARMII::AddrModeT2_i7s2 // v8.1-M MVE - || AddrMode == ARMII::AddrModeT2_i7s4 // v8.1-M sys regs VLDR/VSTR - || AddrMode == ARMII::AddrModeNone) - return false; - - unsigned NumOps = MI->getDesc().getNumOperands(); - unsigned ImmIdx = NumOps - 3; - - const MachineOperand &Offset = MI->getOperand(ImmIdx); - assert(Offset.isImm() && "Is not an immediate"); - int64_t OffVal = Offset.getImm(); - - if (OffVal < 0) - // Don't override data if the are below SP. - return false; - - unsigned NumBits = 0; - unsigned Scale = 1; - - switch (AddrMode) { - case ARMII::AddrMode3: - if (ARM_AM::getAM3Op(OffVal) == ARM_AM::sub) - return false; - OffVal = ARM_AM::getAM3Offset(OffVal); - NumBits = 8; - break; - case ARMII::AddrMode5: - if (ARM_AM::getAM5Op(OffVal) == ARM_AM::sub) - return false; - OffVal = ARM_AM::getAM5Offset(OffVal); - NumBits = 8; - Scale = 4; - break; - case ARMII::AddrMode5FP16: - if (ARM_AM::getAM5FP16Op(OffVal) == ARM_AM::sub) - return false; - OffVal = ARM_AM::getAM5FP16Offset(OffVal); - NumBits = 8; - Scale = 2; - break; - case ARMII::AddrModeT2_i8: - NumBits = 8; - break; - case ARMII::AddrModeT2_i8s4: - // FIXME: Values are already scaled in this addressing mode. - assert((Fixup & 3) == 0 && "Can't encode this offset!"); - NumBits = 10; - break; - case ARMII::AddrModeT2_ldrex: - NumBits = 8; - Scale = 4; - break; - case ARMII::AddrModeT2_i12: - case ARMII::AddrMode_i12: - NumBits = 12; - break; - case ARMII::AddrModeT1_s: // SP-relative LD/ST - NumBits = 8; - Scale = 4; - break; - default: - llvm_unreachable("Unsupported addressing mode!"); - } - // Make sure the offset is encodable for instructions that scale the - // immediate. - assert(((OffVal * Scale + Fixup) & (Scale - 1)) == 0 && - "Can't encode this offset!"); - OffVal += Fixup / Scale; - - unsigned Mask = (1 << NumBits) - 1; - - if (OffVal <= Mask) { - if (Updt) - MI->getOperand(ImmIdx).setImm(OffVal); - return true; - } - - return false; - -} - +bool ARMBaseInstrInfo::checkAndUpdateStackOffset(MachineInstr *MI, + int64_t Fixup, + bool Updt) const { + int SPIdx = MI->findRegisterUseOperandIdx(ARM::SP); + unsigned AddrMode = (MI->getDesc().TSFlags & ARMII::AddrModeMask); + if (SPIdx < 0) + // No SP operand + return true; + else if (SPIdx != 1 && (AddrMode != ARMII::AddrModeT2_i8s4 || SPIdx != 2)) + // If SP is not the base register we can't do much + return false; + + // Stack might be involved but addressing mode doesn't handle any offset. + // Rq: AddrModeT1_[1|2|4] don't operate on SP + if (AddrMode == ARMII::AddrMode1 // Arithmetic instructions + || AddrMode == ARMII::AddrMode4 // Load/Store Multiple + || AddrMode == ARMII::AddrMode6 // Neon Load/Store Multiple + || AddrMode == ARMII::AddrModeT2_so // SP can't be used as based register + || AddrMode == ARMII::AddrModeT2_pc // PCrel access + || AddrMode == ARMII::AddrMode2 // Used by PRE and POST indexed LD/ST + || AddrMode == ARMII::AddrModeT2_i7 // v8.1-M MVE + || AddrMode == ARMII::AddrModeT2_i7s2 // v8.1-M MVE + || AddrMode == ARMII::AddrModeT2_i7s4 // v8.1-M sys regs VLDR/VSTR + || AddrMode == ARMII::AddrModeNone) + return false; + + unsigned NumOps = MI->getDesc().getNumOperands(); + unsigned ImmIdx = NumOps - 3; + + const MachineOperand &Offset = MI->getOperand(ImmIdx); + assert(Offset.isImm() && "Is not an immediate"); + int64_t OffVal = Offset.getImm(); + + if (OffVal < 0) + // Don't override data if the are below SP. + return false; + + unsigned NumBits = 0; + unsigned Scale = 1; + + switch (AddrMode) { + case ARMII::AddrMode3: + if (ARM_AM::getAM3Op(OffVal) == ARM_AM::sub) + return false; + OffVal = ARM_AM::getAM3Offset(OffVal); + NumBits = 8; + break; + case ARMII::AddrMode5: + if (ARM_AM::getAM5Op(OffVal) == ARM_AM::sub) + return false; + OffVal = ARM_AM::getAM5Offset(OffVal); + NumBits = 8; + Scale = 4; + break; + case ARMII::AddrMode5FP16: + if (ARM_AM::getAM5FP16Op(OffVal) == ARM_AM::sub) + return false; + OffVal = ARM_AM::getAM5FP16Offset(OffVal); + NumBits = 8; + Scale = 2; + break; + case ARMII::AddrModeT2_i8: + NumBits = 8; + break; + case ARMII::AddrModeT2_i8s4: + // FIXME: Values are already scaled in this addressing mode. + assert((Fixup & 3) == 0 && "Can't encode this offset!"); + NumBits = 10; + break; + case ARMII::AddrModeT2_ldrex: + NumBits = 8; + Scale = 4; + break; + case ARMII::AddrModeT2_i12: + case ARMII::AddrMode_i12: + NumBits = 12; + break; + case ARMII::AddrModeT1_s: // SP-relative LD/ST + NumBits = 8; + Scale = 4; + break; + default: + llvm_unreachable("Unsupported addressing mode!"); + } + // Make sure the offset is encodable for instructions that scale the + // immediate. + assert(((OffVal * Scale + Fixup) & (Scale - 1)) == 0 && + "Can't encode this offset!"); + OffVal += Fixup / Scale; + + unsigned Mask = (1 << NumBits) - 1; + + if (OffVal <= Mask) { + if (Updt) + MI->getOperand(ImmIdx).setImm(OffVal); + return true; + } + + return false; + +} + bool ARMBaseInstrInfo::isFunctionSafeToOutlineFrom( MachineFunction &MF, bool OutlineFromLinkOnceODRs) const { const Function &F = MF.getFunction(); @@ -6075,13 +6075,13 @@ bool ARMBaseInstrInfo::isMBBSafeToOutlineFrom(MachineBasicBlock &MBB, if (any_of(MBB, [](MachineInstr &MI) { return MI.isCall(); })) Flags |= MachineOutlinerMBBFlags::HasCalls; - // LR liveness is overestimated in return blocks. - - bool LRIsAvailable = - MBB.isReturnBlock() && !MBB.back().isCall() - ? isLRAvailable(getRegisterInfo(), MBB.rbegin(), MBB.rend()) - : LRU.available(ARM::LR); - if (!LRIsAvailable) + // LR liveness is overestimated in return blocks. + + bool LRIsAvailable = + MBB.isReturnBlock() && !MBB.back().isCall() + ? isLRAvailable(getRegisterInfo(), MBB.rbegin(), MBB.rend()) + : LRU.available(ARM::LR); + if (!LRIsAvailable) Flags |= MachineOutlinerMBBFlags::LRUnavailableSomewhere; return true; @@ -6119,9 +6119,9 @@ ARMBaseInstrInfo::getOutliningType(MachineBasicBlock::iterator &MIT, // Be conservative with ARMv8.1 MVE instructions. if (Opc == ARM::t2BF_LabelPseudo || Opc == ARM::t2DoLoopStart || - Opc == ARM::t2DoLoopStartTP || Opc == ARM::t2WhileLoopStart || - Opc == ARM::t2LoopDec || Opc == ARM::t2LoopEnd || - Opc == ARM::t2LoopEndDec) + Opc == ARM::t2DoLoopStartTP || Opc == ARM::t2WhileLoopStart || + Opc == ARM::t2LoopDec || Opc == ARM::t2LoopEnd || + Opc == ARM::t2LoopEndDec) return outliner::InstrType::Illegal; const MCInstrDesc &MCID = MI.getDesc(); @@ -6155,56 +6155,56 @@ ARMBaseInstrInfo::getOutliningType(MachineBasicBlock::iterator &MIT, return outliner::InstrType::Illegal; if (MI.isCall()) { - // Get the function associated with the call. Look at each operand and find - // the one that represents the calle and get its name. - const Function *Callee = nullptr; - for (const MachineOperand &MOP : MI.operands()) { - if (MOP.isGlobal()) { - Callee = dyn_cast<Function>(MOP.getGlobal()); - break; - } - } - - // Dont't outline calls to "mcount" like functions, in particular Linux - // kernel function tracing relies on it. - if (Callee && - (Callee->getName() == "\01__gnu_mcount_nc" || - Callee->getName() == "\01mcount" || Callee->getName() == "__mcount")) - return outliner::InstrType::Illegal; - + // Get the function associated with the call. Look at each operand and find + // the one that represents the calle and get its name. + const Function *Callee = nullptr; + for (const MachineOperand &MOP : MI.operands()) { + if (MOP.isGlobal()) { + Callee = dyn_cast<Function>(MOP.getGlobal()); + break; + } + } + + // Dont't outline calls to "mcount" like functions, in particular Linux + // kernel function tracing relies on it. + if (Callee && + (Callee->getName() == "\01__gnu_mcount_nc" || + Callee->getName() == "\01mcount" || Callee->getName() == "__mcount")) + return outliner::InstrType::Illegal; + // If we don't know anything about the callee, assume it depends on the // stack layout of the caller. In that case, it's only legal to outline // as a tail-call. Explicitly list the call instructions we know about so // we don't get unexpected results with call pseudo-instructions. auto UnknownCallOutlineType = outliner::InstrType::Illegal; if (Opc == ARM::BL || Opc == ARM::tBL || Opc == ARM::BLX || - Opc == ARM::BLX_noip || Opc == ARM::tBLXr || Opc == ARM::tBLXr_noip || - Opc == ARM::tBLXi) + Opc == ARM::BLX_noip || Opc == ARM::tBLXr || Opc == ARM::tBLXr_noip || + Opc == ARM::tBLXi) UnknownCallOutlineType = outliner::InstrType::LegalTerminator; - if (!Callee) - return UnknownCallOutlineType; - - // We have a function we have information about. Check if it's something we - // can safely outline. - MachineFunction *MF = MI.getParent()->getParent(); - MachineFunction *CalleeMF = MF->getMMI().getMachineFunction(*Callee); - - // We don't know what's going on with the callee at all. Don't touch it. - if (!CalleeMF) - return UnknownCallOutlineType; - - // Check if we know anything about the callee saves on the function. If we - // don't, then don't touch it, since that implies that we haven't computed - // anything about its stack frame yet. - MachineFrameInfo &MFI = CalleeMF->getFrameInfo(); - if (!MFI.isCalleeSavedInfoValid() || MFI.getStackSize() > 0 || - MFI.getNumObjects() > 0) - return UnknownCallOutlineType; - - // At this point, we can say that CalleeMF ought to not pass anything on the - // stack. Therefore, we can outline it. - return outliner::InstrType::Legal; + if (!Callee) + return UnknownCallOutlineType; + + // We have a function we have information about. Check if it's something we + // can safely outline. + MachineFunction *MF = MI.getParent()->getParent(); + MachineFunction *CalleeMF = MF->getMMI().getMachineFunction(*Callee); + + // We don't know what's going on with the callee at all. Don't touch it. + if (!CalleeMF) + return UnknownCallOutlineType; + + // Check if we know anything about the callee saves on the function. If we + // don't, then don't touch it, since that implies that we haven't computed + // anything about its stack frame yet. + MachineFrameInfo &MFI = CalleeMF->getFrameInfo(); + if (!MFI.isCalleeSavedInfoValid() || MFI.getStackSize() > 0 || + MFI.getNumObjects() > 0) + return UnknownCallOutlineType; + + // At this point, we can say that CalleeMF ought to not pass anything on the + // stack. Therefore, we can outline it. + return outliner::InstrType::Legal; } // Since calls are handled, don't touch LR or PC @@ -6227,19 +6227,19 @@ ARMBaseInstrInfo::getOutliningType(MachineBasicBlock::iterator &MIT, if (!MightNeedStackFixUp) return outliner::InstrType::Legal; - // Any modification of SP will break our code to save/restore LR. - // FIXME: We could handle some instructions which add a constant offset to - // SP, with a bit more work. - if (MI.modifiesRegister(ARM::SP, TRI)) - return outliner::InstrType::Illegal; - - // At this point, we have a stack instruction that we might need to fix up. - // up. We'll handle it if it's a load or store. - if (checkAndUpdateStackOffset(&MI, Subtarget.getStackAlignment().value(), - false)) - return outliner::InstrType::Legal; - - // We can't fix it up, so don't outline it. + // Any modification of SP will break our code to save/restore LR. + // FIXME: We could handle some instructions which add a constant offset to + // SP, with a bit more work. + if (MI.modifiesRegister(ARM::SP, TRI)) + return outliner::InstrType::Illegal; + + // At this point, we have a stack instruction that we might need to fix up. + // up. We'll handle it if it's a load or store. + if (checkAndUpdateStackOffset(&MI, Subtarget.getStackAlignment().value(), + false)) + return outliner::InstrType::Legal; + + // We can't fix it up, so don't outline it. return outliner::InstrType::Illegal; } @@ -6255,104 +6255,104 @@ ARMBaseInstrInfo::getOutliningType(MachineBasicBlock::iterator &MIT, return outliner::InstrType::Legal; } -void ARMBaseInstrInfo::fixupPostOutline(MachineBasicBlock &MBB) const { - for (MachineInstr &MI : MBB) { - checkAndUpdateStackOffset(&MI, Subtarget.getStackAlignment().value(), true); - } -} - -void ARMBaseInstrInfo::saveLROnStack(MachineBasicBlock &MBB, - MachineBasicBlock::iterator It) const { - unsigned Opc = Subtarget.isThumb() ? ARM::t2STR_PRE : ARM::STR_PRE_IMM; - int Align = -Subtarget.getStackAlignment().value(); - BuildMI(MBB, It, DebugLoc(), get(Opc), ARM::SP) - .addReg(ARM::LR, RegState::Kill) - .addReg(ARM::SP) - .addImm(Align) - .add(predOps(ARMCC::AL)); -} - -void ARMBaseInstrInfo::emitCFIForLRSaveOnStack( - MachineBasicBlock &MBB, MachineBasicBlock::iterator It) const { - MachineFunction &MF = *MBB.getParent(); - const MCRegisterInfo *MRI = Subtarget.getRegisterInfo(); - unsigned DwarfLR = MRI->getDwarfRegNum(ARM::LR, true); - int Align = Subtarget.getStackAlignment().value(); - // Add a CFI saying the stack was moved down. - int64_t StackPosEntry = - MF.addFrameInst(MCCFIInstruction::cfiDefCfaOffset(nullptr, Align)); - BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION)) - .addCFIIndex(StackPosEntry) - .setMIFlags(MachineInstr::FrameSetup); - - // Add a CFI saying that the LR that we want to find is now higher than - // before. - int64_t LRPosEntry = - MF.addFrameInst(MCCFIInstruction::createOffset(nullptr, DwarfLR, -Align)); - BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION)) - .addCFIIndex(LRPosEntry) - .setMIFlags(MachineInstr::FrameSetup); -} - -void ARMBaseInstrInfo::emitCFIForLRSaveToReg(MachineBasicBlock &MBB, - MachineBasicBlock::iterator It, - Register Reg) const { - MachineFunction &MF = *MBB.getParent(); - const MCRegisterInfo *MRI = Subtarget.getRegisterInfo(); - unsigned DwarfLR = MRI->getDwarfRegNum(ARM::LR, true); - unsigned DwarfReg = MRI->getDwarfRegNum(Reg, true); - - int64_t LRPosEntry = MF.addFrameInst( - MCCFIInstruction::createRegister(nullptr, DwarfLR, DwarfReg)); - BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION)) - .addCFIIndex(LRPosEntry) - .setMIFlags(MachineInstr::FrameSetup); -} - -void ARMBaseInstrInfo::restoreLRFromStack( - MachineBasicBlock &MBB, MachineBasicBlock::iterator It) const { - unsigned Opc = Subtarget.isThumb() ? ARM::t2LDR_POST : ARM::LDR_POST_IMM; - MachineInstrBuilder MIB = BuildMI(MBB, It, DebugLoc(), get(Opc), ARM::LR) - .addReg(ARM::SP, RegState::Define) - .addReg(ARM::SP); - if (!Subtarget.isThumb()) - MIB.addReg(0); - MIB.addImm(Subtarget.getStackAlignment().value()).add(predOps(ARMCC::AL)); -} - -void ARMBaseInstrInfo::emitCFIForLRRestoreFromStack( - MachineBasicBlock &MBB, MachineBasicBlock::iterator It) const { - // Now stack has moved back up... - MachineFunction &MF = *MBB.getParent(); - const MCRegisterInfo *MRI = Subtarget.getRegisterInfo(); - unsigned DwarfLR = MRI->getDwarfRegNum(ARM::LR, true); - int64_t StackPosEntry = - MF.addFrameInst(MCCFIInstruction::cfiDefCfaOffset(nullptr, 0)); - BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION)) - .addCFIIndex(StackPosEntry) - .setMIFlags(MachineInstr::FrameDestroy); - - // ... and we have restored LR. - int64_t LRPosEntry = - MF.addFrameInst(MCCFIInstruction::createRestore(nullptr, DwarfLR)); - BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION)) - .addCFIIndex(LRPosEntry) - .setMIFlags(MachineInstr::FrameDestroy); -} - -void ARMBaseInstrInfo::emitCFIForLRRestoreFromReg( - MachineBasicBlock &MBB, MachineBasicBlock::iterator It) const { - MachineFunction &MF = *MBB.getParent(); - const MCRegisterInfo *MRI = Subtarget.getRegisterInfo(); - unsigned DwarfLR = MRI->getDwarfRegNum(ARM::LR, true); - - int64_t LRPosEntry = - MF.addFrameInst(MCCFIInstruction::createRestore(nullptr, DwarfLR)); - BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION)) - .addCFIIndex(LRPosEntry) - .setMIFlags(MachineInstr::FrameDestroy); -} - +void ARMBaseInstrInfo::fixupPostOutline(MachineBasicBlock &MBB) const { + for (MachineInstr &MI : MBB) { + checkAndUpdateStackOffset(&MI, Subtarget.getStackAlignment().value(), true); + } +} + +void ARMBaseInstrInfo::saveLROnStack(MachineBasicBlock &MBB, + MachineBasicBlock::iterator It) const { + unsigned Opc = Subtarget.isThumb() ? ARM::t2STR_PRE : ARM::STR_PRE_IMM; + int Align = -Subtarget.getStackAlignment().value(); + BuildMI(MBB, It, DebugLoc(), get(Opc), ARM::SP) + .addReg(ARM::LR, RegState::Kill) + .addReg(ARM::SP) + .addImm(Align) + .add(predOps(ARMCC::AL)); +} + +void ARMBaseInstrInfo::emitCFIForLRSaveOnStack( + MachineBasicBlock &MBB, MachineBasicBlock::iterator It) const { + MachineFunction &MF = *MBB.getParent(); + const MCRegisterInfo *MRI = Subtarget.getRegisterInfo(); + unsigned DwarfLR = MRI->getDwarfRegNum(ARM::LR, true); + int Align = Subtarget.getStackAlignment().value(); + // Add a CFI saying the stack was moved down. + int64_t StackPosEntry = + MF.addFrameInst(MCCFIInstruction::cfiDefCfaOffset(nullptr, Align)); + BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION)) + .addCFIIndex(StackPosEntry) + .setMIFlags(MachineInstr::FrameSetup); + + // Add a CFI saying that the LR that we want to find is now higher than + // before. + int64_t LRPosEntry = + MF.addFrameInst(MCCFIInstruction::createOffset(nullptr, DwarfLR, -Align)); + BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION)) + .addCFIIndex(LRPosEntry) + .setMIFlags(MachineInstr::FrameSetup); +} + +void ARMBaseInstrInfo::emitCFIForLRSaveToReg(MachineBasicBlock &MBB, + MachineBasicBlock::iterator It, + Register Reg) const { + MachineFunction &MF = *MBB.getParent(); + const MCRegisterInfo *MRI = Subtarget.getRegisterInfo(); + unsigned DwarfLR = MRI->getDwarfRegNum(ARM::LR, true); + unsigned DwarfReg = MRI->getDwarfRegNum(Reg, true); + + int64_t LRPosEntry = MF.addFrameInst( + MCCFIInstruction::createRegister(nullptr, DwarfLR, DwarfReg)); + BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION)) + .addCFIIndex(LRPosEntry) + .setMIFlags(MachineInstr::FrameSetup); +} + +void ARMBaseInstrInfo::restoreLRFromStack( + MachineBasicBlock &MBB, MachineBasicBlock::iterator It) const { + unsigned Opc = Subtarget.isThumb() ? ARM::t2LDR_POST : ARM::LDR_POST_IMM; + MachineInstrBuilder MIB = BuildMI(MBB, It, DebugLoc(), get(Opc), ARM::LR) + .addReg(ARM::SP, RegState::Define) + .addReg(ARM::SP); + if (!Subtarget.isThumb()) + MIB.addReg(0); + MIB.addImm(Subtarget.getStackAlignment().value()).add(predOps(ARMCC::AL)); +} + +void ARMBaseInstrInfo::emitCFIForLRRestoreFromStack( + MachineBasicBlock &MBB, MachineBasicBlock::iterator It) const { + // Now stack has moved back up... + MachineFunction &MF = *MBB.getParent(); + const MCRegisterInfo *MRI = Subtarget.getRegisterInfo(); + unsigned DwarfLR = MRI->getDwarfRegNum(ARM::LR, true); + int64_t StackPosEntry = + MF.addFrameInst(MCCFIInstruction::cfiDefCfaOffset(nullptr, 0)); + BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION)) + .addCFIIndex(StackPosEntry) + .setMIFlags(MachineInstr::FrameDestroy); + + // ... and we have restored LR. + int64_t LRPosEntry = + MF.addFrameInst(MCCFIInstruction::createRestore(nullptr, DwarfLR)); + BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION)) + .addCFIIndex(LRPosEntry) + .setMIFlags(MachineInstr::FrameDestroy); +} + +void ARMBaseInstrInfo::emitCFIForLRRestoreFromReg( + MachineBasicBlock &MBB, MachineBasicBlock::iterator It) const { + MachineFunction &MF = *MBB.getParent(); + const MCRegisterInfo *MRI = Subtarget.getRegisterInfo(); + unsigned DwarfLR = MRI->getDwarfRegNum(ARM::LR, true); + + int64_t LRPosEntry = + MF.addFrameInst(MCCFIInstruction::createRestore(nullptr, DwarfLR)); + BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION)) + .addCFIIndex(LRPosEntry) + .setMIFlags(MachineInstr::FrameDestroy); +} + void ARMBaseInstrInfo::buildOutlinedFrame( MachineBasicBlock &MBB, MachineFunction &MF, const outliner::OutlinedFunction &OF) const { @@ -6374,57 +6374,57 @@ void ARMBaseInstrInfo::buildOutlinedFrame( Call->eraseFromParent(); } - // Is there a call in the outlined range? - auto IsNonTailCall = [](MachineInstr &MI) { - return MI.isCall() && !MI.isReturn(); - }; - if (llvm::any_of(MBB.instrs(), IsNonTailCall)) { - MachineBasicBlock::iterator It = MBB.begin(); - MachineBasicBlock::iterator Et = MBB.end(); - - if (OF.FrameConstructionID == MachineOutlinerTailCall || - OF.FrameConstructionID == MachineOutlinerThunk) - Et = std::prev(MBB.end()); - - // We have to save and restore LR, we need to add it to the liveins if it - // is not already part of the set. This is suffient since outlined - // functions only have one block. - if (!MBB.isLiveIn(ARM::LR)) - MBB.addLiveIn(ARM::LR); - - // Insert a save before the outlined region - saveLROnStack(MBB, It); - emitCFIForLRSaveOnStack(MBB, It); - - // Fix up the instructions in the range, since we're going to modify the - // stack. - assert(OF.FrameConstructionID != MachineOutlinerDefault && - "Can only fix up stack references once"); - fixupPostOutline(MBB); - - // Insert a restore before the terminator for the function. Restore LR. - restoreLRFromStack(MBB, Et); - emitCFIForLRRestoreFromStack(MBB, Et); - } - - // If this is a tail call outlined function, then there's already a return. - if (OF.FrameConstructionID == MachineOutlinerTailCall || - OF.FrameConstructionID == MachineOutlinerThunk) - return; - + // Is there a call in the outlined range? + auto IsNonTailCall = [](MachineInstr &MI) { + return MI.isCall() && !MI.isReturn(); + }; + if (llvm::any_of(MBB.instrs(), IsNonTailCall)) { + MachineBasicBlock::iterator It = MBB.begin(); + MachineBasicBlock::iterator Et = MBB.end(); + + if (OF.FrameConstructionID == MachineOutlinerTailCall || + OF.FrameConstructionID == MachineOutlinerThunk) + Et = std::prev(MBB.end()); + + // We have to save and restore LR, we need to add it to the liveins if it + // is not already part of the set. This is suffient since outlined + // functions only have one block. + if (!MBB.isLiveIn(ARM::LR)) + MBB.addLiveIn(ARM::LR); + + // Insert a save before the outlined region + saveLROnStack(MBB, It); + emitCFIForLRSaveOnStack(MBB, It); + + // Fix up the instructions in the range, since we're going to modify the + // stack. + assert(OF.FrameConstructionID != MachineOutlinerDefault && + "Can only fix up stack references once"); + fixupPostOutline(MBB); + + // Insert a restore before the terminator for the function. Restore LR. + restoreLRFromStack(MBB, Et); + emitCFIForLRRestoreFromStack(MBB, Et); + } + + // If this is a tail call outlined function, then there's already a return. + if (OF.FrameConstructionID == MachineOutlinerTailCall || + OF.FrameConstructionID == MachineOutlinerThunk) + return; + // Here we have to insert the return ourselves. Get the correct opcode from // current feature set. BuildMI(MBB, MBB.end(), DebugLoc(), get(Subtarget.getReturnOpcode())) .add(predOps(ARMCC::AL)); - - // Did we have to modify the stack by saving the link register? - if (OF.FrameConstructionID != MachineOutlinerDefault && - OF.Candidates[0].CallConstructionID != MachineOutlinerDefault) - return; - - // We modified the stack. - // Walk over the basic block and fix up all the stack accesses. - fixupPostOutline(MBB); + + // Did we have to modify the stack by saving the link register? + if (OF.FrameConstructionID != MachineOutlinerDefault && + OF.Candidates[0].CallConstructionID != MachineOutlinerDefault) + return; + + // We modified the stack. + // Walk over the basic block and fix up all the stack accesses. + fixupPostOutline(MBB); } MachineBasicBlock::iterator ARMBaseInstrInfo::insertOutlinedCall( @@ -6456,14 +6456,14 @@ MachineBasicBlock::iterator ARMBaseInstrInfo::insertOutlinedCall( CallMIB.add(predOps(ARMCC::AL)); CallMIB.addGlobalAddress(M.getNamedValue(MF.getName())); - if (C.CallConstructionID == MachineOutlinerNoLRSave || - C.CallConstructionID == MachineOutlinerThunk) { - // No, so just insert the call. - It = MBB.insert(It, CallMIB); - return It; - } - - const ARMFunctionInfo &AFI = *C.getMF()->getInfo<ARMFunctionInfo>(); + if (C.CallConstructionID == MachineOutlinerNoLRSave || + C.CallConstructionID == MachineOutlinerThunk) { + // No, so just insert the call. + It = MBB.insert(It, CallMIB); + return It; + } + + const ARMFunctionInfo &AFI = *C.getMF()->getInfo<ARMFunctionInfo>(); // Can we save to a register? if (C.CallConstructionID == MachineOutlinerRegSave) { unsigned Reg = findRegisterToSaveLRTo(C); @@ -6471,55 +6471,55 @@ MachineBasicBlock::iterator ARMBaseInstrInfo::insertOutlinedCall( // Save and restore LR from that register. copyPhysReg(MBB, It, DebugLoc(), Reg, ARM::LR, true); - if (!AFI.isLRSpilled()) - emitCFIForLRSaveToReg(MBB, It, Reg); + if (!AFI.isLRSpilled()) + emitCFIForLRSaveToReg(MBB, It, Reg); CallPt = MBB.insert(It, CallMIB); copyPhysReg(MBB, It, DebugLoc(), ARM::LR, Reg, true); - if (!AFI.isLRSpilled()) - emitCFIForLRRestoreFromReg(MBB, It); + if (!AFI.isLRSpilled()) + emitCFIForLRRestoreFromReg(MBB, It); It--; return CallPt; } - // We have the default case. Save and restore from SP. - if (!MBB.isLiveIn(ARM::LR)) - MBB.addLiveIn(ARM::LR); - saveLROnStack(MBB, It); - if (!AFI.isLRSpilled()) - emitCFIForLRSaveOnStack(MBB, It); - CallPt = MBB.insert(It, CallMIB); - restoreLRFromStack(MBB, It); - if (!AFI.isLRSpilled()) - emitCFIForLRRestoreFromStack(MBB, It); - It--; - return CallPt; -} - -bool ARMBaseInstrInfo::shouldOutlineFromFunctionByDefault( - MachineFunction &MF) const { - return Subtarget.isMClass() && MF.getFunction().hasMinSize(); -} - -bool ARMBaseInstrInfo::isReallyTriviallyReMaterializable(const MachineInstr &MI, - AAResults *AA) const { - // Try hard to rematerialize any VCTPs because if we spill P0, it will block - // the tail predication conversion. This means that the element count - // register has to be live for longer, but that has to be better than - // spill/restore and VPT predication. - return isVCTP(&MI) && !isPredicated(MI); -} - -unsigned llvm::getBLXOpcode(const MachineFunction &MF) { - return (MF.getSubtarget<ARMSubtarget>().hardenSlsBlr()) ? ARM::BLX_noip - : ARM::BLX; -} - -unsigned llvm::gettBLXrOpcode(const MachineFunction &MF) { - return (MF.getSubtarget<ARMSubtarget>().hardenSlsBlr()) ? ARM::tBLXr_noip - : ARM::tBLXr; -} - -unsigned llvm::getBLXpredOpcode(const MachineFunction &MF) { - return (MF.getSubtarget<ARMSubtarget>().hardenSlsBlr()) ? ARM::BLX_pred_noip - : ARM::BLX_pred; -} - + // We have the default case. Save and restore from SP. + if (!MBB.isLiveIn(ARM::LR)) + MBB.addLiveIn(ARM::LR); + saveLROnStack(MBB, It); + if (!AFI.isLRSpilled()) + emitCFIForLRSaveOnStack(MBB, It); + CallPt = MBB.insert(It, CallMIB); + restoreLRFromStack(MBB, It); + if (!AFI.isLRSpilled()) + emitCFIForLRRestoreFromStack(MBB, It); + It--; + return CallPt; +} + +bool ARMBaseInstrInfo::shouldOutlineFromFunctionByDefault( + MachineFunction &MF) const { + return Subtarget.isMClass() && MF.getFunction().hasMinSize(); +} + +bool ARMBaseInstrInfo::isReallyTriviallyReMaterializable(const MachineInstr &MI, + AAResults *AA) const { + // Try hard to rematerialize any VCTPs because if we spill P0, it will block + // the tail predication conversion. This means that the element count + // register has to be live for longer, but that has to be better than + // spill/restore and VPT predication. + return isVCTP(&MI) && !isPredicated(MI); +} + +unsigned llvm::getBLXOpcode(const MachineFunction &MF) { + return (MF.getSubtarget<ARMSubtarget>().hardenSlsBlr()) ? ARM::BLX_noip + : ARM::BLX; +} + +unsigned llvm::gettBLXrOpcode(const MachineFunction &MF) { + return (MF.getSubtarget<ARMSubtarget>().hardenSlsBlr()) ? ARM::tBLXr_noip + : ARM::tBLXr; +} + +unsigned llvm::getBLXpredOpcode(const MachineFunction &MF) { + return (MF.getSubtarget<ARMSubtarget>().hardenSlsBlr()) ? ARM::BLX_pred_noip + : ARM::BLX_pred; +} + |