diff options
author | shadchin <shadchin@yandex-team.ru> | 2022-02-10 16:44:30 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:44:30 +0300 |
commit | 2598ef1d0aee359b4b6d5fdd1758916d5907d04f (patch) | |
tree | 012bb94d777798f1f56ac1cec429509766d05181 /contrib/libs/llvm12/lib/CodeGen/RegAllocGreedy.cpp | |
parent | 6751af0b0c1b952fede40b19b71da8025b5d8bcf (diff) | |
download | ydb-2598ef1d0aee359b4b6d5fdd1758916d5907d04f.tar.gz |
Restoring authorship annotation for <shadchin@yandex-team.ru>. Commit 1 of 2.
Diffstat (limited to 'contrib/libs/llvm12/lib/CodeGen/RegAllocGreedy.cpp')
-rw-r--r-- | contrib/libs/llvm12/lib/CodeGen/RegAllocGreedy.cpp | 550 |
1 files changed, 275 insertions, 275 deletions
diff --git a/contrib/libs/llvm12/lib/CodeGen/RegAllocGreedy.cpp b/contrib/libs/llvm12/lib/CodeGen/RegAllocGreedy.cpp index 166414e4ff..8a991fdbba 100644 --- a/contrib/libs/llvm12/lib/CodeGen/RegAllocGreedy.cpp +++ b/contrib/libs/llvm12/lib/CodeGen/RegAllocGreedy.cpp @@ -147,7 +147,7 @@ class RAGreedy : public MachineFunctionPass, // Convenient shortcuts. using PQueue = std::priority_queue<std::pair<unsigned, unsigned>>; using SmallLISet = SmallPtrSet<LiveInterval *, 4>; - using SmallVirtRegSet = SmallSet<Register, 16>; + using SmallVirtRegSet = SmallSet<Register, 16>; // context MachineFunction *MF; @@ -172,7 +172,7 @@ class RAGreedy : public MachineFunctionPass, std::unique_ptr<Spiller> SpillerInstance; PQueue Queue; unsigned NextCascade; - std::unique_ptr<VirtRegAuxInfo> VRAI; + std::unique_ptr<VirtRegAuxInfo> VRAI; // Live ranges pass through a number of stages as we try to allocate them. // Some of the stages may also create new live ranges: @@ -248,19 +248,19 @@ class RAGreedy : public MachineFunctionPass, IndexedMap<RegInfo, VirtReg2IndexFunctor> ExtraRegInfo; LiveRangeStage getStage(const LiveInterval &VirtReg) const { - return ExtraRegInfo[VirtReg.reg()].Stage; + return ExtraRegInfo[VirtReg.reg()].Stage; } void setStage(const LiveInterval &VirtReg, LiveRangeStage Stage) { ExtraRegInfo.resize(MRI->getNumVirtRegs()); - ExtraRegInfo[VirtReg.reg()].Stage = Stage; + ExtraRegInfo[VirtReg.reg()].Stage = Stage; } template<typename Iterator> void setStage(Iterator Begin, Iterator End, LiveRangeStage NewStage) { ExtraRegInfo.resize(MRI->getNumVirtRegs()); for (;Begin != End; ++Begin) { - Register Reg = *Begin; + Register Reg = *Begin; if (ExtraRegInfo[Reg].Stage == RS_New) ExtraRegInfo[Reg].Stage = NewStage; } @@ -291,8 +291,8 @@ class RAGreedy : public MachineFunctionPass, public: using EvictorInfo = - std::pair<Register /* evictor */, MCRegister /* physreg */>; - using EvicteeInfo = llvm::DenseMap<Register /* evictee */, EvictorInfo>; + std::pair<Register /* evictor */, MCRegister /* physreg */>; + using EvicteeInfo = llvm::DenseMap<Register /* evictee */, EvictorInfo>; private: /// Each Vreg that has been evicted in the last stage of selectOrSplit will @@ -308,14 +308,14 @@ class RAGreedy : public MachineFunctionPass, /// longer relevant. /// \param Evictee The evictee Vreg for whom we want to clear collected /// eviction info. - void clearEvicteeInfo(Register Evictee) { Evictees.erase(Evictee); } + void clearEvicteeInfo(Register Evictee) { Evictees.erase(Evictee); } /// Track new eviction. /// The Evictor vreg has evicted the Evictee vreg from Physreg. /// \param PhysReg The physical register Evictee was evicted from. /// \param Evictor The evictor Vreg that evicted Evictee. /// \param Evictee The evictee Vreg. - void addEviction(MCRegister PhysReg, Register Evictor, Register Evictee) { + void addEviction(MCRegister PhysReg, Register Evictor, Register Evictee) { Evictees[Evictee].first = Evictor; Evictees[Evictee].second = PhysReg; } @@ -324,7 +324,7 @@ class RAGreedy : public MachineFunctionPass, /// \param Evictee The evictee vreg. /// \return The Evictor vreg which evicted Evictee vreg from PhysReg. 0 if /// nobody has evicted Evictee from PhysReg. - EvictorInfo getEvictor(Register Evictee) { + EvictorInfo getEvictor(Register Evictee) { if (Evictees.count(Evictee)) { return Evictees[Evictee]; } @@ -349,7 +349,7 @@ class RAGreedy : public MachineFunctionPass, /// Global live range splitting candidate info. struct GlobalSplitCandidate { // Register intended for assignment, or 0. - MCRegister PhysReg; + MCRegister PhysReg; // SplitKit interval index for this candidate. unsigned IntvIdx; @@ -361,7 +361,7 @@ class RAGreedy : public MachineFunctionPass, BitVector LiveBundles; SmallVector<unsigned, 8> ActiveBlocks; - void reset(InterferenceCache &Cache, MCRegister Reg) { + void reset(InterferenceCache &Cache, MCRegister Reg) { PhysReg = Reg; IntvIdx = 0; Intf.setPhysReg(Cache, Reg); @@ -369,12 +369,12 @@ class RAGreedy : public MachineFunctionPass, ActiveBlocks.clear(); } - // Set B[I] = C for every live bundle where B[I] was NoCand. + // Set B[I] = C for every live bundle where B[I] was NoCand. unsigned getBundles(SmallVectorImpl<unsigned> &B, unsigned C) { unsigned Count = 0; - for (unsigned I : LiveBundles.set_bits()) - if (B[I] == NoCand) { - B[I] = C; + for (unsigned I : LiveBundles.set_bits()) + if (B[I] == NoCand) { + B[I] = C; Count++; } return Count; @@ -418,8 +418,8 @@ public: Spiller &spiller() override { return *SpillerInstance; } void enqueue(LiveInterval *LI) override; LiveInterval *dequeue() override; - MCRegister selectOrSplit(LiveInterval &, - SmallVectorImpl<Register> &) override; + MCRegister selectOrSplit(LiveInterval &, + SmallVectorImpl<Register> &) override; void aboutToRemoveInterval(LiveInterval &) override; /// Perform register allocation. @@ -430,20 +430,20 @@ public: MachineFunctionProperties::Property::NoPHIs); } - MachineFunctionProperties getClearedProperties() const override { - return MachineFunctionProperties().set( - MachineFunctionProperties::Property::IsSSA); - } - + MachineFunctionProperties getClearedProperties() const override { + return MachineFunctionProperties().set( + MachineFunctionProperties::Property::IsSSA); + } + static char ID; private: - MCRegister selectOrSplitImpl(LiveInterval &, SmallVectorImpl<Register> &, - SmallVirtRegSet &, unsigned = 0); + MCRegister selectOrSplitImpl(LiveInterval &, SmallVectorImpl<Register> &, + SmallVirtRegSet &, unsigned = 0); - bool LRE_CanEraseVirtReg(Register) override; - void LRE_WillShrinkVirtReg(Register) override; - void LRE_DidCloneVirtReg(Register, Register) override; + bool LRE_CanEraseVirtReg(Register) override; + void LRE_WillShrinkVirtReg(Register) override; + void LRE_DidCloneVirtReg(Register, Register) override; void enqueue(PQueue &CurQueue, LiveInterval *LI); LiveInterval *dequeue(PQueue &CurQueue); @@ -451,7 +451,7 @@ private: bool addSplitConstraints(InterferenceCache::Cursor, BlockFrequency&); bool addThroughConstraints(InterferenceCache::Cursor, ArrayRef<unsigned>); bool growRegion(GlobalSplitCandidate &Cand); - bool splitCanCauseEvictionChain(Register Evictee, GlobalSplitCandidate &Cand, + bool splitCanCauseEvictionChain(Register Evictee, GlobalSplitCandidate &Cand, unsigned BBNumber, const AllocationOrder &Order); bool splitCanCauseLocalSpill(unsigned VirtRegToSplit, @@ -462,20 +462,20 @@ private: bool *CanCauseEvictionChain); bool calcCompactRegion(GlobalSplitCandidate&); void splitAroundRegion(LiveRangeEdit&, ArrayRef<unsigned>); - void calcGapWeights(MCRegister, SmallVectorImpl<float> &); + void calcGapWeights(MCRegister, SmallVectorImpl<float> &); Register canReassign(LiveInterval &VirtReg, Register PrevReg); bool shouldEvict(LiveInterval &A, bool, LiveInterval &B, bool); - bool canEvictInterference(LiveInterval &, MCRegister, bool, EvictionCost &, - const SmallVirtRegSet &); - bool canEvictInterferenceInRange(LiveInterval &VirtReg, MCRegister PhysReg, + bool canEvictInterference(LiveInterval &, MCRegister, bool, EvictionCost &, + const SmallVirtRegSet &); + bool canEvictInterferenceInRange(LiveInterval &VirtReg, MCRegister PhysReg, SlotIndex Start, SlotIndex End, EvictionCost &MaxCost); - MCRegister getCheapestEvicteeWeight(const AllocationOrder &Order, - LiveInterval &VirtReg, SlotIndex Start, - SlotIndex End, float *BestEvictWeight); - void evictInterference(LiveInterval &, MCRegister, - SmallVectorImpl<Register> &); - bool mayRecolorAllInterferences(MCRegister PhysReg, LiveInterval &VirtReg, + MCRegister getCheapestEvicteeWeight(const AllocationOrder &Order, + LiveInterval &VirtReg, SlotIndex Start, + SlotIndex End, float *BestEvictWeight); + void evictInterference(LiveInterval &, MCRegister, + SmallVectorImpl<Register> &); + bool mayRecolorAllInterferences(MCRegister PhysReg, LiveInterval &VirtReg, SmallLISet &RecoloringCandidates, const SmallVirtRegSet &FixedRegisters); @@ -485,8 +485,8 @@ private: unsigned tryEvict(LiveInterval&, AllocationOrder&, SmallVectorImpl<Register>&, unsigned, const SmallVirtRegSet&); - MCRegister tryRegionSplit(LiveInterval &, AllocationOrder &, - SmallVectorImpl<Register> &); + MCRegister tryRegionSplit(LiveInterval &, AllocationOrder &, + SmallVectorImpl<Register> &); /// Calculate cost of region splitting. unsigned calculateRegionSplitCost(LiveInterval &VirtReg, AllocationOrder &Order, @@ -499,10 +499,10 @@ private: SmallVectorImpl<Register> &NewVRegs); /// Check other options before using a callee-saved register for the first /// time. - MCRegister tryAssignCSRFirstTime(LiveInterval &VirtReg, - AllocationOrder &Order, MCRegister PhysReg, - unsigned &CostPerUseLimit, - SmallVectorImpl<Register> &NewVRegs); + MCRegister tryAssignCSRFirstTime(LiveInterval &VirtReg, + AllocationOrder &Order, MCRegister PhysReg, + unsigned &CostPerUseLimit, + SmallVectorImpl<Register> &NewVRegs); void initializeCSRCost(); unsigned tryBlockSplit(LiveInterval&, AllocationOrder&, SmallVectorImpl<Register>&); @@ -536,8 +536,8 @@ private: }; using HintsInfo = SmallVector<HintInfo, 4>; - BlockFrequency getBrokenHintFreq(const HintsInfo &, MCRegister); - void collectHintInfo(Register, HintsInfo &); + BlockFrequency getBrokenHintFreq(const HintsInfo &, MCRegister); + void collectHintInfo(Register, HintsInfo &); bool isUnusedCalleeSavedReg(MCRegister PhysReg) const; @@ -634,7 +634,7 @@ void RAGreedy::getAnalysisUsage(AnalysisUsage &AU) const { // LiveRangeEdit delegate methods //===----------------------------------------------------------------------===// -bool RAGreedy::LRE_CanEraseVirtReg(Register VirtReg) { +bool RAGreedy::LRE_CanEraseVirtReg(Register VirtReg) { LiveInterval &LI = LIS->getInterval(VirtReg); if (VRM->hasPhys(VirtReg)) { Matrix->unassign(LI); @@ -649,7 +649,7 @@ bool RAGreedy::LRE_CanEraseVirtReg(Register VirtReg) { return false; } -void RAGreedy::LRE_WillShrinkVirtReg(Register VirtReg) { +void RAGreedy::LRE_WillShrinkVirtReg(Register VirtReg) { if (!VRM->hasPhys(VirtReg)) return; @@ -659,7 +659,7 @@ void RAGreedy::LRE_WillShrinkVirtReg(Register VirtReg) { enqueue(&LI); } -void RAGreedy::LRE_DidCloneVirtReg(Register New, Register Old) { +void RAGreedy::LRE_DidCloneVirtReg(Register New, Register Old) { // Cloning a register we haven't even heard about yet? Just ignore it. if (!ExtraRegInfo.inBounds(Old)) return; @@ -685,8 +685,8 @@ void RAGreedy::enqueue(PQueue &CurQueue, LiveInterval *LI) { // Prioritize live ranges by size, assigning larger ranges first. // The queue holds (size, reg) pairs. const unsigned Size = LI->getSize(); - const Register Reg = LI->reg(); - assert(Reg.isVirtual() && "Can only enqueue virtual registers"); + const Register Reg = LI->reg(); + assert(Reg.isVirtual() && "Can only enqueue virtual registers"); unsigned Prio; ExtraRegInfo.grow(Reg); @@ -764,32 +764,32 @@ Register RAGreedy::tryAssign(LiveInterval &VirtReg, SmallVectorImpl<Register> &NewVRegs, const SmallVirtRegSet &FixedRegisters) { Register PhysReg; - for (auto I = Order.begin(), E = Order.end(); I != E && !PhysReg; ++I) { - assert(*I); - if (!Matrix->checkInterference(VirtReg, *I)) { - if (I.isHint()) - return *I; - else - PhysReg = *I; - } - } - if (!PhysReg.isValid()) + for (auto I = Order.begin(), E = Order.end(); I != E && !PhysReg; ++I) { + assert(*I); + if (!Matrix->checkInterference(VirtReg, *I)) { + if (I.isHint()) + return *I; + else + PhysReg = *I; + } + } + if (!PhysReg.isValid()) return PhysReg; // PhysReg is available, but there may be a better choice. // If we missed a simple hint, try to cheaply evict interference from the // preferred register. - if (Register Hint = MRI->getSimpleHint(VirtReg.reg())) + if (Register Hint = MRI->getSimpleHint(VirtReg.reg())) if (Order.isHint(Hint)) { - MCRegister PhysHint = Hint.asMCReg(); - LLVM_DEBUG(dbgs() << "missed hint " << printReg(PhysHint, TRI) << '\n'); + MCRegister PhysHint = Hint.asMCReg(); + LLVM_DEBUG(dbgs() << "missed hint " << printReg(PhysHint, TRI) << '\n'); EvictionCost MaxCost; MaxCost.setBrokenHints(1); - if (canEvictInterference(VirtReg, PhysHint, true, MaxCost, - FixedRegisters)) { - evictInterference(VirtReg, PhysHint, NewVRegs); - return PhysHint; + if (canEvictInterference(VirtReg, PhysHint, true, MaxCost, + FixedRegisters)) { + evictInterference(VirtReg, PhysHint, NewVRegs); + return PhysHint; } // Record the missed hint, we may be able to recover // at the end if the surrounding allocation changed. @@ -814,14 +814,14 @@ Register RAGreedy::tryAssign(LiveInterval &VirtReg, //===----------------------------------------------------------------------===// Register RAGreedy::canReassign(LiveInterval &VirtReg, Register PrevReg) { - auto Order = - AllocationOrder::create(VirtReg.reg(), *VRM, RegClassInfo, Matrix); - MCRegister PhysReg; - for (auto I = Order.begin(), E = Order.end(); I != E && !PhysReg; ++I) { - if ((*I).id() == PrevReg.id()) + auto Order = + AllocationOrder::create(VirtReg.reg(), *VRM, RegClassInfo, Matrix); + MCRegister PhysReg; + for (auto I = Order.begin(), E = Order.end(); I != E && !PhysReg; ++I) { + if ((*I).id() == PrevReg.id()) continue; - MCRegUnitIterator Units(*I, TRI); + MCRegUnitIterator Units(*I, TRI); for (; Units.isValid(); ++Units) { // Instantiate a "subquery", not to be confused with the Queries array. LiveIntervalUnion::Query subQ(VirtReg, Matrix->getLiveUnions()[*Units]); @@ -830,7 +830,7 @@ Register RAGreedy::canReassign(LiveInterval &VirtReg, Register PrevReg) { } // If no units have interference, break out with the current PhysReg. if (!Units.isValid()) - PhysReg = *I; + PhysReg = *I; } if (PhysReg) LLVM_DEBUG(dbgs() << "can reassign: " << VirtReg << " from " @@ -861,8 +861,8 @@ bool RAGreedy::shouldEvict(LiveInterval &A, bool IsHint, if (CanSplit && IsHint && !BreaksHint) return true; - if (A.weight() > B.weight()) { - LLVM_DEBUG(dbgs() << "should evict: " << B << " w= " << B.weight() << '\n'); + if (A.weight() > B.weight()) { + LLVM_DEBUG(dbgs() << "should evict: " << B << " w= " << B.weight() << '\n'); return true; } return false; @@ -877,7 +877,7 @@ bool RAGreedy::shouldEvict(LiveInterval &A, bool IsHint, /// @param MaxCost Only look for cheaper candidates and update with new cost /// when returning true. /// @returns True when interference can be evicted cheaper than MaxCost. -bool RAGreedy::canEvictInterference(LiveInterval &VirtReg, MCRegister PhysReg, +bool RAGreedy::canEvictInterference(LiveInterval &VirtReg, MCRegister PhysReg, bool IsHint, EvictionCost &MaxCost, const SmallVirtRegSet &FixedRegisters) { // It is only possible to evict virtual register interference. @@ -893,7 +893,7 @@ bool RAGreedy::canEvictInterference(LiveInterval &VirtReg, MCRegister PhysReg, // // This works out so a register without a cascade number is allowed to evict // anything, and it can be evicted by anything. - unsigned Cascade = ExtraRegInfo[VirtReg.reg()].Cascade; + unsigned Cascade = ExtraRegInfo[VirtReg.reg()].Cascade; if (!Cascade) Cascade = NextCascade; @@ -905,14 +905,14 @@ bool RAGreedy::canEvictInterference(LiveInterval &VirtReg, MCRegister PhysReg, return false; // Check if any interfering live range is heavier than MaxWeight. - for (LiveInterval *Intf : reverse(Q.interferingVRegs())) { - assert(Register::isVirtualRegister(Intf->reg()) && + for (LiveInterval *Intf : reverse(Q.interferingVRegs())) { + assert(Register::isVirtualRegister(Intf->reg()) && "Only expecting virtual register interference from query"); // Do not allow eviction of a virtual register if we are in the middle // of last-chance recoloring and this virtual register is one that we // have scavenged a physical register for. - if (FixedRegisters.count(Intf->reg())) + if (FixedRegisters.count(Intf->reg())) return false; // Never evict spill products. They cannot split or spill. @@ -924,14 +924,14 @@ bool RAGreedy::canEvictInterference(LiveInterval &VirtReg, MCRegister PhysReg, // // Also allow urgent evictions of unspillable ranges from a strictly // larger allocation order. - bool Urgent = - !VirtReg.isSpillable() && - (Intf->isSpillable() || - RegClassInfo.getNumAllocatableRegs(MRI->getRegClass(VirtReg.reg())) < - RegClassInfo.getNumAllocatableRegs( - MRI->getRegClass(Intf->reg()))); + bool Urgent = + !VirtReg.isSpillable() && + (Intf->isSpillable() || + RegClassInfo.getNumAllocatableRegs(MRI->getRegClass(VirtReg.reg())) < + RegClassInfo.getNumAllocatableRegs( + MRI->getRegClass(Intf->reg()))); // Only evict older cascades or live ranges without a cascade. - unsigned IntfCascade = ExtraRegInfo[Intf->reg()].Cascade; + unsigned IntfCascade = ExtraRegInfo[Intf->reg()].Cascade; if (Cascade <= IntfCascade) { if (!Urgent) return false; @@ -940,10 +940,10 @@ bool RAGreedy::canEvictInterference(LiveInterval &VirtReg, MCRegister PhysReg, Cost.BrokenHints += 10; } // Would this break a satisfied hint? - bool BreaksHint = VRM->hasPreferredPhys(Intf->reg()); + bool BreaksHint = VRM->hasPreferredPhys(Intf->reg()); // Update eviction cost. Cost.BrokenHints += BreaksHint; - Cost.MaxWeight = std::max(Cost.MaxWeight, Intf->weight()); + Cost.MaxWeight = std::max(Cost.MaxWeight, Intf->weight()); // Abort if this would be too expensive. if (!(Cost < MaxCost)) return false; @@ -976,7 +976,7 @@ bool RAGreedy::canEvictInterference(LiveInterval &VirtReg, MCRegister PhysReg, /// when returning true. /// \return True when interference can be evicted cheaper than MaxCost. bool RAGreedy::canEvictInterferenceInRange(LiveInterval &VirtReg, - MCRegister PhysReg, SlotIndex Start, + MCRegister PhysReg, SlotIndex Start, SlotIndex End, EvictionCost &MaxCost) { EvictionCost Cost; @@ -985,23 +985,23 @@ bool RAGreedy::canEvictInterferenceInRange(LiveInterval &VirtReg, LiveIntervalUnion::Query &Q = Matrix->query(VirtReg, *Units); // Check if any interfering live range is heavier than MaxWeight. - for (const LiveInterval *Intf : reverse(Q.interferingVRegs())) { + for (const LiveInterval *Intf : reverse(Q.interferingVRegs())) { // Check if interference overlast the segment in interest. if (!Intf->overlaps(Start, End)) continue; // Cannot evict non virtual reg interference. - if (!Register::isVirtualRegister(Intf->reg())) + if (!Register::isVirtualRegister(Intf->reg())) return false; // Never evict spill products. They cannot split or spill. if (getStage(*Intf) == RS_Done) return false; // Would this break a satisfied hint? - bool BreaksHint = VRM->hasPreferredPhys(Intf->reg()); + bool BreaksHint = VRM->hasPreferredPhys(Intf->reg()); // Update eviction cost. Cost.BrokenHints += BreaksHint; - Cost.MaxWeight = std::max(Cost.MaxWeight, Intf->weight()); + Cost.MaxWeight = std::max(Cost.MaxWeight, Intf->weight()); // Abort if this would be too expensive. if (!(Cost < MaxCost)) return false; @@ -1026,17 +1026,17 @@ bool RAGreedy::canEvictInterferenceInRange(LiveInterval &VirtReg, /// \param BestEvictweight The eviction cost of that eviction /// \return The PhysReg which is the best candidate for eviction and the /// eviction cost in BestEvictweight -MCRegister RAGreedy::getCheapestEvicteeWeight(const AllocationOrder &Order, - LiveInterval &VirtReg, - SlotIndex Start, SlotIndex End, - float *BestEvictweight) { +MCRegister RAGreedy::getCheapestEvicteeWeight(const AllocationOrder &Order, + LiveInterval &VirtReg, + SlotIndex Start, SlotIndex End, + float *BestEvictweight) { EvictionCost BestEvictCost; BestEvictCost.setMax(); - BestEvictCost.MaxWeight = VirtReg.weight(); - MCRegister BestEvicteePhys; + BestEvictCost.MaxWeight = VirtReg.weight(); + MCRegister BestEvicteePhys; // Go over all physical registers and find the best candidate for eviction - for (MCRegister PhysReg : Order.getOrder()) { + for (MCRegister PhysReg : Order.getOrder()) { if (!canEvictInterferenceInRange(VirtReg, PhysReg, Start, End, BestEvictCost)) @@ -1052,14 +1052,14 @@ MCRegister RAGreedy::getCheapestEvicteeWeight(const AllocationOrder &Order, /// evictInterference - Evict any interferring registers that prevent VirtReg /// from being assigned to Physreg. This assumes that canEvictInterference /// returned true. -void RAGreedy::evictInterference(LiveInterval &VirtReg, MCRegister PhysReg, +void RAGreedy::evictInterference(LiveInterval &VirtReg, MCRegister PhysReg, SmallVectorImpl<Register> &NewVRegs) { // Make sure that VirtReg has a cascade number, and assign that cascade // number to every evicted register. These live ranges than then only be // evicted by a newer cascade, preventing infinite loops. - unsigned Cascade = ExtraRegInfo[VirtReg.reg()].Cascade; + unsigned Cascade = ExtraRegInfo[VirtReg.reg()].Cascade; if (!Cascade) - Cascade = ExtraRegInfo[VirtReg.reg()].Cascade = NextCascade++; + Cascade = ExtraRegInfo[VirtReg.reg()].Cascade = NextCascade++; LLVM_DEBUG(dbgs() << "evicting " << printReg(PhysReg, TRI) << " interference: Cascade " << Cascade << '\n'); @@ -1078,20 +1078,20 @@ void RAGreedy::evictInterference(LiveInterval &VirtReg, MCRegister PhysReg, } // Evict them second. This will invalidate the queries. - for (LiveInterval *Intf : Intfs) { + for (LiveInterval *Intf : Intfs) { // The same VirtReg may be present in multiple RegUnits. Skip duplicates. - if (!VRM->hasPhys(Intf->reg())) + if (!VRM->hasPhys(Intf->reg())) continue; - LastEvicted.addEviction(PhysReg, VirtReg.reg(), Intf->reg()); + LastEvicted.addEviction(PhysReg, VirtReg.reg(), Intf->reg()); Matrix->unassign(*Intf); - assert((ExtraRegInfo[Intf->reg()].Cascade < Cascade || + assert((ExtraRegInfo[Intf->reg()].Cascade < Cascade || VirtReg.isSpillable() < Intf->isSpillable()) && "Cannot decrease cascade number, illegal eviction"); - ExtraRegInfo[Intf->reg()].Cascade = Cascade; + ExtraRegInfo[Intf->reg()].Cascade = Cascade; ++NumEvicted; - NewVRegs.push_back(Intf->reg()); + NewVRegs.push_back(Intf->reg()); } } @@ -1120,17 +1120,17 @@ unsigned RAGreedy::tryEvict(LiveInterval &VirtReg, // Keep track of the cheapest interference seen so far. EvictionCost BestCost; BestCost.setMax(); - MCRegister BestPhys; + MCRegister BestPhys; unsigned OrderLimit = Order.getOrder().size(); // When we are just looking for a reduced cost per use, don't break any // hints, and only evict smaller spill weights. if (CostPerUseLimit < ~0u) { BestCost.BrokenHints = 0; - BestCost.MaxWeight = VirtReg.weight(); + BestCost.MaxWeight = VirtReg.weight(); // Check of any registers in RC are below CostPerUseLimit. - const TargetRegisterClass *RC = MRI->getRegClass(VirtReg.reg()); + const TargetRegisterClass *RC = MRI->getRegClass(VirtReg.reg()); unsigned MinCost = RegClassInfo.getMinCost(RC); if (MinCost >= CostPerUseLimit) { LLVM_DEBUG(dbgs() << TRI->getRegClassName(RC) << " minimum cost = " @@ -1147,10 +1147,10 @@ unsigned RAGreedy::tryEvict(LiveInterval &VirtReg, } } - for (auto I = Order.begin(), E = Order.getOrderLimitEnd(OrderLimit); I != E; - ++I) { - MCRegister PhysReg = *I; - assert(PhysReg); + for (auto I = Order.begin(), E = Order.getOrderLimitEnd(OrderLimit); I != E; + ++I) { + MCRegister PhysReg = *I; + assert(PhysReg); if (TRI->getCostPerUse(PhysReg) >= CostPerUseLimit) continue; // The first use of a callee-saved register in a function has cost 1. @@ -1171,7 +1171,7 @@ unsigned RAGreedy::tryEvict(LiveInterval &VirtReg, BestPhys = PhysReg; // Stop if the hint can be used. - if (I.isHint()) + if (I.isHint()) break; } @@ -1198,9 +1198,9 @@ bool RAGreedy::addSplitConstraints(InterferenceCache::Cursor Intf, // Reset interference dependent info. SplitConstraints.resize(UseBlocks.size()); BlockFrequency StaticCost = 0; - for (unsigned I = 0; I != UseBlocks.size(); ++I) { - const SplitAnalysis::BlockInfo &BI = UseBlocks[I]; - SpillPlacement::BlockConstraint &BC = SplitConstraints[I]; + for (unsigned I = 0; I != UseBlocks.size(); ++I) { + const SplitAnalysis::BlockInfo &BI = UseBlocks[I]; + SpillPlacement::BlockConstraint &BC = SplitConstraints[I]; BC.Number = BI.MBB->getNumber(); Intf.moveToBlock(BC.Number); @@ -1271,7 +1271,7 @@ bool RAGreedy::addThroughConstraints(InterferenceCache::Cursor Intf, unsigned TBS[GroupSize]; unsigned B = 0, T = 0; - for (unsigned Number : Blocks) { + for (unsigned Number : Blocks) { Intf.moveToBlock(Number); if (!Intf.hasInterference()) { @@ -1328,7 +1328,7 @@ bool RAGreedy::growRegion(GlobalSplitCandidate &Cand) { while (true) { ArrayRef<unsigned> NewBundles = SpillPlacer->getRecentPositive(); // Find new through blocks in the periphery of PrefRegBundles. - for (unsigned Bundle : NewBundles) { + for (unsigned Bundle : NewBundles) { // Look at all blocks connected to Bundle in the full graph. ArrayRef<unsigned> Blocks = Bundles->getBlocks(Bundle); for (ArrayRef<unsigned>::iterator I = Blocks.begin(), E = Blocks.end(); @@ -1380,7 +1380,7 @@ bool RAGreedy::calcCompactRegion(GlobalSplitCandidate &Cand) { return false; // Compact regions don't correspond to any physreg. - Cand.reset(IntfCache, MCRegister::NoRegister); + Cand.reset(IntfCache, MCRegister::NoRegister); LLVM_DEBUG(dbgs() << "Compact region bundles"); @@ -1408,8 +1408,8 @@ bool RAGreedy::calcCompactRegion(GlobalSplitCandidate &Cand) { } LLVM_DEBUG({ - for (int I : Cand.LiveBundles.set_bits()) - dbgs() << " EB#" << I; + for (int I : Cand.LiveBundles.set_bits()) + dbgs() << " EB#" << I; dbgs() << ".\n"; }); return true; @@ -1420,7 +1420,7 @@ bool RAGreedy::calcCompactRegion(GlobalSplitCandidate &Cand) { BlockFrequency RAGreedy::calcSpillCost() { BlockFrequency Cost = 0; ArrayRef<SplitAnalysis::BlockInfo> UseBlocks = SA->getUseBlocks(); - for (const SplitAnalysis::BlockInfo &BI : UseBlocks) { + for (const SplitAnalysis::BlockInfo &BI : UseBlocks) { unsigned Number = BI.MBB->getNumber(); // We normally only need one spill instruction - a load or a store. Cost += SpillPlacer->getBlockFrequency(Number); @@ -1485,20 +1485,20 @@ BlockFrequency RAGreedy::calcSpillCost() { /// artifact of Evictee. /// \return True if splitting Evictee may cause a bad eviction chain, false /// otherwise. -bool RAGreedy::splitCanCauseEvictionChain(Register Evictee, +bool RAGreedy::splitCanCauseEvictionChain(Register Evictee, GlobalSplitCandidate &Cand, unsigned BBNumber, const AllocationOrder &Order) { EvictionTrack::EvictorInfo VregEvictorInfo = LastEvicted.getEvictor(Evictee); unsigned Evictor = VregEvictorInfo.first; - MCRegister PhysReg = VregEvictorInfo.second; + MCRegister PhysReg = VregEvictorInfo.second; // No actual evictor. if (!Evictor || !PhysReg) return false; float MaxWeight = 0; - MCRegister FutureEvictedPhysReg = + MCRegister FutureEvictedPhysReg = getCheapestEvicteeWeight(Order, LIS->getInterval(Evictee), Cand.Intf.first(), Cand.Intf.last(), &MaxWeight); @@ -1524,8 +1524,8 @@ bool RAGreedy::splitCanCauseEvictionChain(Register Evictee, // expensive enough to evict somebody If so, this may cause a bad eviction // chain. float splitArtifactWeight = - VRAI->futureWeight(LIS->getInterval(Evictee), - Cand.Intf.first().getPrevIndex(), Cand.Intf.last()); + VRAI->futureWeight(LIS->getInterval(Evictee), + Cand.Intf.first().getPrevIndex(), Cand.Intf.last()); if (splitArtifactWeight >= 0 && splitArtifactWeight < MaxWeight) return false; @@ -1559,15 +1559,15 @@ bool RAGreedy::splitCanCauseLocalSpill(unsigned VirtRegToSplit, // Check if the local interval will evict a cheaper interval. float CheapestEvictWeight = 0; - MCRegister FutureEvictedPhysReg = getCheapestEvicteeWeight( + MCRegister FutureEvictedPhysReg = getCheapestEvicteeWeight( Order, LIS->getInterval(VirtRegToSplit), Cand.Intf.first(), Cand.Intf.last(), &CheapestEvictWeight); // Have we found an interval that can be evicted? if (FutureEvictedPhysReg) { float splitArtifactWeight = - VRAI->futureWeight(LIS->getInterval(VirtRegToSplit), - Cand.Intf.first().getPrevIndex(), Cand.Intf.last()); + VRAI->futureWeight(LIS->getInterval(VirtRegToSplit), + Cand.Intf.first().getPrevIndex(), Cand.Intf.last()); // Will the weight of the local interval be higher than the cheapest evictee // weight? If so it will evict it and will not cause a spill. if (splitArtifactWeight >= 0 && splitArtifactWeight > CheapestEvictWeight) @@ -1588,11 +1588,11 @@ BlockFrequency RAGreedy::calcGlobalSplitCost(GlobalSplitCandidate &Cand, bool *CanCauseEvictionChain) { BlockFrequency GlobalCost = 0; const BitVector &LiveBundles = Cand.LiveBundles; - Register VirtRegToSplit = SA->getParent().reg(); + Register VirtRegToSplit = SA->getParent().reg(); ArrayRef<SplitAnalysis::BlockInfo> UseBlocks = SA->getUseBlocks(); - for (unsigned I = 0; I != UseBlocks.size(); ++I) { - const SplitAnalysis::BlockInfo &BI = UseBlocks[I]; - SpillPlacement::BlockConstraint &BC = SplitConstraints[I]; + for (unsigned I = 0; I != UseBlocks.size(); ++I) { + const SplitAnalysis::BlockInfo &BI = UseBlocks[I]; + SpillPlacement::BlockConstraint &BC = SplitConstraints[I]; bool RegIn = LiveBundles[Bundles->getBundle(BC.Number, false)]; bool RegOut = LiveBundles[Bundles->getBundle(BC.Number, true)]; unsigned Ins = 0; @@ -1630,7 +1630,7 @@ BlockFrequency RAGreedy::calcGlobalSplitCost(GlobalSplitCandidate &Cand, GlobalCost += SpillPlacer->getBlockFrequency(BC.Number); } - for (unsigned Number : Cand.ActiveBlocks) { + for (unsigned Number : Cand.ActiveBlocks) { bool RegIn = LiveBundles[Bundles->getBundle(Number, false)]; bool RegOut = LiveBundles[Bundles->getBundle(Number, true)]; if (!RegIn && !RegOut) @@ -1688,12 +1688,12 @@ void RAGreedy::splitAroundRegion(LiveRangeEdit &LREdit, // Isolate even single instructions when dealing with a proper sub-class. // That guarantees register class inflation for the stack interval because it // is all copies. - Register Reg = SA->getParent().reg(); + Register Reg = SA->getParent().reg(); bool SingleInstrs = RegClassInfo.isProperSubClass(MRI->getRegClass(Reg)); // First handle all the blocks with uses. ArrayRef<SplitAnalysis::BlockInfo> UseBlocks = SA->getUseBlocks(); - for (const SplitAnalysis::BlockInfo &BI : UseBlocks) { + for (const SplitAnalysis::BlockInfo &BI : UseBlocks) { unsigned Number = BI.MBB->getNumber(); unsigned IntvIn = 0, IntvOut = 0; SlotIndex IntfIn, IntfOut; @@ -1738,7 +1738,7 @@ void RAGreedy::splitAroundRegion(LiveRangeEdit &LREdit, BitVector Todo = SA->getThroughBlocks(); for (unsigned c = 0; c != UsedCands.size(); ++c) { ArrayRef<unsigned> Blocks = GlobalCand[UsedCands[c]].ActiveBlocks; - for (unsigned Number : Blocks) { + for (unsigned Number : Blocks) { if (!Todo.test(Number)) continue; Todo.reset(Number); @@ -1781,8 +1781,8 @@ void RAGreedy::splitAroundRegion(LiveRangeEdit &LREdit, // - Candidate intervals can be assigned to Cand.PhysReg. // - Block-local splits are candidates for local splitting. // - DCE leftovers should go back on the queue. - for (unsigned I = 0, E = LREdit.size(); I != E; ++I) { - LiveInterval &Reg = LIS->getInterval(LREdit.get(I)); + for (unsigned I = 0, E = LREdit.size(); I != E; ++I) { + LiveInterval &Reg = LIS->getInterval(LREdit.get(I)); // Ignore old intervals from DCE. if (getStage(Reg) != RS_New) @@ -1790,14 +1790,14 @@ void RAGreedy::splitAroundRegion(LiveRangeEdit &LREdit, // Remainder interval. Don't try splitting again, spill if it doesn't // allocate. - if (IntvMap[I] == 0) { + if (IntvMap[I] == 0) { setStage(Reg, RS_Spill); continue; } // Global intervals. Allow repeated splitting as long as the number of live // blocks is strictly decreasing. - if (IntvMap[I] < NumGlobalIntvs) { + if (IntvMap[I] < NumGlobalIntvs) { if (SA->countLiveBlocks(&Reg) >= OrigBlocks) { LLVM_DEBUG(dbgs() << "Main interval covers the same " << OrigBlocks << " blocks as original.\n"); @@ -1815,11 +1815,11 @@ void RAGreedy::splitAroundRegion(LiveRangeEdit &LREdit, MF->verify(this, "After splitting live range around region"); } -MCRegister RAGreedy::tryRegionSplit(LiveInterval &VirtReg, - AllocationOrder &Order, - SmallVectorImpl<Register> &NewVRegs) { +MCRegister RAGreedy::tryRegionSplit(LiveInterval &VirtReg, + AllocationOrder &Order, + SmallVectorImpl<Register> &NewVRegs) { if (!TRI->shouldRegionSplitForVirtReg(*MF, VirtReg)) - return MCRegister::NoRegister; + return MCRegister::NoRegister; unsigned NumCands = 0; BlockFrequency SpillCost = calcSpillCost(); BlockFrequency BestCost; @@ -1849,12 +1849,12 @@ MCRegister RAGreedy::tryRegionSplit(LiveInterval &VirtReg, // current max frequency. if (HasCompact && (BestCost > SpillCost) && (BestCand != NoCand) && CanCauseEvictionChain) { - return MCRegister::NoRegister; + return MCRegister::NoRegister; } // No solutions found, fall back to single block splitting. if (!HasCompact && BestCand == NoCand) - return MCRegister::NoRegister; + return MCRegister::NoRegister; return doRegionSplit(VirtReg, BestCand, HasCompact, NewVRegs); } @@ -1865,8 +1865,8 @@ unsigned RAGreedy::calculateRegionSplitCost(LiveInterval &VirtReg, unsigned &NumCands, bool IgnoreCSR, bool *CanCauseEvictionChain) { unsigned BestCand = NoCand; - for (MCPhysReg PhysReg : Order) { - assert(PhysReg); + for (MCPhysReg PhysReg : Order) { + assert(PhysReg); if (IgnoreCSR && isUnusedCalleeSavedReg(PhysReg)) continue; @@ -1875,12 +1875,12 @@ unsigned RAGreedy::calculateRegionSplitCost(LiveInterval &VirtReg, if (NumCands == IntfCache.getMaxCursors()) { unsigned WorstCount = ~0u; unsigned Worst = 0; - for (unsigned CandIndex = 0; CandIndex != NumCands; ++CandIndex) { - if (CandIndex == BestCand || !GlobalCand[CandIndex].PhysReg) + for (unsigned CandIndex = 0; CandIndex != NumCands; ++CandIndex) { + if (CandIndex == BestCand || !GlobalCand[CandIndex].PhysReg) continue; - unsigned Count = GlobalCand[CandIndex].LiveBundles.count(); + unsigned Count = GlobalCand[CandIndex].LiveBundles.count(); if (Count < WorstCount) { - Worst = CandIndex; + Worst = CandIndex; WorstCount = Count; } } @@ -1931,8 +1931,8 @@ unsigned RAGreedy::calculateRegionSplitCost(LiveInterval &VirtReg, LLVM_DEBUG({ dbgs() << ", total = "; MBFI->printBlockFreq(dbgs(), Cost) << " with bundles"; - for (int I : Cand.LiveBundles.set_bits()) - dbgs() << " EB#" << I; + for (int I : Cand.LiveBundles.set_bits()) + dbgs() << " EB#" << I; dbgs() << ".\n"; }); if (Cost < BestCost) { @@ -1950,7 +1950,7 @@ unsigned RAGreedy::calculateRegionSplitCost(LiveInterval &VirtReg, // See splitCanCauseEvictionChain for detailed description of bad // eviction chain scenarios. LLVM_DEBUG(dbgs() << "Best split candidate of vreg " - << printReg(VirtReg.reg(), TRI) << " may "); + << printReg(VirtReg.reg(), TRI) << " may "); if (!(*CanCauseEvictionChain)) LLVM_DEBUG(dbgs() << "not "); LLVM_DEBUG(dbgs() << "cause bad eviction chain\n"); @@ -2009,12 +2009,12 @@ unsigned RAGreedy::doRegionSplit(LiveInterval &VirtReg, unsigned BestCand, unsigned RAGreedy::tryBlockSplit(LiveInterval &VirtReg, AllocationOrder &Order, SmallVectorImpl<Register> &NewVRegs) { assert(&SA->getParent() == &VirtReg && "Live range wasn't analyzed"); - Register Reg = VirtReg.reg(); + Register Reg = VirtReg.reg(); bool SingleInstrs = RegClassInfo.isProperSubClass(MRI->getRegClass(Reg)); LiveRangeEdit LREdit(&VirtReg, NewVRegs, *MF, *LIS, VRM, this, &DeadRemats); SE->reset(LREdit, SplitSpillMode); ArrayRef<SplitAnalysis::BlockInfo> UseBlocks = SA->getUseBlocks(); - for (const SplitAnalysis::BlockInfo &BI : UseBlocks) { + for (const SplitAnalysis::BlockInfo &BI : UseBlocks) { if (SA->shouldSplitSingleBlock(BI, SingleInstrs)) SE->splitSingleBlock(BI); } @@ -2033,9 +2033,9 @@ unsigned RAGreedy::tryBlockSplit(LiveInterval &VirtReg, AllocationOrder &Order, // Sort out the new intervals created by splitting. The remainder interval // goes straight to spilling, the new local ranges get to stay RS_New. - for (unsigned I = 0, E = LREdit.size(); I != E; ++I) { - LiveInterval &LI = LIS->getInterval(LREdit.get(I)); - if (getStage(LI) == RS_New && IntvMap[I] == 0) + for (unsigned I = 0, E = LREdit.size(); I != E; ++I) { + LiveInterval &LI = LIS->getInterval(LREdit.get(I)); + if (getStage(LI) == RS_New && IntvMap[I] == 0) setStage(LI, RS_Spill); } @@ -2051,7 +2051,7 @@ unsigned RAGreedy::tryBlockSplit(LiveInterval &VirtReg, AllocationOrder &Order, /// Get the number of allocatable registers that match the constraints of \p Reg /// on \p MI and that are also in \p SuperRC. static unsigned getNumAllocatableRegsForConstraints( - const MachineInstr *MI, Register Reg, const TargetRegisterClass *SuperRC, + const MachineInstr *MI, Register Reg, const TargetRegisterClass *SuperRC, const TargetInstrInfo *TII, const TargetRegisterInfo *TRI, const RegisterClassInfo &RCI) { assert(SuperRC && "Invalid register class"); @@ -2074,7 +2074,7 @@ static unsigned getNumAllocatableRegsForConstraints( unsigned RAGreedy::tryInstructionSplit(LiveInterval &VirtReg, AllocationOrder &Order, SmallVectorImpl<Register> &NewVRegs) { - const TargetRegisterClass *CurRC = MRI->getRegClass(VirtReg.reg()); + const TargetRegisterClass *CurRC = MRI->getRegClass(VirtReg.reg()); // There is no point to this if there are no larger sub-classes. if (!RegClassInfo.isProperSubClass(CurRC)) return 0; @@ -2098,18 +2098,18 @@ RAGreedy::tryInstructionSplit(LiveInterval &VirtReg, AllocationOrder &Order, // the constraints on the virtual register. // Otherwise, splitting just inserts uncoalescable copies that do not help // the allocation. - for (const auto &Use : Uses) { - if (const MachineInstr *MI = Indexes->getInstructionFromIndex(Use)) + for (const auto &Use : Uses) { + if (const MachineInstr *MI = Indexes->getInstructionFromIndex(Use)) if (MI->isFullCopy() || SuperRCNumAllocatableRegs == - getNumAllocatableRegsForConstraints(MI, VirtReg.reg(), SuperRC, - TII, TRI, RCI)) { - LLVM_DEBUG(dbgs() << " skip:\t" << Use << '\t' << *MI); + getNumAllocatableRegsForConstraints(MI, VirtReg.reg(), SuperRC, + TII, TRI, RCI)) { + LLVM_DEBUG(dbgs() << " skip:\t" << Use << '\t' << *MI); continue; } SE->openIntv(); - SlotIndex SegStart = SE->enterIntvBefore(Use); - SlotIndex SegStop = SE->leaveIntvAfter(Use); + SlotIndex SegStart = SE->enterIntvBefore(Use); + SlotIndex SegStop = SE->leaveIntvAfter(Use); SE->useIntv(SegStart, SegStop); } @@ -2120,7 +2120,7 @@ RAGreedy::tryInstructionSplit(LiveInterval &VirtReg, AllocationOrder &Order, SmallVector<unsigned, 8> IntvMap; SE->finish(&IntvMap); - DebugVars->splitRegister(VirtReg.reg(), LREdit.regs(), *LIS); + DebugVars->splitRegister(VirtReg.reg(), LREdit.regs(), *LIS); ExtraRegInfo.resize(MRI->getNumVirtRegs()); // Assign all new registers to RS_Spill. This was the last chance. @@ -2135,9 +2135,9 @@ RAGreedy::tryInstructionSplit(LiveInterval &VirtReg, AllocationOrder &Order, /// calcGapWeights - Compute the maximum spill weight that needs to be evicted /// in order to use PhysReg between two entries in SA->UseSlots. /// -/// GapWeight[I] represents the gap between UseSlots[I] and UseSlots[I + 1]. +/// GapWeight[I] represents the gap between UseSlots[I] and UseSlots[I + 1]. /// -void RAGreedy::calcGapWeights(MCRegister PhysReg, +void RAGreedy::calcGapWeights(MCRegister PhysReg, SmallVectorImpl<float> &GapWeight) { assert(SA->getUseBlocks().size() == 1 && "Not a local interval"); const SplitAnalysis::BlockInfo &BI = SA->getUseBlocks().front(); @@ -2176,7 +2176,7 @@ void RAGreedy::calcGapWeights(MCRegister PhysReg, break; // Update the gaps covered by IntI. - const float weight = IntI.value()->weight(); + const float weight = IntI.value()->weight(); for (; Gap != NumGaps; ++Gap) { GapWeight[Gap] = std::max(GapWeight[Gap], weight); if (Uses[Gap+1].getBaseIndex() >= IntI.stop()) @@ -2238,8 +2238,8 @@ unsigned RAGreedy::tryLocalSplit(LiveInterval &VirtReg, AllocationOrder &Order, LLVM_DEBUG({ dbgs() << "tryLocalSplit: "; - for (const auto &Use : Uses) - dbgs() << ' ' << Use; + for (const auto &Use : Uses) + dbgs() << ' ' << Use; dbgs() << '\n'; }); @@ -2251,25 +2251,25 @@ unsigned RAGreedy::tryLocalSplit(LiveInterval &VirtReg, AllocationOrder &Order, ArrayRef<SlotIndex> RMS = LIS->getRegMaskSlotsInBlock(BI.MBB->getNumber()); LLVM_DEBUG(dbgs() << RMS.size() << " regmasks in block:"); // Constrain to VirtReg's live range. - unsigned RI = + unsigned RI = llvm::lower_bound(RMS, Uses.front().getRegSlot()) - RMS.begin(); - unsigned RE = RMS.size(); - for (unsigned I = 0; I != NumGaps && RI != RE; ++I) { - // Look for Uses[I] <= RMS <= Uses[I + 1]. - assert(!SlotIndex::isEarlierInstr(RMS[RI], Uses[I])); - if (SlotIndex::isEarlierInstr(Uses[I + 1], RMS[RI])) + unsigned RE = RMS.size(); + for (unsigned I = 0; I != NumGaps && RI != RE; ++I) { + // Look for Uses[I] <= RMS <= Uses[I + 1]. + assert(!SlotIndex::isEarlierInstr(RMS[RI], Uses[I])); + if (SlotIndex::isEarlierInstr(Uses[I + 1], RMS[RI])) continue; // Skip a regmask on the same instruction as the last use. It doesn't // overlap the live range. - if (SlotIndex::isSameInstr(Uses[I + 1], RMS[RI]) && I + 1 == NumGaps) + if (SlotIndex::isSameInstr(Uses[I + 1], RMS[RI]) && I + 1 == NumGaps) break; - LLVM_DEBUG(dbgs() << ' ' << RMS[RI] << ':' << Uses[I] << '-' - << Uses[I + 1]); - RegMaskGaps.push_back(I); + LLVM_DEBUG(dbgs() << ' ' << RMS[RI] << ':' << Uses[I] << '-' + << Uses[I + 1]); + RegMaskGaps.push_back(I); // Advance ri to the next gap. A regmask on one of the uses counts in // both gaps. - while (RI != RE && SlotIndex::isEarlierInstr(RMS[RI], Uses[I + 1])) - ++RI; + while (RI != RE && SlotIndex::isEarlierInstr(RMS[RI], Uses[I + 1])) + ++RI; } LLVM_DEBUG(dbgs() << '\n'); } @@ -2304,16 +2304,16 @@ unsigned RAGreedy::tryLocalSplit(LiveInterval &VirtReg, AllocationOrder &Order, (1.0f / MBFI->getEntryFreq()); SmallVector<float, 8> GapWeight; - for (MCPhysReg PhysReg : Order) { - assert(PhysReg); + for (MCPhysReg PhysReg : Order) { + assert(PhysReg); // Keep track of the largest spill weight that would need to be evicted in - // order to make use of PhysReg between UseSlots[I] and UseSlots[I + 1]. + // order to make use of PhysReg between UseSlots[I] and UseSlots[I + 1]. calcGapWeights(PhysReg, GapWeight); // Remove any gaps with regmask clobbers. if (Matrix->checkRegMaskInterference(VirtReg, PhysReg)) - for (unsigned I = 0, E = RegMaskGaps.size(); I != E; ++I) - GapWeight[RegMaskGaps[I]] = huge_valf; + for (unsigned I = 0, E = RegMaskGaps.size(); I != E; ++I) + GapWeight[RegMaskGaps[I]] = huge_valf; // Try to find the best sequence of gaps to close. // The new spill weight must be larger than any gap interference. @@ -2331,7 +2331,7 @@ unsigned RAGreedy::tryLocalSplit(LiveInterval &VirtReg, AllocationOrder &Order, const bool LiveAfter = SplitAfter != NumGaps || BI.LiveOut; LLVM_DEBUG(dbgs() << printReg(PhysReg, TRI) << ' ' << Uses[SplitBefore] - << '-' << Uses[SplitAfter] << " I=" << MaxGap); + << '-' << Uses[SplitAfter] << " I=" << MaxGap); // Stop before the interval gets so big we wouldn't be making progress. if (!LiveBefore && !LiveAfter) { @@ -2380,8 +2380,8 @@ unsigned RAGreedy::tryLocalSplit(LiveInterval &VirtReg, AllocationOrder &Order, // Recompute the max when necessary. if (GapWeight[SplitBefore - 1] >= MaxGap) { MaxGap = GapWeight[SplitBefore]; - for (unsigned I = SplitBefore + 1; I != SplitAfter; ++I) - MaxGap = std::max(MaxGap, GapWeight[I]); + for (unsigned I = SplitBefore + 1; I != SplitAfter; ++I) + MaxGap = std::max(MaxGap, GapWeight[I]); } continue; } @@ -2416,7 +2416,7 @@ unsigned RAGreedy::tryLocalSplit(LiveInterval &VirtReg, AllocationOrder &Order, SE->useIntv(SegStart, SegStop); SmallVector<unsigned, 8> IntvMap; SE->finish(&IntvMap); - DebugVars->splitRegister(VirtReg.reg(), LREdit.regs(), *LIS); + DebugVars->splitRegister(VirtReg.reg(), LREdit.regs(), *LIS); // If the new range has the same number of instructions as before, mark it as // RS_Split2 so the next split will be forced to make progress. Otherwise, @@ -2427,10 +2427,10 @@ unsigned RAGreedy::tryLocalSplit(LiveInterval &VirtReg, AllocationOrder &Order, if (NewGaps >= NumGaps) { LLVM_DEBUG(dbgs() << "Tagging non-progress ranges: "); assert(!ProgressRequired && "Didn't make progress when it was required."); - for (unsigned I = 0, E = IntvMap.size(); I != E; ++I) - if (IntvMap[I] == 1) { - setStage(LIS->getInterval(LREdit.get(I)), RS_Split2); - LLVM_DEBUG(dbgs() << printReg(LREdit.get(I))); + for (unsigned I = 0, E = IntvMap.size(); I != E; ++I) + if (IntvMap[I] == 1) { + setStage(LIS->getInterval(LREdit.get(I)), RS_Split2); + LLVM_DEBUG(dbgs() << printReg(LREdit.get(I))); } LLVM_DEBUG(dbgs() << '\n'); } @@ -2484,7 +2484,7 @@ unsigned RAGreedy::trySplit(LiveInterval &VirtReg, AllocationOrder &Order, // ranges already made dubious progress with region splitting, so they go // straight to single block splitting. if (getStage(VirtReg) < RS_Split2) { - MCRegister PhysReg = tryRegionSplit(VirtReg, Order, NewVRegs); + MCRegister PhysReg = tryRegionSplit(VirtReg, Order, NewVRegs); if (PhysReg || !NewVRegs.empty()) return PhysReg; } @@ -2514,10 +2514,10 @@ static bool hasTiedDef(MachineRegisterInfo *MRI, unsigned reg) { /// for \p VirtReg. /// \p FixedRegisters contains all the virtual registers that cannot be /// recolored. -bool RAGreedy::mayRecolorAllInterferences( - MCRegister PhysReg, LiveInterval &VirtReg, SmallLISet &RecoloringCandidates, - const SmallVirtRegSet &FixedRegisters) { - const TargetRegisterClass *CurRC = MRI->getRegClass(VirtReg.reg()); +bool RAGreedy::mayRecolorAllInterferences( + MCRegister PhysReg, LiveInterval &VirtReg, SmallLISet &RecoloringCandidates, + const SmallVirtRegSet &FixedRegisters) { + const TargetRegisterClass *CurRC = MRI->getRegClass(VirtReg.reg()); for (MCRegUnitIterator Units(PhysReg, TRI); Units.isValid(); ++Units) { LiveIntervalUnion::Query &Q = Matrix->query(VirtReg, *Units); @@ -2529,16 +2529,16 @@ bool RAGreedy::mayRecolorAllInterferences( CutOffInfo |= CO_Interf; return false; } - for (LiveInterval *Intf : reverse(Q.interferingVRegs())) { + for (LiveInterval *Intf : reverse(Q.interferingVRegs())) { // If Intf is done and sit on the same register class as VirtReg, // it would not be recolorable as it is in the same state as VirtReg. // However, if VirtReg has tied defs and Intf doesn't, then // there is still a point in examining if it can be recolorable. if (((getStage(*Intf) == RS_Done && - MRI->getRegClass(Intf->reg()) == CurRC) && - !(hasTiedDef(MRI, VirtReg.reg()) && - !hasTiedDef(MRI, Intf->reg()))) || - FixedRegisters.count(Intf->reg())) { + MRI->getRegClass(Intf->reg()) == CurRC) && + !(hasTiedDef(MRI, VirtReg.reg()) && + !hasTiedDef(MRI, Intf->reg()))) || + FixedRegisters.count(Intf->reg())) { LLVM_DEBUG( dbgs() << "Early abort: the interference is not recolorable.\n"); return false; @@ -2593,9 +2593,9 @@ unsigned RAGreedy::tryLastChanceRecoloring(LiveInterval &VirtReg, SmallVectorImpl<Register> &NewVRegs, SmallVirtRegSet &FixedRegisters, unsigned Depth) { - if (!TRI->shouldUseLastChanceRecoloringForVirtReg(*MF, VirtReg)) - return ~0u; - + if (!TRI->shouldUseLastChanceRecoloringForVirtReg(*MF, VirtReg)) + return ~0u; + LLVM_DEBUG(dbgs() << "Try last chance recoloring for " << VirtReg << '\n'); // Ranges must be Done. assert((getStage(VirtReg) >= RS_Done || !VirtReg.isSpillable()) && @@ -2614,15 +2614,15 @@ unsigned RAGreedy::tryLastChanceRecoloring(LiveInterval &VirtReg, SmallLISet RecoloringCandidates; // Record the original mapping virtual register to physical register in case // the recoloring fails. - DenseMap<Register, MCRegister> VirtRegToPhysReg; + DenseMap<Register, MCRegister> VirtRegToPhysReg; // Mark VirtReg as fixed, i.e., it will not be recolored pass this point in // this recoloring "session". - assert(!FixedRegisters.count(VirtReg.reg())); - FixedRegisters.insert(VirtReg.reg()); + assert(!FixedRegisters.count(VirtReg.reg())); + FixedRegisters.insert(VirtReg.reg()); SmallVector<Register, 4> CurrentNewVRegs; - for (MCRegister PhysReg : Order) { - assert(PhysReg.isValid()); + for (MCRegister PhysReg : Order) { + assert(PhysReg.isValid()); LLVM_DEBUG(dbgs() << "Try to assign: " << VirtReg << " to " << printReg(PhysReg, TRI) << '\n'); RecoloringCandidates.clear(); @@ -2653,7 +2653,7 @@ unsigned RAGreedy::tryLastChanceRecoloring(LiveInterval &VirtReg, for (SmallLISet::iterator It = RecoloringCandidates.begin(), EndIt = RecoloringCandidates.end(); It != EndIt; ++It) { - Register ItVirtReg = (*It)->reg(); + Register ItVirtReg = (*It)->reg(); enqueue(RecoloringQueue, *It); assert(VRM->hasPhys(ItVirtReg) && "Interferences are supposed to be with allocated variables"); @@ -2706,10 +2706,10 @@ unsigned RAGreedy::tryLastChanceRecoloring(LiveInterval &VirtReg, for (SmallLISet::iterator It = RecoloringCandidates.begin(), EndIt = RecoloringCandidates.end(); It != EndIt; ++It) { - Register ItVirtReg = (*It)->reg(); + Register ItVirtReg = (*It)->reg(); if (VRM->hasPhys(ItVirtReg)) Matrix->unassign(**It); - MCRegister ItPhysReg = VirtRegToPhysReg[ItVirtReg]; + MCRegister ItPhysReg = VirtRegToPhysReg[ItVirtReg]; Matrix->assign(**It, ItPhysReg); } } @@ -2733,8 +2733,8 @@ bool RAGreedy::tryRecoloringCandidates(PQueue &RecoloringQueue, while (!RecoloringQueue.empty()) { LiveInterval *LI = dequeue(RecoloringQueue); LLVM_DEBUG(dbgs() << "Try to recolor: " << *LI << '\n'); - MCRegister PhysReg = - selectOrSplitImpl(*LI, NewVRegs, FixedRegisters, Depth + 1); + MCRegister PhysReg = + selectOrSplitImpl(*LI, NewVRegs, FixedRegisters, Depth + 1); // When splitting happens, the live-range may actually be empty. // In that case, this is okay to continue the recoloring even // if we did not find an alternative color for it. Indeed, @@ -2752,7 +2752,7 @@ bool RAGreedy::tryRecoloringCandidates(PQueue &RecoloringQueue, << " succeeded with: " << printReg(PhysReg, TRI) << '\n'); Matrix->assign(*LI, PhysReg); - FixedRegisters.insert(LI->reg()); + FixedRegisters.insert(LI->reg()); } return true; } @@ -2761,12 +2761,12 @@ bool RAGreedy::tryRecoloringCandidates(PQueue &RecoloringQueue, // Main Entry Point //===----------------------------------------------------------------------===// -MCRegister RAGreedy::selectOrSplit(LiveInterval &VirtReg, - SmallVectorImpl<Register> &NewVRegs) { +MCRegister RAGreedy::selectOrSplit(LiveInterval &VirtReg, + SmallVectorImpl<Register> &NewVRegs) { CutOffInfo = CO_None; LLVMContext &Ctx = MF->getFunction().getContext(); SmallVirtRegSet FixedRegisters; - MCRegister Reg = selectOrSplitImpl(VirtReg, NewVRegs, FixedRegisters); + MCRegister Reg = selectOrSplitImpl(VirtReg, NewVRegs, FixedRegisters); if (Reg == ~0U && (CutOffInfo != CO_None)) { uint8_t CutOffEncountered = CutOffInfo & (CO_Depth | CO_Interf); if (CutOffEncountered == CO_Depth) @@ -2791,10 +2791,10 @@ MCRegister RAGreedy::selectOrSplit(LiveInterval &VirtReg, /// Spilling a live range in the cold path can have lower cost than using /// the CSR for the first time. Returns the physical register if we decide /// to use the CSR; otherwise return 0. -MCRegister -RAGreedy::tryAssignCSRFirstTime(LiveInterval &VirtReg, AllocationOrder &Order, - MCRegister PhysReg, unsigned &CostPerUseLimit, - SmallVectorImpl<Register> &NewVRegs) { +MCRegister +RAGreedy::tryAssignCSRFirstTime(LiveInterval &VirtReg, AllocationOrder &Order, + MCRegister PhysReg, unsigned &CostPerUseLimit, + SmallVectorImpl<Register> &NewVRegs) { if (getStage(VirtReg) == RS_Spill && VirtReg.isSpillable()) { // We choose spill over using the CSR for the first time if the spill cost // is lower than CSRCost. @@ -2859,7 +2859,7 @@ void RAGreedy::initializeCSRCost() { /// Collect the hint info for \p Reg. /// The results are stored into \p Out. /// \p Out is not cleared before being populated. -void RAGreedy::collectHintInfo(Register Reg, HintsInfo &Out) { +void RAGreedy::collectHintInfo(Register Reg, HintsInfo &Out) { for (const MachineInstr &Instr : MRI->reg_nodbg_instructions(Reg)) { if (!Instr.isFullCopy()) continue; @@ -2871,8 +2871,8 @@ void RAGreedy::collectHintInfo(Register Reg, HintsInfo &Out) { continue; } // Get the current assignment. - MCRegister OtherPhysReg = - OtherReg.isPhysical() ? OtherReg.asMCReg() : VRM->getPhys(OtherReg); + MCRegister OtherPhysReg = + OtherReg.isPhysical() ? OtherReg.asMCReg() : VRM->getPhys(OtherReg); // Push the collected information. Out.push_back(HintInfo(MBFI->getBlockFreq(Instr.getParent()), OtherReg, OtherPhysReg)); @@ -2883,7 +2883,7 @@ void RAGreedy::collectHintInfo(Register Reg, HintsInfo &Out) { /// \p PhysReg was used. /// \return The cost of \p List for \p PhysReg. BlockFrequency RAGreedy::getBrokenHintFreq(const HintsInfo &List, - MCRegister PhysReg) { + MCRegister PhysReg) { BlockFrequency Cost = 0; for (const HintInfo &Info : List) { if (Info.PhysReg != PhysReg) @@ -2904,11 +2904,11 @@ void RAGreedy::tryHintRecoloring(LiveInterval &VirtReg) { // We have a broken hint, check if it is possible to fix it by // reusing PhysReg for the copy-related live-ranges. Indeed, we evicted // some register and PhysReg may be available for the other live-ranges. - SmallSet<Register, 4> Visited; + SmallSet<Register, 4> Visited; SmallVector<unsigned, 2> RecoloringCandidates; HintsInfo Info; - Register Reg = VirtReg.reg(); - MCRegister PhysReg = VRM->getPhys(Reg); + Register Reg = VirtReg.reg(); + MCRegister PhysReg = VRM->getPhys(Reg); // Start the recoloring algorithm from the input live-interval, then // it will propagate to the ones that are copy-related with it. Visited.insert(Reg); @@ -2929,7 +2929,7 @@ void RAGreedy::tryHintRecoloring(LiveInterval &VirtReg) { // Get the live interval mapped with this virtual register to be able // to check for the interference with the new color. LiveInterval &LI = LIS->getInterval(Reg); - MCRegister CurrPhys = VRM->getPhys(Reg); + MCRegister CurrPhys = VRM->getPhys(Reg); // Check that the new color matches the register class constraints and // that it is free for this live range. if (CurrPhys != PhysReg && (!MRI->getRegClass(Reg)->contains(PhysReg) || @@ -3010,35 +3010,35 @@ void RAGreedy::tryHintRecoloring(LiveInterval &VirtReg) { /// getting rid of 2 copies. void RAGreedy::tryHintsRecoloring() { for (LiveInterval *LI : SetOfBrokenHints) { - assert(Register::isVirtualRegister(LI->reg()) && + assert(Register::isVirtualRegister(LI->reg()) && "Recoloring is possible only for virtual registers"); // Some dead defs may be around (e.g., because of debug uses). // Ignore those. - if (!VRM->hasPhys(LI->reg())) + if (!VRM->hasPhys(LI->reg())) continue; tryHintRecoloring(*LI); } } -MCRegister RAGreedy::selectOrSplitImpl(LiveInterval &VirtReg, - SmallVectorImpl<Register> &NewVRegs, - SmallVirtRegSet &FixedRegisters, - unsigned Depth) { +MCRegister RAGreedy::selectOrSplitImpl(LiveInterval &VirtReg, + SmallVectorImpl<Register> &NewVRegs, + SmallVirtRegSet &FixedRegisters, + unsigned Depth) { unsigned CostPerUseLimit = ~0u; // First try assigning a free register. - auto Order = - AllocationOrder::create(VirtReg.reg(), *VRM, RegClassInfo, Matrix); - if (MCRegister PhysReg = - tryAssign(VirtReg, Order, NewVRegs, FixedRegisters)) { + auto Order = + AllocationOrder::create(VirtReg.reg(), *VRM, RegClassInfo, Matrix); + if (MCRegister PhysReg = + tryAssign(VirtReg, Order, NewVRegs, FixedRegisters)) { // If VirtReg got an assignment, the eviction info is no longre relevant. - LastEvicted.clearEvicteeInfo(VirtReg.reg()); + LastEvicted.clearEvicteeInfo(VirtReg.reg()); // When NewVRegs is not empty, we may have made decisions such as evicting // a virtual register, go with the earlier decisions and use the physical // register. if (CSRCost.getFrequency() && isUnusedCalleeSavedReg(PhysReg) && NewVRegs.empty()) { - MCRegister CSRReg = tryAssignCSRFirstTime(VirtReg, Order, PhysReg, - CostPerUseLimit, NewVRegs); + MCRegister CSRReg = tryAssignCSRFirstTime(VirtReg, Order, PhysReg, + CostPerUseLimit, NewVRegs); if (CSRReg || !NewVRegs.empty()) // Return now if we decide to use a CSR or create new vregs due to // pre-splitting. @@ -3049,7 +3049,7 @@ MCRegister RAGreedy::selectOrSplitImpl(LiveInterval &VirtReg, LiveRangeStage Stage = getStage(VirtReg); LLVM_DEBUG(dbgs() << StageName[Stage] << " Cascade " - << ExtraRegInfo[VirtReg.reg()].Cascade << '\n'); + << ExtraRegInfo[VirtReg.reg()].Cascade << '\n'); // Try to evict a less worthy live range, but only for ranges from the primary // queue. The RS_Split ranges already failed to do this, and they should not @@ -3058,7 +3058,7 @@ MCRegister RAGreedy::selectOrSplitImpl(LiveInterval &VirtReg, if (Register PhysReg = tryEvict(VirtReg, Order, NewVRegs, CostPerUseLimit, FixedRegisters)) { - Register Hint = MRI->getSimpleHint(VirtReg.reg()); + Register Hint = MRI->getSimpleHint(VirtReg.reg()); // If VirtReg has a hint and that hint is broken record this // virtual register as a recoloring candidate for broken hint. // Indeed, since we evicted a variable in its neighborhood it is @@ -3068,7 +3068,7 @@ MCRegister RAGreedy::selectOrSplitImpl(LiveInterval &VirtReg, SetOfBrokenHints.insert(&VirtReg); // If VirtReg eviction someone, the eviction info for it as an evictee is // no longre relevant. - LastEvicted.clearEvicteeInfo(VirtReg.reg()); + LastEvicted.clearEvicteeInfo(VirtReg.reg()); return PhysReg; } @@ -3080,7 +3080,7 @@ MCRegister RAGreedy::selectOrSplitImpl(LiveInterval &VirtReg, if (Stage < RS_Split) { setStage(VirtReg, RS_Split); LLVM_DEBUG(dbgs() << "wait for second round\n"); - NewVRegs.push_back(VirtReg.reg()); + NewVRegs.push_back(VirtReg.reg()); return 0; } @@ -3090,7 +3090,7 @@ MCRegister RAGreedy::selectOrSplitImpl(LiveInterval &VirtReg, Register PhysReg = trySplit(VirtReg, Order, NewVRegs, FixedRegisters); if (PhysReg || (NewVRegs.size() - NewVRegSizeBefore)) { // If VirtReg got split, the eviction info is no longer relevant. - LastEvicted.clearEvicteeInfo(VirtReg.reg()); + LastEvicted.clearEvicteeInfo(VirtReg.reg()); return PhysReg; } } @@ -3102,16 +3102,16 @@ MCRegister RAGreedy::selectOrSplitImpl(LiveInterval &VirtReg, Depth); // Finally spill VirtReg itself. - if ((EnableDeferredSpilling || - TRI->shouldUseDeferredSpillingForVirtReg(*MF, VirtReg)) && - getStage(VirtReg) < RS_Memory) { + if ((EnableDeferredSpilling || + TRI->shouldUseDeferredSpillingForVirtReg(*MF, VirtReg)) && + getStage(VirtReg) < RS_Memory) { // TODO: This is experimental and in particular, we do not model // the live range splitting done by spilling correctly. // We would need a deep integration with the spiller to do the // right thing here. Anyway, that is still good for early testing. setStage(VirtReg, RS_Memory); LLVM_DEBUG(dbgs() << "Do as if this register is in memory\n"); - NewVRegs.push_back(VirtReg.reg()); + NewVRegs.push_back(VirtReg.reg()); } else { NamedRegionTimer T("spill", "Spiller", TimerGroupName, TimerGroupDescription, TimePassesIsEnabled); @@ -3122,7 +3122,7 @@ MCRegister RAGreedy::selectOrSplitImpl(LiveInterval &VirtReg, // Tell LiveDebugVariables about the new ranges. Ranges not being covered by // the new regs are kept in LDV (still mapping to the old register), until // we rewrite spilled locations in LDV at a later stage. - DebugVars->splitRegister(VirtReg.reg(), LRE.regs(), *LIS); + DebugVars->splitRegister(VirtReg.reg(), LRE.regs(), *LIS); if (VerifyEnabled) MF->verify(this, "After spilling"); @@ -3241,10 +3241,10 @@ bool RAGreedy::runOnMachineFunction(MachineFunction &mf) { initializeCSRCost(); - VRAI = std::make_unique<VirtRegAuxInfo>(*MF, *LIS, *VRM, *Loops, *MBFI); - - VRAI->calculateSpillWeightsAndHints(); + VRAI = std::make_unique<VirtRegAuxInfo>(*MF, *LIS, *VRM, *Loops, *MBFI); + VRAI->calculateSpillWeightsAndHints(); + LLVM_DEBUG(LIS->dump()); SA.reset(new SplitAnalysis(*VRM, *LIS, *Loops)); |