summaryrefslogtreecommitdiffstats
path: root/contrib/libs/llvm16/lib/Passes
diff options
context:
space:
mode:
authorvvvv <[email protected]>2024-02-06 20:01:22 +0300
committervvvv <[email protected]>2024-02-06 20:22:16 +0300
commit0203b7a9a40828bb2bd4c32029b79ff0ea3d1f8f (patch)
treee630d0d5bd0bd29fc8c2d2842ed2cfde781b993a /contrib/libs/llvm16/lib/Passes
parentba27db76d99d12a4f1c06960b5449423218614c4 (diff)
llvm16 targets
Diffstat (limited to 'contrib/libs/llvm16/lib/Passes')
-rw-r--r--contrib/libs/llvm16/lib/Passes/OptimizationLevel.cpp30
-rw-r--r--contrib/libs/llvm16/lib/Passes/PassBuilder.cpp1890
-rw-r--r--contrib/libs/llvm16/lib/Passes/PassBuilderBindings.cpp144
-rw-r--r--contrib/libs/llvm16/lib/Passes/PassBuilderPipelines.cpp1998
-rw-r--r--contrib/libs/llvm16/lib/Passes/PassPlugin.cpp54
-rw-r--r--contrib/libs/llvm16/lib/Passes/PassRegistry.def576
-rw-r--r--contrib/libs/llvm16/lib/Passes/StandardInstrumentations.cpp2191
-rw-r--r--contrib/libs/llvm16/lib/Passes/ya.make46
8 files changed, 6929 insertions, 0 deletions
diff --git a/contrib/libs/llvm16/lib/Passes/OptimizationLevel.cpp b/contrib/libs/llvm16/lib/Passes/OptimizationLevel.cpp
new file mode 100644
index 00000000000..a1f8c1e14b1
--- /dev/null
+++ b/contrib/libs/llvm16/lib/Passes/OptimizationLevel.cpp
@@ -0,0 +1,30 @@
+//===- OptimizationLevel.cpp ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Passes/OptimizationLevel.h"
+
+using namespace llvm;
+
+const OptimizationLevel OptimizationLevel::O0 = {
+ /*SpeedLevel*/ 0,
+ /*SizeLevel*/ 0};
+const OptimizationLevel OptimizationLevel::O1 = {
+ /*SpeedLevel*/ 1,
+ /*SizeLevel*/ 0};
+const OptimizationLevel OptimizationLevel::O2 = {
+ /*SpeedLevel*/ 2,
+ /*SizeLevel*/ 0};
+const OptimizationLevel OptimizationLevel::O3 = {
+ /*SpeedLevel*/ 3,
+ /*SizeLevel*/ 0};
+const OptimizationLevel OptimizationLevel::Os = {
+ /*SpeedLevel*/ 2,
+ /*SizeLevel*/ 1};
+const OptimizationLevel OptimizationLevel::Oz = {
+ /*SpeedLevel*/ 2,
+ /*SizeLevel*/ 2};
diff --git a/contrib/libs/llvm16/lib/Passes/PassBuilder.cpp b/contrib/libs/llvm16/lib/Passes/PassBuilder.cpp
new file mode 100644
index 00000000000..e251d56463a
--- /dev/null
+++ b/contrib/libs/llvm16/lib/Passes/PassBuilder.cpp
@@ -0,0 +1,1890 @@
+//===- Parsing and selection of pass pipelines ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+/// \file
+///
+/// This file provides the implementation of the PassBuilder based on our
+/// static pass registry as well as related functionality. It also provides
+/// helpers to aid in analyzing, debugging, and testing passes and pass
+/// pipelines.
+///
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Passes/PassBuilder.h"
+#include "llvm/ADT/StringSwitch.h"
+#include "llvm/Analysis/AliasAnalysisEvaluator.h"
+#include "llvm/Analysis/AliasSetTracker.h"
+#include "llvm/Analysis/AssumptionCache.h"
+#include "llvm/Analysis/BasicAliasAnalysis.h"
+#include "llvm/Analysis/BlockFrequencyInfo.h"
+#include "llvm/Analysis/BranchProbabilityInfo.h"
+#include "llvm/Analysis/CFGPrinter.h"
+#include "llvm/Analysis/CFGSCCPrinter.h"
+#include "llvm/Analysis/CGSCCPassManager.h"
+#include "llvm/Analysis/CallGraph.h"
+#include "llvm/Analysis/CallPrinter.h"
+#include "llvm/Analysis/CostModel.h"
+#include "llvm/Analysis/CycleAnalysis.h"
+#include "llvm/Analysis/DDG.h"
+#include "llvm/Analysis/DDGPrinter.h"
+#include "llvm/Analysis/Delinearization.h"
+#include "llvm/Analysis/DemandedBits.h"
+#include "llvm/Analysis/DependenceAnalysis.h"
+#include "llvm/Analysis/DivergenceAnalysis.h"
+#include "llvm/Analysis/DomPrinter.h"
+#include "llvm/Analysis/DominanceFrontier.h"
+#include "llvm/Analysis/FunctionPropertiesAnalysis.h"
+#include "llvm/Analysis/GlobalsModRef.h"
+#include "llvm/Analysis/IRSimilarityIdentifier.h"
+#include "llvm/Analysis/IVUsers.h"
+#include "llvm/Analysis/InlineAdvisor.h"
+#include "llvm/Analysis/InlineSizeEstimatorAnalysis.h"
+#include "llvm/Analysis/InstCount.h"
+#include "llvm/Analysis/LazyCallGraph.h"
+#include "llvm/Analysis/LazyValueInfo.h"
+#include "llvm/Analysis/LegacyDivergenceAnalysis.h"
+#include "llvm/Analysis/Lint.h"
+#include "llvm/Analysis/LoopAccessAnalysis.h"
+#include "llvm/Analysis/LoopCacheAnalysis.h"
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/LoopNestAnalysis.h"
+#include "llvm/Analysis/MemDerefPrinter.h"
+#include "llvm/Analysis/MemoryDependenceAnalysis.h"
+#include "llvm/Analysis/MemorySSA.h"
+#include "llvm/Analysis/ModuleDebugInfoPrinter.h"
+#include "llvm/Analysis/ModuleSummaryAnalysis.h"
+#include "llvm/Analysis/MustExecute.h"
+#include "llvm/Analysis/ObjCARCAliasAnalysis.h"
+#include "llvm/Analysis/OptimizationRemarkEmitter.h"
+#include "llvm/Analysis/PhiValues.h"
+#include "llvm/Analysis/PostDominators.h"
+#include "llvm/Analysis/ProfileSummaryInfo.h"
+#include "llvm/Analysis/RegionInfo.h"
+#include "llvm/Analysis/ScalarEvolution.h"
+#include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h"
+#include "llvm/Analysis/ScopedNoAliasAA.h"
+#include "llvm/Analysis/StackLifetime.h"
+#include "llvm/Analysis/StackSafetyAnalysis.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/Analysis/TypeBasedAliasAnalysis.h"
+#include "llvm/Analysis/UniformityAnalysis.h"
+#include "llvm/CodeGen/TypePromotion.h"
+#include "llvm/IR/DebugInfo.h"
+#include "llvm/IR/Dominators.h"
+#include "llvm/IR/PassManager.h"
+#include "llvm/IR/PrintPasses.h"
+#include "llvm/IR/SafepointIRVerifier.h"
+#include "llvm/IR/Verifier.h"
+#include "llvm/IRPrinter/IRPrintingPasses.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/FormatVariadic.h"
+#include "llvm/Support/Regex.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h"
+#include "llvm/Transforms/Coroutines/CoroCleanup.h"
+#include "llvm/Transforms/Coroutines/CoroConditionalWrapper.h"
+#include "llvm/Transforms/Coroutines/CoroEarly.h"
+#include "llvm/Transforms/Coroutines/CoroElide.h"
+#include "llvm/Transforms/Coroutines/CoroSplit.h"
+#include "llvm/Transforms/IPO/AlwaysInliner.h"
+#include "llvm/Transforms/IPO/Annotation2Metadata.h"
+#include "llvm/Transforms/IPO/ArgumentPromotion.h"
+#include "llvm/Transforms/IPO/Attributor.h"
+#include "llvm/Transforms/IPO/BlockExtractor.h"
+#include "llvm/Transforms/IPO/CalledValuePropagation.h"
+#include "llvm/Transforms/IPO/ConstantMerge.h"
+#include "llvm/Transforms/IPO/CrossDSOCFI.h"
+#include "llvm/Transforms/IPO/DeadArgumentElimination.h"
+#include "llvm/Transforms/IPO/ElimAvailExtern.h"
+#include "llvm/Transforms/IPO/ForceFunctionAttrs.h"
+#include "llvm/Transforms/IPO/FunctionAttrs.h"
+#include "llvm/Transforms/IPO/FunctionImport.h"
+#include "llvm/Transforms/IPO/GlobalDCE.h"
+#include "llvm/Transforms/IPO/GlobalOpt.h"
+#include "llvm/Transforms/IPO/GlobalSplit.h"
+#include "llvm/Transforms/IPO/HotColdSplitting.h"
+#include "llvm/Transforms/IPO/IROutliner.h"
+#include "llvm/Transforms/IPO/InferFunctionAttrs.h"
+#include "llvm/Transforms/IPO/Inliner.h"
+#include "llvm/Transforms/IPO/Internalize.h"
+#include "llvm/Transforms/IPO/LoopExtractor.h"
+#include "llvm/Transforms/IPO/LowerTypeTests.h"
+#include "llvm/Transforms/IPO/MergeFunctions.h"
+#include "llvm/Transforms/IPO/ModuleInliner.h"
+#include "llvm/Transforms/IPO/OpenMPOpt.h"
+#include "llvm/Transforms/IPO/PartialInlining.h"
+#include "llvm/Transforms/IPO/SCCP.h"
+#include "llvm/Transforms/IPO/SampleProfile.h"
+#include "llvm/Transforms/IPO/SampleProfileProbe.h"
+#include "llvm/Transforms/IPO/StripDeadPrototypes.h"
+#include "llvm/Transforms/IPO/StripSymbols.h"
+#include "llvm/Transforms/IPO/SyntheticCountsPropagation.h"
+#include "llvm/Transforms/IPO/WholeProgramDevirt.h"
+#include "llvm/Transforms/InstCombine/InstCombine.h"
+#include "llvm/Transforms/Instrumentation.h"
+#include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
+#include "llvm/Transforms/Instrumentation/BoundsChecking.h"
+#include "llvm/Transforms/Instrumentation/CGProfile.h"
+#include "llvm/Transforms/Instrumentation/ControlHeightReduction.h"
+#include "llvm/Transforms/Instrumentation/DataFlowSanitizer.h"
+#include "llvm/Transforms/Instrumentation/GCOVProfiler.h"
+#include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
+#include "llvm/Transforms/Instrumentation/InstrOrderFile.h"
+#include "llvm/Transforms/Instrumentation/InstrProfiling.h"
+#include "llvm/Transforms/Instrumentation/KCFI.h"
+#include "llvm/Transforms/Instrumentation/MemProfiler.h"
+#include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
+#include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
+#include "llvm/Transforms/Instrumentation/PoisonChecking.h"
+#include "llvm/Transforms/Instrumentation/SanitizerBinaryMetadata.h"
+#include "llvm/Transforms/Instrumentation/SanitizerCoverage.h"
+#include "llvm/Transforms/Instrumentation/ThreadSanitizer.h"
+#include "llvm/Transforms/ObjCARC.h"
+#include "llvm/Transforms/Scalar/ADCE.h"
+#include "llvm/Transforms/Scalar/AlignmentFromAssumptions.h"
+#include "llvm/Transforms/Scalar/AnnotationRemarks.h"
+#include "llvm/Transforms/Scalar/BDCE.h"
+#include "llvm/Transforms/Scalar/CallSiteSplitting.h"
+#include "llvm/Transforms/Scalar/ConstantHoisting.h"
+#include "llvm/Transforms/Scalar/ConstraintElimination.h"
+#include "llvm/Transforms/Scalar/CorrelatedValuePropagation.h"
+#include "llvm/Transforms/Scalar/DCE.h"
+#include "llvm/Transforms/Scalar/DFAJumpThreading.h"
+#include "llvm/Transforms/Scalar/DeadStoreElimination.h"
+#include "llvm/Transforms/Scalar/DivRemPairs.h"
+#include "llvm/Transforms/Scalar/EarlyCSE.h"
+#include "llvm/Transforms/Scalar/FlattenCFG.h"
+#include "llvm/Transforms/Scalar/Float2Int.h"
+#include "llvm/Transforms/Scalar/GVN.h"
+#include "llvm/Transforms/Scalar/GuardWidening.h"
+#include "llvm/Transforms/Scalar/IVUsersPrinter.h"
+#include "llvm/Transforms/Scalar/IndVarSimplify.h"
+#include "llvm/Transforms/Scalar/InductiveRangeCheckElimination.h"
+#include "llvm/Transforms/Scalar/InferAddressSpaces.h"
+#include "llvm/Transforms/Scalar/InstSimplifyPass.h"
+#include "llvm/Transforms/Scalar/JumpThreading.h"
+#include "llvm/Transforms/Scalar/LICM.h"
+#include "llvm/Transforms/Scalar/LoopAccessAnalysisPrinter.h"
+#include "llvm/Transforms/Scalar/LoopBoundSplit.h"
+#include "llvm/Transforms/Scalar/LoopDataPrefetch.h"
+#include "llvm/Transforms/Scalar/LoopDeletion.h"
+#include "llvm/Transforms/Scalar/LoopDistribute.h"
+#include "llvm/Transforms/Scalar/LoopFlatten.h"
+#include "llvm/Transforms/Scalar/LoopFuse.h"
+#include "llvm/Transforms/Scalar/LoopIdiomRecognize.h"
+#include "llvm/Transforms/Scalar/LoopInstSimplify.h"
+#include "llvm/Transforms/Scalar/LoopInterchange.h"
+#include "llvm/Transforms/Scalar/LoopLoadElimination.h"
+#include "llvm/Transforms/Scalar/LoopPassManager.h"
+#include "llvm/Transforms/Scalar/LoopPredication.h"
+#include "llvm/Transforms/Scalar/LoopReroll.h"
+#include "llvm/Transforms/Scalar/LoopRotation.h"
+#include "llvm/Transforms/Scalar/LoopSimplifyCFG.h"
+#include "llvm/Transforms/Scalar/LoopSink.h"
+#include "llvm/Transforms/Scalar/LoopStrengthReduce.h"
+#include "llvm/Transforms/Scalar/LoopUnrollAndJamPass.h"
+#include "llvm/Transforms/Scalar/LoopUnrollPass.h"
+#include "llvm/Transforms/Scalar/LoopVersioningLICM.h"
+#include "llvm/Transforms/Scalar/LowerAtomicPass.h"
+#include "llvm/Transforms/Scalar/LowerConstantIntrinsics.h"
+#include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h"
+#include "llvm/Transforms/Scalar/LowerGuardIntrinsic.h"
+#include "llvm/Transforms/Scalar/LowerMatrixIntrinsics.h"
+#include "llvm/Transforms/Scalar/LowerWidenableCondition.h"
+#include "llvm/Transforms/Scalar/MakeGuardsExplicit.h"
+#include "llvm/Transforms/Scalar/MemCpyOptimizer.h"
+#include "llvm/Transforms/Scalar/MergeICmps.h"
+#include "llvm/Transforms/Scalar/MergedLoadStoreMotion.h"
+#include "llvm/Transforms/Scalar/NaryReassociate.h"
+#include "llvm/Transforms/Scalar/NewGVN.h"
+#include "llvm/Transforms/Scalar/PartiallyInlineLibCalls.h"
+#include "llvm/Transforms/Scalar/Reassociate.h"
+#include "llvm/Transforms/Scalar/Reg2Mem.h"
+#include "llvm/Transforms/Scalar/RewriteStatepointsForGC.h"
+#include "llvm/Transforms/Scalar/SCCP.h"
+#include "llvm/Transforms/Scalar/SROA.h"
+#include "llvm/Transforms/Scalar/ScalarizeMaskedMemIntrin.h"
+#include "llvm/Transforms/Scalar/Scalarizer.h"
+#include "llvm/Transforms/Scalar/SeparateConstOffsetFromGEP.h"
+#include "llvm/Transforms/Scalar/SimpleLoopUnswitch.h"
+#include "llvm/Transforms/Scalar/SimplifyCFG.h"
+#include "llvm/Transforms/Scalar/Sink.h"
+#include "llvm/Transforms/Scalar/SpeculativeExecution.h"
+#include "llvm/Transforms/Scalar/StraightLineStrengthReduce.h"
+#include "llvm/Transforms/Scalar/StructurizeCFG.h"
+#include "llvm/Transforms/Scalar/TLSVariableHoist.h"
+#include "llvm/Transforms/Scalar/TailRecursionElimination.h"
+#include "llvm/Transforms/Scalar/WarnMissedTransforms.h"
+#include "llvm/Transforms/Utils/AddDiscriminators.h"
+#include "llvm/Transforms/Utils/AssumeBundleBuilder.h"
+#include "llvm/Transforms/Utils/BreakCriticalEdges.h"
+#include "llvm/Transforms/Utils/CanonicalizeAliases.h"
+#include "llvm/Transforms/Utils/CanonicalizeFreezeInLoops.h"
+#include "llvm/Transforms/Utils/Debugify.h"
+#include "llvm/Transforms/Utils/EntryExitInstrumenter.h"
+#include "llvm/Transforms/Utils/FixIrreducible.h"
+#include "llvm/Transforms/Utils/HelloWorld.h"
+#include "llvm/Transforms/Utils/InjectTLIMappings.h"
+#include "llvm/Transforms/Utils/InstructionNamer.h"
+#include "llvm/Transforms/Utils/LCSSA.h"
+#include "llvm/Transforms/Utils/LibCallsShrinkWrap.h"
+#include "llvm/Transforms/Utils/LoopSimplify.h"
+#include "llvm/Transforms/Utils/LoopVersioning.h"
+#include "llvm/Transforms/Utils/LowerGlobalDtors.h"
+#include "llvm/Transforms/Utils/LowerIFunc.h"
+#include "llvm/Transforms/Utils/LowerInvoke.h"
+#include "llvm/Transforms/Utils/LowerSwitch.h"
+#include "llvm/Transforms/Utils/Mem2Reg.h"
+#include "llvm/Transforms/Utils/MetaRenamer.h"
+#include "llvm/Transforms/Utils/NameAnonGlobals.h"
+#include "llvm/Transforms/Utils/PredicateInfo.h"
+#include "llvm/Transforms/Utils/RelLookupTableConverter.h"
+#include "llvm/Transforms/Utils/StripGCRelocates.h"
+#include "llvm/Transforms/Utils/StripNonLineTableDebugInfo.h"
+#include "llvm/Transforms/Utils/SymbolRewriter.h"
+#include "llvm/Transforms/Utils/UnifyFunctionExitNodes.h"
+#include "llvm/Transforms/Utils/UnifyLoopExits.h"
+#include "llvm/Transforms/Vectorize/LoadStoreVectorizer.h"
+#include "llvm/Transforms/Vectorize/LoopVectorize.h"
+#include "llvm/Transforms/Vectorize/SLPVectorizer.h"
+#include "llvm/Transforms/Vectorize/VectorCombine.h"
+#include <optional>
+
+using namespace llvm;
+
+static const Regex DefaultAliasRegex(
+ "^(default|thinlto-pre-link|thinlto|lto-pre-link|lto)<(O[0123sz])>$");
+
+namespace llvm {
+cl::opt<bool> PrintPipelinePasses(
+ "print-pipeline-passes",
+ cl::desc("Print a '-passes' compatible string describing the pipeline "
+ "(best-effort only)."));
+} // namespace llvm
+
+namespace {
+
+// The following passes/analyses have custom names, otherwise their name will
+// include `(anonymous namespace)`. These are special since they are only for
+// testing purposes and don't live in a header file.
+
+/// No-op module pass which does nothing.
+struct NoOpModulePass : PassInfoMixin<NoOpModulePass> {
+ PreservedAnalyses run(Module &M, ModuleAnalysisManager &) {
+ return PreservedAnalyses::all();
+ }
+
+ static StringRef name() { return "NoOpModulePass"; }
+};
+
+/// No-op module analysis.
+class NoOpModuleAnalysis : public AnalysisInfoMixin<NoOpModuleAnalysis> {
+ friend AnalysisInfoMixin<NoOpModuleAnalysis>;
+ static AnalysisKey Key;
+
+public:
+ struct Result {};
+ Result run(Module &, ModuleAnalysisManager &) { return Result(); }
+ static StringRef name() { return "NoOpModuleAnalysis"; }
+};
+
+/// No-op CGSCC pass which does nothing.
+struct NoOpCGSCCPass : PassInfoMixin<NoOpCGSCCPass> {
+ PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &,
+ LazyCallGraph &, CGSCCUpdateResult &UR) {
+ return PreservedAnalyses::all();
+ }
+ static StringRef name() { return "NoOpCGSCCPass"; }
+};
+
+/// No-op CGSCC analysis.
+class NoOpCGSCCAnalysis : public AnalysisInfoMixin<NoOpCGSCCAnalysis> {
+ friend AnalysisInfoMixin<NoOpCGSCCAnalysis>;
+ static AnalysisKey Key;
+
+public:
+ struct Result {};
+ Result run(LazyCallGraph::SCC &, CGSCCAnalysisManager &, LazyCallGraph &G) {
+ return Result();
+ }
+ static StringRef name() { return "NoOpCGSCCAnalysis"; }
+};
+
+/// No-op function pass which does nothing.
+struct NoOpFunctionPass : PassInfoMixin<NoOpFunctionPass> {
+ PreservedAnalyses run(Function &F, FunctionAnalysisManager &) {
+ return PreservedAnalyses::all();
+ }
+ static StringRef name() { return "NoOpFunctionPass"; }
+};
+
+/// No-op function analysis.
+class NoOpFunctionAnalysis : public AnalysisInfoMixin<NoOpFunctionAnalysis> {
+ friend AnalysisInfoMixin<NoOpFunctionAnalysis>;
+ static AnalysisKey Key;
+
+public:
+ struct Result {};
+ Result run(Function &, FunctionAnalysisManager &) { return Result(); }
+ static StringRef name() { return "NoOpFunctionAnalysis"; }
+};
+
+/// No-op loop nest pass which does nothing.
+struct NoOpLoopNestPass : PassInfoMixin<NoOpLoopNestPass> {
+ PreservedAnalyses run(LoopNest &L, LoopAnalysisManager &,
+ LoopStandardAnalysisResults &, LPMUpdater &) {
+ return PreservedAnalyses::all();
+ }
+ static StringRef name() { return "NoOpLoopNestPass"; }
+};
+
+/// No-op loop pass which does nothing.
+struct NoOpLoopPass : PassInfoMixin<NoOpLoopPass> {
+ PreservedAnalyses run(Loop &L, LoopAnalysisManager &,
+ LoopStandardAnalysisResults &, LPMUpdater &) {
+ return PreservedAnalyses::all();
+ }
+ static StringRef name() { return "NoOpLoopPass"; }
+};
+
+/// No-op loop analysis.
+class NoOpLoopAnalysis : public AnalysisInfoMixin<NoOpLoopAnalysis> {
+ friend AnalysisInfoMixin<NoOpLoopAnalysis>;
+ static AnalysisKey Key;
+
+public:
+ struct Result {};
+ Result run(Loop &, LoopAnalysisManager &, LoopStandardAnalysisResults &) {
+ return Result();
+ }
+ static StringRef name() { return "NoOpLoopAnalysis"; }
+};
+
+AnalysisKey NoOpModuleAnalysis::Key;
+AnalysisKey NoOpCGSCCAnalysis::Key;
+AnalysisKey NoOpFunctionAnalysis::Key;
+AnalysisKey NoOpLoopAnalysis::Key;
+
+/// Whether or not we should populate a PassInstrumentationCallbacks's class to
+/// pass name map.
+///
+/// This is for optimization purposes so we don't populate it if we never use
+/// it. This should be updated if new pass instrumentation wants to use the map.
+/// We currently only use this for --print-before/after.
+bool shouldPopulateClassToPassNames() {
+ return PrintPipelinePasses || !printBeforePasses().empty() ||
+ !printAfterPasses().empty() || !isFilterPassesEmpty();
+}
+
+// A pass for testing -print-on-crash.
+// DO NOT USE THIS EXCEPT FOR TESTING!
+class TriggerCrashPass : public PassInfoMixin<TriggerCrashPass> {
+public:
+ PreservedAnalyses run(Module &, ModuleAnalysisManager &) {
+ abort();
+ return PreservedAnalyses::all();
+ }
+ static StringRef name() { return "TriggerCrashPass"; }
+};
+
+} // namespace
+
+PassBuilder::PassBuilder(TargetMachine *TM, PipelineTuningOptions PTO,
+ std::optional<PGOOptions> PGOOpt,
+ PassInstrumentationCallbacks *PIC)
+ : TM(TM), PTO(PTO), PGOOpt(PGOOpt), PIC(PIC) {
+ if (TM)
+ TM->registerPassBuilderCallbacks(*this);
+ if (PIC && shouldPopulateClassToPassNames()) {
+#define MODULE_PASS(NAME, CREATE_PASS) \
+ PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
+#define MODULE_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS) \
+ PIC->addClassToPassName(CLASS, NAME);
+#define MODULE_ANALYSIS(NAME, CREATE_PASS) \
+ PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
+#define FUNCTION_PASS(NAME, CREATE_PASS) \
+ PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
+#define FUNCTION_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS) \
+ PIC->addClassToPassName(CLASS, NAME);
+#define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \
+ PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
+#define LOOPNEST_PASS(NAME, CREATE_PASS) \
+ PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
+#define LOOP_PASS(NAME, CREATE_PASS) \
+ PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
+#define LOOP_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS) \
+ PIC->addClassToPassName(CLASS, NAME);
+#define LOOP_ANALYSIS(NAME, CREATE_PASS) \
+ PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
+#define CGSCC_PASS(NAME, CREATE_PASS) \
+ PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
+#define CGSCC_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS) \
+ PIC->addClassToPassName(CLASS, NAME);
+#define CGSCC_ANALYSIS(NAME, CREATE_PASS) \
+ PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
+#include "PassRegistry.def"
+ }
+}
+
+void PassBuilder::registerModuleAnalyses(ModuleAnalysisManager &MAM) {
+#define MODULE_ANALYSIS(NAME, CREATE_PASS) \
+ MAM.registerPass([&] { return CREATE_PASS; });
+#include "PassRegistry.def"
+
+ for (auto &C : ModuleAnalysisRegistrationCallbacks)
+ C(MAM);
+}
+
+void PassBuilder::registerCGSCCAnalyses(CGSCCAnalysisManager &CGAM) {
+#define CGSCC_ANALYSIS(NAME, CREATE_PASS) \
+ CGAM.registerPass([&] { return CREATE_PASS; });
+#include "PassRegistry.def"
+
+ for (auto &C : CGSCCAnalysisRegistrationCallbacks)
+ C(CGAM);
+}
+
+void PassBuilder::registerFunctionAnalyses(FunctionAnalysisManager &FAM) {
+ // We almost always want the default alias analysis pipeline.
+ // If a user wants a different one, they can register their own before calling
+ // registerFunctionAnalyses().
+ FAM.registerPass([&] { return buildDefaultAAPipeline(); });
+
+#define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \
+ FAM.registerPass([&] { return CREATE_PASS; });
+#include "PassRegistry.def"
+
+ for (auto &C : FunctionAnalysisRegistrationCallbacks)
+ C(FAM);
+}
+
+void PassBuilder::registerLoopAnalyses(LoopAnalysisManager &LAM) {
+#define LOOP_ANALYSIS(NAME, CREATE_PASS) \
+ LAM.registerPass([&] { return CREATE_PASS; });
+#include "PassRegistry.def"
+
+ for (auto &C : LoopAnalysisRegistrationCallbacks)
+ C(LAM);
+}
+
+static std::optional<int> parseRepeatPassName(StringRef Name) {
+ if (!Name.consume_front("repeat<") || !Name.consume_back(">"))
+ return std::nullopt;
+ int Count;
+ if (Name.getAsInteger(0, Count) || Count <= 0)
+ return std::nullopt;
+ return Count;
+}
+
+static std::optional<int> parseDevirtPassName(StringRef Name) {
+ if (!Name.consume_front("devirt<") || !Name.consume_back(">"))
+ return std::nullopt;
+ int Count;
+ if (Name.getAsInteger(0, Count) || Count < 0)
+ return std::nullopt;
+ return Count;
+}
+
+static bool checkParametrizedPassName(StringRef Name, StringRef PassName) {
+ if (!Name.consume_front(PassName))
+ return false;
+ // normal pass name w/o parameters == default parameters
+ if (Name.empty())
+ return true;
+ return Name.startswith("<") && Name.endswith(">");
+}
+
+namespace {
+
+/// This performs customized parsing of pass name with parameters.
+///
+/// We do not need parametrization of passes in textual pipeline very often,
+/// yet on a rare occasion ability to specify parameters right there can be
+/// useful.
+///
+/// \p Name - parameterized specification of a pass from a textual pipeline
+/// is a string in a form of :
+/// PassName '<' parameter-list '>'
+///
+/// Parameter list is being parsed by the parser callable argument, \p Parser,
+/// It takes a string-ref of parameters and returns either StringError or a
+/// parameter list in a form of a custom parameters type, all wrapped into
+/// Expected<> template class.
+///
+template <typename ParametersParseCallableT>
+auto parsePassParameters(ParametersParseCallableT &&Parser, StringRef Name,
+ StringRef PassName) -> decltype(Parser(StringRef{})) {
+ using ParametersT = typename decltype(Parser(StringRef{}))::value_type;
+
+ StringRef Params = Name;
+ if (!Params.consume_front(PassName)) {
+ assert(false &&
+ "unable to strip pass name from parametrized pass specification");
+ }
+ if (!Params.empty() &&
+ (!Params.consume_front("<") || !Params.consume_back(">"))) {
+ assert(false && "invalid format for parametrized pass name");
+ }
+
+ Expected<ParametersT> Result = Parser(Params);
+ assert((Result || Result.template errorIsA<StringError>()) &&
+ "Pass parameter parser can only return StringErrors.");
+ return Result;
+}
+
+/// Parser of parameters for LoopUnroll pass.
+Expected<LoopUnrollOptions> parseLoopUnrollOptions(StringRef Params) {
+ LoopUnrollOptions UnrollOpts;
+ while (!Params.empty()) {
+ StringRef ParamName;
+ std::tie(ParamName, Params) = Params.split(';');
+ int OptLevel = StringSwitch<int>(ParamName)
+ .Case("O0", 0)
+ .Case("O1", 1)
+ .Case("O2", 2)
+ .Case("O3", 3)
+ .Default(-1);
+ if (OptLevel >= 0) {
+ UnrollOpts.setOptLevel(OptLevel);
+ continue;
+ }
+ if (ParamName.consume_front("full-unroll-max=")) {
+ int Count;
+ if (ParamName.getAsInteger(0, Count))
+ return make_error<StringError>(
+ formatv("invalid LoopUnrollPass parameter '{0}' ", ParamName).str(),
+ inconvertibleErrorCode());
+ UnrollOpts.setFullUnrollMaxCount(Count);
+ continue;
+ }
+
+ bool Enable = !ParamName.consume_front("no-");
+ if (ParamName == "partial") {
+ UnrollOpts.setPartial(Enable);
+ } else if (ParamName == "peeling") {
+ UnrollOpts.setPeeling(Enable);
+ } else if (ParamName == "profile-peeling") {
+ UnrollOpts.setProfileBasedPeeling(Enable);
+ } else if (ParamName == "runtime") {
+ UnrollOpts.setRuntime(Enable);
+ } else if (ParamName == "upperbound") {
+ UnrollOpts.setUpperBound(Enable);
+ } else {
+ return make_error<StringError>(
+ formatv("invalid LoopUnrollPass parameter '{0}' ", ParamName).str(),
+ inconvertibleErrorCode());
+ }
+ }
+ return UnrollOpts;
+}
+
+Expected<bool> parseSinglePassOption(StringRef Params, StringRef OptionName,
+ StringRef PassName) {
+ bool Result = false;
+ while (!Params.empty()) {
+ StringRef ParamName;
+ std::tie(ParamName, Params) = Params.split(';');
+
+ if (ParamName == OptionName) {
+ Result = true;
+ } else {
+ return make_error<StringError>(
+ formatv("invalid {1} pass parameter '{0}' ", ParamName, PassName)
+ .str(),
+ inconvertibleErrorCode());
+ }
+ }
+ return Result;
+}
+
+Expected<bool> parseInlinerPassOptions(StringRef Params) {
+ return parseSinglePassOption(Params, "only-mandatory", "InlinerPass");
+}
+
+Expected<bool> parseCoroSplitPassOptions(StringRef Params) {
+ return parseSinglePassOption(Params, "reuse-storage", "CoroSplitPass");
+}
+
+Expected<bool> parseEarlyCSEPassOptions(StringRef Params) {
+ return parseSinglePassOption(Params, "memssa", "EarlyCSE");
+}
+
+Expected<bool> parseEntryExitInstrumenterPassOptions(StringRef Params) {
+ return parseSinglePassOption(Params, "post-inline", "EntryExitInstrumenter");
+}
+
+Expected<bool> parseLoopExtractorPassOptions(StringRef Params) {
+ return parseSinglePassOption(Params, "single", "LoopExtractor");
+}
+
+Expected<bool> parseLowerMatrixIntrinsicsPassOptions(StringRef Params) {
+ return parseSinglePassOption(Params, "minimal", "LowerMatrixIntrinsics");
+}
+
+Expected<AddressSanitizerOptions> parseASanPassOptions(StringRef Params) {
+ AddressSanitizerOptions Result;
+ while (!Params.empty()) {
+ StringRef ParamName;
+ std::tie(ParamName, Params) = Params.split(';');
+
+ if (ParamName == "kernel") {
+ Result.CompileKernel = true;
+ } else {
+ return make_error<StringError>(
+ formatv("invalid AddressSanitizer pass parameter '{0}' ", ParamName)
+ .str(),
+ inconvertibleErrorCode());
+ }
+ }
+ return Result;
+}
+
+Expected<HWAddressSanitizerOptions> parseHWASanPassOptions(StringRef Params) {
+ HWAddressSanitizerOptions Result;
+ while (!Params.empty()) {
+ StringRef ParamName;
+ std::tie(ParamName, Params) = Params.split(';');
+
+ if (ParamName == "recover") {
+ Result.Recover = true;
+ } else if (ParamName == "kernel") {
+ Result.CompileKernel = true;
+ } else {
+ return make_error<StringError>(
+ formatv("invalid HWAddressSanitizer pass parameter '{0}' ", ParamName)
+ .str(),
+ inconvertibleErrorCode());
+ }
+ }
+ return Result;
+}
+
+Expected<MemorySanitizerOptions> parseMSanPassOptions(StringRef Params) {
+ MemorySanitizerOptions Result;
+ while (!Params.empty()) {
+ StringRef ParamName;
+ std::tie(ParamName, Params) = Params.split(';');
+
+ if (ParamName == "recover") {
+ Result.Recover = true;
+ } else if (ParamName == "kernel") {
+ Result.Kernel = true;
+ } else if (ParamName.consume_front("track-origins=")) {
+ if (ParamName.getAsInteger(0, Result.TrackOrigins))
+ return make_error<StringError>(
+ formatv("invalid argument to MemorySanitizer pass track-origins "
+ "parameter: '{0}' ",
+ ParamName)
+ .str(),
+ inconvertibleErrorCode());
+ } else if (ParamName == "eager-checks") {
+ Result.EagerChecks = true;
+ } else {
+ return make_error<StringError>(
+ formatv("invalid MemorySanitizer pass parameter '{0}' ", ParamName)
+ .str(),
+ inconvertibleErrorCode());
+ }
+ }
+ return Result;
+}
+
+/// Parser of parameters for SimplifyCFG pass.
+Expected<SimplifyCFGOptions> parseSimplifyCFGOptions(StringRef Params) {
+ SimplifyCFGOptions Result;
+ while (!Params.empty()) {
+ StringRef ParamName;
+ std::tie(ParamName, Params) = Params.split(';');
+
+ bool Enable = !ParamName.consume_front("no-");
+ if (ParamName == "forward-switch-cond") {
+ Result.forwardSwitchCondToPhi(Enable);
+ } else if (ParamName == "switch-range-to-icmp") {
+ Result.convertSwitchRangeToICmp(Enable);
+ } else if (ParamName == "switch-to-lookup") {
+ Result.convertSwitchToLookupTable(Enable);
+ } else if (ParamName == "keep-loops") {
+ Result.needCanonicalLoops(Enable);
+ } else if (ParamName == "hoist-common-insts") {
+ Result.hoistCommonInsts(Enable);
+ } else if (ParamName == "sink-common-insts") {
+ Result.sinkCommonInsts(Enable);
+ } else if (Enable && ParamName.consume_front("bonus-inst-threshold=")) {
+ APInt BonusInstThreshold;
+ if (ParamName.getAsInteger(0, BonusInstThreshold))
+ return make_error<StringError>(
+ formatv("invalid argument to SimplifyCFG pass bonus-threshold "
+ "parameter: '{0}' ",
+ ParamName).str(),
+ inconvertibleErrorCode());
+ Result.bonusInstThreshold(BonusInstThreshold.getSExtValue());
+ } else {
+ return make_error<StringError>(
+ formatv("invalid SimplifyCFG pass parameter '{0}' ", ParamName).str(),
+ inconvertibleErrorCode());
+ }
+ }
+ return Result;
+}
+
+/// Parser of parameters for LoopVectorize pass.
+Expected<LoopVectorizeOptions> parseLoopVectorizeOptions(StringRef Params) {
+ LoopVectorizeOptions Opts;
+ while (!Params.empty()) {
+ StringRef ParamName;
+ std::tie(ParamName, Params) = Params.split(';');
+
+ bool Enable = !ParamName.consume_front("no-");
+ if (ParamName == "interleave-forced-only") {
+ Opts.setInterleaveOnlyWhenForced(Enable);
+ } else if (ParamName == "vectorize-forced-only") {
+ Opts.setVectorizeOnlyWhenForced(Enable);
+ } else {
+ return make_error<StringError>(
+ formatv("invalid LoopVectorize parameter '{0}' ", ParamName).str(),
+ inconvertibleErrorCode());
+ }
+ }
+ return Opts;
+}
+
+Expected<std::pair<bool, bool>> parseLoopUnswitchOptions(StringRef Params) {
+ std::pair<bool, bool> Result = {false, true};
+ while (!Params.empty()) {
+ StringRef ParamName;
+ std::tie(ParamName, Params) = Params.split(';');
+
+ bool Enable = !ParamName.consume_front("no-");
+ if (ParamName == "nontrivial") {
+ Result.first = Enable;
+ } else if (ParamName == "trivial") {
+ Result.second = Enable;
+ } else {
+ return make_error<StringError>(
+ formatv("invalid LoopUnswitch pass parameter '{0}' ", ParamName)
+ .str(),
+ inconvertibleErrorCode());
+ }
+ }
+ return Result;
+}
+
+Expected<LICMOptions> parseLICMOptions(StringRef Params) {
+ LICMOptions Result;
+ while (!Params.empty()) {
+ StringRef ParamName;
+ std::tie(ParamName, Params) = Params.split(';');
+
+ bool Enable = !ParamName.consume_front("no-");
+ if (ParamName == "allowspeculation") {
+ Result.AllowSpeculation = Enable;
+ } else {
+ return make_error<StringError>(
+ formatv("invalid LICM pass parameter '{0}' ", ParamName).str(),
+ inconvertibleErrorCode());
+ }
+ }
+ return Result;
+}
+
+Expected<bool> parseMergedLoadStoreMotionOptions(StringRef Params) {
+ bool Result = false;
+ while (!Params.empty()) {
+ StringRef ParamName;
+ std::tie(ParamName, Params) = Params.split(';');
+
+ bool Enable = !ParamName.consume_front("no-");
+ if (ParamName == "split-footer-bb") {
+ Result = Enable;
+ } else {
+ return make_error<StringError>(
+ formatv("invalid MergedLoadStoreMotion pass parameter '{0}' ",
+ ParamName)
+ .str(),
+ inconvertibleErrorCode());
+ }
+ }
+ return Result;
+}
+
+Expected<GVNOptions> parseGVNOptions(StringRef Params) {
+ GVNOptions Result;
+ while (!Params.empty()) {
+ StringRef ParamName;
+ std::tie(ParamName, Params) = Params.split(';');
+
+ bool Enable = !ParamName.consume_front("no-");
+ if (ParamName == "pre") {
+ Result.setPRE(Enable);
+ } else if (ParamName == "load-pre") {
+ Result.setLoadPRE(Enable);
+ } else if (ParamName == "split-backedge-load-pre") {
+ Result.setLoadPRESplitBackedge(Enable);
+ } else if (ParamName == "memdep") {
+ Result.setMemDep(Enable);
+ } else {
+ return make_error<StringError>(
+ formatv("invalid GVN pass parameter '{0}' ", ParamName).str(),
+ inconvertibleErrorCode());
+ }
+ }
+ return Result;
+}
+
+Expected<IPSCCPOptions> parseIPSCCPOptions(StringRef Params) {
+ IPSCCPOptions Result;
+ while (!Params.empty()) {
+ StringRef ParamName;
+ std::tie(ParamName, Params) = Params.split(';');
+
+ bool Enable = !ParamName.consume_front("no-");
+ if (ParamName == "func-spec")
+ Result.setFuncSpec(Enable);
+ else
+ return make_error<StringError>(
+ formatv("invalid IPSCCP pass parameter '{0}' ", ParamName).str(),
+ inconvertibleErrorCode());
+ }
+ return Result;
+}
+
+Expected<SROAOptions> parseSROAOptions(StringRef Params) {
+ if (Params.empty() || Params == "modify-cfg")
+ return SROAOptions::ModifyCFG;
+ if (Params == "preserve-cfg")
+ return SROAOptions::PreserveCFG;
+ return make_error<StringError>(
+ formatv("invalid SROA pass parameter '{0}' (either preserve-cfg or "
+ "modify-cfg can be specified)",
+ Params)
+ .str(),
+ inconvertibleErrorCode());
+}
+
+Expected<StackLifetime::LivenessType>
+parseStackLifetimeOptions(StringRef Params) {
+ StackLifetime::LivenessType Result = StackLifetime::LivenessType::May;
+ while (!Params.empty()) {
+ StringRef ParamName;
+ std::tie(ParamName, Params) = Params.split(';');
+
+ if (ParamName == "may") {
+ Result = StackLifetime::LivenessType::May;
+ } else if (ParamName == "must") {
+ Result = StackLifetime::LivenessType::Must;
+ } else {
+ return make_error<StringError>(
+ formatv("invalid StackLifetime parameter '{0}' ", ParamName).str(),
+ inconvertibleErrorCode());
+ }
+ }
+ return Result;
+}
+
+Expected<bool> parseDependenceAnalysisPrinterOptions(StringRef Params) {
+ return parseSinglePassOption(Params, "normalized-results",
+ "DependenceAnalysisPrinter");
+}
+
+} // namespace
+
+/// Tests whether a pass name starts with a valid prefix for a default pipeline
+/// alias.
+static bool startsWithDefaultPipelineAliasPrefix(StringRef Name) {
+ return Name.startswith("default") || Name.startswith("thinlto") ||
+ Name.startswith("lto");
+}
+
+/// Tests whether registered callbacks will accept a given pass name.
+///
+/// When parsing a pipeline text, the type of the outermost pipeline may be
+/// omitted, in which case the type is automatically determined from the first
+/// pass name in the text. This may be a name that is handled through one of the
+/// callbacks. We check this through the oridinary parsing callbacks by setting
+/// up a dummy PassManager in order to not force the client to also handle this
+/// type of query.
+template <typename PassManagerT, typename CallbacksT>
+static bool callbacksAcceptPassName(StringRef Name, CallbacksT &Callbacks) {
+ if (!Callbacks.empty()) {
+ PassManagerT DummyPM;
+ for (auto &CB : Callbacks)
+ if (CB(Name, DummyPM, {}))
+ return true;
+ }
+ return false;
+}
+
+template <typename CallbacksT>
+static bool isModulePassName(StringRef Name, CallbacksT &Callbacks) {
+ // Manually handle aliases for pre-configured pipeline fragments.
+ if (startsWithDefaultPipelineAliasPrefix(Name))
+ return DefaultAliasRegex.match(Name);
+
+ // Explicitly handle pass manager names.
+ if (Name == "module")
+ return true;
+ if (Name == "cgscc")
+ return true;
+ if (Name == "function" || Name == "function<eager-inv>")
+ return true;
+ if (Name == "coro-cond")
+ return true;
+
+ // Explicitly handle custom-parsed pass names.
+ if (parseRepeatPassName(Name))
+ return true;
+
+#define MODULE_PASS(NAME, CREATE_PASS) \
+ if (Name == NAME) \
+ return true;
+#define MODULE_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS) \
+ if (checkParametrizedPassName(Name, NAME)) \
+ return true;
+#define MODULE_ANALYSIS(NAME, CREATE_PASS) \
+ if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \
+ return true;
+#include "PassRegistry.def"
+
+ return callbacksAcceptPassName<ModulePassManager>(Name, Callbacks);
+}
+
+template <typename CallbacksT>
+static bool isCGSCCPassName(StringRef Name, CallbacksT &Callbacks) {
+ // Explicitly handle pass manager names.
+ if (Name == "cgscc")
+ return true;
+ if (Name == "function" || Name == "function<eager-inv>")
+ return true;
+
+ // Explicitly handle custom-parsed pass names.
+ if (parseRepeatPassName(Name))
+ return true;
+ if (parseDevirtPassName(Name))
+ return true;
+
+#define CGSCC_PASS(NAME, CREATE_PASS) \
+ if (Name == NAME) \
+ return true;
+#define CGSCC_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS) \
+ if (checkParametrizedPassName(Name, NAME)) \
+ return true;
+#define CGSCC_ANALYSIS(NAME, CREATE_PASS) \
+ if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \
+ return true;
+#include "PassRegistry.def"
+
+ return callbacksAcceptPassName<CGSCCPassManager>(Name, Callbacks);
+}
+
+template <typename CallbacksT>
+static bool isFunctionPassName(StringRef Name, CallbacksT &Callbacks) {
+ // Explicitly handle pass manager names.
+ if (Name == "function" || Name == "function<eager-inv>")
+ return true;
+ if (Name == "loop" || Name == "loop-mssa")
+ return true;
+
+ // Explicitly handle custom-parsed pass names.
+ if (parseRepeatPassName(Name))
+ return true;
+
+#define FUNCTION_PASS(NAME, CREATE_PASS) \
+ if (Name == NAME) \
+ return true;
+#define FUNCTION_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS) \
+ if (checkParametrizedPassName(Name, NAME)) \
+ return true;
+#define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \
+ if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \
+ return true;
+#include "PassRegistry.def"
+
+ return callbacksAcceptPassName<FunctionPassManager>(Name, Callbacks);
+}
+
+template <typename CallbacksT>
+static bool isLoopNestPassName(StringRef Name, CallbacksT &Callbacks,
+ bool &UseMemorySSA) {
+ UseMemorySSA = false;
+
+ // Explicitly handle custom-parsed pass names.
+ if (parseRepeatPassName(Name))
+ return true;
+
+ if (checkParametrizedPassName(Name, "lnicm")) {
+ UseMemorySSA = true;
+ return true;
+ }
+
+#define LOOPNEST_PASS(NAME, CREATE_PASS) \
+ if (Name == NAME) \
+ return true;
+#include "PassRegistry.def"
+
+ return callbacksAcceptPassName<LoopPassManager>(Name, Callbacks);
+}
+
+template <typename CallbacksT>
+static bool isLoopPassName(StringRef Name, CallbacksT &Callbacks,
+ bool &UseMemorySSA) {
+ UseMemorySSA = false;
+
+ // Explicitly handle custom-parsed pass names.
+ if (parseRepeatPassName(Name))
+ return true;
+
+ if (checkParametrizedPassName(Name, "licm")) {
+ UseMemorySSA = true;
+ return true;
+ }
+
+#define LOOP_PASS(NAME, CREATE_PASS) \
+ if (Name == NAME) \
+ return true;
+#define LOOP_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS) \
+ if (checkParametrizedPassName(Name, NAME)) \
+ return true;
+#define LOOP_ANALYSIS(NAME, CREATE_PASS) \
+ if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \
+ return true;
+#include "PassRegistry.def"
+
+ return callbacksAcceptPassName<LoopPassManager>(Name, Callbacks);
+}
+
+std::optional<std::vector<PassBuilder::PipelineElement>>
+PassBuilder::parsePipelineText(StringRef Text) {
+ std::vector<PipelineElement> ResultPipeline;
+
+ SmallVector<std::vector<PipelineElement> *, 4> PipelineStack = {
+ &ResultPipeline};
+ for (;;) {
+ std::vector<PipelineElement> &Pipeline = *PipelineStack.back();
+ size_t Pos = Text.find_first_of(",()");
+ Pipeline.push_back({Text.substr(0, Pos), {}});
+
+ // If we have a single terminating name, we're done.
+ if (Pos == Text.npos)
+ break;
+
+ char Sep = Text[Pos];
+ Text = Text.substr(Pos + 1);
+ if (Sep == ',')
+ // Just a name ending in a comma, continue.
+ continue;
+
+ if (Sep == '(') {
+ // Push the inner pipeline onto the stack to continue processing.
+ PipelineStack.push_back(&Pipeline.back().InnerPipeline);
+ continue;
+ }
+
+ assert(Sep == ')' && "Bogus separator!");
+ // When handling the close parenthesis, we greedily consume them to avoid
+ // empty strings in the pipeline.
+ do {
+ // If we try to pop the outer pipeline we have unbalanced parentheses.
+ if (PipelineStack.size() == 1)
+ return std::nullopt;
+
+ PipelineStack.pop_back();
+ } while (Text.consume_front(")"));
+
+ // Check if we've finished parsing.
+ if (Text.empty())
+ break;
+
+ // Otherwise, the end of an inner pipeline always has to be followed by
+ // a comma, and then we can continue.
+ if (!Text.consume_front(","))
+ return std::nullopt;
+ }
+
+ if (PipelineStack.size() > 1)
+ // Unbalanced paretheses.
+ return std::nullopt;
+
+ assert(PipelineStack.back() == &ResultPipeline &&
+ "Wrong pipeline at the bottom of the stack!");
+ return {std::move(ResultPipeline)};
+}
+
+Error PassBuilder::parseModulePass(ModulePassManager &MPM,
+ const PipelineElement &E) {
+ auto &Name = E.Name;
+ auto &InnerPipeline = E.InnerPipeline;
+
+ // First handle complex passes like the pass managers which carry pipelines.
+ if (!InnerPipeline.empty()) {
+ if (Name == "module") {
+ ModulePassManager NestedMPM;
+ if (auto Err = parseModulePassPipeline(NestedMPM, InnerPipeline))
+ return Err;
+ MPM.addPass(std::move(NestedMPM));
+ return Error::success();
+ }
+ if (Name == "coro-cond") {
+ ModulePassManager NestedMPM;
+ if (auto Err = parseModulePassPipeline(NestedMPM, InnerPipeline))
+ return Err;
+ MPM.addPass(CoroConditionalWrapper(std::move(NestedMPM)));
+ return Error::success();
+ }
+ if (Name == "cgscc") {
+ CGSCCPassManager CGPM;
+ if (auto Err = parseCGSCCPassPipeline(CGPM, InnerPipeline))
+ return Err;
+ MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM)));
+ return Error::success();
+ }
+ if (Name == "function" || Name == "function<eager-inv>") {
+ FunctionPassManager FPM;
+ if (auto Err = parseFunctionPassPipeline(FPM, InnerPipeline))
+ return Err;
+ MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM),
+ Name != "function"));
+ return Error::success();
+ }
+ if (auto Count = parseRepeatPassName(Name)) {
+ ModulePassManager NestedMPM;
+ if (auto Err = parseModulePassPipeline(NestedMPM, InnerPipeline))
+ return Err;
+ MPM.addPass(createRepeatedPass(*Count, std::move(NestedMPM)));
+ return Error::success();
+ }
+
+ for (auto &C : ModulePipelineParsingCallbacks)
+ if (C(Name, MPM, InnerPipeline))
+ return Error::success();
+
+ // Normal passes can't have pipelines.
+ return make_error<StringError>(
+ formatv("invalid use of '{0}' pass as module pipeline", Name).str(),
+ inconvertibleErrorCode());
+ ;
+ }
+
+ // Manually handle aliases for pre-configured pipeline fragments.
+ if (startsWithDefaultPipelineAliasPrefix(Name)) {
+ SmallVector<StringRef, 3> Matches;
+ if (!DefaultAliasRegex.match(Name, &Matches))
+ return make_error<StringError>(
+ formatv("unknown default pipeline alias '{0}'", Name).str(),
+ inconvertibleErrorCode());
+
+ assert(Matches.size() == 3 && "Must capture two matched strings!");
+
+ OptimizationLevel L = StringSwitch<OptimizationLevel>(Matches[2])
+ .Case("O0", OptimizationLevel::O0)
+ .Case("O1", OptimizationLevel::O1)
+ .Case("O2", OptimizationLevel::O2)
+ .Case("O3", OptimizationLevel::O3)
+ .Case("Os", OptimizationLevel::Os)
+ .Case("Oz", OptimizationLevel::Oz);
+ if (L == OptimizationLevel::O0 && Matches[1] != "thinlto" &&
+ Matches[1] != "lto") {
+ MPM.addPass(buildO0DefaultPipeline(L, Matches[1] == "thinlto-pre-link" ||
+ Matches[1] == "lto-pre-link"));
+ return Error::success();
+ }
+
+ // This is consistent with old pass manager invoked via opt, but
+ // inconsistent with clang. Clang doesn't enable loop vectorization
+ // but does enable slp vectorization at Oz.
+ PTO.LoopVectorization =
+ L.getSpeedupLevel() > 1 && L != OptimizationLevel::Oz;
+ PTO.SLPVectorization =
+ L.getSpeedupLevel() > 1 && L != OptimizationLevel::Oz;
+
+ if (Matches[1] == "default") {
+ MPM.addPass(buildPerModuleDefaultPipeline(L));
+ } else if (Matches[1] == "thinlto-pre-link") {
+ MPM.addPass(buildThinLTOPreLinkDefaultPipeline(L));
+ } else if (Matches[1] == "thinlto") {
+ MPM.addPass(buildThinLTODefaultPipeline(L, nullptr));
+ } else if (Matches[1] == "lto-pre-link") {
+ MPM.addPass(buildLTOPreLinkDefaultPipeline(L));
+ } else {
+ assert(Matches[1] == "lto" && "Not one of the matched options!");
+ MPM.addPass(buildLTODefaultPipeline(L, nullptr));
+ }
+ return Error::success();
+ }
+
+ // Finally expand the basic registered passes from the .inc file.
+#define MODULE_PASS(NAME, CREATE_PASS) \
+ if (Name == NAME) { \
+ MPM.addPass(CREATE_PASS); \
+ return Error::success(); \
+ }
+#define MODULE_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS) \
+ if (checkParametrizedPassName(Name, NAME)) { \
+ auto Params = parsePassParameters(PARSER, Name, NAME); \
+ if (!Params) \
+ return Params.takeError(); \
+ MPM.addPass(CREATE_PASS(Params.get())); \
+ return Error::success(); \
+ }
+#define MODULE_ANALYSIS(NAME, CREATE_PASS) \
+ if (Name == "require<" NAME ">") { \
+ MPM.addPass( \
+ RequireAnalysisPass< \
+ std::remove_reference_t<decltype(CREATE_PASS)>, Module>()); \
+ return Error::success(); \
+ } \
+ if (Name == "invalidate<" NAME ">") { \
+ MPM.addPass(InvalidateAnalysisPass< \
+ std::remove_reference_t<decltype(CREATE_PASS)>>()); \
+ return Error::success(); \
+ }
+#define CGSCC_PASS(NAME, CREATE_PASS) \
+ if (Name == NAME) { \
+ MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(CREATE_PASS)); \
+ return Error::success(); \
+ }
+#define CGSCC_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS) \
+ if (checkParametrizedPassName(Name, NAME)) { \
+ auto Params = parsePassParameters(PARSER, Name, NAME); \
+ if (!Params) \
+ return Params.takeError(); \
+ MPM.addPass( \
+ createModuleToPostOrderCGSCCPassAdaptor(CREATE_PASS(Params.get()))); \
+ return Error::success(); \
+ }
+#define FUNCTION_PASS(NAME, CREATE_PASS) \
+ if (Name == NAME) { \
+ MPM.addPass(createModuleToFunctionPassAdaptor(CREATE_PASS)); \
+ return Error::success(); \
+ }
+#define FUNCTION_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS) \
+ if (checkParametrizedPassName(Name, NAME)) { \
+ auto Params = parsePassParameters(PARSER, Name, NAME); \
+ if (!Params) \
+ return Params.takeError(); \
+ MPM.addPass(createModuleToFunctionPassAdaptor(CREATE_PASS(Params.get()))); \
+ return Error::success(); \
+ }
+#define LOOPNEST_PASS(NAME, CREATE_PASS) \
+ if (Name == NAME) { \
+ MPM.addPass(createModuleToFunctionPassAdaptor( \
+ createFunctionToLoopPassAdaptor(CREATE_PASS, false, false))); \
+ return Error::success(); \
+ }
+#define LOOP_PASS(NAME, CREATE_PASS) \
+ if (Name == NAME) { \
+ MPM.addPass(createModuleToFunctionPassAdaptor( \
+ createFunctionToLoopPassAdaptor(CREATE_PASS, false, false))); \
+ return Error::success(); \
+ }
+#define LOOP_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS) \
+ if (checkParametrizedPassName(Name, NAME)) { \
+ auto Params = parsePassParameters(PARSER, Name, NAME); \
+ if (!Params) \
+ return Params.takeError(); \
+ MPM.addPass( \
+ createModuleToFunctionPassAdaptor(createFunctionToLoopPassAdaptor( \
+ CREATE_PASS(Params.get()), false, false))); \
+ return Error::success(); \
+ }
+#include "PassRegistry.def"
+
+ for (auto &C : ModulePipelineParsingCallbacks)
+ if (C(Name, MPM, InnerPipeline))
+ return Error::success();
+ return make_error<StringError>(
+ formatv("unknown module pass '{0}'", Name).str(),
+ inconvertibleErrorCode());
+}
+
+Error PassBuilder::parseCGSCCPass(CGSCCPassManager &CGPM,
+ const PipelineElement &E) {
+ auto &Name = E.Name;
+ auto &InnerPipeline = E.InnerPipeline;
+
+ // First handle complex passes like the pass managers which carry pipelines.
+ if (!InnerPipeline.empty()) {
+ if (Name == "cgscc") {
+ CGSCCPassManager NestedCGPM;
+ if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline))
+ return Err;
+ // Add the nested pass manager with the appropriate adaptor.
+ CGPM.addPass(std::move(NestedCGPM));
+ return Error::success();
+ }
+ if (Name == "function" || Name == "function<eager-inv>") {
+ FunctionPassManager FPM;
+ if (auto Err = parseFunctionPassPipeline(FPM, InnerPipeline))
+ return Err;
+ // Add the nested pass manager with the appropriate adaptor.
+ CGPM.addPass(
+ createCGSCCToFunctionPassAdaptor(std::move(FPM), Name != "function"));
+ return Error::success();
+ }
+ if (auto Count = parseRepeatPassName(Name)) {
+ CGSCCPassManager NestedCGPM;
+ if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline))
+ return Err;
+ CGPM.addPass(createRepeatedPass(*Count, std::move(NestedCGPM)));
+ return Error::success();
+ }
+ if (auto MaxRepetitions = parseDevirtPassName(Name)) {
+ CGSCCPassManager NestedCGPM;
+ if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline))
+ return Err;
+ CGPM.addPass(
+ createDevirtSCCRepeatedPass(std::move(NestedCGPM), *MaxRepetitions));
+ return Error::success();
+ }
+
+ for (auto &C : CGSCCPipelineParsingCallbacks)
+ if (C(Name, CGPM, InnerPipeline))
+ return Error::success();
+
+ // Normal passes can't have pipelines.
+ return make_error<StringError>(
+ formatv("invalid use of '{0}' pass as cgscc pipeline", Name).str(),
+ inconvertibleErrorCode());
+ }
+
+// Now expand the basic registered passes from the .inc file.
+#define CGSCC_PASS(NAME, CREATE_PASS) \
+ if (Name == NAME) { \
+ CGPM.addPass(CREATE_PASS); \
+ return Error::success(); \
+ }
+#define CGSCC_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS) \
+ if (checkParametrizedPassName(Name, NAME)) { \
+ auto Params = parsePassParameters(PARSER, Name, NAME); \
+ if (!Params) \
+ return Params.takeError(); \
+ CGPM.addPass(CREATE_PASS(Params.get())); \
+ return Error::success(); \
+ }
+#define CGSCC_ANALYSIS(NAME, CREATE_PASS) \
+ if (Name == "require<" NAME ">") { \
+ CGPM.addPass(RequireAnalysisPass< \
+ std::remove_reference_t<decltype(CREATE_PASS)>, \
+ LazyCallGraph::SCC, CGSCCAnalysisManager, LazyCallGraph &, \
+ CGSCCUpdateResult &>()); \
+ return Error::success(); \
+ } \
+ if (Name == "invalidate<" NAME ">") { \
+ CGPM.addPass(InvalidateAnalysisPass< \
+ std::remove_reference_t<decltype(CREATE_PASS)>>()); \
+ return Error::success(); \
+ }
+#define FUNCTION_PASS(NAME, CREATE_PASS) \
+ if (Name == NAME) { \
+ CGPM.addPass(createCGSCCToFunctionPassAdaptor(CREATE_PASS)); \
+ return Error::success(); \
+ }
+#define FUNCTION_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS) \
+ if (checkParametrizedPassName(Name, NAME)) { \
+ auto Params = parsePassParameters(PARSER, Name, NAME); \
+ if (!Params) \
+ return Params.takeError(); \
+ CGPM.addPass(createCGSCCToFunctionPassAdaptor(CREATE_PASS(Params.get()))); \
+ return Error::success(); \
+ }
+#define LOOPNEST_PASS(NAME, CREATE_PASS) \
+ if (Name == NAME) { \
+ CGPM.addPass(createCGSCCToFunctionPassAdaptor( \
+ createFunctionToLoopPassAdaptor(CREATE_PASS, false, false))); \
+ return Error::success(); \
+ }
+#define LOOP_PASS(NAME, CREATE_PASS) \
+ if (Name == NAME) { \
+ CGPM.addPass(createCGSCCToFunctionPassAdaptor( \
+ createFunctionToLoopPassAdaptor(CREATE_PASS, false, false))); \
+ return Error::success(); \
+ }
+#define LOOP_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS) \
+ if (checkParametrizedPassName(Name, NAME)) { \
+ auto Params = parsePassParameters(PARSER, Name, NAME); \
+ if (!Params) \
+ return Params.takeError(); \
+ CGPM.addPass( \
+ createCGSCCToFunctionPassAdaptor(createFunctionToLoopPassAdaptor( \
+ CREATE_PASS(Params.get()), false, false))); \
+ return Error::success(); \
+ }
+#include "PassRegistry.def"
+
+ for (auto &C : CGSCCPipelineParsingCallbacks)
+ if (C(Name, CGPM, InnerPipeline))
+ return Error::success();
+ return make_error<StringError>(
+ formatv("unknown cgscc pass '{0}'", Name).str(),
+ inconvertibleErrorCode());
+}
+
+Error PassBuilder::parseFunctionPass(FunctionPassManager &FPM,
+ const PipelineElement &E) {
+ auto &Name = E.Name;
+ auto &InnerPipeline = E.InnerPipeline;
+
+ // First handle complex passes like the pass managers which carry pipelines.
+ if (!InnerPipeline.empty()) {
+ if (Name == "function") {
+ FunctionPassManager NestedFPM;
+ if (auto Err = parseFunctionPassPipeline(NestedFPM, InnerPipeline))
+ return Err;
+ // Add the nested pass manager with the appropriate adaptor.
+ FPM.addPass(std::move(NestedFPM));
+ return Error::success();
+ }
+ if (Name == "loop" || Name == "loop-mssa") {
+ LoopPassManager LPM;
+ if (auto Err = parseLoopPassPipeline(LPM, InnerPipeline))
+ return Err;
+ // Add the nested pass manager with the appropriate adaptor.
+ bool UseMemorySSA = (Name == "loop-mssa");
+ bool UseBFI = llvm::any_of(InnerPipeline, [](auto Pipeline) {
+ return Pipeline.Name.contains("simple-loop-unswitch");
+ });
+ bool UseBPI = llvm::any_of(InnerPipeline, [](auto Pipeline) {
+ return Pipeline.Name == "loop-predication";
+ });
+ FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM), UseMemorySSA,
+ UseBFI, UseBPI));
+ return Error::success();
+ }
+ if (auto Count = parseRepeatPassName(Name)) {
+ FunctionPassManager NestedFPM;
+ if (auto Err = parseFunctionPassPipeline(NestedFPM, InnerPipeline))
+ return Err;
+ FPM.addPass(createRepeatedPass(*Count, std::move(NestedFPM)));
+ return Error::success();
+ }
+
+ for (auto &C : FunctionPipelineParsingCallbacks)
+ if (C(Name, FPM, InnerPipeline))
+ return Error::success();
+
+ // Normal passes can't have pipelines.
+ return make_error<StringError>(
+ formatv("invalid use of '{0}' pass as function pipeline", Name).str(),
+ inconvertibleErrorCode());
+ }
+
+// Now expand the basic registered passes from the .inc file.
+#define FUNCTION_PASS(NAME, CREATE_PASS) \
+ if (Name == NAME) { \
+ FPM.addPass(CREATE_PASS); \
+ return Error::success(); \
+ }
+#define FUNCTION_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS) \
+ if (checkParametrizedPassName(Name, NAME)) { \
+ auto Params = parsePassParameters(PARSER, Name, NAME); \
+ if (!Params) \
+ return Params.takeError(); \
+ FPM.addPass(CREATE_PASS(Params.get())); \
+ return Error::success(); \
+ }
+#define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \
+ if (Name == "require<" NAME ">") { \
+ FPM.addPass( \
+ RequireAnalysisPass< \
+ std::remove_reference_t<decltype(CREATE_PASS)>, Function>()); \
+ return Error::success(); \
+ } \
+ if (Name == "invalidate<" NAME ">") { \
+ FPM.addPass(InvalidateAnalysisPass< \
+ std::remove_reference_t<decltype(CREATE_PASS)>>()); \
+ return Error::success(); \
+ }
+// FIXME: UseMemorySSA is set to false. Maybe we could do things like:
+// bool UseMemorySSA = !("canon-freeze" || "loop-predication" ||
+// "guard-widening");
+// The risk is that it may become obsolete if we're not careful.
+#define LOOPNEST_PASS(NAME, CREATE_PASS) \
+ if (Name == NAME) { \
+ FPM.addPass(createFunctionToLoopPassAdaptor(CREATE_PASS, false, false)); \
+ return Error::success(); \
+ }
+#define LOOP_PASS(NAME, CREATE_PASS) \
+ if (Name == NAME) { \
+ FPM.addPass(createFunctionToLoopPassAdaptor(CREATE_PASS, false, false)); \
+ return Error::success(); \
+ }
+#define LOOP_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS) \
+ if (checkParametrizedPassName(Name, NAME)) { \
+ auto Params = parsePassParameters(PARSER, Name, NAME); \
+ if (!Params) \
+ return Params.takeError(); \
+ FPM.addPass(createFunctionToLoopPassAdaptor(CREATE_PASS(Params.get()), \
+ false, false)); \
+ return Error::success(); \
+ }
+#include "PassRegistry.def"
+
+ for (auto &C : FunctionPipelineParsingCallbacks)
+ if (C(Name, FPM, InnerPipeline))
+ return Error::success();
+ return make_error<StringError>(
+ formatv("unknown function pass '{0}'", Name).str(),
+ inconvertibleErrorCode());
+}
+
+Error PassBuilder::parseLoopPass(LoopPassManager &LPM,
+ const PipelineElement &E) {
+ StringRef Name = E.Name;
+ auto &InnerPipeline = E.InnerPipeline;
+
+ // First handle complex passes like the pass managers which carry pipelines.
+ if (!InnerPipeline.empty()) {
+ if (Name == "loop") {
+ LoopPassManager NestedLPM;
+ if (auto Err = parseLoopPassPipeline(NestedLPM, InnerPipeline))
+ return Err;
+ // Add the nested pass manager with the appropriate adaptor.
+ LPM.addPass(std::move(NestedLPM));
+ return Error::success();
+ }
+ if (auto Count = parseRepeatPassName(Name)) {
+ LoopPassManager NestedLPM;
+ if (auto Err = parseLoopPassPipeline(NestedLPM, InnerPipeline))
+ return Err;
+ LPM.addPass(createRepeatedPass(*Count, std::move(NestedLPM)));
+ return Error::success();
+ }
+
+ for (auto &C : LoopPipelineParsingCallbacks)
+ if (C(Name, LPM, InnerPipeline))
+ return Error::success();
+
+ // Normal passes can't have pipelines.
+ return make_error<StringError>(
+ formatv("invalid use of '{0}' pass as loop pipeline", Name).str(),
+ inconvertibleErrorCode());
+ }
+
+// Now expand the basic registered passes from the .inc file.
+#define LOOPNEST_PASS(NAME, CREATE_PASS) \
+ if (Name == NAME) { \
+ LPM.addPass(CREATE_PASS); \
+ return Error::success(); \
+ }
+#define LOOP_PASS(NAME, CREATE_PASS) \
+ if (Name == NAME) { \
+ LPM.addPass(CREATE_PASS); \
+ return Error::success(); \
+ }
+#define LOOP_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS) \
+ if (checkParametrizedPassName(Name, NAME)) { \
+ auto Params = parsePassParameters(PARSER, Name, NAME); \
+ if (!Params) \
+ return Params.takeError(); \
+ LPM.addPass(CREATE_PASS(Params.get())); \
+ return Error::success(); \
+ }
+#define LOOP_ANALYSIS(NAME, CREATE_PASS) \
+ if (Name == "require<" NAME ">") { \
+ LPM.addPass(RequireAnalysisPass< \
+ std::remove_reference_t<decltype(CREATE_PASS)>, Loop, \
+ LoopAnalysisManager, LoopStandardAnalysisResults &, \
+ LPMUpdater &>()); \
+ return Error::success(); \
+ } \
+ if (Name == "invalidate<" NAME ">") { \
+ LPM.addPass(InvalidateAnalysisPass< \
+ std::remove_reference_t<decltype(CREATE_PASS)>>()); \
+ return Error::success(); \
+ }
+#include "PassRegistry.def"
+
+ for (auto &C : LoopPipelineParsingCallbacks)
+ if (C(Name, LPM, InnerPipeline))
+ return Error::success();
+ return make_error<StringError>(formatv("unknown loop pass '{0}'", Name).str(),
+ inconvertibleErrorCode());
+}
+
+bool PassBuilder::parseAAPassName(AAManager &AA, StringRef Name) {
+#define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS) \
+ if (Name == NAME) { \
+ AA.registerModuleAnalysis< \
+ std::remove_reference_t<decltype(CREATE_PASS)>>(); \
+ return true; \
+ }
+#define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS) \
+ if (Name == NAME) { \
+ AA.registerFunctionAnalysis< \
+ std::remove_reference_t<decltype(CREATE_PASS)>>(); \
+ return true; \
+ }
+#include "PassRegistry.def"
+
+ for (auto &C : AAParsingCallbacks)
+ if (C(Name, AA))
+ return true;
+ return false;
+}
+
+Error PassBuilder::parseLoopPassPipeline(LoopPassManager &LPM,
+ ArrayRef<PipelineElement> Pipeline) {
+ for (const auto &Element : Pipeline) {
+ if (auto Err = parseLoopPass(LPM, Element))
+ return Err;
+ }
+ return Error::success();
+}
+
+Error PassBuilder::parseFunctionPassPipeline(
+ FunctionPassManager &FPM, ArrayRef<PipelineElement> Pipeline) {
+ for (const auto &Element : Pipeline) {
+ if (auto Err = parseFunctionPass(FPM, Element))
+ return Err;
+ }
+ return Error::success();
+}
+
+Error PassBuilder::parseCGSCCPassPipeline(CGSCCPassManager &CGPM,
+ ArrayRef<PipelineElement> Pipeline) {
+ for (const auto &Element : Pipeline) {
+ if (auto Err = parseCGSCCPass(CGPM, Element))
+ return Err;
+ }
+ return Error::success();
+}
+
+void PassBuilder::crossRegisterProxies(LoopAnalysisManager &LAM,
+ FunctionAnalysisManager &FAM,
+ CGSCCAnalysisManager &CGAM,
+ ModuleAnalysisManager &MAM) {
+ MAM.registerPass([&] { return FunctionAnalysisManagerModuleProxy(FAM); });
+ MAM.registerPass([&] { return CGSCCAnalysisManagerModuleProxy(CGAM); });
+ CGAM.registerPass([&] { return ModuleAnalysisManagerCGSCCProxy(MAM); });
+ FAM.registerPass([&] { return CGSCCAnalysisManagerFunctionProxy(CGAM); });
+ FAM.registerPass([&] { return ModuleAnalysisManagerFunctionProxy(MAM); });
+ FAM.registerPass([&] { return LoopAnalysisManagerFunctionProxy(LAM); });
+ LAM.registerPass([&] { return FunctionAnalysisManagerLoopProxy(FAM); });
+}
+
+Error PassBuilder::parseModulePassPipeline(ModulePassManager &MPM,
+ ArrayRef<PipelineElement> Pipeline) {
+ for (const auto &Element : Pipeline) {
+ if (auto Err = parseModulePass(MPM, Element))
+ return Err;
+ }
+ return Error::success();
+}
+
+// Primary pass pipeline description parsing routine for a \c ModulePassManager
+// FIXME: Should this routine accept a TargetMachine or require the caller to
+// pre-populate the analysis managers with target-specific stuff?
+Error PassBuilder::parsePassPipeline(ModulePassManager &MPM,
+ StringRef PipelineText) {
+ auto Pipeline = parsePipelineText(PipelineText);
+ if (!Pipeline || Pipeline->empty())
+ return make_error<StringError>(
+ formatv("invalid pipeline '{0}'", PipelineText).str(),
+ inconvertibleErrorCode());
+
+ // If the first name isn't at the module layer, wrap the pipeline up
+ // automatically.
+ StringRef FirstName = Pipeline->front().Name;
+
+ if (!isModulePassName(FirstName, ModulePipelineParsingCallbacks)) {
+ bool UseMemorySSA;
+ if (isCGSCCPassName(FirstName, CGSCCPipelineParsingCallbacks)) {
+ Pipeline = {{"cgscc", std::move(*Pipeline)}};
+ } else if (isFunctionPassName(FirstName,
+ FunctionPipelineParsingCallbacks)) {
+ Pipeline = {{"function", std::move(*Pipeline)}};
+ } else if (isLoopNestPassName(FirstName, LoopPipelineParsingCallbacks,
+ UseMemorySSA)) {
+ Pipeline = {{"function", {{UseMemorySSA ? "loop-mssa" : "loop",
+ std::move(*Pipeline)}}}};
+ } else if (isLoopPassName(FirstName, LoopPipelineParsingCallbacks,
+ UseMemorySSA)) {
+ Pipeline = {{"function", {{UseMemorySSA ? "loop-mssa" : "loop",
+ std::move(*Pipeline)}}}};
+ } else {
+ for (auto &C : TopLevelPipelineParsingCallbacks)
+ if (C(MPM, *Pipeline))
+ return Error::success();
+
+ // Unknown pass or pipeline name!
+ auto &InnerPipeline = Pipeline->front().InnerPipeline;
+ return make_error<StringError>(
+ formatv("unknown {0} name '{1}'",
+ (InnerPipeline.empty() ? "pass" : "pipeline"), FirstName)
+ .str(),
+ inconvertibleErrorCode());
+ }
+ }
+
+ if (auto Err = parseModulePassPipeline(MPM, *Pipeline))
+ return Err;
+ return Error::success();
+}
+
+// Primary pass pipeline description parsing routine for a \c CGSCCPassManager
+Error PassBuilder::parsePassPipeline(CGSCCPassManager &CGPM,
+ StringRef PipelineText) {
+ auto Pipeline = parsePipelineText(PipelineText);
+ if (!Pipeline || Pipeline->empty())
+ return make_error<StringError>(
+ formatv("invalid pipeline '{0}'", PipelineText).str(),
+ inconvertibleErrorCode());
+
+ StringRef FirstName = Pipeline->front().Name;
+ if (!isCGSCCPassName(FirstName, CGSCCPipelineParsingCallbacks))
+ return make_error<StringError>(
+ formatv("unknown cgscc pass '{0}' in pipeline '{1}'", FirstName,
+ PipelineText)
+ .str(),
+ inconvertibleErrorCode());
+
+ if (auto Err = parseCGSCCPassPipeline(CGPM, *Pipeline))
+ return Err;
+ return Error::success();
+}
+
+// Primary pass pipeline description parsing routine for a \c
+// FunctionPassManager
+Error PassBuilder::parsePassPipeline(FunctionPassManager &FPM,
+ StringRef PipelineText) {
+ auto Pipeline = parsePipelineText(PipelineText);
+ if (!Pipeline || Pipeline->empty())
+ return make_error<StringError>(
+ formatv("invalid pipeline '{0}'", PipelineText).str(),
+ inconvertibleErrorCode());
+
+ StringRef FirstName = Pipeline->front().Name;
+ if (!isFunctionPassName(FirstName, FunctionPipelineParsingCallbacks))
+ return make_error<StringError>(
+ formatv("unknown function pass '{0}' in pipeline '{1}'", FirstName,
+ PipelineText)
+ .str(),
+ inconvertibleErrorCode());
+
+ if (auto Err = parseFunctionPassPipeline(FPM, *Pipeline))
+ return Err;
+ return Error::success();
+}
+
+// Primary pass pipeline description parsing routine for a \c LoopPassManager
+Error PassBuilder::parsePassPipeline(LoopPassManager &CGPM,
+ StringRef PipelineText) {
+ auto Pipeline = parsePipelineText(PipelineText);
+ if (!Pipeline || Pipeline->empty())
+ return make_error<StringError>(
+ formatv("invalid pipeline '{0}'", PipelineText).str(),
+ inconvertibleErrorCode());
+
+ if (auto Err = parseLoopPassPipeline(CGPM, *Pipeline))
+ return Err;
+
+ return Error::success();
+}
+
+Error PassBuilder::parseAAPipeline(AAManager &AA, StringRef PipelineText) {
+ // If the pipeline just consists of the word 'default' just replace the AA
+ // manager with our default one.
+ if (PipelineText == "default") {
+ AA = buildDefaultAAPipeline();
+ return Error::success();
+ }
+
+ while (!PipelineText.empty()) {
+ StringRef Name;
+ std::tie(Name, PipelineText) = PipelineText.split(',');
+ if (!parseAAPassName(AA, Name))
+ return make_error<StringError>(
+ formatv("unknown alias analysis name '{0}'", Name).str(),
+ inconvertibleErrorCode());
+ }
+
+ return Error::success();
+}
+
+static void printPassName(StringRef PassName, raw_ostream &OS) {
+ OS << " " << PassName << "\n";
+}
+static void printPassName(StringRef PassName, StringRef Params,
+ raw_ostream &OS) {
+ OS << " " << PassName << "<" << Params << ">\n";
+}
+
+void PassBuilder::printPassNames(raw_ostream &OS) {
+ // TODO: print pass descriptions when they are available
+
+ OS << "Module passes:\n";
+#define MODULE_PASS(NAME, CREATE_PASS) printPassName(NAME, OS);
+#include "PassRegistry.def"
+
+ OS << "Module passes with params:\n";
+#define MODULE_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS) \
+ printPassName(NAME, PARAMS, OS);
+#include "PassRegistry.def"
+
+ OS << "Module analyses:\n";
+#define MODULE_ANALYSIS(NAME, CREATE_PASS) printPassName(NAME, OS);
+#include "PassRegistry.def"
+
+ OS << "Module alias analyses:\n";
+#define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS) printPassName(NAME, OS);
+#include "PassRegistry.def"
+
+ OS << "CGSCC passes:\n";
+#define CGSCC_PASS(NAME, CREATE_PASS) printPassName(NAME, OS);
+#include "PassRegistry.def"
+
+ OS << "CGSCC passes with params:\n";
+#define CGSCC_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS) \
+ printPassName(NAME, PARAMS, OS);
+#include "PassRegistry.def"
+
+ OS << "CGSCC analyses:\n";
+#define CGSCC_ANALYSIS(NAME, CREATE_PASS) printPassName(NAME, OS);
+#include "PassRegistry.def"
+
+ OS << "Function passes:\n";
+#define FUNCTION_PASS(NAME, CREATE_PASS) printPassName(NAME, OS);
+#include "PassRegistry.def"
+
+ OS << "Function passes with params:\n";
+#define FUNCTION_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS) \
+ printPassName(NAME, PARAMS, OS);
+#include "PassRegistry.def"
+
+ OS << "Function analyses:\n";
+#define FUNCTION_ANALYSIS(NAME, CREATE_PASS) printPassName(NAME, OS);
+#include "PassRegistry.def"
+
+ OS << "Function alias analyses:\n";
+#define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS) printPassName(NAME, OS);
+#include "PassRegistry.def"
+
+ OS << "LoopNest passes:\n";
+#define LOOPNEST_PASS(NAME, CREATE_PASS) printPassName(NAME, OS);
+#include "PassRegistry.def"
+
+ OS << "Loop passes:\n";
+#define LOOP_PASS(NAME, CREATE_PASS) printPassName(NAME, OS);
+#include "PassRegistry.def"
+
+ OS << "Loop passes with params:\n";
+#define LOOP_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS) \
+ printPassName(NAME, PARAMS, OS);
+#include "PassRegistry.def"
+
+ OS << "Loop analyses:\n";
+#define LOOP_ANALYSIS(NAME, CREATE_PASS) printPassName(NAME, OS);
+#include "PassRegistry.def"
+}
+
+void PassBuilder::registerParseTopLevelPipelineCallback(
+ const std::function<bool(ModulePassManager &, ArrayRef<PipelineElement>)>
+ &C) {
+ TopLevelPipelineParsingCallbacks.push_back(C);
+}
diff --git a/contrib/libs/llvm16/lib/Passes/PassBuilderBindings.cpp b/contrib/libs/llvm16/lib/Passes/PassBuilderBindings.cpp
new file mode 100644
index 00000000000..a87c0e6dc0a
--- /dev/null
+++ b/contrib/libs/llvm16/lib/Passes/PassBuilderBindings.cpp
@@ -0,0 +1,144 @@
+//===-------------- PassBuilder bindings for LLVM-C -----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+/// \file
+///
+/// This file defines the C bindings to the new pass manager
+///
+//===----------------------------------------------------------------------===//
+
+#include "llvm-c/Transforms/PassBuilder.h"
+#include "llvm/IR/Verifier.h"
+#include "llvm/Passes/PassBuilder.h"
+#include "llvm/Passes/StandardInstrumentations.h"
+#include "llvm/Support/CBindingWrapping.h"
+
+using namespace llvm;
+
+namespace llvm {
+/// Helper struct for holding a set of builder options for LLVMRunPasses. This
+/// structure is used to keep LLVMRunPasses backwards compatible with future
+/// versions in case we modify the options the new Pass Manager utilizes.
+class LLVMPassBuilderOptions {
+public:
+ explicit LLVMPassBuilderOptions(
+ bool DebugLogging = false, bool VerifyEach = false,
+ PipelineTuningOptions PTO = PipelineTuningOptions())
+ : DebugLogging(DebugLogging), VerifyEach(VerifyEach), PTO(PTO) {}
+
+ bool DebugLogging;
+ bool VerifyEach;
+ PipelineTuningOptions PTO;
+};
+} // namespace llvm
+
+static TargetMachine *unwrap(LLVMTargetMachineRef P) {
+ return reinterpret_cast<TargetMachine *>(P);
+}
+
+DEFINE_SIMPLE_CONVERSION_FUNCTIONS(LLVMPassBuilderOptions,
+ LLVMPassBuilderOptionsRef)
+
+LLVMErrorRef LLVMRunPasses(LLVMModuleRef M, const char *Passes,
+ LLVMTargetMachineRef TM,
+ LLVMPassBuilderOptionsRef Options) {
+ TargetMachine *Machine = unwrap(TM);
+ LLVMPassBuilderOptions *PassOpts = unwrap(Options);
+ bool Debug = PassOpts->DebugLogging;
+ bool VerifyEach = PassOpts->VerifyEach;
+
+ Module *Mod = unwrap(M);
+ PassInstrumentationCallbacks PIC;
+ PassBuilder PB(Machine, PassOpts->PTO, std::nullopt, &PIC);
+
+ LoopAnalysisManager LAM;
+ FunctionAnalysisManager FAM;
+ CGSCCAnalysisManager CGAM;
+ ModuleAnalysisManager MAM;
+ PB.registerLoopAnalyses(LAM);
+ PB.registerFunctionAnalyses(FAM);
+ PB.registerCGSCCAnalyses(CGAM);
+ PB.registerModuleAnalyses(MAM);
+ PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
+
+ StandardInstrumentations SI(Mod->getContext(), Debug, VerifyEach);
+ SI.registerCallbacks(PIC, &FAM);
+ ModulePassManager MPM;
+ if (VerifyEach) {
+ MPM.addPass(VerifierPass());
+ }
+ if (auto Err = PB.parsePassPipeline(MPM, Passes)) {
+ return wrap(std::move(Err));
+ }
+
+ MPM.run(*Mod, MAM);
+ return LLVMErrorSuccess;
+}
+
+LLVMPassBuilderOptionsRef LLVMCreatePassBuilderOptions() {
+ return wrap(new LLVMPassBuilderOptions());
+}
+
+void LLVMPassBuilderOptionsSetVerifyEach(LLVMPassBuilderOptionsRef Options,
+ LLVMBool VerifyEach) {
+ unwrap(Options)->VerifyEach = VerifyEach;
+}
+
+void LLVMPassBuilderOptionsSetDebugLogging(LLVMPassBuilderOptionsRef Options,
+ LLVMBool DebugLogging) {
+ unwrap(Options)->DebugLogging = DebugLogging;
+}
+
+void LLVMPassBuilderOptionsSetLoopInterleaving(
+ LLVMPassBuilderOptionsRef Options, LLVMBool LoopInterleaving) {
+ unwrap(Options)->PTO.LoopInterleaving = LoopInterleaving;
+}
+
+void LLVMPassBuilderOptionsSetLoopVectorization(
+ LLVMPassBuilderOptionsRef Options, LLVMBool LoopVectorization) {
+ unwrap(Options)->PTO.LoopVectorization = LoopVectorization;
+}
+
+void LLVMPassBuilderOptionsSetSLPVectorization(
+ LLVMPassBuilderOptionsRef Options, LLVMBool SLPVectorization) {
+ unwrap(Options)->PTO.SLPVectorization = SLPVectorization;
+}
+
+void LLVMPassBuilderOptionsSetLoopUnrolling(LLVMPassBuilderOptionsRef Options,
+ LLVMBool LoopUnrolling) {
+ unwrap(Options)->PTO.LoopUnrolling = LoopUnrolling;
+}
+
+void LLVMPassBuilderOptionsSetForgetAllSCEVInLoopUnroll(
+ LLVMPassBuilderOptionsRef Options, LLVMBool ForgetAllSCEVInLoopUnroll) {
+ unwrap(Options)->PTO.ForgetAllSCEVInLoopUnroll = ForgetAllSCEVInLoopUnroll;
+}
+
+void LLVMPassBuilderOptionsSetLicmMssaOptCap(LLVMPassBuilderOptionsRef Options,
+ unsigned LicmMssaOptCap) {
+ unwrap(Options)->PTO.LicmMssaOptCap = LicmMssaOptCap;
+}
+
+void LLVMPassBuilderOptionsSetLicmMssaNoAccForPromotionCap(
+ LLVMPassBuilderOptionsRef Options, unsigned LicmMssaNoAccForPromotionCap) {
+ unwrap(Options)->PTO.LicmMssaNoAccForPromotionCap =
+ LicmMssaNoAccForPromotionCap;
+}
+
+void LLVMPassBuilderOptionsSetCallGraphProfile(
+ LLVMPassBuilderOptionsRef Options, LLVMBool CallGraphProfile) {
+ unwrap(Options)->PTO.CallGraphProfile = CallGraphProfile;
+}
+
+void LLVMPassBuilderOptionsSetMergeFunctions(LLVMPassBuilderOptionsRef Options,
+ LLVMBool MergeFunctions) {
+ unwrap(Options)->PTO.MergeFunctions = MergeFunctions;
+}
+
+void LLVMDisposePassBuilderOptions(LLVMPassBuilderOptionsRef Options) {
+ delete unwrap(Options);
+}
diff --git a/contrib/libs/llvm16/lib/Passes/PassBuilderPipelines.cpp b/contrib/libs/llvm16/lib/Passes/PassBuilderPipelines.cpp
new file mode 100644
index 00000000000..eed29c25714
--- /dev/null
+++ b/contrib/libs/llvm16/lib/Passes/PassBuilderPipelines.cpp
@@ -0,0 +1,1998 @@
+//===- Construction of pass pipelines -------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+/// \file
+///
+/// This file provides the implementation of the PassBuilder based on our
+/// static pass registry as well as related functionality. It also provides
+/// helpers to aid in analyzing, debugging, and testing passes and pass
+/// pipelines.
+///
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Analysis/AliasAnalysis.h"
+#include "llvm/Analysis/BasicAliasAnalysis.h"
+#include "llvm/Analysis/CGSCCPassManager.h"
+#include "llvm/Analysis/GlobalsModRef.h"
+#include "llvm/Analysis/InlineAdvisor.h"
+#include "llvm/Analysis/OptimizationRemarkEmitter.h"
+#include "llvm/Analysis/ProfileSummaryInfo.h"
+#include "llvm/Analysis/ScopedNoAliasAA.h"
+#include "llvm/Analysis/TypeBasedAliasAnalysis.h"
+#include "llvm/IR/PassManager.h"
+#include "llvm/Passes/OptimizationLevel.h"
+#include "llvm/Passes/PassBuilder.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/PGOOptions.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h"
+#include "llvm/Transforms/Coroutines/CoroCleanup.h"
+#include "llvm/Transforms/Coroutines/CoroConditionalWrapper.h"
+#include "llvm/Transforms/Coroutines/CoroEarly.h"
+#include "llvm/Transforms/Coroutines/CoroElide.h"
+#include "llvm/Transforms/Coroutines/CoroSplit.h"
+#include "llvm/Transforms/IPO/AlwaysInliner.h"
+#include "llvm/Transforms/IPO/Annotation2Metadata.h"
+#include "llvm/Transforms/IPO/ArgumentPromotion.h"
+#include "llvm/Transforms/IPO/Attributor.h"
+#include "llvm/Transforms/IPO/CalledValuePropagation.h"
+#include "llvm/Transforms/IPO/ConstantMerge.h"
+#include "llvm/Transforms/IPO/CrossDSOCFI.h"
+#include "llvm/Transforms/IPO/DeadArgumentElimination.h"
+#include "llvm/Transforms/IPO/ElimAvailExtern.h"
+#include "llvm/Transforms/IPO/ForceFunctionAttrs.h"
+#include "llvm/Transforms/IPO/FunctionAttrs.h"
+#include "llvm/Transforms/IPO/GlobalDCE.h"
+#include "llvm/Transforms/IPO/GlobalOpt.h"
+#include "llvm/Transforms/IPO/GlobalSplit.h"
+#include "llvm/Transforms/IPO/HotColdSplitting.h"
+#include "llvm/Transforms/IPO/IROutliner.h"
+#include "llvm/Transforms/IPO/InferFunctionAttrs.h"
+#include "llvm/Transforms/IPO/Inliner.h"
+#include "llvm/Transforms/IPO/LowerTypeTests.h"
+#include "llvm/Transforms/IPO/MergeFunctions.h"
+#include "llvm/Transforms/IPO/ModuleInliner.h"
+#include "llvm/Transforms/IPO/OpenMPOpt.h"
+#include "llvm/Transforms/IPO/PartialInlining.h"
+#include "llvm/Transforms/IPO/SCCP.h"
+#include "llvm/Transforms/IPO/SampleProfile.h"
+#include "llvm/Transforms/IPO/SampleProfileProbe.h"
+#include "llvm/Transforms/IPO/SyntheticCountsPropagation.h"
+#include "llvm/Transforms/IPO/WholeProgramDevirt.h"
+#include "llvm/Transforms/InstCombine/InstCombine.h"
+#include "llvm/Transforms/Instrumentation/CGProfile.h"
+#include "llvm/Transforms/Instrumentation/ControlHeightReduction.h"
+#include "llvm/Transforms/Instrumentation/InstrOrderFile.h"
+#include "llvm/Transforms/Instrumentation/InstrProfiling.h"
+#include "llvm/Transforms/Instrumentation/MemProfiler.h"
+#include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
+#include "llvm/Transforms/Scalar/ADCE.h"
+#include "llvm/Transforms/Scalar/AlignmentFromAssumptions.h"
+#include "llvm/Transforms/Scalar/AnnotationRemarks.h"
+#include "llvm/Transforms/Scalar/BDCE.h"
+#include "llvm/Transforms/Scalar/CallSiteSplitting.h"
+#include "llvm/Transforms/Scalar/ConstraintElimination.h"
+#include "llvm/Transforms/Scalar/CorrelatedValuePropagation.h"
+#include "llvm/Transforms/Scalar/DFAJumpThreading.h"
+#include "llvm/Transforms/Scalar/DeadStoreElimination.h"
+#include "llvm/Transforms/Scalar/DivRemPairs.h"
+#include "llvm/Transforms/Scalar/EarlyCSE.h"
+#include "llvm/Transforms/Scalar/Float2Int.h"
+#include "llvm/Transforms/Scalar/GVN.h"
+#include "llvm/Transforms/Scalar/IndVarSimplify.h"
+#include "llvm/Transforms/Scalar/InstSimplifyPass.h"
+#include "llvm/Transforms/Scalar/JumpThreading.h"
+#include "llvm/Transforms/Scalar/LICM.h"
+#include "llvm/Transforms/Scalar/LoopDeletion.h"
+#include "llvm/Transforms/Scalar/LoopDistribute.h"
+#include "llvm/Transforms/Scalar/LoopFlatten.h"
+#include "llvm/Transforms/Scalar/LoopIdiomRecognize.h"
+#include "llvm/Transforms/Scalar/LoopInstSimplify.h"
+#include "llvm/Transforms/Scalar/LoopInterchange.h"
+#include "llvm/Transforms/Scalar/LoopLoadElimination.h"
+#include "llvm/Transforms/Scalar/LoopPassManager.h"
+#include "llvm/Transforms/Scalar/LoopRotation.h"
+#include "llvm/Transforms/Scalar/LoopSimplifyCFG.h"
+#include "llvm/Transforms/Scalar/LoopSink.h"
+#include "llvm/Transforms/Scalar/LoopUnrollAndJamPass.h"
+#include "llvm/Transforms/Scalar/LoopUnrollPass.h"
+#include "llvm/Transforms/Scalar/LowerConstantIntrinsics.h"
+#include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h"
+#include "llvm/Transforms/Scalar/LowerMatrixIntrinsics.h"
+#include "llvm/Transforms/Scalar/MemCpyOptimizer.h"
+#include "llvm/Transforms/Scalar/MergedLoadStoreMotion.h"
+#include "llvm/Transforms/Scalar/NewGVN.h"
+#include "llvm/Transforms/Scalar/Reassociate.h"
+#include "llvm/Transforms/Scalar/SCCP.h"
+#include "llvm/Transforms/Scalar/SROA.h"
+#include "llvm/Transforms/Scalar/SimpleLoopUnswitch.h"
+#include "llvm/Transforms/Scalar/SimplifyCFG.h"
+#include "llvm/Transforms/Scalar/SpeculativeExecution.h"
+#include "llvm/Transforms/Scalar/TailRecursionElimination.h"
+#include "llvm/Transforms/Scalar/WarnMissedTransforms.h"
+#include "llvm/Transforms/Utils/AddDiscriminators.h"
+#include "llvm/Transforms/Utils/AssumeBundleBuilder.h"
+#include "llvm/Transforms/Utils/CanonicalizeAliases.h"
+#include "llvm/Transforms/Utils/InjectTLIMappings.h"
+#include "llvm/Transforms/Utils/LibCallsShrinkWrap.h"
+#include "llvm/Transforms/Utils/Mem2Reg.h"
+#include "llvm/Transforms/Utils/NameAnonGlobals.h"
+#include "llvm/Transforms/Utils/RelLookupTableConverter.h"
+#include "llvm/Transforms/Utils/SimplifyCFGOptions.h"
+#include "llvm/Transforms/Vectorize/LoopVectorize.h"
+#include "llvm/Transforms/Vectorize/SLPVectorizer.h"
+#include "llvm/Transforms/Vectorize/VectorCombine.h"
+
+using namespace llvm;
+
+static cl::opt<InliningAdvisorMode> UseInlineAdvisor(
+ "enable-ml-inliner", cl::init(InliningAdvisorMode::Default), cl::Hidden,
+ cl::desc("Enable ML policy for inliner. Currently trained for -Oz only"),
+ cl::values(clEnumValN(InliningAdvisorMode::Default, "default",
+ "Heuristics-based inliner version"),
+ clEnumValN(InliningAdvisorMode::Development, "development",
+ "Use development mode (runtime-loadable model)"),
+ clEnumValN(InliningAdvisorMode::Release, "release",
+ "Use release mode (AOT-compiled model)")));
+
+static cl::opt<bool> EnableSyntheticCounts(
+ "enable-npm-synthetic-counts", cl::Hidden,
+ cl::desc("Run synthetic function entry count generation "
+ "pass"));
+
+/// Flag to enable inline deferral during PGO.
+static cl::opt<bool>
+ EnablePGOInlineDeferral("enable-npm-pgo-inline-deferral", cl::init(true),
+ cl::Hidden,
+ cl::desc("Enable inline deferral during PGO"));
+
+static cl::opt<bool> EnableMemProfiler("enable-mem-prof", cl::Hidden,
+ cl::desc("Enable memory profiler"));
+
+static cl::opt<bool> EnableModuleInliner("enable-module-inliner",
+ cl::init(false), cl::Hidden,
+ cl::desc("Enable module inliner"));
+
+static cl::opt<bool> PerformMandatoryInliningsFirst(
+ "mandatory-inlining-first", cl::init(true), cl::Hidden,
+ cl::desc("Perform mandatory inlinings module-wide, before performing "
+ "inlining"));
+
+static cl::opt<bool> EnableO3NonTrivialUnswitching(
+ "enable-npm-O3-nontrivial-unswitch", cl::init(true), cl::Hidden,
+ cl::desc("Enable non-trivial loop unswitching for -O3"));
+
+static cl::opt<bool> EnableEagerlyInvalidateAnalyses(
+ "eagerly-invalidate-analyses", cl::init(true), cl::Hidden,
+ cl::desc("Eagerly invalidate more analyses in default pipelines"));
+
+static cl::opt<bool> EnableNoRerunSimplificationPipeline(
+ "enable-no-rerun-simplification-pipeline", cl::init(true), cl::Hidden,
+ cl::desc(
+ "Prevent running the simplification pipeline on a function more "
+ "than once in the case that SCC mutations cause a function to be "
+ "visited multiple times as long as the function has not been changed"));
+
+static cl::opt<bool> EnableMergeFunctions(
+ "enable-merge-functions", cl::init(false), cl::Hidden,
+ cl::desc("Enable function merging as part of the optimization pipeline"));
+
+static cl::opt<bool> EnablePostPGOLoopRotation(
+ "enable-post-pgo-loop-rotation", cl::init(true), cl::Hidden,
+ cl::desc("Run the loop rotation transformation after PGO instrumentation"));
+
+static cl::opt<bool> EnableGlobalAnalyses(
+ "enable-global-analyses", cl::init(true), cl::Hidden,
+ cl::desc("Enable inter-procedural analyses"));
+
+static cl::opt<bool>
+ RunPartialInlining("enable-partial-inlining", cl::init(false), cl::Hidden,
+ cl::desc("Run Partial inlinining pass"));
+
+static cl::opt<bool> ExtraVectorizerPasses(
+ "extra-vectorizer-passes", cl::init(false), cl::Hidden,
+ cl::desc("Run cleanup optimization passes after vectorization"));
+
+static cl::opt<bool> RunNewGVN("enable-newgvn", cl::init(false), cl::Hidden,
+ cl::desc("Run the NewGVN pass"));
+
+static cl::opt<bool> EnableLoopInterchange(
+ "enable-loopinterchange", cl::init(false), cl::Hidden,
+ cl::desc("Enable the experimental LoopInterchange Pass"));
+
+static cl::opt<bool> EnableUnrollAndJam("enable-unroll-and-jam",
+ cl::init(false), cl::Hidden,
+ cl::desc("Enable Unroll And Jam Pass"));
+
+static cl::opt<bool> EnableLoopFlatten("enable-loop-flatten", cl::init(false),
+ cl::Hidden,
+ cl::desc("Enable the LoopFlatten Pass"));
+
+static cl::opt<bool>
+ EnableDFAJumpThreading("enable-dfa-jump-thread",
+ cl::desc("Enable DFA jump threading"),
+ cl::init(false), cl::Hidden);
+
+static cl::opt<bool>
+ EnableHotColdSplit("hot-cold-split",
+ cl::desc("Enable hot-cold splitting pass"));
+
+static cl::opt<bool> EnableIROutliner("ir-outliner", cl::init(false),
+ cl::Hidden,
+ cl::desc("Enable ir outliner pass"));
+
+static cl::opt<bool>
+ DisablePreInliner("disable-preinline", cl::init(false), cl::Hidden,
+ cl::desc("Disable pre-instrumentation inliner"));
+
+static cl::opt<int> PreInlineThreshold(
+ "preinline-threshold", cl::Hidden, cl::init(75),
+ cl::desc("Control the amount of inlining in pre-instrumentation inliner "
+ "(default = 75)"));
+
+static cl::opt<bool>
+ EnableGVNHoist("enable-gvn-hoist",
+ cl::desc("Enable the GVN hoisting pass (default = off)"));
+
+static cl::opt<bool>
+ EnableGVNSink("enable-gvn-sink",
+ cl::desc("Enable the GVN sinking pass (default = off)"));
+
+// This option is used in simplifying testing SampleFDO optimizations for
+// profile loading.
+static cl::opt<bool>
+ EnableCHR("enable-chr", cl::init(true), cl::Hidden,
+ cl::desc("Enable control height reduction optimization (CHR)"));
+
+static cl::opt<bool> FlattenedProfileUsed(
+ "flattened-profile-used", cl::init(false), cl::Hidden,
+ cl::desc("Indicate the sample profile being used is flattened, i.e., "
+ "no inline hierachy exists in the profile"));
+
+static cl::opt<bool> EnableOrderFileInstrumentation(
+ "enable-order-file-instrumentation", cl::init(false), cl::Hidden,
+ cl::desc("Enable order file instrumentation (default = off)"));
+
+static cl::opt<bool>
+ EnableMatrix("enable-matrix", cl::init(false), cl::Hidden,
+ cl::desc("Enable lowering of the matrix intrinsics"));
+
+static cl::opt<bool> EnableConstraintElimination(
+ "enable-constraint-elimination", cl::init(false), cl::Hidden,
+ cl::desc(
+ "Enable pass to eliminate conditions based on linear constraints"));
+
+static cl::opt<AttributorRunOption> AttributorRun(
+ "attributor-enable", cl::Hidden, cl::init(AttributorRunOption::NONE),
+ cl::desc("Enable the attributor inter-procedural deduction pass"),
+ cl::values(clEnumValN(AttributorRunOption::ALL, "all",
+ "enable all attributor runs"),
+ clEnumValN(AttributorRunOption::MODULE, "module",
+ "enable module-wide attributor runs"),
+ clEnumValN(AttributorRunOption::CGSCC, "cgscc",
+ "enable call graph SCC attributor runs"),
+ clEnumValN(AttributorRunOption::NONE, "none",
+ "disable attributor runs")));
+
+PipelineTuningOptions::PipelineTuningOptions() {
+ LoopInterleaving = true;
+ LoopVectorization = true;
+ SLPVectorization = false;
+ LoopUnrolling = true;
+ ForgetAllSCEVInLoopUnroll = ForgetSCEVInLoopUnroll;
+ LicmMssaOptCap = SetLicmMssaOptCap;
+ LicmMssaNoAccForPromotionCap = SetLicmMssaNoAccForPromotionCap;
+ CallGraphProfile = true;
+ MergeFunctions = EnableMergeFunctions;
+ InlinerThreshold = -1;
+ EagerlyInvalidateAnalyses = EnableEagerlyInvalidateAnalyses;
+}
+
+namespace llvm {
+extern cl::opt<unsigned> MaxDevirtIterations;
+extern cl::opt<bool> EnableKnowledgeRetention;
+} // namespace llvm
+
+void PassBuilder::invokePeepholeEPCallbacks(FunctionPassManager &FPM,
+ OptimizationLevel Level) {
+ for (auto &C : PeepholeEPCallbacks)
+ C(FPM, Level);
+}
+
+// Helper to add AnnotationRemarksPass.
+static void addAnnotationRemarksPass(ModulePassManager &MPM) {
+ MPM.addPass(createModuleToFunctionPassAdaptor(AnnotationRemarksPass()));
+}
+
+// Helper to check if the current compilation phase is preparing for LTO
+static bool isLTOPreLink(ThinOrFullLTOPhase Phase) {
+ return Phase == ThinOrFullLTOPhase::ThinLTOPreLink ||
+ Phase == ThinOrFullLTOPhase::FullLTOPreLink;
+}
+
+// TODO: Investigate the cost/benefit of tail call elimination on debugging.
+FunctionPassManager
+PassBuilder::buildO1FunctionSimplificationPipeline(OptimizationLevel Level,
+ ThinOrFullLTOPhase Phase) {
+
+ FunctionPassManager FPM;
+
+ // Form SSA out of local memory accesses after breaking apart aggregates into
+ // scalars.
+ FPM.addPass(SROAPass(SROAOptions::ModifyCFG));
+
+ // Catch trivial redundancies
+ FPM.addPass(EarlyCSEPass(true /* Enable mem-ssa. */));
+
+ // Hoisting of scalars and load expressions.
+ FPM.addPass(
+ SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(true)));
+ FPM.addPass(InstCombinePass());
+
+ FPM.addPass(LibCallsShrinkWrapPass());
+
+ invokePeepholeEPCallbacks(FPM, Level);
+
+ FPM.addPass(
+ SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(true)));
+
+ // Form canonically associated expression trees, and simplify the trees using
+ // basic mathematical properties. For example, this will form (nearly)
+ // minimal multiplication trees.
+ FPM.addPass(ReassociatePass());
+
+ // Add the primary loop simplification pipeline.
+ // FIXME: Currently this is split into two loop pass pipelines because we run
+ // some function passes in between them. These can and should be removed
+ // and/or replaced by scheduling the loop pass equivalents in the correct
+ // positions. But those equivalent passes aren't powerful enough yet.
+ // Specifically, `SimplifyCFGPass` and `InstCombinePass` are currently still
+ // used. We have `LoopSimplifyCFGPass` which isn't yet powerful enough yet to
+ // fully replace `SimplifyCFGPass`, and the closest to the other we have is
+ // `LoopInstSimplify`.
+ LoopPassManager LPM1, LPM2;
+
+ // Simplify the loop body. We do this initially to clean up after other loop
+ // passes run, either when iterating on a loop or on inner loops with
+ // implications on the outer loop.
+ LPM1.addPass(LoopInstSimplifyPass());
+ LPM1.addPass(LoopSimplifyCFGPass());
+
+ // Try to remove as much code from the loop header as possible,
+ // to reduce amount of IR that will have to be duplicated. However,
+ // do not perform speculative hoisting the first time as LICM
+ // will destroy metadata that may not need to be destroyed if run
+ // after loop rotation.
+ // TODO: Investigate promotion cap for O1.
+ LPM1.addPass(LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
+ /*AllowSpeculation=*/false));
+
+ LPM1.addPass(LoopRotatePass(/* Disable header duplication */ true,
+ isLTOPreLink(Phase)));
+ // TODO: Investigate promotion cap for O1.
+ LPM1.addPass(LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
+ /*AllowSpeculation=*/true));
+ LPM1.addPass(SimpleLoopUnswitchPass());
+ if (EnableLoopFlatten)
+ LPM1.addPass(LoopFlattenPass());
+
+ LPM2.addPass(LoopIdiomRecognizePass());
+ LPM2.addPass(IndVarSimplifyPass());
+
+ for (auto &C : LateLoopOptimizationsEPCallbacks)
+ C(LPM2, Level);
+
+ LPM2.addPass(LoopDeletionPass());
+
+ if (EnableLoopInterchange)
+ LPM2.addPass(LoopInterchangePass());
+
+ // Do not enable unrolling in PreLinkThinLTO phase during sample PGO
+ // because it changes IR to makes profile annotation in back compile
+ // inaccurate. The normal unroller doesn't pay attention to forced full unroll
+ // attributes so we need to make sure and allow the full unroll pass to pay
+ // attention to it.
+ if (Phase != ThinOrFullLTOPhase::ThinLTOPreLink || !PGOOpt ||
+ PGOOpt->Action != PGOOptions::SampleUse)
+ LPM2.addPass(LoopFullUnrollPass(Level.getSpeedupLevel(),
+ /* OnlyWhenForced= */ !PTO.LoopUnrolling,
+ PTO.ForgetAllSCEVInLoopUnroll));
+
+ for (auto &C : LoopOptimizerEndEPCallbacks)
+ C(LPM2, Level);
+
+ // We provide the opt remark emitter pass for LICM to use. We only need to do
+ // this once as it is immutable.
+ FPM.addPass(
+ RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>());
+ FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM1),
+ /*UseMemorySSA=*/true,
+ /*UseBlockFrequencyInfo=*/true));
+ FPM.addPass(
+ SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(true)));
+ FPM.addPass(InstCombinePass());
+ // The loop passes in LPM2 (LoopFullUnrollPass) do not preserve MemorySSA.
+ // *All* loop passes must preserve it, in order to be able to use it.
+ FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM2),
+ /*UseMemorySSA=*/false,
+ /*UseBlockFrequencyInfo=*/false));
+
+ // Delete small array after loop unroll.
+ FPM.addPass(SROAPass(SROAOptions::ModifyCFG));
+
+ // Specially optimize memory movement as it doesn't look like dataflow in SSA.
+ FPM.addPass(MemCpyOptPass());
+
+ // Sparse conditional constant propagation.
+ // FIXME: It isn't clear why we do this *after* loop passes rather than
+ // before...
+ FPM.addPass(SCCPPass());
+
+ // Delete dead bit computations (instcombine runs after to fold away the dead
+ // computations, and then ADCE will run later to exploit any new DCE
+ // opportunities that creates).
+ FPM.addPass(BDCEPass());
+
+ // Run instcombine after redundancy and dead bit elimination to exploit
+ // opportunities opened up by them.
+ FPM.addPass(InstCombinePass());
+ invokePeepholeEPCallbacks(FPM, Level);
+
+ FPM.addPass(CoroElidePass());
+
+ for (auto &C : ScalarOptimizerLateEPCallbacks)
+ C(FPM, Level);
+
+ // Finally, do an expensive DCE pass to catch all the dead code exposed by
+ // the simplifications and basic cleanup after all the simplifications.
+ // TODO: Investigate if this is too expensive.
+ FPM.addPass(ADCEPass());
+ FPM.addPass(
+ SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(true)));
+ FPM.addPass(InstCombinePass());
+ invokePeepholeEPCallbacks(FPM, Level);
+
+ return FPM;
+}
+
+FunctionPassManager
+PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level,
+ ThinOrFullLTOPhase Phase) {
+ assert(Level != OptimizationLevel::O0 && "Must request optimizations!");
+
+ // The O1 pipeline has a separate pipeline creation function to simplify
+ // construction readability.
+ if (Level.getSpeedupLevel() == 1)
+ return buildO1FunctionSimplificationPipeline(Level, Phase);
+
+ FunctionPassManager FPM;
+
+ // Form SSA out of local memory accesses after breaking apart aggregates into
+ // scalars.
+ FPM.addPass(SROAPass(SROAOptions::ModifyCFG));
+
+ // Catch trivial redundancies
+ FPM.addPass(EarlyCSEPass(true /* Enable mem-ssa. */));
+ if (EnableKnowledgeRetention)
+ FPM.addPass(AssumeSimplifyPass());
+
+ // Hoisting of scalars and load expressions.
+ if (EnableGVNHoist)
+ FPM.addPass(GVNHoistPass());
+
+ // Global value numbering based sinking.
+ if (EnableGVNSink) {
+ FPM.addPass(GVNSinkPass());
+ FPM.addPass(
+ SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(true)));
+ }
+
+ // Speculative execution if the target has divergent branches; otherwise nop.
+ FPM.addPass(SpeculativeExecutionPass(/* OnlyIfDivergentTarget =*/true));
+
+ // Optimize based on known information about branches, and cleanup afterward.
+ FPM.addPass(JumpThreadingPass());
+ FPM.addPass(CorrelatedValuePropagationPass());
+
+ FPM.addPass(
+ SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(true)));
+ FPM.addPass(InstCombinePass());
+ if (Level == OptimizationLevel::O3)
+ FPM.addPass(AggressiveInstCombinePass());
+
+ if (EnableConstraintElimination)
+ FPM.addPass(ConstraintEliminationPass());
+
+ if (!Level.isOptimizingForSize())
+ FPM.addPass(LibCallsShrinkWrapPass());
+
+ invokePeepholeEPCallbacks(FPM, Level);
+
+ // For PGO use pipeline, try to optimize memory intrinsics such as memcpy
+ // using the size value profile. Don't perform this when optimizing for size.
+ if (PGOOpt && PGOOpt->Action == PGOOptions::IRUse &&
+ !Level.isOptimizingForSize())
+ FPM.addPass(PGOMemOPSizeOpt());
+
+ FPM.addPass(TailCallElimPass());
+ FPM.addPass(
+ SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(true)));
+
+ // Form canonically associated expression trees, and simplify the trees using
+ // basic mathematical properties. For example, this will form (nearly)
+ // minimal multiplication trees.
+ FPM.addPass(ReassociatePass());
+
+ // Add the primary loop simplification pipeline.
+ // FIXME: Currently this is split into two loop pass pipelines because we run
+ // some function passes in between them. These can and should be removed
+ // and/or replaced by scheduling the loop pass equivalents in the correct
+ // positions. But those equivalent passes aren't powerful enough yet.
+ // Specifically, `SimplifyCFGPass` and `InstCombinePass` are currently still
+ // used. We have `LoopSimplifyCFGPass` which isn't yet powerful enough yet to
+ // fully replace `SimplifyCFGPass`, and the closest to the other we have is
+ // `LoopInstSimplify`.
+ LoopPassManager LPM1, LPM2;
+
+ // Simplify the loop body. We do this initially to clean up after other loop
+ // passes run, either when iterating on a loop or on inner loops with
+ // implications on the outer loop.
+ LPM1.addPass(LoopInstSimplifyPass());
+ LPM1.addPass(LoopSimplifyCFGPass());
+
+ // Try to remove as much code from the loop header as possible,
+ // to reduce amount of IR that will have to be duplicated. However,
+ // do not perform speculative hoisting the first time as LICM
+ // will destroy metadata that may not need to be destroyed if run
+ // after loop rotation.
+ // TODO: Investigate promotion cap for O1.
+ LPM1.addPass(LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
+ /*AllowSpeculation=*/false));
+
+ // Disable header duplication in loop rotation at -Oz.
+ LPM1.addPass(
+ LoopRotatePass(Level != OptimizationLevel::Oz, isLTOPreLink(Phase)));
+ // TODO: Investigate promotion cap for O1.
+ LPM1.addPass(LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
+ /*AllowSpeculation=*/true));
+ LPM1.addPass(
+ SimpleLoopUnswitchPass(/* NonTrivial */ Level == OptimizationLevel::O3 &&
+ EnableO3NonTrivialUnswitching));
+ if (EnableLoopFlatten)
+ LPM1.addPass(LoopFlattenPass());
+
+ LPM2.addPass(LoopIdiomRecognizePass());
+ LPM2.addPass(IndVarSimplifyPass());
+
+ for (auto &C : LateLoopOptimizationsEPCallbacks)
+ C(LPM2, Level);
+
+ LPM2.addPass(LoopDeletionPass());
+
+ if (EnableLoopInterchange)
+ LPM2.addPass(LoopInterchangePass());
+
+ // Do not enable unrolling in PreLinkThinLTO phase during sample PGO
+ // because it changes IR to makes profile annotation in back compile
+ // inaccurate. The normal unroller doesn't pay attention to forced full unroll
+ // attributes so we need to make sure and allow the full unroll pass to pay
+ // attention to it.
+ if (Phase != ThinOrFullLTOPhase::ThinLTOPreLink || !PGOOpt ||
+ PGOOpt->Action != PGOOptions::SampleUse)
+ LPM2.addPass(LoopFullUnrollPass(Level.getSpeedupLevel(),
+ /* OnlyWhenForced= */ !PTO.LoopUnrolling,
+ PTO.ForgetAllSCEVInLoopUnroll));
+
+ for (auto &C : LoopOptimizerEndEPCallbacks)
+ C(LPM2, Level);
+
+ // We provide the opt remark emitter pass for LICM to use. We only need to do
+ // this once as it is immutable.
+ FPM.addPass(
+ RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>());
+ FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM1),
+ /*UseMemorySSA=*/true,
+ /*UseBlockFrequencyInfo=*/true));
+ FPM.addPass(
+ SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(true)));
+ FPM.addPass(InstCombinePass());
+ // The loop passes in LPM2 (LoopIdiomRecognizePass, IndVarSimplifyPass,
+ // LoopDeletionPass and LoopFullUnrollPass) do not preserve MemorySSA.
+ // *All* loop passes must preserve it, in order to be able to use it.
+ FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM2),
+ /*UseMemorySSA=*/false,
+ /*UseBlockFrequencyInfo=*/false));
+
+ // Delete small array after loop unroll.
+ FPM.addPass(SROAPass(SROAOptions::ModifyCFG));
+
+ // Try vectorization/scalarization transforms that are both improvements
+ // themselves and can allow further folds with GVN and InstCombine.
+ FPM.addPass(VectorCombinePass(/*TryEarlyFoldsOnly=*/true));
+
+ // Eliminate redundancies.
+ FPM.addPass(MergedLoadStoreMotionPass());
+ if (RunNewGVN)
+ FPM.addPass(NewGVNPass());
+ else
+ FPM.addPass(GVNPass());
+
+ // Sparse conditional constant propagation.
+ // FIXME: It isn't clear why we do this *after* loop passes rather than
+ // before...
+ FPM.addPass(SCCPPass());
+
+ // Delete dead bit computations (instcombine runs after to fold away the dead
+ // computations, and then ADCE will run later to exploit any new DCE
+ // opportunities that creates).
+ FPM.addPass(BDCEPass());
+
+ // Run instcombine after redundancy and dead bit elimination to exploit
+ // opportunities opened up by them.
+ FPM.addPass(InstCombinePass());
+ invokePeepholeEPCallbacks(FPM, Level);
+
+ // Re-consider control flow based optimizations after redundancy elimination,
+ // redo DCE, etc.
+ if (EnableDFAJumpThreading && Level.getSizeLevel() == 0)
+ FPM.addPass(DFAJumpThreadingPass());
+
+ FPM.addPass(JumpThreadingPass());
+ FPM.addPass(CorrelatedValuePropagationPass());
+
+ // Finally, do an expensive DCE pass to catch all the dead code exposed by
+ // the simplifications and basic cleanup after all the simplifications.
+ // TODO: Investigate if this is too expensive.
+ FPM.addPass(ADCEPass());
+
+ // Specially optimize memory movement as it doesn't look like dataflow in SSA.
+ FPM.addPass(MemCpyOptPass());
+
+ FPM.addPass(DSEPass());
+ FPM.addPass(createFunctionToLoopPassAdaptor(
+ LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
+ /*AllowSpeculation=*/true),
+ /*UseMemorySSA=*/true, /*UseBlockFrequencyInfo=*/true));
+
+ FPM.addPass(CoroElidePass());
+
+ for (auto &C : ScalarOptimizerLateEPCallbacks)
+ C(FPM, Level);
+
+ FPM.addPass(SimplifyCFGPass(SimplifyCFGOptions()
+ .convertSwitchRangeToICmp(true)
+ .hoistCommonInsts(true)
+ .sinkCommonInsts(true)));
+ FPM.addPass(InstCombinePass());
+ invokePeepholeEPCallbacks(FPM, Level);
+
+ // Don't add CHR pass for CSIRInstr build in PostLink as the profile
+ // is still the same as the PreLink compilation.
+ if (EnableCHR && Level == OptimizationLevel::O3 && PGOOpt &&
+ ((PGOOpt->Action == PGOOptions::IRUse &&
+ (Phase != ThinOrFullLTOPhase::ThinLTOPostLink ||
+ PGOOpt->CSAction != PGOOptions::CSIRInstr)) ||
+ PGOOpt->Action == PGOOptions::SampleUse))
+ FPM.addPass(ControlHeightReductionPass());
+
+ return FPM;
+}
+
+void PassBuilder::addRequiredLTOPreLinkPasses(ModulePassManager &MPM) {
+ MPM.addPass(CanonicalizeAliasesPass());
+ MPM.addPass(NameAnonGlobalPass());
+}
+
+void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM,
+ OptimizationLevel Level, bool RunProfileGen,
+ bool IsCS, std::string ProfileFile,
+ std::string ProfileRemappingFile,
+ ThinOrFullLTOPhase LTOPhase) {
+ assert(Level != OptimizationLevel::O0 && "Not expecting O0 here!");
+ if (!IsCS && !DisablePreInliner) {
+ InlineParams IP;
+
+ IP.DefaultThreshold = PreInlineThreshold;
+
+ // FIXME: The hint threshold has the same value used by the regular inliner
+ // when not optimzing for size. This should probably be lowered after
+ // performance testing.
+ // FIXME: this comment is cargo culted from the old pass manager, revisit).
+ IP.HintThreshold = Level.isOptimizingForSize() ? PreInlineThreshold : 325;
+ ModuleInlinerWrapperPass MIWP(
+ IP, /* MandatoryFirst */ true,
+ InlineContext{LTOPhase, InlinePass::EarlyInliner});
+ CGSCCPassManager &CGPipeline = MIWP.getPM();
+
+ FunctionPassManager FPM;
+ FPM.addPass(SROAPass(SROAOptions::ModifyCFG));
+ FPM.addPass(EarlyCSEPass()); // Catch trivial redundancies.
+ FPM.addPass(SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(
+ true))); // Merge & remove basic blocks.
+ FPM.addPass(InstCombinePass()); // Combine silly sequences.
+ invokePeepholeEPCallbacks(FPM, Level);
+
+ CGPipeline.addPass(createCGSCCToFunctionPassAdaptor(
+ std::move(FPM), PTO.EagerlyInvalidateAnalyses));
+
+ MPM.addPass(std::move(MIWP));
+
+ // Delete anything that is now dead to make sure that we don't instrument
+ // dead code. Instrumentation can end up keeping dead code around and
+ // dramatically increase code size.
+ MPM.addPass(GlobalDCEPass());
+ }
+
+ if (!RunProfileGen) {
+ assert(!ProfileFile.empty() && "Profile use expecting a profile file!");
+ MPM.addPass(PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS));
+ // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
+ // RequireAnalysisPass for PSI before subsequent non-module passes.
+ MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
+ return;
+ }
+
+ // Perform PGO instrumentation.
+ MPM.addPass(PGOInstrumentationGen(IsCS));
+
+ if (EnablePostPGOLoopRotation) {
+ // Disable header duplication in loop rotation at -Oz.
+ MPM.addPass(createModuleToFunctionPassAdaptor(
+ createFunctionToLoopPassAdaptor(
+ LoopRotatePass(Level != OptimizationLevel::Oz),
+ /*UseMemorySSA=*/false,
+ /*UseBlockFrequencyInfo=*/false),
+ PTO.EagerlyInvalidateAnalyses));
+ }
+
+ // Add the profile lowering pass.
+ InstrProfOptions Options;
+ if (!ProfileFile.empty())
+ Options.InstrProfileOutput = ProfileFile;
+ // Do counter promotion at Level greater than O0.
+ Options.DoCounterPromotion = true;
+ Options.UseBFIInPromotion = IsCS;
+ MPM.addPass(InstrProfiling(Options, IsCS));
+}
+
+void PassBuilder::addPGOInstrPassesForO0(ModulePassManager &MPM,
+ bool RunProfileGen, bool IsCS,
+ std::string ProfileFile,
+ std::string ProfileRemappingFile) {
+ if (!RunProfileGen) {
+ assert(!ProfileFile.empty() && "Profile use expecting a profile file!");
+ MPM.addPass(PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS));
+ // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
+ // RequireAnalysisPass for PSI before subsequent non-module passes.
+ MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
+ return;
+ }
+
+ // Perform PGO instrumentation.
+ MPM.addPass(PGOInstrumentationGen(IsCS));
+ // Add the profile lowering pass.
+ InstrProfOptions Options;
+ if (!ProfileFile.empty())
+ Options.InstrProfileOutput = ProfileFile;
+ // Do not do counter promotion at O0.
+ Options.DoCounterPromotion = false;
+ Options.UseBFIInPromotion = IsCS;
+ MPM.addPass(InstrProfiling(Options, IsCS));
+}
+
+static InlineParams getInlineParamsFromOptLevel(OptimizationLevel Level) {
+ return getInlineParams(Level.getSpeedupLevel(), Level.getSizeLevel());
+}
+
+ModuleInlinerWrapperPass
+PassBuilder::buildInlinerPipeline(OptimizationLevel Level,
+ ThinOrFullLTOPhase Phase) {
+ InlineParams IP;
+ if (PTO.InlinerThreshold == -1)
+ IP = getInlineParamsFromOptLevel(Level);
+ else
+ IP = getInlineParams(PTO.InlinerThreshold);
+ // For PreLinkThinLTO + SamplePGO, set hot-caller threshold to 0 to
+ // disable hot callsite inline (as much as possible [1]) because it makes
+ // profile annotation in the backend inaccurate.
+ //
+ // [1] Note the cost of a function could be below zero due to erased
+ // prologue / epilogue.
+ if (Phase == ThinOrFullLTOPhase::ThinLTOPreLink && PGOOpt &&
+ PGOOpt->Action == PGOOptions::SampleUse)
+ IP.HotCallSiteThreshold = 0;
+
+ if (PGOOpt)
+ IP.EnableDeferral = EnablePGOInlineDeferral;
+
+ ModuleInlinerWrapperPass MIWP(IP, PerformMandatoryInliningsFirst,
+ InlineContext{Phase, InlinePass::CGSCCInliner},
+ UseInlineAdvisor, MaxDevirtIterations);
+
+ // Require the GlobalsAA analysis for the module so we can query it within
+ // the CGSCC pipeline.
+ MIWP.addModulePass(RequireAnalysisPass<GlobalsAA, Module>());
+ // Invalidate AAManager so it can be recreated and pick up the newly available
+ // GlobalsAA.
+ MIWP.addModulePass(
+ createModuleToFunctionPassAdaptor(InvalidateAnalysisPass<AAManager>()));
+
+ // Require the ProfileSummaryAnalysis for the module so we can query it within
+ // the inliner pass.
+ MIWP.addModulePass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
+
+ // Now begin the main postorder CGSCC pipeline.
+ // FIXME: The current CGSCC pipeline has its origins in the legacy pass
+ // manager and trying to emulate its precise behavior. Much of this doesn't
+ // make a lot of sense and we should revisit the core CGSCC structure.
+ CGSCCPassManager &MainCGPipeline = MIWP.getPM();
+
+ // Note: historically, the PruneEH pass was run first to deduce nounwind and
+ // generally clean up exception handling overhead. It isn't clear this is
+ // valuable as the inliner doesn't currently care whether it is inlining an
+ // invoke or a call.
+
+ if (AttributorRun & AttributorRunOption::CGSCC)
+ MainCGPipeline.addPass(AttributorCGSCCPass());
+
+ // Now deduce any function attributes based in the current code.
+ MainCGPipeline.addPass(PostOrderFunctionAttrsPass());
+
+ // When at O3 add argument promotion to the pass pipeline.
+ // FIXME: It isn't at all clear why this should be limited to O3.
+ if (Level == OptimizationLevel::O3)
+ MainCGPipeline.addPass(ArgumentPromotionPass());
+
+ // Try to perform OpenMP specific optimizations. This is a (quick!) no-op if
+ // there are no OpenMP runtime calls present in the module.
+ if (Level == OptimizationLevel::O2 || Level == OptimizationLevel::O3)
+ MainCGPipeline.addPass(OpenMPOptCGSCCPass());
+
+ for (auto &C : CGSCCOptimizerLateEPCallbacks)
+ C(MainCGPipeline, Level);
+
+ // Lastly, add the core function simplification pipeline nested inside the
+ // CGSCC walk.
+ MainCGPipeline.addPass(createCGSCCToFunctionPassAdaptor(
+ buildFunctionSimplificationPipeline(Level, Phase),
+ PTO.EagerlyInvalidateAnalyses, EnableNoRerunSimplificationPipeline));
+
+ MainCGPipeline.addPass(CoroSplitPass(Level != OptimizationLevel::O0));
+
+ if (EnableNoRerunSimplificationPipeline)
+ MIWP.addLateModulePass(createModuleToFunctionPassAdaptor(
+ InvalidateAnalysisPass<ShouldNotRunFunctionPassesAnalysis>()));
+
+ return MIWP;
+}
+
+ModulePassManager
+PassBuilder::buildModuleInlinerPipeline(OptimizationLevel Level,
+ ThinOrFullLTOPhase Phase) {
+ ModulePassManager MPM;
+
+ InlineParams IP = getInlineParamsFromOptLevel(Level);
+ // For PreLinkThinLTO + SamplePGO, set hot-caller threshold to 0 to
+ // disable hot callsite inline (as much as possible [1]) because it makes
+ // profile annotation in the backend inaccurate.
+ //
+ // [1] Note the cost of a function could be below zero due to erased
+ // prologue / epilogue.
+ if (Phase == ThinOrFullLTOPhase::ThinLTOPreLink && PGOOpt &&
+ PGOOpt->Action == PGOOptions::SampleUse)
+ IP.HotCallSiteThreshold = 0;
+
+ if (PGOOpt)
+ IP.EnableDeferral = EnablePGOInlineDeferral;
+
+ // The inline deferral logic is used to avoid losing some
+ // inlining chance in future. It is helpful in SCC inliner, in which
+ // inlining is processed in bottom-up order.
+ // While in module inliner, the inlining order is a priority-based order
+ // by default. The inline deferral is unnecessary there. So we disable the
+ // inline deferral logic in module inliner.
+ IP.EnableDeferral = false;
+
+ MPM.addPass(ModuleInlinerPass(IP, UseInlineAdvisor, Phase));
+
+ MPM.addPass(createModuleToFunctionPassAdaptor(
+ buildFunctionSimplificationPipeline(Level, Phase),
+ PTO.EagerlyInvalidateAnalyses));
+
+ MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
+ CoroSplitPass(Level != OptimizationLevel::O0)));
+
+ return MPM;
+}
+
+ModulePassManager
+PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level,
+ ThinOrFullLTOPhase Phase) {
+ ModulePassManager MPM;
+
+ // Place pseudo probe instrumentation as the first pass of the pipeline to
+ // minimize the impact of optimization changes.
+ if (PGOOpt && PGOOpt->PseudoProbeForProfiling &&
+ Phase != ThinOrFullLTOPhase::ThinLTOPostLink)
+ MPM.addPass(SampleProfileProbePass(TM));
+
+ bool HasSampleProfile = PGOOpt && (PGOOpt->Action == PGOOptions::SampleUse);
+
+ // In ThinLTO mode, when flattened profile is used, all the available
+ // profile information will be annotated in PreLink phase so there is
+ // no need to load the profile again in PostLink.
+ bool LoadSampleProfile =
+ HasSampleProfile &&
+ !(FlattenedProfileUsed && Phase == ThinOrFullLTOPhase::ThinLTOPostLink);
+
+ // During the ThinLTO backend phase we perform early indirect call promotion
+ // here, before globalopt. Otherwise imported available_externally functions
+ // look unreferenced and are removed. If we are going to load the sample
+ // profile then defer until later.
+ // TODO: See if we can move later and consolidate with the location where
+ // we perform ICP when we are loading a sample profile.
+ // TODO: We pass HasSampleProfile (whether there was a sample profile file
+ // passed to the compile) to the SamplePGO flag of ICP. This is used to
+ // determine whether the new direct calls are annotated with prof metadata.
+ // Ideally this should be determined from whether the IR is annotated with
+ // sample profile, and not whether the a sample profile was provided on the
+ // command line. E.g. for flattened profiles where we will not be reloading
+ // the sample profile in the ThinLTO backend, we ideally shouldn't have to
+ // provide the sample profile file.
+ if (Phase == ThinOrFullLTOPhase::ThinLTOPostLink && !LoadSampleProfile)
+ MPM.addPass(PGOIndirectCallPromotion(true /* InLTO */, HasSampleProfile));
+
+ // Do basic inference of function attributes from known properties of system
+ // libraries and other oracles.
+ MPM.addPass(InferFunctionAttrsPass());
+ MPM.addPass(CoroEarlyPass());
+
+ // Create an early function pass manager to cleanup the output of the
+ // frontend.
+ FunctionPassManager EarlyFPM;
+ // Lower llvm.expect to metadata before attempting transforms.
+ // Compare/branch metadata may alter the behavior of passes like SimplifyCFG.
+ EarlyFPM.addPass(LowerExpectIntrinsicPass());
+ EarlyFPM.addPass(SimplifyCFGPass());
+ EarlyFPM.addPass(SROAPass(SROAOptions::ModifyCFG));
+ EarlyFPM.addPass(EarlyCSEPass());
+ if (Level == OptimizationLevel::O3)
+ EarlyFPM.addPass(CallSiteSplittingPass());
+
+ // In SamplePGO ThinLTO backend, we need instcombine before profile annotation
+ // to convert bitcast to direct calls so that they can be inlined during the
+ // profile annotation prepration step.
+ // More details about SamplePGO design can be found in:
+ // https://research.google.com/pubs/pub45290.html
+ // FIXME: revisit how SampleProfileLoad/Inliner/ICP is structured.
+ if (LoadSampleProfile)
+ EarlyFPM.addPass(InstCombinePass());
+ MPM.addPass(createModuleToFunctionPassAdaptor(std::move(EarlyFPM),
+ PTO.EagerlyInvalidateAnalyses));
+
+ if (LoadSampleProfile) {
+ // Annotate sample profile right after early FPM to ensure freshness of
+ // the debug info.
+ MPM.addPass(SampleProfileLoaderPass(PGOOpt->ProfileFile,
+ PGOOpt->ProfileRemappingFile, Phase));
+ // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
+ // RequireAnalysisPass for PSI before subsequent non-module passes.
+ MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
+ // Do not invoke ICP in the LTOPrelink phase as it makes it hard
+ // for the profile annotation to be accurate in the LTO backend.
+ if (Phase != ThinOrFullLTOPhase::ThinLTOPreLink &&
+ Phase != ThinOrFullLTOPhase::FullLTOPreLink)
+ // We perform early indirect call promotion here, before globalopt.
+ // This is important for the ThinLTO backend phase because otherwise
+ // imported available_externally functions look unreferenced and are
+ // removed.
+ MPM.addPass(
+ PGOIndirectCallPromotion(true /* IsInLTO */, true /* SamplePGO */));
+ }
+
+ // Try to perform OpenMP specific optimizations on the module. This is a
+ // (quick!) no-op if there are no OpenMP runtime calls present in the module.
+ if (Level != OptimizationLevel::O0)
+ MPM.addPass(OpenMPOptPass());
+
+ if (AttributorRun & AttributorRunOption::MODULE)
+ MPM.addPass(AttributorPass());
+
+ // Lower type metadata and the type.test intrinsic in the ThinLTO
+ // post link pipeline after ICP. This is to enable usage of the type
+ // tests in ICP sequences.
+ if (Phase == ThinOrFullLTOPhase::ThinLTOPostLink)
+ MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
+
+ for (auto &C : PipelineEarlySimplificationEPCallbacks)
+ C(MPM, Level);
+
+ // Interprocedural constant propagation now that basic cleanup has occurred
+ // and prior to optimizing globals.
+ // FIXME: This position in the pipeline hasn't been carefully considered in
+ // years, it should be re-analyzed.
+ MPM.addPass(IPSCCPPass(IPSCCPOptions(/*AllowFuncSpec=*/
+ Level != OptimizationLevel::Os &&
+ Level != OptimizationLevel::Oz)));
+
+ // Attach metadata to indirect call sites indicating the set of functions
+ // they may target at run-time. This should follow IPSCCP.
+ MPM.addPass(CalledValuePropagationPass());
+
+ // Optimize globals to try and fold them into constants.
+ MPM.addPass(GlobalOptPass());
+
+ // Promote any localized globals to SSA registers.
+ // FIXME: Should this instead by a run of SROA?
+ // FIXME: We should probably run instcombine and simplifycfg afterward to
+ // delete control flows that are dead once globals have been folded to
+ // constants.
+ MPM.addPass(createModuleToFunctionPassAdaptor(PromotePass()));
+
+ // Create a small function pass pipeline to cleanup after all the global
+ // optimizations.
+ FunctionPassManager GlobalCleanupPM;
+ GlobalCleanupPM.addPass(InstCombinePass());
+ invokePeepholeEPCallbacks(GlobalCleanupPM, Level);
+
+ GlobalCleanupPM.addPass(
+ SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(true)));
+ MPM.addPass(createModuleToFunctionPassAdaptor(std::move(GlobalCleanupPM),
+ PTO.EagerlyInvalidateAnalyses));
+
+ // Add all the requested passes for instrumentation PGO, if requested.
+ if (PGOOpt && Phase != ThinOrFullLTOPhase::ThinLTOPostLink &&
+ (PGOOpt->Action == PGOOptions::IRInstr ||
+ PGOOpt->Action == PGOOptions::IRUse)) {
+ addPGOInstrPasses(MPM, Level,
+ /* RunProfileGen */ PGOOpt->Action == PGOOptions::IRInstr,
+ /* IsCS */ false, PGOOpt->ProfileFile,
+ PGOOpt->ProfileRemappingFile, Phase);
+ MPM.addPass(PGOIndirectCallPromotion(false, false));
+ }
+ if (PGOOpt && Phase != ThinOrFullLTOPhase::ThinLTOPostLink &&
+ PGOOpt->CSAction == PGOOptions::CSIRInstr)
+ MPM.addPass(PGOInstrumentationGenCreateVar(PGOOpt->CSProfileGenFile));
+
+ // Synthesize function entry counts for non-PGO compilation.
+ if (EnableSyntheticCounts && !PGOOpt)
+ MPM.addPass(SyntheticCountsPropagation());
+
+ if (EnableModuleInliner)
+ MPM.addPass(buildModuleInlinerPipeline(Level, Phase));
+ else
+ MPM.addPass(buildInlinerPipeline(Level, Phase));
+
+ // Remove any dead arguments exposed by cleanups, constant folding globals,
+ // and argument promotion.
+ MPM.addPass(DeadArgumentEliminationPass());
+
+ MPM.addPass(CoroCleanupPass());
+
+ if (EnableMemProfiler && Phase != ThinOrFullLTOPhase::ThinLTOPreLink) {
+ MPM.addPass(createModuleToFunctionPassAdaptor(MemProfilerPass()));
+ MPM.addPass(ModuleMemProfilerPass());
+ }
+
+ return MPM;
+}
+
+/// TODO: Should LTO cause any differences to this set of passes?
+void PassBuilder::addVectorPasses(OptimizationLevel Level,
+ FunctionPassManager &FPM, bool IsFullLTO) {
+ FPM.addPass(LoopVectorizePass(
+ LoopVectorizeOptions(!PTO.LoopInterleaving, !PTO.LoopVectorization)));
+
+ if (IsFullLTO) {
+ // The vectorizer may have significantly shortened a loop body; unroll
+ // again. Unroll small loops to hide loop backedge latency and saturate any
+ // parallel execution resources of an out-of-order processor. We also then
+ // need to clean up redundancies and loop invariant code.
+ // FIXME: It would be really good to use a loop-integrated instruction
+ // combiner for cleanup here so that the unrolling and LICM can be pipelined
+ // across the loop nests.
+ // We do UnrollAndJam in a separate LPM to ensure it happens before unroll
+ if (EnableUnrollAndJam && PTO.LoopUnrolling)
+ FPM.addPass(createFunctionToLoopPassAdaptor(
+ LoopUnrollAndJamPass(Level.getSpeedupLevel())));
+ FPM.addPass(LoopUnrollPass(LoopUnrollOptions(
+ Level.getSpeedupLevel(), /*OnlyWhenForced=*/!PTO.LoopUnrolling,
+ PTO.ForgetAllSCEVInLoopUnroll)));
+ FPM.addPass(WarnMissedTransformationsPass());
+ // Now that we are done with loop unrolling, be it either by LoopVectorizer,
+ // or LoopUnroll passes, some variable-offset GEP's into alloca's could have
+ // become constant-offset, thus enabling SROA and alloca promotion. Do so.
+ // NOTE: we are very late in the pipeline, and we don't have any LICM
+ // or SimplifyCFG passes scheduled after us, that would cleanup
+ // the CFG mess this may created if allowed to modify CFG, so forbid that.
+ FPM.addPass(SROAPass(SROAOptions::PreserveCFG));
+ }
+
+ if (!IsFullLTO) {
+ // Eliminate loads by forwarding stores from the previous iteration to loads
+ // of the current iteration.
+ FPM.addPass(LoopLoadEliminationPass());
+ }
+ // Cleanup after the loop optimization passes.
+ FPM.addPass(InstCombinePass());
+
+ if (Level.getSpeedupLevel() > 1 && ExtraVectorizerPasses) {
+ ExtraVectorPassManager ExtraPasses;
+ // At higher optimization levels, try to clean up any runtime overlap and
+ // alignment checks inserted by the vectorizer. We want to track correlated
+ // runtime checks for two inner loops in the same outer loop, fold any
+ // common computations, hoist loop-invariant aspects out of any outer loop,
+ // and unswitch the runtime checks if possible. Once hoisted, we may have
+ // dead (or speculatable) control flows or more combining opportunities.
+ ExtraPasses.addPass(EarlyCSEPass());
+ ExtraPasses.addPass(CorrelatedValuePropagationPass());
+ ExtraPasses.addPass(InstCombinePass());
+ LoopPassManager LPM;
+ LPM.addPass(LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
+ /*AllowSpeculation=*/true));
+ LPM.addPass(SimpleLoopUnswitchPass(/* NonTrivial */ Level ==
+ OptimizationLevel::O3));
+ ExtraPasses.addPass(
+ RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>());
+ ExtraPasses.addPass(
+ createFunctionToLoopPassAdaptor(std::move(LPM), /*UseMemorySSA=*/true,
+ /*UseBlockFrequencyInfo=*/true));
+ ExtraPasses.addPass(
+ SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(true)));
+ ExtraPasses.addPass(InstCombinePass());
+ FPM.addPass(std::move(ExtraPasses));
+ }
+
+ // Now that we've formed fast to execute loop structures, we do further
+ // optimizations. These are run afterward as they might block doing complex
+ // analyses and transforms such as what are needed for loop vectorization.
+
+ // Cleanup after loop vectorization, etc. Simplification passes like CVP and
+ // GVN, loop transforms, and others have already run, so it's now better to
+ // convert to more optimized IR using more aggressive simplify CFG options.
+ // The extra sinking transform can create larger basic blocks, so do this
+ // before SLP vectorization.
+ FPM.addPass(SimplifyCFGPass(SimplifyCFGOptions()
+ .forwardSwitchCondToPhi(true)
+ .convertSwitchRangeToICmp(true)
+ .convertSwitchToLookupTable(true)
+ .needCanonicalLoops(false)
+ .hoistCommonInsts(true)
+ .sinkCommonInsts(true)));
+
+ if (IsFullLTO) {
+ FPM.addPass(SCCPPass());
+ FPM.addPass(InstCombinePass());
+ FPM.addPass(BDCEPass());
+ }
+
+ // Optimize parallel scalar instruction chains into SIMD instructions.
+ if (PTO.SLPVectorization) {
+ FPM.addPass(SLPVectorizerPass());
+ if (Level.getSpeedupLevel() > 1 && ExtraVectorizerPasses) {
+ FPM.addPass(EarlyCSEPass());
+ }
+ }
+ // Enhance/cleanup vector code.
+ FPM.addPass(VectorCombinePass());
+
+ if (!IsFullLTO) {
+ FPM.addPass(InstCombinePass());
+ // Unroll small loops to hide loop backedge latency and saturate any
+ // parallel execution resources of an out-of-order processor. We also then
+ // need to clean up redundancies and loop invariant code.
+ // FIXME: It would be really good to use a loop-integrated instruction
+ // combiner for cleanup here so that the unrolling and LICM can be pipelined
+ // across the loop nests.
+ // We do UnrollAndJam in a separate LPM to ensure it happens before unroll
+ if (EnableUnrollAndJam && PTO.LoopUnrolling) {
+ FPM.addPass(createFunctionToLoopPassAdaptor(
+ LoopUnrollAndJamPass(Level.getSpeedupLevel())));
+ }
+ FPM.addPass(LoopUnrollPass(LoopUnrollOptions(
+ Level.getSpeedupLevel(), /*OnlyWhenForced=*/!PTO.LoopUnrolling,
+ PTO.ForgetAllSCEVInLoopUnroll)));
+ FPM.addPass(WarnMissedTransformationsPass());
+ // Now that we are done with loop unrolling, be it either by LoopVectorizer,
+ // or LoopUnroll passes, some variable-offset GEP's into alloca's could have
+ // become constant-offset, thus enabling SROA and alloca promotion. Do so.
+ // NOTE: we are very late in the pipeline, and we don't have any LICM
+ // or SimplifyCFG passes scheduled after us, that would cleanup
+ // the CFG mess this may created if allowed to modify CFG, so forbid that.
+ FPM.addPass(SROAPass(SROAOptions::PreserveCFG));
+ FPM.addPass(InstCombinePass());
+ FPM.addPass(
+ RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>());
+ FPM.addPass(createFunctionToLoopPassAdaptor(
+ LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
+ /*AllowSpeculation=*/true),
+ /*UseMemorySSA=*/true, /*UseBlockFrequencyInfo=*/true));
+ }
+
+ // Now that we've vectorized and unrolled loops, we may have more refined
+ // alignment information, try to re-derive it here.
+ FPM.addPass(AlignmentFromAssumptionsPass());
+
+ if (IsFullLTO)
+ FPM.addPass(InstCombinePass());
+}
+
+ModulePassManager
+PassBuilder::buildModuleOptimizationPipeline(OptimizationLevel Level,
+ ThinOrFullLTOPhase LTOPhase) {
+ const bool LTOPreLink = (LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink ||
+ LTOPhase == ThinOrFullLTOPhase::FullLTOPreLink);
+ ModulePassManager MPM;
+
+ // Optimize globals now that the module is fully simplified.
+ MPM.addPass(GlobalOptPass());
+ MPM.addPass(GlobalDCEPass());
+
+ // Run partial inlining pass to partially inline functions that have
+ // large bodies.
+ if (RunPartialInlining)
+ MPM.addPass(PartialInlinerPass());
+
+ // Remove avail extern fns and globals definitions since we aren't compiling
+ // an object file for later LTO. For LTO we want to preserve these so they
+ // are eligible for inlining at link-time. Note if they are unreferenced they
+ // will be removed by GlobalDCE later, so this only impacts referenced
+ // available externally globals. Eventually they will be suppressed during
+ // codegen, but eliminating here enables more opportunity for GlobalDCE as it
+ // may make globals referenced by available external functions dead and saves
+ // running remaining passes on the eliminated functions. These should be
+ // preserved during prelinking for link-time inlining decisions.
+ if (!LTOPreLink)
+ MPM.addPass(EliminateAvailableExternallyPass());
+
+ if (EnableOrderFileInstrumentation)
+ MPM.addPass(InstrOrderFilePass());
+
+ // Do RPO function attribute inference across the module to forward-propagate
+ // attributes where applicable.
+ // FIXME: Is this really an optimization rather than a canonicalization?
+ MPM.addPass(ReversePostOrderFunctionAttrsPass());
+
+ // Do a post inline PGO instrumentation and use pass. This is a context
+ // sensitive PGO pass. We don't want to do this in LTOPreLink phrase as
+ // cross-module inline has not been done yet. The context sensitive
+ // instrumentation is after all the inlines are done.
+ if (!LTOPreLink && PGOOpt) {
+ if (PGOOpt->CSAction == PGOOptions::CSIRInstr)
+ addPGOInstrPasses(MPM, Level, /* RunProfileGen */ true,
+ /* IsCS */ true, PGOOpt->CSProfileGenFile,
+ PGOOpt->ProfileRemappingFile, LTOPhase);
+ else if (PGOOpt->CSAction == PGOOptions::CSIRUse)
+ addPGOInstrPasses(MPM, Level, /* RunProfileGen */ false,
+ /* IsCS */ true, PGOOpt->ProfileFile,
+ PGOOpt->ProfileRemappingFile, LTOPhase);
+ }
+
+ // Re-compute GlobalsAA here prior to function passes. This is particularly
+ // useful as the above will have inlined, DCE'ed, and function-attr
+ // propagated everything. We should at this point have a reasonably minimal
+ // and richly annotated call graph. By computing aliasing and mod/ref
+ // information for all local globals here, the late loop passes and notably
+ // the vectorizer will be able to use them to help recognize vectorizable
+ // memory operations.
+ MPM.addPass(RecomputeGlobalsAAPass());
+
+ for (auto &C : OptimizerEarlyEPCallbacks)
+ C(MPM, Level);
+
+ FunctionPassManager OptimizePM;
+ OptimizePM.addPass(Float2IntPass());
+ OptimizePM.addPass(LowerConstantIntrinsicsPass());
+
+ if (EnableMatrix) {
+ OptimizePM.addPass(LowerMatrixIntrinsicsPass());
+ OptimizePM.addPass(EarlyCSEPass());
+ }
+
+ // FIXME: We need to run some loop optimizations to re-rotate loops after
+ // simplifycfg and others undo their rotation.
+
+ // Optimize the loop execution. These passes operate on entire loop nests
+ // rather than on each loop in an inside-out manner, and so they are actually
+ // function passes.
+
+ for (auto &C : VectorizerStartEPCallbacks)
+ C(OptimizePM, Level);
+
+ LoopPassManager LPM;
+ // First rotate loops that may have been un-rotated by prior passes.
+ // Disable header duplication at -Oz.
+ LPM.addPass(LoopRotatePass(Level != OptimizationLevel::Oz, LTOPreLink));
+ // Some loops may have become dead by now. Try to delete them.
+ // FIXME: see discussion in https://reviews.llvm.org/D112851,
+ // this may need to be revisited once we run GVN before loop deletion
+ // in the simplification pipeline.
+ LPM.addPass(LoopDeletionPass());
+ OptimizePM.addPass(createFunctionToLoopPassAdaptor(
+ std::move(LPM), /*UseMemorySSA=*/false, /*UseBlockFrequencyInfo=*/false));
+
+ // Distribute loops to allow partial vectorization. I.e. isolate dependences
+ // into separate loop that would otherwise inhibit vectorization. This is
+ // currently only performed for loops marked with the metadata
+ // llvm.loop.distribute=true or when -enable-loop-distribute is specified.
+ OptimizePM.addPass(LoopDistributePass());
+
+ // Populates the VFABI attribute with the scalar-to-vector mappings
+ // from the TargetLibraryInfo.
+ OptimizePM.addPass(InjectTLIMappings());
+
+ addVectorPasses(Level, OptimizePM, /* IsFullLTO */ false);
+
+ // LoopSink pass sinks instructions hoisted by LICM, which serves as a
+ // canonicalization pass that enables other optimizations. As a result,
+ // LoopSink pass needs to be a very late IR pass to avoid undoing LICM
+ // result too early.
+ OptimizePM.addPass(LoopSinkPass());
+
+ // And finally clean up LCSSA form before generating code.
+ OptimizePM.addPass(InstSimplifyPass());
+
+ // This hoists/decomposes div/rem ops. It should run after other sink/hoist
+ // passes to avoid re-sinking, but before SimplifyCFG because it can allow
+ // flattening of blocks.
+ OptimizePM.addPass(DivRemPairsPass());
+
+ // Try to annotate calls that were created during optimization.
+ OptimizePM.addPass(TailCallElimPass());
+
+ // LoopSink (and other loop passes since the last simplifyCFG) might have
+ // resulted in single-entry-single-exit or empty blocks. Clean up the CFG.
+ OptimizePM.addPass(
+ SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(true)));
+
+ // Add the core optimizing pipeline.
+ MPM.addPass(createModuleToFunctionPassAdaptor(std::move(OptimizePM),
+ PTO.EagerlyInvalidateAnalyses));
+
+ for (auto &C : OptimizerLastEPCallbacks)
+ C(MPM, Level);
+
+ // Split out cold code. Splitting is done late to avoid hiding context from
+ // other optimizations and inadvertently regressing performance. The tradeoff
+ // is that this has a higher code size cost than splitting early.
+ if (EnableHotColdSplit && !LTOPreLink)
+ MPM.addPass(HotColdSplittingPass());
+
+ // Search the code for similar regions of code. If enough similar regions can
+ // be found where extracting the regions into their own function will decrease
+ // the size of the program, we extract the regions, a deduplicate the
+ // structurally similar regions.
+ if (EnableIROutliner)
+ MPM.addPass(IROutlinerPass());
+
+ // Merge functions if requested.
+ if (PTO.MergeFunctions)
+ MPM.addPass(MergeFunctionsPass());
+
+ // Now we need to do some global optimization transforms.
+ // FIXME: It would seem like these should come first in the optimization
+ // pipeline and maybe be the bottom of the canonicalization pipeline? Weird
+ // ordering here.
+ MPM.addPass(GlobalDCEPass());
+ MPM.addPass(ConstantMergePass());
+
+ if (PTO.CallGraphProfile && !LTOPreLink)
+ MPM.addPass(CGProfilePass());
+
+ // TODO: Relative look table converter pass caused an issue when full lto is
+ // enabled. See https://reviews.llvm.org/D94355 for more details.
+ // Until the issue fixed, disable this pass during pre-linking phase.
+ if (!LTOPreLink)
+ MPM.addPass(RelLookupTableConverterPass());
+
+ return MPM;
+}
+
+ModulePassManager
+PassBuilder::buildPerModuleDefaultPipeline(OptimizationLevel Level,
+ bool LTOPreLink) {
+ assert(Level != OptimizationLevel::O0 &&
+ "Must request optimizations for the default pipeline!");
+
+ ModulePassManager MPM;
+
+ // Convert @llvm.global.annotations to !annotation metadata.
+ MPM.addPass(Annotation2MetadataPass());
+
+ // Force any function attributes we want the rest of the pipeline to observe.
+ MPM.addPass(ForceFunctionAttrsPass());
+
+ // Apply module pipeline start EP callback.
+ for (auto &C : PipelineStartEPCallbacks)
+ C(MPM, Level);
+
+ if (PGOOpt && PGOOpt->DebugInfoForProfiling)
+ MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass()));
+
+ const ThinOrFullLTOPhase LTOPhase = LTOPreLink
+ ? ThinOrFullLTOPhase::FullLTOPreLink
+ : ThinOrFullLTOPhase::None;
+ // Add the core simplification pipeline.
+ MPM.addPass(buildModuleSimplificationPipeline(Level, LTOPhase));
+
+ // Now add the optimization pipeline.
+ MPM.addPass(buildModuleOptimizationPipeline(Level, LTOPhase));
+
+ if (PGOOpt && PGOOpt->PseudoProbeForProfiling &&
+ PGOOpt->Action == PGOOptions::SampleUse)
+ MPM.addPass(PseudoProbeUpdatePass());
+
+ // Emit annotation remarks.
+ addAnnotationRemarksPass(MPM);
+
+ if (LTOPreLink)
+ addRequiredLTOPreLinkPasses(MPM);
+
+ return MPM;
+}
+
+ModulePassManager
+PassBuilder::buildThinLTOPreLinkDefaultPipeline(OptimizationLevel Level) {
+ assert(Level != OptimizationLevel::O0 &&
+ "Must request optimizations for the default pipeline!");
+
+ ModulePassManager MPM;
+
+ // Convert @llvm.global.annotations to !annotation metadata.
+ MPM.addPass(Annotation2MetadataPass());
+
+ // Force any function attributes we want the rest of the pipeline to observe.
+ MPM.addPass(ForceFunctionAttrsPass());
+
+ if (PGOOpt && PGOOpt->DebugInfoForProfiling)
+ MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass()));
+
+ // Apply module pipeline start EP callback.
+ for (auto &C : PipelineStartEPCallbacks)
+ C(MPM, Level);
+
+ // If we are planning to perform ThinLTO later, we don't bloat the code with
+ // unrolling/vectorization/... now. Just simplify the module as much as we
+ // can.
+ MPM.addPass(buildModuleSimplificationPipeline(
+ Level, ThinOrFullLTOPhase::ThinLTOPreLink));
+
+ // Run partial inlining pass to partially inline functions that have
+ // large bodies.
+ // FIXME: It isn't clear whether this is really the right place to run this
+ // in ThinLTO. Because there is another canonicalization and simplification
+ // phase that will run after the thin link, running this here ends up with
+ // less information than will be available later and it may grow functions in
+ // ways that aren't beneficial.
+ if (RunPartialInlining)
+ MPM.addPass(PartialInlinerPass());
+
+ // Reduce the size of the IR as much as possible.
+ MPM.addPass(GlobalOptPass());
+
+ if (PGOOpt && PGOOpt->PseudoProbeForProfiling &&
+ PGOOpt->Action == PGOOptions::SampleUse)
+ MPM.addPass(PseudoProbeUpdatePass());
+
+ // Handle Optimizer{Early,Last}EPCallbacks added by clang on PreLink. Actual
+ // optimization is going to be done in PostLink stage, but clang can't add
+ // callbacks there in case of in-process ThinLTO called by linker.
+ for (auto &C : OptimizerEarlyEPCallbacks)
+ C(MPM, Level);
+ for (auto &C : OptimizerLastEPCallbacks)
+ C(MPM, Level);
+
+ // Emit annotation remarks.
+ addAnnotationRemarksPass(MPM);
+
+ addRequiredLTOPreLinkPasses(MPM);
+
+ return MPM;
+}
+
+ModulePassManager PassBuilder::buildThinLTODefaultPipeline(
+ OptimizationLevel Level, const ModuleSummaryIndex *ImportSummary) {
+ ModulePassManager MPM;
+
+ // Convert @llvm.global.annotations to !annotation metadata.
+ MPM.addPass(Annotation2MetadataPass());
+
+ if (ImportSummary) {
+ // These passes import type identifier resolutions for whole-program
+ // devirtualization and CFI. They must run early because other passes may
+ // disturb the specific instruction patterns that these passes look for,
+ // creating dependencies on resolutions that may not appear in the summary.
+ //
+ // For example, GVN may transform the pattern assume(type.test) appearing in
+ // two basic blocks into assume(phi(type.test, type.test)), which would
+ // transform a dependency on a WPD resolution into a dependency on a type
+ // identifier resolution for CFI.
+ //
+ // Also, WPD has access to more precise information than ICP and can
+ // devirtualize more effectively, so it should operate on the IR first.
+ //
+ // The WPD and LowerTypeTest passes need to run at -O0 to lower type
+ // metadata and intrinsics.
+ MPM.addPass(WholeProgramDevirtPass(nullptr, ImportSummary));
+ MPM.addPass(LowerTypeTestsPass(nullptr, ImportSummary));
+ }
+
+ if (Level == OptimizationLevel::O0) {
+ // Run a second time to clean up any type tests left behind by WPD for use
+ // in ICP.
+ MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
+ // Drop available_externally and unreferenced globals. This is necessary
+ // with ThinLTO in order to avoid leaving undefined references to dead
+ // globals in the object file.
+ MPM.addPass(EliminateAvailableExternallyPass());
+ MPM.addPass(GlobalDCEPass());
+ return MPM;
+ }
+
+ // Force any function attributes we want the rest of the pipeline to observe.
+ MPM.addPass(ForceFunctionAttrsPass());
+
+ // Add the core simplification pipeline.
+ MPM.addPass(buildModuleSimplificationPipeline(
+ Level, ThinOrFullLTOPhase::ThinLTOPostLink));
+
+ // Now add the optimization pipeline.
+ MPM.addPass(buildModuleOptimizationPipeline(
+ Level, ThinOrFullLTOPhase::ThinLTOPostLink));
+
+ // Emit annotation remarks.
+ addAnnotationRemarksPass(MPM);
+
+ return MPM;
+}
+
+ModulePassManager
+PassBuilder::buildLTOPreLinkDefaultPipeline(OptimizationLevel Level) {
+ assert(Level != OptimizationLevel::O0 &&
+ "Must request optimizations for the default pipeline!");
+ // FIXME: We should use a customized pre-link pipeline!
+ return buildPerModuleDefaultPipeline(Level,
+ /* LTOPreLink */ true);
+}
+
+ModulePassManager
+PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level,
+ ModuleSummaryIndex *ExportSummary) {
+ ModulePassManager MPM;
+
+ // Convert @llvm.global.annotations to !annotation metadata.
+ MPM.addPass(Annotation2MetadataPass());
+
+ for (auto &C : FullLinkTimeOptimizationEarlyEPCallbacks)
+ C(MPM, Level);
+
+ // Create a function that performs CFI checks for cross-DSO calls with targets
+ // in the current module.
+ MPM.addPass(CrossDSOCFIPass());
+
+ if (Level == OptimizationLevel::O0) {
+ // The WPD and LowerTypeTest passes need to run at -O0 to lower type
+ // metadata and intrinsics.
+ MPM.addPass(WholeProgramDevirtPass(ExportSummary, nullptr));
+ MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
+ // Run a second time to clean up any type tests left behind by WPD for use
+ // in ICP.
+ MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
+
+ for (auto &C : FullLinkTimeOptimizationLastEPCallbacks)
+ C(MPM, Level);
+
+ // Emit annotation remarks.
+ addAnnotationRemarksPass(MPM);
+
+ return MPM;
+ }
+
+ if (PGOOpt && PGOOpt->Action == PGOOptions::SampleUse) {
+ // Load sample profile before running the LTO optimization pipeline.
+ MPM.addPass(SampleProfileLoaderPass(PGOOpt->ProfileFile,
+ PGOOpt->ProfileRemappingFile,
+ ThinOrFullLTOPhase::FullLTOPostLink));
+ // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
+ // RequireAnalysisPass for PSI before subsequent non-module passes.
+ MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
+ }
+
+ // Try to run OpenMP optimizations, quick no-op if no OpenMP metadata present.
+ MPM.addPass(OpenMPOptPass(ThinOrFullLTOPhase::FullLTOPostLink));
+
+ // Remove unused virtual tables to improve the quality of code generated by
+ // whole-program devirtualization and bitset lowering.
+ MPM.addPass(GlobalDCEPass());
+
+ // Force any function attributes we want the rest of the pipeline to observe.
+ MPM.addPass(ForceFunctionAttrsPass());
+
+ // Do basic inference of function attributes from known properties of system
+ // libraries and other oracles.
+ MPM.addPass(InferFunctionAttrsPass());
+
+ if (Level.getSpeedupLevel() > 1) {
+ MPM.addPass(createModuleToFunctionPassAdaptor(
+ CallSiteSplittingPass(), PTO.EagerlyInvalidateAnalyses));
+
+ // Indirect call promotion. This should promote all the targets that are
+ // left by the earlier promotion pass that promotes intra-module targets.
+ // This two-step promotion is to save the compile time. For LTO, it should
+ // produce the same result as if we only do promotion here.
+ MPM.addPass(PGOIndirectCallPromotion(
+ true /* InLTO */, PGOOpt && PGOOpt->Action == PGOOptions::SampleUse));
+
+ // Propagate constants at call sites into the functions they call. This
+ // opens opportunities for globalopt (and inlining) by substituting function
+ // pointers passed as arguments to direct uses of functions.
+ MPM.addPass(IPSCCPPass(IPSCCPOptions(/*AllowFuncSpec=*/
+ Level != OptimizationLevel::Os &&
+ Level != OptimizationLevel::Oz)));
+
+ // Attach metadata to indirect call sites indicating the set of functions
+ // they may target at run-time. This should follow IPSCCP.
+ MPM.addPass(CalledValuePropagationPass());
+ }
+
+ // Now deduce any function attributes based in the current code.
+ MPM.addPass(
+ createModuleToPostOrderCGSCCPassAdaptor(PostOrderFunctionAttrsPass()));
+
+ // Do RPO function attribute inference across the module to forward-propagate
+ // attributes where applicable.
+ // FIXME: Is this really an optimization rather than a canonicalization?
+ MPM.addPass(ReversePostOrderFunctionAttrsPass());
+
+ // Use in-range annotations on GEP indices to split globals where beneficial.
+ MPM.addPass(GlobalSplitPass());
+
+ // Run whole program optimization of virtual call when the list of callees
+ // is fixed.
+ MPM.addPass(WholeProgramDevirtPass(ExportSummary, nullptr));
+
+ // Stop here at -O1.
+ if (Level == OptimizationLevel::O1) {
+ // The LowerTypeTestsPass needs to run to lower type metadata and the
+ // type.test intrinsics. The pass does nothing if CFI is disabled.
+ MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
+ // Run a second time to clean up any type tests left behind by WPD for use
+ // in ICP (which is performed earlier than this in the regular LTO
+ // pipeline).
+ MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
+
+ for (auto &C : FullLinkTimeOptimizationLastEPCallbacks)
+ C(MPM, Level);
+
+ // Emit annotation remarks.
+ addAnnotationRemarksPass(MPM);
+
+ return MPM;
+ }
+
+ // Optimize globals to try and fold them into constants.
+ MPM.addPass(GlobalOptPass());
+
+ // Promote any localized globals to SSA registers.
+ MPM.addPass(createModuleToFunctionPassAdaptor(PromotePass()));
+
+ // Linking modules together can lead to duplicate global constant, only
+ // keep one copy of each constant.
+ MPM.addPass(ConstantMergePass());
+
+ // Reduce the code after globalopt and ipsccp. Both can open up significant
+ // simplification opportunities, and both can propagate functions through
+ // function pointers. When this happens, we often have to resolve varargs
+ // calls, etc, so let instcombine do this.
+ FunctionPassManager PeepholeFPM;
+ PeepholeFPM.addPass(InstCombinePass());
+ if (Level == OptimizationLevel::O3)
+ PeepholeFPM.addPass(AggressiveInstCombinePass());
+ invokePeepholeEPCallbacks(PeepholeFPM, Level);
+
+ MPM.addPass(createModuleToFunctionPassAdaptor(std::move(PeepholeFPM),
+ PTO.EagerlyInvalidateAnalyses));
+
+ // Note: historically, the PruneEH pass was run first to deduce nounwind and
+ // generally clean up exception handling overhead. It isn't clear this is
+ // valuable as the inliner doesn't currently care whether it is inlining an
+ // invoke or a call.
+ // Run the inliner now.
+ MPM.addPass(ModuleInlinerWrapperPass(
+ getInlineParamsFromOptLevel(Level),
+ /* MandatoryFirst */ true,
+ InlineContext{ThinOrFullLTOPhase::FullLTOPostLink,
+ InlinePass::CGSCCInliner}));
+
+ // Optimize globals again after we ran the inliner.
+ MPM.addPass(GlobalOptPass());
+
+ // Run the OpenMPOpt pass again after global optimizations.
+ MPM.addPass(OpenMPOptPass(ThinOrFullLTOPhase::FullLTOPostLink));
+
+ // Garbage collect dead functions.
+ MPM.addPass(GlobalDCEPass());
+
+ // If we didn't decide to inline a function, check to see if we can
+ // transform it to pass arguments by value instead of by reference.
+ MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(ArgumentPromotionPass()));
+
+ // Remove unused arguments from functions.
+ MPM.addPass(DeadArgumentEliminationPass());
+
+ FunctionPassManager FPM;
+ // The IPO Passes may leave cruft around. Clean up after them.
+ FPM.addPass(InstCombinePass());
+ invokePeepholeEPCallbacks(FPM, Level);
+
+ if (EnableConstraintElimination)
+ FPM.addPass(ConstraintEliminationPass());
+
+ FPM.addPass(JumpThreadingPass());
+
+ // Do a post inline PGO instrumentation and use pass. This is a context
+ // sensitive PGO pass.
+ if (PGOOpt) {
+ if (PGOOpt->CSAction == PGOOptions::CSIRInstr)
+ addPGOInstrPasses(MPM, Level, /* RunProfileGen */ true,
+ /* IsCS */ true, PGOOpt->CSProfileGenFile,
+ PGOOpt->ProfileRemappingFile,
+ ThinOrFullLTOPhase::FullLTOPostLink);
+ else if (PGOOpt->CSAction == PGOOptions::CSIRUse)
+ addPGOInstrPasses(MPM, Level, /* RunProfileGen */ false,
+ /* IsCS */ true, PGOOpt->ProfileFile,
+ PGOOpt->ProfileRemappingFile,
+ ThinOrFullLTOPhase::FullLTOPostLink);
+ }
+
+ // Break up allocas
+ FPM.addPass(SROAPass(SROAOptions::ModifyCFG));
+
+ // LTO provides additional opportunities for tailcall elimination due to
+ // link-time inlining, and visibility of nocapture attribute.
+ FPM.addPass(TailCallElimPass());
+
+ // Run a few AA driver optimizations here and now to cleanup the code.
+ MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM),
+ PTO.EagerlyInvalidateAnalyses));
+
+ MPM.addPass(
+ createModuleToPostOrderCGSCCPassAdaptor(PostOrderFunctionAttrsPass()));
+
+ // Require the GlobalsAA analysis for the module so we can query it within
+ // MainFPM.
+ MPM.addPass(RequireAnalysisPass<GlobalsAA, Module>());
+ // Invalidate AAManager so it can be recreated and pick up the newly available
+ // GlobalsAA.
+ MPM.addPass(
+ createModuleToFunctionPassAdaptor(InvalidateAnalysisPass<AAManager>()));
+
+ FunctionPassManager MainFPM;
+ MainFPM.addPass(createFunctionToLoopPassAdaptor(
+ LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
+ /*AllowSpeculation=*/true),
+ /*USeMemorySSA=*/true, /*UseBlockFrequencyInfo=*/true));
+
+ if (RunNewGVN)
+ MainFPM.addPass(NewGVNPass());
+ else
+ MainFPM.addPass(GVNPass());
+
+ // Remove dead memcpy()'s.
+ MainFPM.addPass(MemCpyOptPass());
+
+ // Nuke dead stores.
+ MainFPM.addPass(DSEPass());
+ MainFPM.addPass(MergedLoadStoreMotionPass());
+
+ LoopPassManager LPM;
+ if (EnableLoopFlatten && Level.getSpeedupLevel() > 1)
+ LPM.addPass(LoopFlattenPass());
+ LPM.addPass(IndVarSimplifyPass());
+ LPM.addPass(LoopDeletionPass());
+ // FIXME: Add loop interchange.
+
+ // Unroll small loops and perform peeling.
+ LPM.addPass(LoopFullUnrollPass(Level.getSpeedupLevel(),
+ /* OnlyWhenForced= */ !PTO.LoopUnrolling,
+ PTO.ForgetAllSCEVInLoopUnroll));
+ // The loop passes in LPM (LoopFullUnrollPass) do not preserve MemorySSA.
+ // *All* loop passes must preserve it, in order to be able to use it.
+ MainFPM.addPass(createFunctionToLoopPassAdaptor(
+ std::move(LPM), /*UseMemorySSA=*/false, /*UseBlockFrequencyInfo=*/true));
+
+ MainFPM.addPass(LoopDistributePass());
+
+ addVectorPasses(Level, MainFPM, /* IsFullLTO */ true);
+
+ // Run the OpenMPOpt CGSCC pass again late.
+ MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
+ OpenMPOptCGSCCPass(ThinOrFullLTOPhase::FullLTOPostLink)));
+
+ invokePeepholeEPCallbacks(MainFPM, Level);
+ MainFPM.addPass(JumpThreadingPass());
+ MPM.addPass(createModuleToFunctionPassAdaptor(std::move(MainFPM),
+ PTO.EagerlyInvalidateAnalyses));
+
+ // Lower type metadata and the type.test intrinsic. This pass supports
+ // clang's control flow integrity mechanisms (-fsanitize=cfi*) and needs
+ // to be run at link time if CFI is enabled. This pass does nothing if
+ // CFI is disabled.
+ MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
+ // Run a second time to clean up any type tests left behind by WPD for use
+ // in ICP (which is performed earlier than this in the regular LTO pipeline).
+ MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
+
+ // Enable splitting late in the FullLTO post-link pipeline.
+ if (EnableHotColdSplit)
+ MPM.addPass(HotColdSplittingPass());
+
+ // Add late LTO optimization passes.
+ // Delete basic blocks, which optimization passes may have killed.
+ MPM.addPass(createModuleToFunctionPassAdaptor(SimplifyCFGPass(
+ SimplifyCFGOptions().convertSwitchRangeToICmp(true).hoistCommonInsts(
+ true))));
+
+ // Drop bodies of available eternally objects to improve GlobalDCE.
+ MPM.addPass(EliminateAvailableExternallyPass());
+
+ // Now that we have optimized the program, discard unreachable functions.
+ MPM.addPass(GlobalDCEPass());
+
+ if (PTO.MergeFunctions)
+ MPM.addPass(MergeFunctionsPass());
+
+ if (PTO.CallGraphProfile)
+ MPM.addPass(CGProfilePass());
+
+ for (auto &C : FullLinkTimeOptimizationLastEPCallbacks)
+ C(MPM, Level);
+
+ // Emit annotation remarks.
+ addAnnotationRemarksPass(MPM);
+
+ return MPM;
+}
+
+ModulePassManager PassBuilder::buildO0DefaultPipeline(OptimizationLevel Level,
+ bool LTOPreLink) {
+ assert(Level == OptimizationLevel::O0 &&
+ "buildO0DefaultPipeline should only be used with O0");
+
+ ModulePassManager MPM;
+
+ // Perform pseudo probe instrumentation in O0 mode. This is for the
+ // consistency between different build modes. For example, a LTO build can be
+ // mixed with an O0 prelink and an O2 postlink. Loading a sample profile in
+ // the postlink will require pseudo probe instrumentation in the prelink.
+ if (PGOOpt && PGOOpt->PseudoProbeForProfiling)
+ MPM.addPass(SampleProfileProbePass(TM));
+
+ if (PGOOpt && (PGOOpt->Action == PGOOptions::IRInstr ||
+ PGOOpt->Action == PGOOptions::IRUse))
+ addPGOInstrPassesForO0(
+ MPM,
+ /* RunProfileGen */ (PGOOpt->Action == PGOOptions::IRInstr),
+ /* IsCS */ false, PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile);
+
+ for (auto &C : PipelineStartEPCallbacks)
+ C(MPM, Level);
+
+ if (PGOOpt && PGOOpt->DebugInfoForProfiling)
+ MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass()));
+
+ for (auto &C : PipelineEarlySimplificationEPCallbacks)
+ C(MPM, Level);
+
+ // Build a minimal pipeline based on the semantics required by LLVM,
+ // which is just that always inlining occurs. Further, disable generating
+ // lifetime intrinsics to avoid enabling further optimizations during
+ // code generation.
+ MPM.addPass(AlwaysInlinerPass(
+ /*InsertLifetimeIntrinsics=*/false));
+
+ if (PTO.MergeFunctions)
+ MPM.addPass(MergeFunctionsPass());
+
+ if (EnableMatrix)
+ MPM.addPass(
+ createModuleToFunctionPassAdaptor(LowerMatrixIntrinsicsPass(true)));
+
+ if (!CGSCCOptimizerLateEPCallbacks.empty()) {
+ CGSCCPassManager CGPM;
+ for (auto &C : CGSCCOptimizerLateEPCallbacks)
+ C(CGPM, Level);
+ if (!CGPM.isEmpty())
+ MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM)));
+ }
+ if (!LateLoopOptimizationsEPCallbacks.empty()) {
+ LoopPassManager LPM;
+ for (auto &C : LateLoopOptimizationsEPCallbacks)
+ C(LPM, Level);
+ if (!LPM.isEmpty()) {
+ MPM.addPass(createModuleToFunctionPassAdaptor(
+ createFunctionToLoopPassAdaptor(std::move(LPM))));
+ }
+ }
+ if (!LoopOptimizerEndEPCallbacks.empty()) {
+ LoopPassManager LPM;
+ for (auto &C : LoopOptimizerEndEPCallbacks)
+ C(LPM, Level);
+ if (!LPM.isEmpty()) {
+ MPM.addPass(createModuleToFunctionPassAdaptor(
+ createFunctionToLoopPassAdaptor(std::move(LPM))));
+ }
+ }
+ if (!ScalarOptimizerLateEPCallbacks.empty()) {
+ FunctionPassManager FPM;
+ for (auto &C : ScalarOptimizerLateEPCallbacks)
+ C(FPM, Level);
+ if (!FPM.isEmpty())
+ MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+ }
+
+ for (auto &C : OptimizerEarlyEPCallbacks)
+ C(MPM, Level);
+
+ if (!VectorizerStartEPCallbacks.empty()) {
+ FunctionPassManager FPM;
+ for (auto &C : VectorizerStartEPCallbacks)
+ C(FPM, Level);
+ if (!FPM.isEmpty())
+ MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+ }
+
+ ModulePassManager CoroPM;
+ CoroPM.addPass(CoroEarlyPass());
+ CGSCCPassManager CGPM;
+ CGPM.addPass(CoroSplitPass());
+ CoroPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM)));
+ CoroPM.addPass(CoroCleanupPass());
+ CoroPM.addPass(GlobalDCEPass());
+ MPM.addPass(CoroConditionalWrapper(std::move(CoroPM)));
+
+ for (auto &C : OptimizerLastEPCallbacks)
+ C(MPM, Level);
+
+ if (LTOPreLink)
+ addRequiredLTOPreLinkPasses(MPM);
+
+ MPM.addPass(createModuleToFunctionPassAdaptor(AnnotationRemarksPass()));
+
+ return MPM;
+}
+
+AAManager PassBuilder::buildDefaultAAPipeline() {
+ AAManager AA;
+
+ // The order in which these are registered determines their priority when
+ // being queried.
+
+ // First we register the basic alias analysis that provides the majority of
+ // per-function local AA logic. This is a stateless, on-demand local set of
+ // AA techniques.
+ AA.registerFunctionAnalysis<BasicAA>();
+
+ // Next we query fast, specialized alias analyses that wrap IR-embedded
+ // information about aliasing.
+ AA.registerFunctionAnalysis<ScopedNoAliasAA>();
+ AA.registerFunctionAnalysis<TypeBasedAA>();
+
+ // Add support for querying global aliasing information when available.
+ // Because the `AAManager` is a function analysis and `GlobalsAA` is a module
+ // analysis, all that the `AAManager` can do is query for any *cached*
+ // results from `GlobalsAA` through a readonly proxy.
+ if (EnableGlobalAnalyses)
+ AA.registerModuleAnalysis<GlobalsAA>();
+
+ // Add target-specific alias analyses.
+ if (TM)
+ TM->registerDefaultAliasAnalyses(AA);
+
+ return AA;
+}
diff --git a/contrib/libs/llvm16/lib/Passes/PassPlugin.cpp b/contrib/libs/llvm16/lib/Passes/PassPlugin.cpp
new file mode 100644
index 00000000000..6182cbbb1fd
--- /dev/null
+++ b/contrib/libs/llvm16/lib/Passes/PassPlugin.cpp
@@ -0,0 +1,54 @@
+//===- lib/Passes/PassPluginLoader.cpp - Load Plugins for New PM Passes ---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Passes/PassPlugin.h"
+#include "llvm/Support/raw_ostream.h"
+
+#include <cstdint>
+
+using namespace llvm;
+
+Expected<PassPlugin> PassPlugin::Load(const std::string &Filename) {
+ std::string Error;
+ auto Library =
+ sys::DynamicLibrary::getPermanentLibrary(Filename.c_str(), &Error);
+ if (!Library.isValid())
+ return make_error<StringError>(Twine("Could not load library '") +
+ Filename + "': " + Error,
+ inconvertibleErrorCode());
+
+ PassPlugin P{Filename, Library};
+
+ // llvmGetPassPluginInfo should be resolved to the definition from the plugin
+ // we are currently loading.
+ intptr_t getDetailsFn =
+ (intptr_t)Library.getAddressOfSymbol("llvmGetPassPluginInfo");
+
+ if (!getDetailsFn)
+ // If the symbol isn't found, this is probably a legacy plugin, which is an
+ // error
+ return make_error<StringError>(Twine("Plugin entry point not found in '") +
+ Filename + "'. Is this a legacy plugin?",
+ inconvertibleErrorCode());
+
+ P.Info = reinterpret_cast<decltype(llvmGetPassPluginInfo) *>(getDetailsFn)();
+
+ if (P.Info.APIVersion != LLVM_PLUGIN_API_VERSION)
+ return make_error<StringError>(
+ Twine("Wrong API version on plugin '") + Filename + "'. Got version " +
+ Twine(P.Info.APIVersion) + ", supported version is " +
+ Twine(LLVM_PLUGIN_API_VERSION) + ".",
+ inconvertibleErrorCode());
+
+ if (!P.Info.RegisterPassBuilderCallbacks)
+ return make_error<StringError>(Twine("Empty entry callback in plugin '") +
+ Filename + "'.'",
+ inconvertibleErrorCode());
+
+ return P;
+}
diff --git a/contrib/libs/llvm16/lib/Passes/PassRegistry.def b/contrib/libs/llvm16/lib/Passes/PassRegistry.def
new file mode 100644
index 00000000000..10af4160c54
--- /dev/null
+++ b/contrib/libs/llvm16/lib/Passes/PassRegistry.def
@@ -0,0 +1,576 @@
+//===- PassRegistry.def - Registry of passes --------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is used as the registry of passes that are part of the core LLVM
+// libraries. This file describes both transformation passes and analyses
+// Analyses are registered while transformation passes have names registered
+// that can be used when providing a textual pass pipeline.
+//
+//===----------------------------------------------------------------------===//
+
+// NOTE: NO INCLUDE GUARD DESIRED!
+
+#ifndef MODULE_ANALYSIS
+#define MODULE_ANALYSIS(NAME, CREATE_PASS)
+#endif
+MODULE_ANALYSIS("callgraph", CallGraphAnalysis())
+MODULE_ANALYSIS("lcg", LazyCallGraphAnalysis())
+MODULE_ANALYSIS("module-summary", ModuleSummaryIndexAnalysis())
+MODULE_ANALYSIS("no-op-module", NoOpModuleAnalysis())
+MODULE_ANALYSIS("profile-summary", ProfileSummaryAnalysis())
+MODULE_ANALYSIS("stack-safety", StackSafetyGlobalAnalysis())
+MODULE_ANALYSIS("verify", VerifierAnalysis())
+MODULE_ANALYSIS("pass-instrumentation", PassInstrumentationAnalysis(PIC))
+MODULE_ANALYSIS("inline-advisor", InlineAdvisorAnalysis())
+MODULE_ANALYSIS("ir-similarity", IRSimilarityAnalysis())
+
+#ifndef MODULE_ALIAS_ANALYSIS
+#define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS) \
+ MODULE_ANALYSIS(NAME, CREATE_PASS)
+#endif
+MODULE_ALIAS_ANALYSIS("globals-aa", GlobalsAA())
+#undef MODULE_ALIAS_ANALYSIS
+#undef MODULE_ANALYSIS
+
+#ifndef MODULE_PASS
+#define MODULE_PASS(NAME, CREATE_PASS)
+#endif
+MODULE_PASS("always-inline", AlwaysInlinerPass())
+MODULE_PASS("attributor", AttributorPass())
+MODULE_PASS("annotation2metadata", Annotation2MetadataPass())
+MODULE_PASS("openmp-opt", OpenMPOptPass())
+MODULE_PASS("openmp-opt-postlink", OpenMPOptPass(ThinOrFullLTOPhase::FullLTOPostLink))
+MODULE_PASS("called-value-propagation", CalledValuePropagationPass())
+MODULE_PASS("canonicalize-aliases", CanonicalizeAliasesPass())
+MODULE_PASS("cg-profile", CGProfilePass())
+MODULE_PASS("check-debugify", NewPMCheckDebugifyPass())
+MODULE_PASS("constmerge", ConstantMergePass())
+MODULE_PASS("coro-early", CoroEarlyPass())
+MODULE_PASS("coro-cleanup", CoroCleanupPass())
+MODULE_PASS("cross-dso-cfi", CrossDSOCFIPass())
+MODULE_PASS("deadargelim", DeadArgumentEliminationPass())
+MODULE_PASS("debugify", NewPMDebugifyPass())
+MODULE_PASS("dot-callgraph", CallGraphDOTPrinterPass())
+MODULE_PASS("elim-avail-extern", EliminateAvailableExternallyPass())
+MODULE_PASS("extract-blocks", BlockExtractorPass({}, false))
+MODULE_PASS("forceattrs", ForceFunctionAttrsPass())
+MODULE_PASS("function-import", FunctionImportPass())
+MODULE_PASS("globaldce", GlobalDCEPass())
+MODULE_PASS("globalopt", GlobalOptPass())
+MODULE_PASS("globalsplit", GlobalSplitPass())
+MODULE_PASS("hotcoldsplit", HotColdSplittingPass())
+MODULE_PASS("inferattrs", InferFunctionAttrsPass())
+MODULE_PASS("inliner-wrapper", ModuleInlinerWrapperPass())
+MODULE_PASS("inliner-ml-advisor-release", ModuleInlinerWrapperPass(getInlineParams(), true, {}, InliningAdvisorMode::Release, 0))
+MODULE_PASS("print<inline-advisor>", InlineAdvisorAnalysisPrinterPass(dbgs()))
+MODULE_PASS("inliner-wrapper-no-mandatory-first", ModuleInlinerWrapperPass(
+ getInlineParams(),
+ false))
+MODULE_PASS("insert-gcov-profiling", GCOVProfilerPass())
+MODULE_PASS("instrorderfile", InstrOrderFilePass())
+MODULE_PASS("instrprof", InstrProfiling())
+MODULE_PASS("internalize", InternalizePass())
+MODULE_PASS("invalidate<all>", InvalidateAllAnalysesPass())
+MODULE_PASS("iroutliner", IROutlinerPass())
+MODULE_PASS("print-ir-similarity", IRSimilarityAnalysisPrinterPass(dbgs()))
+MODULE_PASS("lower-global-dtors", LowerGlobalDtorsPass())
+MODULE_PASS("lower-ifunc", LowerIFuncPass())
+MODULE_PASS("lowertypetests", LowerTypeTestsPass())
+MODULE_PASS("metarenamer", MetaRenamerPass())
+MODULE_PASS("mergefunc", MergeFunctionsPass())
+MODULE_PASS("name-anon-globals", NameAnonGlobalPass())
+MODULE_PASS("no-op-module", NoOpModulePass())
+MODULE_PASS("objc-arc-apelim", ObjCARCAPElimPass())
+MODULE_PASS("partial-inliner", PartialInlinerPass())
+MODULE_PASS("pgo-icall-prom", PGOIndirectCallPromotion())
+MODULE_PASS("pgo-instr-gen", PGOInstrumentationGen())
+MODULE_PASS("pgo-instr-use", PGOInstrumentationUse())
+MODULE_PASS("print-profile-summary", ProfileSummaryPrinterPass(dbgs()))
+MODULE_PASS("print-callgraph", CallGraphPrinterPass(dbgs()))
+MODULE_PASS("print-callgraph-sccs", CallGraphSCCsPrinterPass(dbgs()))
+MODULE_PASS("print", PrintModulePass(dbgs()))
+MODULE_PASS("print-lcg", LazyCallGraphPrinterPass(dbgs()))
+MODULE_PASS("print-lcg-dot", LazyCallGraphDOTPrinterPass(dbgs()))
+MODULE_PASS("print-must-be-executed-contexts", MustBeExecutedContextPrinterPass(dbgs()))
+MODULE_PASS("print-stack-safety", StackSafetyGlobalPrinterPass(dbgs()))
+MODULE_PASS("print<module-debuginfo>", ModuleDebugInfoPrinterPass(dbgs()))
+MODULE_PASS("recompute-globalsaa", RecomputeGlobalsAAPass())
+MODULE_PASS("rel-lookup-table-converter", RelLookupTableConverterPass())
+MODULE_PASS("rewrite-statepoints-for-gc", RewriteStatepointsForGC())
+MODULE_PASS("rewrite-symbols", RewriteSymbolPass())
+MODULE_PASS("rpo-function-attrs", ReversePostOrderFunctionAttrsPass())
+MODULE_PASS("sample-profile", SampleProfileLoaderPass())
+MODULE_PASS("scc-oz-module-inliner",
+ buildInlinerPipeline(OptimizationLevel::Oz, ThinOrFullLTOPhase::None))
+MODULE_PASS("strip", StripSymbolsPass())
+MODULE_PASS("strip-dead-debug-info", StripDeadDebugInfoPass())
+MODULE_PASS("pseudo-probe", SampleProfileProbePass(TM))
+MODULE_PASS("strip-dead-prototypes", StripDeadPrototypesPass())
+MODULE_PASS("strip-debug-declare", StripDebugDeclarePass())
+MODULE_PASS("strip-nondebug", StripNonDebugSymbolsPass())
+MODULE_PASS("strip-nonlinetable-debuginfo", StripNonLineTableDebugInfoPass())
+MODULE_PASS("synthetic-counts-propagation", SyntheticCountsPropagation())
+MODULE_PASS("trigger-crash", TriggerCrashPass())
+MODULE_PASS("verify", VerifierPass())
+MODULE_PASS("view-callgraph", CallGraphViewerPass())
+MODULE_PASS("wholeprogramdevirt", WholeProgramDevirtPass())
+MODULE_PASS("dfsan", DataFlowSanitizerPass())
+MODULE_PASS("module-inline", ModuleInlinerPass())
+MODULE_PASS("tsan-module", ModuleThreadSanitizerPass())
+MODULE_PASS("sancov-module", SanitizerCoveragePass())
+MODULE_PASS("sanmd-module", SanitizerBinaryMetadataPass())
+MODULE_PASS("memprof-module", ModuleMemProfilerPass())
+MODULE_PASS("poison-checking", PoisonCheckingPass())
+MODULE_PASS("pseudo-probe-update", PseudoProbeUpdatePass())
+#undef MODULE_PASS
+
+#ifndef MODULE_PASS_WITH_PARAMS
+#define MODULE_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS)
+#endif
+MODULE_PASS_WITH_PARAMS("loop-extract",
+ "LoopExtractorPass",
+ [](bool Single) {
+ if (Single)
+ return LoopExtractorPass(1);
+ return LoopExtractorPass();
+ },
+ parseLoopExtractorPassOptions,
+ "single")
+MODULE_PASS_WITH_PARAMS("hwasan",
+ "HWAddressSanitizerPass",
+ [](HWAddressSanitizerOptions Opts) {
+ return HWAddressSanitizerPass(Opts);
+ },
+ parseHWASanPassOptions,
+ "kernel;recover")
+MODULE_PASS_WITH_PARAMS("asan",
+ "AddressSanitizerPass",
+ [](AddressSanitizerOptions Opts) {
+ return AddressSanitizerPass(Opts);
+ },
+ parseASanPassOptions,
+ "kernel")
+MODULE_PASS_WITH_PARAMS("msan",
+ "MemorySanitizerPass",
+ [](MemorySanitizerOptions Opts) {
+ return MemorySanitizerPass(Opts);
+ },
+ parseMSanPassOptions,
+ "recover;kernel;eager-checks;track-origins=N")
+MODULE_PASS_WITH_PARAMS("ipsccp",
+ "IPSCCPPass",
+ [](IPSCCPOptions Opts) {
+ return IPSCCPPass(Opts);
+ },
+ parseIPSCCPOptions,
+ "no-func-spec;func-spec")
+#undef MODULE_PASS_WITH_PARAMS
+
+#ifndef CGSCC_ANALYSIS
+#define CGSCC_ANALYSIS(NAME, CREATE_PASS)
+#endif
+CGSCC_ANALYSIS("no-op-cgscc", NoOpCGSCCAnalysis())
+CGSCC_ANALYSIS("fam-proxy", FunctionAnalysisManagerCGSCCProxy())
+CGSCC_ANALYSIS("pass-instrumentation", PassInstrumentationAnalysis(PIC))
+#undef CGSCC_ANALYSIS
+
+#ifndef CGSCC_PASS
+#define CGSCC_PASS(NAME, CREATE_PASS)
+#endif
+CGSCC_PASS("argpromotion", ArgumentPromotionPass())
+CGSCC_PASS("invalidate<all>", InvalidateAllAnalysesPass())
+CGSCC_PASS("function-attrs", PostOrderFunctionAttrsPass())
+CGSCC_PASS("attributor-cgscc", AttributorCGSCCPass())
+CGSCC_PASS("openmp-opt-cgscc", OpenMPOptCGSCCPass())
+CGSCC_PASS("no-op-cgscc", NoOpCGSCCPass())
+#undef CGSCC_PASS
+
+#ifndef CGSCC_PASS_WITH_PARAMS
+#define CGSCC_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS)
+#endif
+CGSCC_PASS_WITH_PARAMS("inline",
+ "InlinerPass",
+ [](bool OnlyMandatory) {
+ return InlinerPass(OnlyMandatory);
+ },
+ parseInlinerPassOptions,
+ "only-mandatory")
+CGSCC_PASS_WITH_PARAMS("coro-split",
+ "CoroSplitPass",
+ [](bool OptimizeFrame) {
+ return CoroSplitPass(OptimizeFrame);
+ },
+ parseCoroSplitPassOptions,
+ "reuse-storage")
+#undef CGSCC_PASS_WITH_PARAMS
+
+#ifndef FUNCTION_ANALYSIS
+#define FUNCTION_ANALYSIS(NAME, CREATE_PASS)
+#endif
+FUNCTION_ANALYSIS("aa", AAManager())
+FUNCTION_ANALYSIS("assumptions", AssumptionAnalysis())
+FUNCTION_ANALYSIS("block-freq", BlockFrequencyAnalysis())
+FUNCTION_ANALYSIS("branch-prob", BranchProbabilityAnalysis())
+FUNCTION_ANALYSIS("cycles", CycleAnalysis())
+FUNCTION_ANALYSIS("domtree", DominatorTreeAnalysis())
+FUNCTION_ANALYSIS("postdomtree", PostDominatorTreeAnalysis())
+FUNCTION_ANALYSIS("demanded-bits", DemandedBitsAnalysis())
+FUNCTION_ANALYSIS("domfrontier", DominanceFrontierAnalysis())
+FUNCTION_ANALYSIS("func-properties", FunctionPropertiesAnalysis())
+FUNCTION_ANALYSIS("loops", LoopAnalysis())
+FUNCTION_ANALYSIS("access-info", LoopAccessAnalysis())
+FUNCTION_ANALYSIS("lazy-value-info", LazyValueAnalysis())
+FUNCTION_ANALYSIS("da", DependenceAnalysis())
+FUNCTION_ANALYSIS("inliner-size-estimator", InlineSizeEstimatorAnalysis())
+FUNCTION_ANALYSIS("memdep", MemoryDependenceAnalysis())
+FUNCTION_ANALYSIS("memoryssa", MemorySSAAnalysis())
+FUNCTION_ANALYSIS("phi-values", PhiValuesAnalysis())
+FUNCTION_ANALYSIS("regions", RegionInfoAnalysis())
+FUNCTION_ANALYSIS("no-op-function", NoOpFunctionAnalysis())
+FUNCTION_ANALYSIS("opt-remark-emit", OptimizationRemarkEmitterAnalysis())
+FUNCTION_ANALYSIS("scalar-evolution", ScalarEvolutionAnalysis())
+FUNCTION_ANALYSIS("should-not-run-function-passes", ShouldNotRunFunctionPassesAnalysis())
+FUNCTION_ANALYSIS("should-run-extra-vector-passes", ShouldRunExtraVectorPasses())
+FUNCTION_ANALYSIS("stack-safety-local", StackSafetyAnalysis())
+FUNCTION_ANALYSIS("targetlibinfo", TargetLibraryAnalysis())
+FUNCTION_ANALYSIS("targetir",
+ TM ? TM->getTargetIRAnalysis() : TargetIRAnalysis())
+FUNCTION_ANALYSIS("verify", VerifierAnalysis())
+FUNCTION_ANALYSIS("pass-instrumentation", PassInstrumentationAnalysis(PIC))
+FUNCTION_ANALYSIS("divergence", DivergenceAnalysis())
+FUNCTION_ANALYSIS("uniformity", UniformityInfoAnalysis())
+
+#ifndef FUNCTION_ALIAS_ANALYSIS
+#define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS) \
+ FUNCTION_ANALYSIS(NAME, CREATE_PASS)
+#endif
+FUNCTION_ALIAS_ANALYSIS("basic-aa", BasicAA())
+FUNCTION_ALIAS_ANALYSIS("objc-arc-aa", objcarc::ObjCARCAA())
+FUNCTION_ALIAS_ANALYSIS("scev-aa", SCEVAA())
+FUNCTION_ALIAS_ANALYSIS("scoped-noalias-aa", ScopedNoAliasAA())
+FUNCTION_ALIAS_ANALYSIS("tbaa", TypeBasedAA())
+#undef FUNCTION_ALIAS_ANALYSIS
+#undef FUNCTION_ANALYSIS
+
+#ifndef FUNCTION_PASS
+#define FUNCTION_PASS(NAME, CREATE_PASS)
+#endif
+FUNCTION_PASS("aa-eval", AAEvaluator())
+FUNCTION_PASS("adce", ADCEPass())
+FUNCTION_PASS("add-discriminators", AddDiscriminatorsPass())
+FUNCTION_PASS("aggressive-instcombine", AggressiveInstCombinePass())
+FUNCTION_PASS("assume-builder", AssumeBuilderPass())
+FUNCTION_PASS("assume-simplify", AssumeSimplifyPass())
+FUNCTION_PASS("alignment-from-assumptions", AlignmentFromAssumptionsPass())
+FUNCTION_PASS("annotation-remarks", AnnotationRemarksPass())
+FUNCTION_PASS("bdce", BDCEPass())
+FUNCTION_PASS("bounds-checking", BoundsCheckingPass())
+FUNCTION_PASS("break-crit-edges", BreakCriticalEdgesPass())
+FUNCTION_PASS("callsite-splitting", CallSiteSplittingPass())
+FUNCTION_PASS("consthoist", ConstantHoistingPass())
+FUNCTION_PASS("constraint-elimination", ConstraintEliminationPass())
+FUNCTION_PASS("chr", ControlHeightReductionPass())
+FUNCTION_PASS("coro-elide", CoroElidePass())
+FUNCTION_PASS("correlated-propagation", CorrelatedValuePropagationPass())
+FUNCTION_PASS("dce", DCEPass())
+FUNCTION_PASS("dfa-jump-threading", DFAJumpThreadingPass())
+FUNCTION_PASS("div-rem-pairs", DivRemPairsPass())
+FUNCTION_PASS("dse", DSEPass())
+FUNCTION_PASS("dot-cfg", CFGPrinterPass())
+FUNCTION_PASS("dot-cfg-only", CFGOnlyPrinterPass())
+FUNCTION_PASS("dot-dom", DomPrinter())
+FUNCTION_PASS("dot-dom-only", DomOnlyPrinter())
+FUNCTION_PASS("dot-post-dom", PostDomPrinter())
+FUNCTION_PASS("dot-post-dom-only", PostDomOnlyPrinter())
+FUNCTION_PASS("view-dom", DomViewer())
+FUNCTION_PASS("view-dom-only", DomOnlyViewer())
+FUNCTION_PASS("view-post-dom", PostDomViewer())
+FUNCTION_PASS("view-post-dom-only", PostDomOnlyViewer())
+FUNCTION_PASS("fix-irreducible", FixIrreduciblePass())
+FUNCTION_PASS("flattencfg", FlattenCFGPass())
+FUNCTION_PASS("make-guards-explicit", MakeGuardsExplicitPass())
+FUNCTION_PASS("gvn-hoist", GVNHoistPass())
+FUNCTION_PASS("gvn-sink", GVNSinkPass())
+FUNCTION_PASS("helloworld", HelloWorldPass())
+FUNCTION_PASS("infer-address-spaces", InferAddressSpacesPass())
+FUNCTION_PASS("instcombine", InstCombinePass())
+FUNCTION_PASS("instcount", InstCountPass())
+FUNCTION_PASS("instsimplify", InstSimplifyPass())
+FUNCTION_PASS("invalidate<all>", InvalidateAllAnalysesPass())
+FUNCTION_PASS("irce", IRCEPass())
+FUNCTION_PASS("float2int", Float2IntPass())
+FUNCTION_PASS("no-op-function", NoOpFunctionPass())
+FUNCTION_PASS("libcalls-shrinkwrap", LibCallsShrinkWrapPass())
+FUNCTION_PASS("lint", LintPass())
+FUNCTION_PASS("inject-tli-mappings", InjectTLIMappings())
+FUNCTION_PASS("instnamer", InstructionNamerPass())
+FUNCTION_PASS("legacy-divergence-analysis", LegacyDivergenceAnalysisPass())
+FUNCTION_PASS("loweratomic", LowerAtomicPass())
+FUNCTION_PASS("lower-expect", LowerExpectIntrinsicPass())
+FUNCTION_PASS("lower-guard-intrinsic", LowerGuardIntrinsicPass())
+FUNCTION_PASS("lower-constant-intrinsics", LowerConstantIntrinsicsPass())
+FUNCTION_PASS("lower-widenable-condition", LowerWidenableConditionPass())
+FUNCTION_PASS("guard-widening", GuardWideningPass())
+FUNCTION_PASS("load-store-vectorizer", LoadStoreVectorizerPass())
+FUNCTION_PASS("loop-simplify", LoopSimplifyPass())
+FUNCTION_PASS("loop-sink", LoopSinkPass())
+FUNCTION_PASS("lowerinvoke", LowerInvokePass())
+FUNCTION_PASS("lowerswitch", LowerSwitchPass())
+FUNCTION_PASS("mem2reg", PromotePass())
+FUNCTION_PASS("memcpyopt", MemCpyOptPass())
+FUNCTION_PASS("mergeicmps", MergeICmpsPass())
+FUNCTION_PASS("mergereturn", UnifyFunctionExitNodesPass())
+FUNCTION_PASS("nary-reassociate", NaryReassociatePass())
+FUNCTION_PASS("newgvn", NewGVNPass())
+FUNCTION_PASS("jump-threading", JumpThreadingPass())
+FUNCTION_PASS("partially-inline-libcalls", PartiallyInlineLibCallsPass())
+FUNCTION_PASS("kcfi", KCFIPass())
+FUNCTION_PASS("lcssa", LCSSAPass())
+FUNCTION_PASS("loop-data-prefetch", LoopDataPrefetchPass())
+FUNCTION_PASS("loop-load-elim", LoopLoadEliminationPass())
+FUNCTION_PASS("loop-fusion", LoopFusePass())
+FUNCTION_PASS("loop-distribute", LoopDistributePass())
+FUNCTION_PASS("loop-versioning", LoopVersioningPass())
+FUNCTION_PASS("objc-arc", ObjCARCOptPass())
+FUNCTION_PASS("objc-arc-contract", ObjCARCContractPass())
+FUNCTION_PASS("objc-arc-expand", ObjCARCExpandPass())
+FUNCTION_PASS("pa-eval", PAEvalPass())
+FUNCTION_PASS("pgo-memop-opt", PGOMemOPSizeOpt())
+FUNCTION_PASS("print", PrintFunctionPass(dbgs()))
+FUNCTION_PASS("print<assumptions>", AssumptionPrinterPass(dbgs()))
+FUNCTION_PASS("print<block-freq>", BlockFrequencyPrinterPass(dbgs()))
+FUNCTION_PASS("print<branch-prob>", BranchProbabilityPrinterPass(dbgs()))
+FUNCTION_PASS("print<cost-model>", CostModelPrinterPass(dbgs()))
+FUNCTION_PASS("print<cycles>", CycleInfoPrinterPass(dbgs()))
+FUNCTION_PASS("print<da>", DependenceAnalysisPrinterPass(dbgs()))
+FUNCTION_PASS("print<divergence>", DivergenceAnalysisPrinterPass(dbgs()))
+FUNCTION_PASS("print<domtree>", DominatorTreePrinterPass(dbgs()))
+FUNCTION_PASS("print<postdomtree>", PostDominatorTreePrinterPass(dbgs()))
+FUNCTION_PASS("print<delinearization>", DelinearizationPrinterPass(dbgs()))
+FUNCTION_PASS("print<demanded-bits>", DemandedBitsPrinterPass(dbgs()))
+FUNCTION_PASS("print<domfrontier>", DominanceFrontierPrinterPass(dbgs()))
+FUNCTION_PASS("print<func-properties>", FunctionPropertiesPrinterPass(dbgs()))
+FUNCTION_PASS("print<inline-cost>", InlineCostAnnotationPrinterPass(dbgs()))
+FUNCTION_PASS("print<inliner-size-estimator>",
+ InlineSizeEstimatorAnalysisPrinterPass(dbgs()))
+FUNCTION_PASS("print<loops>", LoopPrinterPass(dbgs()))
+FUNCTION_PASS("print<memoryssa>", MemorySSAPrinterPass(dbgs()))
+FUNCTION_PASS("print<memoryssa-walker>", MemorySSAWalkerPrinterPass(dbgs()))
+FUNCTION_PASS("print<phi-values>", PhiValuesPrinterPass(dbgs()))
+FUNCTION_PASS("print<regions>", RegionInfoPrinterPass(dbgs()))
+FUNCTION_PASS("print<scalar-evolution>", ScalarEvolutionPrinterPass(dbgs()))
+FUNCTION_PASS("print<stack-safety-local>", StackSafetyPrinterPass(dbgs()))
+FUNCTION_PASS("print<access-info>", LoopAccessInfoPrinterPass(dbgs()))
+// TODO: rename to print<foo> after NPM switch
+FUNCTION_PASS("print-alias-sets", AliasSetsPrinterPass(dbgs()))
+FUNCTION_PASS("print-cfg-sccs", CFGSCCPrinterPass(dbgs()))
+FUNCTION_PASS("print-predicateinfo", PredicateInfoPrinterPass(dbgs()))
+FUNCTION_PASS("print-mustexecute", MustExecutePrinterPass(dbgs()))
+FUNCTION_PASS("print-memderefs", MemDerefPrinterPass(dbgs()))
+FUNCTION_PASS("print<uniformity>", UniformityInfoPrinterPass(dbgs()))
+FUNCTION_PASS("reassociate", ReassociatePass())
+FUNCTION_PASS("redundant-dbg-inst-elim", RedundantDbgInstEliminationPass())
+FUNCTION_PASS("reg2mem", RegToMemPass())
+FUNCTION_PASS("scalarize-masked-mem-intrin", ScalarizeMaskedMemIntrinPass())
+FUNCTION_PASS("scalarizer", ScalarizerPass())
+FUNCTION_PASS("separate-const-offset-from-gep", SeparateConstOffsetFromGEPPass())
+FUNCTION_PASS("sccp", SCCPPass())
+FUNCTION_PASS("sink", SinkingPass())
+FUNCTION_PASS("slp-vectorizer", SLPVectorizerPass())
+FUNCTION_PASS("slsr", StraightLineStrengthReducePass())
+FUNCTION_PASS("speculative-execution", SpeculativeExecutionPass())
+FUNCTION_PASS("strip-gc-relocates", StripGCRelocates())
+FUNCTION_PASS("structurizecfg", StructurizeCFGPass())
+FUNCTION_PASS("tailcallelim", TailCallElimPass())
+FUNCTION_PASS("typepromotion", TypePromotionPass(TM))
+FUNCTION_PASS("unify-loop-exits", UnifyLoopExitsPass())
+FUNCTION_PASS("vector-combine", VectorCombinePass())
+FUNCTION_PASS("verify", VerifierPass())
+FUNCTION_PASS("verify<domtree>", DominatorTreeVerifierPass())
+FUNCTION_PASS("verify<loops>", LoopVerifierPass())
+FUNCTION_PASS("verify<memoryssa>", MemorySSAVerifierPass())
+FUNCTION_PASS("verify<regions>", RegionInfoVerifierPass())
+FUNCTION_PASS("verify<safepoint-ir>", SafepointIRVerifierPass())
+FUNCTION_PASS("verify<scalar-evolution>", ScalarEvolutionVerifierPass())
+FUNCTION_PASS("view-cfg", CFGViewerPass())
+FUNCTION_PASS("view-cfg-only", CFGOnlyViewerPass())
+FUNCTION_PASS("tlshoist", TLSVariableHoistPass())
+FUNCTION_PASS("transform-warning", WarnMissedTransformationsPass())
+FUNCTION_PASS("tsan", ThreadSanitizerPass())
+FUNCTION_PASS("memprof", MemProfilerPass())
+FUNCTION_PASS("declare-to-assign", llvm::AssignmentTrackingPass())
+#undef FUNCTION_PASS
+
+#ifndef FUNCTION_PASS_WITH_PARAMS
+#define FUNCTION_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS)
+#endif
+FUNCTION_PASS_WITH_PARAMS("early-cse",
+ "EarlyCSEPass",
+ [](bool UseMemorySSA) {
+ return EarlyCSEPass(UseMemorySSA);
+ },
+ parseEarlyCSEPassOptions,
+ "memssa")
+FUNCTION_PASS_WITH_PARAMS("ee-instrument",
+ "EntryExitInstrumenterPass",
+ [](bool PostInlining) {
+ return EntryExitInstrumenterPass(PostInlining);
+ },
+ parseEntryExitInstrumenterPassOptions,
+ "post-inline")
+FUNCTION_PASS_WITH_PARAMS("lower-matrix-intrinsics",
+ "LowerMatrixIntrinsicsPass",
+ [](bool Minimal) {
+ return LowerMatrixIntrinsicsPass(Minimal);
+ },
+ parseLowerMatrixIntrinsicsPassOptions,
+ "minimal")
+FUNCTION_PASS_WITH_PARAMS("loop-unroll",
+ "LoopUnrollPass",
+ [](LoopUnrollOptions Opts) {
+ return LoopUnrollPass(Opts);
+ },
+ parseLoopUnrollOptions,
+ "O0;O1;O2;O3;full-unroll-max=N;"
+ "no-partial;partial;"
+ "no-peeling;peeling;"
+ "no-profile-peeling;profile-peeling;"
+ "no-runtime;runtime;"
+ "no-upperbound;upperbound")
+FUNCTION_PASS_WITH_PARAMS("simplifycfg",
+ "SimplifyCFGPass",
+ [](SimplifyCFGOptions Opts) {
+ return SimplifyCFGPass(Opts);
+ },
+ parseSimplifyCFGOptions,
+ "no-forward-switch-cond;forward-switch-cond;"
+ "no-switch-range-to-icmp;switch-range-to-icmp;"
+ "no-switch-to-lookup;switch-to-lookup;"
+ "no-keep-loops;keep-loops;"
+ "no-hoist-common-insts;hoist-common-insts;"
+ "no-sink-common-insts;sink-common-insts;"
+ "bonus-inst-threshold=N"
+ )
+FUNCTION_PASS_WITH_PARAMS("loop-vectorize",
+ "LoopVectorizePass",
+ [](LoopVectorizeOptions Opts) {
+ return LoopVectorizePass(Opts);
+ },
+ parseLoopVectorizeOptions,
+ "no-interleave-forced-only;interleave-forced-only;"
+ "no-vectorize-forced-only;vectorize-forced-only")
+FUNCTION_PASS_WITH_PARAMS("mldst-motion",
+ "MergedLoadStoreMotionPass",
+ [](MergedLoadStoreMotionOptions Opts) {
+ return MergedLoadStoreMotionPass(Opts);
+ },
+ parseMergedLoadStoreMotionOptions,
+ "no-split-footer-bb;split-footer-bb")
+FUNCTION_PASS_WITH_PARAMS("gvn",
+ "GVNPass",
+ [](GVNOptions Opts) {
+ return GVNPass(Opts);
+ },
+ parseGVNOptions,
+ "no-pre;pre;"
+ "no-load-pre;load-pre;"
+ "no-split-backedge-load-pre;split-backedge-load-pre;"
+ "no-memdep;memdep")
+FUNCTION_PASS_WITH_PARAMS("sroa",
+ "SROAPass",
+ [](SROAOptions PreserveCFG) {
+ return SROAPass(PreserveCFG);
+ },
+ parseSROAOptions,
+ "preserve-cfg;modify-cfg")
+FUNCTION_PASS_WITH_PARAMS("print<stack-lifetime>",
+ "StackLifetimePrinterPass",
+ [](StackLifetime::LivenessType Type) {
+ return StackLifetimePrinterPass(dbgs(), Type);
+ },
+ parseStackLifetimeOptions,
+ "may;must")
+FUNCTION_PASS_WITH_PARAMS("print<da>",
+ "DependenceAnalysisPrinterPass",
+ [](bool NormalizeResults) {
+ return DependenceAnalysisPrinterPass(dbgs(), NormalizeResults);
+ },
+ parseDependenceAnalysisPrinterOptions,
+ "normalized-results")
+#undef FUNCTION_PASS_WITH_PARAMS
+
+#ifndef LOOPNEST_PASS
+#define LOOPNEST_PASS(NAME, CREATE_PASS)
+#endif
+LOOPNEST_PASS("loop-flatten", LoopFlattenPass())
+LOOPNEST_PASS("loop-interchange", LoopInterchangePass())
+LOOPNEST_PASS("loop-unroll-and-jam", LoopUnrollAndJamPass())
+LOOPNEST_PASS("no-op-loopnest", NoOpLoopNestPass())
+#undef LOOPNEST_PASS
+
+#ifndef LOOP_ANALYSIS
+#define LOOP_ANALYSIS(NAME, CREATE_PASS)
+#endif
+LOOP_ANALYSIS("no-op-loop", NoOpLoopAnalysis())
+LOOP_ANALYSIS("ddg", DDGAnalysis())
+LOOP_ANALYSIS("iv-users", IVUsersAnalysis())
+LOOP_ANALYSIS("pass-instrumentation", PassInstrumentationAnalysis(PIC))
+#undef LOOP_ANALYSIS
+
+#ifndef LOOP_PASS
+#define LOOP_PASS(NAME, CREATE_PASS)
+#endif
+LOOP_PASS("canon-freeze", CanonicalizeFreezeInLoopsPass())
+LOOP_PASS("dot-ddg", DDGDotPrinterPass())
+LOOP_PASS("invalidate<all>", InvalidateAllAnalysesPass())
+LOOP_PASS("loop-idiom", LoopIdiomRecognizePass())
+LOOP_PASS("loop-instsimplify", LoopInstSimplifyPass())
+LOOP_PASS("loop-rotate", LoopRotatePass())
+LOOP_PASS("no-op-loop", NoOpLoopPass())
+LOOP_PASS("print", PrintLoopPass(dbgs()))
+LOOP_PASS("loop-deletion", LoopDeletionPass())
+LOOP_PASS("loop-simplifycfg", LoopSimplifyCFGPass())
+LOOP_PASS("loop-reduce", LoopStrengthReducePass())
+LOOP_PASS("indvars", IndVarSimplifyPass())
+LOOP_PASS("loop-unroll-full", LoopFullUnrollPass())
+LOOP_PASS("print<ddg>", DDGAnalysisPrinterPass(dbgs()))
+LOOP_PASS("print<iv-users>", IVUsersPrinterPass(dbgs()))
+LOOP_PASS("print<loopnest>", LoopNestPrinterPass(dbgs()))
+LOOP_PASS("print<loop-cache-cost>", LoopCachePrinterPass(dbgs()))
+LOOP_PASS("loop-predication", LoopPredicationPass())
+LOOP_PASS("guard-widening", GuardWideningPass())
+LOOP_PASS("loop-bound-split", LoopBoundSplitPass())
+LOOP_PASS("loop-reroll", LoopRerollPass())
+LOOP_PASS("loop-versioning-licm", LoopVersioningLICMPass())
+#undef LOOP_PASS
+
+#ifndef LOOP_PASS_WITH_PARAMS
+#define LOOP_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS)
+#endif
+LOOP_PASS_WITH_PARAMS("simple-loop-unswitch",
+ "SimpleLoopUnswitchPass",
+ [](std::pair<bool, bool> Params) {
+ return SimpleLoopUnswitchPass(Params.first, Params.second);
+ },
+ parseLoopUnswitchOptions,
+ "nontrivial;no-nontrivial;trivial;no-trivial")
+
+LOOP_PASS_WITH_PARAMS("licm", "LICMPass",
+ [](LICMOptions Params) {
+ return LICMPass(Params);
+ },
+ parseLICMOptions,
+ "allowspeculation");
+
+LOOP_PASS_WITH_PARAMS("lnicm", "LNICMPass",
+ [](LICMOptions Params) {
+ return LNICMPass(Params);
+ },
+ parseLICMOptions,
+ "allowspeculation");
+#undef LOOP_PASS_WITH_PARAMS
diff --git a/contrib/libs/llvm16/lib/Passes/StandardInstrumentations.cpp b/contrib/libs/llvm16/lib/Passes/StandardInstrumentations.cpp
new file mode 100644
index 00000000000..ad2504eca2f
--- /dev/null
+++ b/contrib/libs/llvm16/lib/Passes/StandardInstrumentations.cpp
@@ -0,0 +1,2191 @@
+//===- Standard pass instrumentations handling ----------------*- C++ -*--===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+/// \file
+///
+/// This file defines IR-printing pass instrumentation callbacks as well as
+/// StandardInstrumentations class that manages standard pass instrumentations.
+///
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Passes/StandardInstrumentations.h"
+#include "llvm/ADT/Any.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Analysis/CallGraphSCCPass.h"
+#include "llvm/Analysis/LazyCallGraph.h"
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/LegacyPassManager.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/PassInstrumentation.h"
+#include "llvm/IR/PassManager.h"
+#include "llvm/IR/PrintPasses.h"
+#include "llvm/IR/Verifier.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/CrashRecoveryContext.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/FormatVariadic.h"
+#include "llvm/Support/GraphWriter.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/Program.h"
+#include "llvm/Support/Regex.h"
+#include "llvm/Support/Signals.h"
+#include "llvm/Support/raw_ostream.h"
+#include <unordered_map>
+#include <unordered_set>
+#include <utility>
+#include <vector>
+
+using namespace llvm;
+
+cl::opt<bool> PreservedCFGCheckerInstrumentation::VerifyPreservedCFG(
+ "verify-cfg-preserved", cl::Hidden,
+#ifdef NDEBUG
+ cl::init(false)
+#else
+ cl::init(true)
+#endif
+ );
+
+// An option that supports the -print-changed option. See
+// the description for -print-changed for an explanation of the use
+// of this option. Note that this option has no effect without -print-changed.
+static cl::opt<bool>
+ PrintChangedBefore("print-before-changed",
+ cl::desc("Print before passes that change them"),
+ cl::init(false), cl::Hidden);
+
+// An option for specifying the dot used by
+// print-changed=[dot-cfg | dot-cfg-quiet]
+static cl::opt<std::string>
+ DotBinary("print-changed-dot-path", cl::Hidden, cl::init("dot"),
+ cl::desc("system dot used by change reporters"));
+
+// An option that determines the colour used for elements that are only
+// in the before part. Must be a colour named in appendix J of
+// https://graphviz.org/pdf/dotguide.pdf
+static cl::opt<std::string>
+ BeforeColour("dot-cfg-before-color",
+ cl::desc("Color for dot-cfg before elements"), cl::Hidden,
+ cl::init("red"));
+// An option that determines the colour used for elements that are only
+// in the after part. Must be a colour named in appendix J of
+// https://graphviz.org/pdf/dotguide.pdf
+static cl::opt<std::string>
+ AfterColour("dot-cfg-after-color",
+ cl::desc("Color for dot-cfg after elements"), cl::Hidden,
+ cl::init("forestgreen"));
+// An option that determines the colour used for elements that are in both
+// the before and after parts. Must be a colour named in appendix J of
+// https://graphviz.org/pdf/dotguide.pdf
+static cl::opt<std::string>
+ CommonColour("dot-cfg-common-color",
+ cl::desc("Color for dot-cfg common elements"), cl::Hidden,
+ cl::init("black"));
+
+// An option that determines where the generated website file (named
+// passes.html) and the associated pdf files (named diff_*.pdf) are saved.
+static cl::opt<std::string> DotCfgDir(
+ "dot-cfg-dir",
+ cl::desc("Generate dot files into specified directory for changed IRs"),
+ cl::Hidden, cl::init("./"));
+
+// An option to print the IR that was being processed when a pass crashes.
+static cl::opt<bool>
+ PrintCrashIR("print-on-crash",
+ cl::desc("Print the last form of the IR before crash"),
+ cl::Hidden);
+
+static cl::opt<std::string> OptBisectPrintIRPath(
+ "opt-bisect-print-ir-path",
+ cl::desc("Print IR to path when opt-bisect-limit is reached"), cl::Hidden);
+
+namespace {
+
+// An option for specifying an executable that will be called with the IR
+// everytime it changes in the opt pipeline. It will also be called on
+// the initial IR as it enters the pipeline. The executable will be passed
+// the name of a temporary file containing the IR and the PassID. This may
+// be used, for example, to call llc on the IR and run a test to determine
+// which pass makes a change that changes the functioning of the IR.
+// The usual modifier options work as expected.
+static cl::opt<std::string>
+ TestChanged("exec-on-ir-change", cl::Hidden, cl::init(""),
+ cl::desc("exe called with module IR after each pass that "
+ "changes it"));
+
+/// Extract Module out of \p IR unit. May return nullptr if \p IR does not match
+/// certain global filters. Will never return nullptr if \p Force is true.
+const Module *unwrapModule(Any IR, bool Force = false) {
+ if (const auto **M = any_cast<const Module *>(&IR))
+ return *M;
+
+ if (const auto **F = any_cast<const Function *>(&IR)) {
+ if (!Force && !isFunctionInPrintList((*F)->getName()))
+ return nullptr;
+
+ return (*F)->getParent();
+ }
+
+ if (const auto **C = any_cast<const LazyCallGraph::SCC *>(&IR)) {
+ for (const LazyCallGraph::Node &N : **C) {
+ const Function &F = N.getFunction();
+ if (Force || (!F.isDeclaration() && isFunctionInPrintList(F.getName()))) {
+ return F.getParent();
+ }
+ }
+ assert(!Force && "Expected a module");
+ return nullptr;
+ }
+
+ if (const auto **L = any_cast<const Loop *>(&IR)) {
+ const Function *F = (*L)->getHeader()->getParent();
+ if (!Force && !isFunctionInPrintList(F->getName()))
+ return nullptr;
+ return F->getParent();
+ }
+
+ llvm_unreachable("Unknown IR unit");
+}
+
+void printIR(raw_ostream &OS, const Function *F) {
+ if (!isFunctionInPrintList(F->getName()))
+ return;
+ OS << *F;
+}
+
+void printIR(raw_ostream &OS, const Module *M) {
+ if (isFunctionInPrintList("*") || forcePrintModuleIR()) {
+ M->print(OS, nullptr);
+ } else {
+ for (const auto &F : M->functions()) {
+ printIR(OS, &F);
+ }
+ }
+}
+
+void printIR(raw_ostream &OS, const LazyCallGraph::SCC *C) {
+ for (const LazyCallGraph::Node &N : *C) {
+ const Function &F = N.getFunction();
+ if (!F.isDeclaration() && isFunctionInPrintList(F.getName())) {
+ F.print(OS);
+ }
+ }
+}
+
+void printIR(raw_ostream &OS, const Loop *L) {
+ const Function *F = L->getHeader()->getParent();
+ if (!isFunctionInPrintList(F->getName()))
+ return;
+ printLoop(const_cast<Loop &>(*L), OS);
+}
+
+std::string getIRName(Any IR) {
+ if (any_cast<const Module *>(&IR))
+ return "[module]";
+
+ if (const auto **F = any_cast<const Function *>(&IR))
+ return (*F)->getName().str();
+
+ if (const auto **C = any_cast<const LazyCallGraph::SCC *>(&IR))
+ return (*C)->getName();
+
+ if (const auto **L = any_cast<const Loop *>(&IR))
+ return (*L)->getName().str();
+
+ llvm_unreachable("Unknown wrapped IR type");
+}
+
+bool moduleContainsFilterPrintFunc(const Module &M) {
+ return any_of(M.functions(),
+ [](const Function &F) {
+ return isFunctionInPrintList(F.getName());
+ }) ||
+ isFunctionInPrintList("*");
+}
+
+bool sccContainsFilterPrintFunc(const LazyCallGraph::SCC &C) {
+ return any_of(C,
+ [](const LazyCallGraph::Node &N) {
+ return isFunctionInPrintList(N.getName());
+ }) ||
+ isFunctionInPrintList("*");
+}
+
+bool shouldPrintIR(Any IR) {
+ if (const auto **M = any_cast<const Module *>(&IR))
+ return moduleContainsFilterPrintFunc(**M);
+
+ if (const auto **F = any_cast<const Function *>(&IR))
+ return isFunctionInPrintList((*F)->getName());
+
+ if (const auto **C = any_cast<const LazyCallGraph::SCC *>(&IR))
+ return sccContainsFilterPrintFunc(**C);
+
+ if (const auto **L = any_cast<const Loop *>(&IR))
+ return isFunctionInPrintList((*L)->getHeader()->getParent()->getName());
+ llvm_unreachable("Unknown wrapped IR type");
+}
+
+/// Generic IR-printing helper that unpacks a pointer to IRUnit wrapped into
+/// Any and does actual print job.
+void unwrapAndPrint(raw_ostream &OS, Any IR) {
+ if (!shouldPrintIR(IR))
+ return;
+
+ if (forcePrintModuleIR()) {
+ auto *M = unwrapModule(IR);
+ assert(M && "should have unwrapped module");
+ printIR(OS, M);
+ return;
+ }
+
+ if (const auto **M = any_cast<const Module *>(&IR)) {
+ printIR(OS, *M);
+ return;
+ }
+
+ if (const auto **F = any_cast<const Function *>(&IR)) {
+ printIR(OS, *F);
+ return;
+ }
+
+ if (const auto **C = any_cast<const LazyCallGraph::SCC *>(&IR)) {
+ printIR(OS, *C);
+ return;
+ }
+
+ if (const auto **L = any_cast<const Loop *>(&IR)) {
+ printIR(OS, *L);
+ return;
+ }
+ llvm_unreachable("Unknown wrapped IR type");
+}
+
+// Return true when this is a pass for which changes should be ignored
+bool isIgnored(StringRef PassID) {
+ return isSpecialPass(PassID,
+ {"PassManager", "PassAdaptor", "AnalysisManagerProxy",
+ "DevirtSCCRepeatedPass", "ModuleInlinerWrapperPass"});
+}
+
+std::string makeHTMLReady(StringRef SR) {
+ std::string S;
+ while (true) {
+ StringRef Clean =
+ SR.take_until([](char C) { return C == '<' || C == '>'; });
+ S.append(Clean.str());
+ SR = SR.drop_front(Clean.size());
+ if (SR.size() == 0)
+ return S;
+ S.append(SR[0] == '<' ? "&lt;" : "&gt;");
+ SR = SR.drop_front();
+ }
+ llvm_unreachable("problems converting string to HTML");
+}
+
+// Return the module when that is the appropriate level of comparison for \p IR.
+const Module *getModuleForComparison(Any IR) {
+ if (const auto **M = any_cast<const Module *>(&IR))
+ return *M;
+ if (const auto **C = any_cast<const LazyCallGraph::SCC *>(&IR))
+ return (*C)
+ ->begin()
+ ->getFunction()
+ .getParent();
+ return nullptr;
+}
+
+bool isInterestingFunction(const Function &F) {
+ return isFunctionInPrintList(F.getName());
+}
+
+// Return true when this is a pass on IR for which printing
+// of changes is desired.
+bool isInteresting(Any IR, StringRef PassID, StringRef PassName) {
+ if (isIgnored(PassID) || !isPassInPrintList(PassName))
+ return false;
+ if (const auto **F = any_cast<const Function *>(&IR))
+ return isInterestingFunction(**F);
+ return true;
+}
+
+} // namespace
+
+template <typename T> ChangeReporter<T>::~ChangeReporter() {
+ assert(BeforeStack.empty() && "Problem with Change Printer stack.");
+}
+
+template <typename T>
+void ChangeReporter<T>::saveIRBeforePass(Any IR, StringRef PassID,
+ StringRef PassName) {
+ // Is this the initial IR?
+ if (InitialIR) {
+ InitialIR = false;
+ if (VerboseMode)
+ handleInitialIR(IR);
+ }
+
+ // Always need to place something on the stack because invalidated passes
+ // are not given the IR so it cannot be determined whether the pass was for
+ // something that was filtered out.
+ BeforeStack.emplace_back();
+
+ if (!isInteresting(IR, PassID, PassName))
+ return;
+
+ // Save the IR representation on the stack.
+ T &Data = BeforeStack.back();
+ generateIRRepresentation(IR, PassID, Data);
+}
+
+template <typename T>
+void ChangeReporter<T>::handleIRAfterPass(Any IR, StringRef PassID,
+ StringRef PassName) {
+ assert(!BeforeStack.empty() && "Unexpected empty stack encountered.");
+
+ std::string Name = getIRName(IR);
+
+ if (isIgnored(PassID)) {
+ if (VerboseMode)
+ handleIgnored(PassID, Name);
+ } else if (!isInteresting(IR, PassID, PassName)) {
+ if (VerboseMode)
+ handleFiltered(PassID, Name);
+ } else {
+ // Get the before rep from the stack
+ T &Before = BeforeStack.back();
+ // Create the after rep
+ T After;
+ generateIRRepresentation(IR, PassID, After);
+
+ // Was there a change in IR?
+ if (Before == After) {
+ if (VerboseMode)
+ omitAfter(PassID, Name);
+ } else
+ handleAfter(PassID, Name, Before, After, IR);
+ }
+ BeforeStack.pop_back();
+}
+
+template <typename T>
+void ChangeReporter<T>::handleInvalidatedPass(StringRef PassID) {
+ assert(!BeforeStack.empty() && "Unexpected empty stack encountered.");
+
+ // Always flag it as invalidated as we cannot determine when
+ // a pass for a filtered function is invalidated since we do not
+ // get the IR in the call. Also, the output is just alternate
+ // forms of the banner anyway.
+ if (VerboseMode)
+ handleInvalidated(PassID);
+ BeforeStack.pop_back();
+}
+
+template <typename T>
+void ChangeReporter<T>::registerRequiredCallbacks(
+ PassInstrumentationCallbacks &PIC) {
+ PIC.registerBeforeNonSkippedPassCallback([&PIC, this](StringRef P, Any IR) {
+ saveIRBeforePass(IR, P, PIC.getPassNameForClassName(P));
+ });
+
+ PIC.registerAfterPassCallback(
+ [&PIC, this](StringRef P, Any IR, const PreservedAnalyses &) {
+ handleIRAfterPass(IR, P, PIC.getPassNameForClassName(P));
+ });
+ PIC.registerAfterPassInvalidatedCallback(
+ [this](StringRef P, const PreservedAnalyses &) {
+ handleInvalidatedPass(P);
+ });
+}
+
+template <typename T>
+TextChangeReporter<T>::TextChangeReporter(bool Verbose)
+ : ChangeReporter<T>(Verbose), Out(dbgs()) {}
+
+template <typename T> void TextChangeReporter<T>::handleInitialIR(Any IR) {
+ // Always print the module.
+ // Unwrap and print directly to avoid filtering problems in general routines.
+ auto *M = unwrapModule(IR, /*Force=*/true);
+ assert(M && "Expected module to be unwrapped when forced.");
+ Out << "*** IR Dump At Start ***\n";
+ M->print(Out, nullptr);
+}
+
+template <typename T>
+void TextChangeReporter<T>::omitAfter(StringRef PassID, std::string &Name) {
+ Out << formatv("*** IR Dump After {0} on {1} omitted because no change ***\n",
+ PassID, Name);
+}
+
+template <typename T>
+void TextChangeReporter<T>::handleInvalidated(StringRef PassID) {
+ Out << formatv("*** IR Pass {0} invalidated ***\n", PassID);
+}
+
+template <typename T>
+void TextChangeReporter<T>::handleFiltered(StringRef PassID,
+ std::string &Name) {
+ SmallString<20> Banner =
+ formatv("*** IR Dump After {0} on {1} filtered out ***\n", PassID, Name);
+ Out << Banner;
+}
+
+template <typename T>
+void TextChangeReporter<T>::handleIgnored(StringRef PassID, std::string &Name) {
+ Out << formatv("*** IR Pass {0} on {1} ignored ***\n", PassID, Name);
+}
+
+IRChangedPrinter::~IRChangedPrinter() = default;
+
+void IRChangedPrinter::registerCallbacks(PassInstrumentationCallbacks &PIC) {
+ if (PrintChanged == ChangePrinter::Verbose ||
+ PrintChanged == ChangePrinter::Quiet)
+ TextChangeReporter<std::string>::registerRequiredCallbacks(PIC);
+}
+
+void IRChangedPrinter::generateIRRepresentation(Any IR, StringRef PassID,
+ std::string &Output) {
+ raw_string_ostream OS(Output);
+ unwrapAndPrint(OS, IR);
+ OS.str();
+}
+
+void IRChangedPrinter::handleAfter(StringRef PassID, std::string &Name,
+ const std::string &Before,
+ const std::string &After, Any) {
+ // Report the IR before the changes when requested.
+ if (PrintChangedBefore)
+ Out << "*** IR Dump Before " << PassID << " on " << Name << " ***\n"
+ << Before;
+
+ // We might not get anything to print if we only want to print a specific
+ // function but it gets deleted.
+ if (After.empty()) {
+ Out << "*** IR Deleted After " << PassID << " on " << Name << " ***\n";
+ return;
+ }
+
+ Out << "*** IR Dump After " << PassID << " on " << Name << " ***\n" << After;
+}
+
+IRChangedTester::~IRChangedTester() {}
+
+void IRChangedTester::registerCallbacks(PassInstrumentationCallbacks &PIC) {
+ if (TestChanged != "")
+ TextChangeReporter<std::string>::registerRequiredCallbacks(PIC);
+}
+
+void IRChangedTester::handleIR(const std::string &S, StringRef PassID) {
+ // Store the body into a temporary file
+ static SmallVector<int> FD{-1};
+ SmallVector<StringRef> SR{S};
+ static SmallVector<std::string> FileName{""};
+ if (auto Err = prepareTempFiles(FD, SR, FileName)) {
+ dbgs() << "Unable to create temporary file.";
+ return;
+ }
+ static ErrorOr<std::string> Exe = sys::findProgramByName(TestChanged);
+ if (!Exe) {
+ dbgs() << "Unable to find test-changed executable.";
+ return;
+ }
+
+ StringRef Args[] = {TestChanged, FileName[0], PassID};
+ int Result = sys::ExecuteAndWait(*Exe, Args);
+ if (Result < 0) {
+ dbgs() << "Error executing test-changed executable.";
+ return;
+ }
+
+ if (auto Err = cleanUpTempFiles(FileName))
+ dbgs() << "Unable to remove temporary file.";
+}
+
+void IRChangedTester::handleInitialIR(Any IR) {
+ // Always test the initial module.
+ // Unwrap and print directly to avoid filtering problems in general routines.
+ std::string S;
+ generateIRRepresentation(IR, "Initial IR", S);
+ handleIR(S, "Initial IR");
+}
+
+void IRChangedTester::omitAfter(StringRef PassID, std::string &Name) {}
+void IRChangedTester::handleInvalidated(StringRef PassID) {}
+void IRChangedTester::handleFiltered(StringRef PassID, std::string &Name) {}
+void IRChangedTester::handleIgnored(StringRef PassID, std::string &Name) {}
+void IRChangedTester::handleAfter(StringRef PassID, std::string &Name,
+ const std::string &Before,
+ const std::string &After, Any) {
+ handleIR(After, PassID);
+}
+
+template <typename T>
+void OrderedChangedData<T>::report(
+ const OrderedChangedData &Before, const OrderedChangedData &After,
+ function_ref<void(const T *, const T *)> HandlePair) {
+ const auto &BFD = Before.getData();
+ const auto &AFD = After.getData();
+ std::vector<std::string>::const_iterator BI = Before.getOrder().begin();
+ std::vector<std::string>::const_iterator BE = Before.getOrder().end();
+ std::vector<std::string>::const_iterator AI = After.getOrder().begin();
+ std::vector<std::string>::const_iterator AE = After.getOrder().end();
+
+ auto HandlePotentiallyRemovedData = [&](std::string S) {
+ // The order in LLVM may have changed so check if still exists.
+ if (!AFD.count(S)) {
+ // This has been removed.
+ HandlePair(&BFD.find(*BI)->getValue(), nullptr);
+ }
+ };
+ auto HandleNewData = [&](std::vector<const T *> &Q) {
+ // Print out any queued up new sections
+ for (const T *NBI : Q)
+ HandlePair(nullptr, NBI);
+ Q.clear();
+ };
+
+ // Print out the data in the after order, with before ones interspersed
+ // appropriately (ie, somewhere near where they were in the before list).
+ // Start at the beginning of both lists. Loop through the
+ // after list. If an element is common, then advance in the before list
+ // reporting the removed ones until the common one is reached. Report any
+ // queued up new ones and then report the common one. If an element is not
+ // common, then enqueue it for reporting. When the after list is exhausted,
+ // loop through the before list, reporting any removed ones. Finally,
+ // report the rest of the enqueued new ones.
+ std::vector<const T *> NewDataQueue;
+ while (AI != AE) {
+ if (!BFD.count(*AI)) {
+ // This section is new so place it in the queue. This will cause it
+ // to be reported after deleted sections.
+ NewDataQueue.emplace_back(&AFD.find(*AI)->getValue());
+ ++AI;
+ continue;
+ }
+ // This section is in both; advance and print out any before-only
+ // until we get to it.
+ // It's possible that this section has moved to be later than before. This
+ // will mess up printing most blocks side by side, but it's a rare case and
+ // it's better than crashing.
+ while (BI != BE && *BI != *AI) {
+ HandlePotentiallyRemovedData(*BI);
+ ++BI;
+ }
+ // Report any new sections that were queued up and waiting.
+ HandleNewData(NewDataQueue);
+
+ const T &AData = AFD.find(*AI)->getValue();
+ const T &BData = BFD.find(*AI)->getValue();
+ HandlePair(&BData, &AData);
+ if (BI != BE)
+ ++BI;
+ ++AI;
+ }
+
+ // Check any remaining before sections to see if they have been removed
+ while (BI != BE) {
+ HandlePotentiallyRemovedData(*BI);
+ ++BI;
+ }
+
+ HandleNewData(NewDataQueue);
+}
+
+template <typename T>
+void IRComparer<T>::compare(
+ bool CompareModule,
+ std::function<void(bool InModule, unsigned Minor,
+ const FuncDataT<T> &Before, const FuncDataT<T> &After)>
+ CompareFunc) {
+ if (!CompareModule) {
+ // Just handle the single function.
+ assert(Before.getData().size() == 1 && After.getData().size() == 1 &&
+ "Expected only one function.");
+ CompareFunc(false, 0, Before.getData().begin()->getValue(),
+ After.getData().begin()->getValue());
+ return;
+ }
+
+ unsigned Minor = 0;
+ FuncDataT<T> Missing("");
+ IRDataT<T>::report(Before, After,
+ [&](const FuncDataT<T> *B, const FuncDataT<T> *A) {
+ assert((B || A) && "Both functions cannot be missing.");
+ if (!B)
+ B = &Missing;
+ else if (!A)
+ A = &Missing;
+ CompareFunc(true, Minor++, *B, *A);
+ });
+}
+
+template <typename T> void IRComparer<T>::analyzeIR(Any IR, IRDataT<T> &Data) {
+ if (const Module *M = getModuleForComparison(IR)) {
+ // Create data for each existing/interesting function in the module.
+ for (const Function &F : *M)
+ generateFunctionData(Data, F);
+ return;
+ }
+
+ const Function **FPtr = any_cast<const Function *>(&IR);
+ const Function *F = FPtr ? *FPtr : nullptr;
+ if (!F) {
+ const Loop **L = any_cast<const Loop *>(&IR);
+ assert(L && "Unknown IR unit.");
+ F = (*L)->getHeader()->getParent();
+ }
+ assert(F && "Unknown IR unit.");
+ generateFunctionData(Data, *F);
+}
+
+template <typename T>
+bool IRComparer<T>::generateFunctionData(IRDataT<T> &Data, const Function &F) {
+ if (!F.isDeclaration() && isFunctionInPrintList(F.getName())) {
+ FuncDataT<T> FD(F.getEntryBlock().getName().str());
+ int I = 0;
+ for (const auto &B : F) {
+ std::string BBName = B.getName().str();
+ if (BBName.empty()) {
+ BBName = formatv("{0}", I);
+ ++I;
+ }
+ FD.getOrder().emplace_back(BBName);
+ FD.getData().insert({BBName, B});
+ }
+ Data.getOrder().emplace_back(F.getName());
+ Data.getData().insert({F.getName(), FD});
+ return true;
+ }
+ return false;
+}
+
+PrintIRInstrumentation::~PrintIRInstrumentation() {
+ assert(ModuleDescStack.empty() && "ModuleDescStack is not empty at exit");
+}
+
+void PrintIRInstrumentation::pushModuleDesc(StringRef PassID, Any IR) {
+ const Module *M = unwrapModule(IR);
+ ModuleDescStack.emplace_back(M, getIRName(IR), PassID);
+}
+
+PrintIRInstrumentation::PrintModuleDesc
+PrintIRInstrumentation::popModuleDesc(StringRef PassID) {
+ assert(!ModuleDescStack.empty() && "empty ModuleDescStack");
+ PrintModuleDesc ModuleDesc = ModuleDescStack.pop_back_val();
+ assert(std::get<2>(ModuleDesc).equals(PassID) && "malformed ModuleDescStack");
+ return ModuleDesc;
+}
+
+void PrintIRInstrumentation::printBeforePass(StringRef PassID, Any IR) {
+ if (isIgnored(PassID))
+ return;
+
+ // Saving Module for AfterPassInvalidated operations.
+ // Note: here we rely on a fact that we do not change modules while
+ // traversing the pipeline, so the latest captured module is good
+ // for all print operations that has not happen yet.
+ if (shouldPrintAfterPass(PassID))
+ pushModuleDesc(PassID, IR);
+
+ if (!shouldPrintBeforePass(PassID))
+ return;
+
+ if (!shouldPrintIR(IR))
+ return;
+
+ dbgs() << "*** IR Dump Before " << PassID << " on " << getIRName(IR)
+ << " ***\n";
+ unwrapAndPrint(dbgs(), IR);
+}
+
+void PrintIRInstrumentation::printAfterPass(StringRef PassID, Any IR) {
+ if (isIgnored(PassID))
+ return;
+
+ if (!shouldPrintAfterPass(PassID))
+ return;
+
+ const Module *M;
+ std::string IRName;
+ StringRef StoredPassID;
+ std::tie(M, IRName, StoredPassID) = popModuleDesc(PassID);
+ assert(StoredPassID == PassID && "mismatched PassID");
+
+ if (!shouldPrintIR(IR))
+ return;
+
+ dbgs() << "*** IR Dump After " << PassID << " on " << IRName << " ***\n";
+ unwrapAndPrint(dbgs(), IR);
+}
+
+void PrintIRInstrumentation::printAfterPassInvalidated(StringRef PassID) {
+ StringRef PassName = PIC->getPassNameForClassName(PassID);
+ if (!shouldPrintAfterPass(PassName))
+ return;
+
+ if (isIgnored(PassID))
+ return;
+
+ const Module *M;
+ std::string IRName;
+ StringRef StoredPassID;
+ std::tie(M, IRName, StoredPassID) = popModuleDesc(PassID);
+ assert(StoredPassID == PassID && "mismatched PassID");
+ // Additional filtering (e.g. -filter-print-func) can lead to module
+ // printing being skipped.
+ if (!M)
+ return;
+
+ SmallString<20> Banner =
+ formatv("*** IR Dump After {0} on {1} (invalidated) ***", PassID, IRName);
+ dbgs() << Banner << "\n";
+ printIR(dbgs(), M);
+}
+
+bool PrintIRInstrumentation::shouldPrintBeforePass(StringRef PassID) {
+ if (shouldPrintBeforeAll())
+ return true;
+
+ StringRef PassName = PIC->getPassNameForClassName(PassID);
+ return is_contained(printBeforePasses(), PassName);
+}
+
+bool PrintIRInstrumentation::shouldPrintAfterPass(StringRef PassID) {
+ if (shouldPrintAfterAll())
+ return true;
+
+ StringRef PassName = PIC->getPassNameForClassName(PassID);
+ return is_contained(printAfterPasses(), PassName);
+}
+
+void PrintIRInstrumentation::registerCallbacks(
+ PassInstrumentationCallbacks &PIC) {
+ this->PIC = &PIC;
+
+ // BeforePass callback is not just for printing, it also saves a Module
+ // for later use in AfterPassInvalidated.
+ if (shouldPrintBeforeSomePass() || shouldPrintAfterSomePass())
+ PIC.registerBeforeNonSkippedPassCallback(
+ [this](StringRef P, Any IR) { this->printBeforePass(P, IR); });
+
+ if (shouldPrintAfterSomePass()) {
+ PIC.registerAfterPassCallback(
+ [this](StringRef P, Any IR, const PreservedAnalyses &) {
+ this->printAfterPass(P, IR);
+ });
+ PIC.registerAfterPassInvalidatedCallback(
+ [this](StringRef P, const PreservedAnalyses &) {
+ this->printAfterPassInvalidated(P);
+ });
+ }
+}
+
+void OptNoneInstrumentation::registerCallbacks(
+ PassInstrumentationCallbacks &PIC) {
+ PIC.registerShouldRunOptionalPassCallback(
+ [this](StringRef P, Any IR) { return this->shouldRun(P, IR); });
+}
+
+bool OptNoneInstrumentation::shouldRun(StringRef PassID, Any IR) {
+ const Function **FPtr = any_cast<const Function *>(&IR);
+ const Function *F = FPtr ? *FPtr : nullptr;
+ if (!F) {
+ if (const auto **L = any_cast<const Loop *>(&IR))
+ F = (*L)->getHeader()->getParent();
+ }
+ bool ShouldRun = !(F && F->hasOptNone());
+ if (!ShouldRun && DebugLogging) {
+ errs() << "Skipping pass " << PassID << " on " << F->getName()
+ << " due to optnone attribute\n";
+ }
+ return ShouldRun;
+}
+
+bool OptPassGateInstrumentation::shouldRun(StringRef PassName, Any IR) {
+ if (isIgnored(PassName))
+ return true;
+
+ bool ShouldRun =
+ Context.getOptPassGate().shouldRunPass(PassName, getIRName(IR));
+ if (!ShouldRun && !this->HasWrittenIR && !OptBisectPrintIRPath.empty()) {
+ // FIXME: print IR if limit is higher than number of opt-bisect
+ // invocations
+ this->HasWrittenIR = true;
+ const Module *M = unwrapModule(IR, /*Force=*/true);
+ assert((M && &M->getContext() == &Context) && "Missing/Mismatching Module");
+ std::error_code EC;
+ raw_fd_ostream OS(OptBisectPrintIRPath, EC);
+ if (EC)
+ report_fatal_error(errorCodeToError(EC));
+ M->print(OS, nullptr);
+ }
+ return ShouldRun;
+}
+
+void OptPassGateInstrumentation::registerCallbacks(
+ PassInstrumentationCallbacks &PIC) {
+ OptPassGate &PassGate = Context.getOptPassGate();
+ if (!PassGate.isEnabled())
+ return;
+
+ PIC.registerShouldRunOptionalPassCallback([this](StringRef PassName, Any IR) {
+ return this->shouldRun(PassName, IR);
+ });
+}
+
+raw_ostream &PrintPassInstrumentation::print() {
+ if (Opts.Indent) {
+ assert(Indent >= 0);
+ dbgs().indent(Indent);
+ }
+ return dbgs();
+}
+
+void PrintPassInstrumentation::registerCallbacks(
+ PassInstrumentationCallbacks &PIC) {
+ if (!Enabled)
+ return;
+
+ std::vector<StringRef> SpecialPasses;
+ if (!Opts.Verbose) {
+ SpecialPasses.emplace_back("PassManager");
+ SpecialPasses.emplace_back("PassAdaptor");
+ }
+
+ PIC.registerBeforeSkippedPassCallback([this, SpecialPasses](StringRef PassID,
+ Any IR) {
+ assert(!isSpecialPass(PassID, SpecialPasses) &&
+ "Unexpectedly skipping special pass");
+
+ print() << "Skipping pass: " << PassID << " on " << getIRName(IR) << "\n";
+ });
+ PIC.registerBeforeNonSkippedPassCallback([this, SpecialPasses](
+ StringRef PassID, Any IR) {
+ if (isSpecialPass(PassID, SpecialPasses))
+ return;
+
+ auto &OS = print();
+ OS << "Running pass: " << PassID << " on " << getIRName(IR);
+ if (const auto **F = any_cast<const Function *>(&IR)) {
+ unsigned Count = (*F)->getInstructionCount();
+ OS << " (" << Count << " instruction";
+ if (Count != 1)
+ OS << 's';
+ OS << ')';
+ } else if (const auto **C = any_cast<const LazyCallGraph::SCC *>(&IR)) {
+ int Count = (*C)->size();
+ OS << " (" << Count << " node";
+ if (Count != 1)
+ OS << 's';
+ OS << ')';
+ }
+ OS << "\n";
+ Indent += 2;
+ });
+ PIC.registerAfterPassCallback(
+ [this, SpecialPasses](StringRef PassID, Any IR,
+ const PreservedAnalyses &) {
+ if (isSpecialPass(PassID, SpecialPasses))
+ return;
+
+ Indent -= 2;
+ });
+ PIC.registerAfterPassInvalidatedCallback(
+ [this, SpecialPasses](StringRef PassID, Any IR) {
+ if (isSpecialPass(PassID, SpecialPasses))
+ return;
+
+ Indent -= 2;
+ });
+
+ if (!Opts.SkipAnalyses) {
+ PIC.registerBeforeAnalysisCallback([this](StringRef PassID, Any IR) {
+ print() << "Running analysis: " << PassID << " on " << getIRName(IR)
+ << "\n";
+ Indent += 2;
+ });
+ PIC.registerAfterAnalysisCallback(
+ [this](StringRef PassID, Any IR) { Indent -= 2; });
+ PIC.registerAnalysisInvalidatedCallback([this](StringRef PassID, Any IR) {
+ print() << "Invalidating analysis: " << PassID << " on " << getIRName(IR)
+ << "\n";
+ });
+ PIC.registerAnalysesClearedCallback([this](StringRef IRName) {
+ print() << "Clearing all analysis results for: " << IRName << "\n";
+ });
+ }
+}
+
+PreservedCFGCheckerInstrumentation::CFG::CFG(const Function *F,
+ bool TrackBBLifetime) {
+ if (TrackBBLifetime)
+ BBGuards = DenseMap<intptr_t, BBGuard>(F->size());
+ for (const auto &BB : *F) {
+ if (BBGuards)
+ BBGuards->try_emplace(intptr_t(&BB), &BB);
+ for (const auto *Succ : successors(&BB)) {
+ Graph[&BB][Succ]++;
+ if (BBGuards)
+ BBGuards->try_emplace(intptr_t(Succ), Succ);
+ }
+ }
+}
+
+static void printBBName(raw_ostream &out, const BasicBlock *BB) {
+ if (BB->hasName()) {
+ out << BB->getName() << "<" << BB << ">";
+ return;
+ }
+
+ if (!BB->getParent()) {
+ out << "unnamed_removed<" << BB << ">";
+ return;
+ }
+
+ if (BB->isEntryBlock()) {
+ out << "entry"
+ << "<" << BB << ">";
+ return;
+ }
+
+ unsigned FuncOrderBlockNum = 0;
+ for (auto &FuncBB : *BB->getParent()) {
+ if (&FuncBB == BB)
+ break;
+ FuncOrderBlockNum++;
+ }
+ out << "unnamed_" << FuncOrderBlockNum << "<" << BB << ">";
+}
+
+void PreservedCFGCheckerInstrumentation::CFG::printDiff(raw_ostream &out,
+ const CFG &Before,
+ const CFG &After) {
+ assert(!After.isPoisoned());
+ if (Before.isPoisoned()) {
+ out << "Some blocks were deleted\n";
+ return;
+ }
+
+ // Find and print graph differences.
+ if (Before.Graph.size() != After.Graph.size())
+ out << "Different number of non-leaf basic blocks: before="
+ << Before.Graph.size() << ", after=" << After.Graph.size() << "\n";
+
+ for (auto &BB : Before.Graph) {
+ auto BA = After.Graph.find(BB.first);
+ if (BA == After.Graph.end()) {
+ out << "Non-leaf block ";
+ printBBName(out, BB.first);
+ out << " is removed (" << BB.second.size() << " successors)\n";
+ }
+ }
+
+ for (auto &BA : After.Graph) {
+ auto BB = Before.Graph.find(BA.first);
+ if (BB == Before.Graph.end()) {
+ out << "Non-leaf block ";
+ printBBName(out, BA.first);
+ out << " is added (" << BA.second.size() << " successors)\n";
+ continue;
+ }
+
+ if (BB->second == BA.second)
+ continue;
+
+ out << "Different successors of block ";
+ printBBName(out, BA.first);
+ out << " (unordered):\n";
+ out << "- before (" << BB->second.size() << "): ";
+ for (auto &SuccB : BB->second) {
+ printBBName(out, SuccB.first);
+ if (SuccB.second != 1)
+ out << "(" << SuccB.second << "), ";
+ else
+ out << ", ";
+ }
+ out << "\n";
+ out << "- after (" << BA.second.size() << "): ";
+ for (auto &SuccA : BA.second) {
+ printBBName(out, SuccA.first);
+ if (SuccA.second != 1)
+ out << "(" << SuccA.second << "), ";
+ else
+ out << ", ";
+ }
+ out << "\n";
+ }
+}
+
+// PreservedCFGCheckerInstrumentation uses PreservedCFGCheckerAnalysis to check
+// passes, that reported they kept CFG analyses up-to-date, did not actually
+// change CFG. This check is done as follows. Before every functional pass in
+// BeforeNonSkippedPassCallback a CFG snapshot (an instance of
+// PreservedCFGCheckerInstrumentation::CFG) is requested from
+// FunctionAnalysisManager as a result of PreservedCFGCheckerAnalysis. When the
+// functional pass finishes and reports that CFGAnalyses or AllAnalyses are
+// up-to-date then the cached result of PreservedCFGCheckerAnalysis (if
+// available) is checked to be equal to a freshly created CFG snapshot.
+struct PreservedCFGCheckerAnalysis
+ : public AnalysisInfoMixin<PreservedCFGCheckerAnalysis> {
+ friend AnalysisInfoMixin<PreservedCFGCheckerAnalysis>;
+
+ static AnalysisKey Key;
+
+public:
+ /// Provide the result type for this analysis pass.
+ using Result = PreservedCFGCheckerInstrumentation::CFG;
+
+ /// Run the analysis pass over a function and produce CFG.
+ Result run(Function &F, FunctionAnalysisManager &FAM) {
+ return Result(&F, /* TrackBBLifetime */ true);
+ }
+};
+
+AnalysisKey PreservedCFGCheckerAnalysis::Key;
+
+bool PreservedCFGCheckerInstrumentation::CFG::invalidate(
+ Function &F, const PreservedAnalyses &PA,
+ FunctionAnalysisManager::Invalidator &) {
+ auto PAC = PA.getChecker<PreservedCFGCheckerAnalysis>();
+ return !(PAC.preserved() || PAC.preservedSet<AllAnalysesOn<Function>>() ||
+ PAC.preservedSet<CFGAnalyses>());
+}
+
+void PreservedCFGCheckerInstrumentation::registerCallbacks(
+ PassInstrumentationCallbacks &PIC, FunctionAnalysisManager &FAM) {
+ if (!VerifyPreservedCFG)
+ return;
+
+ FAM.registerPass([&] { return PreservedCFGCheckerAnalysis(); });
+
+ auto checkCFG = [](StringRef Pass, StringRef FuncName, const CFG &GraphBefore,
+ const CFG &GraphAfter) {
+ if (GraphAfter == GraphBefore)
+ return;
+
+ dbgs() << "Error: " << Pass
+ << " does not invalidate CFG analyses but CFG changes detected in "
+ "function @"
+ << FuncName << ":\n";
+ CFG::printDiff(dbgs(), GraphBefore, GraphAfter);
+ report_fatal_error(Twine("CFG unexpectedly changed by ", Pass));
+ };
+
+ PIC.registerBeforeNonSkippedPassCallback(
+ [this, &FAM](StringRef P, Any IR) {
+#ifdef LLVM_ENABLE_ABI_BREAKING_CHECKS
+ assert(&PassStack.emplace_back(P));
+#endif
+ (void)this;
+ const auto **F = any_cast<const Function *>(&IR);
+ if (!F)
+ return;
+
+ // Make sure a fresh CFG snapshot is available before the pass.
+ FAM.getResult<PreservedCFGCheckerAnalysis>(*const_cast<Function *>(*F));
+ });
+
+ PIC.registerAfterPassInvalidatedCallback(
+ [this](StringRef P, const PreservedAnalyses &PassPA) {
+#ifdef LLVM_ENABLE_ABI_BREAKING_CHECKS
+ assert(PassStack.pop_back_val() == P &&
+ "Before and After callbacks must correspond");
+#endif
+ (void)this;
+ });
+
+ PIC.registerAfterPassCallback([this, &FAM,
+ checkCFG](StringRef P, Any IR,
+ const PreservedAnalyses &PassPA) {
+#ifdef LLVM_ENABLE_ABI_BREAKING_CHECKS
+ assert(PassStack.pop_back_val() == P &&
+ "Before and After callbacks must correspond");
+#endif
+ (void)this;
+
+ const auto **F = any_cast<const Function *>(&IR);
+ if (!F)
+ return;
+
+ if (!PassPA.allAnalysesInSetPreserved<CFGAnalyses>() &&
+ !PassPA.allAnalysesInSetPreserved<AllAnalysesOn<Function>>())
+ return;
+
+ if (auto *GraphBefore = FAM.getCachedResult<PreservedCFGCheckerAnalysis>(
+ *const_cast<Function *>(*F)))
+ checkCFG(P, (*F)->getName(), *GraphBefore,
+ CFG(*F, /* TrackBBLifetime */ false));
+ });
+}
+
+void VerifyInstrumentation::registerCallbacks(
+ PassInstrumentationCallbacks &PIC) {
+ PIC.registerAfterPassCallback(
+ [this](StringRef P, Any IR, const PreservedAnalyses &PassPA) {
+ if (isIgnored(P) || P == "VerifierPass")
+ return;
+ const Function **FPtr = any_cast<const Function *>(&IR);
+ const Function *F = FPtr ? *FPtr : nullptr;
+ if (!F) {
+ if (const auto **L = any_cast<const Loop *>(&IR))
+ F = (*L)->getHeader()->getParent();
+ }
+
+ if (F) {
+ if (DebugLogging)
+ dbgs() << "Verifying function " << F->getName() << "\n";
+
+ if (verifyFunction(*F, &errs()))
+ report_fatal_error("Broken function found, compilation aborted!");
+ } else {
+ const Module **MPtr = any_cast<const Module *>(&IR);
+ const Module *M = MPtr ? *MPtr : nullptr;
+ if (!M) {
+ if (const auto **C = any_cast<const LazyCallGraph::SCC *>(&IR))
+ M = (*C)->begin()->getFunction().getParent();
+ }
+
+ if (M) {
+ if (DebugLogging)
+ dbgs() << "Verifying module " << M->getName() << "\n";
+
+ if (verifyModule(*M, &errs()))
+ report_fatal_error("Broken module found, compilation aborted!");
+ }
+ }
+ });
+}
+
+InLineChangePrinter::~InLineChangePrinter() = default;
+
+void InLineChangePrinter::generateIRRepresentation(Any IR,
+ StringRef PassID,
+ IRDataT<EmptyData> &D) {
+ IRComparer<EmptyData>::analyzeIR(IR, D);
+}
+
+void InLineChangePrinter::handleAfter(StringRef PassID, std::string &Name,
+ const IRDataT<EmptyData> &Before,
+ const IRDataT<EmptyData> &After,
+ Any IR) {
+ SmallString<20> Banner =
+ formatv("*** IR Dump After {0} on {1} ***\n", PassID, Name);
+ Out << Banner;
+ IRComparer<EmptyData>(Before, After)
+ .compare(getModuleForComparison(IR),
+ [&](bool InModule, unsigned Minor,
+ const FuncDataT<EmptyData> &Before,
+ const FuncDataT<EmptyData> &After) -> void {
+ handleFunctionCompare(Name, "", PassID, " on ", InModule,
+ Minor, Before, After);
+ });
+ Out << "\n";
+}
+
+void InLineChangePrinter::handleFunctionCompare(
+ StringRef Name, StringRef Prefix, StringRef PassID, StringRef Divider,
+ bool InModule, unsigned Minor, const FuncDataT<EmptyData> &Before,
+ const FuncDataT<EmptyData> &After) {
+ // Print a banner when this is being shown in the context of a module
+ if (InModule)
+ Out << "\n*** IR for function " << Name << " ***\n";
+
+ FuncDataT<EmptyData>::report(
+ Before, After,
+ [&](const BlockDataT<EmptyData> *B, const BlockDataT<EmptyData> *A) {
+ StringRef BStr = B ? B->getBody() : "\n";
+ StringRef AStr = A ? A->getBody() : "\n";
+ const std::string Removed =
+ UseColour ? "\033[31m-%l\033[0m\n" : "-%l\n";
+ const std::string Added = UseColour ? "\033[32m+%l\033[0m\n" : "+%l\n";
+ const std::string NoChange = " %l\n";
+ Out << doSystemDiff(BStr, AStr, Removed, Added, NoChange);
+ });
+}
+
+void InLineChangePrinter::registerCallbacks(PassInstrumentationCallbacks &PIC) {
+ if (PrintChanged == ChangePrinter::DiffVerbose ||
+ PrintChanged == ChangePrinter::DiffQuiet ||
+ PrintChanged == ChangePrinter::ColourDiffVerbose ||
+ PrintChanged == ChangePrinter::ColourDiffQuiet)
+ TextChangeReporter<IRDataT<EmptyData>>::registerRequiredCallbacks(PIC);
+}
+
+TimeProfilingPassesHandler::TimeProfilingPassesHandler() {}
+
+void TimeProfilingPassesHandler::registerCallbacks(
+ PassInstrumentationCallbacks &PIC) {
+ if (!getTimeTraceProfilerInstance())
+ return;
+ PIC.registerBeforeNonSkippedPassCallback(
+ [this](StringRef P, Any IR) { this->runBeforePass(P, IR); });
+ PIC.registerAfterPassCallback(
+ [this](StringRef P, Any IR, const PreservedAnalyses &) {
+ this->runAfterPass();
+ },
+ true);
+ PIC.registerAfterPassInvalidatedCallback(
+ [this](StringRef P, const PreservedAnalyses &) { this->runAfterPass(); },
+ true);
+ PIC.registerBeforeAnalysisCallback(
+ [this](StringRef P, Any IR) { this->runBeforePass(P, IR); });
+ PIC.registerAfterAnalysisCallback(
+ [this](StringRef P, Any IR) { this->runAfterPass(); }, true);
+}
+
+void TimeProfilingPassesHandler::runBeforePass(StringRef PassID, Any IR) {
+ timeTraceProfilerBegin(PassID, getIRName(IR));
+}
+
+void TimeProfilingPassesHandler::runAfterPass() { timeTraceProfilerEnd(); }
+
+namespace {
+
+class DisplayNode;
+class DotCfgDiffDisplayGraph;
+
+// Base class for a node or edge in the dot-cfg-changes graph.
+class DisplayElement {
+public:
+ // Is this in before, after, or both?
+ StringRef getColour() const { return Colour; }
+
+protected:
+ DisplayElement(StringRef Colour) : Colour(Colour) {}
+ const StringRef Colour;
+};
+
+// An edge representing a transition between basic blocks in the
+// dot-cfg-changes graph.
+class DisplayEdge : public DisplayElement {
+public:
+ DisplayEdge(std::string Value, DisplayNode &Node, StringRef Colour)
+ : DisplayElement(Colour), Value(Value), Node(Node) {}
+ // The value on which the transition is made.
+ std::string getValue() const { return Value; }
+ // The node (representing a basic block) reached by this transition.
+ const DisplayNode &getDestinationNode() const { return Node; }
+
+protected:
+ std::string Value;
+ const DisplayNode &Node;
+};
+
+// A node in the dot-cfg-changes graph which represents a basic block.
+class DisplayNode : public DisplayElement {
+public:
+ // \p C is the content for the node, \p T indicates the colour for the
+ // outline of the node
+ DisplayNode(std::string Content, StringRef Colour)
+ : DisplayElement(Colour), Content(Content) {}
+
+ // Iterator to the child nodes. Required by GraphWriter.
+ using ChildIterator = std::unordered_set<DisplayNode *>::const_iterator;
+ ChildIterator children_begin() const { return Children.cbegin(); }
+ ChildIterator children_end() const { return Children.cend(); }
+
+ // Iterator for the edges. Required by GraphWriter.
+ using EdgeIterator = std::vector<DisplayEdge *>::const_iterator;
+ EdgeIterator edges_begin() const { return EdgePtrs.cbegin(); }
+ EdgeIterator edges_end() const { return EdgePtrs.cend(); }
+
+ // Create an edge to \p Node on value \p Value, with colour \p Colour.
+ void createEdge(StringRef Value, DisplayNode &Node, StringRef Colour);
+
+ // Return the content of this node.
+ std::string getContent() const { return Content; }
+
+ // Return the edge to node \p S.
+ const DisplayEdge &getEdge(const DisplayNode &To) const {
+ assert(EdgeMap.find(&To) != EdgeMap.end() && "Expected to find edge.");
+ return *EdgeMap.find(&To)->second;
+ }
+
+ // Return the value for the transition to basic block \p S.
+ // Required by GraphWriter.
+ std::string getEdgeSourceLabel(const DisplayNode &Sink) const {
+ return getEdge(Sink).getValue();
+ }
+
+ void createEdgeMap();
+
+protected:
+ const std::string Content;
+
+ // Place to collect all of the edges. Once they are all in the vector,
+ // the vector will not reallocate so then we can use pointers to them,
+ // which are required by the graph writing routines.
+ std::vector<DisplayEdge> Edges;
+
+ std::vector<DisplayEdge *> EdgePtrs;
+ std::unordered_set<DisplayNode *> Children;
+ std::unordered_map<const DisplayNode *, const DisplayEdge *> EdgeMap;
+
+ // Safeguard adding of edges.
+ bool AllEdgesCreated = false;
+};
+
+// Class representing a difference display (corresponds to a pdf file).
+class DotCfgDiffDisplayGraph {
+public:
+ DotCfgDiffDisplayGraph(std::string Name) : GraphName(Name) {}
+
+ // Generate the file into \p DotFile.
+ void generateDotFile(StringRef DotFile);
+
+ // Iterator to the nodes. Required by GraphWriter.
+ using NodeIterator = std::vector<DisplayNode *>::const_iterator;
+ NodeIterator nodes_begin() const {
+ assert(NodeGenerationComplete && "Unexpected children iterator creation");
+ return NodePtrs.cbegin();
+ }
+ NodeIterator nodes_end() const {
+ assert(NodeGenerationComplete && "Unexpected children iterator creation");
+ return NodePtrs.cend();
+ }
+
+ // Record the index of the entry node. At this point, we can build up
+ // vectors of pointers that are required by the graph routines.
+ void setEntryNode(unsigned N) {
+ // At this point, there will be no new nodes.
+ assert(!NodeGenerationComplete && "Unexpected node creation");
+ NodeGenerationComplete = true;
+ for (auto &N : Nodes)
+ NodePtrs.emplace_back(&N);
+
+ EntryNode = NodePtrs[N];
+ }
+
+ // Create a node.
+ void createNode(std::string C, StringRef Colour) {
+ assert(!NodeGenerationComplete && "Unexpected node creation");
+ Nodes.emplace_back(C, Colour);
+ }
+ // Return the node at index \p N to avoid problems with vectors reallocating.
+ DisplayNode &getNode(unsigned N) {
+ assert(N < Nodes.size() && "Node is out of bounds");
+ return Nodes[N];
+ }
+ unsigned size() const {
+ assert(NodeGenerationComplete && "Unexpected children iterator creation");
+ return Nodes.size();
+ }
+
+ // Return the name of the graph. Required by GraphWriter.
+ std::string getGraphName() const { return GraphName; }
+
+ // Return the string representing the differences for basic block \p Node.
+ // Required by GraphWriter.
+ std::string getNodeLabel(const DisplayNode &Node) const {
+ return Node.getContent();
+ }
+
+ // Return a string with colour information for Dot. Required by GraphWriter.
+ std::string getNodeAttributes(const DisplayNode &Node) const {
+ return attribute(Node.getColour());
+ }
+
+ // Return a string with colour information for Dot. Required by GraphWriter.
+ std::string getEdgeColorAttr(const DisplayNode &From,
+ const DisplayNode &To) const {
+ return attribute(From.getEdge(To).getColour());
+ }
+
+ // Get the starting basic block. Required by GraphWriter.
+ DisplayNode *getEntryNode() const {
+ assert(NodeGenerationComplete && "Unexpected children iterator creation");
+ return EntryNode;
+ }
+
+protected:
+ // Return the string containing the colour to use as a Dot attribute.
+ std::string attribute(StringRef Colour) const {
+ return "color=" + Colour.str();
+ }
+
+ bool NodeGenerationComplete = false;
+ const std::string GraphName;
+ std::vector<DisplayNode> Nodes;
+ std::vector<DisplayNode *> NodePtrs;
+ DisplayNode *EntryNode = nullptr;
+};
+
+void DisplayNode::createEdge(StringRef Value, DisplayNode &Node,
+ StringRef Colour) {
+ assert(!AllEdgesCreated && "Expected to be able to still create edges.");
+ Edges.emplace_back(Value.str(), Node, Colour);
+ Children.insert(&Node);
+}
+
+void DisplayNode::createEdgeMap() {
+ // No more edges will be added so we can now use pointers to the edges
+ // as the vector will not grow and reallocate.
+ AllEdgesCreated = true;
+ for (auto &E : Edges)
+ EdgeMap.insert({&E.getDestinationNode(), &E});
+}
+
+class DotCfgDiffNode;
+class DotCfgDiff;
+
+// A class representing a basic block in the Dot difference graph.
+class DotCfgDiffNode {
+public:
+ DotCfgDiffNode() = delete;
+
+ // Create a node in Dot difference graph \p G representing the basic block
+ // represented by \p BD with colour \p Colour (where it exists).
+ DotCfgDiffNode(DotCfgDiff &G, unsigned N, const BlockDataT<DCData> &BD,
+ StringRef Colour)
+ : Graph(G), N(N), Data{&BD, nullptr}, Colour(Colour) {}
+ DotCfgDiffNode(const DotCfgDiffNode &DN)
+ : Graph(DN.Graph), N(DN.N), Data{DN.Data[0], DN.Data[1]},
+ Colour(DN.Colour), EdgesMap(DN.EdgesMap), Children(DN.Children),
+ Edges(DN.Edges) {}
+
+ unsigned getIndex() const { return N; }
+
+ // The label of the basic block
+ StringRef getLabel() const {
+ assert(Data[0] && "Expected Data[0] to be set.");
+ return Data[0]->getLabel();
+ }
+ // Return the colour for this block
+ StringRef getColour() const { return Colour; }
+ // Change this basic block from being only in before to being common.
+ // Save the pointer to \p Other.
+ void setCommon(const BlockDataT<DCData> &Other) {
+ assert(!Data[1] && "Expected only one block datum");
+ Data[1] = &Other;
+ Colour = CommonColour;
+ }
+ // Add an edge to \p E of colour {\p Value, \p Colour}.
+ void addEdge(unsigned E, StringRef Value, StringRef Colour) {
+ // This is a new edge or it is an edge being made common.
+ assert((EdgesMap.count(E) == 0 || Colour == CommonColour) &&
+ "Unexpected edge count and color.");
+ EdgesMap[E] = {Value.str(), Colour};
+ }
+ // Record the children and create edges.
+ void finalize(DotCfgDiff &G);
+
+ // Return the colour of the edge to node \p S.
+ StringRef getEdgeColour(const unsigned S) const {
+ assert(EdgesMap.count(S) == 1 && "Expected to find edge.");
+ return EdgesMap.at(S).second;
+ }
+
+ // Return the string representing the basic block.
+ std::string getBodyContent() const;
+
+ void createDisplayEdges(DotCfgDiffDisplayGraph &Graph, unsigned DisplayNode,
+ std::map<const unsigned, unsigned> &NodeMap) const;
+
+protected:
+ DotCfgDiff &Graph;
+ const unsigned N;
+ const BlockDataT<DCData> *Data[2];
+ StringRef Colour;
+ std::map<const unsigned, std::pair<std::string, StringRef>> EdgesMap;
+ std::vector<unsigned> Children;
+ std::vector<unsigned> Edges;
+};
+
+// Class representing the difference graph between two functions.
+class DotCfgDiff {
+public:
+ // \p Title is the title given to the graph. \p EntryNodeName is the
+ // entry node for the function. \p Before and \p After are the before
+ // after versions of the function, respectively. \p Dir is the directory
+ // in which to store the results.
+ DotCfgDiff(StringRef Title, const FuncDataT<DCData> &Before,
+ const FuncDataT<DCData> &After);
+
+ DotCfgDiff(const DotCfgDiff &) = delete;
+ DotCfgDiff &operator=(const DotCfgDiff &) = delete;
+
+ DotCfgDiffDisplayGraph createDisplayGraph(StringRef Title,
+ StringRef EntryNodeName);
+
+ // Return a string consisting of the labels for the \p Source and \p Sink.
+ // The combination allows distinguishing changing transitions on the
+ // same value (ie, a transition went to X before and goes to Y after).
+ // Required by GraphWriter.
+ StringRef getEdgeSourceLabel(const unsigned &Source,
+ const unsigned &Sink) const {
+ std::string S =
+ getNode(Source).getLabel().str() + " " + getNode(Sink).getLabel().str();
+ assert(EdgeLabels.count(S) == 1 && "Expected to find edge label.");
+ return EdgeLabels.find(S)->getValue();
+ }
+
+ // Return the number of basic blocks (nodes). Required by GraphWriter.
+ unsigned size() const { return Nodes.size(); }
+
+ const DotCfgDiffNode &getNode(unsigned N) const {
+ assert(N < Nodes.size() && "Unexpected index for node reference");
+ return Nodes[N];
+ }
+
+protected:
+ // Return the string surrounded by HTML to make it the appropriate colour.
+ std::string colourize(std::string S, StringRef Colour) const;
+
+ void createNode(StringRef Label, const BlockDataT<DCData> &BD, StringRef C) {
+ unsigned Pos = Nodes.size();
+ Nodes.emplace_back(*this, Pos, BD, C);
+ NodePosition.insert({Label, Pos});
+ }
+
+ // TODO Nodes should probably be a StringMap<DotCfgDiffNode> after the
+ // display graph is separated out, which would remove the need for
+ // NodePosition.
+ std::vector<DotCfgDiffNode> Nodes;
+ StringMap<unsigned> NodePosition;
+ const std::string GraphName;
+
+ StringMap<std::string> EdgeLabels;
+};
+
+std::string DotCfgDiffNode::getBodyContent() const {
+ if (Colour == CommonColour) {
+ assert(Data[1] && "Expected Data[1] to be set.");
+
+ StringRef SR[2];
+ for (unsigned I = 0; I < 2; ++I) {
+ SR[I] = Data[I]->getBody();
+ // drop initial '\n' if present
+ if (SR[I][0] == '\n')
+ SR[I] = SR[I].drop_front();
+ // drop predecessors as they can be big and are redundant
+ SR[I] = SR[I].drop_until([](char C) { return C == '\n'; }).drop_front();
+ }
+
+ SmallString<80> OldLineFormat = formatv(
+ "<FONT COLOR=\"{0}\">%l</FONT><BR align=\"left\"/>", BeforeColour);
+ SmallString<80> NewLineFormat = formatv(
+ "<FONT COLOR=\"{0}\">%l</FONT><BR align=\"left\"/>", AfterColour);
+ SmallString<80> UnchangedLineFormat = formatv(
+ "<FONT COLOR=\"{0}\">%l</FONT><BR align=\"left\"/>", CommonColour);
+ std::string Diff = Data[0]->getLabel().str();
+ Diff += ":\n<BR align=\"left\"/>" +
+ doSystemDiff(makeHTMLReady(SR[0]), makeHTMLReady(SR[1]),
+ OldLineFormat, NewLineFormat, UnchangedLineFormat);
+
+ // Diff adds in some empty colour changes which are not valid HTML
+ // so remove them. Colours are all lowercase alpha characters (as
+ // listed in https://graphviz.org/pdf/dotguide.pdf).
+ Regex R("<FONT COLOR=\"\\w+\"></FONT>");
+ while (true) {
+ std::string Error;
+ std::string S = R.sub("", Diff, &Error);
+ if (Error != "")
+ return Error;
+ if (S == Diff)
+ return Diff;
+ Diff = S;
+ }
+ llvm_unreachable("Should not get here");
+ }
+
+ // Put node out in the appropriate colour.
+ assert(!Data[1] && "Data[1] is set unexpectedly.");
+ std::string Body = makeHTMLReady(Data[0]->getBody());
+ const StringRef BS = Body;
+ StringRef BS1 = BS;
+ // Drop leading newline, if present.
+ if (BS.front() == '\n')
+ BS1 = BS1.drop_front(1);
+ // Get label.
+ StringRef Label = BS1.take_until([](char C) { return C == ':'; });
+ // drop predecessors as they can be big and are redundant
+ BS1 = BS1.drop_until([](char C) { return C == '\n'; }).drop_front();
+
+ std::string S = "<FONT COLOR=\"" + Colour.str() + "\">" + Label.str() + ":";
+
+ // align each line to the left.
+ while (BS1.size()) {
+ S.append("<BR align=\"left\"/>");
+ StringRef Line = BS1.take_until([](char C) { return C == '\n'; });
+ S.append(Line.str());
+ BS1 = BS1.drop_front(Line.size() + 1);
+ }
+ S.append("<BR align=\"left\"/></FONT>");
+ return S;
+}
+
+std::string DotCfgDiff::colourize(std::string S, StringRef Colour) const {
+ if (S.length() == 0)
+ return S;
+ return "<FONT COLOR=\"" + Colour.str() + "\">" + S + "</FONT>";
+}
+
+DotCfgDiff::DotCfgDiff(StringRef Title, const FuncDataT<DCData> &Before,
+ const FuncDataT<DCData> &After)
+ : GraphName(Title.str()) {
+ StringMap<StringRef> EdgesMap;
+
+ // Handle each basic block in the before IR.
+ for (auto &B : Before.getData()) {
+ StringRef Label = B.getKey();
+ const BlockDataT<DCData> &BD = B.getValue();
+ createNode(Label, BD, BeforeColour);
+
+ // Create transitions with names made up of the from block label, the value
+ // on which the transition is made and the to block label.
+ for (StringMap<std::string>::const_iterator Sink = BD.getData().begin(),
+ E = BD.getData().end();
+ Sink != E; ++Sink) {
+ std::string Key = (Label + " " + Sink->getKey().str()).str() + " " +
+ BD.getData().getSuccessorLabel(Sink->getKey()).str();
+ EdgesMap.insert({Key, BeforeColour});
+ }
+ }
+
+ // Handle each basic block in the after IR
+ for (auto &A : After.getData()) {
+ StringRef Label = A.getKey();
+ const BlockDataT<DCData> &BD = A.getValue();
+ unsigned C = NodePosition.count(Label);
+ if (C == 0)
+ // This only exists in the after IR. Create the node.
+ createNode(Label, BD, AfterColour);
+ else {
+ assert(C == 1 && "Unexpected multiple nodes.");
+ Nodes[NodePosition[Label]].setCommon(BD);
+ }
+ // Add in the edges between the nodes (as common or only in after).
+ for (StringMap<std::string>::const_iterator Sink = BD.getData().begin(),
+ E = BD.getData().end();
+ Sink != E; ++Sink) {
+ std::string Key = (Label + " " + Sink->getKey().str()).str() + " " +
+ BD.getData().getSuccessorLabel(Sink->getKey()).str();
+ unsigned C = EdgesMap.count(Key);
+ if (C == 0)
+ EdgesMap.insert({Key, AfterColour});
+ else {
+ EdgesMap[Key] = CommonColour;
+ }
+ }
+ }
+
+ // Now go through the map of edges and add them to the node.
+ for (auto &E : EdgesMap) {
+ // Extract the source, sink and value from the edge key.
+ StringRef S = E.getKey();
+ auto SP1 = S.rsplit(' ');
+ auto &SourceSink = SP1.first;
+ auto SP2 = SourceSink.split(' ');
+ StringRef Source = SP2.first;
+ StringRef Sink = SP2.second;
+ StringRef Value = SP1.second;
+
+ assert(NodePosition.count(Source) == 1 && "Expected to find node.");
+ DotCfgDiffNode &SourceNode = Nodes[NodePosition[Source]];
+ assert(NodePosition.count(Sink) == 1 && "Expected to find node.");
+ unsigned SinkNode = NodePosition[Sink];
+ StringRef Colour = E.second;
+
+ // Look for an edge from Source to Sink
+ if (EdgeLabels.count(SourceSink) == 0)
+ EdgeLabels.insert({SourceSink, colourize(Value.str(), Colour)});
+ else {
+ StringRef V = EdgeLabels.find(SourceSink)->getValue();
+ std::string NV = colourize(V.str() + " " + Value.str(), Colour);
+ Colour = CommonColour;
+ EdgeLabels[SourceSink] = NV;
+ }
+ SourceNode.addEdge(SinkNode, Value, Colour);
+ }
+ for (auto &I : Nodes)
+ I.finalize(*this);
+}
+
+DotCfgDiffDisplayGraph DotCfgDiff::createDisplayGraph(StringRef Title,
+ StringRef EntryNodeName) {
+ assert(NodePosition.count(EntryNodeName) == 1 &&
+ "Expected to find entry block in map.");
+ unsigned Entry = NodePosition[EntryNodeName];
+ assert(Entry < Nodes.size() && "Expected to find entry node");
+ DotCfgDiffDisplayGraph G(Title.str());
+
+ std::map<const unsigned, unsigned> NodeMap;
+
+ int EntryIndex = -1;
+ unsigned Index = 0;
+ for (auto &I : Nodes) {
+ if (I.getIndex() == Entry)
+ EntryIndex = Index;
+ G.createNode(I.getBodyContent(), I.getColour());
+ NodeMap.insert({I.getIndex(), Index++});
+ }
+ assert(EntryIndex >= 0 && "Expected entry node index to be set.");
+ G.setEntryNode(EntryIndex);
+
+ for (auto &I : NodeMap) {
+ unsigned SourceNode = I.first;
+ unsigned DisplayNode = I.second;
+ getNode(SourceNode).createDisplayEdges(G, DisplayNode, NodeMap);
+ }
+ return G;
+}
+
+void DotCfgDiffNode::createDisplayEdges(
+ DotCfgDiffDisplayGraph &DisplayGraph, unsigned DisplayNodeIndex,
+ std::map<const unsigned, unsigned> &NodeMap) const {
+
+ DisplayNode &SourceDisplayNode = DisplayGraph.getNode(DisplayNodeIndex);
+
+ for (auto I : Edges) {
+ unsigned SinkNodeIndex = I;
+ StringRef Colour = getEdgeColour(SinkNodeIndex);
+ const DotCfgDiffNode *SinkNode = &Graph.getNode(SinkNodeIndex);
+
+ StringRef Label = Graph.getEdgeSourceLabel(getIndex(), SinkNodeIndex);
+ DisplayNode &SinkDisplayNode = DisplayGraph.getNode(SinkNode->getIndex());
+ SourceDisplayNode.createEdge(Label, SinkDisplayNode, Colour);
+ }
+ SourceDisplayNode.createEdgeMap();
+}
+
+void DotCfgDiffNode::finalize(DotCfgDiff &G) {
+ for (auto E : EdgesMap) {
+ Children.emplace_back(E.first);
+ Edges.emplace_back(E.first);
+ }
+}
+
+} // namespace
+
+namespace llvm {
+
+template <> struct GraphTraits<DotCfgDiffDisplayGraph *> {
+ using NodeRef = const DisplayNode *;
+ using ChildIteratorType = DisplayNode::ChildIterator;
+ using nodes_iterator = DotCfgDiffDisplayGraph::NodeIterator;
+ using EdgeRef = const DisplayEdge *;
+ using ChildEdgeIterator = DisplayNode::EdgeIterator;
+
+ static NodeRef getEntryNode(const DotCfgDiffDisplayGraph *G) {
+ return G->getEntryNode();
+ }
+ static ChildIteratorType child_begin(NodeRef N) {
+ return N->children_begin();
+ }
+ static ChildIteratorType child_end(NodeRef N) { return N->children_end(); }
+ static nodes_iterator nodes_begin(const DotCfgDiffDisplayGraph *G) {
+ return G->nodes_begin();
+ }
+ static nodes_iterator nodes_end(const DotCfgDiffDisplayGraph *G) {
+ return G->nodes_end();
+ }
+ static ChildEdgeIterator child_edge_begin(NodeRef N) {
+ return N->edges_begin();
+ }
+ static ChildEdgeIterator child_edge_end(NodeRef N) { return N->edges_end(); }
+ static NodeRef edge_dest(EdgeRef E) { return &E->getDestinationNode(); }
+ static unsigned size(const DotCfgDiffDisplayGraph *G) { return G->size(); }
+};
+
+template <>
+struct DOTGraphTraits<DotCfgDiffDisplayGraph *> : public DefaultDOTGraphTraits {
+ explicit DOTGraphTraits(bool Simple = false)
+ : DefaultDOTGraphTraits(Simple) {}
+
+ static bool renderNodesUsingHTML() { return true; }
+ static std::string getGraphName(const DotCfgDiffDisplayGraph *DiffData) {
+ return DiffData->getGraphName();
+ }
+ static std::string
+ getGraphProperties(const DotCfgDiffDisplayGraph *DiffData) {
+ return "\tsize=\"190, 190\";\n";
+ }
+ static std::string getNodeLabel(const DisplayNode *Node,
+ const DotCfgDiffDisplayGraph *DiffData) {
+ return DiffData->getNodeLabel(*Node);
+ }
+ static std::string getNodeAttributes(const DisplayNode *Node,
+ const DotCfgDiffDisplayGraph *DiffData) {
+ return DiffData->getNodeAttributes(*Node);
+ }
+ static std::string getEdgeSourceLabel(const DisplayNode *From,
+ DisplayNode::ChildIterator &To) {
+ return From->getEdgeSourceLabel(**To);
+ }
+ static std::string getEdgeAttributes(const DisplayNode *From,
+ DisplayNode::ChildIterator &To,
+ const DotCfgDiffDisplayGraph *DiffData) {
+ return DiffData->getEdgeColorAttr(*From, **To);
+ }
+};
+
+} // namespace llvm
+
+namespace {
+
+void DotCfgDiffDisplayGraph::generateDotFile(StringRef DotFile) {
+ std::error_code EC;
+ raw_fd_ostream OutStream(DotFile, EC);
+ if (EC) {
+ errs() << "Error: " << EC.message() << "\n";
+ return;
+ }
+ WriteGraph(OutStream, this, false);
+ OutStream.flush();
+ OutStream.close();
+}
+
+} // namespace
+
+namespace llvm {
+
+DCData::DCData(const BasicBlock &B) {
+ // Build up transition labels.
+ const Instruction *Term = B.getTerminator();
+ if (const BranchInst *Br = dyn_cast<const BranchInst>(Term))
+ if (Br->isUnconditional())
+ addSuccessorLabel(Br->getSuccessor(0)->getName().str(), "");
+ else {
+ addSuccessorLabel(Br->getSuccessor(0)->getName().str(), "true");
+ addSuccessorLabel(Br->getSuccessor(1)->getName().str(), "false");
+ }
+ else if (const SwitchInst *Sw = dyn_cast<const SwitchInst>(Term)) {
+ addSuccessorLabel(Sw->case_default()->getCaseSuccessor()->getName().str(),
+ "default");
+ for (auto &C : Sw->cases()) {
+ assert(C.getCaseValue() && "Expected to find case value.");
+ SmallString<20> Value = formatv("{0}", C.getCaseValue()->getSExtValue());
+ addSuccessorLabel(C.getCaseSuccessor()->getName().str(), Value);
+ }
+ } else
+ for (const_succ_iterator I = succ_begin(&B), E = succ_end(&B); I != E; ++I)
+ addSuccessorLabel((*I)->getName().str(), "");
+}
+
+DotCfgChangeReporter::DotCfgChangeReporter(bool Verbose)
+ : ChangeReporter<IRDataT<DCData>>(Verbose) {}
+
+void DotCfgChangeReporter::handleFunctionCompare(
+ StringRef Name, StringRef Prefix, StringRef PassID, StringRef Divider,
+ bool InModule, unsigned Minor, const FuncDataT<DCData> &Before,
+ const FuncDataT<DCData> &After) {
+ assert(HTML && "Expected outstream to be set");
+ SmallString<8> Extender;
+ SmallString<8> Number;
+ // Handle numbering and file names.
+ if (InModule) {
+ Extender = formatv("{0}_{1}", N, Minor);
+ Number = formatv("{0}.{1}", N, Minor);
+ } else {
+ Extender = formatv("{0}", N);
+ Number = formatv("{0}", N);
+ }
+ // Create a temporary file name for the dot file.
+ SmallVector<char, 128> SV;
+ sys::fs::createUniquePath("cfgdot-%%%%%%.dot", SV, true);
+ std::string DotFile = Twine(SV).str();
+
+ SmallString<20> PDFFileName = formatv("diff_{0}.pdf", Extender);
+ SmallString<200> Text;
+
+ Text = formatv("{0}.{1}{2}{3}{4}", Number, Prefix, makeHTMLReady(PassID),
+ Divider, Name);
+
+ DotCfgDiff Diff(Text, Before, After);
+ std::string EntryBlockName = After.getEntryBlockName();
+ // Use the before entry block if the after entry block was removed.
+ if (EntryBlockName == "")
+ EntryBlockName = Before.getEntryBlockName();
+ assert(EntryBlockName != "" && "Expected to find entry block");
+
+ DotCfgDiffDisplayGraph DG = Diff.createDisplayGraph(Text, EntryBlockName);
+ DG.generateDotFile(DotFile);
+
+ *HTML << genHTML(Text, DotFile, PDFFileName);
+ std::error_code EC = sys::fs::remove(DotFile);
+ if (EC)
+ errs() << "Error: " << EC.message() << "\n";
+}
+
+std::string DotCfgChangeReporter::genHTML(StringRef Text, StringRef DotFile,
+ StringRef PDFFileName) {
+ SmallString<20> PDFFile = formatv("{0}/{1}", DotCfgDir, PDFFileName);
+ // Create the PDF file.
+ static ErrorOr<std::string> DotExe = sys::findProgramByName(DotBinary);
+ if (!DotExe)
+ return "Unable to find dot executable.";
+
+ StringRef Args[] = {DotBinary, "-Tpdf", "-o", PDFFile, DotFile};
+ int Result = sys::ExecuteAndWait(*DotExe, Args, std::nullopt);
+ if (Result < 0)
+ return "Error executing system dot.";
+
+ // Create the HTML tag refering to the PDF file.
+ SmallString<200> S = formatv(
+ " <a href=\"{0}\" target=\"_blank\">{1}</a><br/>\n", PDFFileName, Text);
+ return S.c_str();
+}
+
+void DotCfgChangeReporter::handleInitialIR(Any IR) {
+ assert(HTML && "Expected outstream to be set");
+ *HTML << "<button type=\"button\" class=\"collapsible\">0. "
+ << "Initial IR (by function)</button>\n"
+ << "<div class=\"content\">\n"
+ << " <p>\n";
+ // Create representation of IR
+ IRDataT<DCData> Data;
+ IRComparer<DCData>::analyzeIR(IR, Data);
+ // Now compare it against itself, which will have everything the
+ // same and will generate the files.
+ IRComparer<DCData>(Data, Data)
+ .compare(getModuleForComparison(IR),
+ [&](bool InModule, unsigned Minor,
+ const FuncDataT<DCData> &Before,
+ const FuncDataT<DCData> &After) -> void {
+ handleFunctionCompare("", " ", "Initial IR", "", InModule,
+ Minor, Before, After);
+ });
+ *HTML << " </p>\n"
+ << "</div><br/>\n";
+ ++N;
+}
+
+void DotCfgChangeReporter::generateIRRepresentation(Any IR, StringRef PassID,
+ IRDataT<DCData> &Data) {
+ IRComparer<DCData>::analyzeIR(IR, Data);
+}
+
+void DotCfgChangeReporter::omitAfter(StringRef PassID, std::string &Name) {
+ assert(HTML && "Expected outstream to be set");
+ SmallString<20> Banner =
+ formatv(" <a>{0}. Pass {1} on {2} omitted because no change</a><br/>\n",
+ N, makeHTMLReady(PassID), Name);
+ *HTML << Banner;
+ ++N;
+}
+
+void DotCfgChangeReporter::handleAfter(StringRef PassID, std::string &Name,
+ const IRDataT<DCData> &Before,
+ const IRDataT<DCData> &After, Any IR) {
+ assert(HTML && "Expected outstream to be set");
+ IRComparer<DCData>(Before, After)
+ .compare(getModuleForComparison(IR),
+ [&](bool InModule, unsigned Minor,
+ const FuncDataT<DCData> &Before,
+ const FuncDataT<DCData> &After) -> void {
+ handleFunctionCompare(Name, " Pass ", PassID, " on ", InModule,
+ Minor, Before, After);
+ });
+ *HTML << " </p></div>\n";
+ ++N;
+}
+
+void DotCfgChangeReporter::handleInvalidated(StringRef PassID) {
+ assert(HTML && "Expected outstream to be set");
+ SmallString<20> Banner =
+ formatv(" <a>{0}. {1} invalidated</a><br/>\n", N, makeHTMLReady(PassID));
+ *HTML << Banner;
+ ++N;
+}
+
+void DotCfgChangeReporter::handleFiltered(StringRef PassID, std::string &Name) {
+ assert(HTML && "Expected outstream to be set");
+ SmallString<20> Banner =
+ formatv(" <a>{0}. Pass {1} on {2} filtered out</a><br/>\n", N,
+ makeHTMLReady(PassID), Name);
+ *HTML << Banner;
+ ++N;
+}
+
+void DotCfgChangeReporter::handleIgnored(StringRef PassID, std::string &Name) {
+ assert(HTML && "Expected outstream to be set");
+ SmallString<20> Banner = formatv(" <a>{0}. {1} on {2} ignored</a><br/>\n", N,
+ makeHTMLReady(PassID), Name);
+ *HTML << Banner;
+ ++N;
+}
+
+bool DotCfgChangeReporter::initializeHTML() {
+ std::error_code EC;
+ HTML = std::make_unique<raw_fd_ostream>(DotCfgDir + "/passes.html", EC);
+ if (EC) {
+ HTML = nullptr;
+ return false;
+ }
+
+ *HTML << "<!doctype html>"
+ << "<html>"
+ << "<head>"
+ << "<style>.collapsible { "
+ << "background-color: #777;"
+ << " color: white;"
+ << " cursor: pointer;"
+ << " padding: 18px;"
+ << " width: 100%;"
+ << " border: none;"
+ << " text-align: left;"
+ << " outline: none;"
+ << " font-size: 15px;"
+ << "} .active, .collapsible:hover {"
+ << " background-color: #555;"
+ << "} .content {"
+ << " padding: 0 18px;"
+ << " display: none;"
+ << " overflow: hidden;"
+ << " background-color: #f1f1f1;"
+ << "}"
+ << "</style>"
+ << "<title>passes.html</title>"
+ << "</head>\n"
+ << "<body>";
+ return true;
+}
+
+DotCfgChangeReporter::~DotCfgChangeReporter() {
+ if (!HTML)
+ return;
+ *HTML
+ << "<script>var coll = document.getElementsByClassName(\"collapsible\");"
+ << "var i;"
+ << "for (i = 0; i < coll.length; i++) {"
+ << "coll[i].addEventListener(\"click\", function() {"
+ << " this.classList.toggle(\"active\");"
+ << " var content = this.nextElementSibling;"
+ << " if (content.style.display === \"block\"){"
+ << " content.style.display = \"none\";"
+ << " }"
+ << " else {"
+ << " content.style.display= \"block\";"
+ << " }"
+ << " });"
+ << " }"
+ << "</script>"
+ << "</body>"
+ << "</html>\n";
+ HTML->flush();
+ HTML->close();
+}
+
+void DotCfgChangeReporter::registerCallbacks(
+ PassInstrumentationCallbacks &PIC) {
+ if (PrintChanged == ChangePrinter::DotCfgVerbose ||
+ PrintChanged == ChangePrinter::DotCfgQuiet) {
+ SmallString<128> OutputDir;
+ sys::fs::expand_tilde(DotCfgDir, OutputDir);
+ sys::fs::make_absolute(OutputDir);
+ assert(!OutputDir.empty() && "expected output dir to be non-empty");
+ DotCfgDir = OutputDir.c_str();
+ if (initializeHTML()) {
+ ChangeReporter<IRDataT<DCData>>::registerRequiredCallbacks(PIC);
+ return;
+ }
+ dbgs() << "Unable to open output stream for -cfg-dot-changed\n";
+ }
+}
+
+StandardInstrumentations::StandardInstrumentations(
+ LLVMContext &Context, bool DebugLogging, bool VerifyEach,
+ PrintPassOptions PrintPassOpts)
+ : PrintPass(DebugLogging, PrintPassOpts),
+ OptNone(DebugLogging),
+ OptPassGate(Context),
+ PrintChangedIR(PrintChanged == ChangePrinter::Verbose),
+ PrintChangedDiff(PrintChanged == ChangePrinter::DiffVerbose ||
+ PrintChanged == ChangePrinter::ColourDiffVerbose,
+ PrintChanged == ChangePrinter::ColourDiffVerbose ||
+ PrintChanged == ChangePrinter::ColourDiffQuiet),
+ WebsiteChangeReporter(PrintChanged == ChangePrinter::DotCfgVerbose),
+ Verify(DebugLogging), VerifyEach(VerifyEach) {}
+
+PrintCrashIRInstrumentation *PrintCrashIRInstrumentation::CrashReporter =
+ nullptr;
+
+void PrintCrashIRInstrumentation::reportCrashIR() { dbgs() << SavedIR; }
+
+void PrintCrashIRInstrumentation::SignalHandler(void *) {
+ // Called by signal handlers so do not lock here
+ // Is the PrintCrashIRInstrumentation still alive?
+ if (!CrashReporter)
+ return;
+
+ assert(PrintCrashIR && "Did not expect to get here without option set.");
+ CrashReporter->reportCrashIR();
+}
+
+PrintCrashIRInstrumentation::~PrintCrashIRInstrumentation() {
+ if (!CrashReporter)
+ return;
+
+ assert(PrintCrashIR && "Did not expect to get here without option set.");
+ CrashReporter = nullptr;
+}
+
+void PrintCrashIRInstrumentation::registerCallbacks(
+ PassInstrumentationCallbacks &PIC) {
+ if (!PrintCrashIR || CrashReporter)
+ return;
+
+ sys::AddSignalHandler(SignalHandler, nullptr);
+ CrashReporter = this;
+
+ PIC.registerBeforeNonSkippedPassCallback([&PIC, this](StringRef PassID,
+ Any IR) {
+ SavedIR.clear();
+ raw_string_ostream OS(SavedIR);
+ OS << formatv("*** Dump of {0}IR Before Last Pass {1}",
+ llvm::forcePrintModuleIR() ? "Module " : "", PassID);
+ if (!isInteresting(IR, PassID, PIC.getPassNameForClassName(PassID))) {
+ OS << " Filtered Out ***\n";
+ return;
+ }
+ OS << " Started ***\n";
+ unwrapAndPrint(OS, IR);
+ });
+}
+
+void StandardInstrumentations::registerCallbacks(
+ PassInstrumentationCallbacks &PIC, FunctionAnalysisManager *FAM) {
+ PrintIR.registerCallbacks(PIC);
+ PrintPass.registerCallbacks(PIC);
+ TimePasses.registerCallbacks(PIC);
+ OptNone.registerCallbacks(PIC);
+ OptPassGate.registerCallbacks(PIC);
+ if (FAM)
+ PreservedCFGChecker.registerCallbacks(PIC, *FAM);
+ PrintChangedIR.registerCallbacks(PIC);
+ PseudoProbeVerification.registerCallbacks(PIC);
+ if (VerifyEach)
+ Verify.registerCallbacks(PIC);
+ PrintChangedDiff.registerCallbacks(PIC);
+ WebsiteChangeReporter.registerCallbacks(PIC);
+
+ ChangeTester.registerCallbacks(PIC);
+
+ PrintCrashIR.registerCallbacks(PIC);
+ // TimeProfiling records the pass running time cost.
+ // Its 'BeforePassCallback' can be appended at the tail of all the
+ // BeforeCallbacks by calling `registerCallbacks` in the end.
+ // Its 'AfterPassCallback' is put at the front of all the
+ // AfterCallbacks by its `registerCallbacks`. This is necessary
+ // to ensure that other callbacks are not included in the timings.
+ TimeProfilingPasses.registerCallbacks(PIC);
+}
+
+template class ChangeReporter<std::string>;
+template class TextChangeReporter<std::string>;
+
+template class BlockDataT<EmptyData>;
+template class FuncDataT<EmptyData>;
+template class IRDataT<EmptyData>;
+template class ChangeReporter<IRDataT<EmptyData>>;
+template class TextChangeReporter<IRDataT<EmptyData>>;
+template class IRComparer<EmptyData>;
+
+} // namespace llvm
diff --git a/contrib/libs/llvm16/lib/Passes/ya.make b/contrib/libs/llvm16/lib/Passes/ya.make
new file mode 100644
index 00000000000..73a09137a69
--- /dev/null
+++ b/contrib/libs/llvm16/lib/Passes/ya.make
@@ -0,0 +1,46 @@
+# Generated by devtools/yamaker.
+
+LIBRARY()
+
+LICENSE(Apache-2.0 WITH LLVM-exception)
+
+LICENSE_TEXTS(.yandex_meta/licenses.list.txt)
+
+PEERDIR(
+ contrib/libs/llvm16
+ contrib/libs/llvm16/include
+ contrib/libs/llvm16/lib/Analysis
+ contrib/libs/llvm16/lib/CodeGen
+ contrib/libs/llvm16/lib/IR
+ contrib/libs/llvm16/lib/IRPrinter
+ contrib/libs/llvm16/lib/Support
+ contrib/libs/llvm16/lib/Target
+ contrib/libs/llvm16/lib/Transforms/AggressiveInstCombine
+ contrib/libs/llvm16/lib/Transforms/Coroutines
+ contrib/libs/llvm16/lib/Transforms/IPO
+ contrib/libs/llvm16/lib/Transforms/InstCombine
+ contrib/libs/llvm16/lib/Transforms/Instrumentation
+ contrib/libs/llvm16/lib/Transforms/ObjCARC
+ contrib/libs/llvm16/lib/Transforms/Scalar
+ contrib/libs/llvm16/lib/Transforms/Utils
+ contrib/libs/llvm16/lib/Transforms/Vectorize
+)
+
+ADDINCL(
+ contrib/libs/llvm16/lib/Passes
+)
+
+NO_COMPILER_WARNINGS()
+
+NO_UTIL()
+
+SRCS(
+ OptimizationLevel.cpp
+ PassBuilder.cpp
+ PassBuilderBindings.cpp
+ PassBuilderPipelines.cpp
+ PassPlugin.cpp
+ StandardInstrumentations.cpp
+)
+
+END()
generated by cgit v1.2.3 (git 2.25.1) at 2025-11-01 20:08:22 +0000