diff options
author | shadchin <shadchin@yandex-team.ru> | 2022-02-10 16:44:30 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:44:30 +0300 |
commit | 2598ef1d0aee359b4b6d5fdd1758916d5907d04f (patch) | |
tree | 012bb94d777798f1f56ac1cec429509766d05181 /contrib/libs/llvm12/include/llvm/ProfileData | |
parent | 6751af0b0c1b952fede40b19b71da8025b5d8bcf (diff) | |
download | ydb-2598ef1d0aee359b4b6d5fdd1758916d5907d04f.tar.gz |
Restoring authorship annotation for <shadchin@yandex-team.ru>. Commit 1 of 2.
Diffstat (limited to 'contrib/libs/llvm12/include/llvm/ProfileData')
10 files changed, 813 insertions, 813 deletions
diff --git a/contrib/libs/llvm12/include/llvm/ProfileData/Coverage/CoverageMapping.h b/contrib/libs/llvm12/include/llvm/ProfileData/Coverage/CoverageMapping.h index 99bc8cd9c7..4cd3afe58f 100644 --- a/contrib/libs/llvm12/include/llvm/ProfileData/Coverage/CoverageMapping.h +++ b/contrib/libs/llvm12/include/llvm/ProfileData/Coverage/CoverageMapping.h @@ -62,8 +62,8 @@ enum class coveragemap_error { unsupported_version, truncated, malformed, - decompression_failed, - invalid_or_missing_arch_specifier + decompression_failed, + invalid_or_missing_arch_specifier }; const std::error_category &coveragemap_category(); @@ -97,8 +97,8 @@ private: /// A Counter is an abstract value that describes how to compute the /// execution count for a region of code using the collected profile count data. struct Counter { - /// The CounterExpression kind (Add or Subtract) is encoded in bit 0 next to - /// the CounterKind. This means CounterKind has to leave bit 0 free. + /// The CounterExpression kind (Add or Subtract) is encoded in bit 0 next to + /// the CounterKind. This means CounterKind has to leave bit 0 free. enum CounterKind { Zero, CounterValueReference, Expression }; static const unsigned EncodingTagBits = 2; static const unsigned EncodingTagMask = 0x3; @@ -228,20 +228,20 @@ struct CounterMappingRegion { /// A GapRegion is like a CodeRegion, but its count is only set as the /// line execution count when its the only region in the line. - GapRegion, - - /// A BranchRegion represents leaf-level boolean expressions and is - /// associated with two counters, each representing the number of times the - /// expression evaluates to true or false. - BranchRegion + GapRegion, + + /// A BranchRegion represents leaf-level boolean expressions and is + /// associated with two counters, each representing the number of times the + /// expression evaluates to true or false. + BranchRegion }; - /// Primary Counter that is also used for Branch Regions (TrueCount). + /// Primary Counter that is also used for Branch Regions (TrueCount). Counter Count; - - /// Secondary Counter used for Branch Regions (FalseCount). - Counter FalseCount; - + + /// Secondary Counter used for Branch Regions (FalseCount). + Counter FalseCount; + unsigned FileID, ExpandedFileID; unsigned LineStart, ColumnStart, LineEnd, ColumnEnd; RegionKind Kind; @@ -253,15 +253,15 @@ struct CounterMappingRegion { LineStart(LineStart), ColumnStart(ColumnStart), LineEnd(LineEnd), ColumnEnd(ColumnEnd), Kind(Kind) {} - CounterMappingRegion(Counter Count, Counter FalseCount, unsigned FileID, - unsigned ExpandedFileID, unsigned LineStart, - unsigned ColumnStart, unsigned LineEnd, - unsigned ColumnEnd, RegionKind Kind) - : Count(Count), FalseCount(FalseCount), FileID(FileID), - ExpandedFileID(ExpandedFileID), LineStart(LineStart), - ColumnStart(ColumnStart), LineEnd(LineEnd), ColumnEnd(ColumnEnd), - Kind(Kind) {} - + CounterMappingRegion(Counter Count, Counter FalseCount, unsigned FileID, + unsigned ExpandedFileID, unsigned LineStart, + unsigned ColumnStart, unsigned LineEnd, + unsigned ColumnEnd, RegionKind Kind) + : Count(Count), FalseCount(FalseCount), FileID(FileID), + ExpandedFileID(ExpandedFileID), LineStart(LineStart), + ColumnStart(ColumnStart), LineEnd(LineEnd), ColumnEnd(ColumnEnd), + Kind(Kind) {} + static CounterMappingRegion makeRegion(Counter Count, unsigned FileID, unsigned LineStart, unsigned ColumnStart, unsigned LineEnd, unsigned ColumnEnd) { @@ -291,14 +291,14 @@ struct CounterMappingRegion { LineEnd, (1U << 31) | ColumnEnd, GapRegion); } - static CounterMappingRegion - makeBranchRegion(Counter Count, Counter FalseCount, unsigned FileID, - unsigned LineStart, unsigned ColumnStart, unsigned LineEnd, - unsigned ColumnEnd) { - return CounterMappingRegion(Count, FalseCount, FileID, 0, LineStart, - ColumnStart, LineEnd, ColumnEnd, BranchRegion); - } - + static CounterMappingRegion + makeBranchRegion(Counter Count, Counter FalseCount, unsigned FileID, + unsigned LineStart, unsigned ColumnStart, unsigned LineEnd, + unsigned ColumnEnd) { + return CounterMappingRegion(Count, FalseCount, FileID, 0, LineStart, + ColumnStart, LineEnd, ColumnEnd, BranchRegion); + } + inline LineColPair startLoc() const { return LineColPair(LineStart, ColumnStart); } @@ -309,17 +309,17 @@ struct CounterMappingRegion { /// Associates a source range with an execution count. struct CountedRegion : public CounterMappingRegion { uint64_t ExecutionCount; - uint64_t FalseExecutionCount; - bool Folded; + uint64_t FalseExecutionCount; + bool Folded; CountedRegion(const CounterMappingRegion &R, uint64_t ExecutionCount) - : CounterMappingRegion(R), ExecutionCount(ExecutionCount), - FalseExecutionCount(0), Folded(false) {} - - CountedRegion(const CounterMappingRegion &R, uint64_t ExecutionCount, - uint64_t FalseExecutionCount) - : CounterMappingRegion(R), ExecutionCount(ExecutionCount), - FalseExecutionCount(FalseExecutionCount), Folded(false) {} + : CounterMappingRegion(R), ExecutionCount(ExecutionCount), + FalseExecutionCount(0), Folded(false) {} + + CountedRegion(const CounterMappingRegion &R, uint64_t ExecutionCount, + uint64_t FalseExecutionCount) + : CounterMappingRegion(R), ExecutionCount(ExecutionCount), + FalseExecutionCount(FalseExecutionCount), Folded(false) {} }; /// A Counter mapping context is used to connect the counters, expressions @@ -356,8 +356,8 @@ struct FunctionRecord { std::vector<std::string> Filenames; /// Regions in the function along with their counts. std::vector<CountedRegion> CountedRegions; - /// Branch Regions in the function along with their counts. - std::vector<CountedRegion> CountedBranchRegions; + /// Branch Regions in the function along with their counts. + std::vector<CountedRegion> CountedBranchRegions; /// The number of times this function was executed. uint64_t ExecutionCount = 0; @@ -367,19 +367,19 @@ struct FunctionRecord { FunctionRecord(FunctionRecord &&FR) = default; FunctionRecord &operator=(FunctionRecord &&) = default; - void pushRegion(CounterMappingRegion Region, uint64_t Count, - uint64_t FalseCount) { - if (Region.Kind == CounterMappingRegion::BranchRegion) { - CountedBranchRegions.emplace_back(Region, Count, FalseCount); - // If both counters are hard-coded to zero, then this region represents a - // constant-folded branch. - if (Region.Count.isZero() && Region.FalseCount.isZero()) - CountedBranchRegions.back().Folded = true; - return; - } + void pushRegion(CounterMappingRegion Region, uint64_t Count, + uint64_t FalseCount) { + if (Region.Kind == CounterMappingRegion::BranchRegion) { + CountedBranchRegions.emplace_back(Region, Count, FalseCount); + // If both counters are hard-coded to zero, then this region represents a + // constant-folded branch. + if (Region.Count.isZero() && Region.FalseCount.isZero()) + CountedBranchRegions.back().Folded = true; + return; + } if (CountedRegions.empty()) ExecutionCount = Count; - CountedRegions.emplace_back(Region, Count, FalseCount); + CountedRegions.emplace_back(Region, Count, FalseCount); } }; @@ -458,8 +458,8 @@ struct CoverageSegment { IsRegionEntry(IsRegionEntry), IsGapRegion(false) {} CoverageSegment(unsigned Line, unsigned Col, uint64_t Count, - bool IsRegionEntry, bool IsGapRegion = false, - bool IsBranchRegion = false) + bool IsRegionEntry, bool IsGapRegion = false, + bool IsBranchRegion = false) : Line(Line), Col(Col), Count(Count), HasCount(true), IsRegionEntry(IsRegionEntry), IsGapRegion(IsGapRegion) {} @@ -539,7 +539,7 @@ class CoverageData { std::string Filename; std::vector<CoverageSegment> Segments; std::vector<ExpansionRecord> Expansions; - std::vector<CountedRegion> BranchRegions; + std::vector<CountedRegion> BranchRegions; public: CoverageData() = default; @@ -563,9 +563,9 @@ public: /// Expansions that can be further processed. ArrayRef<ExpansionRecord> getExpansions() const { return Expansions; } - - /// Branches that can be further processed. - ArrayRef<CountedRegion> getBranches() const { return BranchRegions; } + + /// Branches that can be further processed. + ArrayRef<CountedRegion> getBranches() const { return BranchRegions; } }; /// The mapping of profile information to coverage data. @@ -1001,9 +1001,9 @@ enum CovMapVersion { Version3 = 2, // Function records are named, uniqued, and moved to a dedicated section. Version4 = 3, - // Branch regions referring to two counters are added - Version5 = 4, - // The current version is Version5. + // Branch regions referring to two counters are added + Version5 = 4, + // The current version is Version5. CurrentVersion = INSTR_PROF_COVMAP_VERSION }; diff --git a/contrib/libs/llvm12/include/llvm/ProfileData/GCOV.h b/contrib/libs/llvm12/include/llvm/ProfileData/GCOV.h index a3c2569f13..d984b3c696 100644 --- a/contrib/libs/llvm12/include/llvm/ProfileData/GCOV.h +++ b/contrib/libs/llvm12/include/llvm/ProfileData/GCOV.h @@ -22,7 +22,7 @@ #define LLVM_PROFILEDATA_GCOV_H #include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/DenseSet.h" #include "llvm/ADT/MapVector.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringMap.h" @@ -54,11 +54,11 @@ enum GCOVVersion { V304, V407, V408, V800, V900 }; /// A struct for passing gcov options between functions. struct Options { Options(bool A, bool B, bool C, bool F, bool P, bool U, bool I, bool L, - bool M, bool N, bool R, bool T, bool X, std::string SourcePrefix) + bool M, bool N, bool R, bool T, bool X, std::string SourcePrefix) : AllBlocks(A), BranchInfo(B), BranchCount(C), FuncCoverage(F), PreservePaths(P), UncondBranch(U), Intermediate(I), LongFileNames(L), - Demangle(M), NoOutput(N), RelativeOnly(R), UseStdout(T), - HashFilenames(X), SourcePrefix(std::move(SourcePrefix)) {} + Demangle(M), NoOutput(N), RelativeOnly(R), UseStdout(T), + HashFilenames(X), SourcePrefix(std::move(SourcePrefix)) {} bool AllBlocks; bool BranchInfo; @@ -68,12 +68,12 @@ struct Options { bool UncondBranch; bool Intermediate; bool LongFileNames; - bool Demangle; + bool Demangle; bool NoOutput; - bool RelativeOnly; + bool RelativeOnly; bool UseStdout; bool HashFilenames; - std::string SourcePrefix; + std::string SourcePrefix; }; } // end namespace GCOV @@ -204,32 +204,32 @@ public: std::vector<std::string> filenames; StringMap<unsigned> filenameToIdx; -public: +public: bool GCNOInitialized = false; GCOV::GCOVVersion Version; uint32_t Checksum = 0; StringRef cwd; - SmallVector<std::unique_ptr<GCOVFunction>, 16> functions; + SmallVector<std::unique_ptr<GCOVFunction>, 16> functions; std::map<uint32_t, GCOVFunction *> IdentToFunction; uint32_t RunCount = 0; uint32_t ProgramCount = 0; using iterator = pointee_iterator< SmallVectorImpl<std::unique_ptr<GCOVFunction>>::const_iterator>; - iterator begin() const { return iterator(functions.begin()); } - iterator end() const { return iterator(functions.end()); } + iterator begin() const { return iterator(functions.begin()); } + iterator end() const { return iterator(functions.end()); } }; struct GCOVArc { - GCOVArc(GCOVBlock &src, GCOVBlock &dst, uint32_t flags) - : src(src), dst(dst), flags(flags) {} - bool onTree() const; + GCOVArc(GCOVBlock &src, GCOVBlock &dst, uint32_t flags) + : src(src), dst(dst), flags(flags) {} + bool onTree() const; GCOVBlock &src; GCOVBlock &dst; - uint32_t flags; - uint64_t count = 0; - uint64_t cycleCount = 0; + uint32_t flags; + uint64_t count = 0; + uint64_t cycleCount = 0; }; /// GCOVFunction - Collects function information. @@ -240,16 +240,16 @@ public: GCOVFunction(GCOVFile &file) : file(file) {} - StringRef getName(bool demangle) const; + StringRef getName(bool demangle) const; StringRef getFilename() const; uint64_t getEntryCount() const; - GCOVBlock &getExitBlock() const; + GCOVBlock &getExitBlock() const; - iterator_range<BlockIterator> blocksRange() const { - return make_range(blocks.begin(), blocks.end()); + iterator_range<BlockIterator> blocksRange() const { + return make_range(blocks.begin(), blocks.end()); } - uint64_t propagateCounts(const GCOVBlock &v, GCOVArc *pred); + uint64_t propagateCounts(const GCOVBlock &v, GCOVArc *pred); void print(raw_ostream &OS) const; void dump() const; @@ -263,26 +263,26 @@ public: uint32_t endColumn = 0; uint8_t artificial = 0; StringRef Name; - mutable SmallString<0> demangled; + mutable SmallString<0> demangled; unsigned srcIdx; - SmallVector<std::unique_ptr<GCOVBlock>, 0> blocks; + SmallVector<std::unique_ptr<GCOVBlock>, 0> blocks; SmallVector<std::unique_ptr<GCOVArc>, 0> arcs, treeArcs; - DenseSet<const GCOVBlock *> visited; + DenseSet<const GCOVBlock *> visited; }; /// GCOVBlock - Collects block information. class GCOVBlock { public: using EdgeIterator = SmallVectorImpl<GCOVArc *>::const_iterator; - using BlockVector = SmallVector<const GCOVBlock *, 1>; + using BlockVector = SmallVector<const GCOVBlock *, 1>; using BlockVectorLists = SmallVector<BlockVector, 4>; using Edges = SmallVector<GCOVArc *, 4>; - GCOVBlock(uint32_t N) : number(N) {} + GCOVBlock(uint32_t N) : number(N) {} - void addLine(uint32_t N) { lines.push_back(N); } - uint32_t getLastLine() const { return lines.back(); } - uint64_t getCount() const { return count; } + void addLine(uint32_t N) { lines.push_back(N); } + uint32_t getLastLine() const { return lines.back(); } + uint64_t getCount() const { return count; } void addSrcEdge(GCOVArc *Edge) { pred.push_back(Edge); } @@ -299,24 +299,24 @@ public: void print(raw_ostream &OS) const; void dump() const; - static uint64_t - augmentOneCycle(GCOVBlock *src, - std::vector<std::pair<GCOVBlock *, size_t>> &stack); - static uint64_t getCyclesCount(const BlockVector &blocks); + static uint64_t + augmentOneCycle(GCOVBlock *src, + std::vector<std::pair<GCOVBlock *, size_t>> &stack); + static uint64_t getCyclesCount(const BlockVector &blocks); static uint64_t getLineCount(const BlockVector &Blocks); public: - uint32_t number; - uint64_t count = 0; + uint32_t number; + uint64_t count = 0; SmallVector<GCOVArc *, 2> pred; SmallVector<GCOVArc *, 2> succ; - SmallVector<uint32_t, 4> lines; - bool traversable = false; - GCOVArc *incoming = nullptr; + SmallVector<uint32_t, 4> lines; + bool traversable = false; + GCOVArc *incoming = nullptr; }; -void gcovOneInput(const GCOV::Options &options, StringRef filename, - StringRef gcno, StringRef gcda, GCOVFile &file); +void gcovOneInput(const GCOV::Options &options, StringRef filename, + StringRef gcno, StringRef gcda, GCOVFile &file); } // end namespace llvm diff --git a/contrib/libs/llvm12/include/llvm/ProfileData/InstrProf.h b/contrib/libs/llvm12/include/llvm/ProfileData/InstrProf.h index 99ac560437..ba9c0570ad 100644 --- a/contrib/libs/llvm12/include/llvm/ProfileData/InstrProf.h +++ b/contrib/libs/llvm12/include/llvm/ProfileData/InstrProf.h @@ -81,10 +81,10 @@ inline StringRef getInstrProfValueProfFuncName() { return INSTR_PROF_VALUE_PROF_FUNC_STR; } -/// Return the name profile runtime entry point to do memop size value -/// profiling. -inline StringRef getInstrProfValueProfMemOpFuncName() { - return INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR; +/// Return the name profile runtime entry point to do memop size value +/// profiling. +inline StringRef getInstrProfValueProfMemOpFuncName() { + return INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR; } /// Return the name prefix of variables containing instrumented function names. @@ -569,9 +569,9 @@ StringRef InstrProfSymtab::getFuncNameOrExternalSymbol(uint64_t FuncMD5Hash) { StringRef InstrProfSymtab::getFuncName(uint64_t FuncMD5Hash) { finalizeSymtab(); - auto Result = llvm::lower_bound(MD5NameMap, FuncMD5Hash, - [](const std::pair<uint64_t, StringRef> &LHS, - uint64_t RHS) { return LHS.first < RHS; }); + auto Result = llvm::lower_bound(MD5NameMap, FuncMD5Hash, + [](const std::pair<uint64_t, StringRef> &LHS, + uint64_t RHS) { return LHS.first < RHS; }); if (Result != MD5NameMap.end() && Result->first == FuncMD5Hash) return Result->second; return StringRef(); @@ -579,9 +579,9 @@ StringRef InstrProfSymtab::getFuncName(uint64_t FuncMD5Hash) { Function* InstrProfSymtab::getFunction(uint64_t FuncMD5Hash) { finalizeSymtab(); - auto Result = llvm::lower_bound(MD5FuncMap, FuncMD5Hash, - [](const std::pair<uint64_t, Function *> &LHS, - uint64_t RHS) { return LHS.first < RHS; }); + auto Result = llvm::lower_bound(MD5FuncMap, FuncMD5Hash, + [](const std::pair<uint64_t, Function *> &LHS, + uint64_t RHS) { return LHS.first < RHS; }); if (Result != MD5FuncMap.end() && Result->first == FuncMD5Hash) return Result->second; return nullptr; @@ -684,8 +684,8 @@ struct InstrProfValueSiteRecord { /// Optionally scale merged counts by \p Weight. void merge(InstrProfValueSiteRecord &Input, uint64_t Weight, function_ref<void(instrprof_error)> Warn); - /// Scale up value profile data counts by N (Numerator) / D (Denominator). - void scale(uint64_t N, uint64_t D, function_ref<void(instrprof_error)> Warn); + /// Scale up value profile data counts by N (Numerator) / D (Denominator). + void scale(uint64_t N, uint64_t D, function_ref<void(instrprof_error)> Warn); /// Compute the overlap b/w this record and Input record. void overlap(InstrProfValueSiteRecord &Input, uint32_t ValueKind, @@ -759,8 +759,8 @@ struct InstrProfRecord { function_ref<void(instrprof_error)> Warn); /// Scale up profile counts (including value profile data) by - /// a factor of (N / D). - void scale(uint64_t N, uint64_t D, function_ref<void(instrprof_error)> Warn); + /// a factor of (N / D). + void scale(uint64_t N, uint64_t D, function_ref<void(instrprof_error)> Warn); /// Sort value profile data (per site) by count. void sortValueData() { @@ -845,8 +845,8 @@ private: uint64_t Weight, function_ref<void(instrprof_error)> Warn); - // Scale up value profile data count by N (Numerator) / D (Denominator). - void scaleValueProfData(uint32_t ValueKind, uint64_t N, uint64_t D, + // Scale up value profile data count by N (Numerator) / D (Denominator). + void scaleValueProfData(uint32_t ValueKind, uint64_t N, uint64_t D, function_ref<void(instrprof_error)> Warn); }; @@ -988,9 +988,9 @@ enum ProfVersion { // In this version, the frontend PGO stable hash algorithm got fixed and // may produce hashes different from Version5. Version6 = 6, - // An additional counter is added around logical operators. - Version7 = 7, - // The current version is 7. + // An additional counter is added around logical operators. + Version7 = 7, + // The current version is 7. CurrentVersion = INSTR_PROF_INDEX_VERSION }; const uint64_t Version = ProfVersion::CurrentVersion; @@ -1146,8 +1146,8 @@ void getMemOPSizeRangeFromOption(StringRef Str, int64_t &RangeStart, // Create a COMDAT variable INSTR_PROF_RAW_VERSION_VAR to make the runtime // aware this is an ir_level profile so it can set the version flag. -void createIRLevelProfileFlagVar(Module &M, bool IsCS, - bool InstrEntryBBEnabled); +void createIRLevelProfileFlagVar(Module &M, bool IsCS, + bool InstrEntryBBEnabled); // Create the variable for the profile file name. void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput); diff --git a/contrib/libs/llvm12/include/llvm/ProfileData/InstrProfData.inc b/contrib/libs/llvm12/include/llvm/ProfileData/InstrProfData.inc index f715505ba5..727723cfe1 100644 --- a/contrib/libs/llvm12/include/llvm/ProfileData/InstrProfData.inc +++ b/contrib/libs/llvm12/include/llvm/ProfileData/InstrProfData.inc @@ -647,9 +647,9 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, /* Raw profile format version (start from 1). */ #define INSTR_PROF_RAW_VERSION 5 /* Indexed profile format version (start from 1). */ -#define INSTR_PROF_INDEX_VERSION 7 +#define INSTR_PROF_INDEX_VERSION 7 /* Coverage mapping format version (start from 0). */ -#define INSTR_PROF_COVMAP_VERSION 4 +#define INSTR_PROF_COVMAP_VERSION 4 /* Profile version is always of type uint64_t. Reserve the upper 8 bits in the * version for other variants of profile. We set the lowest bit of the upper 8 @@ -661,7 +661,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL) #define VARIANT_MASK_IR_PROF (0x1ULL << 56) #define VARIANT_MASK_CSIR_PROF (0x1ULL << 57) -#define VARIANT_MASK_INSTR_ENTRY (0x1ULL << 58) +#define VARIANT_MASK_INSTR_ENTRY (0x1ULL << 58) #define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version #define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime @@ -744,9 +744,9 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target #define INSTR_PROF_VALUE_PROF_FUNC_STR \ INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC) -#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC __llvm_profile_instrument_memop -#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR \ - INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_MEMOP_FUNC) +#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC __llvm_profile_instrument_memop +#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR \ + INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_MEMOP_FUNC) /* InstrProfile per-function control data alignment. */ #define INSTR_PROF_DATA_ALIGNMENT 8 @@ -774,121 +774,121 @@ typedef struct InstrProfValueData { #endif #undef COVMAP_V2_OR_V3 - -#ifdef INSTR_PROF_VALUE_PROF_MEMOP_API - -#ifdef __cplusplus -#define INSTR_PROF_INLINE inline -#else -#define INSTR_PROF_INLINE -#endif - -/* The value range buckets (22 buckets) for the memop size value profiling looks - * like: - * - * [0, 0] - * [1, 1] - * [2, 2] - * [3, 3] - * [4, 4] - * [5, 5] - * [6, 6] - * [7, 7] - * [8, 8] - * [9, 15] - * [16, 16] - * [17, 31] - * [32, 32] - * [33, 63] - * [64, 64] - * [65, 127] - * [128, 128] - * [129, 255] - * [256, 256] - * [257, 511] - * [512, 512] - * [513, UINT64_MAX] - * - * Each range has a 'representative value' which is the lower end value of the - * range and used to store in the runtime profile data records and the VP - * metadata. For example, it's 2 for [2, 2] and 64 for [65, 127]. - */ - -/* - * Clz and Popcount. This code was copied from - * compiler-rt/lib/fuzzer/{FuzzerBuiltins.h,FuzzerBuiltinsMsvc.h} and - * llvm/include/llvm/Support/MathExtras.h. - */ -#if defined(_MSC_VER) && !defined(__clang__) - -#include <intrin.h> -INSTR_PROF_VISIBILITY INSTR_PROF_INLINE -int InstProfClzll(unsigned long long X) { - unsigned long LeadZeroIdx = 0; -#if !defined(_M_ARM64) && !defined(_M_X64) - // Scan the high 32 bits. - if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X >> 32))) - return (int)(63 - (LeadZeroIdx + 32)); // Create a bit offset - // from the MSB. - // Scan the low 32 bits. - if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X))) - return (int)(63 - LeadZeroIdx); -#else - if (_BitScanReverse64(&LeadZeroIdx, X)) return 63 - LeadZeroIdx; -#endif - return 64; -} -INSTR_PROF_VISIBILITY INSTR_PROF_INLINE -int InstProfPopcountll(unsigned long long X) { - // This code originates from https://reviews.llvm.org/rG30626254510f. - unsigned long long v = X; - v = v - ((v >> 1) & 0x5555555555555555ULL); - v = (v & 0x3333333333333333ULL) + ((v >> 2) & 0x3333333333333333ULL); - v = (v + (v >> 4)) & 0x0F0F0F0F0F0F0F0FULL; - return (int)((unsigned long long)(v * 0x0101010101010101ULL) >> 56); -} - -#else - -INSTR_PROF_VISIBILITY INSTR_PROF_INLINE -int InstProfClzll(unsigned long long X) { return __builtin_clzll(X); } -INSTR_PROF_VISIBILITY INSTR_PROF_INLINE -int InstProfPopcountll(unsigned long long X) { return __builtin_popcountll(X); } - -#endif /* defined(_MSC_VER) && !defined(__clang__) */ - -/* Map an (observed) memop size value to the representative value of its range. - * For example, 5 -> 5, 22 -> 17, 99 -> 65, 256 -> 256, 1001 -> 513. */ -INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint64_t -InstrProfGetRangeRepValue(uint64_t Value) { - if (Value <= 8) - // The first ranges are individually tracked. Use the value as is. - return Value; - else if (Value >= 513) - // The last range is mapped to its lowest value. - return 513; - else if (InstProfPopcountll(Value) == 1) - // If it's a power of two, use it as is. - return Value; - else - // Otherwise, take to the previous power of two + 1. - return (1 << (64 - InstProfClzll(Value) - 1)) + 1; -} - -/* Return true if the range that an (observed) memop size value belongs to has - * only a single value in the range. For example, 0 -> true, 8 -> true, 10 -> - * false, 64 -> true, 100 -> false, 513 -> false. */ -INSTR_PROF_VISIBILITY INSTR_PROF_INLINE unsigned -InstrProfIsSingleValRange(uint64_t Value) { - if (Value <= 8) - // The first ranges are individually tracked. - return 1; - else if (InstProfPopcountll(Value) == 1) - // If it's a power of two, there's only one value. - return 1; - else - // Otherwise, there's more than one value in the range. - return 0; -} - -#endif /* INSTR_PROF_VALUE_PROF_MEMOP_API */ + +#ifdef INSTR_PROF_VALUE_PROF_MEMOP_API + +#ifdef __cplusplus +#define INSTR_PROF_INLINE inline +#else +#define INSTR_PROF_INLINE +#endif + +/* The value range buckets (22 buckets) for the memop size value profiling looks + * like: + * + * [0, 0] + * [1, 1] + * [2, 2] + * [3, 3] + * [4, 4] + * [5, 5] + * [6, 6] + * [7, 7] + * [8, 8] + * [9, 15] + * [16, 16] + * [17, 31] + * [32, 32] + * [33, 63] + * [64, 64] + * [65, 127] + * [128, 128] + * [129, 255] + * [256, 256] + * [257, 511] + * [512, 512] + * [513, UINT64_MAX] + * + * Each range has a 'representative value' which is the lower end value of the + * range and used to store in the runtime profile data records and the VP + * metadata. For example, it's 2 for [2, 2] and 64 for [65, 127]. + */ + +/* + * Clz and Popcount. This code was copied from + * compiler-rt/lib/fuzzer/{FuzzerBuiltins.h,FuzzerBuiltinsMsvc.h} and + * llvm/include/llvm/Support/MathExtras.h. + */ +#if defined(_MSC_VER) && !defined(__clang__) + +#include <intrin.h> +INSTR_PROF_VISIBILITY INSTR_PROF_INLINE +int InstProfClzll(unsigned long long X) { + unsigned long LeadZeroIdx = 0; +#if !defined(_M_ARM64) && !defined(_M_X64) + // Scan the high 32 bits. + if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X >> 32))) + return (int)(63 - (LeadZeroIdx + 32)); // Create a bit offset + // from the MSB. + // Scan the low 32 bits. + if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X))) + return (int)(63 - LeadZeroIdx); +#else + if (_BitScanReverse64(&LeadZeroIdx, X)) return 63 - LeadZeroIdx; +#endif + return 64; +} +INSTR_PROF_VISIBILITY INSTR_PROF_INLINE +int InstProfPopcountll(unsigned long long X) { + // This code originates from https://reviews.llvm.org/rG30626254510f. + unsigned long long v = X; + v = v - ((v >> 1) & 0x5555555555555555ULL); + v = (v & 0x3333333333333333ULL) + ((v >> 2) & 0x3333333333333333ULL); + v = (v + (v >> 4)) & 0x0F0F0F0F0F0F0F0FULL; + return (int)((unsigned long long)(v * 0x0101010101010101ULL) >> 56); +} + +#else + +INSTR_PROF_VISIBILITY INSTR_PROF_INLINE +int InstProfClzll(unsigned long long X) { return __builtin_clzll(X); } +INSTR_PROF_VISIBILITY INSTR_PROF_INLINE +int InstProfPopcountll(unsigned long long X) { return __builtin_popcountll(X); } + +#endif /* defined(_MSC_VER) && !defined(__clang__) */ + +/* Map an (observed) memop size value to the representative value of its range. + * For example, 5 -> 5, 22 -> 17, 99 -> 65, 256 -> 256, 1001 -> 513. */ +INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint64_t +InstrProfGetRangeRepValue(uint64_t Value) { + if (Value <= 8) + // The first ranges are individually tracked. Use the value as is. + return Value; + else if (Value >= 513) + // The last range is mapped to its lowest value. + return 513; + else if (InstProfPopcountll(Value) == 1) + // If it's a power of two, use it as is. + return Value; + else + // Otherwise, take to the previous power of two + 1. + return (1 << (64 - InstProfClzll(Value) - 1)) + 1; +} + +/* Return true if the range that an (observed) memop size value belongs to has + * only a single value in the range. For example, 0 -> true, 8 -> true, 10 -> + * false, 64 -> true, 100 -> false, 513 -> false. */ +INSTR_PROF_VISIBILITY INSTR_PROF_INLINE unsigned +InstrProfIsSingleValRange(uint64_t Value) { + if (Value <= 8) + // The first ranges are individually tracked. + return 1; + else if (InstProfPopcountll(Value) == 1) + // If it's a power of two, there's only one value. + return 1; + else + // Otherwise, there's more than one value in the range. + return 0; +} + +#endif /* INSTR_PROF_VALUE_PROF_MEMOP_API */ diff --git a/contrib/libs/llvm12/include/llvm/ProfileData/InstrProfReader.h b/contrib/libs/llvm12/include/llvm/ProfileData/InstrProfReader.h index 29555e07d8..a95a7657b0 100644 --- a/contrib/libs/llvm12/include/llvm/ProfileData/InstrProfReader.h +++ b/contrib/libs/llvm12/include/llvm/ProfileData/InstrProfReader.h @@ -90,8 +90,8 @@ public: virtual bool hasCSIRLevelProfile() const = 0; - virtual bool instrEntryBBEnabled() const = 0; - + virtual bool instrEntryBBEnabled() const = 0; + /// Return the PGO symtab. There are three different readers: /// Raw, Text, and Indexed profile readers. The first two types /// of readers are used only by llvm-profdata tool, while the indexed @@ -161,7 +161,7 @@ private: line_iterator Line; bool IsIRLevelProfile = false; bool HasCSIRLevelProfile = false; - bool InstrEntryBBEnabled = false; + bool InstrEntryBBEnabled = false; Error readValueProfileData(InstrProfRecord &Record); @@ -178,8 +178,8 @@ public: bool hasCSIRLevelProfile() const override { return HasCSIRLevelProfile; } - bool instrEntryBBEnabled() const override { return InstrEntryBBEnabled; } - + bool instrEntryBBEnabled() const override { return InstrEntryBBEnabled; } + /// Read the header. Error readHeader() override; @@ -240,10 +240,10 @@ public: return (Version & VARIANT_MASK_CSIR_PROF) != 0; } - bool instrEntryBBEnabled() const override { - return (Version & VARIANT_MASK_INSTR_ENTRY) != 0; - } - + bool instrEntryBBEnabled() const override { + return (Version & VARIANT_MASK_INSTR_ENTRY) != 0; + } + InstrProfSymtab &getSymtab() override { assert(Symtab.get()); return *Symtab.get(); @@ -380,7 +380,7 @@ struct InstrProfReaderIndexBase { virtual uint64_t getVersion() const = 0; virtual bool isIRLevelProfile() const = 0; virtual bool hasCSIRLevelProfile() const = 0; - virtual bool instrEntryBBEnabled() const = 0; + virtual bool instrEntryBBEnabled() const = 0; virtual Error populateSymtab(InstrProfSymtab &) = 0; }; @@ -429,10 +429,10 @@ public: return (FormatVersion & VARIANT_MASK_CSIR_PROF) != 0; } - bool instrEntryBBEnabled() const override { - return (FormatVersion & VARIANT_MASK_INSTR_ENTRY) != 0; - } - + bool instrEntryBBEnabled() const override { + return (FormatVersion & VARIANT_MASK_INSTR_ENTRY) != 0; + } + Error populateSymtab(InstrProfSymtab &Symtab) override { return Symtab.create(HashTable->keys()); } @@ -487,10 +487,10 @@ public: return Index->hasCSIRLevelProfile(); } - bool instrEntryBBEnabled() const override { - return Index->instrEntryBBEnabled(); - } - + bool instrEntryBBEnabled() const override { + return Index->instrEntryBBEnabled(); + } + /// Return true if the given buffer is in an indexed instrprof format. static bool hasFormat(const MemoryBuffer &DataBuffer); diff --git a/contrib/libs/llvm12/include/llvm/ProfileData/InstrProfWriter.h b/contrib/libs/llvm12/include/llvm/ProfileData/InstrProfWriter.h index d9824d565b..cc4668f7f3 100644 --- a/contrib/libs/llvm12/include/llvm/ProfileData/InstrProfWriter.h +++ b/contrib/libs/llvm12/include/llvm/ProfileData/InstrProfWriter.h @@ -47,16 +47,16 @@ private: bool Sparse; StringMap<ProfilingData> FunctionData; ProfKind ProfileKind = PF_Unknown; - bool InstrEntryBBEnabled; + bool InstrEntryBBEnabled; // Use raw pointer here for the incomplete type object. InstrProfRecordWriterTrait *InfoObj; public: - InstrProfWriter(bool Sparse = false, bool InstrEntryBBEnabled = false); + InstrProfWriter(bool Sparse = false, bool InstrEntryBBEnabled = false); ~InstrProfWriter(); - StringMap<ProfilingData> &getProfileData() { return FunctionData; } - + StringMap<ProfilingData> &getProfileData() { return FunctionData; } + /// Add function counts for the given function. If there are already counts /// for this function and the hash and number of counts match, each counter is /// summed. Optionally scale counts by \p Weight. @@ -107,7 +107,7 @@ public: return Error::success(); } - void setInstrEntryBBEnabled(bool Enabled) { InstrEntryBBEnabled = Enabled; } + void setInstrEntryBBEnabled(bool Enabled) { InstrEntryBBEnabled = Enabled; } // Internal interface for testing purpose only. void setValueProfDataEndianness(support::endianness Endianness); void setOutputSparse(bool Sparse); diff --git a/contrib/libs/llvm12/include/llvm/ProfileData/ProfileCommon.h b/contrib/libs/llvm12/include/llvm/ProfileData/ProfileCommon.h index 649b18830a..6c4713add1 100644 --- a/contrib/libs/llvm12/include/llvm/ProfileData/ProfileCommon.h +++ b/contrib/libs/llvm12/include/llvm/ProfileData/ProfileCommon.h @@ -24,7 +24,7 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/IR/ProfileSummary.h" #include "llvm/ProfileData/InstrProf.h" -#include "llvm/ProfileData/SampleProf.h" +#include "llvm/ProfileData/SampleProf.h" #include "llvm/Support/Error.h" #include <algorithm> #include <cstdint> @@ -41,8 +41,8 @@ class FunctionSamples; } // end namespace sampleprof -inline const char *getHotSectionPrefix() { return "hot"; } -inline const char *getUnlikelySectionPrefix() { return "unlikely"; } +inline const char *getHotSectionPrefix() { return "hot"; } +inline const char *getUnlikelySectionPrefix() { return "unlikely"; } class ProfileSummaryBuilder { private: @@ -97,8 +97,8 @@ public: void addRecord(const sampleprof::FunctionSamples &FS, bool isCallsiteSample = false); - std::unique_ptr<ProfileSummary> computeSummaryForProfiles( - const StringMap<sampleprof::FunctionSamples> &Profiles); + std::unique_ptr<ProfileSummary> computeSummaryForProfiles( + const StringMap<sampleprof::FunctionSamples> &Profiles); std::unique_ptr<ProfileSummary> getSummary(); }; diff --git a/contrib/libs/llvm12/include/llvm/ProfileData/SampleProf.h b/contrib/libs/llvm12/include/llvm/ProfileData/SampleProf.h index 669bd70d3a..5478b9a474 100644 --- a/contrib/libs/llvm12/include/llvm/ProfileData/SampleProf.h +++ b/contrib/libs/llvm12/include/llvm/ProfileData/SampleProf.h @@ -61,8 +61,8 @@ enum class sampleprof_error { ostream_seek_unsupported, compress_failed, uncompress_failed, - zlib_unavailable, - hash_mismatch + zlib_unavailable, + hash_mismatch }; inline std::error_code make_error_code(sampleprof_error E) { @@ -128,7 +128,7 @@ enum SecType { SecNameTable = 2, SecProfileSymbolList = 3, SecFuncOffsetTable = 4, - SecFuncMetadata = 5, + SecFuncMetadata = 5, // marker for the first type of profile. SecFuncProfileFirst = 32, SecLBRProfile = SecFuncProfileFirst @@ -146,8 +146,8 @@ static inline std::string getSecName(SecType Type) { return "ProfileSymbolListSection"; case SecFuncOffsetTable: return "FuncOffsetTableSection"; - case SecFuncMetadata: - return "FunctionMetadata"; + case SecFuncMetadata: + return "FunctionMetadata"; case SecLBRProfile: return "LBRProfileSection"; } @@ -161,9 +161,9 @@ struct SecHdrTableEntry { uint64_t Flags; uint64_t Offset; uint64_t Size; - // The index indicating the location of the current entry in - // SectionHdrLayout table. - uint32_t LayoutIndex; + // The index indicating the location of the current entry in + // SectionHdrLayout table. + uint32_t LayoutIndex; }; // Flags common for all sections are defined here. In SecHdrTableEntry::Flags, @@ -171,9 +171,9 @@ struct SecHdrTableEntry { // will be saved in the higher 32 bits. enum class SecCommonFlags : uint32_t { SecFlagInValid = 0, - SecFlagCompress = (1 << 0), - // Indicate the section contains only profile without context. - SecFlagFlat = (1 << 1) + SecFlagCompress = (1 << 0), + // Indicate the section contains only profile without context. + SecFlagFlat = (1 << 1) }; // Section specific flags are defined here. @@ -181,10 +181,10 @@ enum class SecCommonFlags : uint32_t { // a new check in verifySecFlag. enum class SecNameTableFlags : uint32_t { SecFlagInValid = 0, - SecFlagMD5Name = (1 << 0), - // Store MD5 in fixed length instead of ULEB128 so NameTable can be - // accessed like an array. - SecFlagFixedLengthMD5 = (1 << 1) + SecFlagMD5Name = (1 << 0), + // Store MD5 in fixed length instead of ULEB128 so NameTable can be + // accessed like an array. + SecFlagFixedLengthMD5 = (1 << 1) }; enum class SecProfSummaryFlags : uint32_t { SecFlagInValid = 0, @@ -194,11 +194,11 @@ enum class SecProfSummaryFlags : uint32_t { SecFlagPartial = (1 << 0) }; -enum class SecFuncMetadataFlags : uint32_t { - SecFlagInvalid = 0, - SecFlagIsProbeBased = (1 << 0), -}; - +enum class SecFuncMetadataFlags : uint32_t { + SecFlagInvalid = 0, + SecFlagIsProbeBased = (1 << 0), +}; + // Verify section specific flag is used for the correct section. template <class SecFlagType> static inline void verifySecFlag(SecType Type, SecFlagType Flag) { @@ -215,9 +215,9 @@ static inline void verifySecFlag(SecType Type, SecFlagType Flag) { case SecProfSummary: IsFlagLegal = std::is_same<SecProfSummaryFlags, SecFlagType>(); break; - case SecFuncMetadata: - IsFlagLegal = std::is_same<SecFuncMetadataFlags, SecFlagType>(); - break; + case SecFuncMetadata: + IsFlagLegal = std::is_same<SecFuncMetadataFlags, SecFlagType>(); + break; default: break; } @@ -269,14 +269,14 @@ struct LineLocation { (LineOffset == O.LineOffset && Discriminator < O.Discriminator); } - bool operator==(const LineLocation &O) const { - return LineOffset == O.LineOffset && Discriminator == O.Discriminator; - } - - bool operator!=(const LineLocation &O) const { - return LineOffset != O.LineOffset || Discriminator != O.Discriminator; - } - + bool operator==(const LineLocation &O) const { + return LineOffset == O.LineOffset && Discriminator == O.Discriminator; + } + + bool operator!=(const LineLocation &O) const { + return LineOffset != O.LineOffset || Discriminator != O.Discriminator; + } + uint32_t LineOffset; uint32_t Discriminator; }; @@ -354,16 +354,16 @@ public: return SortedTargets; } - /// Prorate call targets by a distribution factor. - static const CallTargetMap adjustCallTargets(const CallTargetMap &Targets, - float DistributionFactor) { - CallTargetMap AdjustedTargets; - for (const auto &I : Targets) { - AdjustedTargets[I.first()] = I.second * DistributionFactor; - } - return AdjustedTargets; - } - + /// Prorate call targets by a distribution factor. + static const CallTargetMap adjustCallTargets(const CallTargetMap &Targets, + float DistributionFactor) { + CallTargetMap AdjustedTargets; + for (const auto &I : Targets) { + AdjustedTargets[I.first()] = I.second * DistributionFactor; + } + return AdjustedTargets; + } + /// Merge the samples in \p Other into this record. /// Optionally scale sample counts by \p Weight. sampleprof_error merge(const SampleRecord &Other, uint64_t Weight = 1) { @@ -384,137 +384,137 @@ private: raw_ostream &operator<<(raw_ostream &OS, const SampleRecord &Sample); -// State of context associated with FunctionSamples -enum ContextStateMask { - UnknownContext = 0x0, // Profile without context - RawContext = 0x1, // Full context profile from input profile - SyntheticContext = 0x2, // Synthetic context created for context promotion - InlinedContext = 0x4, // Profile for context that is inlined into caller - MergedContext = 0x8 // Profile for context merged into base profile -}; - -// Sample context for FunctionSamples. It consists of the calling context, -// the function name and context state. Internally sample context is represented -// using StringRef, which is also the input for constructing a `SampleContext`. -// It can accept and represent both full context string as well as context-less -// function name. -// Example of full context string (note the wrapping `[]`): -// `[main:3 @ _Z5funcAi:1 @ _Z8funcLeafi]` -// Example of context-less function name (same as AutoFDO): -// `_Z8funcLeafi` -class SampleContext { -public: - SampleContext() : State(UnknownContext) {} - SampleContext(StringRef ContextStr, - ContextStateMask CState = UnknownContext) { - setContext(ContextStr, CState); - } - - // Promote context by removing top frames (represented by `ContextStrToRemove`). - // Note that with string representation of context, the promotion is effectively - // a substr operation with `ContextStrToRemove` removed from left. - void promoteOnPath(StringRef ContextStrToRemove) { - assert(FullContext.startswith(ContextStrToRemove)); - - // Remove leading context and frame separator " @ ". - FullContext = FullContext.substr(ContextStrToRemove.size() + 3); - CallingContext = CallingContext.substr(ContextStrToRemove.size() + 3); - } - - // Split the top context frame (left-most substr) from context. - static std::pair<StringRef, StringRef> - splitContextString(StringRef ContextStr) { - return ContextStr.split(" @ "); - } - - // Decode context string for a frame to get function name and location. - // `ContextStr` is in the form of `FuncName:StartLine.Discriminator`. - static void decodeContextString(StringRef ContextStr, StringRef &FName, - LineLocation &LineLoc) { - // Get function name - auto EntrySplit = ContextStr.split(':'); - FName = EntrySplit.first; - - LineLoc = {0, 0}; - if (!EntrySplit.second.empty()) { - // Get line offset, use signed int for getAsInteger so string will - // be parsed as signed. - int LineOffset = 0; - auto LocSplit = EntrySplit.second.split('.'); - LocSplit.first.getAsInteger(10, LineOffset); - LineLoc.LineOffset = LineOffset; - - // Get discriminator - if (!LocSplit.second.empty()) - LocSplit.second.getAsInteger(10, LineLoc.Discriminator); - } - } - - operator StringRef() const { return FullContext; } - bool hasState(ContextStateMask S) { return State & (uint32_t)S; } - void setState(ContextStateMask S) { State |= (uint32_t)S; } - void clearState(ContextStateMask S) { State &= (uint32_t)~S; } - bool hasContext() const { return State != UnknownContext; } - bool isBaseContext() const { return CallingContext.empty(); } - StringRef getNameWithoutContext() const { return Name; } - StringRef getCallingContext() const { return CallingContext; } - StringRef getNameWithContext(bool WithBracket = false) const { - return WithBracket ? InputContext : FullContext; - } - -private: - // Give a context string, decode and populate internal states like - // Function name, Calling context and context state. Example of input - // `ContextStr`: `[main:3 @ _Z5funcAi:1 @ _Z8funcLeafi]` - void setContext(StringRef ContextStr, ContextStateMask CState) { - assert(!ContextStr.empty()); - InputContext = ContextStr; - // Note that `[]` wrapped input indicates a full context string, otherwise - // it's treated as context-less function name only. - bool HasContext = ContextStr.startswith("["); - if (!HasContext && CState == UnknownContext) { - State = UnknownContext; - Name = FullContext = ContextStr; - } else { - // Assume raw context profile if unspecified - if (CState == UnknownContext) - State = RawContext; - else - State = CState; - - // Remove encapsulating '[' and ']' if any - if (HasContext) - FullContext = ContextStr.substr(1, ContextStr.size() - 2); - else - FullContext = ContextStr; - - // Caller is to the left of callee in context string - auto NameContext = FullContext.rsplit(" @ "); - if (NameContext.second.empty()) { - Name = NameContext.first; - CallingContext = NameContext.second; - } else { - Name = NameContext.second; - CallingContext = NameContext.first; - } - } - } - - // Input context string including bracketed calling context and leaf function - // name - StringRef InputContext; - // Full context string including calling context and leaf function name - StringRef FullContext; - // Function name for the associated sample profile - StringRef Name; - // Calling context (leaf function excluded) for the associated sample profile - StringRef CallingContext; - // State of the associated sample profile - uint32_t State; -}; - +// State of context associated with FunctionSamples +enum ContextStateMask { + UnknownContext = 0x0, // Profile without context + RawContext = 0x1, // Full context profile from input profile + SyntheticContext = 0x2, // Synthetic context created for context promotion + InlinedContext = 0x4, // Profile for context that is inlined into caller + MergedContext = 0x8 // Profile for context merged into base profile +}; + +// Sample context for FunctionSamples. It consists of the calling context, +// the function name and context state. Internally sample context is represented +// using StringRef, which is also the input for constructing a `SampleContext`. +// It can accept and represent both full context string as well as context-less +// function name. +// Example of full context string (note the wrapping `[]`): +// `[main:3 @ _Z5funcAi:1 @ _Z8funcLeafi]` +// Example of context-less function name (same as AutoFDO): +// `_Z8funcLeafi` +class SampleContext { +public: + SampleContext() : State(UnknownContext) {} + SampleContext(StringRef ContextStr, + ContextStateMask CState = UnknownContext) { + setContext(ContextStr, CState); + } + + // Promote context by removing top frames (represented by `ContextStrToRemove`). + // Note that with string representation of context, the promotion is effectively + // a substr operation with `ContextStrToRemove` removed from left. + void promoteOnPath(StringRef ContextStrToRemove) { + assert(FullContext.startswith(ContextStrToRemove)); + + // Remove leading context and frame separator " @ ". + FullContext = FullContext.substr(ContextStrToRemove.size() + 3); + CallingContext = CallingContext.substr(ContextStrToRemove.size() + 3); + } + + // Split the top context frame (left-most substr) from context. + static std::pair<StringRef, StringRef> + splitContextString(StringRef ContextStr) { + return ContextStr.split(" @ "); + } + + // Decode context string for a frame to get function name and location. + // `ContextStr` is in the form of `FuncName:StartLine.Discriminator`. + static void decodeContextString(StringRef ContextStr, StringRef &FName, + LineLocation &LineLoc) { + // Get function name + auto EntrySplit = ContextStr.split(':'); + FName = EntrySplit.first; + + LineLoc = {0, 0}; + if (!EntrySplit.second.empty()) { + // Get line offset, use signed int for getAsInteger so string will + // be parsed as signed. + int LineOffset = 0; + auto LocSplit = EntrySplit.second.split('.'); + LocSplit.first.getAsInteger(10, LineOffset); + LineLoc.LineOffset = LineOffset; + + // Get discriminator + if (!LocSplit.second.empty()) + LocSplit.second.getAsInteger(10, LineLoc.Discriminator); + } + } + + operator StringRef() const { return FullContext; } + bool hasState(ContextStateMask S) { return State & (uint32_t)S; } + void setState(ContextStateMask S) { State |= (uint32_t)S; } + void clearState(ContextStateMask S) { State &= (uint32_t)~S; } + bool hasContext() const { return State != UnknownContext; } + bool isBaseContext() const { return CallingContext.empty(); } + StringRef getNameWithoutContext() const { return Name; } + StringRef getCallingContext() const { return CallingContext; } + StringRef getNameWithContext(bool WithBracket = false) const { + return WithBracket ? InputContext : FullContext; + } + +private: + // Give a context string, decode and populate internal states like + // Function name, Calling context and context state. Example of input + // `ContextStr`: `[main:3 @ _Z5funcAi:1 @ _Z8funcLeafi]` + void setContext(StringRef ContextStr, ContextStateMask CState) { + assert(!ContextStr.empty()); + InputContext = ContextStr; + // Note that `[]` wrapped input indicates a full context string, otherwise + // it's treated as context-less function name only. + bool HasContext = ContextStr.startswith("["); + if (!HasContext && CState == UnknownContext) { + State = UnknownContext; + Name = FullContext = ContextStr; + } else { + // Assume raw context profile if unspecified + if (CState == UnknownContext) + State = RawContext; + else + State = CState; + + // Remove encapsulating '[' and ']' if any + if (HasContext) + FullContext = ContextStr.substr(1, ContextStr.size() - 2); + else + FullContext = ContextStr; + + // Caller is to the left of callee in context string + auto NameContext = FullContext.rsplit(" @ "); + if (NameContext.second.empty()) { + Name = NameContext.first; + CallingContext = NameContext.second; + } else { + Name = NameContext.second; + CallingContext = NameContext.first; + } + } + } + + // Input context string including bracketed calling context and leaf function + // name + StringRef InputContext; + // Full context string including calling context and leaf function name + StringRef FullContext; + // Function name for the associated sample profile + StringRef Name; + // Calling context (leaf function excluded) for the associated sample profile + StringRef CallingContext; + // State of the associated sample profile + uint32_t State; +}; + class FunctionSamples; -class SampleProfileReaderItaniumRemapper; +class SampleProfileReaderItaniumRemapper; using BodySampleMap = std::map<LineLocation, SampleRecord>; // NOTE: Using a StringMap here makes parsed profiles consume around 17% more @@ -542,8 +542,8 @@ public: : sampleprof_error::success; } - void setTotalSamples(uint64_t Num) { TotalSamples = Num; } - + void setTotalSamples(uint64_t Num) { TotalSamples = Num; } + sampleprof_error addHeadSamples(uint64_t Num, uint64_t Weight = 1) { bool Overflowed; TotalHeadSamples = @@ -572,22 +572,22 @@ public: ErrorOr<uint64_t> findSamplesAt(uint32_t LineOffset, uint32_t Discriminator) const { const auto &ret = BodySamples.find(LineLocation(LineOffset, Discriminator)); - if (ret == BodySamples.end()) { - // For CSSPGO, in order to conserve profile size, we no longer write out - // locations profile for those not hit during training, so we need to - // treat them as zero instead of error here. - if (ProfileIsCS) - return 0; - return std::error_code(); - // A missing counter for a probe likely means the probe was not executed. - // Treat it as a zero count instead of an unknown count to help edge - // weight inference. - if (FunctionSamples::ProfileIsProbeBased) - return 0; + if (ret == BodySamples.end()) { + // For CSSPGO, in order to conserve profile size, we no longer write out + // locations profile for those not hit during training, so we need to + // treat them as zero instead of error here. + if (ProfileIsCS) + return 0; return std::error_code(); - } else { + // A missing counter for a probe likely means the probe was not executed. + // Treat it as a zero count instead of an unknown count to help edge + // weight inference. + if (FunctionSamples::ProfileIsProbeBased) + return 0; + return std::error_code(); + } else { return ret->second.getSamples(); - } + } } /// Returns the call target map collected at a given location. @@ -601,16 +601,16 @@ public: return ret->second.getCallTargets(); } - /// Returns the call target map collected at a given location specified by \p - /// CallSite. If the location is not found in profile, return error. - ErrorOr<SampleRecord::CallTargetMap> - findCallTargetMapAt(const LineLocation &CallSite) const { - const auto &Ret = BodySamples.find(CallSite); - if (Ret == BodySamples.end()) - return std::error_code(); - return Ret->second.getCallTargets(); - } - + /// Returns the call target map collected at a given location specified by \p + /// CallSite. If the location is not found in profile, return error. + ErrorOr<SampleRecord::CallTargetMap> + findCallTargetMapAt(const LineLocation &CallSite) const { + const auto &Ret = BodySamples.find(CallSite); + if (Ret == BodySamples.end()) + return std::error_code(); + return Ret->second.getCallTargets(); + } + /// Return the function samples at the given callsite location. FunctionSamplesMap &functionSamplesAt(const LineLocation &Loc) { return CallsiteSamples[Loc]; @@ -625,15 +625,15 @@ public: return &iter->second; } - /// Returns a pointer to FunctionSamples at the given callsite location - /// \p Loc with callee \p CalleeName. If no callsite can be found, relax - /// the restriction to return the FunctionSamples at callsite location - /// \p Loc with the maximum total sample count. If \p Remapper is not - /// nullptr, use \p Remapper to find FunctionSamples with equivalent name - /// as \p CalleeName. - const FunctionSamples * - findFunctionSamplesAt(const LineLocation &Loc, StringRef CalleeName, - SampleProfileReaderItaniumRemapper *Remapper) const; + /// Returns a pointer to FunctionSamples at the given callsite location + /// \p Loc with callee \p CalleeName. If no callsite can be found, relax + /// the restriction to return the FunctionSamples at callsite location + /// \p Loc with the maximum total sample count. If \p Remapper is not + /// nullptr, use \p Remapper to find FunctionSamples with equivalent name + /// as \p CalleeName. + const FunctionSamples * + findFunctionSamplesAt(const LineLocation &Loc, StringRef CalleeName, + SampleProfileReaderItaniumRemapper *Remapper) const; bool empty() const { return TotalSamples == 0; } @@ -650,11 +650,11 @@ public: /// Return the sample count of the first instruction of the function. /// The function can be either a standalone symbol or an inlined function. uint64_t getEntrySamples() const { - if (FunctionSamples::ProfileIsCS && getHeadSamples()) { - // For CS profile, if we already have more accurate head samples - // counted by branch sample from caller, use them as entry samples. - return getHeadSamples(); - } + if (FunctionSamples::ProfileIsCS && getHeadSamples()) { + // For CS profile, if we already have more accurate head samples + // counted by branch sample from caller, use them as entry samples. + return getHeadSamples(); + } uint64_t Count = 0; // Use either BodySamples or CallsiteSamples which ever has the smaller // lineno. @@ -697,24 +697,24 @@ public: sampleprof_error merge(const FunctionSamples &Other, uint64_t Weight = 1) { sampleprof_error Result = sampleprof_error::success; Name = Other.getName(); - if (!GUIDToFuncNameMap) - GUIDToFuncNameMap = Other.GUIDToFuncNameMap; - if (Context.getNameWithContext(true).empty()) - Context = Other.getContext(); - if (FunctionHash == 0) { - // Set the function hash code for the target profile. - FunctionHash = Other.getFunctionHash(); - } else if (FunctionHash != Other.getFunctionHash()) { - // The two profiles coming with different valid hash codes indicates - // either: - // 1. They are same-named static functions from different compilation - // units (without using -unique-internal-linkage-names), or - // 2. They are really the same function but from different compilations. - // Let's bail out in either case for now, which means one profile is - // dropped. - return sampleprof_error::hash_mismatch; - } - + if (!GUIDToFuncNameMap) + GUIDToFuncNameMap = Other.GUIDToFuncNameMap; + if (Context.getNameWithContext(true).empty()) + Context = Other.getContext(); + if (FunctionHash == 0) { + // Set the function hash code for the target profile. + FunctionHash = Other.getFunctionHash(); + } else if (FunctionHash != Other.getFunctionHash()) { + // The two profiles coming with different valid hash codes indicates + // either: + // 1. They are same-named static functions from different compilation + // units (without using -unique-internal-linkage-names), or + // 2. They are really the same function but from different compilations. + // Let's bail out in either case for now, which means one profile is + // dropped. + return sampleprof_error::hash_mismatch; + } + MergeResult(Result, addTotalSamples(Other.getTotalSamples(), Weight)); MergeResult(Result, addHeadSamples(Other.getHeadSamples(), Weight)); for (const auto &I : Other.getBodySamples()) { @@ -766,34 +766,34 @@ public: /// Return the function name. StringRef getName() const { return Name; } - /// Return function name with context. - StringRef getNameWithContext(bool WithBracket = false) const { - return FunctionSamples::ProfileIsCS - ? Context.getNameWithContext(WithBracket) - : Name; - } - + /// Return function name with context. + StringRef getNameWithContext(bool WithBracket = false) const { + return FunctionSamples::ProfileIsCS + ? Context.getNameWithContext(WithBracket) + : Name; + } + /// Return the original function name. StringRef getFuncName() const { return getFuncName(Name); } - void setFunctionHash(uint64_t Hash) { FunctionHash = Hash; } - - uint64_t getFunctionHash() const { return FunctionHash; } - + void setFunctionHash(uint64_t Hash) { FunctionHash = Hash; } + + uint64_t getFunctionHash() const { return FunctionHash; } + /// Return the canonical name for a function, taking into account /// suffix elision policy attributes. static StringRef getCanonicalFnName(const Function &F) { auto AttrName = "sample-profile-suffix-elision-policy"; auto Attr = F.getFnAttribute(AttrName).getValueAsString(); - return getCanonicalFnName(F.getName(), Attr); - } - - static StringRef getCanonicalFnName(StringRef FnName, StringRef Attr = "") { - static const char *knownSuffixes[] = { ".llvm.", ".part." }; + return getCanonicalFnName(F.getName(), Attr); + } + + static StringRef getCanonicalFnName(StringRef FnName, StringRef Attr = "") { + static const char *knownSuffixes[] = { ".llvm.", ".part." }; if (Attr == "" || Attr == "all") { - return FnName.split('.').first; + return FnName.split('.').first; } else if (Attr == "selected") { - StringRef Cand(FnName); + StringRef Cand(FnName); for (const auto &Suf : knownSuffixes) { StringRef Suffix(Suf); auto It = Cand.rfind(Suffix); @@ -805,11 +805,11 @@ public: } return Cand; } else if (Attr == "none") { - return FnName; + return FnName; } else { assert(false && "internal error: unknown suffix elision policy"); } - return FnName; + return FnName; } /// Translate \p Name into its original name. @@ -824,19 +824,19 @@ public: return Name; assert(GUIDToFuncNameMap && "GUIDToFuncNameMap needs to be popluated first"); - return GUIDToFuncNameMap->lookup(std::stoull(Name.data())); + return GUIDToFuncNameMap->lookup(std::stoull(Name.data())); } /// Returns the line offset to the start line of the subprogram. /// We assume that a single function will not exceed 65535 LOC. static unsigned getOffset(const DILocation *DIL); - /// Returns a unique call site identifier for a given debug location of a call - /// instruction. This is wrapper of two scenarios, the probe-based profile and - /// regular profile, to hide implementation details from the sample loader and - /// the context tracker. - static LineLocation getCallSiteIdentifier(const DILocation *DIL); - + /// Returns a unique call site identifier for a given debug location of a call + /// instruction. This is wrapper of two scenarios, the probe-based profile and + /// regular profile, to hide implementation details from the sample loader and + /// the context tracker. + static LineLocation getCallSiteIdentifier(const DILocation *DIL); + /// Get the FunctionSamples of the inline instance where DIL originates /// from. /// @@ -846,20 +846,20 @@ public: /// tree nodes in the profile. /// /// \returns the FunctionSamples pointer to the inlined instance. - /// If \p Remapper is not nullptr, it will be used to find matching - /// FunctionSamples with not exactly the same but equivalent name. - const FunctionSamples *findFunctionSamples( - const DILocation *DIL, - SampleProfileReaderItaniumRemapper *Remapper = nullptr) const; - - static bool ProfileIsProbeBased; - - static bool ProfileIsCS; - - SampleContext &getContext() const { return Context; } - - void setContext(const SampleContext &FContext) { Context = FContext; } - + /// If \p Remapper is not nullptr, it will be used to find matching + /// FunctionSamples with not exactly the same but equivalent name. + const FunctionSamples *findFunctionSamples( + const DILocation *DIL, + SampleProfileReaderItaniumRemapper *Remapper = nullptr) const; + + static bool ProfileIsProbeBased; + + static bool ProfileIsCS; + + SampleContext &getContext() const { return Context; } + + void setContext(const SampleContext &FContext) { Context = FContext; } + static SampleProfileFormat Format; /// Whether the profile uses MD5 to represent string. @@ -876,20 +876,20 @@ public: return UseMD5 ? std::stoull(Name.data()) : Function::getGUID(Name); } - // Find all the names in the current FunctionSamples including names in - // all the inline instances and names of call targets. - void findAllNames(DenseSet<StringRef> &NameSet) const; - + // Find all the names in the current FunctionSamples including names in + // all the inline instances and names of call targets. + void findAllNames(DenseSet<StringRef> &NameSet) const; + private: /// Mangled name of the function. StringRef Name; - /// CFG hash value for the function. - uint64_t FunctionHash = 0; - - /// Calling context for function profile - mutable SampleContext Context; - + /// CFG hash value for the function. + uint64_t FunctionHash = 0; + + /// Calling context for function profile + mutable SampleContext Context; + /// Total number of samples collected inside this function. /// /// Samples are cumulative, they include all the samples collected diff --git a/contrib/libs/llvm12/include/llvm/ProfileData/SampleProfReader.h b/contrib/libs/llvm12/include/llvm/ProfileData/SampleProfReader.h index 7cece5e376..62cca31d48 100644 --- a/contrib/libs/llvm12/include/llvm/ProfileData/SampleProfReader.h +++ b/contrib/libs/llvm12/include/llvm/ProfileData/SampleProfReader.h @@ -34,9 +34,9 @@ // offsetA[.discriminator]: fnA:num_of_total_samples // offsetA1[.discriminator]: number_of_samples [fn7:num fn8:num ... ] // ... -// !CFGChecksum: num +// !CFGChecksum: num // -// This is a nested tree in which the indentation represents the nesting level +// This is a nested tree in which the indentation represents the nesting level // of the inline stack. There are no blank lines in the file. And the spacing // within a single line is fixed. Additional spaces will result in an error // while reading the file. @@ -55,11 +55,11 @@ // in the prologue of the function (second number). This head sample // count provides an indicator of how frequently the function is invoked. // -// There are three types of lines in the function body. +// There are three types of lines in the function body. // // * Sampled line represents the profile information of a source location. // * Callsite line represents the profile information of a callsite. -// * Metadata line represents extra metadata of the function. +// * Metadata line represents extra metadata of the function. // // Each sampled line may contain several items. Some are optional (marked // below): @@ -123,19 +123,19 @@ // total number of samples collected for the inlined instance at this // callsite // -// Metadata line can occur in lines with one indent only, containing extra -// information for the top-level function. Furthermore, metadata can only -// occur after all the body samples and callsite samples. -// Each metadata line may contain a particular type of metadata, marked by -// the starting characters annotated with !. We process each metadata line -// independently, hence each metadata line has to form an independent piece -// of information that does not require cross-line reference. -// We support the following types of metadata: -// -// a. CFG Checksum (a.k.a. function hash): -// !CFGChecksum: 12345 -// -// +// Metadata line can occur in lines with one indent only, containing extra +// information for the top-level function. Furthermore, metadata can only +// occur after all the body samples and callsite samples. +// Each metadata line may contain a particular type of metadata, marked by +// the starting characters annotated with !. We process each metadata line +// independently, hence each metadata line has to form an independent piece +// of information that does not require cross-line reference. +// We support the following types of metadata: +// +// a. CFG Checksum (a.k.a. function hash): +// !CFGChecksum: 12345 +// +// // Binary format // ------------- // @@ -229,7 +229,7 @@ #ifndef LLVM_PROFILEDATA_SAMPLEPROFREADER_H #define LLVM_PROFILEDATA_SAMPLEPROFREADER_H -#include "llvm/ADT/Optional.h" +#include "llvm/ADT/Optional.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" @@ -253,7 +253,7 @@ namespace llvm { class raw_ostream; -class Twine; +class Twine; namespace sampleprof { @@ -297,18 +297,18 @@ public: return Remappings->lookup(FunctionName); } - /// Return the equivalent name in the profile for \p FunctionName if - /// it exists. - Optional<StringRef> lookUpNameInProfile(StringRef FunctionName); + /// Return the equivalent name in the profile for \p FunctionName if + /// it exists. + Optional<StringRef> lookUpNameInProfile(StringRef FunctionName); private: // The buffer holding the content read from remapping file. std::unique_ptr<MemoryBuffer> Buffer; std::unique_ptr<SymbolRemappingReader> Remappings; - // Map remapping key to the name in the profile. By looking up the - // key in the remapper, a given new name can be mapped to the - // cannonical name using the NameMap. - DenseMap<SymbolRemappingReader::Key, StringRef> NameMap; + // Map remapping key to the name in the profile. By looking up the + // key in the remapper, a given new name can be mapped to the + // cannonical name using the NameMap. + DenseMap<SymbolRemappingReader::Key, StringRef> NameMap; // The Reader the remapper is servicing. SampleProfileReader &Reader; // Indicate whether remapping has been applied to the profile read @@ -400,14 +400,14 @@ public: auto It = Profiles.find(Fname); if (It != Profiles.end()) return &It->second; - - if (Remapper) { - if (auto NameInProfile = Remapper->lookUpNameInProfile(Fname)) { - auto It = Profiles.find(*NameInProfile); - if (It != Profiles.end()) - return &It->second; - } - } + + if (Remapper) { + if (auto NameInProfile = Remapper->lookUpNameInProfile(Fname)) { + auto It = Profiles.find(*NameInProfile); + if (It != Profiles.end()) + return &It->second; + } + } return nullptr; } @@ -415,7 +415,7 @@ public: StringMap<FunctionSamples> &getProfiles() { return Profiles; } /// Report a parse error message. - void reportError(int64_t LineNumber, const Twine &Msg) const { + void reportError(int64_t LineNumber, const Twine &Msg) const { Ctx.diagnose(DiagnosticInfoSampleProfile(Buffer->getBufferIdentifier(), LineNumber, Msg)); } @@ -440,12 +440,12 @@ public: /// \brief Return the profile format. SampleProfileFormat getFormat() const { return Format; } - /// Whether input profile is based on pseudo probes. - bool profileIsProbeBased() const { return ProfileIsProbeBased; } - - /// Whether input profile is fully context-sensitive - bool profileIsCS() const { return ProfileIsCS; } - + /// Whether input profile is based on pseudo probes. + bool profileIsProbeBased() const { return ProfileIsProbeBased; } + + /// Whether input profile is fully context-sensitive + bool profileIsCS() const { return ProfileIsCS; } + virtual std::unique_ptr<ProfileSymbolList> getProfileSymbolList() { return nullptr; }; @@ -458,12 +458,12 @@ public: /// Return whether names in the profile are all MD5 numbers. virtual bool useMD5() { return false; } - /// Don't read profile without context if the flag is set. This is only meaningful - /// for ExtBinary format. - virtual void setSkipFlatProf(bool Skip) {} - - SampleProfileReaderItaniumRemapper *getRemapper() { return Remapper.get(); } - + /// Don't read profile without context if the flag is set. This is only meaningful + /// for ExtBinary format. + virtual void setSkipFlatProf(bool Skip) {} + + SampleProfileReaderItaniumRemapper *getRemapper() { return Remapper.get(); } + protected: /// Map every function to its associated profile. /// @@ -492,15 +492,15 @@ protected: std::unique_ptr<SampleProfileReaderItaniumRemapper> Remapper; - /// \brief Whether samples are collected based on pseudo probes. - bool ProfileIsProbeBased = false; - - /// Whether function profiles are context-sensitive. - bool ProfileIsCS = false; - - /// Number of context-sensitive profiles. - uint32_t CSProfileCount = 0; - + /// \brief Whether samples are collected based on pseudo probes. + bool ProfileIsProbeBased = false; + + /// Whether function profiles are context-sensitive. + bool ProfileIsCS = false; + + /// Number of context-sensitive profiles. + uint32_t CSProfileCount = 0; + /// \brief The format of sample. SampleProfileFormat Format = SPF_None; }; @@ -638,26 +638,26 @@ private: protected: std::vector<SecHdrTableEntry> SecHdrTable; - std::error_code readSecHdrTableEntry(uint32_t Idx); + std::error_code readSecHdrTableEntry(uint32_t Idx); std::error_code readSecHdrTable(); - std::error_code readFuncMetadata(); + std::error_code readFuncMetadata(); std::error_code readFuncOffsetTable(); std::error_code readFuncProfiles(); std::error_code readMD5NameTable(); std::error_code readNameTableSec(bool IsMD5); - std::error_code readProfileSymbolList(); - - virtual std::error_code readHeader() override; - virtual std::error_code verifySPMagic(uint64_t Magic) override = 0; - virtual std::error_code readOneSection(const uint8_t *Start, uint64_t Size, - const SecHdrTableEntry &Entry); - // placeholder for subclasses to dispatch their own section readers. - virtual std::error_code readCustomSection(const SecHdrTableEntry &Entry) = 0; - virtual ErrorOr<StringRef> readStringFromTable() override; - - std::unique_ptr<ProfileSymbolList> ProfSymList; - + std::error_code readProfileSymbolList(); + + virtual std::error_code readHeader() override; + virtual std::error_code verifySPMagic(uint64_t Magic) override = 0; + virtual std::error_code readOneSection(const uint8_t *Start, uint64_t Size, + const SecHdrTableEntry &Entry); + // placeholder for subclasses to dispatch their own section readers. + virtual std::error_code readCustomSection(const SecHdrTableEntry &Entry) = 0; + virtual ErrorOr<StringRef> readStringFromTable() override; + + std::unique_ptr<ProfileSymbolList> ProfSymList; + /// The table mapping from function name to the offset of its FunctionSample /// towards file start. DenseMap<StringRef, uint64_t> FuncOffsetTable; @@ -666,12 +666,12 @@ protected: /// Use all functions from the input profile. bool UseAllFuncs = true; - /// Use fixed length MD5 instead of ULEB128 encoding so NameTable doesn't - /// need to be read in up front and can be directly accessed using index. - bool FixedLengthMD5 = false; - /// The starting address of NameTable containing fixed length MD5. - const uint8_t *MD5NameMemStart = nullptr; - + /// Use fixed length MD5 instead of ULEB128 encoding so NameTable doesn't + /// need to be read in up front and can be directly accessed using index. + bool FixedLengthMD5 = false; + /// The starting address of NameTable containing fixed length MD5. + const uint8_t *MD5NameMemStart = nullptr; + /// If MD5 is used in NameTable section, the section saves uint64_t data. /// The uint64_t data has to be converted to a string and then the string /// will be used to initialize StringRef in NameTable. @@ -681,54 +681,54 @@ protected: /// the lifetime of MD5StringBuf is not shorter than that of NameTable. std::unique_ptr<std::vector<std::string>> MD5StringBuf; - /// If SkipFlatProf is true, skip the sections with - /// SecFlagFlat flag. - bool SkipFlatProf = false; - + /// If SkipFlatProf is true, skip the sections with + /// SecFlagFlat flag. + bool SkipFlatProf = false; + public: - SampleProfileReaderExtBinaryBase(std::unique_ptr<MemoryBuffer> B, - LLVMContext &C, SampleProfileFormat Format) - : SampleProfileReaderBinary(std::move(B), C, Format) {} + SampleProfileReaderExtBinaryBase(std::unique_ptr<MemoryBuffer> B, + LLVMContext &C, SampleProfileFormat Format) + : SampleProfileReaderBinary(std::move(B), C, Format) {} - /// Read sample profiles in extensible format from the associated file. - std::error_code readImpl() override; + /// Read sample profiles in extensible format from the associated file. + std::error_code readImpl() override; - /// Get the total size of all \p Type sections. - uint64_t getSectionSize(SecType Type); - /// Get the total size of header and all sections. - uint64_t getFileSize(); - virtual bool dumpSectionInfo(raw_ostream &OS = dbgs()) override; + /// Get the total size of all \p Type sections. + uint64_t getSectionSize(SecType Type); + /// Get the total size of header and all sections. + uint64_t getFileSize(); + virtual bool dumpSectionInfo(raw_ostream &OS = dbgs()) override; /// Collect functions with definitions in Module \p M. void collectFuncsFrom(const Module &M) override; /// Return whether names in the profile are all MD5 numbers. - virtual bool useMD5() override { return MD5StringBuf.get(); } - - virtual std::unique_ptr<ProfileSymbolList> getProfileSymbolList() override { - return std::move(ProfSymList); - }; - - virtual void setSkipFlatProf(bool Skip) override { SkipFlatProf = Skip; } -}; - -class SampleProfileReaderExtBinary : public SampleProfileReaderExtBinaryBase { -private: - virtual std::error_code verifySPMagic(uint64_t Magic) override; - virtual std::error_code - readCustomSection(const SecHdrTableEntry &Entry) override { - return sampleprof_error::success; - }; - -public: - SampleProfileReaderExtBinary(std::unique_ptr<MemoryBuffer> B, LLVMContext &C, - SampleProfileFormat Format = SPF_Ext_Binary) - : SampleProfileReaderExtBinaryBase(std::move(B), C, Format) {} - - /// \brief Return true if \p Buffer is in the format supported by this class. - static bool hasFormat(const MemoryBuffer &Buffer); + virtual bool useMD5() override { return MD5StringBuf.get(); } + + virtual std::unique_ptr<ProfileSymbolList> getProfileSymbolList() override { + return std::move(ProfSymList); + }; + + virtual void setSkipFlatProf(bool Skip) override { SkipFlatProf = Skip; } }; +class SampleProfileReaderExtBinary : public SampleProfileReaderExtBinaryBase { +private: + virtual std::error_code verifySPMagic(uint64_t Magic) override; + virtual std::error_code + readCustomSection(const SecHdrTableEntry &Entry) override { + return sampleprof_error::success; + }; + +public: + SampleProfileReaderExtBinary(std::unique_ptr<MemoryBuffer> B, LLVMContext &C, + SampleProfileFormat Format = SPF_Ext_Binary) + : SampleProfileReaderExtBinaryBase(std::move(B), C, Format) {} + + /// \brief Return true if \p Buffer is in the format supported by this class. + static bool hasFormat(const MemoryBuffer &Buffer); +}; + class SampleProfileReaderCompactBinary : public SampleProfileReaderBinary { private: /// Function name table. diff --git a/contrib/libs/llvm12/include/llvm/ProfileData/SampleProfWriter.h b/contrib/libs/llvm12/include/llvm/ProfileData/SampleProfWriter.h index ad64621ae7..e00c52f1d9 100644 --- a/contrib/libs/llvm12/include/llvm/ProfileData/SampleProfWriter.h +++ b/contrib/libs/llvm12/include/llvm/ProfileData/SampleProfWriter.h @@ -22,7 +22,7 @@ #include "llvm/ADT/MapVector.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" -#include "llvm/ADT/StringSet.h" +#include "llvm/ADT/StringSet.h" #include "llvm/IR/ProfileSummary.h" #include "llvm/ProfileData/SampleProf.h" #include "llvm/Support/ErrorOr.h" @@ -36,15 +36,15 @@ namespace llvm { namespace sampleprof { -enum SectionLayout { - DefaultLayout, - // The layout splits profile with context information from profile without - // context information. When Thinlto is enabled, ThinLTO postlink phase only - // has to load profile with context information and can skip the other part. - CtxSplitLayout, - NumOfLayout, -}; - +enum SectionLayout { + DefaultLayout, + // The layout splits profile with context information from profile without + // context information. When Thinlto is enabled, ThinLTO postlink phase only + // has to load profile with context information and can skip the other part. + CtxSplitLayout, + NumOfLayout, +}; + /// Sample-based profile writer. Base class. class SampleProfileWriter { public: @@ -77,7 +77,7 @@ public: virtual void setToCompressAllSections() {} virtual void setUseMD5() {} virtual void setPartialProfile() {} - virtual void resetSecLayout(SectionLayout SL) {} + virtual void resetSecLayout(SectionLayout SL) {} protected: SampleProfileWriter(std::unique_ptr<raw_ostream> &OS) @@ -162,36 +162,36 @@ class SampleProfileWriterRawBinary : public SampleProfileWriterBinary { using SampleProfileWriterBinary::SampleProfileWriterBinary; }; -const std::array<SmallVector<SecHdrTableEntry, 8>, NumOfLayout> - ExtBinaryHdrLayoutTable = { - // Note that SecFuncOffsetTable section is written after SecLBRProfile - // in the profile, but is put before SecLBRProfile in SectionHdrLayout. - // This is because sample reader follows the order in SectionHdrLayout - // to read each section. To read function profiles on demand, sample - // reader need to get the offset of each function profile first. - // - // DefaultLayout - SmallVector<SecHdrTableEntry, 8>({{SecProfSummary, 0, 0, 0, 0}, - {SecNameTable, 0, 0, 0, 0}, - {SecFuncOffsetTable, 0, 0, 0, 0}, - {SecLBRProfile, 0, 0, 0, 0}, - {SecProfileSymbolList, 0, 0, 0, 0}, - {SecFuncMetadata, 0, 0, 0, 0}}), - // CtxSplitLayout - SmallVector<SecHdrTableEntry, 8>({{SecProfSummary, 0, 0, 0, 0}, - {SecNameTable, 0, 0, 0, 0}, - // profile with context - // for next two sections - {SecFuncOffsetTable, 0, 0, 0, 0}, - {SecLBRProfile, 0, 0, 0, 0}, - // profile without context - // for next two sections - {SecFuncOffsetTable, 0, 0, 0, 0}, - {SecLBRProfile, 0, 0, 0, 0}, - {SecProfileSymbolList, 0, 0, 0, 0}, - {SecFuncMetadata, 0, 0, 0, 0}}), -}; - +const std::array<SmallVector<SecHdrTableEntry, 8>, NumOfLayout> + ExtBinaryHdrLayoutTable = { + // Note that SecFuncOffsetTable section is written after SecLBRProfile + // in the profile, but is put before SecLBRProfile in SectionHdrLayout. + // This is because sample reader follows the order in SectionHdrLayout + // to read each section. To read function profiles on demand, sample + // reader need to get the offset of each function profile first. + // + // DefaultLayout + SmallVector<SecHdrTableEntry, 8>({{SecProfSummary, 0, 0, 0, 0}, + {SecNameTable, 0, 0, 0, 0}, + {SecFuncOffsetTable, 0, 0, 0, 0}, + {SecLBRProfile, 0, 0, 0, 0}, + {SecProfileSymbolList, 0, 0, 0, 0}, + {SecFuncMetadata, 0, 0, 0, 0}}), + // CtxSplitLayout + SmallVector<SecHdrTableEntry, 8>({{SecProfSummary, 0, 0, 0, 0}, + {SecNameTable, 0, 0, 0, 0}, + // profile with context + // for next two sections + {SecFuncOffsetTable, 0, 0, 0, 0}, + {SecLBRProfile, 0, 0, 0, 0}, + // profile without context + // for next two sections + {SecFuncOffsetTable, 0, 0, 0, 0}, + {SecLBRProfile, 0, 0, 0, 0}, + {SecProfileSymbolList, 0, 0, 0, 0}, + {SecFuncMetadata, 0, 0, 0, 0}}), +}; + class SampleProfileWriterExtBinaryBase : public SampleProfileWriterBinary { using SampleProfileWriterBinary::SampleProfileWriterBinary; public: @@ -200,45 +200,45 @@ public: virtual void setToCompressAllSections() override; void setToCompressSection(SecType Type); - virtual std::error_code writeSample(const FunctionSamples &S) override; - - // Set to use MD5 to represent string in NameTable. - virtual void setUseMD5() override { - UseMD5 = true; - addSectionFlag(SecNameTable, SecNameTableFlags::SecFlagMD5Name); - // MD5 will be stored as plain uint64_t instead of variable-length - // quantity format in NameTable section. - addSectionFlag(SecNameTable, SecNameTableFlags::SecFlagFixedLengthMD5); - } - - // Set the profile to be partial. It means the profile is for - // common/shared code. The common profile is usually merged from - // profiles collected from running other targets. - virtual void setPartialProfile() override { - addSectionFlag(SecProfSummary, SecProfSummaryFlags::SecFlagPartial); - } - - virtual void setProfileSymbolList(ProfileSymbolList *PSL) override { - ProfSymList = PSL; - }; - - virtual void resetSecLayout(SectionLayout SL) override { - verifySecLayout(SL); -#ifndef NDEBUG - // Make sure resetSecLayout is called before any flag setting. - for (auto &Entry : SectionHdrLayout) { - assert(Entry.Flags == 0 && - "resetSecLayout has to be called before any flag setting"); - } -#endif - SecLayout = SL; - SectionHdrLayout = ExtBinaryHdrLayoutTable[SL]; - } - + virtual std::error_code writeSample(const FunctionSamples &S) override; + + // Set to use MD5 to represent string in NameTable. + virtual void setUseMD5() override { + UseMD5 = true; + addSectionFlag(SecNameTable, SecNameTableFlags::SecFlagMD5Name); + // MD5 will be stored as plain uint64_t instead of variable-length + // quantity format in NameTable section. + addSectionFlag(SecNameTable, SecNameTableFlags::SecFlagFixedLengthMD5); + } + + // Set the profile to be partial. It means the profile is for + // common/shared code. The common profile is usually merged from + // profiles collected from running other targets. + virtual void setPartialProfile() override { + addSectionFlag(SecProfSummary, SecProfSummaryFlags::SecFlagPartial); + } + + virtual void setProfileSymbolList(ProfileSymbolList *PSL) override { + ProfSymList = PSL; + }; + + virtual void resetSecLayout(SectionLayout SL) override { + verifySecLayout(SL); +#ifndef NDEBUG + // Make sure resetSecLayout is called before any flag setting. + for (auto &Entry : SectionHdrLayout) { + assert(Entry.Flags == 0 && + "resetSecLayout has to be called before any flag setting"); + } +#endif + SecLayout = SL; + SectionHdrLayout = ExtBinaryHdrLayoutTable[SL]; + } + protected: - uint64_t markSectionStart(SecType Type, uint32_t LayoutIdx); - std::error_code addNewSection(SecType Sec, uint32_t LayoutIdx, - uint64_t SectionStart); + uint64_t markSectionStart(SecType Type, uint32_t LayoutIdx); + std::error_code addNewSection(SecType Sec, uint32_t LayoutIdx, + uint64_t SectionStart); template <class SecFlagType> void addSectionFlag(SecType Type, SecFlagType Flag) { for (auto &Entry : SectionHdrLayout) { @@ -246,50 +246,50 @@ protected: addSecFlag(Entry, Flag); } } - template <class SecFlagType> - void addSectionFlag(uint32_t SectionIdx, SecFlagType Flag) { - addSecFlag(SectionHdrLayout[SectionIdx], Flag); - } - - // placeholder for subclasses to dispatch their own section writers. - virtual std::error_code writeCustomSection(SecType Type) = 0; - // Verify the SecLayout is supported by the format. - virtual void verifySecLayout(SectionLayout SL) = 0; - - // specify the order to write sections. + template <class SecFlagType> + void addSectionFlag(uint32_t SectionIdx, SecFlagType Flag) { + addSecFlag(SectionHdrLayout[SectionIdx], Flag); + } + + // placeholder for subclasses to dispatch their own section writers. + virtual std::error_code writeCustomSection(SecType Type) = 0; + // Verify the SecLayout is supported by the format. + virtual void verifySecLayout(SectionLayout SL) = 0; + + // specify the order to write sections. virtual std::error_code writeSections(const StringMap<FunctionSamples> &ProfileMap) = 0; - // Dispatch section writer for each section. \p LayoutIdx is the sequence - // number indicating where the section is located in SectionHdrLayout. - virtual std::error_code - writeOneSection(SecType Type, uint32_t LayoutIdx, - const StringMap<FunctionSamples> &ProfileMap); - - // Helper function to write name table. - virtual std::error_code writeNameTable() override; - - std::error_code writeFuncMetadata(const StringMap<FunctionSamples> &Profiles); - - // Functions to write various kinds of sections. - std::error_code - writeNameTableSection(const StringMap<FunctionSamples> &ProfileMap); - std::error_code writeFuncOffsetTable(); - std::error_code writeProfileSymbolListSection(); - - SectionLayout SecLayout = DefaultLayout; + // Dispatch section writer for each section. \p LayoutIdx is the sequence + // number indicating where the section is located in SectionHdrLayout. + virtual std::error_code + writeOneSection(SecType Type, uint32_t LayoutIdx, + const StringMap<FunctionSamples> &ProfileMap); + + // Helper function to write name table. + virtual std::error_code writeNameTable() override; + + std::error_code writeFuncMetadata(const StringMap<FunctionSamples> &Profiles); + + // Functions to write various kinds of sections. + std::error_code + writeNameTableSection(const StringMap<FunctionSamples> &ProfileMap); + std::error_code writeFuncOffsetTable(); + std::error_code writeProfileSymbolListSection(); + + SectionLayout SecLayout = DefaultLayout; // Specifiy the order of sections in section header table. Note - // the order of sections in SecHdrTable may be different that the + // the order of sections in SecHdrTable may be different that the // order in SectionHdrLayout. sample Reader will follow the order // in SectionHdrLayout to read each section. - SmallVector<SecHdrTableEntry, 8> SectionHdrLayout = - ExtBinaryHdrLayoutTable[DefaultLayout]; - - // Save the start of SecLBRProfile so we can compute the offset to the - // start of SecLBRProfile for each Function's Profile and will keep it - // in FuncOffsetTable. - uint64_t SecLBRProfileStart = 0; - + SmallVector<SecHdrTableEntry, 8> SectionHdrLayout = + ExtBinaryHdrLayoutTable[DefaultLayout]; + + // Save the start of SecLBRProfile so we can compute the offset to the + // start of SecLBRProfile for each Function's Profile and will keep it + // in FuncOffsetTable. + uint64_t SecLBRProfileStart = 0; + private: void allocSecHdrTable(); std::error_code writeSecHdrTable(); @@ -311,43 +311,43 @@ private: // The location in the output stream where the SecHdrTable should be // written to. uint64_t SecHdrTableOffset; - // The table contains SecHdrTableEntry entries in order of how they are - // populated in the writer. It may be different from the order in - // SectionHdrLayout which specifies the sequence in which sections will - // be read. + // The table contains SecHdrTableEntry entries in order of how they are + // populated in the writer. It may be different from the order in + // SectionHdrLayout which specifies the sequence in which sections will + // be read. std::vector<SecHdrTableEntry> SecHdrTable; - - // FuncOffsetTable maps function name to its profile offset in SecLBRProfile - // section. It is used to load function profile on demand. - MapVector<StringRef, uint64_t> FuncOffsetTable; - // Whether to use MD5 to represent string. - bool UseMD5 = false; - - ProfileSymbolList *ProfSymList = nullptr; + + // FuncOffsetTable maps function name to its profile offset in SecLBRProfile + // section. It is used to load function profile on demand. + MapVector<StringRef, uint64_t> FuncOffsetTable; + // Whether to use MD5 to represent string. + bool UseMD5 = false; + + ProfileSymbolList *ProfSymList = nullptr; }; class SampleProfileWriterExtBinary : public SampleProfileWriterExtBinaryBase { public: SampleProfileWriterExtBinary(std::unique_ptr<raw_ostream> &OS) - : SampleProfileWriterExtBinaryBase(OS) {} + : SampleProfileWriterExtBinaryBase(OS) {} -private: - std::error_code - writeDefaultLayout(const StringMap<FunctionSamples> &ProfileMap); - std::error_code - writeCtxSplitLayout(const StringMap<FunctionSamples> &ProfileMap); +private: + std::error_code + writeDefaultLayout(const StringMap<FunctionSamples> &ProfileMap); + std::error_code + writeCtxSplitLayout(const StringMap<FunctionSamples> &ProfileMap); virtual std::error_code writeSections(const StringMap<FunctionSamples> &ProfileMap) override; - virtual std::error_code writeCustomSection(SecType Type) override { - return sampleprof_error::success; - }; + virtual std::error_code writeCustomSection(SecType Type) override { + return sampleprof_error::success; + }; - virtual void verifySecLayout(SectionLayout SL) override { - assert((SL == DefaultLayout || SL == CtxSplitLayout) && - "Unsupported layout"); - } + virtual void verifySecLayout(SectionLayout SL) override { + assert((SL == DefaultLayout || SL == CtxSplitLayout) && + "Unsupported layout"); + } }; // CompactBinary is a compact format of binary profile which both reduces |