diff options
author | monster <monster@ydb.tech> | 2022-07-07 14:41:37 +0300 |
---|---|---|
committer | monster <monster@ydb.tech> | 2022-07-07 14:41:37 +0300 |
commit | 06e5c21a835c0e923506c4ff27929f34e00761c2 (patch) | |
tree | 75efcbc6854ef9bd476eb8bf00cc5c900da436a2 /contrib/libs/llvm12/lib/ExecutionEngine | |
parent | 03f024c4412e3aa613bb543cf1660176320ba8f4 (diff) | |
download | ydb-06e5c21a835c0e923506c4ff27929f34e00761c2.tar.gz |
fix ya.make
Diffstat (limited to 'contrib/libs/llvm12/lib/ExecutionEngine')
53 files changed, 19581 insertions, 22 deletions
diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/Interpreter/Execution.cpp b/contrib/libs/llvm12/lib/ExecutionEngine/Interpreter/Execution.cpp new file mode 100644 index 0000000000..62e1ea6e0f --- /dev/null +++ b/contrib/libs/llvm12/lib/ExecutionEngine/Interpreter/Execution.cpp @@ -0,0 +1,2168 @@ +//===-- Execution.cpp - Implement code to simulate the program ------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains the actual instruction interpreter. +// +//===----------------------------------------------------------------------===// + +#include "Interpreter.h" +#include "llvm/ADT/APInt.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/CodeGen/IntrinsicLowering.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/GetElementPtrTypeIterator.h" +#include "llvm/IR/Instructions.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/MathExtras.h" +#include "llvm/Support/raw_ostream.h" +#include <algorithm> +#include <cmath> +using namespace llvm; + +#define DEBUG_TYPE "interpreter" + +STATISTIC(NumDynamicInsts, "Number of dynamic instructions executed"); + +static cl::opt<bool> PrintVolatile("interpreter-print-volatile", cl::Hidden, + cl::desc("make the interpreter print every volatile load and store")); + +//===----------------------------------------------------------------------===// +// Various Helper Functions +//===----------------------------------------------------------------------===// + +static void SetValue(Value *V, GenericValue Val, ExecutionContext &SF) { + SF.Values[V] = Val; +} + +//===----------------------------------------------------------------------===// +// Unary Instruction Implementations +//===----------------------------------------------------------------------===// + +static void executeFNegInst(GenericValue &Dest, GenericValue Src, Type *Ty) { + switch (Ty->getTypeID()) { + case Type::FloatTyID: + Dest.FloatVal = -Src.FloatVal; + break; + case Type::DoubleTyID: + Dest.DoubleVal = -Src.DoubleVal; + break; + default: + llvm_unreachable("Unhandled type for FNeg instruction"); + } +} + +void Interpreter::visitUnaryOperator(UnaryOperator &I) { + ExecutionContext &SF = ECStack.back(); + Type *Ty = I.getOperand(0)->getType(); + GenericValue Src = getOperandValue(I.getOperand(0), SF); + GenericValue R; // Result + + // First process vector operation + if (Ty->isVectorTy()) { + R.AggregateVal.resize(Src.AggregateVal.size()); + + switch(I.getOpcode()) { + default: + llvm_unreachable("Don't know how to handle this unary operator"); + break; + case Instruction::FNeg: + if (cast<VectorType>(Ty)->getElementType()->isFloatTy()) { + for (unsigned i = 0; i < R.AggregateVal.size(); ++i) + R.AggregateVal[i].FloatVal = -Src.AggregateVal[i].FloatVal; + } else if (cast<VectorType>(Ty)->getElementType()->isDoubleTy()) { + for (unsigned i = 0; i < R.AggregateVal.size(); ++i) + R.AggregateVal[i].DoubleVal = -Src.AggregateVal[i].DoubleVal; + } else { + llvm_unreachable("Unhandled type for FNeg instruction"); + } + break; + } + } else { + switch (I.getOpcode()) { + default: + llvm_unreachable("Don't know how to handle this unary operator"); + break; + case Instruction::FNeg: executeFNegInst(R, Src, Ty); break; + } + } + SetValue(&I, R, SF); +} + +//===----------------------------------------------------------------------===// +// Binary Instruction Implementations +//===----------------------------------------------------------------------===// + +#define IMPLEMENT_BINARY_OPERATOR(OP, TY) \ + case Type::TY##TyID: \ + Dest.TY##Val = Src1.TY##Val OP Src2.TY##Val; \ + break + +static void executeFAddInst(GenericValue &Dest, GenericValue Src1, + GenericValue Src2, Type *Ty) { + switch (Ty->getTypeID()) { + IMPLEMENT_BINARY_OPERATOR(+, Float); + IMPLEMENT_BINARY_OPERATOR(+, Double); + default: + dbgs() << "Unhandled type for FAdd instruction: " << *Ty << "\n"; + llvm_unreachable(nullptr); + } +} + +static void executeFSubInst(GenericValue &Dest, GenericValue Src1, + GenericValue Src2, Type *Ty) { + switch (Ty->getTypeID()) { + IMPLEMENT_BINARY_OPERATOR(-, Float); + IMPLEMENT_BINARY_OPERATOR(-, Double); + default: + dbgs() << "Unhandled type for FSub instruction: " << *Ty << "\n"; + llvm_unreachable(nullptr); + } +} + +static void executeFMulInst(GenericValue &Dest, GenericValue Src1, + GenericValue Src2, Type *Ty) { + switch (Ty->getTypeID()) { + IMPLEMENT_BINARY_OPERATOR(*, Float); + IMPLEMENT_BINARY_OPERATOR(*, Double); + default: + dbgs() << "Unhandled type for FMul instruction: " << *Ty << "\n"; + llvm_unreachable(nullptr); + } +} + +static void executeFDivInst(GenericValue &Dest, GenericValue Src1, + GenericValue Src2, Type *Ty) { + switch (Ty->getTypeID()) { + IMPLEMENT_BINARY_OPERATOR(/, Float); + IMPLEMENT_BINARY_OPERATOR(/, Double); + default: + dbgs() << "Unhandled type for FDiv instruction: " << *Ty << "\n"; + llvm_unreachable(nullptr); + } +} + +static void executeFRemInst(GenericValue &Dest, GenericValue Src1, + GenericValue Src2, Type *Ty) { + switch (Ty->getTypeID()) { + case Type::FloatTyID: + Dest.FloatVal = fmod(Src1.FloatVal, Src2.FloatVal); + break; + case Type::DoubleTyID: + Dest.DoubleVal = fmod(Src1.DoubleVal, Src2.DoubleVal); + break; + default: + dbgs() << "Unhandled type for Rem instruction: " << *Ty << "\n"; + llvm_unreachable(nullptr); + } +} + +#define IMPLEMENT_INTEGER_ICMP(OP, TY) \ + case Type::IntegerTyID: \ + Dest.IntVal = APInt(1,Src1.IntVal.OP(Src2.IntVal)); \ + break; + +#define IMPLEMENT_VECTOR_INTEGER_ICMP(OP, TY) \ + case Type::FixedVectorTyID: \ + case Type::ScalableVectorTyID: { \ + assert(Src1.AggregateVal.size() == Src2.AggregateVal.size()); \ + Dest.AggregateVal.resize(Src1.AggregateVal.size()); \ + for (uint32_t _i = 0; _i < Src1.AggregateVal.size(); _i++) \ + Dest.AggregateVal[_i].IntVal = APInt( \ + 1, Src1.AggregateVal[_i].IntVal.OP(Src2.AggregateVal[_i].IntVal)); \ + } break; + +// Handle pointers specially because they must be compared with only as much +// width as the host has. We _do not_ want to be comparing 64 bit values when +// running on a 32-bit target, otherwise the upper 32 bits might mess up +// comparisons if they contain garbage. +#define IMPLEMENT_POINTER_ICMP(OP) \ + case Type::PointerTyID: \ + Dest.IntVal = APInt(1,(void*)(intptr_t)Src1.PointerVal OP \ + (void*)(intptr_t)Src2.PointerVal); \ + break; + +static GenericValue executeICMP_EQ(GenericValue Src1, GenericValue Src2, + Type *Ty) { + GenericValue Dest; + switch (Ty->getTypeID()) { + IMPLEMENT_INTEGER_ICMP(eq,Ty); + IMPLEMENT_VECTOR_INTEGER_ICMP(eq,Ty); + IMPLEMENT_POINTER_ICMP(==); + default: + dbgs() << "Unhandled type for ICMP_EQ predicate: " << *Ty << "\n"; + llvm_unreachable(nullptr); + } + return Dest; +} + +static GenericValue executeICMP_NE(GenericValue Src1, GenericValue Src2, + Type *Ty) { + GenericValue Dest; + switch (Ty->getTypeID()) { + IMPLEMENT_INTEGER_ICMP(ne,Ty); + IMPLEMENT_VECTOR_INTEGER_ICMP(ne,Ty); + IMPLEMENT_POINTER_ICMP(!=); + default: + dbgs() << "Unhandled type for ICMP_NE predicate: " << *Ty << "\n"; + llvm_unreachable(nullptr); + } + return Dest; +} + +static GenericValue executeICMP_ULT(GenericValue Src1, GenericValue Src2, + Type *Ty) { + GenericValue Dest; + switch (Ty->getTypeID()) { + IMPLEMENT_INTEGER_ICMP(ult,Ty); + IMPLEMENT_VECTOR_INTEGER_ICMP(ult,Ty); + IMPLEMENT_POINTER_ICMP(<); + default: + dbgs() << "Unhandled type for ICMP_ULT predicate: " << *Ty << "\n"; + llvm_unreachable(nullptr); + } + return Dest; +} + +static GenericValue executeICMP_SLT(GenericValue Src1, GenericValue Src2, + Type *Ty) { + GenericValue Dest; + switch (Ty->getTypeID()) { + IMPLEMENT_INTEGER_ICMP(slt,Ty); + IMPLEMENT_VECTOR_INTEGER_ICMP(slt,Ty); + IMPLEMENT_POINTER_ICMP(<); + default: + dbgs() << "Unhandled type for ICMP_SLT predicate: " << *Ty << "\n"; + llvm_unreachable(nullptr); + } + return Dest; +} + +static GenericValue executeICMP_UGT(GenericValue Src1, GenericValue Src2, + Type *Ty) { + GenericValue Dest; + switch (Ty->getTypeID()) { + IMPLEMENT_INTEGER_ICMP(ugt,Ty); + IMPLEMENT_VECTOR_INTEGER_ICMP(ugt,Ty); + IMPLEMENT_POINTER_ICMP(>); + default: + dbgs() << "Unhandled type for ICMP_UGT predicate: " << *Ty << "\n"; + llvm_unreachable(nullptr); + } + return Dest; +} + +static GenericValue executeICMP_SGT(GenericValue Src1, GenericValue Src2, + Type *Ty) { + GenericValue Dest; + switch (Ty->getTypeID()) { + IMPLEMENT_INTEGER_ICMP(sgt,Ty); + IMPLEMENT_VECTOR_INTEGER_ICMP(sgt,Ty); + IMPLEMENT_POINTER_ICMP(>); + default: + dbgs() << "Unhandled type for ICMP_SGT predicate: " << *Ty << "\n"; + llvm_unreachable(nullptr); + } + return Dest; +} + +static GenericValue executeICMP_ULE(GenericValue Src1, GenericValue Src2, + Type *Ty) { + GenericValue Dest; + switch (Ty->getTypeID()) { + IMPLEMENT_INTEGER_ICMP(ule,Ty); + IMPLEMENT_VECTOR_INTEGER_ICMP(ule,Ty); + IMPLEMENT_POINTER_ICMP(<=); + default: + dbgs() << "Unhandled type for ICMP_ULE predicate: " << *Ty << "\n"; + llvm_unreachable(nullptr); + } + return Dest; +} + +static GenericValue executeICMP_SLE(GenericValue Src1, GenericValue Src2, + Type *Ty) { + GenericValue Dest; + switch (Ty->getTypeID()) { + IMPLEMENT_INTEGER_ICMP(sle,Ty); + IMPLEMENT_VECTOR_INTEGER_ICMP(sle,Ty); + IMPLEMENT_POINTER_ICMP(<=); + default: + dbgs() << "Unhandled type for ICMP_SLE predicate: " << *Ty << "\n"; + llvm_unreachable(nullptr); + } + return Dest; +} + +static GenericValue executeICMP_UGE(GenericValue Src1, GenericValue Src2, + Type *Ty) { + GenericValue Dest; + switch (Ty->getTypeID()) { + IMPLEMENT_INTEGER_ICMP(uge,Ty); + IMPLEMENT_VECTOR_INTEGER_ICMP(uge,Ty); + IMPLEMENT_POINTER_ICMP(>=); + default: + dbgs() << "Unhandled type for ICMP_UGE predicate: " << *Ty << "\n"; + llvm_unreachable(nullptr); + } + return Dest; +} + +static GenericValue executeICMP_SGE(GenericValue Src1, GenericValue Src2, + Type *Ty) { + GenericValue Dest; + switch (Ty->getTypeID()) { + IMPLEMENT_INTEGER_ICMP(sge,Ty); + IMPLEMENT_VECTOR_INTEGER_ICMP(sge,Ty); + IMPLEMENT_POINTER_ICMP(>=); + default: + dbgs() << "Unhandled type for ICMP_SGE predicate: " << *Ty << "\n"; + llvm_unreachable(nullptr); + } + return Dest; +} + +void Interpreter::visitICmpInst(ICmpInst &I) { + ExecutionContext &SF = ECStack.back(); + Type *Ty = I.getOperand(0)->getType(); + GenericValue Src1 = getOperandValue(I.getOperand(0), SF); + GenericValue Src2 = getOperandValue(I.getOperand(1), SF); + GenericValue R; // Result + + switch (I.getPredicate()) { + case ICmpInst::ICMP_EQ: R = executeICMP_EQ(Src1, Src2, Ty); break; + case ICmpInst::ICMP_NE: R = executeICMP_NE(Src1, Src2, Ty); break; + case ICmpInst::ICMP_ULT: R = executeICMP_ULT(Src1, Src2, Ty); break; + case ICmpInst::ICMP_SLT: R = executeICMP_SLT(Src1, Src2, Ty); break; + case ICmpInst::ICMP_UGT: R = executeICMP_UGT(Src1, Src2, Ty); break; + case ICmpInst::ICMP_SGT: R = executeICMP_SGT(Src1, Src2, Ty); break; + case ICmpInst::ICMP_ULE: R = executeICMP_ULE(Src1, Src2, Ty); break; + case ICmpInst::ICMP_SLE: R = executeICMP_SLE(Src1, Src2, Ty); break; + case ICmpInst::ICMP_UGE: R = executeICMP_UGE(Src1, Src2, Ty); break; + case ICmpInst::ICMP_SGE: R = executeICMP_SGE(Src1, Src2, Ty); break; + default: + dbgs() << "Don't know how to handle this ICmp predicate!\n-->" << I; + llvm_unreachable(nullptr); + } + + SetValue(&I, R, SF); +} + +#define IMPLEMENT_FCMP(OP, TY) \ + case Type::TY##TyID: \ + Dest.IntVal = APInt(1,Src1.TY##Val OP Src2.TY##Val); \ + break + +#define IMPLEMENT_VECTOR_FCMP_T(OP, TY) \ + assert(Src1.AggregateVal.size() == Src2.AggregateVal.size()); \ + Dest.AggregateVal.resize( Src1.AggregateVal.size() ); \ + for( uint32_t _i=0;_i<Src1.AggregateVal.size();_i++) \ + Dest.AggregateVal[_i].IntVal = APInt(1, \ + Src1.AggregateVal[_i].TY##Val OP Src2.AggregateVal[_i].TY##Val);\ + break; + +#define IMPLEMENT_VECTOR_FCMP(OP) \ + case Type::FixedVectorTyID: \ + case Type::ScalableVectorTyID: \ + if (cast<VectorType>(Ty)->getElementType()->isFloatTy()) { \ + IMPLEMENT_VECTOR_FCMP_T(OP, Float); \ + } else { \ + IMPLEMENT_VECTOR_FCMP_T(OP, Double); \ + } + +static GenericValue executeFCMP_OEQ(GenericValue Src1, GenericValue Src2, + Type *Ty) { + GenericValue Dest; + switch (Ty->getTypeID()) { + IMPLEMENT_FCMP(==, Float); + IMPLEMENT_FCMP(==, Double); + IMPLEMENT_VECTOR_FCMP(==); + default: + dbgs() << "Unhandled type for FCmp EQ instruction: " << *Ty << "\n"; + llvm_unreachable(nullptr); + } + return Dest; +} + +#define IMPLEMENT_SCALAR_NANS(TY, X,Y) \ + if (TY->isFloatTy()) { \ + if (X.FloatVal != X.FloatVal || Y.FloatVal != Y.FloatVal) { \ + Dest.IntVal = APInt(1,false); \ + return Dest; \ + } \ + } else { \ + if (X.DoubleVal != X.DoubleVal || Y.DoubleVal != Y.DoubleVal) { \ + Dest.IntVal = APInt(1,false); \ + return Dest; \ + } \ + } + +#define MASK_VECTOR_NANS_T(X,Y, TZ, FLAG) \ + assert(X.AggregateVal.size() == Y.AggregateVal.size()); \ + Dest.AggregateVal.resize( X.AggregateVal.size() ); \ + for( uint32_t _i=0;_i<X.AggregateVal.size();_i++) { \ + if (X.AggregateVal[_i].TZ##Val != X.AggregateVal[_i].TZ##Val || \ + Y.AggregateVal[_i].TZ##Val != Y.AggregateVal[_i].TZ##Val) \ + Dest.AggregateVal[_i].IntVal = APInt(1,FLAG); \ + else { \ + Dest.AggregateVal[_i].IntVal = APInt(1,!FLAG); \ + } \ + } + +#define MASK_VECTOR_NANS(TY, X,Y, FLAG) \ + if (TY->isVectorTy()) { \ + if (cast<VectorType>(TY)->getElementType()->isFloatTy()) { \ + MASK_VECTOR_NANS_T(X, Y, Float, FLAG) \ + } else { \ + MASK_VECTOR_NANS_T(X, Y, Double, FLAG) \ + } \ + } \ + + + +static GenericValue executeFCMP_ONE(GenericValue Src1, GenericValue Src2, + Type *Ty) +{ + GenericValue Dest; + // if input is scalar value and Src1 or Src2 is NaN return false + IMPLEMENT_SCALAR_NANS(Ty, Src1, Src2) + // if vector input detect NaNs and fill mask + MASK_VECTOR_NANS(Ty, Src1, Src2, false) + GenericValue DestMask = Dest; + switch (Ty->getTypeID()) { + IMPLEMENT_FCMP(!=, Float); + IMPLEMENT_FCMP(!=, Double); + IMPLEMENT_VECTOR_FCMP(!=); + default: + dbgs() << "Unhandled type for FCmp NE instruction: " << *Ty << "\n"; + llvm_unreachable(nullptr); + } + // in vector case mask out NaN elements + if (Ty->isVectorTy()) + for( size_t _i=0; _i<Src1.AggregateVal.size(); _i++) + if (DestMask.AggregateVal[_i].IntVal == false) + Dest.AggregateVal[_i].IntVal = APInt(1,false); + + return Dest; +} + +static GenericValue executeFCMP_OLE(GenericValue Src1, GenericValue Src2, + Type *Ty) { + GenericValue Dest; + switch (Ty->getTypeID()) { + IMPLEMENT_FCMP(<=, Float); + IMPLEMENT_FCMP(<=, Double); + IMPLEMENT_VECTOR_FCMP(<=); + default: + dbgs() << "Unhandled type for FCmp LE instruction: " << *Ty << "\n"; + llvm_unreachable(nullptr); + } + return Dest; +} + +static GenericValue executeFCMP_OGE(GenericValue Src1, GenericValue Src2, + Type *Ty) { + GenericValue Dest; + switch (Ty->getTypeID()) { + IMPLEMENT_FCMP(>=, Float); + IMPLEMENT_FCMP(>=, Double); + IMPLEMENT_VECTOR_FCMP(>=); + default: + dbgs() << "Unhandled type for FCmp GE instruction: " << *Ty << "\n"; + llvm_unreachable(nullptr); + } + return Dest; +} + +static GenericValue executeFCMP_OLT(GenericValue Src1, GenericValue Src2, + Type *Ty) { + GenericValue Dest; + switch (Ty->getTypeID()) { + IMPLEMENT_FCMP(<, Float); + IMPLEMENT_FCMP(<, Double); + IMPLEMENT_VECTOR_FCMP(<); + default: + dbgs() << "Unhandled type for FCmp LT instruction: " << *Ty << "\n"; + llvm_unreachable(nullptr); + } + return Dest; +} + +static GenericValue executeFCMP_OGT(GenericValue Src1, GenericValue Src2, + Type *Ty) { + GenericValue Dest; + switch (Ty->getTypeID()) { + IMPLEMENT_FCMP(>, Float); + IMPLEMENT_FCMP(>, Double); + IMPLEMENT_VECTOR_FCMP(>); + default: + dbgs() << "Unhandled type for FCmp GT instruction: " << *Ty << "\n"; + llvm_unreachable(nullptr); + } + return Dest; +} + +#define IMPLEMENT_UNORDERED(TY, X,Y) \ + if (TY->isFloatTy()) { \ + if (X.FloatVal != X.FloatVal || Y.FloatVal != Y.FloatVal) { \ + Dest.IntVal = APInt(1,true); \ + return Dest; \ + } \ + } else if (X.DoubleVal != X.DoubleVal || Y.DoubleVal != Y.DoubleVal) { \ + Dest.IntVal = APInt(1,true); \ + return Dest; \ + } + +#define IMPLEMENT_VECTOR_UNORDERED(TY, X, Y, FUNC) \ + if (TY->isVectorTy()) { \ + GenericValue DestMask = Dest; \ + Dest = FUNC(Src1, Src2, Ty); \ + for (size_t _i = 0; _i < Src1.AggregateVal.size(); _i++) \ + if (DestMask.AggregateVal[_i].IntVal == true) \ + Dest.AggregateVal[_i].IntVal = APInt(1, true); \ + return Dest; \ + } + +static GenericValue executeFCMP_UEQ(GenericValue Src1, GenericValue Src2, + Type *Ty) { + GenericValue Dest; + IMPLEMENT_UNORDERED(Ty, Src1, Src2) + MASK_VECTOR_NANS(Ty, Src1, Src2, true) + IMPLEMENT_VECTOR_UNORDERED(Ty, Src1, Src2, executeFCMP_OEQ) + return executeFCMP_OEQ(Src1, Src2, Ty); + +} + +static GenericValue executeFCMP_UNE(GenericValue Src1, GenericValue Src2, + Type *Ty) { + GenericValue Dest; + IMPLEMENT_UNORDERED(Ty, Src1, Src2) + MASK_VECTOR_NANS(Ty, Src1, Src2, true) + IMPLEMENT_VECTOR_UNORDERED(Ty, Src1, Src2, executeFCMP_ONE) + return executeFCMP_ONE(Src1, Src2, Ty); +} + +static GenericValue executeFCMP_ULE(GenericValue Src1, GenericValue Src2, + Type *Ty) { + GenericValue Dest; + IMPLEMENT_UNORDERED(Ty, Src1, Src2) + MASK_VECTOR_NANS(Ty, Src1, Src2, true) + IMPLEMENT_VECTOR_UNORDERED(Ty, Src1, Src2, executeFCMP_OLE) + return executeFCMP_OLE(Src1, Src2, Ty); +} + +static GenericValue executeFCMP_UGE(GenericValue Src1, GenericValue Src2, + Type *Ty) { + GenericValue Dest; + IMPLEMENT_UNORDERED(Ty, Src1, Src2) + MASK_VECTOR_NANS(Ty, Src1, Src2, true) + IMPLEMENT_VECTOR_UNORDERED(Ty, Src1, Src2, executeFCMP_OGE) + return executeFCMP_OGE(Src1, Src2, Ty); +} + +static GenericValue executeFCMP_ULT(GenericValue Src1, GenericValue Src2, + Type *Ty) { + GenericValue Dest; + IMPLEMENT_UNORDERED(Ty, Src1, Src2) + MASK_VECTOR_NANS(Ty, Src1, Src2, true) + IMPLEMENT_VECTOR_UNORDERED(Ty, Src1, Src2, executeFCMP_OLT) + return executeFCMP_OLT(Src1, Src2, Ty); +} + +static GenericValue executeFCMP_UGT(GenericValue Src1, GenericValue Src2, + Type *Ty) { + GenericValue Dest; + IMPLEMENT_UNORDERED(Ty, Src1, Src2) + MASK_VECTOR_NANS(Ty, Src1, Src2, true) + IMPLEMENT_VECTOR_UNORDERED(Ty, Src1, Src2, executeFCMP_OGT) + return executeFCMP_OGT(Src1, Src2, Ty); +} + +static GenericValue executeFCMP_ORD(GenericValue Src1, GenericValue Src2, + Type *Ty) { + GenericValue Dest; + if(Ty->isVectorTy()) { + assert(Src1.AggregateVal.size() == Src2.AggregateVal.size()); + Dest.AggregateVal.resize( Src1.AggregateVal.size() ); + if (cast<VectorType>(Ty)->getElementType()->isFloatTy()) { + for( size_t _i=0;_i<Src1.AggregateVal.size();_i++) + Dest.AggregateVal[_i].IntVal = APInt(1, + ( (Src1.AggregateVal[_i].FloatVal == + Src1.AggregateVal[_i].FloatVal) && + (Src2.AggregateVal[_i].FloatVal == + Src2.AggregateVal[_i].FloatVal))); + } else { + for( size_t _i=0;_i<Src1.AggregateVal.size();_i++) + Dest.AggregateVal[_i].IntVal = APInt(1, + ( (Src1.AggregateVal[_i].DoubleVal == + Src1.AggregateVal[_i].DoubleVal) && + (Src2.AggregateVal[_i].DoubleVal == + Src2.AggregateVal[_i].DoubleVal))); + } + } else if (Ty->isFloatTy()) + Dest.IntVal = APInt(1,(Src1.FloatVal == Src1.FloatVal && + Src2.FloatVal == Src2.FloatVal)); + else { + Dest.IntVal = APInt(1,(Src1.DoubleVal == Src1.DoubleVal && + Src2.DoubleVal == Src2.DoubleVal)); + } + return Dest; +} + +static GenericValue executeFCMP_UNO(GenericValue Src1, GenericValue Src2, + Type *Ty) { + GenericValue Dest; + if(Ty->isVectorTy()) { + assert(Src1.AggregateVal.size() == Src2.AggregateVal.size()); + Dest.AggregateVal.resize( Src1.AggregateVal.size() ); + if (cast<VectorType>(Ty)->getElementType()->isFloatTy()) { + for( size_t _i=0;_i<Src1.AggregateVal.size();_i++) + Dest.AggregateVal[_i].IntVal = APInt(1, + ( (Src1.AggregateVal[_i].FloatVal != + Src1.AggregateVal[_i].FloatVal) || + (Src2.AggregateVal[_i].FloatVal != + Src2.AggregateVal[_i].FloatVal))); + } else { + for( size_t _i=0;_i<Src1.AggregateVal.size();_i++) + Dest.AggregateVal[_i].IntVal = APInt(1, + ( (Src1.AggregateVal[_i].DoubleVal != + Src1.AggregateVal[_i].DoubleVal) || + (Src2.AggregateVal[_i].DoubleVal != + Src2.AggregateVal[_i].DoubleVal))); + } + } else if (Ty->isFloatTy()) + Dest.IntVal = APInt(1,(Src1.FloatVal != Src1.FloatVal || + Src2.FloatVal != Src2.FloatVal)); + else { + Dest.IntVal = APInt(1,(Src1.DoubleVal != Src1.DoubleVal || + Src2.DoubleVal != Src2.DoubleVal)); + } + return Dest; +} + +static GenericValue executeFCMP_BOOL(GenericValue Src1, GenericValue Src2, + Type *Ty, const bool val) { + GenericValue Dest; + if(Ty->isVectorTy()) { + assert(Src1.AggregateVal.size() == Src2.AggregateVal.size()); + Dest.AggregateVal.resize( Src1.AggregateVal.size() ); + for( size_t _i=0; _i<Src1.AggregateVal.size(); _i++) + Dest.AggregateVal[_i].IntVal = APInt(1,val); + } else { + Dest.IntVal = APInt(1, val); + } + + return Dest; +} + +void Interpreter::visitFCmpInst(FCmpInst &I) { + ExecutionContext &SF = ECStack.back(); + Type *Ty = I.getOperand(0)->getType(); + GenericValue Src1 = getOperandValue(I.getOperand(0), SF); + GenericValue Src2 = getOperandValue(I.getOperand(1), SF); + GenericValue R; // Result + + switch (I.getPredicate()) { + default: + dbgs() << "Don't know how to handle this FCmp predicate!\n-->" << I; + llvm_unreachable(nullptr); + break; + case FCmpInst::FCMP_FALSE: R = executeFCMP_BOOL(Src1, Src2, Ty, false); + break; + case FCmpInst::FCMP_TRUE: R = executeFCMP_BOOL(Src1, Src2, Ty, true); + break; + case FCmpInst::FCMP_ORD: R = executeFCMP_ORD(Src1, Src2, Ty); break; + case FCmpInst::FCMP_UNO: R = executeFCMP_UNO(Src1, Src2, Ty); break; + case FCmpInst::FCMP_UEQ: R = executeFCMP_UEQ(Src1, Src2, Ty); break; + case FCmpInst::FCMP_OEQ: R = executeFCMP_OEQ(Src1, Src2, Ty); break; + case FCmpInst::FCMP_UNE: R = executeFCMP_UNE(Src1, Src2, Ty); break; + case FCmpInst::FCMP_ONE: R = executeFCMP_ONE(Src1, Src2, Ty); break; + case FCmpInst::FCMP_ULT: R = executeFCMP_ULT(Src1, Src2, Ty); break; + case FCmpInst::FCMP_OLT: R = executeFCMP_OLT(Src1, Src2, Ty); break; + case FCmpInst::FCMP_UGT: R = executeFCMP_UGT(Src1, Src2, Ty); break; + case FCmpInst::FCMP_OGT: R = executeFCMP_OGT(Src1, Src2, Ty); break; + case FCmpInst::FCMP_ULE: R = executeFCMP_ULE(Src1, Src2, Ty); break; + case FCmpInst::FCMP_OLE: R = executeFCMP_OLE(Src1, Src2, Ty); break; + case FCmpInst::FCMP_UGE: R = executeFCMP_UGE(Src1, Src2, Ty); break; + case FCmpInst::FCMP_OGE: R = executeFCMP_OGE(Src1, Src2, Ty); break; + } + + SetValue(&I, R, SF); +} + +static GenericValue executeCmpInst(unsigned predicate, GenericValue Src1, + GenericValue Src2, Type *Ty) { + GenericValue Result; + switch (predicate) { + case ICmpInst::ICMP_EQ: return executeICMP_EQ(Src1, Src2, Ty); + case ICmpInst::ICMP_NE: return executeICMP_NE(Src1, Src2, Ty); + case ICmpInst::ICMP_UGT: return executeICMP_UGT(Src1, Src2, Ty); + case ICmpInst::ICMP_SGT: return executeICMP_SGT(Src1, Src2, Ty); + case ICmpInst::ICMP_ULT: return executeICMP_ULT(Src1, Src2, Ty); + case ICmpInst::ICMP_SLT: return executeICMP_SLT(Src1, Src2, Ty); + case ICmpInst::ICMP_UGE: return executeICMP_UGE(Src1, Src2, Ty); + case ICmpInst::ICMP_SGE: return executeICMP_SGE(Src1, Src2, Ty); + case ICmpInst::ICMP_ULE: return executeICMP_ULE(Src1, Src2, Ty); + case ICmpInst::ICMP_SLE: return executeICMP_SLE(Src1, Src2, Ty); + case FCmpInst::FCMP_ORD: return executeFCMP_ORD(Src1, Src2, Ty); + case FCmpInst::FCMP_UNO: return executeFCMP_UNO(Src1, Src2, Ty); + case FCmpInst::FCMP_OEQ: return executeFCMP_OEQ(Src1, Src2, Ty); + case FCmpInst::FCMP_UEQ: return executeFCMP_UEQ(Src1, Src2, Ty); + case FCmpInst::FCMP_ONE: return executeFCMP_ONE(Src1, Src2, Ty); + case FCmpInst::FCMP_UNE: return executeFCMP_UNE(Src1, Src2, Ty); + case FCmpInst::FCMP_OLT: return executeFCMP_OLT(Src1, Src2, Ty); + case FCmpInst::FCMP_ULT: return executeFCMP_ULT(Src1, Src2, Ty); + case FCmpInst::FCMP_OGT: return executeFCMP_OGT(Src1, Src2, Ty); + case FCmpInst::FCMP_UGT: return executeFCMP_UGT(Src1, Src2, Ty); + case FCmpInst::FCMP_OLE: return executeFCMP_OLE(Src1, Src2, Ty); + case FCmpInst::FCMP_ULE: return executeFCMP_ULE(Src1, Src2, Ty); + case FCmpInst::FCMP_OGE: return executeFCMP_OGE(Src1, Src2, Ty); + case FCmpInst::FCMP_UGE: return executeFCMP_UGE(Src1, Src2, Ty); + case FCmpInst::FCMP_FALSE: return executeFCMP_BOOL(Src1, Src2, Ty, false); + case FCmpInst::FCMP_TRUE: return executeFCMP_BOOL(Src1, Src2, Ty, true); + default: + dbgs() << "Unhandled Cmp predicate\n"; + llvm_unreachable(nullptr); + } +} + +void Interpreter::visitBinaryOperator(BinaryOperator &I) { + ExecutionContext &SF = ECStack.back(); + Type *Ty = I.getOperand(0)->getType(); + GenericValue Src1 = getOperandValue(I.getOperand(0), SF); + GenericValue Src2 = getOperandValue(I.getOperand(1), SF); + GenericValue R; // Result + + // First process vector operation + if (Ty->isVectorTy()) { + assert(Src1.AggregateVal.size() == Src2.AggregateVal.size()); + R.AggregateVal.resize(Src1.AggregateVal.size()); + + // Macros to execute binary operation 'OP' over integer vectors +#define INTEGER_VECTOR_OPERATION(OP) \ + for (unsigned i = 0; i < R.AggregateVal.size(); ++i) \ + R.AggregateVal[i].IntVal = \ + Src1.AggregateVal[i].IntVal OP Src2.AggregateVal[i].IntVal; + + // Additional macros to execute binary operations udiv/sdiv/urem/srem since + // they have different notation. +#define INTEGER_VECTOR_FUNCTION(OP) \ + for (unsigned i = 0; i < R.AggregateVal.size(); ++i) \ + R.AggregateVal[i].IntVal = \ + Src1.AggregateVal[i].IntVal.OP(Src2.AggregateVal[i].IntVal); + + // Macros to execute binary operation 'OP' over floating point type TY + // (float or double) vectors +#define FLOAT_VECTOR_FUNCTION(OP, TY) \ + for (unsigned i = 0; i < R.AggregateVal.size(); ++i) \ + R.AggregateVal[i].TY = \ + Src1.AggregateVal[i].TY OP Src2.AggregateVal[i].TY; + + // Macros to choose appropriate TY: float or double and run operation + // execution +#define FLOAT_VECTOR_OP(OP) { \ + if (cast<VectorType>(Ty)->getElementType()->isFloatTy()) \ + FLOAT_VECTOR_FUNCTION(OP, FloatVal) \ + else { \ + if (cast<VectorType>(Ty)->getElementType()->isDoubleTy()) \ + FLOAT_VECTOR_FUNCTION(OP, DoubleVal) \ + else { \ + dbgs() << "Unhandled type for OP instruction: " << *Ty << "\n"; \ + llvm_unreachable(0); \ + } \ + } \ +} + + switch(I.getOpcode()){ + default: + dbgs() << "Don't know how to handle this binary operator!\n-->" << I; + llvm_unreachable(nullptr); + break; + case Instruction::Add: INTEGER_VECTOR_OPERATION(+) break; + case Instruction::Sub: INTEGER_VECTOR_OPERATION(-) break; + case Instruction::Mul: INTEGER_VECTOR_OPERATION(*) break; + case Instruction::UDiv: INTEGER_VECTOR_FUNCTION(udiv) break; + case Instruction::SDiv: INTEGER_VECTOR_FUNCTION(sdiv) break; + case Instruction::URem: INTEGER_VECTOR_FUNCTION(urem) break; + case Instruction::SRem: INTEGER_VECTOR_FUNCTION(srem) break; + case Instruction::And: INTEGER_VECTOR_OPERATION(&) break; + case Instruction::Or: INTEGER_VECTOR_OPERATION(|) break; + case Instruction::Xor: INTEGER_VECTOR_OPERATION(^) break; + case Instruction::FAdd: FLOAT_VECTOR_OP(+) break; + case Instruction::FSub: FLOAT_VECTOR_OP(-) break; + case Instruction::FMul: FLOAT_VECTOR_OP(*) break; + case Instruction::FDiv: FLOAT_VECTOR_OP(/) break; + case Instruction::FRem: + if (cast<VectorType>(Ty)->getElementType()->isFloatTy()) + for (unsigned i = 0; i < R.AggregateVal.size(); ++i) + R.AggregateVal[i].FloatVal = + fmod(Src1.AggregateVal[i].FloatVal, Src2.AggregateVal[i].FloatVal); + else { + if (cast<VectorType>(Ty)->getElementType()->isDoubleTy()) + for (unsigned i = 0; i < R.AggregateVal.size(); ++i) + R.AggregateVal[i].DoubleVal = + fmod(Src1.AggregateVal[i].DoubleVal, Src2.AggregateVal[i].DoubleVal); + else { + dbgs() << "Unhandled type for Rem instruction: " << *Ty << "\n"; + llvm_unreachable(nullptr); + } + } + break; + } + } else { + switch (I.getOpcode()) { + default: + dbgs() << "Don't know how to handle this binary operator!\n-->" << I; + llvm_unreachable(nullptr); + break; + case Instruction::Add: R.IntVal = Src1.IntVal + Src2.IntVal; break; + case Instruction::Sub: R.IntVal = Src1.IntVal - Src2.IntVal; break; + case Instruction::Mul: R.IntVal = Src1.IntVal * Src2.IntVal; break; + case Instruction::FAdd: executeFAddInst(R, Src1, Src2, Ty); break; + case Instruction::FSub: executeFSubInst(R, Src1, Src2, Ty); break; + case Instruction::FMul: executeFMulInst(R, Src1, Src2, Ty); break; + case Instruction::FDiv: executeFDivInst(R, Src1, Src2, Ty); break; + case Instruction::FRem: executeFRemInst(R, Src1, Src2, Ty); break; + case Instruction::UDiv: R.IntVal = Src1.IntVal.udiv(Src2.IntVal); break; + case Instruction::SDiv: R.IntVal = Src1.IntVal.sdiv(Src2.IntVal); break; + case Instruction::URem: R.IntVal = Src1.IntVal.urem(Src2.IntVal); break; + case Instruction::SRem: R.IntVal = Src1.IntVal.srem(Src2.IntVal); break; + case Instruction::And: R.IntVal = Src1.IntVal & Src2.IntVal; break; + case Instruction::Or: R.IntVal = Src1.IntVal | Src2.IntVal; break; + case Instruction::Xor: R.IntVal = Src1.IntVal ^ Src2.IntVal; break; + } + } + SetValue(&I, R, SF); +} + +static GenericValue executeSelectInst(GenericValue Src1, GenericValue Src2, + GenericValue Src3, Type *Ty) { + GenericValue Dest; + if(Ty->isVectorTy()) { + assert(Src1.AggregateVal.size() == Src2.AggregateVal.size()); + assert(Src2.AggregateVal.size() == Src3.AggregateVal.size()); + Dest.AggregateVal.resize( Src1.AggregateVal.size() ); + for (size_t i = 0; i < Src1.AggregateVal.size(); ++i) + Dest.AggregateVal[i] = (Src1.AggregateVal[i].IntVal == 0) ? + Src3.AggregateVal[i] : Src2.AggregateVal[i]; + } else { + Dest = (Src1.IntVal == 0) ? Src3 : Src2; + } + return Dest; +} + +void Interpreter::visitSelectInst(SelectInst &I) { + ExecutionContext &SF = ECStack.back(); + Type * Ty = I.getOperand(0)->getType(); + GenericValue Src1 = getOperandValue(I.getOperand(0), SF); + GenericValue Src2 = getOperandValue(I.getOperand(1), SF); + GenericValue Src3 = getOperandValue(I.getOperand(2), SF); + GenericValue R = executeSelectInst(Src1, Src2, Src3, Ty); + SetValue(&I, R, SF); +} + +//===----------------------------------------------------------------------===// +// Terminator Instruction Implementations +//===----------------------------------------------------------------------===// + +void Interpreter::exitCalled(GenericValue GV) { + // runAtExitHandlers() assumes there are no stack frames, but + // if exit() was called, then it had a stack frame. Blow away + // the stack before interpreting atexit handlers. + ECStack.clear(); + runAtExitHandlers(); + exit(GV.IntVal.zextOrTrunc(32).getZExtValue()); +} + +/// Pop the last stack frame off of ECStack and then copy the result +/// back into the result variable if we are not returning void. The +/// result variable may be the ExitValue, or the Value of the calling +/// CallInst if there was a previous stack frame. This method may +/// invalidate any ECStack iterators you have. This method also takes +/// care of switching to the normal destination BB, if we are returning +/// from an invoke. +/// +void Interpreter::popStackAndReturnValueToCaller(Type *RetTy, + GenericValue Result) { + // Pop the current stack frame. + ECStack.pop_back(); + + if (ECStack.empty()) { // Finished main. Put result into exit code... + if (RetTy && !RetTy->isVoidTy()) { // Nonvoid return type? + ExitValue = Result; // Capture the exit value of the program + } else { + memset(&ExitValue.Untyped, 0, sizeof(ExitValue.Untyped)); + } + } else { + // If we have a previous stack frame, and we have a previous call, + // fill in the return value... + ExecutionContext &CallingSF = ECStack.back(); + if (CallingSF.Caller) { + // Save result... + if (!CallingSF.Caller->getType()->isVoidTy()) + SetValue(CallingSF.Caller, Result, CallingSF); + if (InvokeInst *II = dyn_cast<InvokeInst>(CallingSF.Caller)) + SwitchToNewBasicBlock (II->getNormalDest (), CallingSF); + CallingSF.Caller = nullptr; // We returned from the call... + } + } +} + +void Interpreter::visitReturnInst(ReturnInst &I) { + ExecutionContext &SF = ECStack.back(); + Type *RetTy = Type::getVoidTy(I.getContext()); + GenericValue Result; + + // Save away the return value... (if we are not 'ret void') + if (I.getNumOperands()) { + RetTy = I.getReturnValue()->getType(); + Result = getOperandValue(I.getReturnValue(), SF); + } + + popStackAndReturnValueToCaller(RetTy, Result); +} + +void Interpreter::visitUnreachableInst(UnreachableInst &I) { + report_fatal_error("Program executed an 'unreachable' instruction!"); +} + +void Interpreter::visitBranchInst(BranchInst &I) { + ExecutionContext &SF = ECStack.back(); + BasicBlock *Dest; + + Dest = I.getSuccessor(0); // Uncond branches have a fixed dest... + if (!I.isUnconditional()) { + Value *Cond = I.getCondition(); + if (getOperandValue(Cond, SF).IntVal == 0) // If false cond... + Dest = I.getSuccessor(1); + } + SwitchToNewBasicBlock(Dest, SF); +} + +void Interpreter::visitSwitchInst(SwitchInst &I) { + ExecutionContext &SF = ECStack.back(); + Value* Cond = I.getCondition(); + Type *ElTy = Cond->getType(); + GenericValue CondVal = getOperandValue(Cond, SF); + + // Check to see if any of the cases match... + BasicBlock *Dest = nullptr; + for (auto Case : I.cases()) { + GenericValue CaseVal = getOperandValue(Case.getCaseValue(), SF); + if (executeICMP_EQ(CondVal, CaseVal, ElTy).IntVal != 0) { + Dest = cast<BasicBlock>(Case.getCaseSuccessor()); + break; + } + } + if (!Dest) Dest = I.getDefaultDest(); // No cases matched: use default + SwitchToNewBasicBlock(Dest, SF); +} + +void Interpreter::visitIndirectBrInst(IndirectBrInst &I) { + ExecutionContext &SF = ECStack.back(); + void *Dest = GVTOP(getOperandValue(I.getAddress(), SF)); + SwitchToNewBasicBlock((BasicBlock*)Dest, SF); +} + + +// SwitchToNewBasicBlock - This method is used to jump to a new basic block. +// This function handles the actual updating of block and instruction iterators +// as well as execution of all of the PHI nodes in the destination block. +// +// This method does this because all of the PHI nodes must be executed +// atomically, reading their inputs before any of the results are updated. Not +// doing this can cause problems if the PHI nodes depend on other PHI nodes for +// their inputs. If the input PHI node is updated before it is read, incorrect +// results can happen. Thus we use a two phase approach. +// +void Interpreter::SwitchToNewBasicBlock(BasicBlock *Dest, ExecutionContext &SF){ + BasicBlock *PrevBB = SF.CurBB; // Remember where we came from... + SF.CurBB = Dest; // Update CurBB to branch destination + SF.CurInst = SF.CurBB->begin(); // Update new instruction ptr... + + if (!isa<PHINode>(SF.CurInst)) return; // Nothing fancy to do + + // Loop over all of the PHI nodes in the current block, reading their inputs. + std::vector<GenericValue> ResultValues; + + for (; PHINode *PN = dyn_cast<PHINode>(SF.CurInst); ++SF.CurInst) { + // Search for the value corresponding to this previous bb... + int i = PN->getBasicBlockIndex(PrevBB); + assert(i != -1 && "PHINode doesn't contain entry for predecessor??"); + Value *IncomingValue = PN->getIncomingValue(i); + + // Save the incoming value for this PHI node... + ResultValues.push_back(getOperandValue(IncomingValue, SF)); + } + + // Now loop over all of the PHI nodes setting their values... + SF.CurInst = SF.CurBB->begin(); + for (unsigned i = 0; isa<PHINode>(SF.CurInst); ++SF.CurInst, ++i) { + PHINode *PN = cast<PHINode>(SF.CurInst); + SetValue(PN, ResultValues[i], SF); + } +} + +//===----------------------------------------------------------------------===// +// Memory Instruction Implementations +//===----------------------------------------------------------------------===// + +void Interpreter::visitAllocaInst(AllocaInst &I) { + ExecutionContext &SF = ECStack.back(); + + Type *Ty = I.getType()->getElementType(); // Type to be allocated + + // Get the number of elements being allocated by the array... + unsigned NumElements = + getOperandValue(I.getOperand(0), SF).IntVal.getZExtValue(); + + unsigned TypeSize = (size_t)getDataLayout().getTypeAllocSize(Ty); + + // Avoid malloc-ing zero bytes, use max()... + unsigned MemToAlloc = std::max(1U, NumElements * TypeSize); + + // Allocate enough memory to hold the type... + void *Memory = safe_malloc(MemToAlloc); + + LLVM_DEBUG(dbgs() << "Allocated Type: " << *Ty << " (" << TypeSize + << " bytes) x " << NumElements << " (Total: " << MemToAlloc + << ") at " << uintptr_t(Memory) << '\n'); + + GenericValue Result = PTOGV(Memory); + assert(Result.PointerVal && "Null pointer returned by malloc!"); + SetValue(&I, Result, SF); + + if (I.getOpcode() == Instruction::Alloca) + ECStack.back().Allocas.add(Memory); +} + +// getElementOffset - The workhorse for getelementptr. +// +GenericValue Interpreter::executeGEPOperation(Value *Ptr, gep_type_iterator I, + gep_type_iterator E, + ExecutionContext &SF) { + assert(Ptr->getType()->isPointerTy() && + "Cannot getElementOffset of a nonpointer type!"); + + uint64_t Total = 0; + + for (; I != E; ++I) { + if (StructType *STy = I.getStructTypeOrNull()) { + const StructLayout *SLO = getDataLayout().getStructLayout(STy); + + const ConstantInt *CPU = cast<ConstantInt>(I.getOperand()); + unsigned Index = unsigned(CPU->getZExtValue()); + + Total += SLO->getElementOffset(Index); + } else { + // Get the index number for the array... which must be long type... + GenericValue IdxGV = getOperandValue(I.getOperand(), SF); + + int64_t Idx; + unsigned BitWidth = + cast<IntegerType>(I.getOperand()->getType())->getBitWidth(); + if (BitWidth == 32) + Idx = (int64_t)(int32_t)IdxGV.IntVal.getZExtValue(); + else { + assert(BitWidth == 64 && "Invalid index type for getelementptr"); + Idx = (int64_t)IdxGV.IntVal.getZExtValue(); + } + Total += getDataLayout().getTypeAllocSize(I.getIndexedType()) * Idx; + } + } + + GenericValue Result; + Result.PointerVal = ((char*)getOperandValue(Ptr, SF).PointerVal) + Total; + LLVM_DEBUG(dbgs() << "GEP Index " << Total << " bytes.\n"); + return Result; +} + +void Interpreter::visitGetElementPtrInst(GetElementPtrInst &I) { + ExecutionContext &SF = ECStack.back(); + SetValue(&I, executeGEPOperation(I.getPointerOperand(), + gep_type_begin(I), gep_type_end(I), SF), SF); +} + +void Interpreter::visitLoadInst(LoadInst &I) { + ExecutionContext &SF = ECStack.back(); + GenericValue SRC = getOperandValue(I.getPointerOperand(), SF); + GenericValue *Ptr = (GenericValue*)GVTOP(SRC); + GenericValue Result; + LoadValueFromMemory(Result, Ptr, I.getType()); + SetValue(&I, Result, SF); + if (I.isVolatile() && PrintVolatile) + dbgs() << "Volatile load " << I; +} + +void Interpreter::visitStoreInst(StoreInst &I) { + ExecutionContext &SF = ECStack.back(); + GenericValue Val = getOperandValue(I.getOperand(0), SF); + GenericValue SRC = getOperandValue(I.getPointerOperand(), SF); + StoreValueToMemory(Val, (GenericValue *)GVTOP(SRC), + I.getOperand(0)->getType()); + if (I.isVolatile() && PrintVolatile) + dbgs() << "Volatile store: " << I; +} + +//===----------------------------------------------------------------------===// +// Miscellaneous Instruction Implementations +//===----------------------------------------------------------------------===// + +void Interpreter::visitVAStartInst(VAStartInst &I) { + ExecutionContext &SF = ECStack.back(); + GenericValue ArgIndex; + ArgIndex.UIntPairVal.first = ECStack.size() - 1; + ArgIndex.UIntPairVal.second = 0; + SetValue(&I, ArgIndex, SF); +} + +void Interpreter::visitVAEndInst(VAEndInst &I) { + // va_end is a noop for the interpreter +} + +void Interpreter::visitVACopyInst(VACopyInst &I) { + ExecutionContext &SF = ECStack.back(); + SetValue(&I, getOperandValue(*I.arg_begin(), SF), SF); +} + +void Interpreter::visitIntrinsicInst(IntrinsicInst &I) { + ExecutionContext &SF = ECStack.back(); + + // If it is an unknown intrinsic function, use the intrinsic lowering + // class to transform it into hopefully tasty LLVM code. + // + BasicBlock::iterator Me(&I); + BasicBlock *Parent = I.getParent(); + bool atBegin(Parent->begin() == Me); + if (!atBegin) + --Me; + IL->LowerIntrinsicCall(&I); + + // Restore the CurInst pointer to the first instruction newly inserted, if + // any. + if (atBegin) { + SF.CurInst = Parent->begin(); + } else { + SF.CurInst = Me; + ++SF.CurInst; + } +} + +void Interpreter::visitCallBase(CallBase &I) { + ExecutionContext &SF = ECStack.back(); + + SF.Caller = &I; + std::vector<GenericValue> ArgVals; + const unsigned NumArgs = SF.Caller->arg_size(); + ArgVals.reserve(NumArgs); + for (Value *V : SF.Caller->args()) + ArgVals.push_back(getOperandValue(V, SF)); + + // To handle indirect calls, we must get the pointer value from the argument + // and treat it as a function pointer. + GenericValue SRC = getOperandValue(SF.Caller->getCalledOperand(), SF); + callFunction((Function*)GVTOP(SRC), ArgVals); +} + +// auxiliary function for shift operations +static unsigned getShiftAmount(uint64_t orgShiftAmount, + llvm::APInt valueToShift) { + unsigned valueWidth = valueToShift.getBitWidth(); + if (orgShiftAmount < (uint64_t)valueWidth) + return orgShiftAmount; + // according to the llvm documentation, if orgShiftAmount > valueWidth, + // the result is undfeined. but we do shift by this rule: + return (NextPowerOf2(valueWidth-1) - 1) & orgShiftAmount; +} + + +void Interpreter::visitShl(BinaryOperator &I) { + ExecutionContext &SF = ECStack.back(); + GenericValue Src1 = getOperandValue(I.getOperand(0), SF); + GenericValue Src2 = getOperandValue(I.getOperand(1), SF); + GenericValue Dest; + Type *Ty = I.getType(); + + if (Ty->isVectorTy()) { + uint32_t src1Size = uint32_t(Src1.AggregateVal.size()); + assert(src1Size == Src2.AggregateVal.size()); + for (unsigned i = 0; i < src1Size; i++) { + GenericValue Result; + uint64_t shiftAmount = Src2.AggregateVal[i].IntVal.getZExtValue(); + llvm::APInt valueToShift = Src1.AggregateVal[i].IntVal; + Result.IntVal = valueToShift.shl(getShiftAmount(shiftAmount, valueToShift)); + Dest.AggregateVal.push_back(Result); + } + } else { + // scalar + uint64_t shiftAmount = Src2.IntVal.getZExtValue(); + llvm::APInt valueToShift = Src1.IntVal; + Dest.IntVal = valueToShift.shl(getShiftAmount(shiftAmount, valueToShift)); + } + + SetValue(&I, Dest, SF); +} + +void Interpreter::visitLShr(BinaryOperator &I) { + ExecutionContext &SF = ECStack.back(); + GenericValue Src1 = getOperandValue(I.getOperand(0), SF); + GenericValue Src2 = getOperandValue(I.getOperand(1), SF); + GenericValue Dest; + Type *Ty = I.getType(); + + if (Ty->isVectorTy()) { + uint32_t src1Size = uint32_t(Src1.AggregateVal.size()); + assert(src1Size == Src2.AggregateVal.size()); + for (unsigned i = 0; i < src1Size; i++) { + GenericValue Result; + uint64_t shiftAmount = Src2.AggregateVal[i].IntVal.getZExtValue(); + llvm::APInt valueToShift = Src1.AggregateVal[i].IntVal; + Result.IntVal = valueToShift.lshr(getShiftAmount(shiftAmount, valueToShift)); + Dest.AggregateVal.push_back(Result); + } + } else { + // scalar + uint64_t shiftAmount = Src2.IntVal.getZExtValue(); + llvm::APInt valueToShift = Src1.IntVal; + Dest.IntVal = valueToShift.lshr(getShiftAmount(shiftAmount, valueToShift)); + } + + SetValue(&I, Dest, SF); +} + +void Interpreter::visitAShr(BinaryOperator &I) { + ExecutionContext &SF = ECStack.back(); + GenericValue Src1 = getOperandValue(I.getOperand(0), SF); + GenericValue Src2 = getOperandValue(I.getOperand(1), SF); + GenericValue Dest; + Type *Ty = I.getType(); + + if (Ty->isVectorTy()) { + size_t src1Size = Src1.AggregateVal.size(); + assert(src1Size == Src2.AggregateVal.size()); + for (unsigned i = 0; i < src1Size; i++) { + GenericValue Result; + uint64_t shiftAmount = Src2.AggregateVal[i].IntVal.getZExtValue(); + llvm::APInt valueToShift = Src1.AggregateVal[i].IntVal; + Result.IntVal = valueToShift.ashr(getShiftAmount(shiftAmount, valueToShift)); + Dest.AggregateVal.push_back(Result); + } + } else { + // scalar + uint64_t shiftAmount = Src2.IntVal.getZExtValue(); + llvm::APInt valueToShift = Src1.IntVal; + Dest.IntVal = valueToShift.ashr(getShiftAmount(shiftAmount, valueToShift)); + } + + SetValue(&I, Dest, SF); +} + +GenericValue Interpreter::executeTruncInst(Value *SrcVal, Type *DstTy, + ExecutionContext &SF) { + GenericValue Dest, Src = getOperandValue(SrcVal, SF); + Type *SrcTy = SrcVal->getType(); + if (SrcTy->isVectorTy()) { + Type *DstVecTy = DstTy->getScalarType(); + unsigned DBitWidth = cast<IntegerType>(DstVecTy)->getBitWidth(); + unsigned NumElts = Src.AggregateVal.size(); + // the sizes of src and dst vectors must be equal + Dest.AggregateVal.resize(NumElts); + for (unsigned i = 0; i < NumElts; i++) + Dest.AggregateVal[i].IntVal = Src.AggregateVal[i].IntVal.trunc(DBitWidth); + } else { + IntegerType *DITy = cast<IntegerType>(DstTy); + unsigned DBitWidth = DITy->getBitWidth(); + Dest.IntVal = Src.IntVal.trunc(DBitWidth); + } + return Dest; +} + +GenericValue Interpreter::executeSExtInst(Value *SrcVal, Type *DstTy, + ExecutionContext &SF) { + Type *SrcTy = SrcVal->getType(); + GenericValue Dest, Src = getOperandValue(SrcVal, SF); + if (SrcTy->isVectorTy()) { + Type *DstVecTy = DstTy->getScalarType(); + unsigned DBitWidth = cast<IntegerType>(DstVecTy)->getBitWidth(); + unsigned size = Src.AggregateVal.size(); + // the sizes of src and dst vectors must be equal. + Dest.AggregateVal.resize(size); + for (unsigned i = 0; i < size; i++) + Dest.AggregateVal[i].IntVal = Src.AggregateVal[i].IntVal.sext(DBitWidth); + } else { + auto *DITy = cast<IntegerType>(DstTy); + unsigned DBitWidth = DITy->getBitWidth(); + Dest.IntVal = Src.IntVal.sext(DBitWidth); + } + return Dest; +} + +GenericValue Interpreter::executeZExtInst(Value *SrcVal, Type *DstTy, + ExecutionContext &SF) { + Type *SrcTy = SrcVal->getType(); + GenericValue Dest, Src = getOperandValue(SrcVal, SF); + if (SrcTy->isVectorTy()) { + Type *DstVecTy = DstTy->getScalarType(); + unsigned DBitWidth = cast<IntegerType>(DstVecTy)->getBitWidth(); + + unsigned size = Src.AggregateVal.size(); + // the sizes of src and dst vectors must be equal. + Dest.AggregateVal.resize(size); + for (unsigned i = 0; i < size; i++) + Dest.AggregateVal[i].IntVal = Src.AggregateVal[i].IntVal.zext(DBitWidth); + } else { + auto *DITy = cast<IntegerType>(DstTy); + unsigned DBitWidth = DITy->getBitWidth(); + Dest.IntVal = Src.IntVal.zext(DBitWidth); + } + return Dest; +} + +GenericValue Interpreter::executeFPTruncInst(Value *SrcVal, Type *DstTy, + ExecutionContext &SF) { + GenericValue Dest, Src = getOperandValue(SrcVal, SF); + + if (isa<VectorType>(SrcVal->getType())) { + assert(SrcVal->getType()->getScalarType()->isDoubleTy() && + DstTy->getScalarType()->isFloatTy() && + "Invalid FPTrunc instruction"); + + unsigned size = Src.AggregateVal.size(); + // the sizes of src and dst vectors must be equal. + Dest.AggregateVal.resize(size); + for (unsigned i = 0; i < size; i++) + Dest.AggregateVal[i].FloatVal = (float)Src.AggregateVal[i].DoubleVal; + } else { + assert(SrcVal->getType()->isDoubleTy() && DstTy->isFloatTy() && + "Invalid FPTrunc instruction"); + Dest.FloatVal = (float)Src.DoubleVal; + } + + return Dest; +} + +GenericValue Interpreter::executeFPExtInst(Value *SrcVal, Type *DstTy, + ExecutionContext &SF) { + GenericValue Dest, Src = getOperandValue(SrcVal, SF); + + if (isa<VectorType>(SrcVal->getType())) { + assert(SrcVal->getType()->getScalarType()->isFloatTy() && + DstTy->getScalarType()->isDoubleTy() && "Invalid FPExt instruction"); + + unsigned size = Src.AggregateVal.size(); + // the sizes of src and dst vectors must be equal. + Dest.AggregateVal.resize(size); + for (unsigned i = 0; i < size; i++) + Dest.AggregateVal[i].DoubleVal = (double)Src.AggregateVal[i].FloatVal; + } else { + assert(SrcVal->getType()->isFloatTy() && DstTy->isDoubleTy() && + "Invalid FPExt instruction"); + Dest.DoubleVal = (double)Src.FloatVal; + } + + return Dest; +} + +GenericValue Interpreter::executeFPToUIInst(Value *SrcVal, Type *DstTy, + ExecutionContext &SF) { + Type *SrcTy = SrcVal->getType(); + GenericValue Dest, Src = getOperandValue(SrcVal, SF); + + if (isa<VectorType>(SrcTy)) { + Type *DstVecTy = DstTy->getScalarType(); + Type *SrcVecTy = SrcTy->getScalarType(); + uint32_t DBitWidth = cast<IntegerType>(DstVecTy)->getBitWidth(); + unsigned size = Src.AggregateVal.size(); + // the sizes of src and dst vectors must be equal. + Dest.AggregateVal.resize(size); + + if (SrcVecTy->getTypeID() == Type::FloatTyID) { + assert(SrcVecTy->isFloatingPointTy() && "Invalid FPToUI instruction"); + for (unsigned i = 0; i < size; i++) + Dest.AggregateVal[i].IntVal = APIntOps::RoundFloatToAPInt( + Src.AggregateVal[i].FloatVal, DBitWidth); + } else { + for (unsigned i = 0; i < size; i++) + Dest.AggregateVal[i].IntVal = APIntOps::RoundDoubleToAPInt( + Src.AggregateVal[i].DoubleVal, DBitWidth); + } + } else { + // scalar + uint32_t DBitWidth = cast<IntegerType>(DstTy)->getBitWidth(); + assert(SrcTy->isFloatingPointTy() && "Invalid FPToUI instruction"); + + if (SrcTy->getTypeID() == Type::FloatTyID) + Dest.IntVal = APIntOps::RoundFloatToAPInt(Src.FloatVal, DBitWidth); + else { + Dest.IntVal = APIntOps::RoundDoubleToAPInt(Src.DoubleVal, DBitWidth); + } + } + + return Dest; +} + +GenericValue Interpreter::executeFPToSIInst(Value *SrcVal, Type *DstTy, + ExecutionContext &SF) { + Type *SrcTy = SrcVal->getType(); + GenericValue Dest, Src = getOperandValue(SrcVal, SF); + + if (isa<VectorType>(SrcTy)) { + Type *DstVecTy = DstTy->getScalarType(); + Type *SrcVecTy = SrcTy->getScalarType(); + uint32_t DBitWidth = cast<IntegerType>(DstVecTy)->getBitWidth(); + unsigned size = Src.AggregateVal.size(); + // the sizes of src and dst vectors must be equal + Dest.AggregateVal.resize(size); + + if (SrcVecTy->getTypeID() == Type::FloatTyID) { + assert(SrcVecTy->isFloatingPointTy() && "Invalid FPToSI instruction"); + for (unsigned i = 0; i < size; i++) + Dest.AggregateVal[i].IntVal = APIntOps::RoundFloatToAPInt( + Src.AggregateVal[i].FloatVal, DBitWidth); + } else { + for (unsigned i = 0; i < size; i++) + Dest.AggregateVal[i].IntVal = APIntOps::RoundDoubleToAPInt( + Src.AggregateVal[i].DoubleVal, DBitWidth); + } + } else { + // scalar + unsigned DBitWidth = cast<IntegerType>(DstTy)->getBitWidth(); + assert(SrcTy->isFloatingPointTy() && "Invalid FPToSI instruction"); + + if (SrcTy->getTypeID() == Type::FloatTyID) + Dest.IntVal = APIntOps::RoundFloatToAPInt(Src.FloatVal, DBitWidth); + else { + Dest.IntVal = APIntOps::RoundDoubleToAPInt(Src.DoubleVal, DBitWidth); + } + } + return Dest; +} + +GenericValue Interpreter::executeUIToFPInst(Value *SrcVal, Type *DstTy, + ExecutionContext &SF) { + GenericValue Dest, Src = getOperandValue(SrcVal, SF); + + if (isa<VectorType>(SrcVal->getType())) { + Type *DstVecTy = DstTy->getScalarType(); + unsigned size = Src.AggregateVal.size(); + // the sizes of src and dst vectors must be equal + Dest.AggregateVal.resize(size); + + if (DstVecTy->getTypeID() == Type::FloatTyID) { + assert(DstVecTy->isFloatingPointTy() && "Invalid UIToFP instruction"); + for (unsigned i = 0; i < size; i++) + Dest.AggregateVal[i].FloatVal = + APIntOps::RoundAPIntToFloat(Src.AggregateVal[i].IntVal); + } else { + for (unsigned i = 0; i < size; i++) + Dest.AggregateVal[i].DoubleVal = + APIntOps::RoundAPIntToDouble(Src.AggregateVal[i].IntVal); + } + } else { + // scalar + assert(DstTy->isFloatingPointTy() && "Invalid UIToFP instruction"); + if (DstTy->getTypeID() == Type::FloatTyID) + Dest.FloatVal = APIntOps::RoundAPIntToFloat(Src.IntVal); + else { + Dest.DoubleVal = APIntOps::RoundAPIntToDouble(Src.IntVal); + } + } + return Dest; +} + +GenericValue Interpreter::executeSIToFPInst(Value *SrcVal, Type *DstTy, + ExecutionContext &SF) { + GenericValue Dest, Src = getOperandValue(SrcVal, SF); + + if (isa<VectorType>(SrcVal->getType())) { + Type *DstVecTy = DstTy->getScalarType(); + unsigned size = Src.AggregateVal.size(); + // the sizes of src and dst vectors must be equal + Dest.AggregateVal.resize(size); + + if (DstVecTy->getTypeID() == Type::FloatTyID) { + assert(DstVecTy->isFloatingPointTy() && "Invalid SIToFP instruction"); + for (unsigned i = 0; i < size; i++) + Dest.AggregateVal[i].FloatVal = + APIntOps::RoundSignedAPIntToFloat(Src.AggregateVal[i].IntVal); + } else { + for (unsigned i = 0; i < size; i++) + Dest.AggregateVal[i].DoubleVal = + APIntOps::RoundSignedAPIntToDouble(Src.AggregateVal[i].IntVal); + } + } else { + // scalar + assert(DstTy->isFloatingPointTy() && "Invalid SIToFP instruction"); + + if (DstTy->getTypeID() == Type::FloatTyID) + Dest.FloatVal = APIntOps::RoundSignedAPIntToFloat(Src.IntVal); + else { + Dest.DoubleVal = APIntOps::RoundSignedAPIntToDouble(Src.IntVal); + } + } + + return Dest; +} + +GenericValue Interpreter::executePtrToIntInst(Value *SrcVal, Type *DstTy, + ExecutionContext &SF) { + uint32_t DBitWidth = cast<IntegerType>(DstTy)->getBitWidth(); + GenericValue Dest, Src = getOperandValue(SrcVal, SF); + assert(SrcVal->getType()->isPointerTy() && "Invalid PtrToInt instruction"); + + Dest.IntVal = APInt(DBitWidth, (intptr_t) Src.PointerVal); + return Dest; +} + +GenericValue Interpreter::executeIntToPtrInst(Value *SrcVal, Type *DstTy, + ExecutionContext &SF) { + GenericValue Dest, Src = getOperandValue(SrcVal, SF); + assert(DstTy->isPointerTy() && "Invalid PtrToInt instruction"); + + uint32_t PtrSize = getDataLayout().getPointerSizeInBits(); + if (PtrSize != Src.IntVal.getBitWidth()) + Src.IntVal = Src.IntVal.zextOrTrunc(PtrSize); + + Dest.PointerVal = PointerTy(intptr_t(Src.IntVal.getZExtValue())); + return Dest; +} + +GenericValue Interpreter::executeBitCastInst(Value *SrcVal, Type *DstTy, + ExecutionContext &SF) { + + // This instruction supports bitwise conversion of vectors to integers and + // to vectors of other types (as long as they have the same size) + Type *SrcTy = SrcVal->getType(); + GenericValue Dest, Src = getOperandValue(SrcVal, SF); + + if (isa<VectorType>(SrcTy) || isa<VectorType>(DstTy)) { + // vector src bitcast to vector dst or vector src bitcast to scalar dst or + // scalar src bitcast to vector dst + bool isLittleEndian = getDataLayout().isLittleEndian(); + GenericValue TempDst, TempSrc, SrcVec; + Type *SrcElemTy; + Type *DstElemTy; + unsigned SrcBitSize; + unsigned DstBitSize; + unsigned SrcNum; + unsigned DstNum; + + if (isa<VectorType>(SrcTy)) { + SrcElemTy = SrcTy->getScalarType(); + SrcBitSize = SrcTy->getScalarSizeInBits(); + SrcNum = Src.AggregateVal.size(); + SrcVec = Src; + } else { + // if src is scalar value, make it vector <1 x type> + SrcElemTy = SrcTy; + SrcBitSize = SrcTy->getPrimitiveSizeInBits(); + SrcNum = 1; + SrcVec.AggregateVal.push_back(Src); + } + + if (isa<VectorType>(DstTy)) { + DstElemTy = DstTy->getScalarType(); + DstBitSize = DstTy->getScalarSizeInBits(); + DstNum = (SrcNum * SrcBitSize) / DstBitSize; + } else { + DstElemTy = DstTy; + DstBitSize = DstTy->getPrimitiveSizeInBits(); + DstNum = 1; + } + + if (SrcNum * SrcBitSize != DstNum * DstBitSize) + llvm_unreachable("Invalid BitCast"); + + // If src is floating point, cast to integer first. + TempSrc.AggregateVal.resize(SrcNum); + if (SrcElemTy->isFloatTy()) { + for (unsigned i = 0; i < SrcNum; i++) + TempSrc.AggregateVal[i].IntVal = + APInt::floatToBits(SrcVec.AggregateVal[i].FloatVal); + + } else if (SrcElemTy->isDoubleTy()) { + for (unsigned i = 0; i < SrcNum; i++) + TempSrc.AggregateVal[i].IntVal = + APInt::doubleToBits(SrcVec.AggregateVal[i].DoubleVal); + } else if (SrcElemTy->isIntegerTy()) { + for (unsigned i = 0; i < SrcNum; i++) + TempSrc.AggregateVal[i].IntVal = SrcVec.AggregateVal[i].IntVal; + } else { + // Pointers are not allowed as the element type of vector. + llvm_unreachable("Invalid Bitcast"); + } + + // now TempSrc is integer type vector + if (DstNum < SrcNum) { + // Example: bitcast <4 x i32> <i32 0, i32 1, i32 2, i32 3> to <2 x i64> + unsigned Ratio = SrcNum / DstNum; + unsigned SrcElt = 0; + for (unsigned i = 0; i < DstNum; i++) { + GenericValue Elt; + Elt.IntVal = 0; + Elt.IntVal = Elt.IntVal.zext(DstBitSize); + unsigned ShiftAmt = isLittleEndian ? 0 : SrcBitSize * (Ratio - 1); + for (unsigned j = 0; j < Ratio; j++) { + APInt Tmp; + Tmp = Tmp.zext(SrcBitSize); + Tmp = TempSrc.AggregateVal[SrcElt++].IntVal; + Tmp = Tmp.zext(DstBitSize); + Tmp <<= ShiftAmt; + ShiftAmt += isLittleEndian ? SrcBitSize : -SrcBitSize; + Elt.IntVal |= Tmp; + } + TempDst.AggregateVal.push_back(Elt); + } + } else { + // Example: bitcast <2 x i64> <i64 0, i64 1> to <4 x i32> + unsigned Ratio = DstNum / SrcNum; + for (unsigned i = 0; i < SrcNum; i++) { + unsigned ShiftAmt = isLittleEndian ? 0 : DstBitSize * (Ratio - 1); + for (unsigned j = 0; j < Ratio; j++) { + GenericValue Elt; + Elt.IntVal = Elt.IntVal.zext(SrcBitSize); + Elt.IntVal = TempSrc.AggregateVal[i].IntVal; + Elt.IntVal.lshrInPlace(ShiftAmt); + // it could be DstBitSize == SrcBitSize, so check it + if (DstBitSize < SrcBitSize) + Elt.IntVal = Elt.IntVal.trunc(DstBitSize); + ShiftAmt += isLittleEndian ? DstBitSize : -DstBitSize; + TempDst.AggregateVal.push_back(Elt); + } + } + } + + // convert result from integer to specified type + if (isa<VectorType>(DstTy)) { + if (DstElemTy->isDoubleTy()) { + Dest.AggregateVal.resize(DstNum); + for (unsigned i = 0; i < DstNum; i++) + Dest.AggregateVal[i].DoubleVal = + TempDst.AggregateVal[i].IntVal.bitsToDouble(); + } else if (DstElemTy->isFloatTy()) { + Dest.AggregateVal.resize(DstNum); + for (unsigned i = 0; i < DstNum; i++) + Dest.AggregateVal[i].FloatVal = + TempDst.AggregateVal[i].IntVal.bitsToFloat(); + } else { + Dest = TempDst; + } + } else { + if (DstElemTy->isDoubleTy()) + Dest.DoubleVal = TempDst.AggregateVal[0].IntVal.bitsToDouble(); + else if (DstElemTy->isFloatTy()) { + Dest.FloatVal = TempDst.AggregateVal[0].IntVal.bitsToFloat(); + } else { + Dest.IntVal = TempDst.AggregateVal[0].IntVal; + } + } + } else { // if (isa<VectorType>(SrcTy)) || isa<VectorType>(DstTy)) + + // scalar src bitcast to scalar dst + if (DstTy->isPointerTy()) { + assert(SrcTy->isPointerTy() && "Invalid BitCast"); + Dest.PointerVal = Src.PointerVal; + } else if (DstTy->isIntegerTy()) { + if (SrcTy->isFloatTy()) + Dest.IntVal = APInt::floatToBits(Src.FloatVal); + else if (SrcTy->isDoubleTy()) { + Dest.IntVal = APInt::doubleToBits(Src.DoubleVal); + } else if (SrcTy->isIntegerTy()) { + Dest.IntVal = Src.IntVal; + } else { + llvm_unreachable("Invalid BitCast"); + } + } else if (DstTy->isFloatTy()) { + if (SrcTy->isIntegerTy()) + Dest.FloatVal = Src.IntVal.bitsToFloat(); + else { + Dest.FloatVal = Src.FloatVal; + } + } else if (DstTy->isDoubleTy()) { + if (SrcTy->isIntegerTy()) + Dest.DoubleVal = Src.IntVal.bitsToDouble(); + else { + Dest.DoubleVal = Src.DoubleVal; + } + } else { + llvm_unreachable("Invalid Bitcast"); + } + } + + return Dest; +} + +void Interpreter::visitTruncInst(TruncInst &I) { + ExecutionContext &SF = ECStack.back(); + SetValue(&I, executeTruncInst(I.getOperand(0), I.getType(), SF), SF); +} + +void Interpreter::visitSExtInst(SExtInst &I) { + ExecutionContext &SF = ECStack.back(); + SetValue(&I, executeSExtInst(I.getOperand(0), I.getType(), SF), SF); +} + +void Interpreter::visitZExtInst(ZExtInst &I) { + ExecutionContext &SF = ECStack.back(); + SetValue(&I, executeZExtInst(I.getOperand(0), I.getType(), SF), SF); +} + +void Interpreter::visitFPTruncInst(FPTruncInst &I) { + ExecutionContext &SF = ECStack.back(); + SetValue(&I, executeFPTruncInst(I.getOperand(0), I.getType(), SF), SF); +} + +void Interpreter::visitFPExtInst(FPExtInst &I) { + ExecutionContext &SF = ECStack.back(); + SetValue(&I, executeFPExtInst(I.getOperand(0), I.getType(), SF), SF); +} + +void Interpreter::visitUIToFPInst(UIToFPInst &I) { + ExecutionContext &SF = ECStack.back(); + SetValue(&I, executeUIToFPInst(I.getOperand(0), I.getType(), SF), SF); +} + +void Interpreter::visitSIToFPInst(SIToFPInst &I) { + ExecutionContext &SF = ECStack.back(); + SetValue(&I, executeSIToFPInst(I.getOperand(0), I.getType(), SF), SF); +} + +void Interpreter::visitFPToUIInst(FPToUIInst &I) { + ExecutionContext &SF = ECStack.back(); + SetValue(&I, executeFPToUIInst(I.getOperand(0), I.getType(), SF), SF); +} + +void Interpreter::visitFPToSIInst(FPToSIInst &I) { + ExecutionContext &SF = ECStack.back(); + SetValue(&I, executeFPToSIInst(I.getOperand(0), I.getType(), SF), SF); +} + +void Interpreter::visitPtrToIntInst(PtrToIntInst &I) { + ExecutionContext &SF = ECStack.back(); + SetValue(&I, executePtrToIntInst(I.getOperand(0), I.getType(), SF), SF); +} + +void Interpreter::visitIntToPtrInst(IntToPtrInst &I) { + ExecutionContext &SF = ECStack.back(); + SetValue(&I, executeIntToPtrInst(I.getOperand(0), I.getType(), SF), SF); +} + +void Interpreter::visitBitCastInst(BitCastInst &I) { + ExecutionContext &SF = ECStack.back(); + SetValue(&I, executeBitCastInst(I.getOperand(0), I.getType(), SF), SF); +} + +#define IMPLEMENT_VAARG(TY) \ + case Type::TY##TyID: Dest.TY##Val = Src.TY##Val; break + +void Interpreter::visitVAArgInst(VAArgInst &I) { + ExecutionContext &SF = ECStack.back(); + + // Get the incoming valist parameter. LLI treats the valist as a + // (ec-stack-depth var-arg-index) pair. + GenericValue VAList = getOperandValue(I.getOperand(0), SF); + GenericValue Dest; + GenericValue Src = ECStack[VAList.UIntPairVal.first] + .VarArgs[VAList.UIntPairVal.second]; + Type *Ty = I.getType(); + switch (Ty->getTypeID()) { + case Type::IntegerTyID: + Dest.IntVal = Src.IntVal; + break; + IMPLEMENT_VAARG(Pointer); + IMPLEMENT_VAARG(Float); + IMPLEMENT_VAARG(Double); + default: + dbgs() << "Unhandled dest type for vaarg instruction: " << *Ty << "\n"; + llvm_unreachable(nullptr); + } + + // Set the Value of this Instruction. + SetValue(&I, Dest, SF); + + // Move the pointer to the next vararg. + ++VAList.UIntPairVal.second; +} + +void Interpreter::visitExtractElementInst(ExtractElementInst &I) { + ExecutionContext &SF = ECStack.back(); + GenericValue Src1 = getOperandValue(I.getOperand(0), SF); + GenericValue Src2 = getOperandValue(I.getOperand(1), SF); + GenericValue Dest; + + Type *Ty = I.getType(); + const unsigned indx = unsigned(Src2.IntVal.getZExtValue()); + + if(Src1.AggregateVal.size() > indx) { + switch (Ty->getTypeID()) { + default: + dbgs() << "Unhandled destination type for extractelement instruction: " + << *Ty << "\n"; + llvm_unreachable(nullptr); + break; + case Type::IntegerTyID: + Dest.IntVal = Src1.AggregateVal[indx].IntVal; + break; + case Type::FloatTyID: + Dest.FloatVal = Src1.AggregateVal[indx].FloatVal; + break; + case Type::DoubleTyID: + Dest.DoubleVal = Src1.AggregateVal[indx].DoubleVal; + break; + } + } else { + dbgs() << "Invalid index in extractelement instruction\n"; + } + + SetValue(&I, Dest, SF); +} + +void Interpreter::visitInsertElementInst(InsertElementInst &I) { + ExecutionContext &SF = ECStack.back(); + VectorType *Ty = cast<VectorType>(I.getType()); + + GenericValue Src1 = getOperandValue(I.getOperand(0), SF); + GenericValue Src2 = getOperandValue(I.getOperand(1), SF); + GenericValue Src3 = getOperandValue(I.getOperand(2), SF); + GenericValue Dest; + + Type *TyContained = Ty->getElementType(); + + const unsigned indx = unsigned(Src3.IntVal.getZExtValue()); + Dest.AggregateVal = Src1.AggregateVal; + + if(Src1.AggregateVal.size() <= indx) + llvm_unreachable("Invalid index in insertelement instruction"); + switch (TyContained->getTypeID()) { + default: + llvm_unreachable("Unhandled dest type for insertelement instruction"); + case Type::IntegerTyID: + Dest.AggregateVal[indx].IntVal = Src2.IntVal; + break; + case Type::FloatTyID: + Dest.AggregateVal[indx].FloatVal = Src2.FloatVal; + break; + case Type::DoubleTyID: + Dest.AggregateVal[indx].DoubleVal = Src2.DoubleVal; + break; + } + SetValue(&I, Dest, SF); +} + +void Interpreter::visitShuffleVectorInst(ShuffleVectorInst &I){ + ExecutionContext &SF = ECStack.back(); + + VectorType *Ty = cast<VectorType>(I.getType()); + + GenericValue Src1 = getOperandValue(I.getOperand(0), SF); + GenericValue Src2 = getOperandValue(I.getOperand(1), SF); + GenericValue Dest; + + // There is no need to check types of src1 and src2, because the compiled + // bytecode can't contain different types for src1 and src2 for a + // shufflevector instruction. + + Type *TyContained = Ty->getElementType(); + unsigned src1Size = (unsigned)Src1.AggregateVal.size(); + unsigned src2Size = (unsigned)Src2.AggregateVal.size(); + unsigned src3Size = I.getShuffleMask().size(); + + Dest.AggregateVal.resize(src3Size); + + switch (TyContained->getTypeID()) { + default: + llvm_unreachable("Unhandled dest type for insertelement instruction"); + break; + case Type::IntegerTyID: + for( unsigned i=0; i<src3Size; i++) { + unsigned j = std::max(0, I.getMaskValue(i)); + if(j < src1Size) + Dest.AggregateVal[i].IntVal = Src1.AggregateVal[j].IntVal; + else if(j < src1Size + src2Size) + Dest.AggregateVal[i].IntVal = Src2.AggregateVal[j-src1Size].IntVal; + else + // The selector may not be greater than sum of lengths of first and + // second operands and llasm should not allow situation like + // %tmp = shufflevector <2 x i32> <i32 3, i32 4>, <2 x i32> undef, + // <2 x i32> < i32 0, i32 5 >, + // where i32 5 is invalid, but let it be additional check here: + llvm_unreachable("Invalid mask in shufflevector instruction"); + } + break; + case Type::FloatTyID: + for( unsigned i=0; i<src3Size; i++) { + unsigned j = std::max(0, I.getMaskValue(i)); + if(j < src1Size) + Dest.AggregateVal[i].FloatVal = Src1.AggregateVal[j].FloatVal; + else if(j < src1Size + src2Size) + Dest.AggregateVal[i].FloatVal = Src2.AggregateVal[j-src1Size].FloatVal; + else + llvm_unreachable("Invalid mask in shufflevector instruction"); + } + break; + case Type::DoubleTyID: + for( unsigned i=0; i<src3Size; i++) { + unsigned j = std::max(0, I.getMaskValue(i)); + if(j < src1Size) + Dest.AggregateVal[i].DoubleVal = Src1.AggregateVal[j].DoubleVal; + else if(j < src1Size + src2Size) + Dest.AggregateVal[i].DoubleVal = + Src2.AggregateVal[j-src1Size].DoubleVal; + else + llvm_unreachable("Invalid mask in shufflevector instruction"); + } + break; + } + SetValue(&I, Dest, SF); +} + +void Interpreter::visitExtractValueInst(ExtractValueInst &I) { + ExecutionContext &SF = ECStack.back(); + Value *Agg = I.getAggregateOperand(); + GenericValue Dest; + GenericValue Src = getOperandValue(Agg, SF); + + ExtractValueInst::idx_iterator IdxBegin = I.idx_begin(); + unsigned Num = I.getNumIndices(); + GenericValue *pSrc = &Src; + + for (unsigned i = 0 ; i < Num; ++i) { + pSrc = &pSrc->AggregateVal[*IdxBegin]; + ++IdxBegin; + } + + Type *IndexedType = ExtractValueInst::getIndexedType(Agg->getType(), I.getIndices()); + switch (IndexedType->getTypeID()) { + default: + llvm_unreachable("Unhandled dest type for extractelement instruction"); + break; + case Type::IntegerTyID: + Dest.IntVal = pSrc->IntVal; + break; + case Type::FloatTyID: + Dest.FloatVal = pSrc->FloatVal; + break; + case Type::DoubleTyID: + Dest.DoubleVal = pSrc->DoubleVal; + break; + case Type::ArrayTyID: + case Type::StructTyID: + case Type::FixedVectorTyID: + case Type::ScalableVectorTyID: + Dest.AggregateVal = pSrc->AggregateVal; + break; + case Type::PointerTyID: + Dest.PointerVal = pSrc->PointerVal; + break; + } + + SetValue(&I, Dest, SF); +} + +void Interpreter::visitInsertValueInst(InsertValueInst &I) { + + ExecutionContext &SF = ECStack.back(); + Value *Agg = I.getAggregateOperand(); + + GenericValue Src1 = getOperandValue(Agg, SF); + GenericValue Src2 = getOperandValue(I.getOperand(1), SF); + GenericValue Dest = Src1; // Dest is a slightly changed Src1 + + ExtractValueInst::idx_iterator IdxBegin = I.idx_begin(); + unsigned Num = I.getNumIndices(); + + GenericValue *pDest = &Dest; + for (unsigned i = 0 ; i < Num; ++i) { + pDest = &pDest->AggregateVal[*IdxBegin]; + ++IdxBegin; + } + // pDest points to the target value in the Dest now + + Type *IndexedType = ExtractValueInst::getIndexedType(Agg->getType(), I.getIndices()); + + switch (IndexedType->getTypeID()) { + default: + llvm_unreachable("Unhandled dest type for insertelement instruction"); + break; + case Type::IntegerTyID: + pDest->IntVal = Src2.IntVal; + break; + case Type::FloatTyID: + pDest->FloatVal = Src2.FloatVal; + break; + case Type::DoubleTyID: + pDest->DoubleVal = Src2.DoubleVal; + break; + case Type::ArrayTyID: + case Type::StructTyID: + case Type::FixedVectorTyID: + case Type::ScalableVectorTyID: + pDest->AggregateVal = Src2.AggregateVal; + break; + case Type::PointerTyID: + pDest->PointerVal = Src2.PointerVal; + break; + } + + SetValue(&I, Dest, SF); +} + +GenericValue Interpreter::getConstantExprValue (ConstantExpr *CE, + ExecutionContext &SF) { + switch (CE->getOpcode()) { + case Instruction::Trunc: + return executeTruncInst(CE->getOperand(0), CE->getType(), SF); + case Instruction::ZExt: + return executeZExtInst(CE->getOperand(0), CE->getType(), SF); + case Instruction::SExt: + return executeSExtInst(CE->getOperand(0), CE->getType(), SF); + case Instruction::FPTrunc: + return executeFPTruncInst(CE->getOperand(0), CE->getType(), SF); + case Instruction::FPExt: + return executeFPExtInst(CE->getOperand(0), CE->getType(), SF); + case Instruction::UIToFP: + return executeUIToFPInst(CE->getOperand(0), CE->getType(), SF); + case Instruction::SIToFP: + return executeSIToFPInst(CE->getOperand(0), CE->getType(), SF); + case Instruction::FPToUI: + return executeFPToUIInst(CE->getOperand(0), CE->getType(), SF); + case Instruction::FPToSI: + return executeFPToSIInst(CE->getOperand(0), CE->getType(), SF); + case Instruction::PtrToInt: + return executePtrToIntInst(CE->getOperand(0), CE->getType(), SF); + case Instruction::IntToPtr: + return executeIntToPtrInst(CE->getOperand(0), CE->getType(), SF); + case Instruction::BitCast: + return executeBitCastInst(CE->getOperand(0), CE->getType(), SF); + case Instruction::GetElementPtr: + return executeGEPOperation(CE->getOperand(0), gep_type_begin(CE), + gep_type_end(CE), SF); + case Instruction::FCmp: + case Instruction::ICmp: + return executeCmpInst(CE->getPredicate(), + getOperandValue(CE->getOperand(0), SF), + getOperandValue(CE->getOperand(1), SF), + CE->getOperand(0)->getType()); + case Instruction::Select: + return executeSelectInst(getOperandValue(CE->getOperand(0), SF), + getOperandValue(CE->getOperand(1), SF), + getOperandValue(CE->getOperand(2), SF), + CE->getOperand(0)->getType()); + default : + break; + } + + // The cases below here require a GenericValue parameter for the result + // so we initialize one, compute it and then return it. + GenericValue Op0 = getOperandValue(CE->getOperand(0), SF); + GenericValue Op1 = getOperandValue(CE->getOperand(1), SF); + GenericValue Dest; + Type * Ty = CE->getOperand(0)->getType(); + switch (CE->getOpcode()) { + case Instruction::Add: Dest.IntVal = Op0.IntVal + Op1.IntVal; break; + case Instruction::Sub: Dest.IntVal = Op0.IntVal - Op1.IntVal; break; + case Instruction::Mul: Dest.IntVal = Op0.IntVal * Op1.IntVal; break; + case Instruction::FAdd: executeFAddInst(Dest, Op0, Op1, Ty); break; + case Instruction::FSub: executeFSubInst(Dest, Op0, Op1, Ty); break; + case Instruction::FMul: executeFMulInst(Dest, Op0, Op1, Ty); break; + case Instruction::FDiv: executeFDivInst(Dest, Op0, Op1, Ty); break; + case Instruction::FRem: executeFRemInst(Dest, Op0, Op1, Ty); break; + case Instruction::SDiv: Dest.IntVal = Op0.IntVal.sdiv(Op1.IntVal); break; + case Instruction::UDiv: Dest.IntVal = Op0.IntVal.udiv(Op1.IntVal); break; + case Instruction::URem: Dest.IntVal = Op0.IntVal.urem(Op1.IntVal); break; + case Instruction::SRem: Dest.IntVal = Op0.IntVal.srem(Op1.IntVal); break; + case Instruction::And: Dest.IntVal = Op0.IntVal & Op1.IntVal; break; + case Instruction::Or: Dest.IntVal = Op0.IntVal | Op1.IntVal; break; + case Instruction::Xor: Dest.IntVal = Op0.IntVal ^ Op1.IntVal; break; + case Instruction::Shl: + Dest.IntVal = Op0.IntVal.shl(Op1.IntVal.getZExtValue()); + break; + case Instruction::LShr: + Dest.IntVal = Op0.IntVal.lshr(Op1.IntVal.getZExtValue()); + break; + case Instruction::AShr: + Dest.IntVal = Op0.IntVal.ashr(Op1.IntVal.getZExtValue()); + break; + default: + dbgs() << "Unhandled ConstantExpr: " << *CE << "\n"; + llvm_unreachable("Unhandled ConstantExpr"); + } + return Dest; +} + +GenericValue Interpreter::getOperandValue(Value *V, ExecutionContext &SF) { + if (ConstantExpr *CE = dyn_cast<ConstantExpr>(V)) { + return getConstantExprValue(CE, SF); + } else if (Constant *CPV = dyn_cast<Constant>(V)) { + return getConstantValue(CPV); + } else if (GlobalValue *GV = dyn_cast<GlobalValue>(V)) { + return PTOGV(getPointerToGlobal(GV)); + } else { + return SF.Values[V]; + } +} + +//===----------------------------------------------------------------------===// +// Dispatch and Execution Code +//===----------------------------------------------------------------------===// + +//===----------------------------------------------------------------------===// +// callFunction - Execute the specified function... +// +void Interpreter::callFunction(Function *F, ArrayRef<GenericValue> ArgVals) { + assert((ECStack.empty() || !ECStack.back().Caller || + ECStack.back().Caller->arg_size() == ArgVals.size()) && + "Incorrect number of arguments passed into function call!"); + // Make a new stack frame... and fill it in. + ECStack.emplace_back(); + ExecutionContext &StackFrame = ECStack.back(); + StackFrame.CurFunction = F; + + // Special handling for external functions. + if (F->isDeclaration()) { + GenericValue Result = callExternalFunction (F, ArgVals); + // Simulate a 'ret' instruction of the appropriate type. + popStackAndReturnValueToCaller (F->getReturnType (), Result); + return; + } + + // Get pointers to first LLVM BB & Instruction in function. + StackFrame.CurBB = &F->front(); + StackFrame.CurInst = StackFrame.CurBB->begin(); + + // Run through the function arguments and initialize their values... + assert((ArgVals.size() == F->arg_size() || + (ArgVals.size() > F->arg_size() && F->getFunctionType()->isVarArg()))&& + "Invalid number of values passed to function invocation!"); + + // Handle non-varargs arguments... + unsigned i = 0; + for (Function::arg_iterator AI = F->arg_begin(), E = F->arg_end(); + AI != E; ++AI, ++i) + SetValue(&*AI, ArgVals[i], StackFrame); + + // Handle varargs arguments... + StackFrame.VarArgs.assign(ArgVals.begin()+i, ArgVals.end()); +} + + +void Interpreter::run() { + while (!ECStack.empty()) { + // Interpret a single instruction & increment the "PC". + ExecutionContext &SF = ECStack.back(); // Current stack frame + Instruction &I = *SF.CurInst++; // Increment before execute + + // Track the number of dynamic instructions executed. + ++NumDynamicInsts; + + LLVM_DEBUG(dbgs() << "About to interpret: " << I << "\n"); + visit(I); // Dispatch to one of the visit* methods... + } +} diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp b/contrib/libs/llvm12/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp new file mode 100644 index 0000000000..3aa7755786 --- /dev/null +++ b/contrib/libs/llvm12/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp @@ -0,0 +1,509 @@ +//===-- ExternalFunctions.cpp - Implement External Functions --------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains both code to deal with invoking "external" functions, but +// also contains code that implements "exported" external functions. +// +// There are currently two mechanisms for handling external functions in the +// Interpreter. The first is to implement lle_* wrapper functions that are +// specific to well-known library functions which manually translate the +// arguments from GenericValues and make the call. If such a wrapper does +// not exist, and libffi is available, then the Interpreter will attempt to +// invoke the function using libffi, after finding its address. +// +//===----------------------------------------------------------------------===// + +#include "Interpreter.h" +#include "llvm/ADT/APInt.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/Config/config.h" // Detect libffi +#include "llvm/ExecutionEngine/GenericValue.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/Type.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/DynamicLibrary.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/ManagedStatic.h" +#include "llvm/Support/Mutex.h" +#include "llvm/Support/raw_ostream.h" +#include <cassert> +#include <cmath> +#include <csignal> +#include <cstdint> +#include <cstdio> +#include <cstring> +#include <map> +#include <mutex> +#include <string> +#include <utility> +#include <vector> + +#ifdef HAVE_FFI_CALL +#ifdef HAVE_FFI_H +#include <ffi.h> +#define USE_LIBFFI +#elif HAVE_FFI_FFI_H +#include <ffi/ffi.h> +#define USE_LIBFFI +#endif +#endif + +using namespace llvm; + +static ManagedStatic<sys::Mutex> FunctionsLock; + +typedef GenericValue (*ExFunc)(FunctionType *, ArrayRef<GenericValue>); +static ManagedStatic<std::map<const Function *, ExFunc> > ExportedFunctions; +static ManagedStatic<std::map<std::string, ExFunc> > FuncNames; + +#ifdef USE_LIBFFI +typedef void (*RawFunc)(); +static ManagedStatic<std::map<const Function *, RawFunc> > RawFunctions; +#endif + +static Interpreter *TheInterpreter; + +static char getTypeID(Type *Ty) { + switch (Ty->getTypeID()) { + case Type::VoidTyID: return 'V'; + case Type::IntegerTyID: + switch (cast<IntegerType>(Ty)->getBitWidth()) { + case 1: return 'o'; + case 8: return 'B'; + case 16: return 'S'; + case 32: return 'I'; + case 64: return 'L'; + default: return 'N'; + } + case Type::FloatTyID: return 'F'; + case Type::DoubleTyID: return 'D'; + case Type::PointerTyID: return 'P'; + case Type::FunctionTyID:return 'M'; + case Type::StructTyID: return 'T'; + case Type::ArrayTyID: return 'A'; + default: return 'U'; + } +} + +// Try to find address of external function given a Function object. +// Please note, that interpreter doesn't know how to assemble a +// real call in general case (this is JIT job), that's why it assumes, +// that all external functions has the same (and pretty "general") signature. +// The typical example of such functions are "lle_X_" ones. +static ExFunc lookupFunction(const Function *F) { + // Function not found, look it up... start by figuring out what the + // composite function name should be. + std::string ExtName = "lle_"; + FunctionType *FT = F->getFunctionType(); + ExtName += getTypeID(FT->getReturnType()); + for (Type *T : FT->params()) + ExtName += getTypeID(T); + ExtName += ("_" + F->getName()).str(); + + sys::ScopedLock Writer(*FunctionsLock); + ExFunc FnPtr = (*FuncNames)[ExtName]; + if (!FnPtr) + FnPtr = (*FuncNames)[("lle_X_" + F->getName()).str()]; + if (!FnPtr) // Try calling a generic function... if it exists... + FnPtr = (ExFunc)(intptr_t)sys::DynamicLibrary::SearchForAddressOfSymbol( + ("lle_X_" + F->getName()).str()); + if (FnPtr) + ExportedFunctions->insert(std::make_pair(F, FnPtr)); // Cache for later + return FnPtr; +} + +#ifdef USE_LIBFFI +static ffi_type *ffiTypeFor(Type *Ty) { + switch (Ty->getTypeID()) { + case Type::VoidTyID: return &ffi_type_void; + case Type::IntegerTyID: + switch (cast<IntegerType>(Ty)->getBitWidth()) { + case 8: return &ffi_type_sint8; + case 16: return &ffi_type_sint16; + case 32: return &ffi_type_sint32; + case 64: return &ffi_type_sint64; + } + case Type::FloatTyID: return &ffi_type_float; + case Type::DoubleTyID: return &ffi_type_double; + case Type::PointerTyID: return &ffi_type_pointer; + default: break; + } + // TODO: Support other types such as StructTyID, ArrayTyID, OpaqueTyID, etc. + report_fatal_error("Type could not be mapped for use with libffi."); + return NULL; +} + +static void *ffiValueFor(Type *Ty, const GenericValue &AV, + void *ArgDataPtr) { + switch (Ty->getTypeID()) { + case Type::IntegerTyID: + switch (cast<IntegerType>(Ty)->getBitWidth()) { + case 8: { + int8_t *I8Ptr = (int8_t *) ArgDataPtr; + *I8Ptr = (int8_t) AV.IntVal.getZExtValue(); + return ArgDataPtr; + } + case 16: { + int16_t *I16Ptr = (int16_t *) ArgDataPtr; + *I16Ptr = (int16_t) AV.IntVal.getZExtValue(); + return ArgDataPtr; + } + case 32: { + int32_t *I32Ptr = (int32_t *) ArgDataPtr; + *I32Ptr = (int32_t) AV.IntVal.getZExtValue(); + return ArgDataPtr; + } + case 64: { + int64_t *I64Ptr = (int64_t *) ArgDataPtr; + *I64Ptr = (int64_t) AV.IntVal.getZExtValue(); + return ArgDataPtr; + } + } + case Type::FloatTyID: { + float *FloatPtr = (float *) ArgDataPtr; + *FloatPtr = AV.FloatVal; + return ArgDataPtr; + } + case Type::DoubleTyID: { + double *DoublePtr = (double *) ArgDataPtr; + *DoublePtr = AV.DoubleVal; + return ArgDataPtr; + } + case Type::PointerTyID: { + void **PtrPtr = (void **) ArgDataPtr; + *PtrPtr = GVTOP(AV); + return ArgDataPtr; + } + default: break; + } + // TODO: Support other types such as StructTyID, ArrayTyID, OpaqueTyID, etc. + report_fatal_error("Type value could not be mapped for use with libffi."); + return NULL; +} + +static bool ffiInvoke(RawFunc Fn, Function *F, ArrayRef<GenericValue> ArgVals, + const DataLayout &TD, GenericValue &Result) { + ffi_cif cif; + FunctionType *FTy = F->getFunctionType(); + const unsigned NumArgs = F->arg_size(); + + // TODO: We don't have type information about the remaining arguments, because + // this information is never passed into ExecutionEngine::runFunction(). + if (ArgVals.size() > NumArgs && F->isVarArg()) { + report_fatal_error("Calling external var arg function '" + F->getName() + + "' is not supported by the Interpreter."); + } + + unsigned ArgBytes = 0; + + std::vector<ffi_type*> args(NumArgs); + for (Function::const_arg_iterator A = F->arg_begin(), E = F->arg_end(); + A != E; ++A) { + const unsigned ArgNo = A->getArgNo(); + Type *ArgTy = FTy->getParamType(ArgNo); + args[ArgNo] = ffiTypeFor(ArgTy); + ArgBytes += TD.getTypeStoreSize(ArgTy); + } + + SmallVector<uint8_t, 128> ArgData; + ArgData.resize(ArgBytes); + uint8_t *ArgDataPtr = ArgData.data(); + SmallVector<void*, 16> values(NumArgs); + for (Function::const_arg_iterator A = F->arg_begin(), E = F->arg_end(); + A != E; ++A) { + const unsigned ArgNo = A->getArgNo(); + Type *ArgTy = FTy->getParamType(ArgNo); + values[ArgNo] = ffiValueFor(ArgTy, ArgVals[ArgNo], ArgDataPtr); + ArgDataPtr += TD.getTypeStoreSize(ArgTy); + } + + Type *RetTy = FTy->getReturnType(); + ffi_type *rtype = ffiTypeFor(RetTy); + + if (ffi_prep_cif(&cif, FFI_DEFAULT_ABI, NumArgs, rtype, args.data()) == + FFI_OK) { + SmallVector<uint8_t, 128> ret; + if (RetTy->getTypeID() != Type::VoidTyID) + ret.resize(TD.getTypeStoreSize(RetTy)); + ffi_call(&cif, Fn, ret.data(), values.data()); + switch (RetTy->getTypeID()) { + case Type::IntegerTyID: + switch (cast<IntegerType>(RetTy)->getBitWidth()) { + case 8: Result.IntVal = APInt(8 , *(int8_t *) ret.data()); break; + case 16: Result.IntVal = APInt(16, *(int16_t*) ret.data()); break; + case 32: Result.IntVal = APInt(32, *(int32_t*) ret.data()); break; + case 64: Result.IntVal = APInt(64, *(int64_t*) ret.data()); break; + } + break; + case Type::FloatTyID: Result.FloatVal = *(float *) ret.data(); break; + case Type::DoubleTyID: Result.DoubleVal = *(double*) ret.data(); break; + case Type::PointerTyID: Result.PointerVal = *(void **) ret.data(); break; + default: break; + } + return true; + } + + return false; +} +#endif // USE_LIBFFI + +GenericValue Interpreter::callExternalFunction(Function *F, + ArrayRef<GenericValue> ArgVals) { + TheInterpreter = this; + + std::unique_lock<sys::Mutex> Guard(*FunctionsLock); + + // Do a lookup to see if the function is in our cache... this should just be a + // deferred annotation! + std::map<const Function *, ExFunc>::iterator FI = ExportedFunctions->find(F); + if (ExFunc Fn = (FI == ExportedFunctions->end()) ? lookupFunction(F) + : FI->second) { + Guard.unlock(); + return Fn(F->getFunctionType(), ArgVals); + } + +#ifdef USE_LIBFFI + std::map<const Function *, RawFunc>::iterator RF = RawFunctions->find(F); + RawFunc RawFn; + if (RF == RawFunctions->end()) { + RawFn = (RawFunc)(intptr_t) + sys::DynamicLibrary::SearchForAddressOfSymbol(std::string(F->getName())); + if (!RawFn) + RawFn = (RawFunc)(intptr_t)getPointerToGlobalIfAvailable(F); + if (RawFn != 0) + RawFunctions->insert(std::make_pair(F, RawFn)); // Cache for later + } else { + RawFn = RF->second; + } + + Guard.unlock(); + + GenericValue Result; + if (RawFn != 0 && ffiInvoke(RawFn, F, ArgVals, getDataLayout(), Result)) + return Result; +#endif // USE_LIBFFI + + if (F->getName() == "__main") + errs() << "Tried to execute an unknown external function: " + << *F->getType() << " __main\n"; + else + report_fatal_error("Tried to execute an unknown external function: " + + F->getName()); +#ifndef USE_LIBFFI + errs() << "Recompiling LLVM with --enable-libffi might help.\n"; +#endif + return GenericValue(); +} + +//===----------------------------------------------------------------------===// +// Functions "exported" to the running application... +// + +// void atexit(Function*) +static GenericValue lle_X_atexit(FunctionType *FT, + ArrayRef<GenericValue> Args) { + assert(Args.size() == 1); + TheInterpreter->addAtExitHandler((Function*)GVTOP(Args[0])); + GenericValue GV; + GV.IntVal = 0; + return GV; +} + +// void exit(int) +static GenericValue lle_X_exit(FunctionType *FT, ArrayRef<GenericValue> Args) { + TheInterpreter->exitCalled(Args[0]); + return GenericValue(); +} + +// void abort(void) +static GenericValue lle_X_abort(FunctionType *FT, ArrayRef<GenericValue> Args) { + //FIXME: should we report or raise here? + //report_fatal_error("Interpreted program raised SIGABRT"); + raise (SIGABRT); + return GenericValue(); +} + +// int sprintf(char *, const char *, ...) - a very rough implementation to make +// output useful. +static GenericValue lle_X_sprintf(FunctionType *FT, + ArrayRef<GenericValue> Args) { + char *OutputBuffer = (char *)GVTOP(Args[0]); + const char *FmtStr = (const char *)GVTOP(Args[1]); + unsigned ArgNo = 2; + + // printf should return # chars printed. This is completely incorrect, but + // close enough for now. + GenericValue GV; + GV.IntVal = APInt(32, strlen(FmtStr)); + while (true) { + switch (*FmtStr) { + case 0: return GV; // Null terminator... + default: // Normal nonspecial character + sprintf(OutputBuffer++, "%c", *FmtStr++); + break; + case '\\': { // Handle escape codes + sprintf(OutputBuffer, "%c%c", *FmtStr, *(FmtStr+1)); + FmtStr += 2; OutputBuffer += 2; + break; + } + case '%': { // Handle format specifiers + char FmtBuf[100] = "", Buffer[1000] = ""; + char *FB = FmtBuf; + *FB++ = *FmtStr++; + char Last = *FB++ = *FmtStr++; + unsigned HowLong = 0; + while (Last != 'c' && Last != 'd' && Last != 'i' && Last != 'u' && + Last != 'o' && Last != 'x' && Last != 'X' && Last != 'e' && + Last != 'E' && Last != 'g' && Last != 'G' && Last != 'f' && + Last != 'p' && Last != 's' && Last != '%') { + if (Last == 'l' || Last == 'L') HowLong++; // Keep track of l's + Last = *FB++ = *FmtStr++; + } + *FB = 0; + + switch (Last) { + case '%': + memcpy(Buffer, "%", 2); break; + case 'c': + sprintf(Buffer, FmtBuf, uint32_t(Args[ArgNo++].IntVal.getZExtValue())); + break; + case 'd': case 'i': + case 'u': case 'o': + case 'x': case 'X': + if (HowLong >= 1) { + if (HowLong == 1 && + TheInterpreter->getDataLayout().getPointerSizeInBits() == 64 && + sizeof(long) < sizeof(int64_t)) { + // Make sure we use %lld with a 64 bit argument because we might be + // compiling LLI on a 32 bit compiler. + unsigned Size = strlen(FmtBuf); + FmtBuf[Size] = FmtBuf[Size-1]; + FmtBuf[Size+1] = 0; + FmtBuf[Size-1] = 'l'; + } + sprintf(Buffer, FmtBuf, Args[ArgNo++].IntVal.getZExtValue()); + } else + sprintf(Buffer, FmtBuf,uint32_t(Args[ArgNo++].IntVal.getZExtValue())); + break; + case 'e': case 'E': case 'g': case 'G': case 'f': + sprintf(Buffer, FmtBuf, Args[ArgNo++].DoubleVal); break; + case 'p': + sprintf(Buffer, FmtBuf, (void*)GVTOP(Args[ArgNo++])); break; + case 's': + sprintf(Buffer, FmtBuf, (char*)GVTOP(Args[ArgNo++])); break; + default: + errs() << "<unknown printf code '" << *FmtStr << "'!>"; + ArgNo++; break; + } + size_t Len = strlen(Buffer); + memcpy(OutputBuffer, Buffer, Len + 1); + OutputBuffer += Len; + } + break; + } + } + return GV; +} + +// int printf(const char *, ...) - a very rough implementation to make output +// useful. +static GenericValue lle_X_printf(FunctionType *FT, + ArrayRef<GenericValue> Args) { + char Buffer[10000]; + std::vector<GenericValue> NewArgs; + NewArgs.push_back(PTOGV((void*)&Buffer[0])); + llvm::append_range(NewArgs, Args); + GenericValue GV = lle_X_sprintf(FT, NewArgs); + outs() << Buffer; + return GV; +} + +// int sscanf(const char *format, ...); +static GenericValue lle_X_sscanf(FunctionType *FT, + ArrayRef<GenericValue> args) { + assert(args.size() < 10 && "Only handle up to 10 args to sscanf right now!"); + + char *Args[10]; + for (unsigned i = 0; i < args.size(); ++i) + Args[i] = (char*)GVTOP(args[i]); + + GenericValue GV; + GV.IntVal = APInt(32, sscanf(Args[0], Args[1], Args[2], Args[3], Args[4], + Args[5], Args[6], Args[7], Args[8], Args[9])); + return GV; +} + +// int scanf(const char *format, ...); +static GenericValue lle_X_scanf(FunctionType *FT, ArrayRef<GenericValue> args) { + assert(args.size() < 10 && "Only handle up to 10 args to scanf right now!"); + + char *Args[10]; + for (unsigned i = 0; i < args.size(); ++i) + Args[i] = (char*)GVTOP(args[i]); + + GenericValue GV; + GV.IntVal = APInt(32, scanf( Args[0], Args[1], Args[2], Args[3], Args[4], + Args[5], Args[6], Args[7], Args[8], Args[9])); + return GV; +} + +// int fprintf(FILE *, const char *, ...) - a very rough implementation to make +// output useful. +static GenericValue lle_X_fprintf(FunctionType *FT, + ArrayRef<GenericValue> Args) { + assert(Args.size() >= 2); + char Buffer[10000]; + std::vector<GenericValue> NewArgs; + NewArgs.push_back(PTOGV(Buffer)); + NewArgs.insert(NewArgs.end(), Args.begin()+1, Args.end()); + GenericValue GV = lle_X_sprintf(FT, NewArgs); + + fputs(Buffer, (FILE *) GVTOP(Args[0])); + return GV; +} + +static GenericValue lle_X_memset(FunctionType *FT, + ArrayRef<GenericValue> Args) { + int val = (int)Args[1].IntVal.getSExtValue(); + size_t len = (size_t)Args[2].IntVal.getZExtValue(); + memset((void *)GVTOP(Args[0]), val, len); + // llvm.memset.* returns void, lle_X_* returns GenericValue, + // so here we return GenericValue with IntVal set to zero + GenericValue GV; + GV.IntVal = 0; + return GV; +} + +static GenericValue lle_X_memcpy(FunctionType *FT, + ArrayRef<GenericValue> Args) { + memcpy(GVTOP(Args[0]), GVTOP(Args[1]), + (size_t)(Args[2].IntVal.getLimitedValue())); + + // llvm.memcpy* returns void, lle_X_* returns GenericValue, + // so here we return GenericValue with IntVal set to zero + GenericValue GV; + GV.IntVal = 0; + return GV; +} + +void Interpreter::initializeExternalFunctions() { + sys::ScopedLock Writer(*FunctionsLock); + (*FuncNames)["lle_X_atexit"] = lle_X_atexit; + (*FuncNames)["lle_X_exit"] = lle_X_exit; + (*FuncNames)["lle_X_abort"] = lle_X_abort; + + (*FuncNames)["lle_X_printf"] = lle_X_printf; + (*FuncNames)["lle_X_sprintf"] = lle_X_sprintf; + (*FuncNames)["lle_X_sscanf"] = lle_X_sscanf; + (*FuncNames)["lle_X_scanf"] = lle_X_scanf; + (*FuncNames)["lle_X_fprintf"] = lle_X_fprintf; + (*FuncNames)["lle_X_memset"] = lle_X_memset; + (*FuncNames)["lle_X_memcpy"] = lle_X_memcpy; +} diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/Interpreter/Interpreter.cpp b/contrib/libs/llvm12/lib/ExecutionEngine/Interpreter/Interpreter.cpp new file mode 100644 index 0000000000..5727f7adb4 --- /dev/null +++ b/contrib/libs/llvm12/lib/ExecutionEngine/Interpreter/Interpreter.cpp @@ -0,0 +1,102 @@ +//===- Interpreter.cpp - Top-Level LLVM Interpreter Implementation --------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the top-level functionality for the LLVM interpreter. +// This interpreter is designed to be a very simple, portable, inefficient +// interpreter. +// +//===----------------------------------------------------------------------===// + +#include "Interpreter.h" +#include "llvm/CodeGen/IntrinsicLowering.h" +#include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/Module.h" +#include <cstring> +using namespace llvm; + +namespace { + +static struct RegisterInterp { + RegisterInterp() { Interpreter::Register(); } +} InterpRegistrator; + +} + +extern "C" void LLVMLinkInInterpreter() { } + +/// Create a new interpreter object. +/// +ExecutionEngine *Interpreter::create(std::unique_ptr<Module> M, + std::string *ErrStr) { + // Tell this Module to materialize everything and release the GVMaterializer. + if (Error Err = M->materializeAll()) { + std::string Msg; + handleAllErrors(std::move(Err), [&](ErrorInfoBase &EIB) { + Msg = EIB.message(); + }); + if (ErrStr) + *ErrStr = Msg; + // We got an error, just return 0 + return nullptr; + } + + return new Interpreter(std::move(M)); +} + +//===----------------------------------------------------------------------===// +// Interpreter ctor - Initialize stuff +// +Interpreter::Interpreter(std::unique_ptr<Module> M) + : ExecutionEngine(std::move(M)) { + + memset(&ExitValue.Untyped, 0, sizeof(ExitValue.Untyped)); + // Initialize the "backend" + initializeExecutionEngine(); + initializeExternalFunctions(); + emitGlobals(); + + IL = new IntrinsicLowering(getDataLayout()); +} + +Interpreter::~Interpreter() { + delete IL; +} + +void Interpreter::runAtExitHandlers () { + while (!AtExitHandlers.empty()) { + callFunction(AtExitHandlers.back(), None); + AtExitHandlers.pop_back(); + run(); + } +} + +/// run - Start execution with the specified function and arguments. +/// +GenericValue Interpreter::runFunction(Function *F, + ArrayRef<GenericValue> ArgValues) { + assert (F && "Function *F was null at entry to run()"); + + // Try extra hard not to pass extra args to a function that isn't + // expecting them. C programmers frequently bend the rules and + // declare main() with fewer parameters than it actually gets + // passed, and the interpreter barfs if you pass a function more + // parameters than it is declared to take. This does not attempt to + // take into account gratuitous differences in declared types, + // though. + const size_t ArgCount = F->getFunctionType()->getNumParams(); + ArrayRef<GenericValue> ActualArgs = + ArgValues.slice(0, std::min(ArgValues.size(), ArgCount)); + + // Set up the function call. + callFunction(F, ActualArgs); + + // Start executing the function. + run(); + + return ExitValue; +} diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/Interpreter/Interpreter.h b/contrib/libs/llvm12/lib/ExecutionEngine/Interpreter/Interpreter.h new file mode 100644 index 0000000000..fd7fa21df1 --- /dev/null +++ b/contrib/libs/llvm12/lib/ExecutionEngine/Interpreter/Interpreter.h @@ -0,0 +1,235 @@ +//===-- Interpreter.h ------------------------------------------*- C++ -*--===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This header file defines the interpreter structure +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_EXECUTIONENGINE_INTERPRETER_INTERPRETER_H +#define LLVM_LIB_EXECUTIONENGINE_INTERPRETER_INTERPRETER_H + +#include "llvm/ExecutionEngine/ExecutionEngine.h" +#include "llvm/ExecutionEngine/GenericValue.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/InstVisitor.h" +#include "llvm/Support/DataTypes.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/raw_ostream.h" +namespace llvm { + +class IntrinsicLowering; +template<typename T> class generic_gep_type_iterator; +class ConstantExpr; +typedef generic_gep_type_iterator<User::const_op_iterator> gep_type_iterator; + + +// AllocaHolder - Object to track all of the blocks of memory allocated by +// alloca. When the function returns, this object is popped off the execution +// stack, which causes the dtor to be run, which frees all the alloca'd memory. +// +class AllocaHolder { + std::vector<void *> Allocations; + +public: + AllocaHolder() {} + + // Make this type move-only. + AllocaHolder(AllocaHolder &&) = default; + AllocaHolder &operator=(AllocaHolder &&RHS) = default; + + ~AllocaHolder() { + for (void *Allocation : Allocations) + free(Allocation); + } + + void add(void *Mem) { Allocations.push_back(Mem); } +}; + +typedef std::vector<GenericValue> ValuePlaneTy; + +// ExecutionContext struct - This struct represents one stack frame currently +// executing. +// +struct ExecutionContext { + Function *CurFunction;// The currently executing function + BasicBlock *CurBB; // The currently executing BB + BasicBlock::iterator CurInst; // The next instruction to execute + CallBase *Caller; // Holds the call that called subframes. + // NULL if main func or debugger invoked fn + std::map<Value *, GenericValue> Values; // LLVM values used in this invocation + std::vector<GenericValue> VarArgs; // Values passed through an ellipsis + AllocaHolder Allocas; // Track memory allocated by alloca + + ExecutionContext() : CurFunction(nullptr), CurBB(nullptr), CurInst(nullptr) {} +}; + +// Interpreter - This class represents the entirety of the interpreter. +// +class Interpreter : public ExecutionEngine, public InstVisitor<Interpreter> { + GenericValue ExitValue; // The return value of the called function + IntrinsicLowering *IL; + + // The runtime stack of executing code. The top of the stack is the current + // function record. + std::vector<ExecutionContext> ECStack; + + // AtExitHandlers - List of functions to call when the program exits, + // registered with the atexit() library function. + std::vector<Function*> AtExitHandlers; + +public: + explicit Interpreter(std::unique_ptr<Module> M); + ~Interpreter() override; + + /// runAtExitHandlers - Run any functions registered by the program's calls to + /// atexit(3), which we intercept and store in AtExitHandlers. + /// + void runAtExitHandlers(); + + static void Register() { + InterpCtor = create; + } + + /// Create an interpreter ExecutionEngine. + /// + static ExecutionEngine *create(std::unique_ptr<Module> M, + std::string *ErrorStr = nullptr); + + /// run - Start execution with the specified function and arguments. + /// + GenericValue runFunction(Function *F, + ArrayRef<GenericValue> ArgValues) override; + + void *getPointerToNamedFunction(StringRef Name, + bool AbortOnFailure = true) override { + // FIXME: not implemented. + return nullptr; + } + + // Methods used to execute code: + // Place a call on the stack + void callFunction(Function *F, ArrayRef<GenericValue> ArgVals); + void run(); // Execute instructions until nothing left to do + + // Opcode Implementations + void visitReturnInst(ReturnInst &I); + void visitBranchInst(BranchInst &I); + void visitSwitchInst(SwitchInst &I); + void visitIndirectBrInst(IndirectBrInst &I); + + void visitUnaryOperator(UnaryOperator &I); + void visitBinaryOperator(BinaryOperator &I); + void visitICmpInst(ICmpInst &I); + void visitFCmpInst(FCmpInst &I); + void visitAllocaInst(AllocaInst &I); + void visitLoadInst(LoadInst &I); + void visitStoreInst(StoreInst &I); + void visitGetElementPtrInst(GetElementPtrInst &I); + void visitPHINode(PHINode &PN) { + llvm_unreachable("PHI nodes already handled!"); + } + void visitTruncInst(TruncInst &I); + void visitZExtInst(ZExtInst &I); + void visitSExtInst(SExtInst &I); + void visitFPTruncInst(FPTruncInst &I); + void visitFPExtInst(FPExtInst &I); + void visitUIToFPInst(UIToFPInst &I); + void visitSIToFPInst(SIToFPInst &I); + void visitFPToUIInst(FPToUIInst &I); + void visitFPToSIInst(FPToSIInst &I); + void visitPtrToIntInst(PtrToIntInst &I); + void visitIntToPtrInst(IntToPtrInst &I); + void visitBitCastInst(BitCastInst &I); + void visitSelectInst(SelectInst &I); + + void visitVAStartInst(VAStartInst &I); + void visitVAEndInst(VAEndInst &I); + void visitVACopyInst(VACopyInst &I); + void visitIntrinsicInst(IntrinsicInst &I); + void visitCallBase(CallBase &I); + void visitUnreachableInst(UnreachableInst &I); + + void visitShl(BinaryOperator &I); + void visitLShr(BinaryOperator &I); + void visitAShr(BinaryOperator &I); + + void visitVAArgInst(VAArgInst &I); + void visitExtractElementInst(ExtractElementInst &I); + void visitInsertElementInst(InsertElementInst &I); + void visitShuffleVectorInst(ShuffleVectorInst &I); + + void visitExtractValueInst(ExtractValueInst &I); + void visitInsertValueInst(InsertValueInst &I); + + void visitInstruction(Instruction &I) { + errs() << I << "\n"; + llvm_unreachable("Instruction not interpretable yet!"); + } + + GenericValue callExternalFunction(Function *F, + ArrayRef<GenericValue> ArgVals); + void exitCalled(GenericValue GV); + + void addAtExitHandler(Function *F) { + AtExitHandlers.push_back(F); + } + + GenericValue *getFirstVarArg () { + return &(ECStack.back ().VarArgs[0]); + } + +private: // Helper functions + GenericValue executeGEPOperation(Value *Ptr, gep_type_iterator I, + gep_type_iterator E, ExecutionContext &SF); + + // SwitchToNewBasicBlock - Start execution in a new basic block and run any + // PHI nodes in the top of the block. This is used for intraprocedural + // control flow. + // + void SwitchToNewBasicBlock(BasicBlock *Dest, ExecutionContext &SF); + + void *getPointerToFunction(Function *F) override { return (void*)F; } + + void initializeExecutionEngine() { } + void initializeExternalFunctions(); + GenericValue getConstantExprValue(ConstantExpr *CE, ExecutionContext &SF); + GenericValue getOperandValue(Value *V, ExecutionContext &SF); + GenericValue executeTruncInst(Value *SrcVal, Type *DstTy, + ExecutionContext &SF); + GenericValue executeSExtInst(Value *SrcVal, Type *DstTy, + ExecutionContext &SF); + GenericValue executeZExtInst(Value *SrcVal, Type *DstTy, + ExecutionContext &SF); + GenericValue executeFPTruncInst(Value *SrcVal, Type *DstTy, + ExecutionContext &SF); + GenericValue executeFPExtInst(Value *SrcVal, Type *DstTy, + ExecutionContext &SF); + GenericValue executeFPToUIInst(Value *SrcVal, Type *DstTy, + ExecutionContext &SF); + GenericValue executeFPToSIInst(Value *SrcVal, Type *DstTy, + ExecutionContext &SF); + GenericValue executeUIToFPInst(Value *SrcVal, Type *DstTy, + ExecutionContext &SF); + GenericValue executeSIToFPInst(Value *SrcVal, Type *DstTy, + ExecutionContext &SF); + GenericValue executePtrToIntInst(Value *SrcVal, Type *DstTy, + ExecutionContext &SF); + GenericValue executeIntToPtrInst(Value *SrcVal, Type *DstTy, + ExecutionContext &SF); + GenericValue executeBitCastInst(Value *SrcVal, Type *DstTy, + ExecutionContext &SF); + GenericValue executeCastOperation(Instruction::CastOps opcode, Value *SrcVal, + Type *Ty, ExecutionContext &SF); + void popStackAndReturnValueToCaller(Type *RetTy, GenericValue Result); + +}; + +} // End llvm namespace + +#endif diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/JITLink/BasicGOTAndStubsBuilder.h b/contrib/libs/llvm12/lib/ExecutionEngine/JITLink/BasicGOTAndStubsBuilder.h new file mode 100644 index 0000000000..82258a35a6 --- /dev/null +++ b/contrib/libs/llvm12/lib/ExecutionEngine/JITLink/BasicGOTAndStubsBuilder.h @@ -0,0 +1,107 @@ +//===--- BasicGOTAndStubsBuilder.h - Generic GOT/Stub creation --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// A base for simple GOT and stub creation. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_EXECUTIONENGINE_JITLINK_BASICGOTANDSTUBSBUILDER_H +#define LLVM_LIB_EXECUTIONENGINE_JITLINK_BASICGOTANDSTUBSBUILDER_H + +#include "llvm/ExecutionEngine/JITLink/JITLink.h" + +#define DEBUG_TYPE "jitlink" + +namespace llvm { +namespace jitlink { + +template <typename BuilderImpl> class BasicGOTAndStubsBuilder { +public: + BasicGOTAndStubsBuilder(LinkGraph &G) : G(G) {} + + void run() { + // We're going to be adding new blocks, but we don't want to iterate over + // the newly added ones, so just copy the existing blocks out. + std::vector<Block *> Blocks(G.blocks().begin(), G.blocks().end()); + + LLVM_DEBUG(dbgs() << "Creating GOT entries and stubs:\n"); + + for (auto *B : Blocks) + for (auto &E : B->edges()) + if (impl().isGOTEdge(E)) { + LLVM_DEBUG({ + dbgs() << " Updating GOT edge "; + printEdge(dbgs(), *B, E, "<target GOT>"); + dbgs() << "\n"; + }); + impl().fixGOTEdge(E, getGOTEntrySymbol(E.getTarget())); + } else if (impl().isExternalBranchEdge(E)) { + LLVM_DEBUG({ + dbgs() << " Updating external branch edge "; + printEdge(dbgs(), *B, E, "<target PC-rel>"); + dbgs() << "\n"; + }); + impl().fixExternalBranchEdge(E, getStubSymbol(E.getTarget())); + } + } + +protected: + Symbol &getGOTEntrySymbol(Symbol &Target) { + assert(Target.hasName() && "GOT edge cannot point to anonymous target"); + + auto GOTEntryI = GOTEntries.find(Target.getName()); + + // Build the entry if it doesn't exist. + if (GOTEntryI == GOTEntries.end()) { + auto &GOTEntry = impl().createGOTEntry(Target); + LLVM_DEBUG({ + dbgs() << " Created GOT entry for " << Target.getName() << ": " + << GOTEntry << "\n"; + }); + GOTEntryI = + GOTEntries.insert(std::make_pair(Target.getName(), &GOTEntry)).first; + } + + assert(GOTEntryI != GOTEntries.end() && "Could not get GOT entry symbol"); + LLVM_DEBUG( + { dbgs() << " Using GOT entry " << *GOTEntryI->second << "\n"; }); + return *GOTEntryI->second; + } + + Symbol &getStubSymbol(Symbol &Target) { + assert(Target.hasName() && + "External branch edge can not point to an anonymous target"); + auto StubI = Stubs.find(Target.getName()); + + if (StubI == Stubs.end()) { + auto &StubSymbol = impl().createStub(Target); + LLVM_DEBUG({ + dbgs() << " Created stub for " << Target.getName() << ": " + << StubSymbol << "\n"; + }); + StubI = Stubs.insert(std::make_pair(Target.getName(), &StubSymbol)).first; + } + + assert(StubI != Stubs.end() && "Count not get stub symbol"); + LLVM_DEBUG({ dbgs() << " Using stub " << *StubI->second << "\n"; }); + return *StubI->second; + } + + LinkGraph &G; + +private: + BuilderImpl &impl() { return static_cast<BuilderImpl &>(*this); } + + DenseMap<StringRef, Symbol *> GOTEntries; + DenseMap<StringRef, Symbol *> Stubs; +}; + +} // end namespace jitlink +} // end namespace llvm + +#endif // LLVM_LIB_EXECUTIONENGINE_JITLINK_BASICGOTANDSTUBSBUILDER_H diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp b/contrib/libs/llvm12/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp new file mode 100644 index 0000000000..3602601287 --- /dev/null +++ b/contrib/libs/llvm12/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp @@ -0,0 +1,781 @@ +//===-------- JITLink_EHFrameSupport.cpp - JITLink eh-frame utils ---------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "EHFrameSupportImpl.h" + +#include "llvm/BinaryFormat/Dwarf.h" +#include "llvm/Config/config.h" +#include "llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h" +#include "llvm/Support/DynamicLibrary.h" + +#define DEBUG_TYPE "jitlink" + +namespace llvm { +namespace jitlink { + +EHFrameSplitter::EHFrameSplitter(StringRef EHFrameSectionName) + : EHFrameSectionName(EHFrameSectionName) {} + +Error EHFrameSplitter::operator()(LinkGraph &G) { + auto *EHFrame = G.findSectionByName(EHFrameSectionName); + + if (!EHFrame) { + LLVM_DEBUG({ + dbgs() << "EHFrameSplitter: No " << EHFrameSectionName + << " section. Nothing to do\n"; + }); + return Error::success(); + } + + LLVM_DEBUG({ + dbgs() << "EHFrameSplitter: Processing " << EHFrameSectionName << "...\n"; + }); + + DenseMap<Block *, LinkGraph::SplitBlockCache> Caches; + + { + // Pre-build the split caches. + for (auto *B : EHFrame->blocks()) + Caches[B] = LinkGraph::SplitBlockCache::value_type(); + for (auto *Sym : EHFrame->symbols()) + Caches[&Sym->getBlock()]->push_back(Sym); + for (auto *B : EHFrame->blocks()) + llvm::sort(*Caches[B], [](const Symbol *LHS, const Symbol *RHS) { + return LHS->getOffset() > RHS->getOffset(); + }); + } + + // Iterate over blocks (we do this by iterating over Caches entries rather + // than EHFrame->blocks() as we will be inserting new blocks along the way, + // which would invalidate iterators in the latter sequence. + for (auto &KV : Caches) { + auto &B = *KV.first; + auto &BCache = KV.second; + if (auto Err = processBlock(G, B, BCache)) + return Err; + } + + return Error::success(); +} + +Error EHFrameSplitter::processBlock(LinkGraph &G, Block &B, + LinkGraph::SplitBlockCache &Cache) { + LLVM_DEBUG({ + dbgs() << " Processing block at " << formatv("{0:x16}", B.getAddress()) + << "\n"; + }); + + // eh-frame should not contain zero-fill blocks. + if (B.isZeroFill()) + return make_error<JITLinkError>("Unexpected zero-fill block in " + + EHFrameSectionName + " section"); + + if (B.getSize() == 0) { + LLVM_DEBUG(dbgs() << " Block is empty. Skipping.\n"); + return Error::success(); + } + + BinaryStreamReader BlockReader(B.getContent(), G.getEndianness()); + + while (true) { + uint64_t RecordStartOffset = BlockReader.getOffset(); + + LLVM_DEBUG({ + dbgs() << " Processing CFI record at " + << formatv("{0:x16}", B.getAddress()) << "\n"; + }); + + uint32_t Length; + if (auto Err = BlockReader.readInteger(Length)) + return Err; + if (Length != 0xffffffff) { + if (auto Err = BlockReader.skip(Length)) + return Err; + } else { + uint64_t ExtendedLength; + if (auto Err = BlockReader.readInteger(ExtendedLength)) + return Err; + if (auto Err = BlockReader.skip(ExtendedLength)) + return Err; + } + + // If this was the last block then there's nothing to split + if (BlockReader.empty()) { + LLVM_DEBUG(dbgs() << " Extracted " << B << "\n"); + return Error::success(); + } + + uint64_t BlockSize = BlockReader.getOffset() - RecordStartOffset; + auto &NewBlock = G.splitBlock(B, BlockSize); + (void)NewBlock; + LLVM_DEBUG(dbgs() << " Extracted " << NewBlock << "\n"); + } +} + +EHFrameEdgeFixer::EHFrameEdgeFixer(StringRef EHFrameSectionName, + unsigned PointerSize, Edge::Kind Delta64, + Edge::Kind Delta32, Edge::Kind NegDelta32) + : EHFrameSectionName(EHFrameSectionName), PointerSize(PointerSize), + Delta64(Delta64), Delta32(Delta32), NegDelta32(NegDelta32) {} + +Error EHFrameEdgeFixer::operator()(LinkGraph &G) { + auto *EHFrame = G.findSectionByName(EHFrameSectionName); + + if (!EHFrame) { + LLVM_DEBUG({ + dbgs() << "EHFrameEdgeFixer: No " << EHFrameSectionName + << " section. Nothing to do\n"; + }); + return Error::success(); + } + + // Check that we support the graph's pointer size. + if (G.getPointerSize() != 4 && G.getPointerSize() != 8) + return make_error<JITLinkError>( + "EHFrameEdgeFixer only supports 32 and 64 bit targets"); + + LLVM_DEBUG({ + dbgs() << "EHFrameEdgeFixer: Processing " << EHFrameSectionName << "...\n"; + }); + + ParseContext PC(G); + + // Build a map of all blocks and symbols in the text sections. We will use + // these for finding / building edge targets when processing FDEs. + for (auto &Sec : G.sections()) { + PC.AddrToSyms.addSymbols(Sec.symbols()); + if (auto Err = PC.AddrToBlock.addBlocks(Sec.blocks(), + BlockAddressMap::includeNonNull)) + return Err; + } + + // Sort eh-frame blocks into address order to ensure we visit CIEs before + // their child FDEs. + std::vector<Block *> EHFrameBlocks; + for (auto *B : EHFrame->blocks()) + EHFrameBlocks.push_back(B); + llvm::sort(EHFrameBlocks, [](const Block *LHS, const Block *RHS) { + return LHS->getAddress() < RHS->getAddress(); + }); + + // Loop over the blocks in address order. + for (auto *B : EHFrameBlocks) + if (auto Err = processBlock(PC, *B)) + return Err; + + return Error::success(); +} + +Error EHFrameEdgeFixer::processBlock(ParseContext &PC, Block &B) { + + LLVM_DEBUG({ + dbgs() << " Processing block at " << formatv("{0:x16}", B.getAddress()) + << "\n"; + }); + + // eh-frame should not contain zero-fill blocks. + if (B.isZeroFill()) + return make_error<JITLinkError>("Unexpected zero-fill block in " + + EHFrameSectionName + " section"); + + if (B.getSize() == 0) { + LLVM_DEBUG(dbgs() << " Block is empty. Skipping.\n"); + return Error::success(); + } + + // Find the offsets of any existing edges from this block. + BlockEdgeMap BlockEdges; + for (auto &E : B.edges()) + if (E.isRelocation()) { + if (BlockEdges.count(E.getOffset())) + return make_error<JITLinkError>( + "Multiple relocations at offset " + + formatv("{0:x16}", E.getOffset()) + " in " + EHFrameSectionName + + " block at address " + formatv("{0:x16}", B.getAddress())); + + BlockEdges[E.getOffset()] = EdgeTarget(E); + } + + CIEInfosMap CIEInfos; + BinaryStreamReader BlockReader(B.getContent(), PC.G.getEndianness()); + while (!BlockReader.empty()) { + size_t RecordStartOffset = BlockReader.getOffset(); + + LLVM_DEBUG({ + dbgs() << " Processing CFI record at " + << formatv("{0:x16}", B.getAddress() + RecordStartOffset) << "\n"; + }); + + // Get the record length. + size_t RecordRemaining; + { + uint32_t Length; + if (auto Err = BlockReader.readInteger(Length)) + return Err; + // If Length < 0xffffffff then use the regular length field, otherwise + // read the extended length field. + if (Length != 0xffffffff) + RecordRemaining = Length; + else { + uint64_t ExtendedLength; + if (auto Err = BlockReader.readInteger(ExtendedLength)) + return Err; + RecordRemaining = ExtendedLength; + } + } + + if (BlockReader.bytesRemaining() < RecordRemaining) + return make_error<JITLinkError>( + "Incomplete CFI record at " + + formatv("{0:x16}", B.getAddress() + RecordStartOffset)); + + // Read the CIE delta for this record. + uint64_t CIEDeltaFieldOffset = BlockReader.getOffset() - RecordStartOffset; + uint32_t CIEDelta; + if (auto Err = BlockReader.readInteger(CIEDelta)) + return Err; + + if (CIEDelta == 0) { + if (auto Err = processCIE(PC, B, RecordStartOffset, + CIEDeltaFieldOffset + RecordRemaining, + CIEDeltaFieldOffset)) + return Err; + } else { + if (auto Err = processFDE(PC, B, RecordStartOffset, + CIEDeltaFieldOffset + RecordRemaining, + CIEDeltaFieldOffset, CIEDelta, BlockEdges)) + return Err; + } + + // Move to the next record. + BlockReader.setOffset(RecordStartOffset + CIEDeltaFieldOffset + + RecordRemaining); + } + + return Error::success(); +} + +Error EHFrameEdgeFixer::processCIE(ParseContext &PC, Block &B, + size_t RecordOffset, size_t RecordLength, + size_t CIEDeltaFieldOffset) { + + LLVM_DEBUG(dbgs() << " Record is CIE\n"); + + auto RecordContent = B.getContent().substr(RecordOffset, RecordLength); + BinaryStreamReader RecordReader(RecordContent, PC.G.getEndianness()); + + // Skip past the CIE delta field: we've already processed this far. + RecordReader.setOffset(CIEDeltaFieldOffset + 4); + + auto &CIESymbol = + PC.G.addAnonymousSymbol(B, RecordOffset, RecordLength, false, false); + CIEInformation CIEInfo(CIESymbol); + + uint8_t Version = 0; + if (auto Err = RecordReader.readInteger(Version)) + return Err; + + if (Version != 0x01) + return make_error<JITLinkError>("Bad CIE version " + Twine(Version) + + " (should be 0x01) in eh-frame"); + + auto AugInfo = parseAugmentationString(RecordReader); + if (!AugInfo) + return AugInfo.takeError(); + + // Skip the EH Data field if present. + if (AugInfo->EHDataFieldPresent) + if (auto Err = RecordReader.skip(PC.G.getPointerSize())) + return Err; + + // Read and sanity check the code alignment factor. + { + uint64_t CodeAlignmentFactor = 0; + if (auto Err = RecordReader.readULEB128(CodeAlignmentFactor)) + return Err; + if (CodeAlignmentFactor != 1) + return make_error<JITLinkError>("Unsupported CIE code alignment factor " + + Twine(CodeAlignmentFactor) + + " (expected 1)"); + } + + // Read and sanity check the data alignment factor. + { + int64_t DataAlignmentFactor = 0; + if (auto Err = RecordReader.readSLEB128(DataAlignmentFactor)) + return Err; + if (DataAlignmentFactor != -8) + return make_error<JITLinkError>("Unsupported CIE data alignment factor " + + Twine(DataAlignmentFactor) + + " (expected -8)"); + } + + // Skip the return address register field. + if (auto Err = RecordReader.skip(1)) + return Err; + + uint64_t AugmentationDataLength = 0; + if (auto Err = RecordReader.readULEB128(AugmentationDataLength)) + return Err; + + uint32_t AugmentationDataStartOffset = RecordReader.getOffset(); + + uint8_t *NextField = &AugInfo->Fields[0]; + while (uint8_t Field = *NextField++) { + switch (Field) { + case 'L': { + CIEInfo.FDEsHaveLSDAField = true; + uint8_t LSDAPointerEncoding; + if (auto Err = RecordReader.readInteger(LSDAPointerEncoding)) + return Err; + if (!isSupportedPointerEncoding(LSDAPointerEncoding)) + return make_error<JITLinkError>( + "Unsupported LSDA pointer encoding " + + formatv("{0:x2}", LSDAPointerEncoding) + " in CIE at " + + formatv("{0:x16}", CIESymbol.getAddress())); + CIEInfo.LSDAPointerEncoding = LSDAPointerEncoding; + break; + } + case 'P': { + uint8_t PersonalityPointerEncoding = 0; + if (auto Err = RecordReader.readInteger(PersonalityPointerEncoding)) + return Err; + if (PersonalityPointerEncoding != + (dwarf::DW_EH_PE_indirect | dwarf::DW_EH_PE_pcrel | + dwarf::DW_EH_PE_sdata4)) + return make_error<JITLinkError>( + "Unspported personality pointer " + "encoding " + + formatv("{0:x2}", PersonalityPointerEncoding) + " in CIE at " + + formatv("{0:x16}", CIESymbol.getAddress())); + uint32_t PersonalityPointerAddress; + if (auto Err = RecordReader.readInteger(PersonalityPointerAddress)) + return Err; + break; + } + case 'R': { + uint8_t FDEPointerEncoding; + if (auto Err = RecordReader.readInteger(FDEPointerEncoding)) + return Err; + if (!isSupportedPointerEncoding(FDEPointerEncoding)) + return make_error<JITLinkError>( + "Unsupported FDE pointer encoding " + + formatv("{0:x2}", FDEPointerEncoding) + " in CIE at " + + formatv("{0:x16}", CIESymbol.getAddress())); + CIEInfo.FDEPointerEncoding = FDEPointerEncoding; + break; + } + default: + llvm_unreachable("Invalid augmentation string field"); + } + } + + if (RecordReader.getOffset() - AugmentationDataStartOffset > + AugmentationDataLength) + return make_error<JITLinkError>("Read past the end of the augmentation " + "data while parsing fields"); + + assert(!PC.CIEInfos.count(CIESymbol.getAddress()) && + "Multiple CIEs recorded at the same address?"); + PC.CIEInfos[CIESymbol.getAddress()] = std::move(CIEInfo); + + return Error::success(); +} + +Error EHFrameEdgeFixer::processFDE(ParseContext &PC, Block &B, + size_t RecordOffset, size_t RecordLength, + size_t CIEDeltaFieldOffset, + uint32_t CIEDelta, + BlockEdgeMap &BlockEdges) { + LLVM_DEBUG(dbgs() << " Record is FDE\n"); + + JITTargetAddress RecordAddress = B.getAddress() + RecordOffset; + + auto RecordContent = B.getContent().substr(RecordOffset, RecordLength); + BinaryStreamReader RecordReader(RecordContent, PC.G.getEndianness()); + + // Skip past the CIE delta field: we've already read this far. + RecordReader.setOffset(CIEDeltaFieldOffset + 4); + + auto &FDESymbol = + PC.G.addAnonymousSymbol(B, RecordOffset, RecordLength, false, false); + + CIEInformation *CIEInfo = nullptr; + + { + // Process the CIE pointer field. + auto CIEEdgeItr = BlockEdges.find(RecordOffset + CIEDeltaFieldOffset); + JITTargetAddress CIEAddress = + RecordAddress + CIEDeltaFieldOffset - CIEDelta; + if (CIEEdgeItr == BlockEdges.end()) { + + LLVM_DEBUG({ + dbgs() << " Adding edge at " + << formatv("{0:x16}", RecordAddress + CIEDeltaFieldOffset) + << " to CIE at: " << formatv("{0:x16}", CIEAddress) << "\n"; + }); + if (auto CIEInfoOrErr = PC.findCIEInfo(CIEAddress)) + CIEInfo = *CIEInfoOrErr; + else + return CIEInfoOrErr.takeError(); + assert(CIEInfo->CIESymbol && "CIEInfo has no CIE symbol set"); + B.addEdge(NegDelta32, RecordOffset + CIEDeltaFieldOffset, + *CIEInfo->CIESymbol, 0); + } else { + LLVM_DEBUG({ + dbgs() << " Already has edge at " + << formatv("{0:x16}", RecordAddress + CIEDeltaFieldOffset) + << " to CIE at " << formatv("{0:x16}", CIEAddress) << "\n"; + }); + auto &EI = CIEEdgeItr->second; + if (EI.Addend) + return make_error<JITLinkError>( + "CIE edge at " + + formatv("{0:x16}", RecordAddress + CIEDeltaFieldOffset) + + " has non-zero addend"); + if (auto CIEInfoOrErr = PC.findCIEInfo(EI.Target->getAddress())) + CIEInfo = *CIEInfoOrErr; + else + return CIEInfoOrErr.takeError(); + } + } + + { + // Process the PC-Begin field. + Block *PCBeginBlock = nullptr; + JITTargetAddress PCBeginFieldOffset = RecordReader.getOffset(); + auto PCEdgeItr = BlockEdges.find(RecordOffset + PCBeginFieldOffset); + if (PCEdgeItr == BlockEdges.end()) { + auto PCBeginPtrInfo = + readEncodedPointer(CIEInfo->FDEPointerEncoding, + RecordAddress + PCBeginFieldOffset, RecordReader); + if (!PCBeginPtrInfo) + return PCBeginPtrInfo.takeError(); + JITTargetAddress PCBegin = PCBeginPtrInfo->first; + Edge::Kind PCBeginEdgeKind = PCBeginPtrInfo->second; + LLVM_DEBUG({ + dbgs() << " Adding edge at " + << formatv("{0:x16}", RecordAddress + PCBeginFieldOffset) + << " to PC at " << formatv("{0:x16}", PCBegin) << "\n"; + }); + auto PCBeginSym = getOrCreateSymbol(PC, PCBegin); + if (!PCBeginSym) + return PCBeginSym.takeError(); + B.addEdge(PCBeginEdgeKind, RecordOffset + PCBeginFieldOffset, *PCBeginSym, + 0); + PCBeginBlock = &PCBeginSym->getBlock(); + } else { + auto &EI = PCEdgeItr->second; + LLVM_DEBUG({ + dbgs() << " Already has edge at " + << formatv("{0:x16}", RecordAddress + PCBeginFieldOffset) + << " to PC at " << formatv("{0:x16}", EI.Target->getAddress()); + if (EI.Addend) + dbgs() << " + " << formatv("{0:x16}", EI.Addend); + dbgs() << "\n"; + }); + + // Make sure the existing edge points at a defined block. + if (!EI.Target->isDefined()) { + auto EdgeAddr = RecordAddress + PCBeginFieldOffset; + return make_error<JITLinkError>("FDE edge at " + + formatv("{0:x16}", EdgeAddr) + + " points at external block"); + } + PCBeginBlock = &EI.Target->getBlock(); + if (auto Err = RecordReader.skip( + getPointerEncodingDataSize(CIEInfo->FDEPointerEncoding))) + return Err; + } + + // Add a keep-alive edge from the FDE target to the FDE to ensure that the + // FDE is kept alive if its target is. + assert(PCBeginBlock && "PC-begin block not recorded"); + LLVM_DEBUG({ + dbgs() << " Adding keep-alive edge from target at " + << formatv("{0:x16}", PCBeginBlock->getAddress()) << " to FDE at " + << formatv("{0:x16}", RecordAddress) << "\n"; + }); + PCBeginBlock->addEdge(Edge::KeepAlive, 0, FDESymbol, 0); + } + + // Skip over the PC range size field. + if (auto Err = RecordReader.skip( + getPointerEncodingDataSize(CIEInfo->FDEPointerEncoding))) + return Err; + + if (CIEInfo->FDEsHaveLSDAField) { + uint64_t AugmentationDataSize; + if (auto Err = RecordReader.readULEB128(AugmentationDataSize)) + return Err; + + JITTargetAddress LSDAFieldOffset = RecordReader.getOffset(); + auto LSDAEdgeItr = BlockEdges.find(RecordOffset + LSDAFieldOffset); + if (LSDAEdgeItr == BlockEdges.end()) { + auto LSDAPointerInfo = + readEncodedPointer(CIEInfo->LSDAPointerEncoding, + RecordAddress + LSDAFieldOffset, RecordReader); + if (!LSDAPointerInfo) + return LSDAPointerInfo.takeError(); + JITTargetAddress LSDA = LSDAPointerInfo->first; + Edge::Kind LSDAEdgeKind = LSDAPointerInfo->second; + auto LSDASym = getOrCreateSymbol(PC, LSDA); + if (!LSDASym) + return LSDASym.takeError(); + LLVM_DEBUG({ + dbgs() << " Adding edge at " + << formatv("{0:x16}", RecordAddress + LSDAFieldOffset) + << " to LSDA at " << formatv("{0:x16}", LSDA) << "\n"; + }); + B.addEdge(LSDAEdgeKind, RecordOffset + LSDAFieldOffset, *LSDASym, 0); + } else { + LLVM_DEBUG({ + auto &EI = LSDAEdgeItr->second; + dbgs() << " Already has edge at " + << formatv("{0:x16}", RecordAddress + LSDAFieldOffset) + << " to LSDA at " << formatv("{0:x16}", EI.Target->getAddress()); + if (EI.Addend) + dbgs() << " + " << formatv("{0:x16}", EI.Addend); + dbgs() << "\n"; + }); + if (auto Err = RecordReader.skip(AugmentationDataSize)) + return Err; + } + } else { + LLVM_DEBUG(dbgs() << " Record does not have LSDA field.\n"); + } + + return Error::success(); +} + +Expected<EHFrameEdgeFixer::AugmentationInfo> +EHFrameEdgeFixer::parseAugmentationString(BinaryStreamReader &RecordReader) { + AugmentationInfo AugInfo; + uint8_t NextChar; + uint8_t *NextField = &AugInfo.Fields[0]; + + if (auto Err = RecordReader.readInteger(NextChar)) + return std::move(Err); + + while (NextChar != 0) { + switch (NextChar) { + case 'z': + AugInfo.AugmentationDataPresent = true; + break; + case 'e': + if (auto Err = RecordReader.readInteger(NextChar)) + return std::move(Err); + if (NextChar != 'h') + return make_error<JITLinkError>("Unrecognized substring e" + + Twine(NextChar) + + " in augmentation string"); + AugInfo.EHDataFieldPresent = true; + break; + case 'L': + case 'P': + case 'R': + *NextField++ = NextChar; + break; + default: + return make_error<JITLinkError>("Unrecognized character " + + Twine(NextChar) + + " in augmentation string"); + } + + if (auto Err = RecordReader.readInteger(NextChar)) + return std::move(Err); + } + + return std::move(AugInfo); +} + +bool EHFrameEdgeFixer::isSupportedPointerEncoding(uint8_t PointerEncoding) { + using namespace dwarf; + + // We only support PC-rel for now. + if ((PointerEncoding & 0x70) != DW_EH_PE_pcrel) + return false; + + // readEncodedPointer does not handle indirect. + if (PointerEncoding & DW_EH_PE_indirect) + return false; + + // Supported datatypes. + switch (PointerEncoding & 0xf) { + case DW_EH_PE_absptr: + case DW_EH_PE_udata4: + case DW_EH_PE_udata8: + case DW_EH_PE_sdata4: + case DW_EH_PE_sdata8: + return true; + } + + return false; +} + +unsigned EHFrameEdgeFixer::getPointerEncodingDataSize(uint8_t PointerEncoding) { + using namespace dwarf; + + assert(isSupportedPointerEncoding(PointerEncoding) && + "Unsupported pointer encoding"); + switch (PointerEncoding & 0xf) { + case DW_EH_PE_absptr: + return PointerSize; + case DW_EH_PE_udata4: + case DW_EH_PE_sdata4: + return 4; + case DW_EH_PE_udata8: + case DW_EH_PE_sdata8: + return 8; + default: + llvm_unreachable("Unsupported encoding"); + } +} + +Expected<std::pair<JITTargetAddress, Edge::Kind>> +EHFrameEdgeFixer::readEncodedPointer(uint8_t PointerEncoding, + JITTargetAddress PointerFieldAddress, + BinaryStreamReader &RecordReader) { + static_assert(sizeof(JITTargetAddress) == sizeof(uint64_t), + "Result must be able to hold a uint64_t"); + assert(isSupportedPointerEncoding(PointerEncoding) && + "Unsupported pointer encoding"); + + using namespace dwarf; + + // Isolate data type, remap absptr to udata4 or udata8. This relies on us + // having verified that the graph uses 32-bit or 64-bit pointers only at the + // start of this pass. + uint8_t EffectiveType = PointerEncoding & 0xf; + if (EffectiveType == DW_EH_PE_absptr) + EffectiveType = (PointerSize == 8) ? DW_EH_PE_udata8 : DW_EH_PE_udata4; + + JITTargetAddress Addr; + Edge::Kind PointerEdgeKind; + switch (EffectiveType) { + case DW_EH_PE_udata4: { + uint32_t Val; + if (auto Err = RecordReader.readInteger(Val)) + return std::move(Err); + Addr = PointerFieldAddress + Val; + PointerEdgeKind = Delta32; + break; + } + case DW_EH_PE_udata8: { + uint64_t Val; + if (auto Err = RecordReader.readInteger(Val)) + return std::move(Err); + Addr = PointerFieldAddress + Val; + PointerEdgeKind = Delta64; + break; + } + case DW_EH_PE_sdata4: { + int32_t Val; + if (auto Err = RecordReader.readInteger(Val)) + return std::move(Err); + Addr = PointerFieldAddress + Val; + PointerEdgeKind = Delta32; + break; + } + case DW_EH_PE_sdata8: { + int64_t Val; + if (auto Err = RecordReader.readInteger(Val)) + return std::move(Err); + Addr = PointerFieldAddress + Val; + PointerEdgeKind = Delta64; + break; + } + } + + if (PointerEdgeKind == Edge::Invalid) + return make_error<JITLinkError>( + "Unspported edge kind for encoded pointer at " + + formatv("{0:x}", PointerFieldAddress)); + + return std::make_pair(Addr, Delta64); +} + +Expected<Symbol &> EHFrameEdgeFixer::getOrCreateSymbol(ParseContext &PC, + JITTargetAddress Addr) { + Symbol *CanonicalSym = nullptr; + + auto UpdateCanonicalSym = [&](Symbol *Sym) { + if (!CanonicalSym || Sym->getLinkage() < CanonicalSym->getLinkage() || + Sym->getScope() < CanonicalSym->getScope() || + (Sym->hasName() && !CanonicalSym->hasName()) || + Sym->getName() < CanonicalSym->getName()) + CanonicalSym = Sym; + }; + + if (auto *SymbolsAtAddr = PC.AddrToSyms.getSymbolsAt(Addr)) + for (auto *Sym : *SymbolsAtAddr) + UpdateCanonicalSym(Sym); + + // If we found an existing symbol at the given address then use it. + if (CanonicalSym) + return *CanonicalSym; + + // Otherwise search for a block covering the address and create a new symbol. + auto *B = PC.AddrToBlock.getBlockCovering(Addr); + if (!B) + return make_error<JITLinkError>("No symbol or block covering address " + + formatv("{0:x16}", Addr)); + + return PC.G.addAnonymousSymbol(*B, Addr - B->getAddress(), 0, false, false); +} + +EHFrameRegistrar::~EHFrameRegistrar() {} + +Error InProcessEHFrameRegistrar::registerEHFrames( + JITTargetAddress EHFrameSectionAddr, size_t EHFrameSectionSize) { + return orc::registerEHFrameSection( + jitTargetAddressToPointer<void *>(EHFrameSectionAddr), + EHFrameSectionSize); +} + +Error InProcessEHFrameRegistrar::deregisterEHFrames( + JITTargetAddress EHFrameSectionAddr, size_t EHFrameSectionSize) { + return orc::deregisterEHFrameSection( + jitTargetAddressToPointer<void *>(EHFrameSectionAddr), + EHFrameSectionSize); +} + +LinkGraphPassFunction +createEHFrameRecorderPass(const Triple &TT, + StoreFrameRangeFunction StoreRangeAddress) { + const char *EHFrameSectionName = nullptr; + if (TT.getObjectFormat() == Triple::MachO) + EHFrameSectionName = "__eh_frame"; + else + EHFrameSectionName = ".eh_frame"; + + auto RecordEHFrame = + [EHFrameSectionName, + StoreFrameRange = std::move(StoreRangeAddress)](LinkGraph &G) -> Error { + // Search for a non-empty eh-frame and record the address of the first + // symbol in it. + JITTargetAddress Addr = 0; + size_t Size = 0; + if (auto *S = G.findSectionByName(EHFrameSectionName)) { + auto R = SectionRange(*S); + Addr = R.getStart(); + Size = R.getSize(); + } + if (Addr == 0 && Size != 0) + return make_error<JITLinkError>("__eh_frame section can not have zero " + "address with non-zero size"); + StoreFrameRange(Addr, Size); + return Error::success(); + }; + + return RecordEHFrame; +} + +} // end namespace jitlink +} // end namespace llvm diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h b/contrib/libs/llvm12/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h new file mode 100644 index 0000000000..5e68e72ba1 --- /dev/null +++ b/contrib/libs/llvm12/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h @@ -0,0 +1,122 @@ +//===------- EHFrameSupportImpl.h - JITLink eh-frame utils ------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// EHFrame registration support for JITLink. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_EXECUTIONENGINE_JITLINK_EHFRAMESUPPORTIMPL_H +#define LLVM_LIB_EXECUTIONENGINE_JITLINK_EHFRAMESUPPORTIMPL_H + +#include "llvm/ExecutionEngine/JITLink/EHFrameSupport.h" + +#include "llvm/ExecutionEngine/JITLink/JITLink.h" +#include "llvm/Support/BinaryStreamReader.h" + +namespace llvm { +namespace jitlink { + +/// A LinkGraph pass that splits blocks in an eh-frame section into sub-blocks +/// representing individual eh-frames. +/// EHFrameSplitter should not be run without EHFrameEdgeFixer, which is +/// responsible for adding FDE-to-CIE edges. +class EHFrameSplitter { +public: + EHFrameSplitter(StringRef EHFrameSectionName); + Error operator()(LinkGraph &G); + +private: + Error processBlock(LinkGraph &G, Block &B, LinkGraph::SplitBlockCache &Cache); + + StringRef EHFrameSectionName; +}; + +/// A LinkGraph pass that adds missing FDE-to-CIE, FDE-to-PC and FDE-to-LSDA +/// edges. +class EHFrameEdgeFixer { +public: + EHFrameEdgeFixer(StringRef EHFrameSectionName, unsigned PointerSize, + Edge::Kind Delta64, Edge::Kind Delta32, + Edge::Kind NegDelta32); + Error operator()(LinkGraph &G); + +private: + + struct AugmentationInfo { + bool AugmentationDataPresent = false; + bool EHDataFieldPresent = false; + uint8_t Fields[4] = {0x0, 0x0, 0x0, 0x0}; + }; + + struct CIEInformation { + CIEInformation() = default; + CIEInformation(Symbol &CIESymbol) : CIESymbol(&CIESymbol) {} + Symbol *CIESymbol = nullptr; + bool FDEsHaveLSDAField = false; + uint8_t FDEPointerEncoding = 0; + uint8_t LSDAPointerEncoding = 0; + }; + + struct EdgeTarget { + EdgeTarget() = default; + EdgeTarget(const Edge &E) : Target(&E.getTarget()), Addend(E.getAddend()) {} + + Symbol *Target = nullptr; + Edge::AddendT Addend = 0; + }; + + using BlockEdgeMap = DenseMap<Edge::OffsetT, EdgeTarget>; + using CIEInfosMap = DenseMap<JITTargetAddress, CIEInformation>; + + struct ParseContext { + ParseContext(LinkGraph &G) : G(G) {} + + Expected<CIEInformation *> findCIEInfo(JITTargetAddress Address) { + auto I = CIEInfos.find(Address); + if (I == CIEInfos.end()) + return make_error<JITLinkError>("No CIE found at address " + + formatv("{0:x16}", Address)); + return &I->second; + } + + LinkGraph &G; + CIEInfosMap CIEInfos; + BlockAddressMap AddrToBlock; + SymbolAddressMap AddrToSyms; + }; + + Error processBlock(ParseContext &PC, Block &B); + Error processCIE(ParseContext &PC, Block &B, size_t RecordOffset, + size_t RecordLength, size_t CIEDeltaFieldOffset); + Error processFDE(ParseContext &PC, Block &B, size_t RecordOffset, + size_t RecordLength, size_t CIEDeltaFieldOffset, + uint32_t CIEDelta, BlockEdgeMap &BlockEdges); + + Expected<AugmentationInfo> + parseAugmentationString(BinaryStreamReader &RecordReader); + + static bool isSupportedPointerEncoding(uint8_t PointerEncoding); + unsigned getPointerEncodingDataSize(uint8_t PointerEncoding); + Expected<std::pair<JITTargetAddress, Edge::Kind>> + readEncodedPointer(uint8_t PointerEncoding, + JITTargetAddress PointerFieldAddress, + BinaryStreamReader &RecordReader); + + Expected<Symbol &> getOrCreateSymbol(ParseContext &PC, JITTargetAddress Addr); + + StringRef EHFrameSectionName; + unsigned PointerSize; + Edge::Kind Delta64; + Edge::Kind Delta32; + Edge::Kind NegDelta32; +}; + +} // end namespace jitlink +} // end namespace llvm + +#endif // LLVM_LIB_EXECUTIONENGINE_JITLINK_EHFRAMESUPPORTIMPL_H diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/JITLink/ELF.cpp b/contrib/libs/llvm12/lib/ExecutionEngine/JITLink/ELF.cpp new file mode 100644 index 0000000000..27eb7d576e --- /dev/null +++ b/contrib/libs/llvm12/lib/ExecutionEngine/JITLink/ELF.cpp @@ -0,0 +1,91 @@ +//===-------------- ELF.cpp - JIT linker function for ELF -------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// ELF jit-link function. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/JITLink/ELF.h" + +#include "llvm/BinaryFormat/ELF.h" +#include "llvm/ExecutionEngine/JITLink/ELF_x86_64.h" +#include "llvm/Object/ELF.h" +#include "llvm/Support/Endian.h" +#include "llvm/Support/Format.h" +#include "llvm/Support/MemoryBuffer.h" +#include <cstring> + +using namespace llvm; + +#define DEBUG_TYPE "jitlink" + +namespace llvm { +namespace jitlink { + +Expected<uint16_t> readTargetMachineArch(StringRef Buffer) { + const char *Data = Buffer.data(); + + if (Data[ELF::EI_DATA] == ELF::ELFDATA2LSB) { + if (Data[ELF::EI_CLASS] == ELF::ELFCLASS64) { + if (auto File = llvm::object::ELF64LEFile::create(Buffer)) { + return File->getHeader().e_machine; + } else { + return File.takeError(); + } + } else if (Data[ELF::EI_CLASS] == ELF::ELFCLASS32) { + if (auto File = llvm::object::ELF32LEFile::create(Buffer)) { + return File->getHeader().e_machine; + } else { + return File.takeError(); + } + } + } + + return ELF::EM_NONE; +} + +Expected<std::unique_ptr<LinkGraph>> +createLinkGraphFromELFObject(MemoryBufferRef ObjectBuffer) { + StringRef Buffer = ObjectBuffer.getBuffer(); + if (Buffer.size() < ELF::EI_MAG3 + 1) + return make_error<JITLinkError>("Truncated ELF buffer"); + + if (memcmp(Buffer.data(), ELF::ElfMagic, strlen(ELF::ElfMagic)) != 0) + return make_error<JITLinkError>("ELF magic not valid"); + + Expected<uint16_t> TargetMachineArch = readTargetMachineArch(Buffer); + if (!TargetMachineArch) + return TargetMachineArch.takeError(); + + switch (*TargetMachineArch) { + case ELF::EM_X86_64: + return createLinkGraphFromELFObject_x86_64(std::move(ObjectBuffer)); + default: + return make_error<JITLinkError>( + "Unsupported target machine architecture in ELF object " + + ObjectBuffer.getBufferIdentifier()); + } +} + +void link_ELF(std::unique_ptr<LinkGraph> G, + std::unique_ptr<JITLinkContext> Ctx) { + switch (G->getTargetTriple().getArch()) { + case Triple::x86_64: + link_ELF_x86_64(std::move(G), std::move(Ctx)); + return; + default: + Ctx->notifyFailed(make_error<JITLinkError>( + "Unsupported target machine architecture in ELF link graph " + + G->getName())); + return; + } +} + +} // end namespace jitlink +} // end namespace llvm diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp b/contrib/libs/llvm12/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp new file mode 100644 index 0000000000..2a6b3eb19d --- /dev/null +++ b/contrib/libs/llvm12/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp @@ -0,0 +1,813 @@ +//===---- ELF_x86_64.cpp -JIT linker implementation for ELF/x86-64 ----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// ELF/x86-64 jit-link implementation. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/JITLink/ELF_x86_64.h" +#include "llvm/ExecutionEngine/JITLink/JITLink.h" +#include "llvm/Object/ELFObjectFile.h" +#include "llvm/Support/Endian.h" + +#include "BasicGOTAndStubsBuilder.h" +#include "EHFrameSupportImpl.h" +#include "JITLinkGeneric.h" + +#define DEBUG_TYPE "jitlink" + +using namespace llvm; +using namespace llvm::jitlink; +using namespace llvm::jitlink::ELF_x86_64_Edges; + +namespace { + +class ELF_x86_64_GOTAndStubsBuilder + : public BasicGOTAndStubsBuilder<ELF_x86_64_GOTAndStubsBuilder> { +public: + static const uint8_t NullGOTEntryContent[8]; + static const uint8_t StubContent[6]; + + ELF_x86_64_GOTAndStubsBuilder(LinkGraph &G) + : BasicGOTAndStubsBuilder<ELF_x86_64_GOTAndStubsBuilder>(G) {} + + bool isGOTEdge(Edge &E) const { + return E.getKind() == PCRel32GOT || E.getKind() == PCRel32GOTLoad; + } + + Symbol &createGOTEntry(Symbol &Target) { + auto &GOTEntryBlock = G.createContentBlock( + getGOTSection(), getGOTEntryBlockContent(), 0, 8, 0); + GOTEntryBlock.addEdge(Pointer64, 0, Target, 0); + return G.addAnonymousSymbol(GOTEntryBlock, 0, 8, false, false); + } + + void fixGOTEdge(Edge &E, Symbol &GOTEntry) { + assert((E.getKind() == PCRel32GOT || E.getKind() == PCRel32GOTLoad) && + "Not a GOT edge?"); + // If this is a PCRel32GOT then change it to an ordinary PCRel32. If it is + // a PCRel32GOTLoad then leave it as-is for now. We will use the kind to + // check for GOT optimization opportunities in the + // optimizeMachO_x86_64_GOTAndStubs pass below. + if (E.getKind() == PCRel32GOT) + E.setKind(PCRel32); + + E.setTarget(GOTEntry); + // Leave the edge addend as-is. + } + + bool isExternalBranchEdge(Edge &E) { + return E.getKind() == Branch32 && !E.getTarget().isDefined(); + } + + Symbol &createStub(Symbol &Target) { + auto &StubContentBlock = + G.createContentBlock(getStubsSection(), getStubBlockContent(), 0, 1, 0); + // Re-use GOT entries for stub targets. + auto &GOTEntrySymbol = getGOTEntrySymbol(Target); + StubContentBlock.addEdge(PCRel32, 2, GOTEntrySymbol, -4); + return G.addAnonymousSymbol(StubContentBlock, 0, 6, true, false); + } + + void fixExternalBranchEdge(Edge &E, Symbol &Stub) { + assert(E.getKind() == Branch32 && "Not a Branch32 edge?"); + + // Set the edge kind to Branch32ToStub. We will use this to check for stub + // optimization opportunities in the optimize ELF_x86_64_GOTAndStubs pass + // below. + E.setKind(Branch32ToStub); + E.setTarget(Stub); + } + +private: + Section &getGOTSection() { + if (!GOTSection) + GOTSection = &G.createSection("$__GOT", sys::Memory::MF_READ); + return *GOTSection; + } + + Section &getStubsSection() { + if (!StubsSection) { + auto StubsProt = static_cast<sys::Memory::ProtectionFlags>( + sys::Memory::MF_READ | sys::Memory::MF_EXEC); + StubsSection = &G.createSection("$__STUBS", StubsProt); + } + return *StubsSection; + } + + StringRef getGOTEntryBlockContent() { + return StringRef(reinterpret_cast<const char *>(NullGOTEntryContent), + sizeof(NullGOTEntryContent)); + } + + StringRef getStubBlockContent() { + return StringRef(reinterpret_cast<const char *>(StubContent), + sizeof(StubContent)); + } + + Section *GOTSection = nullptr; + Section *StubsSection = nullptr; +}; + +const char *const DwarfSectionNames[] = { +#define HANDLE_DWARF_SECTION(ENUM_NAME, ELF_NAME, CMDLINE_NAME, OPTION) \ + ELF_NAME, +#include "llvm/BinaryFormat/Dwarf.def" +#undef HANDLE_DWARF_SECTION +}; + +} // namespace + +const uint8_t ELF_x86_64_GOTAndStubsBuilder::NullGOTEntryContent[8] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; +const uint8_t ELF_x86_64_GOTAndStubsBuilder::StubContent[6] = { + 0xFF, 0x25, 0x00, 0x00, 0x00, 0x00}; + +static const char *CommonSectionName = "__common"; +static Error optimizeELF_x86_64_GOTAndStubs(LinkGraph &G) { + LLVM_DEBUG(dbgs() << "Optimizing GOT entries and stubs:\n"); + + for (auto *B : G.blocks()) + for (auto &E : B->edges()) + if (E.getKind() == PCRel32GOTLoad) { + // Replace GOT load with LEA only for MOVQ instructions. + constexpr uint8_t MOVQRIPRel[] = {0x48, 0x8b}; + if (E.getOffset() < 3 || + strncmp(B->getContent().data() + E.getOffset() - 3, + reinterpret_cast<const char *>(MOVQRIPRel), 2) != 0) + continue; + + auto &GOTBlock = E.getTarget().getBlock(); + assert(GOTBlock.getSize() == G.getPointerSize() && + "GOT entry block should be pointer sized"); + assert(GOTBlock.edges_size() == 1 && + "GOT entry should only have one outgoing edge"); + + auto &GOTTarget = GOTBlock.edges().begin()->getTarget(); + JITTargetAddress EdgeAddr = B->getAddress() + E.getOffset(); + JITTargetAddress TargetAddr = GOTTarget.getAddress(); + + int64_t Displacement = TargetAddr - EdgeAddr + 4; + if (Displacement >= std::numeric_limits<int32_t>::min() && + Displacement <= std::numeric_limits<int32_t>::max()) { + // Change the edge kind as we don't go through GOT anymore. This is + // for formal correctness only. Technically, the two relocation kinds + // are resolved the same way. + E.setKind(PCRel32); + E.setTarget(GOTTarget); + auto *BlockData = reinterpret_cast<uint8_t *>( + const_cast<char *>(B->getContent().data())); + BlockData[E.getOffset() - 2] = 0x8d; + LLVM_DEBUG({ + dbgs() << " Replaced GOT load wih LEA:\n "; + printEdge(dbgs(), *B, E, getELFX86RelocationKindName(E.getKind())); + dbgs() << "\n"; + }); + } + } else if (E.getKind() == Branch32ToStub) { + auto &StubBlock = E.getTarget().getBlock(); + assert(StubBlock.getSize() == + sizeof(ELF_x86_64_GOTAndStubsBuilder::StubContent) && + "Stub block should be stub sized"); + assert(StubBlock.edges_size() == 1 && + "Stub block should only have one outgoing edge"); + + auto &GOTBlock = StubBlock.edges().begin()->getTarget().getBlock(); + assert(GOTBlock.getSize() == G.getPointerSize() && + "GOT block should be pointer sized"); + assert(GOTBlock.edges_size() == 1 && + "GOT block should only have one outgoing edge"); + + auto &GOTTarget = GOTBlock.edges().begin()->getTarget(); + JITTargetAddress EdgeAddr = B->getAddress() + E.getOffset(); + JITTargetAddress TargetAddr = GOTTarget.getAddress(); + + int64_t Displacement = TargetAddr - EdgeAddr + 4; + if (Displacement >= std::numeric_limits<int32_t>::min() && + Displacement <= std::numeric_limits<int32_t>::max()) { + E.setKind(Branch32); + E.setTarget(GOTTarget); + LLVM_DEBUG({ + dbgs() << " Replaced stub branch with direct branch:\n "; + printEdge(dbgs(), *B, E, getELFX86RelocationKindName(E.getKind())); + dbgs() << "\n"; + }); + } + } + + return Error::success(); +} + +static bool isDwarfSection(StringRef SectionName) { + for (auto &DwarfSectionName : DwarfSectionNames) + if (SectionName == DwarfSectionName) + return true; + return false; +} + +namespace llvm { +namespace jitlink { + +// This should become a template as the ELFFile is so a lot of this could become +// generic +class ELFLinkGraphBuilder_x86_64 { + +private: + Section *CommonSection = nullptr; + // TODO hack to get this working + // Find a better way + using SymbolTable = object::ELFFile<object::ELF64LE>::Elf_Shdr; + // For now we just assume + using SymbolMap = std::map<int32_t, Symbol *>; + SymbolMap JITSymbolTable; + + Section &getCommonSection() { + if (!CommonSection) { + auto Prot = static_cast<sys::Memory::ProtectionFlags>( + sys::Memory::MF_READ | sys::Memory::MF_WRITE); + CommonSection = &G->createSection(CommonSectionName, Prot); + } + return *CommonSection; + } + + static Expected<ELF_x86_64_Edges::ELFX86RelocationKind> + getRelocationKind(const uint32_t Type) { + switch (Type) { + case ELF::R_X86_64_PC32: + return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32; + case ELF::R_X86_64_PC64: + return ELF_x86_64_Edges::ELFX86RelocationKind::Delta64; + case ELF::R_X86_64_64: + return ELF_x86_64_Edges::ELFX86RelocationKind::Pointer64; + case ELF::R_X86_64_GOTPCREL: + case ELF::R_X86_64_GOTPCRELX: + case ELF::R_X86_64_REX_GOTPCRELX: + return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32GOTLoad; + case ELF::R_X86_64_PLT32: + return ELF_x86_64_Edges::ELFX86RelocationKind::Branch32; + } + return make_error<JITLinkError>("Unsupported x86-64 relocation:" + + formatv("{0:d}", Type)); + } + + std::unique_ptr<LinkGraph> G; + // This could be a template + const object::ELFFile<object::ELF64LE> &Obj; + object::ELFFile<object::ELF64LE>::Elf_Shdr_Range sections; + SymbolTable SymTab; + + bool isRelocatable() { return Obj.getHeader().e_type == llvm::ELF::ET_REL; } + + support::endianness + getEndianness(const object::ELFFile<object::ELF64LE> &Obj) { + return Obj.isLE() ? support::little : support::big; + } + + // This could also just become part of a template + unsigned getPointerSize(const object::ELFFile<object::ELF64LE> &Obj) { + return Obj.getHeader().getFileClass() == ELF::ELFCLASS64 ? 8 : 4; + } + + // We don't technically need this right now + // But for now going to keep it as it helps me to debug things + + Error createNormalizedSymbols() { + LLVM_DEBUG(dbgs() << "Creating normalized symbols...\n"); + + for (auto SecRef : sections) { + if (SecRef.sh_type != ELF::SHT_SYMTAB && + SecRef.sh_type != ELF::SHT_DYNSYM) + continue; + + auto Symbols = Obj.symbols(&SecRef); + // TODO: Currently I use this function to test things + // I also want to leave it to see if its common between MACH and elf + // so for now I just want to continue even if there is an error + if (errorToBool(Symbols.takeError())) + continue; + + auto StrTabSec = Obj.getSection(SecRef.sh_link); + if (!StrTabSec) + return StrTabSec.takeError(); + auto StringTable = Obj.getStringTable(**StrTabSec); + if (!StringTable) + return StringTable.takeError(); + + for (auto SymRef : *Symbols) { + Optional<StringRef> Name; + + if (auto NameOrErr = SymRef.getName(*StringTable)) + Name = *NameOrErr; + else + return NameOrErr.takeError(); + + LLVM_DEBUG({ + dbgs() << " value = " << formatv("{0:x16}", SymRef.getValue()) + << ", type = " << formatv("{0:x2}", SymRef.getType()) + << ", binding = " << formatv("{0:x2}", SymRef.getBinding()) + << ", size = " + << formatv("{0:x16}", static_cast<uint64_t>(SymRef.st_size)) + << ", info = " << formatv("{0:x2}", SymRef.st_info) + << " :" << (Name ? *Name : "<anonymous symbol>") << "\n"; + }); + } + } + return Error::success(); + } + + Error createNormalizedSections() { + LLVM_DEBUG(dbgs() << "Creating normalized sections...\n"); + for (auto &SecRef : sections) { + auto Name = Obj.getSectionName(SecRef); + if (!Name) + return Name.takeError(); + + // Skip Dwarf sections. + if (isDwarfSection(*Name)) { + LLVM_DEBUG({ + dbgs() << *Name + << " is a debug section: No graph section will be created.\n"; + }); + continue; + } + + sys::Memory::ProtectionFlags Prot; + if (SecRef.sh_flags & ELF::SHF_EXECINSTR) { + Prot = static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | + sys::Memory::MF_EXEC); + } else { + Prot = static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | + sys::Memory::MF_WRITE); + } + uint64_t Address = SecRef.sh_addr; + uint64_t Size = SecRef.sh_size; + uint64_t Flags = SecRef.sh_flags; + uint64_t Alignment = SecRef.sh_addralign; + const char *Data = nullptr; + // for now we just use this to skip the "undefined" section, probably need + // to revist + if (Size == 0) + continue; + + // FIXME: Use flags. + (void)Flags; + + LLVM_DEBUG({ + dbgs() << " " << *Name << ": " << formatv("{0:x16}", Address) << " -- " + << formatv("{0:x16}", Address + Size) << ", align: " << Alignment + << " Flags: " << formatv("{0:x}", Flags) << "\n"; + }); + + if (SecRef.sh_type != ELF::SHT_NOBITS) { + // .sections() already checks that the data is not beyond the end of + // file + auto contents = Obj.getSectionContentsAsArray<char>(SecRef); + if (!contents) + return contents.takeError(); + + Data = contents->data(); + // TODO protection flags. + // for now everything is + auto §ion = G->createSection(*Name, Prot); + // Do this here because we have it, but move it into graphify later + G->createContentBlock(section, StringRef(Data, Size), Address, + Alignment, 0); + if (SecRef.sh_type == ELF::SHT_SYMTAB) + // TODO: Dynamic? + SymTab = SecRef; + } else { + auto &Section = G->createSection(*Name, Prot); + G->createZeroFillBlock(Section, Size, Address, Alignment, 0); + } + } + + return Error::success(); + } + + Error addRelocations() { + LLVM_DEBUG(dbgs() << "Adding relocations\n"); + // TODO a partern is forming of iterate some sections but only give me + // ones I am interested, i should abstract that concept some where + for (auto &SecRef : sections) { + if (SecRef.sh_type != ELF::SHT_RELA && SecRef.sh_type != ELF::SHT_REL) + continue; + // TODO can the elf obj file do this for me? + if (SecRef.sh_type == ELF::SHT_REL) + return make_error<llvm::StringError>("Shouldn't have REL in x64", + llvm::inconvertibleErrorCode()); + + auto RelSectName = Obj.getSectionName(SecRef); + if (!RelSectName) + return RelSectName.takeError(); + + LLVM_DEBUG({ + dbgs() << "Adding relocations from section " << *RelSectName << "\n"; + }); + + auto UpdateSection = Obj.getSection(SecRef.sh_info); + if (!UpdateSection) + return UpdateSection.takeError(); + + auto UpdateSectionName = Obj.getSectionName(**UpdateSection); + if (!UpdateSectionName) + return UpdateSectionName.takeError(); + + // Don't process relocations for debug sections. + if (isDwarfSection(*UpdateSectionName)) { + LLVM_DEBUG({ + dbgs() << " Target is dwarf section " << *UpdateSectionName + << ". Skipping.\n"; + }); + continue; + } else + LLVM_DEBUG({ + dbgs() << " For target section " << *UpdateSectionName << "\n"; + }); + + auto JITSection = G->findSectionByName(*UpdateSectionName); + if (!JITSection) + return make_error<llvm::StringError>( + "Refencing a a section that wasn't added to graph" + + *UpdateSectionName, + llvm::inconvertibleErrorCode()); + + auto Relocations = Obj.relas(SecRef); + if (!Relocations) + return Relocations.takeError(); + + for (const auto &Rela : *Relocations) { + auto Type = Rela.getType(false); + + LLVM_DEBUG({ + dbgs() << "Relocation Type: " << Type << "\n" + << "Name: " << Obj.getRelocationTypeName(Type) << "\n"; + }); + auto SymbolIndex = Rela.getSymbol(false); + auto Symbol = Obj.getRelocationSymbol(Rela, &SymTab); + if (!Symbol) + return Symbol.takeError(); + + auto BlockToFix = *(JITSection->blocks().begin()); + auto *TargetSymbol = JITSymbolTable[SymbolIndex]; + + if (!TargetSymbol) { + return make_error<llvm::StringError>( + "Could not find symbol at given index, did you add it to " + "JITSymbolTable? index: " + std::to_string(SymbolIndex) + + ", shndx: " + std::to_string((*Symbol)->st_shndx) + + " Size of table: " + std::to_string(JITSymbolTable.size()), + llvm::inconvertibleErrorCode()); + } + uint64_t Addend = Rela.r_addend; + JITTargetAddress FixupAddress = + (*UpdateSection)->sh_addr + Rela.r_offset; + + LLVM_DEBUG({ + dbgs() << "Processing relocation at " + << format("0x%016" PRIx64, FixupAddress) << "\n"; + }); + auto Kind = getRelocationKind(Type); + if (!Kind) + return Kind.takeError(); + + LLVM_DEBUG({ + Edge GE(*Kind, FixupAddress - BlockToFix->getAddress(), *TargetSymbol, + Addend); + printEdge(dbgs(), *BlockToFix, GE, + getELFX86RelocationKindName(*Kind)); + dbgs() << "\n"; + }); + BlockToFix->addEdge(*Kind, FixupAddress - BlockToFix->getAddress(), + *TargetSymbol, Addend); + } + } + return Error::success(); + } + + Error graphifyRegularSymbols() { + + // TODO: ELF supports beyond SHN_LORESERVE, + // need to perf test how a vector vs map handles those cases + + std::vector<std::vector<object::ELFFile<object::ELF64LE>::Elf_Shdr_Range *>> + SecIndexToSymbols; + + LLVM_DEBUG(dbgs() << "Creating graph symbols...\n"); + + for (auto SecRef : sections) { + + if (SecRef.sh_type != ELF::SHT_SYMTAB && + SecRef.sh_type != ELF::SHT_DYNSYM) + continue; + auto Symbols = Obj.symbols(&SecRef); + if (!Symbols) + return Symbols.takeError(); + + auto StrTabSec = Obj.getSection(SecRef.sh_link); + if (!StrTabSec) + return StrTabSec.takeError(); + auto StringTable = Obj.getStringTable(**StrTabSec); + if (!StringTable) + return StringTable.takeError(); + auto Name = Obj.getSectionName(SecRef); + if (!Name) + return Name.takeError(); + + LLVM_DEBUG(dbgs() << "Processing symbol section " << *Name << ":\n"); + + auto Section = G->findSectionByName(*Name); + if (!Section) + return make_error<llvm::StringError>("Could not find a section " + + *Name, + llvm::inconvertibleErrorCode()); + // we only have one for now + auto blocks = Section->blocks(); + if (blocks.empty()) + return make_error<llvm::StringError>("Section has no block", + llvm::inconvertibleErrorCode()); + int SymbolIndex = -1; + for (auto SymRef : *Symbols) { + ++SymbolIndex; + auto Type = SymRef.getType(); + + if (Type == ELF::STT_FILE || SymbolIndex == 0) + continue; + // these should do it for now + // if(Type != ELF::STT_NOTYPE && + // Type != ELF::STT_OBJECT && + // Type != ELF::STT_FUNC && + // Type != ELF::STT_SECTION && + // Type != ELF::STT_COMMON) { + // continue; + // } + auto Name = SymRef.getName(*StringTable); + // I am not sure on If this is going to hold as an invariant. Revisit. + if (!Name) + return Name.takeError(); + + if (SymRef.isCommon()) { + // Symbols in SHN_COMMON refer to uninitialized data. The st_value + // field holds alignment constraints. + Symbol &S = + G->addCommonSymbol(*Name, Scope::Default, getCommonSection(), 0, + SymRef.st_size, SymRef.getValue(), false); + JITSymbolTable[SymbolIndex] = &S; + continue; + } + + // Map Visibility and Binding to Scope and Linkage: + Linkage L = Linkage::Strong; + Scope S = Scope::Default; + + switch (SymRef.getBinding()) { + case ELF::STB_LOCAL: + S = Scope::Local; + break; + case ELF::STB_GLOBAL: + // Nothing to do here. + break; + case ELF::STB_WEAK: + L = Linkage::Weak; + break; + default: + return make_error<StringError>("Unrecognized symbol binding for " + + *Name, + inconvertibleErrorCode()); + } + + switch (SymRef.getVisibility()) { + case ELF::STV_DEFAULT: + case ELF::STV_PROTECTED: + // FIXME: Make STV_DEFAULT symbols pre-emptible? This probably needs + // Orc support. + // Otherwise nothing to do here. + break; + case ELF::STV_HIDDEN: + // Default scope -> Hidden scope. No effect on local scope. + if (S == Scope::Default) + S = Scope::Hidden; + break; + case ELF::STV_INTERNAL: + return make_error<StringError>("Unrecognized symbol visibility for " + + *Name, + inconvertibleErrorCode()); + } + + if (SymRef.isDefined() && + (Type == ELF::STT_FUNC || Type == ELF::STT_OBJECT || + Type == ELF::STT_SECTION)) { + + auto DefinedSection = Obj.getSection(SymRef.st_shndx); + if (!DefinedSection) + return DefinedSection.takeError(); + auto sectName = Obj.getSectionName(**DefinedSection); + if (!sectName) + return Name.takeError(); + + // Skip debug section symbols. + if (isDwarfSection(*sectName)) + continue; + + auto JitSection = G->findSectionByName(*sectName); + if (!JitSection) + return make_error<llvm::StringError>( + "Could not find the JitSection " + *sectName, + llvm::inconvertibleErrorCode()); + auto bs = JitSection->blocks(); + if (bs.empty()) + return make_error<llvm::StringError>( + "Section has no block", llvm::inconvertibleErrorCode()); + + auto *B = *bs.begin(); + LLVM_DEBUG({ dbgs() << " " << *Name << " at index " << SymbolIndex << "\n"; }); + if (SymRef.getType() == ELF::STT_SECTION) + *Name = *sectName; + auto &Sym = G->addDefinedSymbol( + *B, SymRef.getValue(), *Name, SymRef.st_size, L, S, + SymRef.getType() == ELF::STT_FUNC, false); + JITSymbolTable[SymbolIndex] = &Sym; + } else if (SymRef.isUndefined() && SymRef.isExternal()) { + auto &Sym = G->addExternalSymbol(*Name, SymRef.st_size, L); + JITSymbolTable[SymbolIndex] = &Sym; + } else + LLVM_DEBUG({ + dbgs() + << "Not creating graph symbol for normalized symbol at index " + << SymbolIndex << ", \"" << *Name << "\"\n"; + }); + + // TODO: The following has to be implmented. + // leaving commented out to save time for future patchs + /* + G->addAbsoluteSymbol(*Name, SymRef.getValue(), SymRef.st_size, + Linkage::Strong, Scope::Default, false); + */ + } + } + return Error::success(); + } + +public: + ELFLinkGraphBuilder_x86_64(StringRef FileName, + const object::ELFFile<object::ELF64LE> &Obj) + : G(std::make_unique<LinkGraph>(FileName.str(), + Triple("x86_64-unknown-linux"), + getPointerSize(Obj), getEndianness(Obj))), + Obj(Obj) {} + + Expected<std::unique_ptr<LinkGraph>> buildGraph() { + // Sanity check: we only operate on relocatable objects. + if (!isRelocatable()) + return make_error<JITLinkError>("Object is not a relocatable ELF"); + + auto Secs = Obj.sections(); + + if (!Secs) { + return Secs.takeError(); + } + sections = *Secs; + + if (auto Err = createNormalizedSections()) + return std::move(Err); + + if (auto Err = createNormalizedSymbols()) + return std::move(Err); + + if (auto Err = graphifyRegularSymbols()) + return std::move(Err); + + if (auto Err = addRelocations()) + return std::move(Err); + + return std::move(G); + } +}; + +class ELFJITLinker_x86_64 : public JITLinker<ELFJITLinker_x86_64> { + friend class JITLinker<ELFJITLinker_x86_64>; + +public: + ELFJITLinker_x86_64(std::unique_ptr<JITLinkContext> Ctx, + std::unique_ptr<LinkGraph> G, + PassConfiguration PassConfig) + : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) {} + +private: + StringRef getEdgeKindName(Edge::Kind R) const override { + return getELFX86RelocationKindName(R); + } + + static Error targetOutOfRangeError(const Block &B, const Edge &E) { + std::string ErrMsg; + { + raw_string_ostream ErrStream(ErrMsg); + ErrStream << "Relocation target out of range: "; + printEdge(ErrStream, B, E, getELFX86RelocationKindName(E.getKind())); + ErrStream << "\n"; + } + return make_error<JITLinkError>(std::move(ErrMsg)); + } + + Error applyFixup(Block &B, const Edge &E, char *BlockWorkingMem) const { + using namespace ELF_x86_64_Edges; + using namespace llvm::support; + char *FixupPtr = BlockWorkingMem + E.getOffset(); + JITTargetAddress FixupAddress = B.getAddress() + E.getOffset(); + switch (E.getKind()) { + case ELFX86RelocationKind::Branch32: + case ELFX86RelocationKind::Branch32ToStub: + case ELFX86RelocationKind::PCRel32: + case ELFX86RelocationKind::PCRel32GOTLoad: { + int64_t Value = E.getTarget().getAddress() + E.getAddend() - FixupAddress; + if (Value < std::numeric_limits<int32_t>::min() || + Value > std::numeric_limits<int32_t>::max()) + return targetOutOfRangeError(B, E); + *(little32_t *)FixupPtr = Value; + break; + } + case ELFX86RelocationKind::Pointer64: { + int64_t Value = E.getTarget().getAddress() + E.getAddend(); + *(ulittle64_t *)FixupPtr = Value; + break; + } + case ELFX86RelocationKind::Delta64: { + int64_t Value = E.getTarget().getAddress() + E.getAddend() - FixupAddress; + *(little64_t *)FixupPtr = Value; + break; + } + } + return Error::success(); + } +}; + +Expected<std::unique_ptr<LinkGraph>> +createLinkGraphFromELFObject_x86_64(MemoryBufferRef ObjectBuffer) { + LLVM_DEBUG({ + dbgs() << "Building jitlink graph for new input " + << ObjectBuffer.getBufferIdentifier() << "...\n"; + }); + + auto ELFObj = object::ObjectFile::createELFObjectFile(ObjectBuffer); + if (!ELFObj) + return ELFObj.takeError(); + + auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF64LE>>(**ELFObj); + return ELFLinkGraphBuilder_x86_64((*ELFObj)->getFileName(), + ELFObjFile.getELFFile()) + .buildGraph(); +} + +void link_ELF_x86_64(std::unique_ptr<LinkGraph> G, + std::unique_ptr<JITLinkContext> Ctx) { + PassConfiguration Config; + + if (Ctx->shouldAddDefaultTargetPasses(G->getTargetTriple())) { + + Config.PrePrunePasses.push_back(EHFrameSplitter(".eh_frame")); + Config.PrePrunePasses.push_back(EHFrameEdgeFixer( + ".eh_frame", G->getPointerSize(), Delta64, Delta32, NegDelta32)); + + // Construct a JITLinker and run the link function. + // Add a mark-live pass. + if (auto MarkLive = Ctx->getMarkLivePass(G->getTargetTriple())) + Config.PrePrunePasses.push_back(std::move(MarkLive)); + else + Config.PrePrunePasses.push_back(markAllSymbolsLive); + + // Add an in-place GOT/Stubs pass. + Config.PostPrunePasses.push_back([](LinkGraph &G) -> Error { + ELF_x86_64_GOTAndStubsBuilder(G).run(); + return Error::success(); + }); + + // Add GOT/Stubs optimizer pass. + Config.PreFixupPasses.push_back(optimizeELF_x86_64_GOTAndStubs); + } + + if (auto Err = Ctx->modifyPassConfig(G->getTargetTriple(), Config)) + return Ctx->notifyFailed(std::move(Err)); + + ELFJITLinker_x86_64::link(std::move(Ctx), std::move(G), std::move(Config)); +} +StringRef getELFX86RelocationKindName(Edge::Kind R) { + switch (R) { + case PCRel32: + return "PCRel32"; + case Pointer64: + return "Pointer64"; + case PCRel32GOTLoad: + return "PCRel32GOTLoad"; + case Branch32: + return "Branch32"; + case Branch32ToStub: + return "Branch32ToStub"; + } + return getGenericEdgeKindName(static_cast<Edge::Kind>(R)); +} +} // end namespace jitlink +} // end namespace llvm diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/JITLink/JITLink.cpp b/contrib/libs/llvm12/lib/ExecutionEngine/JITLink/JITLink.cpp new file mode 100644 index 0000000000..93dfba9c75 --- /dev/null +++ b/contrib/libs/llvm12/lib/ExecutionEngine/JITLink/JITLink.cpp @@ -0,0 +1,350 @@ +//===------------- JITLink.cpp - Core Run-time JIT linker APIs ------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/JITLink/JITLink.h" + +#include "llvm/BinaryFormat/Magic.h" +#include "llvm/ExecutionEngine/JITLink/ELF.h" +#include "llvm/ExecutionEngine/JITLink/MachO.h" +#include "llvm/Support/Format.h" +#include "llvm/Support/ManagedStatic.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/raw_ostream.h" + +using namespace llvm; +using namespace llvm::object; + +#define DEBUG_TYPE "jitlink" + +namespace { + +enum JITLinkErrorCode { GenericJITLinkError = 1 }; + +// FIXME: This class is only here to support the transition to llvm::Error. It +// will be removed once this transition is complete. Clients should prefer to +// deal with the Error value directly, rather than converting to error_code. +class JITLinkerErrorCategory : public std::error_category { +public: + const char *name() const noexcept override { return "runtimedyld"; } + + std::string message(int Condition) const override { + switch (static_cast<JITLinkErrorCode>(Condition)) { + case GenericJITLinkError: + return "Generic JITLink error"; + } + llvm_unreachable("Unrecognized JITLinkErrorCode"); + } +}; + +static ManagedStatic<JITLinkerErrorCategory> JITLinkerErrorCategory; + +} // namespace + +namespace llvm { +namespace jitlink { + +char JITLinkError::ID = 0; + +void JITLinkError::log(raw_ostream &OS) const { OS << ErrMsg << "\n"; } + +std::error_code JITLinkError::convertToErrorCode() const { + return std::error_code(GenericJITLinkError, *JITLinkerErrorCategory); +} + +const char *getGenericEdgeKindName(Edge::Kind K) { + switch (K) { + case Edge::Invalid: + return "INVALID RELOCATION"; + case Edge::KeepAlive: + return "Keep-Alive"; + default: + return "<Unrecognized edge kind>"; + } +} + +const char *getLinkageName(Linkage L) { + switch (L) { + case Linkage::Strong: + return "strong"; + case Linkage::Weak: + return "weak"; + } + llvm_unreachable("Unrecognized llvm.jitlink.Linkage enum"); +} + +const char *getScopeName(Scope S) { + switch (S) { + case Scope::Default: + return "default"; + case Scope::Hidden: + return "hidden"; + case Scope::Local: + return "local"; + } + llvm_unreachable("Unrecognized llvm.jitlink.Scope enum"); +} + +raw_ostream &operator<<(raw_ostream &OS, const Block &B) { + return OS << formatv("{0:x16}", B.getAddress()) << " -- " + << formatv("{0:x16}", B.getAddress() + B.getSize()) << ": " + << "size = " << formatv("{0:x}", B.getSize()) << ", " + << (B.isZeroFill() ? "zero-fill" : "content") + << ", align = " << B.getAlignment() + << ", align-ofs = " << B.getAlignmentOffset() + << ", section = " << B.getSection().getName(); +} + +raw_ostream &operator<<(raw_ostream &OS, const Symbol &Sym) { + OS << "<"; + if (Sym.getName().empty()) + OS << "*anon*"; + else + OS << Sym.getName(); + OS << ": flags = "; + switch (Sym.getLinkage()) { + case Linkage::Strong: + OS << 'S'; + break; + case Linkage::Weak: + OS << 'W'; + break; + } + switch (Sym.getScope()) { + case Scope::Default: + OS << 'D'; + break; + case Scope::Hidden: + OS << 'H'; + break; + case Scope::Local: + OS << 'L'; + break; + } + OS << (Sym.isLive() ? '+' : '-') + << ", size = " << formatv("{0:x}", Sym.getSize()) + << ", addr = " << formatv("{0:x16}", Sym.getAddress()) << " (" + << formatv("{0:x16}", Sym.getAddressable().getAddress()) << " + " + << formatv("{0:x}", Sym.getOffset()); + if (Sym.isDefined()) + OS << " " << Sym.getBlock().getSection().getName(); + OS << ")>"; + return OS; +} + +void printEdge(raw_ostream &OS, const Block &B, const Edge &E, + StringRef EdgeKindName) { + OS << "edge@" << formatv("{0:x16}", B.getAddress() + E.getOffset()) << ": " + << formatv("{0:x16}", B.getAddress()) << " + " + << formatv("{0:x}", E.getOffset()) << " -- " << EdgeKindName << " -> "; + + auto &TargetSym = E.getTarget(); + if (TargetSym.hasName()) + OS << TargetSym.getName(); + else { + auto &TargetBlock = TargetSym.getBlock(); + auto &TargetSec = TargetBlock.getSection(); + JITTargetAddress SecAddress = ~JITTargetAddress(0); + for (auto *B : TargetSec.blocks()) + if (B->getAddress() < SecAddress) + SecAddress = B->getAddress(); + + JITTargetAddress SecDelta = TargetSym.getAddress() - SecAddress; + OS << formatv("{0:x16}", TargetSym.getAddress()) << " (section " + << TargetSec.getName(); + if (SecDelta) + OS << " + " << formatv("{0:x}", SecDelta); + OS << " / block " << formatv("{0:x16}", TargetBlock.getAddress()); + if (TargetSym.getOffset()) + OS << " + " << formatv("{0:x}", TargetSym.getOffset()); + OS << ")"; + } + + if (E.getAddend() != 0) + OS << " + " << E.getAddend(); +} + +Section::~Section() { + for (auto *Sym : Symbols) + Sym->~Symbol(); + for (auto *B : Blocks) + B->~Block(); +} + +Block &LinkGraph::splitBlock(Block &B, size_t SplitIndex, + SplitBlockCache *Cache) { + + assert(SplitIndex > 0 && "splitBlock can not be called with SplitIndex == 0"); + + // If the split point covers all of B then just return B. + if (SplitIndex == B.getSize()) + return B; + + assert(SplitIndex < B.getSize() && "SplitIndex out of range"); + + // Create the new block covering [ 0, SplitIndex ). + auto &NewBlock = + B.isZeroFill() + ? createZeroFillBlock(B.getSection(), SplitIndex, B.getAddress(), + B.getAlignment(), B.getAlignmentOffset()) + : createContentBlock( + B.getSection(), B.getContent().substr(0, SplitIndex), + B.getAddress(), B.getAlignment(), B.getAlignmentOffset()); + + // Modify B to cover [ SplitIndex, B.size() ). + B.setAddress(B.getAddress() + SplitIndex); + B.setContent(B.getContent().substr(SplitIndex)); + B.setAlignmentOffset((B.getAlignmentOffset() + SplitIndex) % + B.getAlignment()); + + // Handle edge transfer/update. + { + // Copy edges to NewBlock (recording their iterators so that we can remove + // them from B), and update of Edges remaining on B. + std::vector<Block::edge_iterator> EdgesToRemove; + for (auto I = B.edges().begin(); I != B.edges().end();) { + if (I->getOffset() < SplitIndex) { + NewBlock.addEdge(*I); + I = B.removeEdge(I); + } else { + I->setOffset(I->getOffset() - SplitIndex); + ++I; + } + } + } + + // Handle symbol transfer/update. + { + // Initialize the symbols cache if necessary. + SplitBlockCache LocalBlockSymbolsCache; + if (!Cache) + Cache = &LocalBlockSymbolsCache; + if (*Cache == None) { + *Cache = SplitBlockCache::value_type(); + for (auto *Sym : B.getSection().symbols()) + if (&Sym->getBlock() == &B) + (*Cache)->push_back(Sym); + + llvm::sort(**Cache, [](const Symbol *LHS, const Symbol *RHS) { + return LHS->getOffset() > RHS->getOffset(); + }); + } + auto &BlockSymbols = **Cache; + + // Transfer all symbols with offset less than SplitIndex to NewBlock. + while (!BlockSymbols.empty() && + BlockSymbols.back()->getOffset() < SplitIndex) { + BlockSymbols.back()->setBlock(NewBlock); + BlockSymbols.pop_back(); + } + + // Update offsets for all remaining symbols in B. + for (auto *Sym : BlockSymbols) + Sym->setOffset(Sym->getOffset() - SplitIndex); + } + + return NewBlock; +} + +void LinkGraph::dump(raw_ostream &OS, + std::function<StringRef(Edge::Kind)> EdgeKindToName) { + if (!EdgeKindToName) + EdgeKindToName = [](Edge::Kind K) { return StringRef(); }; + + OS << "Symbols:\n"; + for (auto *Sym : defined_symbols()) { + OS << " " << format("0x%016" PRIx64, Sym->getAddress()) << ": " << *Sym + << "\n"; + if (Sym->isDefined()) { + for (auto &E : Sym->getBlock().edges()) { + OS << " "; + StringRef EdgeName = (E.getKind() < Edge::FirstRelocation + ? getGenericEdgeKindName(E.getKind()) + : EdgeKindToName(E.getKind())); + + if (!EdgeName.empty()) + printEdge(OS, Sym->getBlock(), E, EdgeName); + else { + auto EdgeNumberString = std::to_string(E.getKind()); + printEdge(OS, Sym->getBlock(), E, EdgeNumberString); + } + OS << "\n"; + } + } + } + + OS << "Absolute symbols:\n"; + for (auto *Sym : absolute_symbols()) + OS << " " << format("0x%016" PRIx64, Sym->getAddress()) << ": " << *Sym + << "\n"; + + OS << "External symbols:\n"; + for (auto *Sym : external_symbols()) + OS << " " << format("0x%016" PRIx64, Sym->getAddress()) << ": " << *Sym + << "\n"; +} + +raw_ostream &operator<<(raw_ostream &OS, const SymbolLookupFlags &LF) { + switch (LF) { + case SymbolLookupFlags::RequiredSymbol: + return OS << "RequiredSymbol"; + case SymbolLookupFlags::WeaklyReferencedSymbol: + return OS << "WeaklyReferencedSymbol"; + } + llvm_unreachable("Unrecognized lookup flags"); +} + +void JITLinkAsyncLookupContinuation::anchor() {} + +JITLinkContext::~JITLinkContext() {} + +bool JITLinkContext::shouldAddDefaultTargetPasses(const Triple &TT) const { + return true; +} + +LinkGraphPassFunction JITLinkContext::getMarkLivePass(const Triple &TT) const { + return LinkGraphPassFunction(); +} + +Error JITLinkContext::modifyPassConfig(const Triple &TT, + PassConfiguration &Config) { + return Error::success(); +} + +Error markAllSymbolsLive(LinkGraph &G) { + for (auto *Sym : G.defined_symbols()) + Sym->setLive(true); + return Error::success(); +} + +Expected<std::unique_ptr<LinkGraph>> +createLinkGraphFromObject(MemoryBufferRef ObjectBuffer) { + auto Magic = identify_magic(ObjectBuffer.getBuffer()); + switch (Magic) { + case file_magic::macho_object: + return createLinkGraphFromMachOObject(std::move(ObjectBuffer)); + case file_magic::elf_relocatable: + return createLinkGraphFromELFObject(std::move(ObjectBuffer)); + default: + return make_error<JITLinkError>("Unsupported file format"); + }; +} + +void link(std::unique_ptr<LinkGraph> G, std::unique_ptr<JITLinkContext> Ctx) { + switch (G->getTargetTriple().getObjectFormat()) { + case Triple::MachO: + return link_MachO(std::move(G), std::move(Ctx)); + case Triple::ELF: + return link_ELF(std::move(G), std::move(Ctx)); + default: + Ctx->notifyFailed(make_error<JITLinkError>("Unsupported object format")); + }; +} + +} // end namespace jitlink +} // end namespace llvm diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp b/contrib/libs/llvm12/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp new file mode 100644 index 0000000000..7a5e014f22 --- /dev/null +++ b/contrib/libs/llvm12/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp @@ -0,0 +1,497 @@ +//===--------- JITLinkGeneric.cpp - Generic JIT linker utilities ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Generic JITLinker utility class. +// +//===----------------------------------------------------------------------===// + +#include "JITLinkGeneric.h" + +#include "llvm/Support/BinaryStreamReader.h" +#include "llvm/Support/MemoryBuffer.h" + +#define DEBUG_TYPE "jitlink" + +namespace llvm { +namespace jitlink { + +JITLinkerBase::~JITLinkerBase() {} + +void JITLinkerBase::linkPhase1(std::unique_ptr<JITLinkerBase> Self) { + + LLVM_DEBUG({ + dbgs() << "Starting link phase 1 for graph " << G->getName() << "\n"; + }); + + // Prune and optimize the graph. + if (auto Err = runPasses(Passes.PrePrunePasses)) + return Ctx->notifyFailed(std::move(Err)); + + LLVM_DEBUG({ + dbgs() << "Link graph \"" << G->getName() << "\" pre-pruning:\n"; + dumpGraph(dbgs()); + }); + + prune(*G); + + LLVM_DEBUG({ + dbgs() << "Link graph \"" << G->getName() << "\" post-pruning:\n"; + dumpGraph(dbgs()); + }); + + // Run post-pruning passes. + if (auto Err = runPasses(Passes.PostPrunePasses)) + return Ctx->notifyFailed(std::move(Err)); + + // Sort blocks into segments. + auto Layout = layOutBlocks(); + + // Allocate memory for segments. + if (auto Err = allocateSegments(Layout)) + return Ctx->notifyFailed(std::move(Err)); + + LLVM_DEBUG({ + dbgs() << "Link graph \"" << G->getName() + << "\" before post-allocation passes:\n"; + dumpGraph(dbgs()); + }); + + // Run post-allocation passes. + if (auto Err = runPasses(Passes.PostAllocationPasses)) + return Ctx->notifyFailed(std::move(Err)); + + // Notify client that the defined symbols have been assigned addresses. + LLVM_DEBUG( + { dbgs() << "Resolving symbols defined in " << G->getName() << "\n"; }); + + if (auto Err = Ctx->notifyResolved(*G)) + return Ctx->notifyFailed(std::move(Err)); + + auto ExternalSymbols = getExternalSymbolNames(); + + LLVM_DEBUG({ + dbgs() << "Issuing lookup for external symbols for " << G->getName() + << " (may trigger materialization/linking of other graphs)...\n"; + }); + + // We're about to hand off ownership of ourself to the continuation. Grab a + // pointer to the context so that we can call it to initiate the lookup. + // + // FIXME: Once callee expressions are defined to be sequenced before argument + // expressions (c++17) we can simplify all this to: + // + // Ctx->lookup(std::move(UnresolvedExternals), + // [Self=std::move(Self)](Expected<AsyncLookupResult> Result) { + // Self->linkPhase2(std::move(Self), std::move(Result)); + // }); + auto *TmpCtx = Ctx.get(); + TmpCtx->lookup(std::move(ExternalSymbols), + createLookupContinuation( + [S = std::move(Self), L = std::move(Layout)]( + Expected<AsyncLookupResult> LookupResult) mutable { + auto &TmpSelf = *S; + TmpSelf.linkPhase2(std::move(S), std::move(LookupResult), + std::move(L)); + })); +} + +void JITLinkerBase::linkPhase2(std::unique_ptr<JITLinkerBase> Self, + Expected<AsyncLookupResult> LR, + SegmentLayoutMap Layout) { + + LLVM_DEBUG({ + dbgs() << "Starting link phase 2 for graph " << G->getName() << "\n"; + }); + + // If the lookup failed, bail out. + if (!LR) + return deallocateAndBailOut(LR.takeError()); + + // Assign addresses to external addressables. + applyLookupResult(*LR); + + // Copy block content to working memory. + copyBlockContentToWorkingMemory(Layout, *Alloc); + + LLVM_DEBUG({ + dbgs() << "Link graph \"" << G->getName() + << "\" before pre-fixup passes:\n"; + dumpGraph(dbgs()); + }); + + if (auto Err = runPasses(Passes.PreFixupPasses)) + return deallocateAndBailOut(std::move(Err)); + + LLVM_DEBUG({ + dbgs() << "Link graph \"" << G->getName() << "\" before copy-and-fixup:\n"; + dumpGraph(dbgs()); + }); + + // Fix up block content. + if (auto Err = fixUpBlocks(*G)) + return deallocateAndBailOut(std::move(Err)); + + LLVM_DEBUG({ + dbgs() << "Link graph \"" << G->getName() << "\" after copy-and-fixup:\n"; + dumpGraph(dbgs()); + }); + + if (auto Err = runPasses(Passes.PostFixupPasses)) + return deallocateAndBailOut(std::move(Err)); + + // FIXME: Use move capture once we have c++14. + auto *UnownedSelf = Self.release(); + auto Phase3Continuation = [UnownedSelf](Error Err) { + std::unique_ptr<JITLinkerBase> Self(UnownedSelf); + UnownedSelf->linkPhase3(std::move(Self), std::move(Err)); + }; + + Alloc->finalizeAsync(std::move(Phase3Continuation)); +} + +void JITLinkerBase::linkPhase3(std::unique_ptr<JITLinkerBase> Self, Error Err) { + + LLVM_DEBUG({ + dbgs() << "Starting link phase 3 for graph " << G->getName() << "\n"; + }); + + if (Err) + return deallocateAndBailOut(std::move(Err)); + Ctx->notifyFinalized(std::move(Alloc)); + + LLVM_DEBUG({ dbgs() << "Link of graph " << G->getName() << " complete\n"; }); +} + +Error JITLinkerBase::runPasses(LinkGraphPassList &Passes) { + for (auto &P : Passes) + if (auto Err = P(*G)) + return Err; + return Error::success(); +} + +JITLinkerBase::SegmentLayoutMap JITLinkerBase::layOutBlocks() { + + SegmentLayoutMap Layout; + + /// Partition blocks based on permissions and content vs. zero-fill. + for (auto *B : G->blocks()) { + auto &SegLists = Layout[B->getSection().getProtectionFlags()]; + if (!B->isZeroFill()) + SegLists.ContentBlocks.push_back(B); + else + SegLists.ZeroFillBlocks.push_back(B); + } + + /// Sort blocks within each list. + for (auto &KV : Layout) { + + auto CompareBlocks = [](const Block *LHS, const Block *RHS) { + // Sort by section, address and size + if (LHS->getSection().getOrdinal() != RHS->getSection().getOrdinal()) + return LHS->getSection().getOrdinal() < RHS->getSection().getOrdinal(); + if (LHS->getAddress() != RHS->getAddress()) + return LHS->getAddress() < RHS->getAddress(); + return LHS->getSize() < RHS->getSize(); + }; + + auto &SegLists = KV.second; + llvm::sort(SegLists.ContentBlocks, CompareBlocks); + llvm::sort(SegLists.ZeroFillBlocks, CompareBlocks); + } + + LLVM_DEBUG({ + dbgs() << "Computed segment ordering:\n"; + for (auto &KV : Layout) { + dbgs() << " Segment " + << static_cast<sys::Memory::ProtectionFlags>(KV.first) << ":\n"; + auto &SL = KV.second; + for (auto &SIEntry : + {std::make_pair(&SL.ContentBlocks, "content block"), + std::make_pair(&SL.ZeroFillBlocks, "zero-fill block")}) { + dbgs() << " " << SIEntry.second << ":\n"; + for (auto *B : *SIEntry.first) + dbgs() << " " << *B << "\n"; + } + } + }); + + return Layout; +} + +Error JITLinkerBase::allocateSegments(const SegmentLayoutMap &Layout) { + + // Compute segment sizes and allocate memory. + LLVM_DEBUG(dbgs() << "JIT linker requesting: { "); + JITLinkMemoryManager::SegmentsRequestMap Segments; + for (auto &KV : Layout) { + auto &Prot = KV.first; + auto &SegLists = KV.second; + + uint64_t SegAlign = 1; + + // Calculate segment content size. + size_t SegContentSize = 0; + for (auto *B : SegLists.ContentBlocks) { + SegAlign = std::max(SegAlign, B->getAlignment()); + SegContentSize = alignToBlock(SegContentSize, *B); + SegContentSize += B->getSize(); + } + + uint64_t SegZeroFillStart = SegContentSize; + uint64_t SegZeroFillEnd = SegZeroFillStart; + + for (auto *B : SegLists.ZeroFillBlocks) { + SegAlign = std::max(SegAlign, B->getAlignment()); + SegZeroFillEnd = alignToBlock(SegZeroFillEnd, *B); + SegZeroFillEnd += B->getSize(); + } + + Segments[Prot] = {SegAlign, SegContentSize, + SegZeroFillEnd - SegZeroFillStart}; + + LLVM_DEBUG({ + dbgs() << (&KV == &*Layout.begin() ? "" : "; ") + << static_cast<sys::Memory::ProtectionFlags>(Prot) + << ": alignment = " << SegAlign + << ", content size = " << SegContentSize + << ", zero-fill size = " << (SegZeroFillEnd - SegZeroFillStart); + }); + } + LLVM_DEBUG(dbgs() << " }\n"); + + if (auto AllocOrErr = + Ctx->getMemoryManager().allocate(Ctx->getJITLinkDylib(), Segments)) + Alloc = std::move(*AllocOrErr); + else + return AllocOrErr.takeError(); + + LLVM_DEBUG({ + dbgs() << "JIT linker got memory (working -> target):\n"; + for (auto &KV : Layout) { + auto Prot = static_cast<sys::Memory::ProtectionFlags>(KV.first); + dbgs() << " " << Prot << ": " + << (const void *)Alloc->getWorkingMemory(Prot).data() << " -> " + << formatv("{0:x16}", Alloc->getTargetMemory(Prot)) << "\n"; + } + }); + + // Update block target addresses. + for (auto &KV : Layout) { + auto &Prot = KV.first; + auto &SL = KV.second; + + JITTargetAddress NextBlockAddr = + Alloc->getTargetMemory(static_cast<sys::Memory::ProtectionFlags>(Prot)); + + for (auto *SIList : {&SL.ContentBlocks, &SL.ZeroFillBlocks}) + for (auto *B : *SIList) { + NextBlockAddr = alignToBlock(NextBlockAddr, *B); + B->setAddress(NextBlockAddr); + NextBlockAddr += B->getSize(); + } + } + + return Error::success(); +} + +JITLinkContext::LookupMap JITLinkerBase::getExternalSymbolNames() const { + // Identify unresolved external symbols. + JITLinkContext::LookupMap UnresolvedExternals; + for (auto *Sym : G->external_symbols()) { + assert(Sym->getAddress() == 0 && + "External has already been assigned an address"); + assert(Sym->getName() != StringRef() && Sym->getName() != "" && + "Externals must be named"); + SymbolLookupFlags LookupFlags = + Sym->getLinkage() == Linkage::Weak + ? SymbolLookupFlags::WeaklyReferencedSymbol + : SymbolLookupFlags::RequiredSymbol; + UnresolvedExternals[Sym->getName()] = LookupFlags; + } + return UnresolvedExternals; +} + +void JITLinkerBase::applyLookupResult(AsyncLookupResult Result) { + for (auto *Sym : G->external_symbols()) { + assert(Sym->getOffset() == 0 && + "External symbol is not at the start of its addressable block"); + assert(Sym->getAddress() == 0 && "Symbol already resolved"); + assert(!Sym->isDefined() && "Symbol being resolved is already defined"); + auto ResultI = Result.find(Sym->getName()); + if (ResultI != Result.end()) + Sym->getAddressable().setAddress(ResultI->second.getAddress()); + else + assert(Sym->getLinkage() == Linkage::Weak && + "Failed to resolve non-weak reference"); + } + + LLVM_DEBUG({ + dbgs() << "Externals after applying lookup result:\n"; + for (auto *Sym : G->external_symbols()) + dbgs() << " " << Sym->getName() << ": " + << formatv("{0:x16}", Sym->getAddress()) << "\n"; + }); +} + +void JITLinkerBase::copyBlockContentToWorkingMemory( + const SegmentLayoutMap &Layout, JITLinkMemoryManager::Allocation &Alloc) { + + LLVM_DEBUG(dbgs() << "Copying block content:\n"); + for (auto &KV : Layout) { + auto &Prot = KV.first; + auto &SegLayout = KV.second; + + auto SegMem = + Alloc.getWorkingMemory(static_cast<sys::Memory::ProtectionFlags>(Prot)); + char *LastBlockEnd = SegMem.data(); + char *BlockDataPtr = LastBlockEnd; + + LLVM_DEBUG({ + dbgs() << " Processing segment " + << static_cast<sys::Memory::ProtectionFlags>(Prot) << " [ " + << (const void *)SegMem.data() << " .. " + << (const void *)((char *)SegMem.data() + SegMem.size()) + << " ]\n Processing content sections:\n"; + }); + + for (auto *B : SegLayout.ContentBlocks) { + LLVM_DEBUG(dbgs() << " " << *B << ":\n"); + + // Pad to alignment/alignment-offset. + BlockDataPtr = alignToBlock(BlockDataPtr, *B); + + LLVM_DEBUG({ + dbgs() << " Bumped block pointer to " << (const void *)BlockDataPtr + << " to meet block alignment " << B->getAlignment() + << " and alignment offset " << B->getAlignmentOffset() << "\n"; + }); + + // Zero pad up to alignment. + LLVM_DEBUG({ + if (LastBlockEnd != BlockDataPtr) + dbgs() << " Zero padding from " << (const void *)LastBlockEnd + << " to " << (const void *)BlockDataPtr << "\n"; + }); + + while (LastBlockEnd != BlockDataPtr) + *LastBlockEnd++ = 0; + + // Copy initial block content. + LLVM_DEBUG({ + dbgs() << " Copying block " << *B << " content, " + << B->getContent().size() << " bytes, from " + << (const void *)B->getContent().data() << " to " + << (const void *)BlockDataPtr << "\n"; + }); + memcpy(BlockDataPtr, B->getContent().data(), B->getContent().size()); + + // Point the block's content to the fixed up buffer. + B->setContent(StringRef(BlockDataPtr, B->getContent().size())); + + // Update block end pointer. + LastBlockEnd = BlockDataPtr + B->getContent().size(); + BlockDataPtr = LastBlockEnd; + } + + // Zero pad the rest of the segment. + LLVM_DEBUG({ + dbgs() << " Zero padding end of segment from " + << (const void *)LastBlockEnd << " to " + << (const void *)((char *)SegMem.data() + SegMem.size()) << "\n"; + }); + while (LastBlockEnd != SegMem.data() + SegMem.size()) + *LastBlockEnd++ = 0; + } +} + +void JITLinkerBase::deallocateAndBailOut(Error Err) { + assert(Err && "Should not be bailing out on success value"); + assert(Alloc && "can not call deallocateAndBailOut before allocation"); + Ctx->notifyFailed(joinErrors(std::move(Err), Alloc->deallocate())); +} + +void JITLinkerBase::dumpGraph(raw_ostream &OS) { + assert(G && "Graph is not set yet"); + G->dump(dbgs(), [this](Edge::Kind K) { return getEdgeKindName(K); }); +} + +void prune(LinkGraph &G) { + std::vector<Symbol *> Worklist; + DenseSet<Block *> VisitedBlocks; + + // Build the initial worklist from all symbols initially live. + for (auto *Sym : G.defined_symbols()) + if (Sym->isLive()) + Worklist.push_back(Sym); + + // Propagate live flags to all symbols reachable from the initial live set. + while (!Worklist.empty()) { + auto *Sym = Worklist.back(); + Worklist.pop_back(); + + auto &B = Sym->getBlock(); + + // Skip addressables that we've visited before. + if (VisitedBlocks.count(&B)) + continue; + + VisitedBlocks.insert(&B); + + for (auto &E : Sym->getBlock().edges()) { + // If the edge target is a defined symbol that is being newly marked live + // then add it to the worklist. + if (E.getTarget().isDefined() && !E.getTarget().isLive()) + Worklist.push_back(&E.getTarget()); + + // Mark the target live. + E.getTarget().setLive(true); + } + } + + // Collect all defined symbols to remove, then remove them. + { + LLVM_DEBUG(dbgs() << "Dead-stripping defined symbols:\n"); + std::vector<Symbol *> SymbolsToRemove; + for (auto *Sym : G.defined_symbols()) + if (!Sym->isLive()) + SymbolsToRemove.push_back(Sym); + for (auto *Sym : SymbolsToRemove) { + LLVM_DEBUG(dbgs() << " " << *Sym << "...\n"); + G.removeDefinedSymbol(*Sym); + } + } + + // Delete any unused blocks. + { + LLVM_DEBUG(dbgs() << "Dead-stripping blocks:\n"); + std::vector<Block *> BlocksToRemove; + for (auto *B : G.blocks()) + if (!VisitedBlocks.count(B)) + BlocksToRemove.push_back(B); + for (auto *B : BlocksToRemove) { + LLVM_DEBUG(dbgs() << " " << *B << "...\n"); + G.removeBlock(*B); + } + } + + // Collect all external symbols to remove, then remove them. + { + LLVM_DEBUG(dbgs() << "Removing unused external symbols:\n"); + std::vector<Symbol *> SymbolsToRemove; + for (auto *Sym : G.external_symbols()) + if (!Sym->isLive()) + SymbolsToRemove.push_back(Sym); + for (auto *Sym : SymbolsToRemove) { + LLVM_DEBUG(dbgs() << " " << *Sym << "...\n"); + G.removeExternalSymbol(*Sym); + } + } +} + +} // end namespace jitlink +} // end namespace llvm diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/JITLink/JITLinkGeneric.h b/contrib/libs/llvm12/lib/ExecutionEngine/JITLink/JITLinkGeneric.h new file mode 100644 index 0000000000..1d28f5006b --- /dev/null +++ b/contrib/libs/llvm12/lib/ExecutionEngine/JITLink/JITLinkGeneric.h @@ -0,0 +1,180 @@ +//===------ JITLinkGeneric.h - Generic JIT linker utilities -----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Generic JITLinker utilities. E.g. graph pruning, eh-frame parsing. +// +//===----------------------------------------------------------------------===// + +#ifndef LIB_EXECUTIONENGINE_JITLINK_JITLINKGENERIC_H +#define LIB_EXECUTIONENGINE_JITLINK_JITLINKGENERIC_H + +#include "llvm/ADT/DenseSet.h" +#include "llvm/ExecutionEngine/JITLink/JITLink.h" + +#define DEBUG_TYPE "jitlink" + +namespace llvm { + +class MemoryBufferRef; + +namespace jitlink { + +/// Base class for a JIT linker. +/// +/// A JITLinkerBase instance links one object file into an ongoing JIT +/// session. Symbol resolution and finalization operations are pluggable, +/// and called using continuation passing (passing a continuation for the +/// remaining linker work) to allow them to be performed asynchronously. +class JITLinkerBase { +public: + JITLinkerBase(std::unique_ptr<JITLinkContext> Ctx, + std::unique_ptr<LinkGraph> G, PassConfiguration Passes) + : Ctx(std::move(Ctx)), G(std::move(G)), Passes(std::move(Passes)) { + assert(this->Ctx && "Ctx can not be null"); + assert(this->G && "G can not be null"); + } + + virtual ~JITLinkerBase(); + +protected: + struct SegmentLayout { + using BlocksList = std::vector<Block *>; + + BlocksList ContentBlocks; + BlocksList ZeroFillBlocks; + }; + + using SegmentLayoutMap = DenseMap<unsigned, SegmentLayout>; + + // Phase 1: + // 1.1: Run pre-prune passes + // 1.2: Prune graph + // 1.3: Run post-prune passes + // 1.4: Sort blocks into segments + // 1.5: Allocate segment memory + // 1.6: Identify externals and make an async call to resolve function + void linkPhase1(std::unique_ptr<JITLinkerBase> Self); + + // Phase 2: + // 2.1: Apply resolution results + // 2.2: Fix up block contents + // 2.3: Call OnResolved callback + // 2.3: Make an async call to transfer and finalize memory. + void linkPhase2(std::unique_ptr<JITLinkerBase> Self, + Expected<AsyncLookupResult> LookupResult, + SegmentLayoutMap Layout); + + // Phase 3: + // 3.1: Call OnFinalized callback, handing off allocation. + void linkPhase3(std::unique_ptr<JITLinkerBase> Self, Error Err); + + // For debug dumping of the link graph. + virtual StringRef getEdgeKindName(Edge::Kind K) const = 0; + + // Align a JITTargetAddress to conform with block alignment requirements. + static JITTargetAddress alignToBlock(JITTargetAddress Addr, Block &B) { + uint64_t Delta = (B.getAlignmentOffset() - Addr) % B.getAlignment(); + return Addr + Delta; + } + + // Align a pointer to conform with block alignment requirements. + static char *alignToBlock(char *P, Block &B) { + uint64_t PAddr = static_cast<uint64_t>(reinterpret_cast<uintptr_t>(P)); + uint64_t Delta = (B.getAlignmentOffset() - PAddr) % B.getAlignment(); + return P + Delta; + } + +private: + // Run all passes in the given pass list, bailing out immediately if any pass + // returns an error. + Error runPasses(LinkGraphPassList &Passes); + + // Copy block contents and apply relocations. + // Implemented in JITLinker. + virtual Error fixUpBlocks(LinkGraph &G) const = 0; + + SegmentLayoutMap layOutBlocks(); + Error allocateSegments(const SegmentLayoutMap &Layout); + JITLinkContext::LookupMap getExternalSymbolNames() const; + void applyLookupResult(AsyncLookupResult LR); + void copyBlockContentToWorkingMemory(const SegmentLayoutMap &Layout, + JITLinkMemoryManager::Allocation &Alloc); + void deallocateAndBailOut(Error Err); + + void dumpGraph(raw_ostream &OS); + + std::unique_ptr<JITLinkContext> Ctx; + std::unique_ptr<LinkGraph> G; + PassConfiguration Passes; + std::unique_ptr<JITLinkMemoryManager::Allocation> Alloc; +}; + +template <typename LinkerImpl> class JITLinker : public JITLinkerBase { +public: + using JITLinkerBase::JITLinkerBase; + + /// Link constructs a LinkerImpl instance and calls linkPhase1. + /// Link should be called with the constructor arguments for LinkerImpl, which + /// will be forwarded to the constructor. + template <typename... ArgTs> static void link(ArgTs &&... Args) { + auto L = std::make_unique<LinkerImpl>(std::forward<ArgTs>(Args)...); + + // Ownership of the linker is passed into the linker's doLink function to + // allow it to be passed on to async continuations. + // + // FIXME: Remove LTmp once we have c++17. + // C++17 sequencing rules guarantee that function name expressions are + // sequenced before arguments, so L->linkPhase1(std::move(L), ...) will be + // well formed. + auto <mp = *L; + LTmp.linkPhase1(std::move(L)); + } + +private: + const LinkerImpl &impl() const { + return static_cast<const LinkerImpl &>(*this); + } + + Error fixUpBlocks(LinkGraph &G) const override { + LLVM_DEBUG(dbgs() << "Fixing up blocks:\n"); + + for (auto *B : G.blocks()) { + LLVM_DEBUG(dbgs() << " " << *B << ":\n"); + + // Copy Block data and apply fixups. + LLVM_DEBUG(dbgs() << " Applying fixups.\n"); + for (auto &E : B->edges()) { + + // Skip non-relocation edges. + if (!E.isRelocation()) + continue; + + // Dispatch to LinkerImpl for fixup. + auto *BlockData = const_cast<char *>(B->getContent().data()); + if (auto Err = impl().applyFixup(*B, E, BlockData)) + return Err; + } + } + + return Error::success(); + } +}; + +/// Removes dead symbols/blocks/addressables. +/// +/// Finds the set of symbols and addressables reachable from any symbol +/// initially marked live. All symbols/addressables not marked live at the end +/// of this process are removed. +void prune(LinkGraph &G); + +} // end namespace jitlink +} // end namespace llvm + +#undef DEBUG_TYPE // "jitlink" + +#endif // LLVM_EXECUTIONENGINE_JITLINK_JITLINKGENERIC_H diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp b/contrib/libs/llvm12/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp new file mode 100644 index 0000000000..fbbb29e916 --- /dev/null +++ b/contrib/libs/llvm12/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp @@ -0,0 +1,133 @@ +//===--- JITLinkMemoryManager.cpp - JITLinkMemoryManager implementation ---===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h" +#include "llvm/Support/Process.h" + +namespace llvm { +namespace jitlink { + +JITLinkMemoryManager::~JITLinkMemoryManager() = default; +JITLinkMemoryManager::Allocation::~Allocation() = default; + +Expected<std::unique_ptr<JITLinkMemoryManager::Allocation>> +InProcessMemoryManager::allocate(const JITLinkDylib *JD, + const SegmentsRequestMap &Request) { + + using AllocationMap = DenseMap<unsigned, sys::MemoryBlock>; + + // Local class for allocation. + class IPMMAlloc : public Allocation { + public: + IPMMAlloc(AllocationMap SegBlocks) : SegBlocks(std::move(SegBlocks)) {} + MutableArrayRef<char> getWorkingMemory(ProtectionFlags Seg) override { + assert(SegBlocks.count(Seg) && "No allocation for segment"); + return {static_cast<char *>(SegBlocks[Seg].base()), + SegBlocks[Seg].allocatedSize()}; + } + JITTargetAddress getTargetMemory(ProtectionFlags Seg) override { + assert(SegBlocks.count(Seg) && "No allocation for segment"); + return pointerToJITTargetAddress(SegBlocks[Seg].base()); + } + void finalizeAsync(FinalizeContinuation OnFinalize) override { + OnFinalize(applyProtections()); + } + Error deallocate() override { + if (SegBlocks.empty()) + return Error::success(); + void *SlabStart = SegBlocks.begin()->second.base(); + char *SlabEnd = (char *)SlabStart; + for (auto &KV : SegBlocks) { + SlabStart = std::min(SlabStart, KV.second.base()); + SlabEnd = std::max(SlabEnd, (char *)(KV.second.base()) + + KV.second.allocatedSize()); + } + size_t SlabSize = SlabEnd - (char *)SlabStart; + assert((SlabSize % sys::Process::getPageSizeEstimate()) == 0 && + "Slab size is not a multiple of page size"); + sys::MemoryBlock Slab(SlabStart, SlabSize); + if (auto EC = sys::Memory::releaseMappedMemory(Slab)) + return errorCodeToError(EC); + return Error::success(); + } + + private: + Error applyProtections() { + for (auto &KV : SegBlocks) { + auto &Prot = KV.first; + auto &Block = KV.second; + if (auto EC = sys::Memory::protectMappedMemory(Block, Prot)) + return errorCodeToError(EC); + if (Prot & sys::Memory::MF_EXEC) + sys::Memory::InvalidateInstructionCache(Block.base(), + Block.allocatedSize()); + } + return Error::success(); + } + + AllocationMap SegBlocks; + }; + + if (!isPowerOf2_64((uint64_t)sys::Process::getPageSizeEstimate())) + return make_error<StringError>("Page size is not a power of 2", + inconvertibleErrorCode()); + + AllocationMap Blocks; + const sys::Memory::ProtectionFlags ReadWrite = + static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | + sys::Memory::MF_WRITE); + + // Compute the total number of pages to allocate. + size_t TotalSize = 0; + for (auto &KV : Request) { + const auto &Seg = KV.second; + + if (Seg.getAlignment() > sys::Process::getPageSizeEstimate()) + return make_error<StringError>("Cannot request higher than page " + "alignment", + inconvertibleErrorCode()); + + TotalSize = alignTo(TotalSize, sys::Process::getPageSizeEstimate()); + TotalSize += Seg.getContentSize(); + TotalSize += Seg.getZeroFillSize(); + } + + // Allocate one slab to cover all the segments. + std::error_code EC; + auto SlabRemaining = + sys::Memory::allocateMappedMemory(TotalSize, nullptr, ReadWrite, EC); + + if (EC) + return errorCodeToError(EC); + + // Allocate segment memory from the slab. + for (auto &KV : Request) { + + const auto &Seg = KV.second; + + uint64_t SegmentSize = alignTo(Seg.getContentSize() + Seg.getZeroFillSize(), + sys::Process::getPageSizeEstimate()); + + sys::MemoryBlock SegMem(SlabRemaining.base(), SegmentSize); + SlabRemaining = sys::MemoryBlock((char *)SlabRemaining.base() + SegmentSize, + SegmentSize); + + // Zero out the zero-fill memory. + memset(static_cast<char *>(SegMem.base()) + Seg.getContentSize(), 0, + Seg.getZeroFillSize()); + + // Record the block for this segment. + Blocks[KV.first] = std::move(SegMem); + } + return std::unique_ptr<InProcessMemoryManager::Allocation>( + new IPMMAlloc(std::move(Blocks))); +} + +} // end namespace jitlink +} // end namespace llvm diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/JITLink/MachO.cpp b/contrib/libs/llvm12/lib/ExecutionEngine/JITLink/MachO.cpp new file mode 100644 index 0000000000..e9327df6da --- /dev/null +++ b/contrib/libs/llvm12/lib/ExecutionEngine/JITLink/MachO.cpp @@ -0,0 +1,91 @@ +//===-------------- MachO.cpp - JIT linker function for MachO -------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// MachO jit-link function. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/JITLink/MachO.h" + +#include "llvm/BinaryFormat/MachO.h" +#include "llvm/ExecutionEngine/JITLink/MachO_arm64.h" +#include "llvm/ExecutionEngine/JITLink/MachO_x86_64.h" +#include "llvm/Support/Format.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/SwapByteOrder.h" + +using namespace llvm; + +#define DEBUG_TYPE "jitlink" + +namespace llvm { +namespace jitlink { + +Expected<std::unique_ptr<LinkGraph>> +createLinkGraphFromMachOObject(MemoryBufferRef ObjectBuffer) { + StringRef Data = ObjectBuffer.getBuffer(); + if (Data.size() < 4) + return make_error<JITLinkError>("Truncated MachO buffer \"" + + ObjectBuffer.getBufferIdentifier() + "\""); + + uint32_t Magic; + memcpy(&Magic, Data.data(), sizeof(uint32_t)); + LLVM_DEBUG({ + dbgs() << "jitLink_MachO: magic = " << format("0x%08" PRIx32, Magic) + << ", identifier = \"" << ObjectBuffer.getBufferIdentifier() + << "\"\n"; + }); + + if (Magic == MachO::MH_MAGIC || Magic == MachO::MH_CIGAM) + return make_error<JITLinkError>("MachO 32-bit platforms not supported"); + else if (Magic == MachO::MH_MAGIC_64 || Magic == MachO::MH_CIGAM_64) { + + if (Data.size() < sizeof(MachO::mach_header_64)) + return make_error<JITLinkError>("Truncated MachO buffer \"" + + ObjectBuffer.getBufferIdentifier() + + "\""); + + // Read the CPU type from the header. + uint32_t CPUType; + memcpy(&CPUType, Data.data() + 4, sizeof(uint32_t)); + if (Magic == MachO::MH_CIGAM_64) + CPUType = ByteSwap_32(CPUType); + + LLVM_DEBUG({ + dbgs() << "jitLink_MachO: cputype = " << format("0x%08" PRIx32, CPUType) + << "\n"; + }); + + switch (CPUType) { + case MachO::CPU_TYPE_ARM64: + return createLinkGraphFromMachOObject_arm64(std::move(ObjectBuffer)); + case MachO::CPU_TYPE_X86_64: + return createLinkGraphFromMachOObject_x86_64(std::move(ObjectBuffer)); + } + return make_error<JITLinkError>("MachO-64 CPU type not valid"); + } else + return make_error<JITLinkError>("Unrecognized MachO magic value"); +} + +void link_MachO(std::unique_ptr<LinkGraph> G, + std::unique_ptr<JITLinkContext> Ctx) { + + switch (G->getTargetTriple().getArch()) { + case Triple::aarch64: + return link_MachO_arm64(std::move(G), std::move(Ctx)); + case Triple::x86_64: + return link_MachO_x86_64(std::move(G), std::move(Ctx)); + default: + Ctx->notifyFailed(make_error<JITLinkError>("MachO-64 CPU type not valid")); + return; + } +} + +} // end namespace jitlink +} // end namespace llvm diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp b/contrib/libs/llvm12/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp new file mode 100644 index 0000000000..4602154eb5 --- /dev/null +++ b/contrib/libs/llvm12/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp @@ -0,0 +1,583 @@ +//=--------- MachOLinkGraphBuilder.cpp - MachO LinkGraph builder ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Generic MachO LinkGraph buliding code. +// +//===----------------------------------------------------------------------===// + +#include "MachOLinkGraphBuilder.h" + +#define DEBUG_TYPE "jitlink" + +static const char *CommonSectionName = "__common"; + +namespace llvm { +namespace jitlink { + +MachOLinkGraphBuilder::~MachOLinkGraphBuilder() {} + +Expected<std::unique_ptr<LinkGraph>> MachOLinkGraphBuilder::buildGraph() { + + // Sanity check: we only operate on relocatable objects. + if (!Obj.isRelocatableObject()) + return make_error<JITLinkError>("Object is not a relocatable MachO"); + + if (auto Err = createNormalizedSections()) + return std::move(Err); + + if (auto Err = createNormalizedSymbols()) + return std::move(Err); + + if (auto Err = graphifyRegularSymbols()) + return std::move(Err); + + if (auto Err = graphifySectionsWithCustomParsers()) + return std::move(Err); + + if (auto Err = addRelocations()) + return std::move(Err); + + return std::move(G); +} + +MachOLinkGraphBuilder::MachOLinkGraphBuilder(const object::MachOObjectFile &Obj, + Triple TT) + : Obj(Obj), + G(std::make_unique<LinkGraph>(std::string(Obj.getFileName()), + std::move(TT), getPointerSize(Obj), + getEndianness(Obj))) {} + +void MachOLinkGraphBuilder::addCustomSectionParser( + StringRef SectionName, SectionParserFunction Parser) { + assert(!CustomSectionParserFunctions.count(SectionName) && + "Custom parser for this section already exists"); + CustomSectionParserFunctions[SectionName] = std::move(Parser); +} + +Linkage MachOLinkGraphBuilder::getLinkage(uint16_t Desc) { + if ((Desc & MachO::N_WEAK_DEF) || (Desc & MachO::N_WEAK_REF)) + return Linkage::Weak; + return Linkage::Strong; +} + +Scope MachOLinkGraphBuilder::getScope(StringRef Name, uint8_t Type) { + if (Type & MachO::N_EXT) { + if ((Type & MachO::N_PEXT) || Name.startswith("l")) + return Scope::Hidden; + else + return Scope::Default; + } + return Scope::Local; +} + +bool MachOLinkGraphBuilder::isAltEntry(const NormalizedSymbol &NSym) { + return NSym.Desc & MachO::N_ALT_ENTRY; +} + +bool MachOLinkGraphBuilder::isDebugSection(const NormalizedSection &NSec) { + return (NSec.Flags & MachO::S_ATTR_DEBUG && + strcmp(NSec.SegName, "__DWARF") == 0); +} + +unsigned +MachOLinkGraphBuilder::getPointerSize(const object::MachOObjectFile &Obj) { + return Obj.is64Bit() ? 8 : 4; +} + +support::endianness +MachOLinkGraphBuilder::getEndianness(const object::MachOObjectFile &Obj) { + return Obj.isLittleEndian() ? support::little : support::big; +} + +Section &MachOLinkGraphBuilder::getCommonSection() { + if (!CommonSection) { + auto Prot = static_cast<sys::Memory::ProtectionFlags>( + sys::Memory::MF_READ | sys::Memory::MF_WRITE); + CommonSection = &G->createSection(CommonSectionName, Prot); + } + return *CommonSection; +} + +Error MachOLinkGraphBuilder::createNormalizedSections() { + // Build normalized sections. Verifies that section data is in-range (for + // sections with content) and that address ranges are non-overlapping. + + LLVM_DEBUG(dbgs() << "Creating normalized sections...\n"); + + for (auto &SecRef : Obj.sections()) { + NormalizedSection NSec; + uint32_t DataOffset = 0; + + auto SecIndex = Obj.getSectionIndex(SecRef.getRawDataRefImpl()); + + auto Name = SecRef.getName(); + if (!Name) + return Name.takeError(); + + if (Obj.is64Bit()) { + const MachO::section_64 &Sec64 = + Obj.getSection64(SecRef.getRawDataRefImpl()); + + memcpy(&NSec.SectName, &Sec64.sectname, 16); + NSec.SectName[16] = '\0'; + memcpy(&NSec.SegName, Sec64.segname, 16); + NSec.SegName[16] = '\0'; + + NSec.Address = Sec64.addr; + NSec.Size = Sec64.size; + NSec.Alignment = 1ULL << Sec64.align; + NSec.Flags = Sec64.flags; + DataOffset = Sec64.offset; + } else { + const MachO::section &Sec32 = Obj.getSection(SecRef.getRawDataRefImpl()); + + memcpy(&NSec.SectName, &Sec32.sectname, 16); + NSec.SectName[16] = '\0'; + memcpy(&NSec.SegName, Sec32.segname, 16); + NSec.SegName[16] = '\0'; + + NSec.Address = Sec32.addr; + NSec.Size = Sec32.size; + NSec.Alignment = 1ULL << Sec32.align; + NSec.Flags = Sec32.flags; + DataOffset = Sec32.offset; + } + + LLVM_DEBUG({ + dbgs() << " " << *Name << ": " << formatv("{0:x16}", NSec.Address) + << " -- " << formatv("{0:x16}", NSec.Address + NSec.Size) + << ", align: " << NSec.Alignment << ", index: " << SecIndex + << "\n"; + }); + + // Get the section data if any. + { + unsigned SectionType = NSec.Flags & MachO::SECTION_TYPE; + if (SectionType != MachO::S_ZEROFILL && + SectionType != MachO::S_GB_ZEROFILL) { + + if (DataOffset + NSec.Size > Obj.getData().size()) + return make_error<JITLinkError>( + "Section data extends past end of file"); + + NSec.Data = Obj.getData().data() + DataOffset; + } + } + + // Get prot flags. + // FIXME: Make sure this test is correct (it's probably missing cases + // as-is). + sys::Memory::ProtectionFlags Prot; + if (NSec.Flags & MachO::S_ATTR_PURE_INSTRUCTIONS) + Prot = static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | + sys::Memory::MF_EXEC); + else + Prot = static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | + sys::Memory::MF_WRITE); + + if (!isDebugSection(NSec)) + NSec.GraphSection = &G->createSection(*Name, Prot); + else + LLVM_DEBUG({ + dbgs() << " " << *Name + << " is a debug section: No graph section will be created.\n"; + }); + + IndexToSection.insert(std::make_pair(SecIndex, std::move(NSec))); + } + + std::vector<NormalizedSection *> Sections; + Sections.reserve(IndexToSection.size()); + for (auto &KV : IndexToSection) + Sections.push_back(&KV.second); + + // If we didn't end up creating any sections then bail out. The code below + // assumes that we have at least one section. + if (Sections.empty()) + return Error::success(); + + llvm::sort(Sections, + [](const NormalizedSection *LHS, const NormalizedSection *RHS) { + assert(LHS && RHS && "Null section?"); + if (LHS->Address != RHS->Address) + return LHS->Address < RHS->Address; + return LHS->Size < RHS->Size; + }); + + for (unsigned I = 0, E = Sections.size() - 1; I != E; ++I) { + auto &Cur = *Sections[I]; + auto &Next = *Sections[I + 1]; + if (Next.Address < Cur.Address + Cur.Size) + return make_error<JITLinkError>( + "Address range for section " + + formatv("\"{0}/{1}\" [ {2:x16} -- {3:x16} ] ", Cur.SegName, + Cur.SectName, Cur.Address, Cur.Address + Cur.Size) + + "overlaps section \"" + Next.SegName + "/" + Next.SectName + "\"" + + formatv("\"{0}/{1}\" [ {2:x16} -- {3:x16} ] ", Next.SegName, + Next.SectName, Next.Address, Next.Address + Next.Size)); + } + + return Error::success(); +} + +Error MachOLinkGraphBuilder::createNormalizedSymbols() { + LLVM_DEBUG(dbgs() << "Creating normalized symbols...\n"); + + for (auto &SymRef : Obj.symbols()) { + + unsigned SymbolIndex = Obj.getSymbolIndex(SymRef.getRawDataRefImpl()); + uint64_t Value; + uint32_t NStrX; + uint8_t Type; + uint8_t Sect; + uint16_t Desc; + + if (Obj.is64Bit()) { + const MachO::nlist_64 &NL64 = + Obj.getSymbol64TableEntry(SymRef.getRawDataRefImpl()); + Value = NL64.n_value; + NStrX = NL64.n_strx; + Type = NL64.n_type; + Sect = NL64.n_sect; + Desc = NL64.n_desc; + } else { + const MachO::nlist &NL32 = + Obj.getSymbolTableEntry(SymRef.getRawDataRefImpl()); + Value = NL32.n_value; + NStrX = NL32.n_strx; + Type = NL32.n_type; + Sect = NL32.n_sect; + Desc = NL32.n_desc; + } + + // Skip stabs. + // FIXME: Are there other symbols we should be skipping? + if (Type & MachO::N_STAB) + continue; + + Optional<StringRef> Name; + if (NStrX) { + if (auto NameOrErr = SymRef.getName()) + Name = *NameOrErr; + else + return NameOrErr.takeError(); + } + + LLVM_DEBUG({ + dbgs() << " "; + if (!Name) + dbgs() << "<anonymous symbol>"; + else + dbgs() << *Name; + dbgs() << ": value = " << formatv("{0:x16}", Value) + << ", type = " << formatv("{0:x2}", Type) + << ", desc = " << formatv("{0:x4}", Desc) << ", sect = "; + if (Sect) + dbgs() << static_cast<unsigned>(Sect - 1); + else + dbgs() << "none"; + dbgs() << "\n"; + }); + + // If this symbol has a section, sanity check that the addresses line up. + if (Sect != 0) { + auto NSec = findSectionByIndex(Sect - 1); + if (!NSec) + return NSec.takeError(); + + if (Value < NSec->Address || Value > NSec->Address + NSec->Size) + return make_error<JITLinkError>("Symbol address does not fall within " + "section"); + + if (!NSec->GraphSection) { + LLVM_DEBUG({ + dbgs() << " Skipping: Symbol is in section " << NSec->SegName << "/" + << NSec->SectName + << " which has no associated graph section.\n"; + }); + continue; + } + } + + IndexToSymbol[SymbolIndex] = + &createNormalizedSymbol(*Name, Value, Type, Sect, Desc, + getLinkage(Desc), getScope(*Name, Type)); + } + + return Error::success(); +} + +void MachOLinkGraphBuilder::addSectionStartSymAndBlock( + Section &GraphSec, uint64_t Address, const char *Data, uint64_t Size, + uint32_t Alignment, bool IsLive) { + Block &B = + Data ? G->createContentBlock(GraphSec, StringRef(Data, Size), Address, + Alignment, 0) + : G->createZeroFillBlock(GraphSec, Size, Address, Alignment, 0); + auto &Sym = G->addAnonymousSymbol(B, 0, Size, false, IsLive); + assert(!AddrToCanonicalSymbol.count(Sym.getAddress()) && + "Anonymous block start symbol clashes with existing symbol address"); + AddrToCanonicalSymbol[Sym.getAddress()] = &Sym; +} + +Error MachOLinkGraphBuilder::graphifyRegularSymbols() { + + LLVM_DEBUG(dbgs() << "Creating graph symbols...\n"); + + /// We only have 256 section indexes: Use a vector rather than a map. + std::vector<std::vector<NormalizedSymbol *>> SecIndexToSymbols; + SecIndexToSymbols.resize(256); + + // Create commons, externs, and absolutes, and partition all other symbols by + // section. + for (auto &KV : IndexToSymbol) { + auto &NSym = *KV.second; + + switch (NSym.Type & MachO::N_TYPE) { + case MachO::N_UNDF: + if (NSym.Value) { + if (!NSym.Name) + return make_error<JITLinkError>("Anonymous common symbol at index " + + Twine(KV.first)); + NSym.GraphSymbol = &G->addCommonSymbol( + *NSym.Name, NSym.S, getCommonSection(), 0, NSym.Value, + 1ull << MachO::GET_COMM_ALIGN(NSym.Desc), + NSym.Desc & MachO::N_NO_DEAD_STRIP); + } else { + if (!NSym.Name) + return make_error<JITLinkError>("Anonymous external symbol at " + "index " + + Twine(KV.first)); + NSym.GraphSymbol = &G->addExternalSymbol( + *NSym.Name, 0, + NSym.Desc & MachO::N_WEAK_REF ? Linkage::Weak : Linkage::Strong); + } + break; + case MachO::N_ABS: + if (!NSym.Name) + return make_error<JITLinkError>("Anonymous absolute symbol at index " + + Twine(KV.first)); + NSym.GraphSymbol = &G->addAbsoluteSymbol( + *NSym.Name, NSym.Value, 0, Linkage::Strong, Scope::Default, + NSym.Desc & MachO::N_NO_DEAD_STRIP); + break; + case MachO::N_SECT: + SecIndexToSymbols[NSym.Sect - 1].push_back(&NSym); + break; + case MachO::N_PBUD: + return make_error<JITLinkError>( + "Unupported N_PBUD symbol " + + (NSym.Name ? ("\"" + *NSym.Name + "\"") : Twine("<anon>")) + + " at index " + Twine(KV.first)); + case MachO::N_INDR: + return make_error<JITLinkError>( + "Unupported N_INDR symbol " + + (NSym.Name ? ("\"" + *NSym.Name + "\"") : Twine("<anon>")) + + " at index " + Twine(KV.first)); + default: + return make_error<JITLinkError>( + "Unrecognized symbol type " + Twine(NSym.Type & MachO::N_TYPE) + + " for symbol " + + (NSym.Name ? ("\"" + *NSym.Name + "\"") : Twine("<anon>")) + + " at index " + Twine(KV.first)); + } + } + + // Loop over sections performing regular graphification for those that + // don't have custom parsers. + for (auto &KV : IndexToSection) { + auto SecIndex = KV.first; + auto &NSec = KV.second; + + if (!NSec.GraphSection) { + LLVM_DEBUG({ + dbgs() << " " << NSec.SegName << "/" << NSec.SectName + << " has no graph section. Skipping.\n"; + }); + continue; + } + + // Skip sections with custom parsers. + if (CustomSectionParserFunctions.count(NSec.GraphSection->getName())) { + LLVM_DEBUG({ + dbgs() << " Skipping section " << NSec.GraphSection->getName() + << " as it has a custom parser.\n"; + }); + continue; + } else + LLVM_DEBUG({ + dbgs() << " Processing section " << NSec.GraphSection->getName() + << "...\n"; + }); + + bool SectionIsNoDeadStrip = NSec.Flags & MachO::S_ATTR_NO_DEAD_STRIP; + bool SectionIsText = NSec.Flags & MachO::S_ATTR_PURE_INSTRUCTIONS; + + auto &SecNSymStack = SecIndexToSymbols[SecIndex]; + + // If this section is non-empty but there are no symbols covering it then + // create one block and anonymous symbol to cover the entire section. + if (SecNSymStack.empty()) { + if (NSec.Size > 0) { + LLVM_DEBUG({ + dbgs() << " Section non-empty, but contains no symbols. " + "Creating anonymous block to cover " + << formatv("{0:x16}", NSec.Address) << " -- " + << formatv("{0:x16}", NSec.Address + NSec.Size) << "\n"; + }); + addSectionStartSymAndBlock(*NSec.GraphSection, NSec.Address, NSec.Data, + NSec.Size, NSec.Alignment, + SectionIsNoDeadStrip); + } else + LLVM_DEBUG({ + dbgs() << " Section empty and contains no symbols. Skipping.\n"; + }); + continue; + } + + // Sort the symbol stack in by address, alt-entry status, scope, and name. + // We sort in reverse order so that symbols will be visited in the right + // order when we pop off the stack below. + llvm::sort(SecNSymStack, [](const NormalizedSymbol *LHS, + const NormalizedSymbol *RHS) { + if (LHS->Value != RHS->Value) + return LHS->Value > RHS->Value; + if (isAltEntry(*LHS) != isAltEntry(*RHS)) + return isAltEntry(*RHS); + if (LHS->S != RHS->S) + return static_cast<uint8_t>(LHS->S) < static_cast<uint8_t>(RHS->S); + return LHS->Name < RHS->Name; + }); + + // The first symbol in a section can not be an alt-entry symbol. + if (!SecNSymStack.empty() && isAltEntry(*SecNSymStack.back())) + return make_error<JITLinkError>( + "First symbol in " + NSec.GraphSection->getName() + " is alt-entry"); + + // If the section is non-empty but there is no symbol covering the start + // address then add an anonymous one. + if (SecNSymStack.back()->Value != NSec.Address) { + auto AnonBlockSize = SecNSymStack.back()->Value - NSec.Address; + LLVM_DEBUG({ + dbgs() << " Section start not covered by symbol. " + << "Creating anonymous block to cover [ " + << formatv("{0:x16}", NSec.Address) << " -- " + << formatv("{0:x16}", NSec.Address + AnonBlockSize) << " ]\n"; + }); + addSectionStartSymAndBlock(*NSec.GraphSection, NSec.Address, NSec.Data, + AnonBlockSize, NSec.Alignment, + SectionIsNoDeadStrip); + } + + // Visit section symbols in order by popping off the reverse-sorted stack, + // building blocks for each alt-entry chain and creating symbols as we go. + while (!SecNSymStack.empty()) { + SmallVector<NormalizedSymbol *, 8> BlockSyms; + + BlockSyms.push_back(SecNSymStack.back()); + SecNSymStack.pop_back(); + while (!SecNSymStack.empty() && + (isAltEntry(*SecNSymStack.back()) || + SecNSymStack.back()->Value == BlockSyms.back()->Value)) { + BlockSyms.push_back(SecNSymStack.back()); + SecNSymStack.pop_back(); + } + + // BlockNSyms now contains the block symbols in reverse canonical order. + JITTargetAddress BlockStart = BlockSyms.front()->Value; + JITTargetAddress BlockEnd = SecNSymStack.empty() + ? NSec.Address + NSec.Size + : SecNSymStack.back()->Value; + JITTargetAddress BlockOffset = BlockStart - NSec.Address; + JITTargetAddress BlockSize = BlockEnd - BlockStart; + + LLVM_DEBUG({ + dbgs() << " Creating block for " << formatv("{0:x16}", BlockStart) + << " -- " << formatv("{0:x16}", BlockEnd) << ": " + << NSec.GraphSection->getName() << " + " + << formatv("{0:x16}", BlockOffset) << " with " + << BlockSyms.size() << " symbol(s)...\n"; + }); + + Block &B = + NSec.Data + ? G->createContentBlock( + *NSec.GraphSection, + StringRef(NSec.Data + BlockOffset, BlockSize), BlockStart, + NSec.Alignment, BlockStart % NSec.Alignment) + : G->createZeroFillBlock(*NSec.GraphSection, BlockSize, + BlockStart, NSec.Alignment, + BlockStart % NSec.Alignment); + + Optional<JITTargetAddress> LastCanonicalAddr; + JITTargetAddress SymEnd = BlockEnd; + while (!BlockSyms.empty()) { + auto &NSym = *BlockSyms.back(); + BlockSyms.pop_back(); + + bool SymLive = + (NSym.Desc & MachO::N_NO_DEAD_STRIP) || SectionIsNoDeadStrip; + + LLVM_DEBUG({ + dbgs() << " " << formatv("{0:x16}", NSym.Value) << " -- " + << formatv("{0:x16}", SymEnd) << ": "; + if (!NSym.Name) + dbgs() << "<anonymous symbol>"; + else + dbgs() << NSym.Name; + if (SymLive) + dbgs() << " [no-dead-strip]"; + if (LastCanonicalAddr == NSym.Value) + dbgs() << " [non-canonical]"; + dbgs() << "\n"; + }); + + auto &Sym = + NSym.Name + ? G->addDefinedSymbol(B, NSym.Value - BlockStart, *NSym.Name, + SymEnd - NSym.Value, NSym.L, NSym.S, + SectionIsText, SymLive) + : G->addAnonymousSymbol(B, NSym.Value - BlockStart, + SymEnd - NSym.Value, SectionIsText, + SymLive); + NSym.GraphSymbol = &Sym; + if (LastCanonicalAddr != Sym.getAddress()) { + if (LastCanonicalAddr) + SymEnd = *LastCanonicalAddr; + LastCanonicalAddr = Sym.getAddress(); + setCanonicalSymbol(Sym); + } + } + } + } + + return Error::success(); +} + +Error MachOLinkGraphBuilder::graphifySectionsWithCustomParsers() { + // Graphify special sections. + for (auto &KV : IndexToSection) { + auto &NSec = KV.second; + + // Skip non-graph sections. + if (!NSec.GraphSection) + continue; + + auto HI = CustomSectionParserFunctions.find(NSec.GraphSection->getName()); + if (HI != CustomSectionParserFunctions.end()) { + auto &Parse = HI->second; + if (auto Err = Parse(NSec)) + return Err; + } + } + + return Error::success(); +} + +} // end namespace jitlink +} // end namespace llvm diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h b/contrib/libs/llvm12/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h new file mode 100644 index 0000000000..26e6859de9 --- /dev/null +++ b/contrib/libs/llvm12/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h @@ -0,0 +1,223 @@ +//===----- MachOLinkGraphBuilder.h - MachO LinkGraph builder ----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Generic MachO LinkGraph building code. +// +//===----------------------------------------------------------------------===// + +#ifndef LIB_EXECUTIONENGINE_JITLINK_MACHOLINKGRAPHBUILDER_H +#define LIB_EXECUTIONENGINE_JITLINK_MACHOLINKGRAPHBUILDER_H + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ExecutionEngine/JITLink/JITLink.h" +#include "llvm/Object/MachO.h" + +#include "EHFrameSupportImpl.h" +#include "JITLinkGeneric.h" + +#include <list> + +namespace llvm { +namespace jitlink { + +class MachOLinkGraphBuilder { +public: + virtual ~MachOLinkGraphBuilder(); + Expected<std::unique_ptr<LinkGraph>> buildGraph(); + +protected: + + struct NormalizedSymbol { + friend class MachOLinkGraphBuilder; + + private: + NormalizedSymbol(Optional<StringRef> Name, uint64_t Value, uint8_t Type, + uint8_t Sect, uint16_t Desc, Linkage L, Scope S) + : Name(Name), Value(Value), Type(Type), Sect(Sect), Desc(Desc), L(L), + S(S) { + assert((!Name || !Name->empty()) && "Name must be none or non-empty"); + } + + public: + NormalizedSymbol(const NormalizedSymbol &) = delete; + NormalizedSymbol &operator=(const NormalizedSymbol &) = delete; + NormalizedSymbol(NormalizedSymbol &&) = delete; + NormalizedSymbol &operator=(NormalizedSymbol &&) = delete; + + Optional<StringRef> Name; + uint64_t Value = 0; + uint8_t Type = 0; + uint8_t Sect = 0; + uint16_t Desc = 0; + Linkage L = Linkage::Strong; + Scope S = Scope::Default; + Symbol *GraphSymbol = nullptr; + }; + + // Normalized section representation. Section and segment names are guaranteed + // to be null-terminated, hence the extra bytes on SegName and SectName. + class NormalizedSection { + friend class MachOLinkGraphBuilder; + + private: + NormalizedSection() = default; + + public: + char SectName[17]; + char SegName[17]; + uint64_t Address = 0; + uint64_t Size = 0; + uint64_t Alignment = 0; + uint32_t Flags = 0; + const char *Data = nullptr; + Section *GraphSection = nullptr; + }; + + using SectionParserFunction = std::function<Error(NormalizedSection &S)>; + + MachOLinkGraphBuilder(const object::MachOObjectFile &Obj, Triple TT); + + LinkGraph &getGraph() const { return *G; } + + const object::MachOObjectFile &getObject() const { return Obj; } + + void addCustomSectionParser(StringRef SectionName, + SectionParserFunction Parse); + + virtual Error addRelocations() = 0; + + /// Create a symbol. + template <typename... ArgTs> + NormalizedSymbol &createNormalizedSymbol(ArgTs &&... Args) { + NormalizedSymbol *Sym = reinterpret_cast<NormalizedSymbol *>( + Allocator.Allocate<NormalizedSymbol>()); + new (Sym) NormalizedSymbol(std::forward<ArgTs>(Args)...); + return *Sym; + } + + /// Index is zero-based (MachO section indexes are usually one-based) and + /// assumed to be in-range. Client is responsible for checking. + NormalizedSection &getSectionByIndex(unsigned Index) { + auto I = IndexToSection.find(Index); + assert(I != IndexToSection.end() && "No section recorded at index"); + return I->second; + } + + /// Try to get the section at the given index. Will return an error if the + /// given index is out of range, or if no section has been added for the given + /// index. + Expected<NormalizedSection &> findSectionByIndex(unsigned Index) { + auto I = IndexToSection.find(Index); + if (I == IndexToSection.end()) + return make_error<JITLinkError>("No section recorded for index " + + formatv("{0:d}", Index)); + return I->second; + } + + /// Try to get the symbol at the given index. Will return an error if the + /// given index is out of range, or if no symbol has been added for the given + /// index. + Expected<NormalizedSymbol &> findSymbolByIndex(uint64_t Index) { + if (Index >= IndexToSymbol.size()) + return make_error<JITLinkError>("Symbol index out of range"); + auto *Sym = IndexToSymbol[Index]; + if (!Sym) + return make_error<JITLinkError>("No symbol at index " + + formatv("{0:d}", Index)); + return *Sym; + } + + /// Returns the symbol with the highest address not greater than the search + /// address, or null if no such symbol exists. + Symbol *getSymbolByAddress(JITTargetAddress Address) { + auto I = AddrToCanonicalSymbol.upper_bound(Address); + if (I == AddrToCanonicalSymbol.begin()) + return nullptr; + return std::prev(I)->second; + } + + /// Returns the symbol with the highest address not greater than the search + /// address, or an error if no such symbol exists. + Expected<Symbol &> findSymbolByAddress(JITTargetAddress Address) { + auto *Sym = getSymbolByAddress(Address); + if (Sym) + if (Address < Sym->getAddress() + Sym->getSize()) + return *Sym; + return make_error<JITLinkError>("No symbol covering address " + + formatv("{0:x16}", Address)); + } + + static Linkage getLinkage(uint16_t Desc); + static Scope getScope(StringRef Name, uint8_t Type); + static bool isAltEntry(const NormalizedSymbol &NSym); + + static bool isDebugSection(const NormalizedSection &NSec); + + MachO::relocation_info + getRelocationInfo(const object::relocation_iterator RelItr) { + MachO::any_relocation_info ARI = + getObject().getRelocation(RelItr->getRawDataRefImpl()); + MachO::relocation_info RI; + RI.r_address = ARI.r_word0; + RI.r_symbolnum = ARI.r_word1 & 0xffffff; + RI.r_pcrel = (ARI.r_word1 >> 24) & 1; + RI.r_length = (ARI.r_word1 >> 25) & 3; + RI.r_extern = (ARI.r_word1 >> 27) & 1; + RI.r_type = (ARI.r_word1 >> 28); + return RI; + } + +private: + static unsigned getPointerSize(const object::MachOObjectFile &Obj); + static support::endianness getEndianness(const object::MachOObjectFile &Obj); + + void setCanonicalSymbol(Symbol &Sym) { + auto *&CanonicalSymEntry = AddrToCanonicalSymbol[Sym.getAddress()]; + // There should be no symbol at this address, or, if there is, + // it should be a zero-sized symbol from an empty section (which + // we can safely override). + assert((!CanonicalSymEntry || CanonicalSymEntry->getSize() == 0) && + "Duplicate canonical symbol at address"); + CanonicalSymEntry = &Sym; + } + + Section &getCommonSection(); + void addSectionStartSymAndBlock(Section &GraphSec, uint64_t Address, + const char *Data, uint64_t Size, + uint32_t Alignment, bool IsLive); + + Error createNormalizedSections(); + Error createNormalizedSymbols(); + + /// Create graph blocks and symbols for externals, absolutes, commons and + /// all defined symbols in sections without custom parsers. + Error graphifyRegularSymbols(); + + /// Create graph blocks and symbols for all sections. + Error graphifySectionsWithCustomParsers(); + + // Put the BumpPtrAllocator first so that we don't free any of the underlying + // memory until the Symbol/Addressable destructors have been run. + BumpPtrAllocator Allocator; + + const object::MachOObjectFile &Obj; + std::unique_ptr<LinkGraph> G; + + DenseMap<unsigned, NormalizedSection> IndexToSection; + Section *CommonSection = nullptr; + + DenseMap<uint32_t, NormalizedSymbol *> IndexToSymbol; + std::map<JITTargetAddress, Symbol *> AddrToCanonicalSymbol; + StringMap<SectionParserFunction> CustomSectionParserFunctions; +}; + +} // end namespace jitlink +} // end namespace llvm + +#endif // LIB_EXECUTIONENGINE_JITLINK_MACHOLINKGRAPHBUILDER_H diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/JITLink/MachO_arm64.cpp b/contrib/libs/llvm12/lib/ExecutionEngine/JITLink/MachO_arm64.cpp new file mode 100644 index 0000000000..8366e96585 --- /dev/null +++ b/contrib/libs/llvm12/lib/ExecutionEngine/JITLink/MachO_arm64.cpp @@ -0,0 +1,747 @@ +//===---- MachO_arm64.cpp - JIT linker implementation for MachO/arm64 -----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// MachO/arm64 jit-link implementation. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/JITLink/MachO_arm64.h" + +#include "BasicGOTAndStubsBuilder.h" +#include "MachOLinkGraphBuilder.h" + +#define DEBUG_TYPE "jitlink" + +using namespace llvm; +using namespace llvm::jitlink; +using namespace llvm::jitlink::MachO_arm64_Edges; + +namespace { + +class MachOLinkGraphBuilder_arm64 : public MachOLinkGraphBuilder { +public: + MachOLinkGraphBuilder_arm64(const object::MachOObjectFile &Obj) + : MachOLinkGraphBuilder(Obj, Triple("arm64-apple-darwin")), + NumSymbols(Obj.getSymtabLoadCommand().nsyms) {} + +private: + static Expected<MachOARM64RelocationKind> + getRelocationKind(const MachO::relocation_info &RI) { + switch (RI.r_type) { + case MachO::ARM64_RELOC_UNSIGNED: + if (!RI.r_pcrel) { + if (RI.r_length == 3) + return RI.r_extern ? Pointer64 : Pointer64Anon; + else if (RI.r_length == 2) + return Pointer32; + } + break; + case MachO::ARM64_RELOC_SUBTRACTOR: + // SUBTRACTOR must be non-pc-rel, extern, with length 2 or 3. + // Initially represent SUBTRACTOR relocations with 'Delta<W>'. + // They may be turned into NegDelta<W> by parsePairRelocation. + if (!RI.r_pcrel && RI.r_extern) { + if (RI.r_length == 2) + return Delta32; + else if (RI.r_length == 3) + return Delta64; + } + break; + case MachO::ARM64_RELOC_BRANCH26: + if (RI.r_pcrel && RI.r_extern && RI.r_length == 2) + return Branch26; + break; + case MachO::ARM64_RELOC_PAGE21: + if (RI.r_pcrel && RI.r_extern && RI.r_length == 2) + return Page21; + break; + case MachO::ARM64_RELOC_PAGEOFF12: + if (!RI.r_pcrel && RI.r_extern && RI.r_length == 2) + return PageOffset12; + break; + case MachO::ARM64_RELOC_GOT_LOAD_PAGE21: + if (RI.r_pcrel && RI.r_extern && RI.r_length == 2) + return GOTPage21; + break; + case MachO::ARM64_RELOC_GOT_LOAD_PAGEOFF12: + if (!RI.r_pcrel && RI.r_extern && RI.r_length == 2) + return GOTPageOffset12; + break; + case MachO::ARM64_RELOC_POINTER_TO_GOT: + if (RI.r_pcrel && RI.r_extern && RI.r_length == 2) + return PointerToGOT; + break; + case MachO::ARM64_RELOC_ADDEND: + if (!RI.r_pcrel && !RI.r_extern && RI.r_length == 2) + return PairedAddend; + break; + } + + return make_error<JITLinkError>( + "Unsupported arm64 relocation: address=" + + formatv("{0:x8}", RI.r_address) + + ", symbolnum=" + formatv("{0:x6}", RI.r_symbolnum) + + ", kind=" + formatv("{0:x1}", RI.r_type) + + ", pc_rel=" + (RI.r_pcrel ? "true" : "false") + + ", extern=" + (RI.r_extern ? "true" : "false") + + ", length=" + formatv("{0:d}", RI.r_length)); + } + + using PairRelocInfo = + std::tuple<MachOARM64RelocationKind, Symbol *, uint64_t>; + + // Parses paired SUBTRACTOR/UNSIGNED relocations and, on success, + // returns the edge kind and addend to be used. + Expected<PairRelocInfo> + parsePairRelocation(Block &BlockToFix, Edge::Kind SubtractorKind, + const MachO::relocation_info &SubRI, + JITTargetAddress FixupAddress, const char *FixupContent, + object::relocation_iterator &UnsignedRelItr, + object::relocation_iterator &RelEnd) { + using namespace support; + + assert(((SubtractorKind == Delta32 && SubRI.r_length == 2) || + (SubtractorKind == Delta64 && SubRI.r_length == 3)) && + "Subtractor kind should match length"); + assert(SubRI.r_extern && "SUBTRACTOR reloc symbol should be extern"); + assert(!SubRI.r_pcrel && "SUBTRACTOR reloc should not be PCRel"); + + if (UnsignedRelItr == RelEnd) + return make_error<JITLinkError>("arm64 SUBTRACTOR without paired " + "UNSIGNED relocation"); + + auto UnsignedRI = getRelocationInfo(UnsignedRelItr); + + if (SubRI.r_address != UnsignedRI.r_address) + return make_error<JITLinkError>("arm64 SUBTRACTOR and paired UNSIGNED " + "point to different addresses"); + + if (SubRI.r_length != UnsignedRI.r_length) + return make_error<JITLinkError>("length of arm64 SUBTRACTOR and paired " + "UNSIGNED reloc must match"); + + Symbol *FromSymbol; + if (auto FromSymbolOrErr = findSymbolByIndex(SubRI.r_symbolnum)) + FromSymbol = FromSymbolOrErr->GraphSymbol; + else + return FromSymbolOrErr.takeError(); + + // Read the current fixup value. + uint64_t FixupValue = 0; + if (SubRI.r_length == 3) + FixupValue = *(const little64_t *)FixupContent; + else + FixupValue = *(const little32_t *)FixupContent; + + // Find 'ToSymbol' using symbol number or address, depending on whether the + // paired UNSIGNED relocation is extern. + Symbol *ToSymbol = nullptr; + if (UnsignedRI.r_extern) { + // Find target symbol by symbol index. + if (auto ToSymbolOrErr = findSymbolByIndex(UnsignedRI.r_symbolnum)) + ToSymbol = ToSymbolOrErr->GraphSymbol; + else + return ToSymbolOrErr.takeError(); + } else { + auto ToSymbolSec = findSectionByIndex(UnsignedRI.r_symbolnum - 1); + if (!ToSymbolSec) + return ToSymbolSec.takeError(); + ToSymbol = getSymbolByAddress(ToSymbolSec->Address); + assert(ToSymbol && "No symbol for section"); + FixupValue -= ToSymbol->getAddress(); + } + + MachOARM64RelocationKind DeltaKind; + Symbol *TargetSymbol; + uint64_t Addend; + if (&BlockToFix == &FromSymbol->getAddressable()) { + TargetSymbol = ToSymbol; + DeltaKind = (SubRI.r_length == 3) ? Delta64 : Delta32; + Addend = FixupValue + (FixupAddress - FromSymbol->getAddress()); + // FIXME: handle extern 'from'. + } else if (&BlockToFix == &ToSymbol->getAddressable()) { + TargetSymbol = &*FromSymbol; + DeltaKind = (SubRI.r_length == 3) ? NegDelta64 : NegDelta32; + Addend = FixupValue - (FixupAddress - ToSymbol->getAddress()); + } else { + // BlockToFix was neither FromSymbol nor ToSymbol. + return make_error<JITLinkError>("SUBTRACTOR relocation must fix up " + "either 'A' or 'B' (or a symbol in one " + "of their alt-entry groups)"); + } + + return PairRelocInfo(DeltaKind, TargetSymbol, Addend); + } + + Error addRelocations() override { + using namespace support; + auto &Obj = getObject(); + + LLVM_DEBUG(dbgs() << "Processing relocations:\n"); + + for (auto &S : Obj.sections()) { + + JITTargetAddress SectionAddress = S.getAddress(); + + // Skip relocations virtual sections. + if (S.isVirtual()) { + if (S.relocation_begin() != S.relocation_end()) + return make_error<JITLinkError>("Virtual section contains " + "relocations"); + continue; + } + + // Skip relocations for debug symbols. + { + auto &NSec = + getSectionByIndex(Obj.getSectionIndex(S.getRawDataRefImpl())); + if (!NSec.GraphSection) { + LLVM_DEBUG({ + dbgs() << " Skipping relocations for MachO section " + << NSec.SegName << "/" << NSec.SectName + << " which has no associated graph section\n"; + }); + continue; + } + } + + for (auto RelItr = S.relocation_begin(), RelEnd = S.relocation_end(); + RelItr != RelEnd; ++RelItr) { + + MachO::relocation_info RI = getRelocationInfo(RelItr); + + // Sanity check the relocation kind. + auto Kind = getRelocationKind(RI); + if (!Kind) + return Kind.takeError(); + + // Find the address of the value to fix up. + JITTargetAddress FixupAddress = SectionAddress + (uint32_t)RI.r_address; + + LLVM_DEBUG({ + auto &NSec = + getSectionByIndex(Obj.getSectionIndex(S.getRawDataRefImpl())); + dbgs() << " " << NSec.SectName << " + " + << formatv("{0:x8}", RI.r_address) << ":\n"; + }); + + // Find the block that the fixup points to. + Block *BlockToFix = nullptr; + { + auto SymbolToFixOrErr = findSymbolByAddress(FixupAddress); + if (!SymbolToFixOrErr) + return SymbolToFixOrErr.takeError(); + BlockToFix = &SymbolToFixOrErr->getBlock(); + } + + if (FixupAddress + static_cast<JITTargetAddress>(1ULL << RI.r_length) > + BlockToFix->getAddress() + BlockToFix->getContent().size()) + return make_error<JITLinkError>( + "Relocation content extends past end of fixup block"); + + // Get a pointer to the fixup content. + const char *FixupContent = BlockToFix->getContent().data() + + (FixupAddress - BlockToFix->getAddress()); + + // The target symbol and addend will be populated by the switch below. + Symbol *TargetSymbol = nullptr; + uint64_t Addend = 0; + + if (*Kind == PairedAddend) { + // If this is an Addend relocation then process it and move to the + // paired reloc. + + Addend = SignExtend64(RI.r_symbolnum, 24); + + if (RelItr == RelEnd) + return make_error<JITLinkError>("Unpaired Addend reloc at " + + formatv("{0:x16}", FixupAddress)); + ++RelItr; + RI = getRelocationInfo(RelItr); + + Kind = getRelocationKind(RI); + if (!Kind) + return Kind.takeError(); + + if (*Kind != Branch26 && *Kind != Page21 && *Kind != PageOffset12) + return make_error<JITLinkError>( + "Invalid relocation pair: Addend + " + + getMachOARM64RelocationKindName(*Kind)); + + LLVM_DEBUG({ + dbgs() << " Addend: value = " << formatv("{0:x6}", Addend) + << ", pair is " << getMachOARM64RelocationKindName(*Kind) + << "\n"; + }); + + // Find the address of the value to fix up. + JITTargetAddress PairedFixupAddress = + SectionAddress + (uint32_t)RI.r_address; + if (PairedFixupAddress != FixupAddress) + return make_error<JITLinkError>("Paired relocation points at " + "different target"); + } + + switch (*Kind) { + case Branch26: { + if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) + TargetSymbol = TargetSymbolOrErr->GraphSymbol; + else + return TargetSymbolOrErr.takeError(); + uint32_t Instr = *(const ulittle32_t *)FixupContent; + if ((Instr & 0x7fffffff) != 0x14000000) + return make_error<JITLinkError>("BRANCH26 target is not a B or BL " + "instruction with a zero addend"); + break; + } + case Pointer32: + if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) + TargetSymbol = TargetSymbolOrErr->GraphSymbol; + else + return TargetSymbolOrErr.takeError(); + Addend = *(const ulittle32_t *)FixupContent; + break; + case Pointer64: + if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) + TargetSymbol = TargetSymbolOrErr->GraphSymbol; + else + return TargetSymbolOrErr.takeError(); + Addend = *(const ulittle64_t *)FixupContent; + break; + case Pointer64Anon: { + JITTargetAddress TargetAddress = *(const ulittle64_t *)FixupContent; + if (auto TargetSymbolOrErr = findSymbolByAddress(TargetAddress)) + TargetSymbol = &*TargetSymbolOrErr; + else + return TargetSymbolOrErr.takeError(); + Addend = TargetAddress - TargetSymbol->getAddress(); + break; + } + case Page21: + case GOTPage21: { + if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) + TargetSymbol = TargetSymbolOrErr->GraphSymbol; + else + return TargetSymbolOrErr.takeError(); + uint32_t Instr = *(const ulittle32_t *)FixupContent; + if ((Instr & 0xffffffe0) != 0x90000000) + return make_error<JITLinkError>("PAGE21/GOTPAGE21 target is not an " + "ADRP instruction with a zero " + "addend"); + break; + } + case PageOffset12: { + if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) + TargetSymbol = TargetSymbolOrErr->GraphSymbol; + else + return TargetSymbolOrErr.takeError(); + uint32_t Instr = *(const ulittle32_t *)FixupContent; + uint32_t EncodedAddend = (Instr & 0x003FFC00) >> 10; + if (EncodedAddend != 0) + return make_error<JITLinkError>("GOTPAGEOFF12 target has non-zero " + "encoded addend"); + break; + } + case GOTPageOffset12: { + if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) + TargetSymbol = TargetSymbolOrErr->GraphSymbol; + else + return TargetSymbolOrErr.takeError(); + uint32_t Instr = *(const ulittle32_t *)FixupContent; + if ((Instr & 0xfffffc00) != 0xf9400000) + return make_error<JITLinkError>("GOTPAGEOFF12 target is not an LDR " + "immediate instruction with a zero " + "addend"); + break; + } + case PointerToGOT: + if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) + TargetSymbol = TargetSymbolOrErr->GraphSymbol; + else + return TargetSymbolOrErr.takeError(); + break; + case Delta32: + case Delta64: { + // We use Delta32/Delta64 to represent SUBTRACTOR relocations. + // parsePairRelocation handles the paired reloc, and returns the + // edge kind to be used (either Delta32/Delta64, or + // NegDelta32/NegDelta64, depending on the direction of the + // subtraction) along with the addend. + auto PairInfo = + parsePairRelocation(*BlockToFix, *Kind, RI, FixupAddress, + FixupContent, ++RelItr, RelEnd); + if (!PairInfo) + return PairInfo.takeError(); + std::tie(*Kind, TargetSymbol, Addend) = *PairInfo; + assert(TargetSymbol && "No target symbol from parsePairRelocation?"); + break; + } + default: + llvm_unreachable("Special relocation kind should not appear in " + "mach-o file"); + } + + LLVM_DEBUG({ + dbgs() << " "; + Edge GE(*Kind, FixupAddress - BlockToFix->getAddress(), *TargetSymbol, + Addend); + printEdge(dbgs(), *BlockToFix, GE, + getMachOARM64RelocationKindName(*Kind)); + dbgs() << "\n"; + }); + BlockToFix->addEdge(*Kind, FixupAddress - BlockToFix->getAddress(), + *TargetSymbol, Addend); + } + } + return Error::success(); + } + + unsigned NumSymbols = 0; +}; + +class MachO_arm64_GOTAndStubsBuilder + : public BasicGOTAndStubsBuilder<MachO_arm64_GOTAndStubsBuilder> { +public: + MachO_arm64_GOTAndStubsBuilder(LinkGraph &G) + : BasicGOTAndStubsBuilder<MachO_arm64_GOTAndStubsBuilder>(G) {} + + bool isGOTEdge(Edge &E) const { + return E.getKind() == GOTPage21 || E.getKind() == GOTPageOffset12 || + E.getKind() == PointerToGOT; + } + + Symbol &createGOTEntry(Symbol &Target) { + auto &GOTEntryBlock = G.createContentBlock( + getGOTSection(), getGOTEntryBlockContent(), 0, 8, 0); + GOTEntryBlock.addEdge(Pointer64, 0, Target, 0); + return G.addAnonymousSymbol(GOTEntryBlock, 0, 8, false, false); + } + + void fixGOTEdge(Edge &E, Symbol &GOTEntry) { + if (E.getKind() == GOTPage21 || E.getKind() == GOTPageOffset12) { + // Update the target, but leave the edge addend as-is. + E.setTarget(GOTEntry); + } else if (E.getKind() == PointerToGOT) { + E.setTarget(GOTEntry); + E.setKind(Delta32); + } else + llvm_unreachable("Not a GOT edge?"); + } + + bool isExternalBranchEdge(Edge &E) { + return E.getKind() == Branch26 && !E.getTarget().isDefined(); + } + + Symbol &createStub(Symbol &Target) { + auto &StubContentBlock = + G.createContentBlock(getStubsSection(), getStubBlockContent(), 0, 1, 0); + // Re-use GOT entries for stub targets. + auto &GOTEntrySymbol = getGOTEntrySymbol(Target); + StubContentBlock.addEdge(LDRLiteral19, 0, GOTEntrySymbol, 0); + return G.addAnonymousSymbol(StubContentBlock, 0, 8, true, false); + } + + void fixExternalBranchEdge(Edge &E, Symbol &Stub) { + assert(E.getKind() == Branch26 && "Not a Branch32 edge?"); + assert(E.getAddend() == 0 && "Branch32 edge has non-zero addend?"); + E.setTarget(Stub); + } + +private: + Section &getGOTSection() { + if (!GOTSection) + GOTSection = &G.createSection("$__GOT", sys::Memory::MF_READ); + return *GOTSection; + } + + Section &getStubsSection() { + if (!StubsSection) { + auto StubsProt = static_cast<sys::Memory::ProtectionFlags>( + sys::Memory::MF_READ | sys::Memory::MF_EXEC); + StubsSection = &G.createSection("$__STUBS", StubsProt); + } + return *StubsSection; + } + + StringRef getGOTEntryBlockContent() { + return StringRef(reinterpret_cast<const char *>(NullGOTEntryContent), + sizeof(NullGOTEntryContent)); + } + + StringRef getStubBlockContent() { + return StringRef(reinterpret_cast<const char *>(StubContent), + sizeof(StubContent)); + } + + static const uint8_t NullGOTEntryContent[8]; + static const uint8_t StubContent[8]; + Section *GOTSection = nullptr; + Section *StubsSection = nullptr; +}; + +const uint8_t MachO_arm64_GOTAndStubsBuilder::NullGOTEntryContent[8] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; +const uint8_t MachO_arm64_GOTAndStubsBuilder::StubContent[8] = { + 0x10, 0x00, 0x00, 0x58, // LDR x16, <literal> + 0x00, 0x02, 0x1f, 0xd6 // BR x16 +}; + +} // namespace + +namespace llvm { +namespace jitlink { + +class MachOJITLinker_arm64 : public JITLinker<MachOJITLinker_arm64> { + friend class JITLinker<MachOJITLinker_arm64>; + +public: + MachOJITLinker_arm64(std::unique_ptr<JITLinkContext> Ctx, + std::unique_ptr<LinkGraph> G, + PassConfiguration PassConfig) + : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) {} + +private: + StringRef getEdgeKindName(Edge::Kind R) const override { + return getMachOARM64RelocationKindName(R); + } + + static Error targetOutOfRangeError(const Block &B, const Edge &E) { + std::string ErrMsg; + { + raw_string_ostream ErrStream(ErrMsg); + ErrStream << "Relocation target out of range: "; + printEdge(ErrStream, B, E, getMachOARM64RelocationKindName(E.getKind())); + ErrStream << "\n"; + } + return make_error<JITLinkError>(std::move(ErrMsg)); + } + + static unsigned getPageOffset12Shift(uint32_t Instr) { + constexpr uint32_t LoadStoreImm12Mask = 0x3b000000; + constexpr uint32_t Vec128Mask = 0x04800000; + + if ((Instr & LoadStoreImm12Mask) == 0x39000000) { + uint32_t ImplicitShift = Instr >> 30; + if (ImplicitShift == 0) + if ((Instr & Vec128Mask) == Vec128Mask) + ImplicitShift = 4; + + return ImplicitShift; + } + + return 0; + } + + Error applyFixup(Block &B, const Edge &E, char *BlockWorkingMem) const { + using namespace support; + + char *FixupPtr = BlockWorkingMem + E.getOffset(); + JITTargetAddress FixupAddress = B.getAddress() + E.getOffset(); + + switch (E.getKind()) { + case Branch26: { + assert((FixupAddress & 0x3) == 0 && "Branch-inst is not 32-bit aligned"); + + int64_t Value = E.getTarget().getAddress() - FixupAddress + E.getAddend(); + + if (static_cast<uint64_t>(Value) & 0x3) + return make_error<JITLinkError>("Branch26 target is not 32-bit " + "aligned"); + + if (Value < -(1 << 27) || Value > ((1 << 27) - 1)) + return targetOutOfRangeError(B, E); + + uint32_t RawInstr = *(little32_t *)FixupPtr; + assert((RawInstr & 0x7fffffff) == 0x14000000 && + "RawInstr isn't a B or BR immediate instruction"); + uint32_t Imm = (static_cast<uint32_t>(Value) & ((1 << 28) - 1)) >> 2; + uint32_t FixedInstr = RawInstr | Imm; + *(little32_t *)FixupPtr = FixedInstr; + break; + } + case Pointer32: { + uint64_t Value = E.getTarget().getAddress() + E.getAddend(); + if (Value > std::numeric_limits<uint32_t>::max()) + return targetOutOfRangeError(B, E); + *(ulittle32_t *)FixupPtr = Value; + break; + } + case Pointer64: + case Pointer64Anon: { + uint64_t Value = E.getTarget().getAddress() + E.getAddend(); + *(ulittle64_t *)FixupPtr = Value; + break; + } + case Page21: + case GOTPage21: { + assert((E.getKind() != GOTPage21 || E.getAddend() == 0) && + "GOTPAGE21 with non-zero addend"); + uint64_t TargetPage = + (E.getTarget().getAddress() + E.getAddend()) & + ~static_cast<uint64_t>(4096 - 1); + uint64_t PCPage = FixupAddress & ~static_cast<uint64_t>(4096 - 1); + + int64_t PageDelta = TargetPage - PCPage; + if (PageDelta < -(1 << 30) || PageDelta > ((1 << 30) - 1)) + return targetOutOfRangeError(B, E); + + uint32_t RawInstr = *(ulittle32_t *)FixupPtr; + assert((RawInstr & 0xffffffe0) == 0x90000000 && + "RawInstr isn't an ADRP instruction"); + uint32_t ImmLo = (static_cast<uint64_t>(PageDelta) >> 12) & 0x3; + uint32_t ImmHi = (static_cast<uint64_t>(PageDelta) >> 14) & 0x7ffff; + uint32_t FixedInstr = RawInstr | (ImmLo << 29) | (ImmHi << 5); + *(ulittle32_t *)FixupPtr = FixedInstr; + break; + } + case PageOffset12: { + uint64_t TargetOffset = + (E.getTarget().getAddress() + E.getAddend()) & 0xfff; + + uint32_t RawInstr = *(ulittle32_t *)FixupPtr; + unsigned ImmShift = getPageOffset12Shift(RawInstr); + + if (TargetOffset & ((1 << ImmShift) - 1)) + return make_error<JITLinkError>("PAGEOFF12 target is not aligned"); + + uint32_t EncodedImm = (TargetOffset >> ImmShift) << 10; + uint32_t FixedInstr = RawInstr | EncodedImm; + *(ulittle32_t *)FixupPtr = FixedInstr; + break; + } + case GOTPageOffset12: { + assert(E.getAddend() == 0 && "GOTPAGEOF12 with non-zero addend"); + + uint32_t RawInstr = *(ulittle32_t *)FixupPtr; + assert((RawInstr & 0xfffffc00) == 0xf9400000 && + "RawInstr isn't a 64-bit LDR immediate"); + + uint32_t TargetOffset = E.getTarget().getAddress() & 0xfff; + assert((TargetOffset & 0x7) == 0 && "GOT entry is not 8-byte aligned"); + uint32_t EncodedImm = (TargetOffset >> 3) << 10; + uint32_t FixedInstr = RawInstr | EncodedImm; + *(ulittle32_t *)FixupPtr = FixedInstr; + break; + } + case LDRLiteral19: { + assert((FixupAddress & 0x3) == 0 && "LDR is not 32-bit aligned"); + assert(E.getAddend() == 0 && "LDRLiteral19 with non-zero addend"); + uint32_t RawInstr = *(ulittle32_t *)FixupPtr; + assert(RawInstr == 0x58000010 && "RawInstr isn't a 64-bit LDR literal"); + int64_t Delta = E.getTarget().getAddress() - FixupAddress; + if (Delta & 0x3) + return make_error<JITLinkError>("LDR literal target is not 32-bit " + "aligned"); + if (Delta < -(1 << 20) || Delta > ((1 << 20) - 1)) + return targetOutOfRangeError(B, E); + + uint32_t EncodedImm = (static_cast<uint32_t>(Delta) >> 2) << 5; + uint32_t FixedInstr = RawInstr | EncodedImm; + *(ulittle32_t *)FixupPtr = FixedInstr; + break; + } + case Delta32: + case Delta64: + case NegDelta32: + case NegDelta64: { + int64_t Value; + if (E.getKind() == Delta32 || E.getKind() == Delta64) + Value = E.getTarget().getAddress() - FixupAddress + E.getAddend(); + else + Value = FixupAddress - E.getTarget().getAddress() + E.getAddend(); + + if (E.getKind() == Delta32 || E.getKind() == NegDelta32) { + if (Value < std::numeric_limits<int32_t>::min() || + Value > std::numeric_limits<int32_t>::max()) + return targetOutOfRangeError(B, E); + *(little32_t *)FixupPtr = Value; + } else + *(little64_t *)FixupPtr = Value; + break; + } + default: + llvm_unreachable("Unrecognized edge kind"); + } + + return Error::success(); + } + + uint64_t NullValue = 0; +}; + +Expected<std::unique_ptr<LinkGraph>> +createLinkGraphFromMachOObject_arm64(MemoryBufferRef ObjectBuffer) { + auto MachOObj = object::ObjectFile::createMachOObjectFile(ObjectBuffer); + if (!MachOObj) + return MachOObj.takeError(); + return MachOLinkGraphBuilder_arm64(**MachOObj).buildGraph(); +} + +void link_MachO_arm64(std::unique_ptr<LinkGraph> G, + std::unique_ptr<JITLinkContext> Ctx) { + + PassConfiguration Config; + + if (Ctx->shouldAddDefaultTargetPasses(G->getTargetTriple())) { + // Add a mark-live pass. + if (auto MarkLive = Ctx->getMarkLivePass(G->getTargetTriple())) + Config.PrePrunePasses.push_back(std::move(MarkLive)); + else + Config.PrePrunePasses.push_back(markAllSymbolsLive); + + // Add an in-place GOT/Stubs pass. + Config.PostPrunePasses.push_back([](LinkGraph &G) -> Error { + MachO_arm64_GOTAndStubsBuilder(G).run(); + return Error::success(); + }); + } + + if (auto Err = Ctx->modifyPassConfig(G->getTargetTriple(), Config)) + return Ctx->notifyFailed(std::move(Err)); + + // Construct a JITLinker and run the link function. + MachOJITLinker_arm64::link(std::move(Ctx), std::move(G), std::move(Config)); +} + +StringRef getMachOARM64RelocationKindName(Edge::Kind R) { + switch (R) { + case Branch26: + return "Branch26"; + case Pointer64: + return "Pointer64"; + case Pointer64Anon: + return "Pointer64Anon"; + case Page21: + return "Page21"; + case PageOffset12: + return "PageOffset12"; + case GOTPage21: + return "GOTPage21"; + case GOTPageOffset12: + return "GOTPageOffset12"; + case PointerToGOT: + return "PointerToGOT"; + case PairedAddend: + return "PairedAddend"; + case LDRLiteral19: + return "LDRLiteral19"; + case Delta32: + return "Delta32"; + case Delta64: + return "Delta64"; + case NegDelta32: + return "NegDelta32"; + case NegDelta64: + return "NegDelta64"; + default: + return getGenericEdgeKindName(static_cast<Edge::Kind>(R)); + } +} + +} // end namespace jitlink +} // end namespace llvm diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp b/contrib/libs/llvm12/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp new file mode 100644 index 0000000000..bde4a19e71 --- /dev/null +++ b/contrib/libs/llvm12/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp @@ -0,0 +1,746 @@ +//===---- MachO_x86_64.cpp -JIT linker implementation for MachO/x86-64 ----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// MachO/x86-64 jit-link implementation. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/JITLink/MachO_x86_64.h" + +#include "BasicGOTAndStubsBuilder.h" +#include "MachOLinkGraphBuilder.h" + +#define DEBUG_TYPE "jitlink" + +using namespace llvm; +using namespace llvm::jitlink; +using namespace llvm::jitlink::MachO_x86_64_Edges; + +namespace { + +class MachOLinkGraphBuilder_x86_64 : public MachOLinkGraphBuilder { +public: + MachOLinkGraphBuilder_x86_64(const object::MachOObjectFile &Obj) + : MachOLinkGraphBuilder(Obj, Triple("x86_64-apple-darwin")) {} + +private: + static Expected<MachOX86RelocationKind> + getRelocationKind(const MachO::relocation_info &RI) { + switch (RI.r_type) { + case MachO::X86_64_RELOC_UNSIGNED: + if (!RI.r_pcrel) { + if (RI.r_length == 3) + return RI.r_extern ? Pointer64 : Pointer64Anon; + else if (RI.r_extern && RI.r_length == 2) + return Pointer32; + } + break; + case MachO::X86_64_RELOC_SIGNED: + if (RI.r_pcrel && RI.r_length == 2) + return RI.r_extern ? PCRel32 : PCRel32Anon; + break; + case MachO::X86_64_RELOC_BRANCH: + if (RI.r_pcrel && RI.r_extern && RI.r_length == 2) + return Branch32; + break; + case MachO::X86_64_RELOC_GOT_LOAD: + if (RI.r_pcrel && RI.r_extern && RI.r_length == 2) + return PCRel32GOTLoad; + break; + case MachO::X86_64_RELOC_GOT: + if (RI.r_pcrel && RI.r_extern && RI.r_length == 2) + return PCRel32GOT; + break; + case MachO::X86_64_RELOC_SUBTRACTOR: + // SUBTRACTOR must be non-pc-rel, extern, with length 2 or 3. + // Initially represent SUBTRACTOR relocations with 'Delta<W>'. They may + // be turned into NegDelta<W> by parsePairRelocation. + if (!RI.r_pcrel && RI.r_extern) { + if (RI.r_length == 2) + return Delta32; + else if (RI.r_length == 3) + return Delta64; + } + break; + case MachO::X86_64_RELOC_SIGNED_1: + if (RI.r_pcrel && RI.r_length == 2) + return RI.r_extern ? PCRel32Minus1 : PCRel32Minus1Anon; + break; + case MachO::X86_64_RELOC_SIGNED_2: + if (RI.r_pcrel && RI.r_length == 2) + return RI.r_extern ? PCRel32Minus2 : PCRel32Minus2Anon; + break; + case MachO::X86_64_RELOC_SIGNED_4: + if (RI.r_pcrel && RI.r_length == 2) + return RI.r_extern ? PCRel32Minus4 : PCRel32Minus4Anon; + break; + case MachO::X86_64_RELOC_TLV: + if (RI.r_pcrel && RI.r_extern && RI.r_length == 2) + return PCRel32TLV; + break; + } + + return make_error<JITLinkError>( + "Unsupported x86-64 relocation: address=" + + formatv("{0:x8}", RI.r_address) + + ", symbolnum=" + formatv("{0:x6}", RI.r_symbolnum) + + ", kind=" + formatv("{0:x1}", RI.r_type) + + ", pc_rel=" + (RI.r_pcrel ? "true" : "false") + + ", extern=" + (RI.r_extern ? "true" : "false") + + ", length=" + formatv("{0:d}", RI.r_length)); + } + + using PairRelocInfo = std::tuple<MachOX86RelocationKind, Symbol *, uint64_t>; + + // Parses paired SUBTRACTOR/UNSIGNED relocations and, on success, + // returns the edge kind and addend to be used. + Expected<PairRelocInfo> + parsePairRelocation(Block &BlockToFix, Edge::Kind SubtractorKind, + const MachO::relocation_info &SubRI, + JITTargetAddress FixupAddress, const char *FixupContent, + object::relocation_iterator &UnsignedRelItr, + object::relocation_iterator &RelEnd) { + using namespace support; + + assert(((SubtractorKind == Delta32 && SubRI.r_length == 2) || + (SubtractorKind == Delta64 && SubRI.r_length == 3)) && + "Subtractor kind should match length"); + assert(SubRI.r_extern && "SUBTRACTOR reloc symbol should be extern"); + assert(!SubRI.r_pcrel && "SUBTRACTOR reloc should not be PCRel"); + + if (UnsignedRelItr == RelEnd) + return make_error<JITLinkError>("x86_64 SUBTRACTOR without paired " + "UNSIGNED relocation"); + + auto UnsignedRI = getRelocationInfo(UnsignedRelItr); + + if (SubRI.r_address != UnsignedRI.r_address) + return make_error<JITLinkError>("x86_64 SUBTRACTOR and paired UNSIGNED " + "point to different addresses"); + + if (SubRI.r_length != UnsignedRI.r_length) + return make_error<JITLinkError>("length of x86_64 SUBTRACTOR and paired " + "UNSIGNED reloc must match"); + + Symbol *FromSymbol; + if (auto FromSymbolOrErr = findSymbolByIndex(SubRI.r_symbolnum)) + FromSymbol = FromSymbolOrErr->GraphSymbol; + else + return FromSymbolOrErr.takeError(); + + // Read the current fixup value. + uint64_t FixupValue = 0; + if (SubRI.r_length == 3) + FixupValue = *(const little64_t *)FixupContent; + else + FixupValue = *(const little32_t *)FixupContent; + + // Find 'ToSymbol' using symbol number or address, depending on whether the + // paired UNSIGNED relocation is extern. + Symbol *ToSymbol = nullptr; + if (UnsignedRI.r_extern) { + // Find target symbol by symbol index. + if (auto ToSymbolOrErr = findSymbolByIndex(UnsignedRI.r_symbolnum)) + ToSymbol = ToSymbolOrErr->GraphSymbol; + else + return ToSymbolOrErr.takeError(); + } else { + auto ToSymbolSec = findSectionByIndex(UnsignedRI.r_symbolnum - 1); + if (!ToSymbolSec) + return ToSymbolSec.takeError(); + ToSymbol = getSymbolByAddress(ToSymbolSec->Address); + assert(ToSymbol && "No symbol for section"); + FixupValue -= ToSymbol->getAddress(); + } + + MachOX86RelocationKind DeltaKind; + Symbol *TargetSymbol; + uint64_t Addend; + if (&BlockToFix == &FromSymbol->getAddressable()) { + TargetSymbol = ToSymbol; + DeltaKind = (SubRI.r_length == 3) ? Delta64 : Delta32; + Addend = FixupValue + (FixupAddress - FromSymbol->getAddress()); + // FIXME: handle extern 'from'. + } else if (&BlockToFix == &ToSymbol->getAddressable()) { + TargetSymbol = FromSymbol; + DeltaKind = (SubRI.r_length == 3) ? NegDelta64 : NegDelta32; + Addend = FixupValue - (FixupAddress - ToSymbol->getAddress()); + } else { + // BlockToFix was neither FromSymbol nor ToSymbol. + return make_error<JITLinkError>("SUBTRACTOR relocation must fix up " + "either 'A' or 'B' (or a symbol in one " + "of their alt-entry chains)"); + } + + return PairRelocInfo(DeltaKind, TargetSymbol, Addend); + } + + Error addRelocations() override { + using namespace support; + auto &Obj = getObject(); + + LLVM_DEBUG(dbgs() << "Processing relocations:\n"); + + for (auto &S : Obj.sections()) { + + JITTargetAddress SectionAddress = S.getAddress(); + + // Skip relocations virtual sections. + if (S.isVirtual()) { + if (S.relocation_begin() != S.relocation_end()) + return make_error<JITLinkError>("Virtual section contains " + "relocations"); + continue; + } + + // Skip relocations for debug symbols. + { + auto &NSec = + getSectionByIndex(Obj.getSectionIndex(S.getRawDataRefImpl())); + if (!NSec.GraphSection) { + LLVM_DEBUG({ + dbgs() << " Skipping relocations for MachO section " + << NSec.SegName << "/" << NSec.SectName + << " which has no associated graph section\n"; + }); + continue; + } + } + + // Add relocations for section. + for (auto RelItr = S.relocation_begin(), RelEnd = S.relocation_end(); + RelItr != RelEnd; ++RelItr) { + + MachO::relocation_info RI = getRelocationInfo(RelItr); + + // Sanity check the relocation kind. + auto Kind = getRelocationKind(RI); + if (!Kind) + return Kind.takeError(); + + // Find the address of the value to fix up. + JITTargetAddress FixupAddress = SectionAddress + (uint32_t)RI.r_address; + + LLVM_DEBUG({ + auto &NSec = + getSectionByIndex(Obj.getSectionIndex(S.getRawDataRefImpl())); + dbgs() << " " << NSec.SectName << " + " + << formatv("{0:x8}", RI.r_address) << ":\n"; + }); + + // Find the block that the fixup points to. + Block *BlockToFix = nullptr; + { + auto SymbolToFixOrErr = findSymbolByAddress(FixupAddress); + if (!SymbolToFixOrErr) + return SymbolToFixOrErr.takeError(); + BlockToFix = &SymbolToFixOrErr->getBlock(); + } + + if (FixupAddress + static_cast<JITTargetAddress>(1ULL << RI.r_length) > + BlockToFix->getAddress() + BlockToFix->getContent().size()) + return make_error<JITLinkError>( + "Relocation extends past end of fixup block"); + + // Get a pointer to the fixup content. + const char *FixupContent = BlockToFix->getContent().data() + + (FixupAddress - BlockToFix->getAddress()); + + // The target symbol and addend will be populated by the switch below. + Symbol *TargetSymbol = nullptr; + uint64_t Addend = 0; + + switch (*Kind) { + case Branch32: + case PCRel32: + case PCRel32GOTLoad: + case PCRel32GOT: + if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) + TargetSymbol = TargetSymbolOrErr->GraphSymbol; + else + return TargetSymbolOrErr.takeError(); + Addend = *(const little32_t *)FixupContent; + break; + case Pointer32: + if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) + TargetSymbol = TargetSymbolOrErr->GraphSymbol; + else + return TargetSymbolOrErr.takeError(); + Addend = *(const ulittle32_t *)FixupContent; + break; + case Pointer64: + if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) + TargetSymbol = TargetSymbolOrErr->GraphSymbol; + else + return TargetSymbolOrErr.takeError(); + Addend = *(const ulittle64_t *)FixupContent; + break; + case Pointer64Anon: { + JITTargetAddress TargetAddress = *(const ulittle64_t *)FixupContent; + if (auto TargetSymbolOrErr = findSymbolByAddress(TargetAddress)) + TargetSymbol = &*TargetSymbolOrErr; + else + return TargetSymbolOrErr.takeError(); + Addend = TargetAddress - TargetSymbol->getAddress(); + break; + } + case PCRel32Minus1: + case PCRel32Minus2: + case PCRel32Minus4: + if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) + TargetSymbol = TargetSymbolOrErr->GraphSymbol; + else + return TargetSymbolOrErr.takeError(); + Addend = *(const little32_t *)FixupContent + + (1 << (*Kind - PCRel32Minus1)); + break; + case PCRel32Anon: { + JITTargetAddress TargetAddress = + FixupAddress + 4 + *(const little32_t *)FixupContent; + if (auto TargetSymbolOrErr = findSymbolByAddress(TargetAddress)) + TargetSymbol = &*TargetSymbolOrErr; + else + return TargetSymbolOrErr.takeError(); + Addend = TargetAddress - TargetSymbol->getAddress(); + break; + } + case PCRel32Minus1Anon: + case PCRel32Minus2Anon: + case PCRel32Minus4Anon: { + JITTargetAddress Delta = + static_cast<JITTargetAddress>(1ULL << (*Kind - PCRel32Minus1Anon)); + JITTargetAddress TargetAddress = + FixupAddress + 4 + Delta + *(const little32_t *)FixupContent; + if (auto TargetSymbolOrErr = findSymbolByAddress(TargetAddress)) + TargetSymbol = &*TargetSymbolOrErr; + else + return TargetSymbolOrErr.takeError(); + Addend = TargetAddress - TargetSymbol->getAddress(); + break; + } + case Delta32: + case Delta64: { + // We use Delta32/Delta64 to represent SUBTRACTOR relocations. + // parsePairRelocation handles the paired reloc, and returns the + // edge kind to be used (either Delta32/Delta64, or + // NegDelta32/NegDelta64, depending on the direction of the + // subtraction) along with the addend. + auto PairInfo = + parsePairRelocation(*BlockToFix, *Kind, RI, FixupAddress, + FixupContent, ++RelItr, RelEnd); + if (!PairInfo) + return PairInfo.takeError(); + std::tie(*Kind, TargetSymbol, Addend) = *PairInfo; + assert(TargetSymbol && "No target symbol from parsePairRelocation?"); + break; + } + case PCRel32TLV: + return make_error<JITLinkError>( + "MachO TLV relocations not yet supported"); + default: + llvm_unreachable("Special relocation kind should not appear in " + "mach-o file"); + } + + LLVM_DEBUG({ + dbgs() << " "; + Edge GE(*Kind, FixupAddress - BlockToFix->getAddress(), *TargetSymbol, + Addend); + printEdge(dbgs(), *BlockToFix, GE, + getMachOX86RelocationKindName(*Kind)); + dbgs() << "\n"; + }); + BlockToFix->addEdge(*Kind, FixupAddress - BlockToFix->getAddress(), + *TargetSymbol, Addend); + } + } + return Error::success(); + } +}; + +class MachO_x86_64_GOTAndStubsBuilder + : public BasicGOTAndStubsBuilder<MachO_x86_64_GOTAndStubsBuilder> { +public: + static const uint8_t NullGOTEntryContent[8]; + static const uint8_t StubContent[6]; + + MachO_x86_64_GOTAndStubsBuilder(LinkGraph &G) + : BasicGOTAndStubsBuilder<MachO_x86_64_GOTAndStubsBuilder>(G) {} + + bool isGOTEdge(Edge &E) const { + return E.getKind() == PCRel32GOT || E.getKind() == PCRel32GOTLoad; + } + + Symbol &createGOTEntry(Symbol &Target) { + auto &GOTEntryBlock = G.createContentBlock( + getGOTSection(), getGOTEntryBlockContent(), 0, 8, 0); + GOTEntryBlock.addEdge(Pointer64, 0, Target, 0); + return G.addAnonymousSymbol(GOTEntryBlock, 0, 8, false, false); + } + + void fixGOTEdge(Edge &E, Symbol &GOTEntry) { + assert((E.getKind() == PCRel32GOT || E.getKind() == PCRel32GOTLoad) && + "Not a GOT edge?"); + // If this is a PCRel32GOT then change it to an ordinary PCRel32. If it is + // a PCRel32GOTLoad then leave it as-is for now. We will use the kind to + // check for GOT optimization opportunities in the + // optimizeMachO_x86_64_GOTAndStubs pass below. + if (E.getKind() == PCRel32GOT) + E.setKind(PCRel32); + + E.setTarget(GOTEntry); + // Leave the edge addend as-is. + } + + bool isExternalBranchEdge(Edge &E) { + return E.getKind() == Branch32 && !E.getTarget().isDefined(); + } + + Symbol &createStub(Symbol &Target) { + auto &StubContentBlock = + G.createContentBlock(getStubsSection(), getStubBlockContent(), 0, 1, 0); + // Re-use GOT entries for stub targets. + auto &GOTEntrySymbol = getGOTEntrySymbol(Target); + StubContentBlock.addEdge(PCRel32, 2, GOTEntrySymbol, 0); + return G.addAnonymousSymbol(StubContentBlock, 0, 6, true, false); + } + + void fixExternalBranchEdge(Edge &E, Symbol &Stub) { + assert(E.getKind() == Branch32 && "Not a Branch32 edge?"); + assert(E.getAddend() == 0 && "Branch32 edge has non-zero addend?"); + + // Set the edge kind to Branch32ToStub. We will use this to check for stub + // optimization opportunities in the optimizeMachO_x86_64_GOTAndStubs pass + // below. + E.setKind(Branch32ToStub); + E.setTarget(Stub); + } + +private: + Section &getGOTSection() { + if (!GOTSection) + GOTSection = &G.createSection("$__GOT", sys::Memory::MF_READ); + return *GOTSection; + } + + Section &getStubsSection() { + if (!StubsSection) { + auto StubsProt = static_cast<sys::Memory::ProtectionFlags>( + sys::Memory::MF_READ | sys::Memory::MF_EXEC); + StubsSection = &G.createSection("$__STUBS", StubsProt); + } + return *StubsSection; + } + + StringRef getGOTEntryBlockContent() { + return StringRef(reinterpret_cast<const char *>(NullGOTEntryContent), + sizeof(NullGOTEntryContent)); + } + + StringRef getStubBlockContent() { + return StringRef(reinterpret_cast<const char *>(StubContent), + sizeof(StubContent)); + } + + Section *GOTSection = nullptr; + Section *StubsSection = nullptr; +}; + +const uint8_t MachO_x86_64_GOTAndStubsBuilder::NullGOTEntryContent[8] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; +const uint8_t MachO_x86_64_GOTAndStubsBuilder::StubContent[6] = { + 0xFF, 0x25, 0x00, 0x00, 0x00, 0x00}; +} // namespace + +static Error optimizeMachO_x86_64_GOTAndStubs(LinkGraph &G) { + LLVM_DEBUG(dbgs() << "Optimizing GOT entries and stubs:\n"); + + for (auto *B : G.blocks()) + for (auto &E : B->edges()) + if (E.getKind() == PCRel32GOTLoad) { + assert(E.getOffset() >= 3 && "GOT edge occurs too early in block"); + + // Switch the edge kind to PCRel32: Whether we change the edge target + // or not this will be the desired kind. + E.setKind(PCRel32); + + // Optimize GOT references. + auto &GOTBlock = E.getTarget().getBlock(); + assert(GOTBlock.getSize() == G.getPointerSize() && + "GOT entry block should be pointer sized"); + assert(GOTBlock.edges_size() == 1 && + "GOT entry should only have one outgoing edge"); + + auto &GOTTarget = GOTBlock.edges().begin()->getTarget(); + JITTargetAddress EdgeAddr = B->getAddress() + E.getOffset(); + JITTargetAddress TargetAddr = GOTTarget.getAddress(); + + // Check that this is a recognized MOV instruction. + // FIXME: Can we assume this? + constexpr uint8_t MOVQRIPRel[] = {0x48, 0x8b}; + if (strncmp(B->getContent().data() + E.getOffset() - 3, + reinterpret_cast<const char *>(MOVQRIPRel), 2) != 0) + continue; + + int64_t Displacement = TargetAddr - EdgeAddr + 4; + if (Displacement >= std::numeric_limits<int32_t>::min() && + Displacement <= std::numeric_limits<int32_t>::max()) { + E.setTarget(GOTTarget); + auto *BlockData = reinterpret_cast<uint8_t *>( + const_cast<char *>(B->getContent().data())); + BlockData[E.getOffset() - 2] = 0x8d; + LLVM_DEBUG({ + dbgs() << " Replaced GOT load wih LEA:\n "; + printEdge(dbgs(), *B, E, + getMachOX86RelocationKindName(E.getKind())); + dbgs() << "\n"; + }); + } + } else if (E.getKind() == Branch32ToStub) { + + // Switch the edge kind to PCRel32: Whether we change the edge target + // or not this will be the desired kind. + E.setKind(Branch32); + + auto &StubBlock = E.getTarget().getBlock(); + assert(StubBlock.getSize() == + sizeof(MachO_x86_64_GOTAndStubsBuilder::StubContent) && + "Stub block should be stub sized"); + assert(StubBlock.edges_size() == 1 && + "Stub block should only have one outgoing edge"); + + auto &GOTBlock = StubBlock.edges().begin()->getTarget().getBlock(); + assert(GOTBlock.getSize() == G.getPointerSize() && + "GOT block should be pointer sized"); + assert(GOTBlock.edges_size() == 1 && + "GOT block should only have one outgoing edge"); + + auto &GOTTarget = GOTBlock.edges().begin()->getTarget(); + JITTargetAddress EdgeAddr = B->getAddress() + E.getOffset(); + JITTargetAddress TargetAddr = GOTTarget.getAddress(); + + int64_t Displacement = TargetAddr - EdgeAddr + 4; + if (Displacement >= std::numeric_limits<int32_t>::min() && + Displacement <= std::numeric_limits<int32_t>::max()) { + E.setTarget(GOTTarget); + LLVM_DEBUG({ + dbgs() << " Replaced stub branch with direct branch:\n "; + printEdge(dbgs(), *B, E, + getMachOX86RelocationKindName(E.getKind())); + dbgs() << "\n"; + }); + } + } + + return Error::success(); +} + +namespace llvm { +namespace jitlink { + +class MachOJITLinker_x86_64 : public JITLinker<MachOJITLinker_x86_64> { + friend class JITLinker<MachOJITLinker_x86_64>; + +public: + MachOJITLinker_x86_64(std::unique_ptr<JITLinkContext> Ctx, + std::unique_ptr<LinkGraph> G, + PassConfiguration PassConfig) + : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) {} + +private: + StringRef getEdgeKindName(Edge::Kind R) const override { + return getMachOX86RelocationKindName(R); + } + + static Error targetOutOfRangeError(const Block &B, const Edge &E) { + std::string ErrMsg; + { + raw_string_ostream ErrStream(ErrMsg); + ErrStream << "Relocation target out of range: "; + printEdge(ErrStream, B, E, getMachOX86RelocationKindName(E.getKind())); + ErrStream << "\n"; + } + return make_error<JITLinkError>(std::move(ErrMsg)); + } + + Error applyFixup(Block &B, const Edge &E, char *BlockWorkingMem) const { + + using namespace support; + + char *FixupPtr = BlockWorkingMem + E.getOffset(); + JITTargetAddress FixupAddress = B.getAddress() + E.getOffset(); + + switch (E.getKind()) { + case Branch32: + case PCRel32: + case PCRel32Anon: { + int64_t Value = + E.getTarget().getAddress() - (FixupAddress + 4) + E.getAddend(); + if (Value < std::numeric_limits<int32_t>::min() || + Value > std::numeric_limits<int32_t>::max()) + return targetOutOfRangeError(B, E); + *(little32_t *)FixupPtr = Value; + break; + } + case Pointer64: + case Pointer64Anon: { + uint64_t Value = E.getTarget().getAddress() + E.getAddend(); + *(ulittle64_t *)FixupPtr = Value; + break; + } + case PCRel32Minus1: + case PCRel32Minus2: + case PCRel32Minus4: { + int Delta = 4 + (1 << (E.getKind() - PCRel32Minus1)); + int64_t Value = + E.getTarget().getAddress() - (FixupAddress + Delta) + E.getAddend(); + if (Value < std::numeric_limits<int32_t>::min() || + Value > std::numeric_limits<int32_t>::max()) + return targetOutOfRangeError(B, E); + *(little32_t *)FixupPtr = Value; + break; + } + case PCRel32Minus1Anon: + case PCRel32Minus2Anon: + case PCRel32Minus4Anon: { + int Delta = 4 + (1 << (E.getKind() - PCRel32Minus1Anon)); + int64_t Value = + E.getTarget().getAddress() - (FixupAddress + Delta) + E.getAddend(); + if (Value < std::numeric_limits<int32_t>::min() || + Value > std::numeric_limits<int32_t>::max()) + return targetOutOfRangeError(B, E); + *(little32_t *)FixupPtr = Value; + break; + } + case Delta32: + case Delta64: + case NegDelta32: + case NegDelta64: { + int64_t Value; + if (E.getKind() == Delta32 || E.getKind() == Delta64) + Value = E.getTarget().getAddress() - FixupAddress + E.getAddend(); + else + Value = FixupAddress - E.getTarget().getAddress() + E.getAddend(); + + if (E.getKind() == Delta32 || E.getKind() == NegDelta32) { + if (Value < std::numeric_limits<int32_t>::min() || + Value > std::numeric_limits<int32_t>::max()) + return targetOutOfRangeError(B, E); + *(little32_t *)FixupPtr = Value; + } else + *(little64_t *)FixupPtr = Value; + break; + } + case Pointer32: { + uint64_t Value = E.getTarget().getAddress() + E.getAddend(); + if (Value > std::numeric_limits<uint32_t>::max()) + return targetOutOfRangeError(B, E); + *(ulittle32_t *)FixupPtr = Value; + break; + } + default: + llvm_unreachable("Unrecognized edge kind"); + } + + return Error::success(); + } + + uint64_t NullValue = 0; +}; + +Expected<std::unique_ptr<LinkGraph>> +createLinkGraphFromMachOObject_x86_64(MemoryBufferRef ObjectBuffer) { + auto MachOObj = object::ObjectFile::createMachOObjectFile(ObjectBuffer); + if (!MachOObj) + return MachOObj.takeError(); + return MachOLinkGraphBuilder_x86_64(**MachOObj).buildGraph(); +} + +void link_MachO_x86_64(std::unique_ptr<LinkGraph> G, + std::unique_ptr<JITLinkContext> Ctx) { + + PassConfiguration Config; + + if (Ctx->shouldAddDefaultTargetPasses(G->getTargetTriple())) { + // Add eh-frame passses. + Config.PrePrunePasses.push_back(EHFrameSplitter("__eh_frame")); + Config.PrePrunePasses.push_back(EHFrameEdgeFixer( + "__eh_frame", G->getPointerSize(), Delta64, Delta32, NegDelta32)); + + // Add a mark-live pass. + if (auto MarkLive = Ctx->getMarkLivePass(G->getTargetTriple())) + Config.PrePrunePasses.push_back(std::move(MarkLive)); + else + Config.PrePrunePasses.push_back(markAllSymbolsLive); + + // Add an in-place GOT/Stubs pass. + Config.PostPrunePasses.push_back([](LinkGraph &G) -> Error { + MachO_x86_64_GOTAndStubsBuilder(G).run(); + return Error::success(); + }); + + // Add GOT/Stubs optimizer pass. + Config.PreFixupPasses.push_back(optimizeMachO_x86_64_GOTAndStubs); + } + + if (auto Err = Ctx->modifyPassConfig(G->getTargetTriple(), Config)) + return Ctx->notifyFailed(std::move(Err)); + + // Construct a JITLinker and run the link function. + MachOJITLinker_x86_64::link(std::move(Ctx), std::move(G), std::move(Config)); +} + +StringRef getMachOX86RelocationKindName(Edge::Kind R) { + switch (R) { + case Branch32: + return "Branch32"; + case Branch32ToStub: + return "Branch32ToStub"; + case Pointer32: + return "Pointer32"; + case Pointer64: + return "Pointer64"; + case Pointer64Anon: + return "Pointer64Anon"; + case PCRel32: + return "PCRel32"; + case PCRel32Minus1: + return "PCRel32Minus1"; + case PCRel32Minus2: + return "PCRel32Minus2"; + case PCRel32Minus4: + return "PCRel32Minus4"; + case PCRel32Anon: + return "PCRel32Anon"; + case PCRel32Minus1Anon: + return "PCRel32Minus1Anon"; + case PCRel32Minus2Anon: + return "PCRel32Minus2Anon"; + case PCRel32Minus4Anon: + return "PCRel32Minus4Anon"; + case PCRel32GOTLoad: + return "PCRel32GOTLoad"; + case PCRel32GOT: + return "PCRel32GOT"; + case PCRel32TLV: + return "PCRel32TLV"; + case Delta32: + return "Delta32"; + case Delta64: + return "Delta64"; + case NegDelta32: + return "NegDelta32"; + case NegDelta64: + return "NegDelta64"; + default: + return getGenericEdgeKindName(static_cast<Edge::Kind>(R)); + } +} + +} // end namespace jitlink +} // end namespace llvm diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/MCJIT/.yandex_meta/licenses.list.txt b/contrib/libs/llvm12/lib/ExecutionEngine/MCJIT/.yandex_meta/licenses.list.txt deleted file mode 100644 index c62d353021..0000000000 --- a/contrib/libs/llvm12/lib/ExecutionEngine/MCJIT/.yandex_meta/licenses.list.txt +++ /dev/null @@ -1,7 +0,0 @@ -====================Apache-2.0 WITH LLVM-exception==================== -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. - - -====================Apache-2.0 WITH LLVM-exception==================== -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp new file mode 100644 index 0000000000..68878f6729 --- /dev/null +++ b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp @@ -0,0 +1,379 @@ +//===----- CompileOnDemandLayer.cpp - Lazily emit IR on first call --------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h" + +#include "llvm/ADT/Hashing.h" +#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" +#include "llvm/IR/Mangler.h" +#include "llvm/IR/Module.h" +#include "llvm/Support/FormatVariadic.h" + +using namespace llvm; +using namespace llvm::orc; + +static ThreadSafeModule extractSubModule(ThreadSafeModule &TSM, + StringRef Suffix, + GVPredicate ShouldExtract) { + + auto DeleteExtractedDefs = [](GlobalValue &GV) { + // Bump the linkage: this global will be provided by the external module. + GV.setLinkage(GlobalValue::ExternalLinkage); + + // Delete the definition in the source module. + if (isa<Function>(GV)) { + auto &F = cast<Function>(GV); + F.deleteBody(); + F.setPersonalityFn(nullptr); + } else if (isa<GlobalVariable>(GV)) { + cast<GlobalVariable>(GV).setInitializer(nullptr); + } else if (isa<GlobalAlias>(GV)) { + // We need to turn deleted aliases into function or variable decls based + // on the type of their aliasee. + auto &A = cast<GlobalAlias>(GV); + Constant *Aliasee = A.getAliasee(); + assert(A.hasName() && "Anonymous alias?"); + assert(Aliasee->hasName() && "Anonymous aliasee"); + std::string AliasName = std::string(A.getName()); + + if (isa<Function>(Aliasee)) { + auto *F = cloneFunctionDecl(*A.getParent(), *cast<Function>(Aliasee)); + A.replaceAllUsesWith(F); + A.eraseFromParent(); + F->setName(AliasName); + } else if (isa<GlobalVariable>(Aliasee)) { + auto *G = cloneGlobalVariableDecl(*A.getParent(), + *cast<GlobalVariable>(Aliasee)); + A.replaceAllUsesWith(G); + A.eraseFromParent(); + G->setName(AliasName); + } else + llvm_unreachable("Alias to unsupported type"); + } else + llvm_unreachable("Unsupported global type"); + }; + + auto NewTSM = cloneToNewContext(TSM, ShouldExtract, DeleteExtractedDefs); + NewTSM.withModuleDo([&](Module &M) { + M.setModuleIdentifier((M.getModuleIdentifier() + Suffix).str()); + }); + + return NewTSM; +} + +namespace llvm { +namespace orc { + +class PartitioningIRMaterializationUnit : public IRMaterializationUnit { +public: + PartitioningIRMaterializationUnit(ExecutionSession &ES, + const IRSymbolMapper::ManglingOptions &MO, + ThreadSafeModule TSM, + CompileOnDemandLayer &Parent) + : IRMaterializationUnit(ES, MO, std::move(TSM)), Parent(Parent) {} + + PartitioningIRMaterializationUnit( + ThreadSafeModule TSM, SymbolFlagsMap SymbolFlags, + SymbolStringPtr InitSymbol, SymbolNameToDefinitionMap SymbolToDefinition, + CompileOnDemandLayer &Parent) + : IRMaterializationUnit(std::move(TSM), std::move(SymbolFlags), + std::move(InitSymbol), + std::move(SymbolToDefinition)), + Parent(Parent) {} + +private: + void materialize(std::unique_ptr<MaterializationResponsibility> R) override { + Parent.emitPartition(std::move(R), std::move(TSM), + std::move(SymbolToDefinition)); + } + + void discard(const JITDylib &V, const SymbolStringPtr &Name) override { + // All original symbols were materialized by the CODLayer and should be + // final. The function bodies provided by M should never be overridden. + llvm_unreachable("Discard should never be called on an " + "ExtractingIRMaterializationUnit"); + } + + mutable std::mutex SourceModuleMutex; + CompileOnDemandLayer &Parent; +}; + +Optional<CompileOnDemandLayer::GlobalValueSet> +CompileOnDemandLayer::compileRequested(GlobalValueSet Requested) { + return std::move(Requested); +} + +Optional<CompileOnDemandLayer::GlobalValueSet> +CompileOnDemandLayer::compileWholeModule(GlobalValueSet Requested) { + return None; +} + +CompileOnDemandLayer::CompileOnDemandLayer( + ExecutionSession &ES, IRLayer &BaseLayer, LazyCallThroughManager &LCTMgr, + IndirectStubsManagerBuilder BuildIndirectStubsManager) + : IRLayer(ES, BaseLayer.getManglingOptions()), BaseLayer(BaseLayer), + LCTMgr(LCTMgr), + BuildIndirectStubsManager(std::move(BuildIndirectStubsManager)) {} + +void CompileOnDemandLayer::setPartitionFunction(PartitionFunction Partition) { + this->Partition = std::move(Partition); +} + +void CompileOnDemandLayer::setImplMap(ImplSymbolMap *Imp) { + this->AliaseeImpls = Imp; +} +void CompileOnDemandLayer::emit( + std::unique_ptr<MaterializationResponsibility> R, ThreadSafeModule TSM) { + assert(TSM && "Null module"); + + auto &ES = getExecutionSession(); + + // Sort the callables and non-callables, build re-exports and lodge the + // actual module with the implementation dylib. + auto &PDR = getPerDylibResources(R->getTargetJITDylib()); + + SymbolAliasMap NonCallables; + SymbolAliasMap Callables; + TSM.withModuleDo([&](Module &M) { + // First, do some cleanup on the module: + cleanUpModule(M); + }); + + for (auto &KV : R->getSymbols()) { + auto &Name = KV.first; + auto &Flags = KV.second; + if (Flags.isCallable()) + Callables[Name] = SymbolAliasMapEntry(Name, Flags); + else + NonCallables[Name] = SymbolAliasMapEntry(Name, Flags); + } + + // Create a partitioning materialization unit and lodge it with the + // implementation dylib. + if (auto Err = PDR.getImplDylib().define( + std::make_unique<PartitioningIRMaterializationUnit>( + ES, *getManglingOptions(), std::move(TSM), *this))) { + ES.reportError(std::move(Err)); + R->failMaterialization(); + return; + } + + if (!NonCallables.empty()) + if (auto Err = + R->replace(reexports(PDR.getImplDylib(), std::move(NonCallables), + JITDylibLookupFlags::MatchAllSymbols))) { + getExecutionSession().reportError(std::move(Err)); + R->failMaterialization(); + return; + } + if (!Callables.empty()) { + if (auto Err = R->replace( + lazyReexports(LCTMgr, PDR.getISManager(), PDR.getImplDylib(), + std::move(Callables), AliaseeImpls))) { + getExecutionSession().reportError(std::move(Err)); + R->failMaterialization(); + return; + } + } +} + +CompileOnDemandLayer::PerDylibResources & +CompileOnDemandLayer::getPerDylibResources(JITDylib &TargetD) { + auto I = DylibResources.find(&TargetD); + if (I == DylibResources.end()) { + auto &ImplD = + getExecutionSession().createBareJITDylib(TargetD.getName() + ".impl"); + JITDylibSearchOrder NewLinkOrder; + TargetD.withLinkOrderDo([&](const JITDylibSearchOrder &TargetLinkOrder) { + NewLinkOrder = TargetLinkOrder; + }); + + assert(!NewLinkOrder.empty() && NewLinkOrder.front().first == &TargetD && + NewLinkOrder.front().second == + JITDylibLookupFlags::MatchAllSymbols && + "TargetD must be at the front of its own search order and match " + "non-exported symbol"); + NewLinkOrder.insert(std::next(NewLinkOrder.begin()), + {&ImplD, JITDylibLookupFlags::MatchAllSymbols}); + ImplD.setLinkOrder(NewLinkOrder, false); + TargetD.setLinkOrder(std::move(NewLinkOrder), false); + + PerDylibResources PDR(ImplD, BuildIndirectStubsManager()); + I = DylibResources.insert(std::make_pair(&TargetD, std::move(PDR))).first; + } + + return I->second; +} + +void CompileOnDemandLayer::cleanUpModule(Module &M) { + for (auto &F : M.functions()) { + if (F.isDeclaration()) + continue; + + if (F.hasAvailableExternallyLinkage()) { + F.deleteBody(); + F.setPersonalityFn(nullptr); + continue; + } + } +} + +void CompileOnDemandLayer::expandPartition(GlobalValueSet &Partition) { + // Expands the partition to ensure the following rules hold: + // (1) If any alias is in the partition, its aliasee is also in the partition. + // (2) If any aliasee is in the partition, its aliases are also in the + // partiton. + // (3) If any global variable is in the partition then all global variables + // are in the partition. + assert(!Partition.empty() && "Unexpected empty partition"); + + const Module &M = *(*Partition.begin())->getParent(); + bool ContainsGlobalVariables = false; + std::vector<const GlobalValue *> GVsToAdd; + + for (auto *GV : Partition) + if (isa<GlobalAlias>(GV)) + GVsToAdd.push_back( + cast<GlobalValue>(cast<GlobalAlias>(GV)->getAliasee())); + else if (isa<GlobalVariable>(GV)) + ContainsGlobalVariables = true; + + for (auto &A : M.aliases()) + if (Partition.count(cast<GlobalValue>(A.getAliasee()))) + GVsToAdd.push_back(&A); + + if (ContainsGlobalVariables) + for (auto &G : M.globals()) + GVsToAdd.push_back(&G); + + for (auto *GV : GVsToAdd) + Partition.insert(GV); +} + +void CompileOnDemandLayer::emitPartition( + std::unique_ptr<MaterializationResponsibility> R, ThreadSafeModule TSM, + IRMaterializationUnit::SymbolNameToDefinitionMap Defs) { + + // FIXME: Need a 'notify lazy-extracting/emitting' callback to tie the + // extracted module key, extracted module, and source module key + // together. This could be used, for example, to provide a specific + // memory manager instance to the linking layer. + + auto &ES = getExecutionSession(); + GlobalValueSet RequestedGVs; + for (auto &Name : R->getRequestedSymbols()) { + if (Name == R->getInitializerSymbol()) + TSM.withModuleDo([&](Module &M) { + for (auto &GV : getStaticInitGVs(M)) + RequestedGVs.insert(&GV); + }); + else { + assert(Defs.count(Name) && "No definition for symbol"); + RequestedGVs.insert(Defs[Name]); + } + } + + /// Perform partitioning with the context lock held, since the partition + /// function is allowed to access the globals to compute the partition. + auto GVsToExtract = + TSM.withModuleDo([&](Module &M) { return Partition(RequestedGVs); }); + + // Take a 'None' partition to mean the whole module (as opposed to an empty + // partition, which means "materialize nothing"). Emit the whole module + // unmodified to the base layer. + if (GVsToExtract == None) { + Defs.clear(); + BaseLayer.emit(std::move(R), std::move(TSM)); + return; + } + + // If the partition is empty, return the whole module to the symbol table. + if (GVsToExtract->empty()) { + if (auto Err = + R->replace(std::make_unique<PartitioningIRMaterializationUnit>( + std::move(TSM), R->getSymbols(), R->getInitializerSymbol(), + std::move(Defs), *this))) { + getExecutionSession().reportError(std::move(Err)); + R->failMaterialization(); + return; + } + return; + } + + // Ok -- we actually need to partition the symbols. Promote the symbol + // linkages/names, expand the partition to include any required symbols + // (i.e. symbols that can't be separated from our partition), and + // then extract the partition. + // + // FIXME: We apply this promotion once per partitioning. It's safe, but + // overkill. + auto ExtractedTSM = + TSM.withModuleDo([&](Module &M) -> Expected<ThreadSafeModule> { + auto PromotedGlobals = PromoteSymbols(M); + if (!PromotedGlobals.empty()) { + + MangleAndInterner Mangle(ES, M.getDataLayout()); + SymbolFlagsMap SymbolFlags; + IRSymbolMapper::add(ES, *getManglingOptions(), + PromotedGlobals, SymbolFlags); + + if (auto Err = R->defineMaterializing(SymbolFlags)) + return std::move(Err); + } + + expandPartition(*GVsToExtract); + + // Submodule name is given by hashing the names of the globals. + std::string SubModuleName; + { + std::vector<const GlobalValue*> HashGVs; + HashGVs.reserve(GVsToExtract->size()); + for (auto *GV : *GVsToExtract) + HashGVs.push_back(GV); + llvm::sort(HashGVs, [](const GlobalValue *LHS, const GlobalValue *RHS) { + return LHS->getName() < RHS->getName(); + }); + hash_code HC(0); + for (auto *GV : HashGVs) { + assert(GV->hasName() && "All GVs to extract should be named by now"); + auto GVName = GV->getName(); + HC = hash_combine(HC, hash_combine_range(GVName.begin(), GVName.end())); + } + raw_string_ostream(SubModuleName) + << ".submodule." + << formatv(sizeof(size_t) == 8 ? "{0:x16}" : "{0:x8}", + static_cast<size_t>(HC)) + << ".ll"; + } + + // Extract the requested partiton (plus any necessary aliases) and + // put the rest back into the impl dylib. + auto ShouldExtract = [&](const GlobalValue &GV) -> bool { + return GVsToExtract->count(&GV); + }; + + return extractSubModule(TSM, SubModuleName , ShouldExtract); + }); + + if (!ExtractedTSM) { + ES.reportError(ExtractedTSM.takeError()); + R->failMaterialization(); + return; + } + + if (auto Err = R->replace(std::make_unique<PartitioningIRMaterializationUnit>( + ES, *getManglingOptions(), std::move(TSM), *this))) { + ES.reportError(std::move(Err)); + R->failMaterialization(); + return; + } + BaseLayer.emit(std::move(R), std::move(*ExtractedTSM)); +} + +} // end namespace orc +} // end namespace llvm diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/Orc/CompileUtils.cpp b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/CompileUtils.cpp new file mode 100644 index 0000000000..f8efed15ed --- /dev/null +++ b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/CompileUtils.cpp @@ -0,0 +1,94 @@ +//===------ CompileUtils.cpp - Utilities for compiling IR in the JIT ------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/CompileUtils.h" + +#include "llvm/ADT/SmallVector.h" +#include "llvm/ExecutionEngine/ObjectCache.h" +#include "llvm/IR/LegacyPassManager.h" +#include "llvm/IR/Module.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/SmallVectorMemoryBuffer.h" +#include "llvm/Target/TargetMachine.h" + +#include <algorithm> + +namespace llvm { +namespace orc { + +IRSymbolMapper::ManglingOptions +irManglingOptionsFromTargetOptions(const TargetOptions &Opts) { + IRSymbolMapper::ManglingOptions MO; + + MO.EmulatedTLS = Opts.EmulatedTLS; + + return MO; +} + +/// Compile a Module to an ObjectFile. +Expected<SimpleCompiler::CompileResult> SimpleCompiler::operator()(Module &M) { + CompileResult CachedObject = tryToLoadFromObjectCache(M); + if (CachedObject) + return std::move(CachedObject); + + SmallVector<char, 0> ObjBufferSV; + + { + raw_svector_ostream ObjStream(ObjBufferSV); + + legacy::PassManager PM; + MCContext *Ctx; + if (TM.addPassesToEmitMC(PM, Ctx, ObjStream)) + return make_error<StringError>("Target does not support MC emission", + inconvertibleErrorCode()); + PM.run(M); + } + + auto ObjBuffer = std::make_unique<SmallVectorMemoryBuffer>( + std::move(ObjBufferSV), M.getModuleIdentifier() + "-jitted-objectbuffer"); + + auto Obj = object::ObjectFile::createObjectFile(ObjBuffer->getMemBufferRef()); + + if (!Obj) + return Obj.takeError(); + + notifyObjectCompiled(M, *ObjBuffer); + return std::move(ObjBuffer); +} + +SimpleCompiler::CompileResult +SimpleCompiler::tryToLoadFromObjectCache(const Module &M) { + if (!ObjCache) + return CompileResult(); + + return ObjCache->getObject(&M); +} + +void SimpleCompiler::notifyObjectCompiled(const Module &M, + const MemoryBuffer &ObjBuffer) { + if (ObjCache) + ObjCache->notifyObjectCompiled(&M, ObjBuffer.getMemBufferRef()); +} + +ConcurrentIRCompiler::ConcurrentIRCompiler(JITTargetMachineBuilder JTMB, + ObjectCache *ObjCache) + : IRCompiler(irManglingOptionsFromTargetOptions(JTMB.getOptions())), + JTMB(std::move(JTMB)), ObjCache(ObjCache) {} + +Expected<std::unique_ptr<MemoryBuffer>> +ConcurrentIRCompiler::operator()(Module &M) { + auto TM = cantFail(JTMB.createTargetMachine()); + SimpleCompiler C(*TM, ObjCache); + return C(M); +} + +} // end namespace orc +} // end namespace llvm diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/Orc/Core.cpp b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/Core.cpp new file mode 100644 index 0000000000..dfb558808c --- /dev/null +++ b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/Core.cpp @@ -0,0 +1,2777 @@ +//===--- Core.cpp - Core ORC APIs (MaterializationUnit, JITDylib, etc.) ---===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/Core.h" + +#include "llvm/ADT/STLExtras.h" +#include "llvm/Config/llvm-config.h" +#include "llvm/ExecutionEngine/Orc/DebugUtils.h" +#include "llvm/ExecutionEngine/Orc/Shared/OrcError.h" +#include "llvm/Support/FormatVariadic.h" +#include "llvm/Support/MSVCErrorWorkarounds.h" + +#include <condition_variable> +#include <future> + +#define DEBUG_TYPE "orc" + +namespace llvm { +namespace orc { + +char ResourceTrackerDefunct::ID = 0; +char FailedToMaterialize::ID = 0; +char SymbolsNotFound::ID = 0; +char SymbolsCouldNotBeRemoved::ID = 0; +char MissingSymbolDefinitions::ID = 0; +char UnexpectedSymbolDefinitions::ID = 0; + +RegisterDependenciesFunction NoDependenciesToRegister = + RegisterDependenciesFunction(); + +void MaterializationUnit::anchor() {} + +ResourceTracker::ResourceTracker(JITDylibSP JD) { + assert((reinterpret_cast<uintptr_t>(JD.get()) & 0x1) == 0 && + "JITDylib must be two byte aligned"); + JD->Retain(); + JDAndFlag.store(reinterpret_cast<uintptr_t>(JD.get())); +} + +ResourceTracker::~ResourceTracker() { + getJITDylib().getExecutionSession().destroyResourceTracker(*this); + getJITDylib().Release(); +} + +Error ResourceTracker::remove() { + return getJITDylib().getExecutionSession().removeResourceTracker(*this); +} + +void ResourceTracker::transferTo(ResourceTracker &DstRT) { + getJITDylib().getExecutionSession().transferResourceTracker(DstRT, *this); +} + +void ResourceTracker::makeDefunct() { + uintptr_t Val = JDAndFlag.load(); + Val |= 0x1U; + JDAndFlag.store(Val); +} + +ResourceManager::~ResourceManager() {} + +ResourceTrackerDefunct::ResourceTrackerDefunct(ResourceTrackerSP RT) + : RT(std::move(RT)) {} + +std::error_code ResourceTrackerDefunct::convertToErrorCode() const { + return orcError(OrcErrorCode::UnknownORCError); +} + +void ResourceTrackerDefunct::log(raw_ostream &OS) const { + OS << "Resource tracker " << (void *)RT.get() << " became defunct"; +} + +FailedToMaterialize::FailedToMaterialize( + std::shared_ptr<SymbolDependenceMap> Symbols) + : Symbols(std::move(Symbols)) { + assert(!this->Symbols->empty() && "Can not fail to resolve an empty set"); +} + +std::error_code FailedToMaterialize::convertToErrorCode() const { + return orcError(OrcErrorCode::UnknownORCError); +} + +void FailedToMaterialize::log(raw_ostream &OS) const { + OS << "Failed to materialize symbols: " << *Symbols; +} + +SymbolsNotFound::SymbolsNotFound(SymbolNameSet Symbols) { + for (auto &Sym : Symbols) + this->Symbols.push_back(Sym); + assert(!this->Symbols.empty() && "Can not fail to resolve an empty set"); +} + +SymbolsNotFound::SymbolsNotFound(SymbolNameVector Symbols) + : Symbols(std::move(Symbols)) { + assert(!this->Symbols.empty() && "Can not fail to resolve an empty set"); +} + +std::error_code SymbolsNotFound::convertToErrorCode() const { + return orcError(OrcErrorCode::UnknownORCError); +} + +void SymbolsNotFound::log(raw_ostream &OS) const { + OS << "Symbols not found: " << Symbols; +} + +SymbolsCouldNotBeRemoved::SymbolsCouldNotBeRemoved(SymbolNameSet Symbols) + : Symbols(std::move(Symbols)) { + assert(!this->Symbols.empty() && "Can not fail to resolve an empty set"); +} + +std::error_code SymbolsCouldNotBeRemoved::convertToErrorCode() const { + return orcError(OrcErrorCode::UnknownORCError); +} + +void SymbolsCouldNotBeRemoved::log(raw_ostream &OS) const { + OS << "Symbols could not be removed: " << Symbols; +} + +std::error_code MissingSymbolDefinitions::convertToErrorCode() const { + return orcError(OrcErrorCode::MissingSymbolDefinitions); +} + +void MissingSymbolDefinitions::log(raw_ostream &OS) const { + OS << "Missing definitions in module " << ModuleName + << ": " << Symbols; +} + +std::error_code UnexpectedSymbolDefinitions::convertToErrorCode() const { + return orcError(OrcErrorCode::UnexpectedSymbolDefinitions); +} + +void UnexpectedSymbolDefinitions::log(raw_ostream &OS) const { + OS << "Unexpected definitions in module " << ModuleName + << ": " << Symbols; +} + +AsynchronousSymbolQuery::AsynchronousSymbolQuery( + const SymbolLookupSet &Symbols, SymbolState RequiredState, + SymbolsResolvedCallback NotifyComplete) + : NotifyComplete(std::move(NotifyComplete)), RequiredState(RequiredState) { + assert(RequiredState >= SymbolState::Resolved && + "Cannot query for a symbols that have not reached the resolve state " + "yet"); + + OutstandingSymbolsCount = Symbols.size(); + + for (auto &KV : Symbols) + ResolvedSymbols[KV.first] = nullptr; +} + +void AsynchronousSymbolQuery::notifySymbolMetRequiredState( + const SymbolStringPtr &Name, JITEvaluatedSymbol Sym) { + auto I = ResolvedSymbols.find(Name); + assert(I != ResolvedSymbols.end() && + "Resolving symbol outside the requested set"); + assert(I->second.getAddress() == 0 && "Redundantly resolving symbol Name"); + + // If this is a materialization-side-effects-only symbol then drop it, + // otherwise update its map entry with its resolved address. + if (Sym.getFlags().hasMaterializationSideEffectsOnly()) + ResolvedSymbols.erase(I); + else + I->second = std::move(Sym); + --OutstandingSymbolsCount; +} + +void AsynchronousSymbolQuery::handleComplete() { + assert(OutstandingSymbolsCount == 0 && + "Symbols remain, handleComplete called prematurely"); + + auto TmpNotifyComplete = std::move(NotifyComplete); + NotifyComplete = SymbolsResolvedCallback(); + TmpNotifyComplete(std::move(ResolvedSymbols)); +} + +void AsynchronousSymbolQuery::handleFailed(Error Err) { + assert(QueryRegistrations.empty() && ResolvedSymbols.empty() && + OutstandingSymbolsCount == 0 && + "Query should already have been abandoned"); + NotifyComplete(std::move(Err)); + NotifyComplete = SymbolsResolvedCallback(); +} + +void AsynchronousSymbolQuery::addQueryDependence(JITDylib &JD, + SymbolStringPtr Name) { + bool Added = QueryRegistrations[&JD].insert(std::move(Name)).second; + (void)Added; + assert(Added && "Duplicate dependence notification?"); +} + +void AsynchronousSymbolQuery::removeQueryDependence( + JITDylib &JD, const SymbolStringPtr &Name) { + auto QRI = QueryRegistrations.find(&JD); + assert(QRI != QueryRegistrations.end() && + "No dependencies registered for JD"); + assert(QRI->second.count(Name) && "No dependency on Name in JD"); + QRI->second.erase(Name); + if (QRI->second.empty()) + QueryRegistrations.erase(QRI); +} + +void AsynchronousSymbolQuery::dropSymbol(const SymbolStringPtr &Name) { + auto I = ResolvedSymbols.find(Name); + assert(I != ResolvedSymbols.end() && + "Redundant removal of weakly-referenced symbol"); + ResolvedSymbols.erase(I); + --OutstandingSymbolsCount; +} + +void AsynchronousSymbolQuery::detach() { + ResolvedSymbols.clear(); + OutstandingSymbolsCount = 0; + for (auto &KV : QueryRegistrations) + KV.first->detachQueryHelper(*this, KV.second); + QueryRegistrations.clear(); +} + +AbsoluteSymbolsMaterializationUnit::AbsoluteSymbolsMaterializationUnit( + SymbolMap Symbols) + : MaterializationUnit(extractFlags(Symbols), nullptr), + Symbols(std::move(Symbols)) {} + +StringRef AbsoluteSymbolsMaterializationUnit::getName() const { + return "<Absolute Symbols>"; +} + +void AbsoluteSymbolsMaterializationUnit::materialize( + std::unique_ptr<MaterializationResponsibility> R) { + // No dependencies, so these calls can't fail. + cantFail(R->notifyResolved(Symbols)); + cantFail(R->notifyEmitted()); +} + +void AbsoluteSymbolsMaterializationUnit::discard(const JITDylib &JD, + const SymbolStringPtr &Name) { + assert(Symbols.count(Name) && "Symbol is not part of this MU"); + Symbols.erase(Name); +} + +SymbolFlagsMap +AbsoluteSymbolsMaterializationUnit::extractFlags(const SymbolMap &Symbols) { + SymbolFlagsMap Flags; + for (const auto &KV : Symbols) + Flags[KV.first] = KV.second.getFlags(); + return Flags; +} + +ReExportsMaterializationUnit::ReExportsMaterializationUnit( + JITDylib *SourceJD, JITDylibLookupFlags SourceJDLookupFlags, + SymbolAliasMap Aliases) + : MaterializationUnit(extractFlags(Aliases), nullptr), SourceJD(SourceJD), + SourceJDLookupFlags(SourceJDLookupFlags), Aliases(std::move(Aliases)) {} + +StringRef ReExportsMaterializationUnit::getName() const { + return "<Reexports>"; +} + +void ReExportsMaterializationUnit::materialize( + std::unique_ptr<MaterializationResponsibility> R) { + + auto &ES = R->getTargetJITDylib().getExecutionSession(); + JITDylib &TgtJD = R->getTargetJITDylib(); + JITDylib &SrcJD = SourceJD ? *SourceJD : TgtJD; + + // Find the set of requested aliases and aliasees. Return any unrequested + // aliases back to the JITDylib so as to not prematurely materialize any + // aliasees. + auto RequestedSymbols = R->getRequestedSymbols(); + SymbolAliasMap RequestedAliases; + + for (auto &Name : RequestedSymbols) { + auto I = Aliases.find(Name); + assert(I != Aliases.end() && "Symbol not found in aliases map?"); + RequestedAliases[Name] = std::move(I->second); + Aliases.erase(I); + } + + LLVM_DEBUG({ + ES.runSessionLocked([&]() { + dbgs() << "materializing reexports: target = " << TgtJD.getName() + << ", source = " << SrcJD.getName() << " " << RequestedAliases + << "\n"; + }); + }); + + if (!Aliases.empty()) { + auto Err = SourceJD ? R->replace(reexports(*SourceJD, std::move(Aliases), + SourceJDLookupFlags)) + : R->replace(symbolAliases(std::move(Aliases))); + + if (Err) { + // FIXME: Should this be reported / treated as failure to materialize? + // Or should this be treated as a sanctioned bailing-out? + ES.reportError(std::move(Err)); + R->failMaterialization(); + return; + } + } + + // The OnResolveInfo struct will hold the aliases and responsibilty for each + // query in the list. + struct OnResolveInfo { + OnResolveInfo(std::unique_ptr<MaterializationResponsibility> R, + SymbolAliasMap Aliases) + : R(std::move(R)), Aliases(std::move(Aliases)) {} + + std::unique_ptr<MaterializationResponsibility> R; + SymbolAliasMap Aliases; + }; + + // Build a list of queries to issue. In each round we build a query for the + // largest set of aliases that we can resolve without encountering a chain of + // aliases (e.g. Foo -> Bar, Bar -> Baz). Such a chain would deadlock as the + // query would be waiting on a symbol that it itself had to resolve. Creating + // a new query for each link in such a chain eliminates the possibility of + // deadlock. In practice chains are likely to be rare, and this algorithm will + // usually result in a single query to issue. + + std::vector<std::pair<SymbolLookupSet, std::shared_ptr<OnResolveInfo>>> + QueryInfos; + while (!RequestedAliases.empty()) { + SymbolNameSet ResponsibilitySymbols; + SymbolLookupSet QuerySymbols; + SymbolAliasMap QueryAliases; + + // Collect as many aliases as we can without including a chain. + for (auto &KV : RequestedAliases) { + // Chain detected. Skip this symbol for this round. + if (&SrcJD == &TgtJD && (QueryAliases.count(KV.second.Aliasee) || + RequestedAliases.count(KV.second.Aliasee))) + continue; + + ResponsibilitySymbols.insert(KV.first); + QuerySymbols.add(KV.second.Aliasee, + KV.second.AliasFlags.hasMaterializationSideEffectsOnly() + ? SymbolLookupFlags::WeaklyReferencedSymbol + : SymbolLookupFlags::RequiredSymbol); + QueryAliases[KV.first] = std::move(KV.second); + } + + // Remove the aliases collected this round from the RequestedAliases map. + for (auto &KV : QueryAliases) + RequestedAliases.erase(KV.first); + + assert(!QuerySymbols.empty() && "Alias cycle detected!"); + + auto NewR = R->delegate(ResponsibilitySymbols); + if (!NewR) { + ES.reportError(NewR.takeError()); + R->failMaterialization(); + return; + } + + auto QueryInfo = std::make_shared<OnResolveInfo>(std::move(*NewR), + std::move(QueryAliases)); + QueryInfos.push_back( + make_pair(std::move(QuerySymbols), std::move(QueryInfo))); + } + + // Issue the queries. + while (!QueryInfos.empty()) { + auto QuerySymbols = std::move(QueryInfos.back().first); + auto QueryInfo = std::move(QueryInfos.back().second); + + QueryInfos.pop_back(); + + auto RegisterDependencies = [QueryInfo, + &SrcJD](const SymbolDependenceMap &Deps) { + // If there were no materializing symbols, just bail out. + if (Deps.empty()) + return; + + // Otherwise the only deps should be on SrcJD. + assert(Deps.size() == 1 && Deps.count(&SrcJD) && + "Unexpected dependencies for reexports"); + + auto &SrcJDDeps = Deps.find(&SrcJD)->second; + SymbolDependenceMap PerAliasDepsMap; + auto &PerAliasDeps = PerAliasDepsMap[&SrcJD]; + + for (auto &KV : QueryInfo->Aliases) + if (SrcJDDeps.count(KV.second.Aliasee)) { + PerAliasDeps = {KV.second.Aliasee}; + QueryInfo->R->addDependencies(KV.first, PerAliasDepsMap); + } + }; + + auto OnComplete = [QueryInfo](Expected<SymbolMap> Result) { + auto &ES = QueryInfo->R->getTargetJITDylib().getExecutionSession(); + if (Result) { + SymbolMap ResolutionMap; + for (auto &KV : QueryInfo->Aliases) { + assert((KV.second.AliasFlags.hasMaterializationSideEffectsOnly() || + Result->count(KV.second.Aliasee)) && + "Result map missing entry?"); + // Don't try to resolve materialization-side-effects-only symbols. + if (KV.second.AliasFlags.hasMaterializationSideEffectsOnly()) + continue; + + ResolutionMap[KV.first] = JITEvaluatedSymbol( + (*Result)[KV.second.Aliasee].getAddress(), KV.second.AliasFlags); + } + if (auto Err = QueryInfo->R->notifyResolved(ResolutionMap)) { + ES.reportError(std::move(Err)); + QueryInfo->R->failMaterialization(); + return; + } + if (auto Err = QueryInfo->R->notifyEmitted()) { + ES.reportError(std::move(Err)); + QueryInfo->R->failMaterialization(); + return; + } + } else { + ES.reportError(Result.takeError()); + QueryInfo->R->failMaterialization(); + } + }; + + ES.lookup(LookupKind::Static, + JITDylibSearchOrder({{&SrcJD, SourceJDLookupFlags}}), + QuerySymbols, SymbolState::Resolved, std::move(OnComplete), + std::move(RegisterDependencies)); + } +} + +void ReExportsMaterializationUnit::discard(const JITDylib &JD, + const SymbolStringPtr &Name) { + assert(Aliases.count(Name) && + "Symbol not covered by this MaterializationUnit"); + Aliases.erase(Name); +} + +SymbolFlagsMap +ReExportsMaterializationUnit::extractFlags(const SymbolAliasMap &Aliases) { + SymbolFlagsMap SymbolFlags; + for (auto &KV : Aliases) + SymbolFlags[KV.first] = KV.second.AliasFlags; + + return SymbolFlags; +} + +Expected<SymbolAliasMap> buildSimpleReexportsAliasMap(JITDylib &SourceJD, + SymbolNameSet Symbols) { + SymbolLookupSet LookupSet(Symbols); + auto Flags = SourceJD.getExecutionSession().lookupFlags( + LookupKind::Static, {{&SourceJD, JITDylibLookupFlags::MatchAllSymbols}}, + SymbolLookupSet(std::move(Symbols))); + + if (!Flags) + return Flags.takeError(); + + SymbolAliasMap Result; + for (auto &Name : Symbols) { + assert(Flags->count(Name) && "Missing entry in flags map"); + Result[Name] = SymbolAliasMapEntry(Name, (*Flags)[Name]); + } + + return Result; +} + +class InProgressLookupState { +public: + InProgressLookupState(LookupKind K, JITDylibSearchOrder SearchOrder, + SymbolLookupSet LookupSet, SymbolState RequiredState) + : K(K), SearchOrder(std::move(SearchOrder)), + LookupSet(std::move(LookupSet)), RequiredState(RequiredState) { + DefGeneratorCandidates = this->LookupSet; + } + virtual ~InProgressLookupState() {} + virtual void complete(std::unique_ptr<InProgressLookupState> IPLS) = 0; + virtual void fail(Error Err) = 0; + + LookupKind K; + JITDylibSearchOrder SearchOrder; + SymbolLookupSet LookupSet; + SymbolState RequiredState; + + std::unique_lock<std::mutex> GeneratorLock; + size_t CurSearchOrderIndex = 0; + bool NewJITDylib = true; + SymbolLookupSet DefGeneratorCandidates; + SymbolLookupSet DefGeneratorNonCandidates; + std::vector<std::weak_ptr<DefinitionGenerator>> CurDefGeneratorStack; +}; + +class InProgressLookupFlagsState : public InProgressLookupState { +public: + InProgressLookupFlagsState( + LookupKind K, JITDylibSearchOrder SearchOrder, SymbolLookupSet LookupSet, + unique_function<void(Expected<SymbolFlagsMap>)> OnComplete) + : InProgressLookupState(K, std::move(SearchOrder), std::move(LookupSet), + SymbolState::NeverSearched), + OnComplete(std::move(OnComplete)) {} + + void complete(std::unique_ptr<InProgressLookupState> IPLS) override { + GeneratorLock = {}; // Unlock and release. + auto &ES = SearchOrder.front().first->getExecutionSession(); + ES.OL_completeLookupFlags(std::move(IPLS), std::move(OnComplete)); + } + + void fail(Error Err) override { + GeneratorLock = {}; // Unlock and release. + OnComplete(std::move(Err)); + } + +private: + unique_function<void(Expected<SymbolFlagsMap>)> OnComplete; +}; + +class InProgressFullLookupState : public InProgressLookupState { +public: + InProgressFullLookupState(LookupKind K, JITDylibSearchOrder SearchOrder, + SymbolLookupSet LookupSet, + SymbolState RequiredState, + std::shared_ptr<AsynchronousSymbolQuery> Q, + RegisterDependenciesFunction RegisterDependencies) + : InProgressLookupState(K, std::move(SearchOrder), std::move(LookupSet), + RequiredState), + Q(std::move(Q)), RegisterDependencies(std::move(RegisterDependencies)) { + } + + void complete(std::unique_ptr<InProgressLookupState> IPLS) override { + GeneratorLock = {}; // Unlock and release. + auto &ES = SearchOrder.front().first->getExecutionSession(); + ES.OL_completeLookup(std::move(IPLS), std::move(Q), + std::move(RegisterDependencies)); + } + + void fail(Error Err) override { + GeneratorLock = {}; + Q->detach(); + Q->handleFailed(std::move(Err)); + } + +private: + std::shared_ptr<AsynchronousSymbolQuery> Q; + RegisterDependenciesFunction RegisterDependencies; +}; + +ReexportsGenerator::ReexportsGenerator(JITDylib &SourceJD, + JITDylibLookupFlags SourceJDLookupFlags, + SymbolPredicate Allow) + : SourceJD(SourceJD), SourceJDLookupFlags(SourceJDLookupFlags), + Allow(std::move(Allow)) {} + +Error ReexportsGenerator::tryToGenerate(LookupState &LS, LookupKind K, + JITDylib &JD, + JITDylibLookupFlags JDLookupFlags, + const SymbolLookupSet &LookupSet) { + assert(&JD != &SourceJD && "Cannot re-export from the same dylib"); + + // Use lookupFlags to find the subset of symbols that match our lookup. + auto Flags = JD.getExecutionSession().lookupFlags( + K, {{&SourceJD, JDLookupFlags}}, LookupSet); + if (!Flags) + return Flags.takeError(); + + // Create an alias map. + orc::SymbolAliasMap AliasMap; + for (auto &KV : *Flags) + if (!Allow || Allow(KV.first)) + AliasMap[KV.first] = SymbolAliasMapEntry(KV.first, KV.second); + + if (AliasMap.empty()) + return Error::success(); + + // Define the re-exports. + return JD.define(reexports(SourceJD, AliasMap, SourceJDLookupFlags)); +} + +LookupState::LookupState(std::unique_ptr<InProgressLookupState> IPLS) + : IPLS(std::move(IPLS)) {} + +void LookupState::reset(InProgressLookupState *IPLS) { this->IPLS.reset(IPLS); } + +LookupState::LookupState() = default; +LookupState::LookupState(LookupState &&) = default; +LookupState &LookupState::operator=(LookupState &&) = default; +LookupState::~LookupState() = default; + +void LookupState::continueLookup(Error Err) { + assert(IPLS && "Cannot call continueLookup on empty LookupState"); + auto &ES = IPLS->SearchOrder.begin()->first->getExecutionSession(); + ES.OL_applyQueryPhase1(std::move(IPLS), std::move(Err)); +} + +DefinitionGenerator::~DefinitionGenerator() {} + +Error JITDylib::clear() { + std::vector<ResourceTrackerSP> TrackersToRemove; + ES.runSessionLocked([&]() { + for (auto &KV : TrackerSymbols) + TrackersToRemove.push_back(KV.first); + TrackersToRemove.push_back(getDefaultResourceTracker()); + }); + + Error Err = Error::success(); + for (auto &RT : TrackersToRemove) + Err = joinErrors(std::move(Err), RT->remove()); + return Err; +} + +ResourceTrackerSP JITDylib::getDefaultResourceTracker() { + return ES.runSessionLocked([this] { + if (!DefaultTracker) + DefaultTracker = new ResourceTracker(this); + return DefaultTracker; + }); +} + +ResourceTrackerSP JITDylib::createResourceTracker() { + return ES.runSessionLocked([this] { + ResourceTrackerSP RT = new ResourceTracker(this); + return RT; + }); +} + +void JITDylib::removeGenerator(DefinitionGenerator &G) { + std::lock_guard<std::mutex> Lock(GeneratorsMutex); + auto I = llvm::find_if(DefGenerators, + [&](const std::shared_ptr<DefinitionGenerator> &H) { + return H.get() == &G; + }); + assert(I != DefGenerators.end() && "Generator not found"); + DefGenerators.erase(I); +} + +Expected<SymbolFlagsMap> +JITDylib::defineMaterializing(SymbolFlagsMap SymbolFlags) { + + return ES.runSessionLocked([&]() -> Expected<SymbolFlagsMap> { + std::vector<SymbolTable::iterator> AddedSyms; + std::vector<SymbolFlagsMap::iterator> RejectedWeakDefs; + + for (auto SFItr = SymbolFlags.begin(), SFEnd = SymbolFlags.end(); + SFItr != SFEnd; ++SFItr) { + + auto &Name = SFItr->first; + auto &Flags = SFItr->second; + + auto EntryItr = Symbols.find(Name); + + // If the entry already exists... + if (EntryItr != Symbols.end()) { + + // If this is a strong definition then error out. + if (!Flags.isWeak()) { + // Remove any symbols already added. + for (auto &SI : AddedSyms) + Symbols.erase(SI); + + // FIXME: Return all duplicates. + return make_error<DuplicateDefinition>(std::string(*Name)); + } + + // Otherwise just make a note to discard this symbol after the loop. + RejectedWeakDefs.push_back(SFItr); + continue; + } else + EntryItr = + Symbols.insert(std::make_pair(Name, SymbolTableEntry(Flags))).first; + + AddedSyms.push_back(EntryItr); + EntryItr->second.setState(SymbolState::Materializing); + } + + // Remove any rejected weak definitions from the SymbolFlags map. + while (!RejectedWeakDefs.empty()) { + SymbolFlags.erase(RejectedWeakDefs.back()); + RejectedWeakDefs.pop_back(); + } + + return SymbolFlags; + }); +} + +Error JITDylib::replace(MaterializationResponsibility &FromMR, + std::unique_ptr<MaterializationUnit> MU) { + assert(MU != nullptr && "Can not replace with a null MaterializationUnit"); + std::unique_ptr<MaterializationUnit> MustRunMU; + std::unique_ptr<MaterializationResponsibility> MustRunMR; + + auto Err = + ES.runSessionLocked([&, this]() -> Error { + auto RT = getTracker(FromMR); + + if (RT->isDefunct()) + return make_error<ResourceTrackerDefunct>(std::move(RT)); + +#ifndef NDEBUG + for (auto &KV : MU->getSymbols()) { + auto SymI = Symbols.find(KV.first); + assert(SymI != Symbols.end() && "Replacing unknown symbol"); + assert(SymI->second.getState() == SymbolState::Materializing && + "Can not replace a symbol that ha is not materializing"); + assert(!SymI->second.hasMaterializerAttached() && + "Symbol should not have materializer attached already"); + assert(UnmaterializedInfos.count(KV.first) == 0 && + "Symbol being replaced should have no UnmaterializedInfo"); + } +#endif // NDEBUG + + // If the tracker is defunct we need to bail out immediately. + + // If any symbol has pending queries against it then we need to + // materialize MU immediately. + for (auto &KV : MU->getSymbols()) { + auto MII = MaterializingInfos.find(KV.first); + if (MII != MaterializingInfos.end()) { + if (MII->second.hasQueriesPending()) { + MustRunMR = ES.createMaterializationResponsibility( + *RT, std::move(MU->SymbolFlags), std::move(MU->InitSymbol)); + MustRunMU = std::move(MU); + return Error::success(); + } + } + } + + // Otherwise, make MU responsible for all the symbols. + auto RTI = MRTrackers.find(&FromMR); + assert(RTI != MRTrackers.end() && "No tracker for FromMR"); + auto UMI = + std::make_shared<UnmaterializedInfo>(std::move(MU), RTI->second); + for (auto &KV : UMI->MU->getSymbols()) { + auto SymI = Symbols.find(KV.first); + assert(SymI->second.getState() == SymbolState::Materializing && + "Can not replace a symbol that is not materializing"); + assert(!SymI->second.hasMaterializerAttached() && + "Can not replace a symbol that has a materializer attached"); + assert(UnmaterializedInfos.count(KV.first) == 0 && + "Unexpected materializer entry in map"); + SymI->second.setAddress(SymI->second.getAddress()); + SymI->second.setMaterializerAttached(true); + + auto &UMIEntry = UnmaterializedInfos[KV.first]; + assert((!UMIEntry || !UMIEntry->MU) && + "Replacing symbol with materializer still attached"); + UMIEntry = UMI; + } + + return Error::success(); + }); + + if (Err) + return Err; + + if (MustRunMU) { + assert(MustRunMR && "MustRunMU set implies MustRunMR set"); + ES.dispatchMaterialization(std::move(MustRunMU), std::move(MustRunMR)); + } else { + assert(!MustRunMR && "MustRunMU unset implies MustRunMR unset"); + } + + return Error::success(); +} + +Expected<std::unique_ptr<MaterializationResponsibility>> +JITDylib::delegate(MaterializationResponsibility &FromMR, + SymbolFlagsMap SymbolFlags, SymbolStringPtr InitSymbol) { + + return ES.runSessionLocked( + [&]() -> Expected<std::unique_ptr<MaterializationResponsibility>> { + auto RT = getTracker(FromMR); + + if (RT->isDefunct()) + return make_error<ResourceTrackerDefunct>(std::move(RT)); + + return ES.createMaterializationResponsibility( + *RT, std::move(SymbolFlags), std::move(InitSymbol)); + }); +} + +SymbolNameSet +JITDylib::getRequestedSymbols(const SymbolFlagsMap &SymbolFlags) const { + return ES.runSessionLocked([&]() { + SymbolNameSet RequestedSymbols; + + for (auto &KV : SymbolFlags) { + assert(Symbols.count(KV.first) && "JITDylib does not cover this symbol?"); + assert(Symbols.find(KV.first)->second.getState() != + SymbolState::NeverSearched && + Symbols.find(KV.first)->second.getState() != SymbolState::Ready && + "getRequestedSymbols can only be called for symbols that have " + "started materializing"); + auto I = MaterializingInfos.find(KV.first); + if (I == MaterializingInfos.end()) + continue; + + if (I->second.hasQueriesPending()) + RequestedSymbols.insert(KV.first); + } + + return RequestedSymbols; + }); +} + +void JITDylib::addDependencies(const SymbolStringPtr &Name, + const SymbolDependenceMap &Dependencies) { + assert(Symbols.count(Name) && "Name not in symbol table"); + assert(Symbols[Name].getState() < SymbolState::Emitted && + "Can not add dependencies for a symbol that is not materializing"); + + LLVM_DEBUG({ + dbgs() << "In " << getName() << " adding dependencies for " + << *Name << ": " << Dependencies << "\n"; + }); + + // If Name is already in an error state then just bail out. + if (Symbols[Name].getFlags().hasError()) + return; + + auto &MI = MaterializingInfos[Name]; + assert(Symbols[Name].getState() != SymbolState::Emitted && + "Can not add dependencies to an emitted symbol"); + + bool DependsOnSymbolInErrorState = false; + + // Register dependencies, record whether any depenendency is in the error + // state. + for (auto &KV : Dependencies) { + assert(KV.first && "Null JITDylib in dependency?"); + auto &OtherJITDylib = *KV.first; + auto &DepsOnOtherJITDylib = MI.UnemittedDependencies[&OtherJITDylib]; + + for (auto &OtherSymbol : KV.second) { + + // Check the sym entry for the dependency. + auto OtherSymI = OtherJITDylib.Symbols.find(OtherSymbol); + + // Assert that this symbol exists and has not reached the ready state + // already. + assert(OtherSymI != OtherJITDylib.Symbols.end() && + "Dependency on unknown symbol"); + + auto &OtherSymEntry = OtherSymI->second; + + // If the other symbol is already in the Ready state then there's no + // dependency to add. + if (OtherSymEntry.getState() == SymbolState::Ready) + continue; + + // If the dependency is in an error state then note this and continue, + // we will move this symbol to the error state below. + if (OtherSymEntry.getFlags().hasError()) { + DependsOnSymbolInErrorState = true; + continue; + } + + // If the dependency was not in the error state then add it to + // our list of dependencies. + auto &OtherMI = OtherJITDylib.MaterializingInfos[OtherSymbol]; + + if (OtherSymEntry.getState() == SymbolState::Emitted) + transferEmittedNodeDependencies(MI, Name, OtherMI); + else if (&OtherJITDylib != this || OtherSymbol != Name) { + OtherMI.Dependants[this].insert(Name); + DepsOnOtherJITDylib.insert(OtherSymbol); + } + } + + if (DepsOnOtherJITDylib.empty()) + MI.UnemittedDependencies.erase(&OtherJITDylib); + } + + // If this symbol dependended on any symbols in the error state then move + // this symbol to the error state too. + if (DependsOnSymbolInErrorState) + Symbols[Name].setFlags(Symbols[Name].getFlags() | JITSymbolFlags::HasError); +} + +Error JITDylib::resolve(MaterializationResponsibility &MR, + const SymbolMap &Resolved) { + AsynchronousSymbolQuerySet CompletedQueries; + + if (auto Err = ES.runSessionLocked([&, this]() -> Error { + auto RTI = MRTrackers.find(&MR); + assert(RTI != MRTrackers.end() && "No resource tracker for MR?"); + if (RTI->second->isDefunct()) + return make_error<ResourceTrackerDefunct>(RTI->second); + + struct WorklistEntry { + SymbolTable::iterator SymI; + JITEvaluatedSymbol ResolvedSym; + }; + + SymbolNameSet SymbolsInErrorState; + std::vector<WorklistEntry> Worklist; + Worklist.reserve(Resolved.size()); + + // Build worklist and check for any symbols in the error state. + for (const auto &KV : Resolved) { + + assert(!KV.second.getFlags().hasError() && + "Resolution result can not have error flag set"); + + auto SymI = Symbols.find(KV.first); + + assert(SymI != Symbols.end() && "Symbol not found"); + assert(!SymI->second.hasMaterializerAttached() && + "Resolving symbol with materializer attached?"); + assert(SymI->second.getState() == SymbolState::Materializing && + "Symbol should be materializing"); + assert(SymI->second.getAddress() == 0 && + "Symbol has already been resolved"); + + if (SymI->second.getFlags().hasError()) + SymbolsInErrorState.insert(KV.first); + else { + auto Flags = KV.second.getFlags(); + Flags &= ~(JITSymbolFlags::Weak | JITSymbolFlags::Common); + assert(Flags == + (SymI->second.getFlags() & + ~(JITSymbolFlags::Weak | JITSymbolFlags::Common)) && + "Resolved flags should match the declared flags"); + + Worklist.push_back( + {SymI, JITEvaluatedSymbol(KV.second.getAddress(), Flags)}); + } + } + + // If any symbols were in the error state then bail out. + if (!SymbolsInErrorState.empty()) { + auto FailedSymbolsDepMap = std::make_shared<SymbolDependenceMap>(); + (*FailedSymbolsDepMap)[this] = std::move(SymbolsInErrorState); + return make_error<FailedToMaterialize>( + std::move(FailedSymbolsDepMap)); + } + + while (!Worklist.empty()) { + auto SymI = Worklist.back().SymI; + auto ResolvedSym = Worklist.back().ResolvedSym; + Worklist.pop_back(); + + auto &Name = SymI->first; + + // Resolved symbols can not be weak: discard the weak flag. + JITSymbolFlags ResolvedFlags = ResolvedSym.getFlags(); + SymI->second.setAddress(ResolvedSym.getAddress()); + SymI->second.setFlags(ResolvedFlags); + SymI->second.setState(SymbolState::Resolved); + + auto MII = MaterializingInfos.find(Name); + if (MII == MaterializingInfos.end()) + continue; + + auto &MI = MII->second; + for (auto &Q : MI.takeQueriesMeeting(SymbolState::Resolved)) { + Q->notifySymbolMetRequiredState(Name, ResolvedSym); + Q->removeQueryDependence(*this, Name); + if (Q->isComplete()) + CompletedQueries.insert(std::move(Q)); + } + } + + return Error::success(); + })) + return Err; + + // Otherwise notify all the completed queries. + for (auto &Q : CompletedQueries) { + assert(Q->isComplete() && "Q not completed"); + Q->handleComplete(); + } + + return Error::success(); +} + +Error JITDylib::emit(MaterializationResponsibility &MR, + const SymbolFlagsMap &Emitted) { + AsynchronousSymbolQuerySet CompletedQueries; + DenseMap<JITDylib *, SymbolNameVector> ReadySymbols; + + if (auto Err = ES.runSessionLocked([&, this]() -> Error { + auto RTI = MRTrackers.find(&MR); + assert(RTI != MRTrackers.end() && "No resource tracker for MR?"); + if (RTI->second->isDefunct()) + return make_error<ResourceTrackerDefunct>(RTI->second); + + SymbolNameSet SymbolsInErrorState; + std::vector<SymbolTable::iterator> Worklist; + + // Scan to build worklist, record any symbols in the erorr state. + for (const auto &KV : Emitted) { + auto &Name = KV.first; + + auto SymI = Symbols.find(Name); + assert(SymI != Symbols.end() && "No symbol table entry for Name"); + + if (SymI->second.getFlags().hasError()) + SymbolsInErrorState.insert(Name); + else + Worklist.push_back(SymI); + } + + // If any symbols were in the error state then bail out. + if (!SymbolsInErrorState.empty()) { + auto FailedSymbolsDepMap = std::make_shared<SymbolDependenceMap>(); + (*FailedSymbolsDepMap)[this] = std::move(SymbolsInErrorState); + return make_error<FailedToMaterialize>( + std::move(FailedSymbolsDepMap)); + } + + // Otherwise update dependencies and move to the emitted state. + while (!Worklist.empty()) { + auto SymI = Worklist.back(); + Worklist.pop_back(); + + auto &Name = SymI->first; + auto &SymEntry = SymI->second; + + // Move symbol to the emitted state. + assert(((SymEntry.getFlags().hasMaterializationSideEffectsOnly() && + SymEntry.getState() == SymbolState::Materializing) || + SymEntry.getState() == SymbolState::Resolved) && + "Emitting from state other than Resolved"); + SymEntry.setState(SymbolState::Emitted); + + auto MII = MaterializingInfos.find(Name); + + // If this symbol has no MaterializingInfo then it's trivially ready. + // Update its state and continue. + if (MII == MaterializingInfos.end()) { + SymEntry.setState(SymbolState::Ready); + continue; + } + + auto &MI = MII->second; + + // For each dependant, transfer this node's emitted dependencies to + // it. If the dependant node is ready (i.e. has no unemitted + // dependencies) then notify any pending queries. + for (auto &KV : MI.Dependants) { + auto &DependantJD = *KV.first; + auto &DependantJDReadySymbols = ReadySymbols[&DependantJD]; + for (auto &DependantName : KV.second) { + auto DependantMII = + DependantJD.MaterializingInfos.find(DependantName); + assert(DependantMII != DependantJD.MaterializingInfos.end() && + "Dependant should have MaterializingInfo"); + + auto &DependantMI = DependantMII->second; + + // Remove the dependant's dependency on this node. + assert(DependantMI.UnemittedDependencies.count(this) && + "Dependant does not have an unemitted dependencies record " + "for " + "this JITDylib"); + assert(DependantMI.UnemittedDependencies[this].count(Name) && + "Dependant does not count this symbol as a dependency?"); + + DependantMI.UnemittedDependencies[this].erase(Name); + if (DependantMI.UnemittedDependencies[this].empty()) + DependantMI.UnemittedDependencies.erase(this); + + // Transfer unemitted dependencies from this node to the + // dependant. + DependantJD.transferEmittedNodeDependencies(DependantMI, + DependantName, MI); + + auto DependantSymI = DependantJD.Symbols.find(DependantName); + assert(DependantSymI != DependantJD.Symbols.end() && + "Dependant has no entry in the Symbols table"); + auto &DependantSymEntry = DependantSymI->second; + + // If the dependant is emitted and this node was the last of its + // unemitted dependencies then the dependant node is now ready, so + // notify any pending queries on the dependant node. + if (DependantSymEntry.getState() == SymbolState::Emitted && + DependantMI.UnemittedDependencies.empty()) { + assert(DependantMI.Dependants.empty() && + "Dependants should be empty by now"); + + // Since this dependant is now ready, we erase its + // MaterializingInfo and update its materializing state. + DependantSymEntry.setState(SymbolState::Ready); + DependantJDReadySymbols.push_back(DependantName); + + for (auto &Q : + DependantMI.takeQueriesMeeting(SymbolState::Ready)) { + Q->notifySymbolMetRequiredState( + DependantName, DependantSymI->second.getSymbol()); + if (Q->isComplete()) + CompletedQueries.insert(Q); + Q->removeQueryDependence(DependantJD, DependantName); + } + } + } + } + + auto &ThisJDReadySymbols = ReadySymbols[this]; + MI.Dependants.clear(); + if (MI.UnemittedDependencies.empty()) { + SymI->second.setState(SymbolState::Ready); + ThisJDReadySymbols.push_back(Name); + for (auto &Q : MI.takeQueriesMeeting(SymbolState::Ready)) { + Q->notifySymbolMetRequiredState(Name, SymI->second.getSymbol()); + if (Q->isComplete()) + CompletedQueries.insert(Q); + Q->removeQueryDependence(*this, Name); + } + } + } + + return Error::success(); + })) + return Err; + + // Otherwise notify all the completed queries. + for (auto &Q : CompletedQueries) { + assert(Q->isComplete() && "Q is not complete"); + Q->handleComplete(); + } + + return Error::success(); +} + +void JITDylib::unlinkMaterializationResponsibility( + MaterializationResponsibility &MR) { + ES.runSessionLocked([&]() { + auto I = MRTrackers.find(&MR); + assert(I != MRTrackers.end() && "MaterializationResponsibility not linked"); + MRTrackers.erase(I); + }); +} + +std::pair<JITDylib::AsynchronousSymbolQuerySet, + std::shared_ptr<SymbolDependenceMap>> +JITDylib::failSymbols(FailedSymbolsWorklist Worklist) { + AsynchronousSymbolQuerySet FailedQueries; + auto FailedSymbolsMap = std::make_shared<SymbolDependenceMap>(); + + while (!Worklist.empty()) { + assert(Worklist.back().first && "Failed JITDylib can not be null"); + auto &JD = *Worklist.back().first; + auto Name = std::move(Worklist.back().second); + Worklist.pop_back(); + + (*FailedSymbolsMap)[&JD].insert(Name); + + assert(JD.Symbols.count(Name) && "No symbol table entry for Name"); + auto &Sym = JD.Symbols[Name]; + + // Move the symbol into the error state. + // Note that this may be redundant: The symbol might already have been + // moved to this state in response to the failure of a dependence. + Sym.setFlags(Sym.getFlags() | JITSymbolFlags::HasError); + + // FIXME: Come up with a sane mapping of state to + // presence-of-MaterializingInfo so that we can assert presence / absence + // here, rather than testing it. + auto MII = JD.MaterializingInfos.find(Name); + + if (MII == JD.MaterializingInfos.end()) + continue; + + auto &MI = MII->second; + + // Move all dependants to the error state and disconnect from them. + for (auto &KV : MI.Dependants) { + auto &DependantJD = *KV.first; + for (auto &DependantName : KV.second) { + assert(DependantJD.Symbols.count(DependantName) && + "No symbol table entry for DependantName"); + auto &DependantSym = DependantJD.Symbols[DependantName]; + DependantSym.setFlags(DependantSym.getFlags() | + JITSymbolFlags::HasError); + + assert(DependantJD.MaterializingInfos.count(DependantName) && + "No MaterializingInfo for dependant"); + auto &DependantMI = DependantJD.MaterializingInfos[DependantName]; + + auto UnemittedDepI = DependantMI.UnemittedDependencies.find(&JD); + assert(UnemittedDepI != DependantMI.UnemittedDependencies.end() && + "No UnemittedDependencies entry for this JITDylib"); + assert(UnemittedDepI->second.count(Name) && + "No UnemittedDependencies entry for this symbol"); + UnemittedDepI->second.erase(Name); + if (UnemittedDepI->second.empty()) + DependantMI.UnemittedDependencies.erase(UnemittedDepI); + + // If this symbol is already in the emitted state then we need to + // take responsibility for failing its queries, so add it to the + // worklist. + if (DependantSym.getState() == SymbolState::Emitted) { + assert(DependantMI.Dependants.empty() && + "Emitted symbol should not have dependants"); + Worklist.push_back(std::make_pair(&DependantJD, DependantName)); + } + } + } + MI.Dependants.clear(); + + // Disconnect from all unemitted depenencies. + for (auto &KV : MI.UnemittedDependencies) { + auto &UnemittedDepJD = *KV.first; + for (auto &UnemittedDepName : KV.second) { + auto UnemittedDepMII = + UnemittedDepJD.MaterializingInfos.find(UnemittedDepName); + assert(UnemittedDepMII != UnemittedDepJD.MaterializingInfos.end() && + "Missing MII for unemitted dependency"); + assert(UnemittedDepMII->second.Dependants.count(&JD) && + "JD not listed as a dependant of unemitted dependency"); + assert(UnemittedDepMII->second.Dependants[&JD].count(Name) && + "Name is not listed as a dependant of unemitted dependency"); + UnemittedDepMII->second.Dependants[&JD].erase(Name); + if (UnemittedDepMII->second.Dependants[&JD].empty()) + UnemittedDepMII->second.Dependants.erase(&JD); + } + } + MI.UnemittedDependencies.clear(); + + // Collect queries to be failed for this MII. + AsynchronousSymbolQueryList ToDetach; + for (auto &Q : MII->second.pendingQueries()) { + // Add the query to the list to be failed and detach it. + FailedQueries.insert(Q); + ToDetach.push_back(Q); + } + for (auto &Q : ToDetach) + Q->detach(); + + assert(MI.Dependants.empty() && + "Can not delete MaterializingInfo with dependants still attached"); + assert(MI.UnemittedDependencies.empty() && + "Can not delete MaterializingInfo with unemitted dependencies " + "still attached"); + assert(!MI.hasQueriesPending() && + "Can not delete MaterializingInfo with queries pending"); + JD.MaterializingInfos.erase(MII); + } + + return std::make_pair(std::move(FailedQueries), std::move(FailedSymbolsMap)); +} + +void JITDylib::setLinkOrder(JITDylibSearchOrder NewLinkOrder, + bool LinkAgainstThisJITDylibFirst) { + ES.runSessionLocked([&]() { + if (LinkAgainstThisJITDylibFirst) { + LinkOrder.clear(); + if (NewLinkOrder.empty() || NewLinkOrder.front().first != this) + LinkOrder.push_back( + std::make_pair(this, JITDylibLookupFlags::MatchAllSymbols)); + llvm::append_range(LinkOrder, NewLinkOrder); + } else + LinkOrder = std::move(NewLinkOrder); + }); +} + +void JITDylib::addToLinkOrder(JITDylib &JD, JITDylibLookupFlags JDLookupFlags) { + ES.runSessionLocked([&]() { LinkOrder.push_back({&JD, JDLookupFlags}); }); +} + +void JITDylib::replaceInLinkOrder(JITDylib &OldJD, JITDylib &NewJD, + JITDylibLookupFlags JDLookupFlags) { + ES.runSessionLocked([&]() { + for (auto &KV : LinkOrder) + if (KV.first == &OldJD) { + KV = {&NewJD, JDLookupFlags}; + break; + } + }); +} + +void JITDylib::removeFromLinkOrder(JITDylib &JD) { + ES.runSessionLocked([&]() { + auto I = llvm::find_if(LinkOrder, + [&](const JITDylibSearchOrder::value_type &KV) { + return KV.first == &JD; + }); + if (I != LinkOrder.end()) + LinkOrder.erase(I); + }); +} + +Error JITDylib::remove(const SymbolNameSet &Names) { + return ES.runSessionLocked([&]() -> Error { + using SymbolMaterializerItrPair = + std::pair<SymbolTable::iterator, UnmaterializedInfosMap::iterator>; + std::vector<SymbolMaterializerItrPair> SymbolsToRemove; + SymbolNameSet Missing; + SymbolNameSet Materializing; + + for (auto &Name : Names) { + auto I = Symbols.find(Name); + + // Note symbol missing. + if (I == Symbols.end()) { + Missing.insert(Name); + continue; + } + + // Note symbol materializing. + if (I->second.getState() != SymbolState::NeverSearched && + I->second.getState() != SymbolState::Ready) { + Materializing.insert(Name); + continue; + } + + auto UMII = I->second.hasMaterializerAttached() + ? UnmaterializedInfos.find(Name) + : UnmaterializedInfos.end(); + SymbolsToRemove.push_back(std::make_pair(I, UMII)); + } + + // If any of the symbols are not defined, return an error. + if (!Missing.empty()) + return make_error<SymbolsNotFound>(std::move(Missing)); + + // If any of the symbols are currently materializing, return an error. + if (!Materializing.empty()) + return make_error<SymbolsCouldNotBeRemoved>(std::move(Materializing)); + + // Remove the symbols. + for (auto &SymbolMaterializerItrPair : SymbolsToRemove) { + auto UMII = SymbolMaterializerItrPair.second; + + // If there is a materializer attached, call discard. + if (UMII != UnmaterializedInfos.end()) { + UMII->second->MU->doDiscard(*this, UMII->first); + UnmaterializedInfos.erase(UMII); + } + + auto SymI = SymbolMaterializerItrPair.first; + Symbols.erase(SymI); + } + + return Error::success(); + }); +} + +void JITDylib::dump(raw_ostream &OS) { + ES.runSessionLocked([&, this]() { + OS << "JITDylib \"" << JITDylibName << "\" (ES: " + << format("0x%016" PRIx64, reinterpret_cast<uintptr_t>(&ES)) << "):\n" + << "Link order: " << LinkOrder << "\n" + << "Symbol table:\n"; + + for (auto &KV : Symbols) { + OS << " \"" << *KV.first << "\": "; + if (auto Addr = KV.second.getAddress()) + OS << format("0x%016" PRIx64, Addr) << ", " << KV.second.getFlags() + << " "; + else + OS << "<not resolved> "; + + OS << KV.second.getFlags() << " " << KV.second.getState(); + + if (KV.second.hasMaterializerAttached()) { + OS << " (Materializer "; + auto I = UnmaterializedInfos.find(KV.first); + assert(I != UnmaterializedInfos.end() && + "Lazy symbol should have UnmaterializedInfo"); + OS << I->second->MU.get() << ", " << I->second->MU->getName() << ")\n"; + } else + OS << "\n"; + } + + if (!MaterializingInfos.empty()) + OS << " MaterializingInfos entries:\n"; + for (auto &KV : MaterializingInfos) { + OS << " \"" << *KV.first << "\":\n" + << " " << KV.second.pendingQueries().size() + << " pending queries: { "; + for (const auto &Q : KV.second.pendingQueries()) + OS << Q.get() << " (" << Q->getRequiredState() << ") "; + OS << "}\n Dependants:\n"; + for (auto &KV2 : KV.second.Dependants) + OS << " " << KV2.first->getName() << ": " << KV2.second << "\n"; + OS << " Unemitted Dependencies:\n"; + for (auto &KV2 : KV.second.UnemittedDependencies) + OS << " " << KV2.first->getName() << ": " << KV2.second << "\n"; + } + }); +} + +void JITDylib::MaterializingInfo::addQuery( + std::shared_ptr<AsynchronousSymbolQuery> Q) { + + auto I = std::lower_bound( + PendingQueries.rbegin(), PendingQueries.rend(), Q->getRequiredState(), + [](const std::shared_ptr<AsynchronousSymbolQuery> &V, SymbolState S) { + return V->getRequiredState() <= S; + }); + PendingQueries.insert(I.base(), std::move(Q)); +} + +void JITDylib::MaterializingInfo::removeQuery( + const AsynchronousSymbolQuery &Q) { + // FIXME: Implement 'find_as' for shared_ptr<T>/T*. + auto I = llvm::find_if( + PendingQueries, [&Q](const std::shared_ptr<AsynchronousSymbolQuery> &V) { + return V.get() == &Q; + }); + assert(I != PendingQueries.end() && + "Query is not attached to this MaterializingInfo"); + PendingQueries.erase(I); +} + +JITDylib::AsynchronousSymbolQueryList +JITDylib::MaterializingInfo::takeQueriesMeeting(SymbolState RequiredState) { + AsynchronousSymbolQueryList Result; + while (!PendingQueries.empty()) { + if (PendingQueries.back()->getRequiredState() > RequiredState) + break; + + Result.push_back(std::move(PendingQueries.back())); + PendingQueries.pop_back(); + } + + return Result; +} + +JITDylib::JITDylib(ExecutionSession &ES, std::string Name) + : ES(ES), JITDylibName(std::move(Name)) { + LinkOrder.push_back({this, JITDylibLookupFlags::MatchAllSymbols}); +} + +ResourceTrackerSP JITDylib::getTracker(MaterializationResponsibility &MR) { + auto I = MRTrackers.find(&MR); + assert(I != MRTrackers.end() && "MR is not linked"); + assert(I->second && "Linked tracker is null"); + return I->second; +} + +std::pair<JITDylib::AsynchronousSymbolQuerySet, + std::shared_ptr<SymbolDependenceMap>> +JITDylib::removeTracker(ResourceTracker &RT) { + // Note: Should be called under the session lock. + + SymbolNameVector SymbolsToRemove; + std::vector<std::pair<JITDylib *, SymbolStringPtr>> SymbolsToFail; + + if (&RT == DefaultTracker.get()) { + SymbolNameSet TrackedSymbols; + for (auto &KV : TrackerSymbols) + for (auto &Sym : KV.second) + TrackedSymbols.insert(Sym); + + for (auto &KV : Symbols) { + auto &Sym = KV.first; + if (!TrackedSymbols.count(Sym)) + SymbolsToRemove.push_back(Sym); + } + + DefaultTracker.reset(); + } else { + /// Check for a non-default tracker. + auto I = TrackerSymbols.find(&RT); + if (I != TrackerSymbols.end()) { + SymbolsToRemove = std::move(I->second); + TrackerSymbols.erase(I); + } + // ... if not found this tracker was already defunct. Nothing to do. + } + + for (auto &Sym : SymbolsToRemove) { + assert(Symbols.count(Sym) && "Symbol not in symbol table"); + + // If there is a MaterializingInfo then collect any queries to fail. + auto MII = MaterializingInfos.find(Sym); + if (MII != MaterializingInfos.end()) + SymbolsToFail.push_back({this, Sym}); + } + + AsynchronousSymbolQuerySet QueriesToFail; + auto Result = failSymbols(std::move(SymbolsToFail)); + + // Removed symbols should be taken out of the table altogether. + for (auto &Sym : SymbolsToRemove) { + auto I = Symbols.find(Sym); + assert(I != Symbols.end() && "Symbol not present in table"); + + // Remove Materializer if present. + if (I->second.hasMaterializerAttached()) { + // FIXME: Should this discard the symbols? + UnmaterializedInfos.erase(Sym); + } else { + assert(!UnmaterializedInfos.count(Sym) && + "Symbol has materializer attached"); + } + + Symbols.erase(I); + } + + return Result; +} + +void JITDylib::transferTracker(ResourceTracker &DstRT, ResourceTracker &SrcRT) { + assert(&DstRT != &SrcRT && "No-op transfers shouldn't call transferTracker"); + assert(&DstRT.getJITDylib() == this && "DstRT is not for this JITDylib"); + assert(&SrcRT.getJITDylib() == this && "SrcRT is not for this JITDylib"); + + // Update trackers for any not-yet materialized units. + for (auto &KV : UnmaterializedInfos) { + if (KV.second->RT == &SrcRT) + KV.second->RT = &DstRT; + } + + // Update trackers for any active materialization responsibilities. + for (auto &KV : MRTrackers) { + if (KV.second == &SrcRT) + KV.second = &DstRT; + } + + // If we're transfering to the default tracker we just need to delete the + // tracked symbols for the source tracker. + if (&DstRT == DefaultTracker.get()) { + TrackerSymbols.erase(&SrcRT); + return; + } + + // If we're transferring from the default tracker we need to find all + // currently untracked symbols. + if (&SrcRT == DefaultTracker.get()) { + assert(!TrackerSymbols.count(&SrcRT) && + "Default tracker should not appear in TrackerSymbols"); + + SymbolNameVector SymbolsToTrack; + + SymbolNameSet CurrentlyTrackedSymbols; + for (auto &KV : TrackerSymbols) + for (auto &Sym : KV.second) + CurrentlyTrackedSymbols.insert(Sym); + + for (auto &KV : Symbols) { + auto &Sym = KV.first; + if (!CurrentlyTrackedSymbols.count(Sym)) + SymbolsToTrack.push_back(Sym); + } + + TrackerSymbols[&DstRT] = std::move(SymbolsToTrack); + return; + } + + auto &DstTrackedSymbols = TrackerSymbols[&DstRT]; + + // Finally if neither SrtRT or DstRT are the default tracker then + // just append DstRT's tracked symbols to SrtRT's. + auto SI = TrackerSymbols.find(&SrcRT); + if (SI == TrackerSymbols.end()) + return; + + DstTrackedSymbols.reserve(DstTrackedSymbols.size() + SI->second.size()); + for (auto &Sym : SI->second) + DstTrackedSymbols.push_back(std::move(Sym)); + TrackerSymbols.erase(SI); +} + +Error JITDylib::defineImpl(MaterializationUnit &MU) { + + LLVM_DEBUG({ dbgs() << " " << MU.getSymbols() << "\n"; }); + + SymbolNameSet Duplicates; + std::vector<SymbolStringPtr> ExistingDefsOverridden; + std::vector<SymbolStringPtr> MUDefsOverridden; + + for (const auto &KV : MU.getSymbols()) { + auto I = Symbols.find(KV.first); + + if (I != Symbols.end()) { + if (KV.second.isStrong()) { + if (I->second.getFlags().isStrong() || + I->second.getState() > SymbolState::NeverSearched) + Duplicates.insert(KV.first); + else { + assert(I->second.getState() == SymbolState::NeverSearched && + "Overridden existing def should be in the never-searched " + "state"); + ExistingDefsOverridden.push_back(KV.first); + } + } else + MUDefsOverridden.push_back(KV.first); + } + } + + // If there were any duplicate definitions then bail out. + if (!Duplicates.empty()) { + LLVM_DEBUG( + { dbgs() << " Error: Duplicate symbols " << Duplicates << "\n"; }); + return make_error<DuplicateDefinition>(std::string(**Duplicates.begin())); + } + + // Discard any overridden defs in this MU. + LLVM_DEBUG({ + if (!MUDefsOverridden.empty()) + dbgs() << " Defs in this MU overridden: " << MUDefsOverridden << "\n"; + }); + for (auto &S : MUDefsOverridden) + MU.doDiscard(*this, S); + + // Discard existing overridden defs. + LLVM_DEBUG({ + if (!ExistingDefsOverridden.empty()) + dbgs() << " Existing defs overridden by this MU: " << MUDefsOverridden + << "\n"; + }); + for (auto &S : ExistingDefsOverridden) { + + auto UMII = UnmaterializedInfos.find(S); + assert(UMII != UnmaterializedInfos.end() && + "Overridden existing def should have an UnmaterializedInfo"); + UMII->second->MU->doDiscard(*this, S); + } + + // Finally, add the defs from this MU. + for (auto &KV : MU.getSymbols()) { + auto &SymEntry = Symbols[KV.first]; + SymEntry.setFlags(KV.second); + SymEntry.setState(SymbolState::NeverSearched); + SymEntry.setMaterializerAttached(true); + } + + return Error::success(); +} + +void JITDylib::installMaterializationUnit( + std::unique_ptr<MaterializationUnit> MU, ResourceTracker &RT) { + + /// defineImpl succeeded. + if (&RT != DefaultTracker.get()) { + auto &TS = TrackerSymbols[&RT]; + TS.reserve(TS.size() + MU->getSymbols().size()); + for (auto &KV : MU->getSymbols()) + TS.push_back(KV.first); + } + + auto UMI = std::make_shared<UnmaterializedInfo>(std::move(MU), &RT); + for (auto &KV : UMI->MU->getSymbols()) + UnmaterializedInfos[KV.first] = UMI; +} + +void JITDylib::detachQueryHelper(AsynchronousSymbolQuery &Q, + const SymbolNameSet &QuerySymbols) { + for (auto &QuerySymbol : QuerySymbols) { + assert(MaterializingInfos.count(QuerySymbol) && + "QuerySymbol does not have MaterializingInfo"); + auto &MI = MaterializingInfos[QuerySymbol]; + MI.removeQuery(Q); + } +} + +void JITDylib::transferEmittedNodeDependencies( + MaterializingInfo &DependantMI, const SymbolStringPtr &DependantName, + MaterializingInfo &EmittedMI) { + for (auto &KV : EmittedMI.UnemittedDependencies) { + auto &DependencyJD = *KV.first; + SymbolNameSet *UnemittedDependenciesOnDependencyJD = nullptr; + + for (auto &DependencyName : KV.second) { + auto &DependencyMI = DependencyJD.MaterializingInfos[DependencyName]; + + // Do not add self dependencies. + if (&DependencyMI == &DependantMI) + continue; + + // If we haven't looked up the dependencies for DependencyJD yet, do it + // now and cache the result. + if (!UnemittedDependenciesOnDependencyJD) + UnemittedDependenciesOnDependencyJD = + &DependantMI.UnemittedDependencies[&DependencyJD]; + + DependencyMI.Dependants[this].insert(DependantName); + UnemittedDependenciesOnDependencyJD->insert(DependencyName); + } + } +} + +Platform::~Platform() {} + +Expected<DenseMap<JITDylib *, SymbolMap>> Platform::lookupInitSymbols( + ExecutionSession &ES, + const DenseMap<JITDylib *, SymbolLookupSet> &InitSyms) { + + DenseMap<JITDylib *, SymbolMap> CompoundResult; + Error CompoundErr = Error::success(); + std::mutex LookupMutex; + std::condition_variable CV; + uint64_t Count = InitSyms.size(); + + LLVM_DEBUG({ + dbgs() << "Issuing init-symbol lookup:\n"; + for (auto &KV : InitSyms) + dbgs() << " " << KV.first->getName() << ": " << KV.second << "\n"; + }); + + for (auto &KV : InitSyms) { + auto *JD = KV.first; + auto Names = std::move(KV.second); + ES.lookup( + LookupKind::Static, + JITDylibSearchOrder({{JD, JITDylibLookupFlags::MatchAllSymbols}}), + std::move(Names), SymbolState::Ready, + [&, JD](Expected<SymbolMap> Result) { + { + std::lock_guard<std::mutex> Lock(LookupMutex); + --Count; + if (Result) { + assert(!CompoundResult.count(JD) && + "Duplicate JITDylib in lookup?"); + CompoundResult[JD] = std::move(*Result); + } else + CompoundErr = + joinErrors(std::move(CompoundErr), Result.takeError()); + } + CV.notify_one(); + }, + NoDependenciesToRegister); + } + + std::unique_lock<std::mutex> Lock(LookupMutex); + CV.wait(Lock, [&] { return Count == 0 || CompoundErr; }); + + if (CompoundErr) + return std::move(CompoundErr); + + return std::move(CompoundResult); +} + +ExecutionSession::ExecutionSession(std::shared_ptr<SymbolStringPool> SSP) + : SSP(SSP ? std::move(SSP) : std::make_shared<SymbolStringPool>()) {} + +Error ExecutionSession::endSession() { + LLVM_DEBUG(dbgs() << "Ending ExecutionSession " << this << "\n"); + + std::vector<JITDylibSP> JITDylibsToClose = runSessionLocked([&] { + SessionOpen = false; + return std::move(JDs); + }); + + // TODO: notifiy platform? run static deinits? + + Error Err = Error::success(); + for (auto &JD : JITDylibsToClose) + Err = joinErrors(std::move(Err), JD->clear()); + return Err; +} + +void ExecutionSession::registerResourceManager(ResourceManager &RM) { + runSessionLocked([&] { ResourceManagers.push_back(&RM); }); +} + +void ExecutionSession::deregisterResourceManager(ResourceManager &RM) { + runSessionLocked([&] { + assert(!ResourceManagers.empty() && "No managers registered"); + if (ResourceManagers.back() == &RM) + ResourceManagers.pop_back(); + else { + auto I = llvm::find(ResourceManagers, &RM); + assert(I != ResourceManagers.end() && "RM not registered"); + ResourceManagers.erase(I); + } + }); +} + +JITDylib *ExecutionSession::getJITDylibByName(StringRef Name) { + return runSessionLocked([&, this]() -> JITDylib * { + for (auto &JD : JDs) + if (JD->getName() == Name) + return JD.get(); + return nullptr; + }); +} + +JITDylib &ExecutionSession::createBareJITDylib(std::string Name) { + assert(!getJITDylibByName(Name) && "JITDylib with that name already exists"); + return runSessionLocked([&, this]() -> JITDylib & { + JDs.push_back(new JITDylib(*this, std::move(Name))); + return *JDs.back(); + }); +} + +Expected<JITDylib &> ExecutionSession::createJITDylib(std::string Name) { + auto &JD = createBareJITDylib(Name); + if (P) + if (auto Err = P->setupJITDylib(JD)) + return std::move(Err); + return JD; +} + +std::vector<JITDylibSP> JITDylib::getDFSLinkOrder(ArrayRef<JITDylibSP> JDs) { + if (JDs.empty()) + return {}; + + auto &ES = JDs.front()->getExecutionSession(); + return ES.runSessionLocked([&]() { + DenseSet<JITDylib *> Visited; + std::vector<JITDylibSP> Result; + + for (auto &JD : JDs) { + + if (Visited.count(JD.get())) + continue; + + SmallVector<JITDylibSP, 64> WorkStack; + WorkStack.push_back(JD); + Visited.insert(JD.get()); + + while (!WorkStack.empty()) { + Result.push_back(std::move(WorkStack.back())); + WorkStack.pop_back(); + + for (auto &KV : llvm::reverse(Result.back()->LinkOrder)) { + auto &JD = *KV.first; + if (Visited.count(&JD)) + continue; + Visited.insert(&JD); + WorkStack.push_back(&JD); + } + } + } + return Result; + }); +} + +std::vector<JITDylibSP> +JITDylib::getReverseDFSLinkOrder(ArrayRef<JITDylibSP> JDs) { + auto Tmp = getDFSLinkOrder(JDs); + std::reverse(Tmp.begin(), Tmp.end()); + return Tmp; +} + +std::vector<JITDylibSP> JITDylib::getDFSLinkOrder() { + return getDFSLinkOrder({this}); +} + +std::vector<JITDylibSP> JITDylib::getReverseDFSLinkOrder() { + return getReverseDFSLinkOrder({this}); +} + +void ExecutionSession::lookupFlags( + LookupKind K, JITDylibSearchOrder SearchOrder, SymbolLookupSet LookupSet, + unique_function<void(Expected<SymbolFlagsMap>)> OnComplete) { + + OL_applyQueryPhase1(std::make_unique<InProgressLookupFlagsState>( + K, std::move(SearchOrder), std::move(LookupSet), + std::move(OnComplete)), + Error::success()); +} + +Expected<SymbolFlagsMap> +ExecutionSession::lookupFlags(LookupKind K, JITDylibSearchOrder SearchOrder, + SymbolLookupSet LookupSet) { + + std::promise<MSVCPExpected<SymbolFlagsMap>> ResultP; + OL_applyQueryPhase1(std::make_unique<InProgressLookupFlagsState>( + K, std::move(SearchOrder), std::move(LookupSet), + [&ResultP](Expected<SymbolFlagsMap> Result) { + ResultP.set_value(std::move(Result)); + }), + Error::success()); + + auto ResultF = ResultP.get_future(); + return ResultF.get(); +} + +void ExecutionSession::lookup( + LookupKind K, const JITDylibSearchOrder &SearchOrder, + SymbolLookupSet Symbols, SymbolState RequiredState, + SymbolsResolvedCallback NotifyComplete, + RegisterDependenciesFunction RegisterDependencies) { + + LLVM_DEBUG({ + runSessionLocked([&]() { + dbgs() << "Looking up " << Symbols << " in " << SearchOrder + << " (required state: " << RequiredState << ")\n"; + }); + }); + + // lookup can be re-entered recursively if running on a single thread. Run any + // outstanding MUs in case this query depends on them, otherwise this lookup + // will starve waiting for a result from an MU that is stuck in the queue. + dispatchOutstandingMUs(); + + auto Unresolved = std::move(Symbols); + auto Q = std::make_shared<AsynchronousSymbolQuery>(Unresolved, RequiredState, + std::move(NotifyComplete)); + + auto IPLS = std::make_unique<InProgressFullLookupState>( + K, SearchOrder, std::move(Unresolved), RequiredState, std::move(Q), + std::move(RegisterDependencies)); + + OL_applyQueryPhase1(std::move(IPLS), Error::success()); +} + +Expected<SymbolMap> +ExecutionSession::lookup(const JITDylibSearchOrder &SearchOrder, + const SymbolLookupSet &Symbols, LookupKind K, + SymbolState RequiredState, + RegisterDependenciesFunction RegisterDependencies) { +#if LLVM_ENABLE_THREADS + // In the threaded case we use promises to return the results. + std::promise<SymbolMap> PromisedResult; + Error ResolutionError = Error::success(); + + auto NotifyComplete = [&](Expected<SymbolMap> R) { + if (R) + PromisedResult.set_value(std::move(*R)); + else { + ErrorAsOutParameter _(&ResolutionError); + ResolutionError = R.takeError(); + PromisedResult.set_value(SymbolMap()); + } + }; + +#else + SymbolMap Result; + Error ResolutionError = Error::success(); + + auto NotifyComplete = [&](Expected<SymbolMap> R) { + ErrorAsOutParameter _(&ResolutionError); + if (R) + Result = std::move(*R); + else + ResolutionError = R.takeError(); + }; +#endif + + // Perform the asynchronous lookup. + lookup(K, SearchOrder, Symbols, RequiredState, NotifyComplete, + RegisterDependencies); + +#if LLVM_ENABLE_THREADS + auto ResultFuture = PromisedResult.get_future(); + auto Result = ResultFuture.get(); + + if (ResolutionError) + return std::move(ResolutionError); + + return std::move(Result); + +#else + if (ResolutionError) + return std::move(ResolutionError); + + return Result; +#endif +} + +Expected<JITEvaluatedSymbol> +ExecutionSession::lookup(const JITDylibSearchOrder &SearchOrder, + SymbolStringPtr Name, SymbolState RequiredState) { + SymbolLookupSet Names({Name}); + + if (auto ResultMap = lookup(SearchOrder, std::move(Names), LookupKind::Static, + RequiredState, NoDependenciesToRegister)) { + assert(ResultMap->size() == 1 && "Unexpected number of results"); + assert(ResultMap->count(Name) && "Missing result for symbol"); + return std::move(ResultMap->begin()->second); + } else + return ResultMap.takeError(); +} + +Expected<JITEvaluatedSymbol> +ExecutionSession::lookup(ArrayRef<JITDylib *> SearchOrder, SymbolStringPtr Name, + SymbolState RequiredState) { + return lookup(makeJITDylibSearchOrder(SearchOrder), Name, RequiredState); +} + +Expected<JITEvaluatedSymbol> +ExecutionSession::lookup(ArrayRef<JITDylib *> SearchOrder, StringRef Name, + SymbolState RequiredState) { + return lookup(SearchOrder, intern(Name), RequiredState); +} + +void ExecutionSession::dump(raw_ostream &OS) { + runSessionLocked([this, &OS]() { + for (auto &JD : JDs) + JD->dump(OS); + }); +} + +void ExecutionSession::dispatchOutstandingMUs() { + LLVM_DEBUG(dbgs() << "Dispatching MaterializationUnits...\n"); + while (1) { + Optional<std::pair<std::unique_ptr<MaterializationUnit>, + std::unique_ptr<MaterializationResponsibility>>> + JMU; + + { + std::lock_guard<std::recursive_mutex> Lock(OutstandingMUsMutex); + if (!OutstandingMUs.empty()) { + JMU.emplace(std::move(OutstandingMUs.back())); + OutstandingMUs.pop_back(); + } + } + + if (!JMU) + break; + + assert(JMU->first && "No MU?"); + LLVM_DEBUG(dbgs() << " Dispatching \"" << JMU->first->getName() << "\"\n"); + dispatchMaterialization(std::move(JMU->first), std::move(JMU->second)); + } + LLVM_DEBUG(dbgs() << "Done dispatching MaterializationUnits.\n"); +} + +Error ExecutionSession::removeResourceTracker(ResourceTracker &RT) { + LLVM_DEBUG({ + dbgs() << "In " << RT.getJITDylib().getName() << " removing tracker " + << formatv("{0:x}", RT.getKeyUnsafe()) << "\n"; + }); + std::vector<ResourceManager *> CurrentResourceManagers; + + JITDylib::AsynchronousSymbolQuerySet QueriesToFail; + std::shared_ptr<SymbolDependenceMap> FailedSymbols; + + runSessionLocked([&] { + CurrentResourceManagers = ResourceManagers; + RT.makeDefunct(); + std::tie(QueriesToFail, FailedSymbols) = RT.getJITDylib().removeTracker(RT); + }); + + Error Err = Error::success(); + + for (auto *L : reverse(CurrentResourceManagers)) + Err = + joinErrors(std::move(Err), L->handleRemoveResources(RT.getKeyUnsafe())); + + for (auto &Q : QueriesToFail) + Q->handleFailed(make_error<FailedToMaterialize>(FailedSymbols)); + + return Err; +} + +void ExecutionSession::transferResourceTracker(ResourceTracker &DstRT, + ResourceTracker &SrcRT) { + LLVM_DEBUG({ + dbgs() << "In " << SrcRT.getJITDylib().getName() + << " transfering resources from tracker " + << formatv("{0:x}", SrcRT.getKeyUnsafe()) << " to tracker " + << formatv("{0:x}", DstRT.getKeyUnsafe()) << "\n"; + }); + + // No-op transfers are allowed and do not invalidate the source. + if (&DstRT == &SrcRT) + return; + + assert(&DstRT.getJITDylib() == &SrcRT.getJITDylib() && + "Can't transfer resources between JITDylibs"); + runSessionLocked([&]() { + SrcRT.makeDefunct(); + auto &JD = DstRT.getJITDylib(); + JD.transferTracker(DstRT, SrcRT); + for (auto *L : reverse(ResourceManagers)) + L->handleTransferResources(DstRT.getKeyUnsafe(), SrcRT.getKeyUnsafe()); + }); +} + +void ExecutionSession::destroyResourceTracker(ResourceTracker &RT) { + runSessionLocked([&]() { + LLVM_DEBUG({ + dbgs() << "In " << RT.getJITDylib().getName() << " destroying tracker " + << formatv("{0:x}", RT.getKeyUnsafe()) << "\n"; + }); + if (!RT.isDefunct()) + transferResourceTracker(*RT.getJITDylib().getDefaultResourceTracker(), + RT); + }); +} + +Error ExecutionSession::IL_updateCandidatesFor( + JITDylib &JD, JITDylibLookupFlags JDLookupFlags, + SymbolLookupSet &Candidates, SymbolLookupSet *NonCandidates) { + return Candidates.forEachWithRemoval( + [&](const SymbolStringPtr &Name, + SymbolLookupFlags SymLookupFlags) -> Expected<bool> { + /// Search for the symbol. If not found then continue without + /// removal. + auto SymI = JD.Symbols.find(Name); + if (SymI == JD.Symbols.end()) + return false; + + // If this is a non-exported symbol and we're matching exported + // symbols only then remove this symbol from the candidates list. + // + // If we're tracking non-candidates then add this to the non-candidate + // list. + if (!SymI->second.getFlags().isExported() && + JDLookupFlags == JITDylibLookupFlags::MatchExportedSymbolsOnly) { + if (NonCandidates) + NonCandidates->add(Name, SymLookupFlags); + return true; + } + + // If we match against a materialization-side-effects only symbol + // then make sure it is weakly-referenced. Otherwise bail out with + // an error. + // FIXME: Use a "materialization-side-effects-only symbols must be + // weakly referenced" specific error here to reduce confusion. + if (SymI->second.getFlags().hasMaterializationSideEffectsOnly() && + SymLookupFlags != SymbolLookupFlags::WeaklyReferencedSymbol) + return make_error<SymbolsNotFound>(SymbolNameVector({Name})); + + // If we matched against this symbol but it is in the error state + // then bail out and treat it as a failure to materialize. + if (SymI->second.getFlags().hasError()) { + auto FailedSymbolsMap = std::make_shared<SymbolDependenceMap>(); + (*FailedSymbolsMap)[&JD] = {Name}; + return make_error<FailedToMaterialize>(std::move(FailedSymbolsMap)); + } + + // Otherwise this is a match. Remove it from the candidate set. + return true; + }); +} + +void ExecutionSession::OL_applyQueryPhase1( + std::unique_ptr<InProgressLookupState> IPLS, Error Err) { + + LLVM_DEBUG({ + dbgs() << "Entering OL_applyQueryPhase1:\n" + << " Lookup kind: " << IPLS->K << "\n" + << " Search order: " << IPLS->SearchOrder + << ", Current index = " << IPLS->CurSearchOrderIndex + << (IPLS->NewJITDylib ? " (entering new JITDylib)" : "") << "\n" + << " Lookup set: " << IPLS->LookupSet << "\n" + << " Definition generator candidates: " + << IPLS->DefGeneratorCandidates << "\n" + << " Definition generator non-candidates: " + << IPLS->DefGeneratorNonCandidates << "\n"; + }); + + // FIXME: We should attach the query as we go: This provides a result in a + // single pass in the common case where all symbols have already reached the + // required state. The query could be detached again in the 'fail' method on + // IPLS. Phase 2 would be reduced to collecting and dispatching the MUs. + + while (IPLS->CurSearchOrderIndex != IPLS->SearchOrder.size()) { + + // If we've been handed an error or received one back from a generator then + // fail the query. We don't need to unlink: At this stage the query hasn't + // actually been lodged. + if (Err) + return IPLS->fail(std::move(Err)); + + // Get the next JITDylib and lookup flags. + auto &KV = IPLS->SearchOrder[IPLS->CurSearchOrderIndex]; + auto &JD = *KV.first; + auto JDLookupFlags = KV.second; + + LLVM_DEBUG({ + dbgs() << "Visiting \"" << JD.getName() << "\" (" << JDLookupFlags + << ") with lookup set " << IPLS->LookupSet << ":\n"; + }); + + // If we've just reached a new JITDylib then perform some setup. + if (IPLS->NewJITDylib) { + + // Acquire the generator lock for this JITDylib. + IPLS->GeneratorLock = std::unique_lock<std::mutex>(JD.GeneratorsMutex); + + // Add any non-candidates from the last JITDylib (if any) back on to the + // list of definition candidates for this JITDylib, reset definition + // non-candiates to the empty set. + SymbolLookupSet Tmp; + std::swap(IPLS->DefGeneratorNonCandidates, Tmp); + IPLS->DefGeneratorCandidates.append(std::move(Tmp)); + + LLVM_DEBUG({ + dbgs() << " First time visiting " << JD.getName() + << ", resetting candidate sets and building generator stack\n"; + }); + + // Build the definition generator stack for this JITDylib. + for (auto &DG : reverse(JD.DefGenerators)) + IPLS->CurDefGeneratorStack.push_back(DG); + + // Flag that we've done our initialization. + IPLS->NewJITDylib = false; + } + + // Remove any generation candidates that are already defined (and match) in + // this JITDylib. + runSessionLocked([&] { + // Update the list of candidates (and non-candidates) for definition + // generation. + LLVM_DEBUG(dbgs() << " Updating candidate set...\n"); + Err = IL_updateCandidatesFor( + JD, JDLookupFlags, IPLS->DefGeneratorCandidates, + JD.DefGenerators.empty() ? nullptr + : &IPLS->DefGeneratorNonCandidates); + LLVM_DEBUG({ + dbgs() << " Remaining candidates = " << IPLS->DefGeneratorCandidates + << "\n"; + }); + }); + + // If we encountered an error while filtering generation candidates then + // bail out. + if (Err) + return IPLS->fail(std::move(Err)); + + /// Apply any definition generators on the stack. + LLVM_DEBUG({ + if (IPLS->CurDefGeneratorStack.empty()) + LLVM_DEBUG(dbgs() << " No generators to run for this JITDylib.\n"); + else if (IPLS->DefGeneratorCandidates.empty()) + LLVM_DEBUG(dbgs() << " No candidates to generate.\n"); + else + dbgs() << " Running " << IPLS->CurDefGeneratorStack.size() + << " remaining generators for " + << IPLS->DefGeneratorCandidates.size() << " candidates\n"; + }); + while (!IPLS->CurDefGeneratorStack.empty() && + !IPLS->DefGeneratorCandidates.empty()) { + auto DG = IPLS->CurDefGeneratorStack.back().lock(); + IPLS->CurDefGeneratorStack.pop_back(); + + if (!DG) + return IPLS->fail(make_error<StringError>( + "DefinitionGenerator removed while lookup in progress", + inconvertibleErrorCode())); + + auto K = IPLS->K; + auto &LookupSet = IPLS->DefGeneratorCandidates; + + // Run the generator. If the generator takes ownership of QA then this + // will break the loop. + { + LLVM_DEBUG(dbgs() << " Attempting to generate " << LookupSet << "\n"); + LookupState LS(std::move(IPLS)); + Err = DG->tryToGenerate(LS, K, JD, JDLookupFlags, LookupSet); + IPLS = std::move(LS.IPLS); + } + + // If there was an error then fail the query. + if (Err) { + LLVM_DEBUG({ + dbgs() << " Error attempting to generate " << LookupSet << "\n"; + }); + assert(IPLS && "LS cannot be retained if error is returned"); + return IPLS->fail(std::move(Err)); + } + + // Otherwise if QA was captured then break the loop. + if (!IPLS) { + LLVM_DEBUG( + { dbgs() << " LookupState captured. Exiting phase1 for now.\n"; }); + return; + } + + // Otherwise if we're continuing around the loop then update candidates + // for the next round. + runSessionLocked([&] { + LLVM_DEBUG(dbgs() << " Updating candidate set post-generation\n"); + Err = IL_updateCandidatesFor( + JD, JDLookupFlags, IPLS->DefGeneratorCandidates, + JD.DefGenerators.empty() ? nullptr + : &IPLS->DefGeneratorNonCandidates); + }); + + // If updating candidates failed then fail the query. + if (Err) { + LLVM_DEBUG(dbgs() << " Error encountered while updating candidates\n"); + return IPLS->fail(std::move(Err)); + } + } + + // If we get here then we've moved on to the next JITDylib. + LLVM_DEBUG(dbgs() << "Phase 1 moving to next JITDylib.\n"); + ++IPLS->CurSearchOrderIndex; + IPLS->NewJITDylib = true; + } + + // Remove any weakly referenced candidates that could not be found/generated. + IPLS->DefGeneratorCandidates.remove_if( + [](const SymbolStringPtr &Name, SymbolLookupFlags SymLookupFlags) { + return SymLookupFlags == SymbolLookupFlags::WeaklyReferencedSymbol; + }); + + // If we get here then we've finished searching all JITDylibs. + // If we matched all symbols then move to phase 2, otherwise fail the query + // with a SymbolsNotFound error. + if (IPLS->DefGeneratorCandidates.empty()) { + LLVM_DEBUG(dbgs() << "Phase 1 succeeded.\n"); + IPLS->complete(std::move(IPLS)); + } else { + LLVM_DEBUG(dbgs() << "Phase 1 failed with unresolved symbols.\n"); + IPLS->fail(make_error<SymbolsNotFound>( + IPLS->DefGeneratorCandidates.getSymbolNames())); + } +} + +void ExecutionSession::OL_completeLookup( + std::unique_ptr<InProgressLookupState> IPLS, + std::shared_ptr<AsynchronousSymbolQuery> Q, + RegisterDependenciesFunction RegisterDependencies) { + + LLVM_DEBUG({ + dbgs() << "Entering OL_completeLookup:\n" + << " Lookup kind: " << IPLS->K << "\n" + << " Search order: " << IPLS->SearchOrder + << ", Current index = " << IPLS->CurSearchOrderIndex + << (IPLS->NewJITDylib ? " (entering new JITDylib)" : "") << "\n" + << " Lookup set: " << IPLS->LookupSet << "\n" + << " Definition generator candidates: " + << IPLS->DefGeneratorCandidates << "\n" + << " Definition generator non-candidates: " + << IPLS->DefGeneratorNonCandidates << "\n"; + }); + + bool QueryComplete = false; + DenseMap<JITDylib *, JITDylib::UnmaterializedInfosList> CollectedUMIs; + + auto LodgingErr = runSessionLocked([&]() -> Error { + for (auto &KV : IPLS->SearchOrder) { + auto &JD = *KV.first; + auto JDLookupFlags = KV.second; + LLVM_DEBUG({ + dbgs() << "Visiting \"" << JD.getName() << "\" (" << JDLookupFlags + << ") with lookup set " << IPLS->LookupSet << ":\n"; + }); + + auto Err = IPLS->LookupSet.forEachWithRemoval( + [&](const SymbolStringPtr &Name, + SymbolLookupFlags SymLookupFlags) -> Expected<bool> { + LLVM_DEBUG({ + dbgs() << " Attempting to match \"" << Name << "\" (" + << SymLookupFlags << ")... "; + }); + + /// Search for the symbol. If not found then continue without + /// removal. + auto SymI = JD.Symbols.find(Name); + if (SymI == JD.Symbols.end()) { + LLVM_DEBUG(dbgs() << "skipping: not present\n"); + return false; + } + + // If this is a non-exported symbol and we're matching exported + // symbols only then skip this symbol without removal. + if (!SymI->second.getFlags().isExported() && + JDLookupFlags == + JITDylibLookupFlags::MatchExportedSymbolsOnly) { + LLVM_DEBUG(dbgs() << "skipping: not exported\n"); + return false; + } + + // If we match against a materialization-side-effects only symbol + // then make sure it is weakly-referenced. Otherwise bail out with + // an error. + // FIXME: Use a "materialization-side-effects-only symbols must be + // weakly referenced" specific error here to reduce confusion. + if (SymI->second.getFlags().hasMaterializationSideEffectsOnly() && + SymLookupFlags != SymbolLookupFlags::WeaklyReferencedSymbol) { + LLVM_DEBUG({ + dbgs() << "error: " + "required, but symbol is has-side-effects-only\n"; + }); + return make_error<SymbolsNotFound>(SymbolNameVector({Name})); + } + + // If we matched against this symbol but it is in the error state + // then bail out and treat it as a failure to materialize. + if (SymI->second.getFlags().hasError()) { + LLVM_DEBUG(dbgs() << "error: symbol is in error state\n"); + auto FailedSymbolsMap = std::make_shared<SymbolDependenceMap>(); + (*FailedSymbolsMap)[&JD] = {Name}; + return make_error<FailedToMaterialize>( + std::move(FailedSymbolsMap)); + } + + // Otherwise this is a match. + + // If this symbol is already in the requried state then notify the + // query, remove the symbol and continue. + if (SymI->second.getState() >= Q->getRequiredState()) { + LLVM_DEBUG(dbgs() + << "matched, symbol already in required state\n"); + Q->notifySymbolMetRequiredState(Name, SymI->second.getSymbol()); + return true; + } + + // Otherwise this symbol does not yet meet the required state. Check + // whether it has a materializer attached, and if so prepare to run + // it. + if (SymI->second.hasMaterializerAttached()) { + assert(SymI->second.getAddress() == 0 && + "Symbol not resolved but already has address?"); + auto UMII = JD.UnmaterializedInfos.find(Name); + assert(UMII != JD.UnmaterializedInfos.end() && + "Lazy symbol should have UnmaterializedInfo"); + + auto UMI = UMII->second; + assert(UMI->MU && "Materializer should not be null"); + assert(UMI->RT && "Tracker should not be null"); + LLVM_DEBUG({ + dbgs() << "matched, preparing to dispatch MU@" << UMI->MU.get() + << " (" << UMI->MU->getName() << ")\n"; + }); + + // Move all symbols associated with this MaterializationUnit into + // materializing state. + for (auto &KV : UMI->MU->getSymbols()) { + auto SymK = JD.Symbols.find(KV.first); + assert(SymK != JD.Symbols.end() && + "No entry for symbol covered by MaterializationUnit"); + SymK->second.setMaterializerAttached(false); + SymK->second.setState(SymbolState::Materializing); + JD.UnmaterializedInfos.erase(KV.first); + } + + // Add MU to the list of MaterializationUnits to be materialized. + CollectedUMIs[&JD].push_back(std::move(UMI)); + } else + LLVM_DEBUG(dbgs() << "matched, registering query"); + + // Add the query to the PendingQueries list and continue, deleting + // the element from the lookup set. + assert(SymI->second.getState() != SymbolState::NeverSearched && + SymI->second.getState() != SymbolState::Ready && + "By this line the symbol should be materializing"); + auto &MI = JD.MaterializingInfos[Name]; + MI.addQuery(Q); + Q->addQueryDependence(JD, Name); + + return true; + }); + + // Handle failure. + if (Err) { + + LLVM_DEBUG({ + dbgs() << "Lookup failed. Detaching query and replacing MUs.\n"; + }); + + // Detach the query. + Q->detach(); + + // Replace the MUs. + for (auto &KV : CollectedUMIs) { + auto &JD = *KV.first; + for (auto &UMI : KV.second) + for (auto &KV2 : UMI->MU->getSymbols()) { + assert(!JD.UnmaterializedInfos.count(KV2.first) && + "Unexpected materializer in map"); + auto SymI = JD.Symbols.find(KV2.first); + assert(SymI != JD.Symbols.end() && "Missing symbol entry"); + assert(SymI->second.getState() == SymbolState::Materializing && + "Can not replace symbol that is not materializing"); + assert(!SymI->second.hasMaterializerAttached() && + "MaterializerAttached flag should not be set"); + SymI->second.setMaterializerAttached(true); + JD.UnmaterializedInfos[KV2.first] = UMI; + } + } + + return Err; + } + } + + LLVM_DEBUG(dbgs() << "Stripping unmatched weakly-refererced symbols\n"); + IPLS->LookupSet.forEachWithRemoval( + [&](const SymbolStringPtr &Name, SymbolLookupFlags SymLookupFlags) { + if (SymLookupFlags == SymbolLookupFlags::WeaklyReferencedSymbol) { + Q->dropSymbol(Name); + return true; + } else + return false; + }); + + if (!IPLS->LookupSet.empty()) { + LLVM_DEBUG(dbgs() << "Failing due to unresolved symbols\n"); + return make_error<SymbolsNotFound>(IPLS->LookupSet.getSymbolNames()); + } + + // Record whether the query completed. + QueryComplete = Q->isComplete(); + + LLVM_DEBUG({ + dbgs() << "Query successfully " + << (QueryComplete ? "completed" : "lodged") << "\n"; + }); + + // Move the collected MUs to the OutstandingMUs list. + if (!CollectedUMIs.empty()) { + std::lock_guard<std::recursive_mutex> Lock(OutstandingMUsMutex); + + LLVM_DEBUG(dbgs() << "Adding MUs to dispatch:\n"); + for (auto &KV : CollectedUMIs) { + auto &JD = *KV.first; + LLVM_DEBUG({ + dbgs() << " For " << JD.getName() << ": Adding " << KV.second.size() + << " MUs.\n"; + }); + for (auto &UMI : KV.second) { + std::unique_ptr<MaterializationResponsibility> MR( + new MaterializationResponsibility( + &JD, std::move(UMI->MU->SymbolFlags), + std::move(UMI->MU->InitSymbol))); + JD.MRTrackers[MR.get()] = UMI->RT; + OutstandingMUs.push_back( + std::make_pair(std::move(UMI->MU), std::move(MR))); + } + } + } else + LLVM_DEBUG(dbgs() << "No MUs to dispatch.\n"); + + if (RegisterDependencies && !Q->QueryRegistrations.empty()) { + LLVM_DEBUG(dbgs() << "Registering dependencies\n"); + RegisterDependencies(Q->QueryRegistrations); + } else + LLVM_DEBUG(dbgs() << "No dependencies to register\n"); + + return Error::success(); + }); + + if (LodgingErr) { + LLVM_DEBUG(dbgs() << "Failing query\n"); + Q->detach(); + Q->handleFailed(std::move(LodgingErr)); + return; + } + + if (QueryComplete) { + LLVM_DEBUG(dbgs() << "Completing query\n"); + Q->handleComplete(); + } + + dispatchOutstandingMUs(); +} + +void ExecutionSession::OL_completeLookupFlags( + std::unique_ptr<InProgressLookupState> IPLS, + unique_function<void(Expected<SymbolFlagsMap>)> OnComplete) { + + auto Result = runSessionLocked([&]() -> Expected<SymbolFlagsMap> { + LLVM_DEBUG({ + dbgs() << "Entering OL_completeLookupFlags:\n" + << " Lookup kind: " << IPLS->K << "\n" + << " Search order: " << IPLS->SearchOrder + << ", Current index = " << IPLS->CurSearchOrderIndex + << (IPLS->NewJITDylib ? " (entering new JITDylib)" : "") << "\n" + << " Lookup set: " << IPLS->LookupSet << "\n" + << " Definition generator candidates: " + << IPLS->DefGeneratorCandidates << "\n" + << " Definition generator non-candidates: " + << IPLS->DefGeneratorNonCandidates << "\n"; + }); + + SymbolFlagsMap Result; + + // Attempt to find flags for each symbol. + for (auto &KV : IPLS->SearchOrder) { + auto &JD = *KV.first; + auto JDLookupFlags = KV.second; + LLVM_DEBUG({ + dbgs() << "Visiting \"" << JD.getName() << "\" (" << JDLookupFlags + << ") with lookup set " << IPLS->LookupSet << ":\n"; + }); + + IPLS->LookupSet.forEachWithRemoval([&](const SymbolStringPtr &Name, + SymbolLookupFlags SymLookupFlags) { + LLVM_DEBUG({ + dbgs() << " Attempting to match \"" << Name << "\" (" + << SymLookupFlags << ")... "; + }); + + // Search for the symbol. If not found then continue without removing + // from the lookup set. + auto SymI = JD.Symbols.find(Name); + if (SymI == JD.Symbols.end()) { + LLVM_DEBUG(dbgs() << "skipping: not present\n"); + return false; + } + + // If this is a non-exported symbol then it doesn't match. Skip it. + if (!SymI->second.getFlags().isExported() && + JDLookupFlags == JITDylibLookupFlags::MatchExportedSymbolsOnly) { + LLVM_DEBUG(dbgs() << "skipping: not exported\n"); + return false; + } + + LLVM_DEBUG({ + dbgs() << "matched, \"" << Name << "\" -> " << SymI->second.getFlags() + << "\n"; + }); + Result[Name] = SymI->second.getFlags(); + return true; + }); + } + + // Remove any weakly referenced symbols that haven't been resolved. + IPLS->LookupSet.remove_if( + [](const SymbolStringPtr &Name, SymbolLookupFlags SymLookupFlags) { + return SymLookupFlags == SymbolLookupFlags::WeaklyReferencedSymbol; + }); + + if (!IPLS->LookupSet.empty()) { + LLVM_DEBUG(dbgs() << "Failing due to unresolved symbols\n"); + return make_error<SymbolsNotFound>(IPLS->LookupSet.getSymbolNames()); + } + + LLVM_DEBUG(dbgs() << "Succeded, result = " << Result << "\n"); + return Result; + }); + + // Run the callback on the result. + LLVM_DEBUG(dbgs() << "Sending result to handler.\n"); + OnComplete(std::move(Result)); +} + +void ExecutionSession::OL_destroyMaterializationResponsibility( + MaterializationResponsibility &MR) { + + assert(MR.SymbolFlags.empty() && + "All symbols should have been explicitly materialized or failed"); + MR.JD->unlinkMaterializationResponsibility(MR); +} + +SymbolNameSet ExecutionSession::OL_getRequestedSymbols( + const MaterializationResponsibility &MR) { + return MR.JD->getRequestedSymbols(MR.SymbolFlags); +} + +Error ExecutionSession::OL_notifyResolved(MaterializationResponsibility &MR, + const SymbolMap &Symbols) { + LLVM_DEBUG({ + dbgs() << "In " << MR.JD->getName() << " resolving " << Symbols << "\n"; + }); +#ifndef NDEBUG + for (auto &KV : Symbols) { + auto WeakFlags = JITSymbolFlags::Weak | JITSymbolFlags::Common; + auto I = MR.SymbolFlags.find(KV.first); + assert(I != MR.SymbolFlags.end() && + "Resolving symbol outside this responsibility set"); + assert(!I->second.hasMaterializationSideEffectsOnly() && + "Can't resolve materialization-side-effects-only symbol"); + assert((KV.second.getFlags() & ~WeakFlags) == (I->second & ~WeakFlags) && + "Resolving symbol with incorrect flags"); + } +#endif + + return MR.JD->resolve(MR, Symbols); +} + +Error ExecutionSession::OL_notifyEmitted(MaterializationResponsibility &MR) { + LLVM_DEBUG({ + dbgs() << "In " << MR.JD->getName() << " emitting " << MR.SymbolFlags << "\n"; + }); + + if (auto Err = MR.JD->emit(MR, MR.SymbolFlags)) + return Err; + + MR.SymbolFlags.clear(); + return Error::success(); +} + +Error ExecutionSession::OL_defineMaterializing( + MaterializationResponsibility &MR, SymbolFlagsMap NewSymbolFlags) { + + LLVM_DEBUG({ + dbgs() << "In " << MR.JD->getName() << " defining materializing symbols " + << NewSymbolFlags << "\n"; + }); + if (auto AcceptedDefs = MR.JD->defineMaterializing(std::move(NewSymbolFlags))) { + // Add all newly accepted symbols to this responsibility object. + for (auto &KV : *AcceptedDefs) + MR.SymbolFlags.insert(KV); + return Error::success(); + } else + return AcceptedDefs.takeError(); +} + +void ExecutionSession::OL_notifyFailed(MaterializationResponsibility &MR) { + + LLVM_DEBUG({ + dbgs() << "In " << MR.JD->getName() << " failing materialization for " + << MR.SymbolFlags << "\n"; + }); + + JITDylib::FailedSymbolsWorklist Worklist; + + for (auto &KV : MR.SymbolFlags) + Worklist.push_back(std::make_pair(MR.JD.get(), KV.first)); + MR.SymbolFlags.clear(); + + if (Worklist.empty()) + return; + + JITDylib::AsynchronousSymbolQuerySet FailedQueries; + std::shared_ptr<SymbolDependenceMap> FailedSymbols; + + runSessionLocked([&]() { + auto RTI = MR.JD->MRTrackers.find(&MR); + assert(RTI != MR.JD->MRTrackers.end() && "No tracker for this"); + if (RTI->second->isDefunct()) + return; + + std::tie(FailedQueries, FailedSymbols) = + JITDylib::failSymbols(std::move(Worklist)); + }); + + for (auto &Q : FailedQueries) + Q->handleFailed(make_error<FailedToMaterialize>(FailedSymbols)); +} + +Error ExecutionSession::OL_replace(MaterializationResponsibility &MR, + std::unique_ptr<MaterializationUnit> MU) { + for (auto &KV : MU->getSymbols()) { + assert(MR.SymbolFlags.count(KV.first) && + "Replacing definition outside this responsibility set"); + MR.SymbolFlags.erase(KV.first); + } + + if (MU->getInitializerSymbol() == MR.InitSymbol) + MR.InitSymbol = nullptr; + + LLVM_DEBUG(MR.JD->getExecutionSession().runSessionLocked([&]() { + dbgs() << "In " << MR.JD->getName() << " replacing symbols with " << *MU + << "\n"; + });); + + return MR.JD->replace(MR, std::move(MU)); +} + +Expected<std::unique_ptr<MaterializationResponsibility>> +ExecutionSession::OL_delegate(MaterializationResponsibility &MR, + const SymbolNameSet &Symbols) { + + SymbolStringPtr DelegatedInitSymbol; + SymbolFlagsMap DelegatedFlags; + + for (auto &Name : Symbols) { + auto I = MR.SymbolFlags.find(Name); + assert(I != MR.SymbolFlags.end() && + "Symbol is not tracked by this MaterializationResponsibility " + "instance"); + + DelegatedFlags[Name] = std::move(I->second); + if (Name == MR.InitSymbol) + std::swap(MR.InitSymbol, DelegatedInitSymbol); + + MR.SymbolFlags.erase(I); + } + + return MR.JD->delegate(MR, std::move(DelegatedFlags), + std::move(DelegatedInitSymbol)); +} + +void ExecutionSession::OL_addDependencies( + MaterializationResponsibility &MR, const SymbolStringPtr &Name, + const SymbolDependenceMap &Dependencies) { + LLVM_DEBUG({ + dbgs() << "Adding dependencies for " << Name << ": " << Dependencies + << "\n"; + }); + assert(MR.SymbolFlags.count(Name) && + "Symbol not covered by this MaterializationResponsibility instance"); + MR.JD->addDependencies(Name, Dependencies); +} + +void ExecutionSession::OL_addDependenciesForAll( + MaterializationResponsibility &MR, + const SymbolDependenceMap &Dependencies) { + LLVM_DEBUG({ + dbgs() << "Adding dependencies for all symbols in " << MR.SymbolFlags << ": " + << Dependencies << "\n"; + }); + for (auto &KV : MR.SymbolFlags) + MR.JD->addDependencies(KV.first, Dependencies); +} + +#ifndef NDEBUG +void ExecutionSession::dumpDispatchInfo(JITDylib &JD, MaterializationUnit &MU) { + runSessionLocked([&]() { + dbgs() << "Dispatching " << MU << " for " << JD.getName() << "\n"; + }); +} +#endif // NDEBUG + +} // End namespace orc. +} // End namespace llvm. diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/Orc/DebugUtils.cpp b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/DebugUtils.cpp new file mode 100644 index 0000000000..6247158919 --- /dev/null +++ b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/DebugUtils.cpp @@ -0,0 +1,349 @@ +//===---------- DebugUtils.cpp - Utilities for debugging ORC JITs ---------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/DebugUtils.h" + +#include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Format.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/raw_ostream.h" + +#define DEBUG_TYPE "orc" + +using namespace llvm; + +namespace { + +#ifndef NDEBUG + +cl::opt<bool> PrintHidden("debug-orc-print-hidden", cl::init(true), + cl::desc("debug print hidden symbols defined by " + "materialization units"), + cl::Hidden); + +cl::opt<bool> PrintCallable("debug-orc-print-callable", cl::init(true), + cl::desc("debug print callable symbols defined by " + "materialization units"), + cl::Hidden); + +cl::opt<bool> PrintData("debug-orc-print-data", cl::init(true), + cl::desc("debug print data symbols defined by " + "materialization units"), + cl::Hidden); + +#endif // NDEBUG + +// SetPrinter predicate that prints every element. +template <typename T> struct PrintAll { + bool operator()(const T &E) { return true; } +}; + +bool anyPrintSymbolOptionSet() { +#ifndef NDEBUG + return PrintHidden || PrintCallable || PrintData; +#else + return false; +#endif // NDEBUG +} + +bool flagsMatchCLOpts(const JITSymbolFlags &Flags) { +#ifndef NDEBUG + // Bail out early if this is a hidden symbol and we're not printing hiddens. + if (!PrintHidden && !Flags.isExported()) + return false; + + // Return true if this is callable and we're printing callables. + if (PrintCallable && Flags.isCallable()) + return true; + + // Return true if this is data and we're printing data. + if (PrintData && !Flags.isCallable()) + return true; + + // otherwise return false. + return false; +#else + return false; +#endif // NDEBUG +} + +// Prints a sequence of items, filtered by an user-supplied predicate. +template <typename Sequence, + typename Pred = PrintAll<typename Sequence::value_type>> +class SequencePrinter { +public: + SequencePrinter(const Sequence &S, char OpenSeq, char CloseSeq, + Pred ShouldPrint = Pred()) + : S(S), OpenSeq(OpenSeq), CloseSeq(CloseSeq), + ShouldPrint(std::move(ShouldPrint)) {} + + void printTo(llvm::raw_ostream &OS) const { + bool PrintComma = false; + OS << OpenSeq; + for (auto &E : S) { + if (ShouldPrint(E)) { + if (PrintComma) + OS << ','; + OS << ' ' << E; + PrintComma = true; + } + } + OS << ' ' << CloseSeq; + } + +private: + const Sequence &S; + char OpenSeq; + char CloseSeq; + mutable Pred ShouldPrint; +}; + +template <typename Sequence, typename Pred> +SequencePrinter<Sequence, Pred> printSequence(const Sequence &S, char OpenSeq, + char CloseSeq, Pred P = Pred()) { + return SequencePrinter<Sequence, Pred>(S, OpenSeq, CloseSeq, std::move(P)); +} + +// Render a SequencePrinter by delegating to its printTo method. +template <typename Sequence, typename Pred> +llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, + const SequencePrinter<Sequence, Pred> &Printer) { + Printer.printTo(OS); + return OS; +} + +struct PrintSymbolFlagsMapElemsMatchingCLOpts { + bool operator()(const orc::SymbolFlagsMap::value_type &KV) { + return flagsMatchCLOpts(KV.second); + } +}; + +struct PrintSymbolMapElemsMatchingCLOpts { + bool operator()(const orc::SymbolMap::value_type &KV) { + return flagsMatchCLOpts(KV.second.getFlags()); + } +}; + +} // end anonymous namespace + +namespace llvm { +namespace orc { + +raw_ostream &operator<<(raw_ostream &OS, const SymbolStringPtr &Sym) { + return OS << *Sym; +} + +raw_ostream &operator<<(raw_ostream &OS, const SymbolNameSet &Symbols) { + return OS << printSequence(Symbols, '{', '}', PrintAll<SymbolStringPtr>()); +} + +raw_ostream &operator<<(raw_ostream &OS, const SymbolNameVector &Symbols) { + return OS << printSequence(Symbols, '[', ']', PrintAll<SymbolStringPtr>()); +} + +raw_ostream &operator<<(raw_ostream &OS, ArrayRef<SymbolStringPtr> Symbols) { + return OS << printSequence(Symbols, '[', ']', PrintAll<SymbolStringPtr>()); +} + +raw_ostream &operator<<(raw_ostream &OS, const JITSymbolFlags &Flags) { + if (Flags.hasError()) + OS << "[*ERROR*]"; + if (Flags.isCallable()) + OS << "[Callable]"; + else + OS << "[Data]"; + if (Flags.isWeak()) + OS << "[Weak]"; + else if (Flags.isCommon()) + OS << "[Common]"; + + if (!Flags.isExported()) + OS << "[Hidden]"; + + return OS; +} + +raw_ostream &operator<<(raw_ostream &OS, const JITEvaluatedSymbol &Sym) { + return OS << format("0x%016" PRIx64, Sym.getAddress()) << " " + << Sym.getFlags(); +} + +raw_ostream &operator<<(raw_ostream &OS, const SymbolFlagsMap::value_type &KV) { + return OS << "(\"" << KV.first << "\", " << KV.second << ")"; +} + +raw_ostream &operator<<(raw_ostream &OS, const SymbolMap::value_type &KV) { + return OS << "(\"" << KV.first << "\": " << KV.second << ")"; +} + +raw_ostream &operator<<(raw_ostream &OS, const SymbolFlagsMap &SymbolFlags) { + return OS << printSequence(SymbolFlags, '{', '}', + PrintSymbolFlagsMapElemsMatchingCLOpts()); +} + +raw_ostream &operator<<(raw_ostream &OS, const SymbolMap &Symbols) { + return OS << printSequence(Symbols, '{', '}', + PrintSymbolMapElemsMatchingCLOpts()); +} + +raw_ostream &operator<<(raw_ostream &OS, + const SymbolDependenceMap::value_type &KV) { + return OS << "(" << KV.first->getName() << ", " << KV.second << ")"; +} + +raw_ostream &operator<<(raw_ostream &OS, const SymbolDependenceMap &Deps) { + return OS << printSequence(Deps, '{', '}', + PrintAll<SymbolDependenceMap::value_type>()); +} + +raw_ostream &operator<<(raw_ostream &OS, const MaterializationUnit &MU) { + OS << "MU@" << &MU << " (\"" << MU.getName() << "\""; + if (anyPrintSymbolOptionSet()) + OS << ", " << MU.getSymbols(); + return OS << ")"; +} + +raw_ostream &operator<<(raw_ostream &OS, const LookupKind &K) { + switch (K) { + case LookupKind::Static: + return OS << "Static"; + case LookupKind::DLSym: + return OS << "DLSym"; + } + llvm_unreachable("Invalid lookup kind"); +} + +raw_ostream &operator<<(raw_ostream &OS, + const JITDylibLookupFlags &JDLookupFlags) { + switch (JDLookupFlags) { + case JITDylibLookupFlags::MatchExportedSymbolsOnly: + return OS << "MatchExportedSymbolsOnly"; + case JITDylibLookupFlags::MatchAllSymbols: + return OS << "MatchAllSymbols"; + } + llvm_unreachable("Invalid JITDylib lookup flags"); +} + +raw_ostream &operator<<(raw_ostream &OS, const SymbolLookupFlags &LookupFlags) { + switch (LookupFlags) { + case SymbolLookupFlags::RequiredSymbol: + return OS << "RequiredSymbol"; + case SymbolLookupFlags::WeaklyReferencedSymbol: + return OS << "WeaklyReferencedSymbol"; + } + llvm_unreachable("Invalid symbol lookup flags"); +} + +raw_ostream &operator<<(raw_ostream &OS, + const SymbolLookupSet::value_type &KV) { + return OS << "(" << KV.first << ", " << KV.second << ")"; +} + +raw_ostream &operator<<(raw_ostream &OS, const SymbolLookupSet &LookupSet) { + return OS << printSequence(LookupSet, '{', '}', + PrintAll<SymbolLookupSet::value_type>()); +} + +raw_ostream &operator<<(raw_ostream &OS, + const JITDylibSearchOrder &SearchOrder) { + OS << "["; + if (!SearchOrder.empty()) { + assert(SearchOrder.front().first && + "JITDylibList entries must not be null"); + OS << " (\"" << SearchOrder.front().first->getName() << "\", " + << SearchOrder.begin()->second << ")"; + for (auto &KV : + make_range(std::next(SearchOrder.begin(), 1), SearchOrder.end())) { + assert(KV.first && "JITDylibList entries must not be null"); + OS << ", (\"" << KV.first->getName() << "\", " << KV.second << ")"; + } + } + OS << " ]"; + return OS; +} + +raw_ostream &operator<<(raw_ostream &OS, const SymbolAliasMap &Aliases) { + OS << "{"; + for (auto &KV : Aliases) + OS << " " << *KV.first << ": " << KV.second.Aliasee << " " + << KV.second.AliasFlags; + OS << " }"; + return OS; +} + +raw_ostream &operator<<(raw_ostream &OS, const SymbolState &S) { + switch (S) { + case SymbolState::Invalid: + return OS << "Invalid"; + case SymbolState::NeverSearched: + return OS << "Never-Searched"; + case SymbolState::Materializing: + return OS << "Materializing"; + case SymbolState::Resolved: + return OS << "Resolved"; + case SymbolState::Emitted: + return OS << "Emitted"; + case SymbolState::Ready: + return OS << "Ready"; + } + llvm_unreachable("Invalid state"); +} + +DumpObjects::DumpObjects(std::string DumpDir, std::string IdentifierOverride) + : DumpDir(std::move(DumpDir)), + IdentifierOverride(std::move(IdentifierOverride)) { + + /// Discard any trailing separators. + while (!this->DumpDir.empty() && + sys::path::is_separator(this->DumpDir.back())) + this->DumpDir.pop_back(); +} + +Expected<std::unique_ptr<MemoryBuffer>> +DumpObjects::operator()(std::unique_ptr<MemoryBuffer> Obj) { + size_t Idx = 1; + + std::string DumpPathStem; + raw_string_ostream(DumpPathStem) + << DumpDir << (DumpDir.empty() ? "" : "/") << getBufferIdentifier(*Obj); + + std::string DumpPath = DumpPathStem + ".o"; + while (sys::fs::exists(DumpPath)) { + DumpPath.clear(); + raw_string_ostream(DumpPath) << DumpPathStem << "." << (++Idx) << ".o"; + } + + LLVM_DEBUG({ + dbgs() << "Dumping object buffer [ " << (const void *)Obj->getBufferStart() + << " -- " << (const void *)(Obj->getBufferEnd() - 1) << " ] to " + << DumpPath << "\n"; + }); + + std::error_code EC; + raw_fd_ostream DumpStream(DumpPath, EC); + if (EC) + return errorCodeToError(EC); + DumpStream.write(Obj->getBufferStart(), Obj->getBufferSize()); + + return std::move(Obj); +} + +StringRef DumpObjects::getBufferIdentifier(MemoryBuffer &B) { + if (!IdentifierOverride.empty()) + return IdentifierOverride; + StringRef Identifier = B.getBufferIdentifier(); + Identifier.consume_back(".o"); + return Identifier; +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/Orc/ExecutionUtils.cpp b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/ExecutionUtils.cpp new file mode 100644 index 0000000000..6a1a41a13a --- /dev/null +++ b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/ExecutionUtils.cpp @@ -0,0 +1,387 @@ +//===---- ExecutionUtils.cpp - Utilities for executing functions in Orc ---===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" + +#include "llvm/ExecutionEngine/Orc/Layer.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/GlobalVariable.h" +#include "llvm/IR/Module.h" +#include "llvm/Object/MachOUniversal.h" +#include "llvm/Support/FormatVariadic.h" +#include "llvm/Support/TargetRegistry.h" +#include "llvm/Target/TargetMachine.h" + +namespace llvm { +namespace orc { + +CtorDtorIterator::CtorDtorIterator(const GlobalVariable *GV, bool End) + : InitList( + GV ? dyn_cast_or_null<ConstantArray>(GV->getInitializer()) : nullptr), + I((InitList && End) ? InitList->getNumOperands() : 0) { +} + +bool CtorDtorIterator::operator==(const CtorDtorIterator &Other) const { + assert(InitList == Other.InitList && "Incomparable iterators."); + return I == Other.I; +} + +bool CtorDtorIterator::operator!=(const CtorDtorIterator &Other) const { + return !(*this == Other); +} + +CtorDtorIterator& CtorDtorIterator::operator++() { + ++I; + return *this; +} + +CtorDtorIterator CtorDtorIterator::operator++(int) { + CtorDtorIterator Temp = *this; + ++I; + return Temp; +} + +CtorDtorIterator::Element CtorDtorIterator::operator*() const { + ConstantStruct *CS = dyn_cast<ConstantStruct>(InitList->getOperand(I)); + assert(CS && "Unrecognized type in llvm.global_ctors/llvm.global_dtors"); + + Constant *FuncC = CS->getOperand(1); + Function *Func = nullptr; + + // Extract function pointer, pulling off any casts. + while (FuncC) { + if (Function *F = dyn_cast_or_null<Function>(FuncC)) { + Func = F; + break; + } else if (ConstantExpr *CE = dyn_cast_or_null<ConstantExpr>(FuncC)) { + if (CE->isCast()) + FuncC = dyn_cast_or_null<ConstantExpr>(CE->getOperand(0)); + else + break; + } else { + // This isn't anything we recognize. Bail out with Func left set to null. + break; + } + } + + auto *Priority = cast<ConstantInt>(CS->getOperand(0)); + Value *Data = CS->getNumOperands() == 3 ? CS->getOperand(2) : nullptr; + if (Data && !isa<GlobalValue>(Data)) + Data = nullptr; + return Element(Priority->getZExtValue(), Func, Data); +} + +iterator_range<CtorDtorIterator> getConstructors(const Module &M) { + const GlobalVariable *CtorsList = M.getNamedGlobal("llvm.global_ctors"); + return make_range(CtorDtorIterator(CtorsList, false), + CtorDtorIterator(CtorsList, true)); +} + +iterator_range<CtorDtorIterator> getDestructors(const Module &M) { + const GlobalVariable *DtorsList = M.getNamedGlobal("llvm.global_dtors"); + return make_range(CtorDtorIterator(DtorsList, false), + CtorDtorIterator(DtorsList, true)); +} + +bool StaticInitGVIterator::isStaticInitGlobal(GlobalValue &GV) { + if (GV.isDeclaration()) + return false; + + if (GV.hasName() && (GV.getName() == "llvm.global_ctors" || + GV.getName() == "llvm.global_dtors")) + return true; + + if (ObjFmt == Triple::MachO) { + // FIXME: These section checks are too strict: We should match first and + // second word split by comma. + if (GV.hasSection() && + (GV.getSection().startswith("__DATA,__objc_classlist") || + GV.getSection().startswith("__DATA,__objc_selrefs"))) + return true; + } + + return false; +} + +void CtorDtorRunner::add(iterator_range<CtorDtorIterator> CtorDtors) { + if (CtorDtors.empty()) + return; + + MangleAndInterner Mangle( + JD.getExecutionSession(), + (*CtorDtors.begin()).Func->getParent()->getDataLayout()); + + for (auto CtorDtor : CtorDtors) { + assert(CtorDtor.Func && CtorDtor.Func->hasName() && + "Ctor/Dtor function must be named to be runnable under the JIT"); + + // FIXME: Maybe use a symbol promoter here instead. + if (CtorDtor.Func->hasLocalLinkage()) { + CtorDtor.Func->setLinkage(GlobalValue::ExternalLinkage); + CtorDtor.Func->setVisibility(GlobalValue::HiddenVisibility); + } + + if (CtorDtor.Data && cast<GlobalValue>(CtorDtor.Data)->isDeclaration()) { + dbgs() << " Skipping because why now?\n"; + continue; + } + + CtorDtorsByPriority[CtorDtor.Priority].push_back( + Mangle(CtorDtor.Func->getName())); + } +} + +Error CtorDtorRunner::run() { + using CtorDtorTy = void (*)(); + + SymbolLookupSet LookupSet; + for (auto &KV : CtorDtorsByPriority) + for (auto &Name : KV.second) + LookupSet.add(Name); + assert(!LookupSet.containsDuplicates() && + "Ctor/Dtor list contains duplicates"); + + auto &ES = JD.getExecutionSession(); + if (auto CtorDtorMap = ES.lookup( + makeJITDylibSearchOrder(&JD, JITDylibLookupFlags::MatchAllSymbols), + std::move(LookupSet))) { + for (auto &KV : CtorDtorsByPriority) { + for (auto &Name : KV.second) { + assert(CtorDtorMap->count(Name) && "No entry for Name"); + auto CtorDtor = reinterpret_cast<CtorDtorTy>( + static_cast<uintptr_t>((*CtorDtorMap)[Name].getAddress())); + CtorDtor(); + } + } + CtorDtorsByPriority.clear(); + return Error::success(); + } else + return CtorDtorMap.takeError(); +} + +void LocalCXXRuntimeOverridesBase::runDestructors() { + auto& CXXDestructorDataPairs = DSOHandleOverride; + for (auto &P : CXXDestructorDataPairs) + P.first(P.second); + CXXDestructorDataPairs.clear(); +} + +int LocalCXXRuntimeOverridesBase::CXAAtExitOverride(DestructorPtr Destructor, + void *Arg, + void *DSOHandle) { + auto& CXXDestructorDataPairs = + *reinterpret_cast<CXXDestructorDataPairList*>(DSOHandle); + CXXDestructorDataPairs.push_back(std::make_pair(Destructor, Arg)); + return 0; +} + +Error LocalCXXRuntimeOverrides::enable(JITDylib &JD, + MangleAndInterner &Mangle) { + SymbolMap RuntimeInterposes; + RuntimeInterposes[Mangle("__dso_handle")] = + JITEvaluatedSymbol(toTargetAddress(&DSOHandleOverride), + JITSymbolFlags::Exported); + RuntimeInterposes[Mangle("__cxa_atexit")] = + JITEvaluatedSymbol(toTargetAddress(&CXAAtExitOverride), + JITSymbolFlags::Exported); + + return JD.define(absoluteSymbols(std::move(RuntimeInterposes))); +} + +void ItaniumCXAAtExitSupport::registerAtExit(void (*F)(void *), void *Ctx, + void *DSOHandle) { + std::lock_guard<std::mutex> Lock(AtExitsMutex); + AtExitRecords[DSOHandle].push_back({F, Ctx}); +} + +void ItaniumCXAAtExitSupport::runAtExits(void *DSOHandle) { + std::vector<AtExitRecord> AtExitsToRun; + + { + std::lock_guard<std::mutex> Lock(AtExitsMutex); + auto I = AtExitRecords.find(DSOHandle); + if (I != AtExitRecords.end()) { + AtExitsToRun = std::move(I->second); + AtExitRecords.erase(I); + } + } + + while (!AtExitsToRun.empty()) { + AtExitsToRun.back().F(AtExitsToRun.back().Ctx); + AtExitsToRun.pop_back(); + } +} + +DynamicLibrarySearchGenerator::DynamicLibrarySearchGenerator( + sys::DynamicLibrary Dylib, char GlobalPrefix, SymbolPredicate Allow) + : Dylib(std::move(Dylib)), Allow(std::move(Allow)), + GlobalPrefix(GlobalPrefix) {} + +Expected<std::unique_ptr<DynamicLibrarySearchGenerator>> +DynamicLibrarySearchGenerator::Load(const char *FileName, char GlobalPrefix, + SymbolPredicate Allow) { + std::string ErrMsg; + auto Lib = sys::DynamicLibrary::getPermanentLibrary(FileName, &ErrMsg); + if (!Lib.isValid()) + return make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode()); + return std::make_unique<DynamicLibrarySearchGenerator>( + std::move(Lib), GlobalPrefix, std::move(Allow)); +} + +Error DynamicLibrarySearchGenerator::tryToGenerate( + LookupState &LS, LookupKind K, JITDylib &JD, + JITDylibLookupFlags JDLookupFlags, const SymbolLookupSet &Symbols) { + orc::SymbolMap NewSymbols; + + bool HasGlobalPrefix = (GlobalPrefix != '\0'); + + for (auto &KV : Symbols) { + auto &Name = KV.first; + + if ((*Name).empty()) + continue; + + if (Allow && !Allow(Name)) + continue; + + if (HasGlobalPrefix && (*Name).front() != GlobalPrefix) + continue; + + std::string Tmp((*Name).data() + HasGlobalPrefix, + (*Name).size() - HasGlobalPrefix); + if (void *Addr = Dylib.getAddressOfSymbol(Tmp.c_str())) { + NewSymbols[Name] = JITEvaluatedSymbol( + static_cast<JITTargetAddress>(reinterpret_cast<uintptr_t>(Addr)), + JITSymbolFlags::Exported); + } + } + + if (NewSymbols.empty()) + return Error::success(); + + return JD.define(absoluteSymbols(std::move(NewSymbols))); +} + +Expected<std::unique_ptr<StaticLibraryDefinitionGenerator>> +StaticLibraryDefinitionGenerator::Load(ObjectLayer &L, const char *FileName) { + auto ArchiveBuffer = errorOrToExpected(MemoryBuffer::getFile(FileName)); + + if (!ArchiveBuffer) + return ArchiveBuffer.takeError(); + + return Create(L, std::move(*ArchiveBuffer)); +} + +Expected<std::unique_ptr<StaticLibraryDefinitionGenerator>> +StaticLibraryDefinitionGenerator::Load(ObjectLayer &L, const char *FileName, + const Triple &TT) { + auto B = object::createBinary(FileName); + if (!B) + return B.takeError(); + + // If this is a regular archive then create an instance from it. + if (isa<object::Archive>(B->getBinary())) + return Create(L, std::move(B->takeBinary().second)); + + // If this is a universal binary then search for a slice matching the given + // Triple. + if (auto *UB = cast<object::MachOUniversalBinary>(B->getBinary())) { + for (const auto &Obj : UB->objects()) { + auto ObjTT = Obj.getTriple(); + if (ObjTT.getArch() == TT.getArch() && + ObjTT.getSubArch() == TT.getSubArch() && + (TT.getVendor() == Triple::UnknownVendor || + ObjTT.getVendor() == TT.getVendor())) { + // We found a match. Create an instance from a buffer covering this + // slice. + auto SliceBuffer = MemoryBuffer::getFileSlice(FileName, Obj.getSize(), + Obj.getOffset()); + if (!SliceBuffer) + return make_error<StringError>( + Twine("Could not create buffer for ") + TT.str() + " slice of " + + FileName + ": [ " + formatv("{0:x}", Obj.getOffset()) + + " .. " + formatv("{0:x}", Obj.getOffset() + Obj.getSize()) + + ": " + SliceBuffer.getError().message(), + SliceBuffer.getError()); + return Create(L, std::move(*SliceBuffer)); + } + } + + return make_error<StringError>(Twine("Universal binary ") + FileName + + " does not contain a slice for " + + TT.str(), + inconvertibleErrorCode()); + } + + return make_error<StringError>(Twine("Unrecognized file type for ") + + FileName, + inconvertibleErrorCode()); +} + +Expected<std::unique_ptr<StaticLibraryDefinitionGenerator>> +StaticLibraryDefinitionGenerator::Create( + ObjectLayer &L, std::unique_ptr<MemoryBuffer> ArchiveBuffer) { + Error Err = Error::success(); + + std::unique_ptr<StaticLibraryDefinitionGenerator> ADG( + new StaticLibraryDefinitionGenerator(L, std::move(ArchiveBuffer), Err)); + + if (Err) + return std::move(Err); + + return std::move(ADG); +} + +Error StaticLibraryDefinitionGenerator::tryToGenerate( + LookupState &LS, LookupKind K, JITDylib &JD, + JITDylibLookupFlags JDLookupFlags, const SymbolLookupSet &Symbols) { + + // Don't materialize symbols from static archives unless this is a static + // lookup. + if (K != LookupKind::Static) + return Error::success(); + + // Bail out early if we've already freed the archive. + if (!Archive) + return Error::success(); + + DenseSet<std::pair<StringRef, StringRef>> ChildBufferInfos; + + for (const auto &KV : Symbols) { + const auto &Name = KV.first; + auto Child = Archive->findSym(*Name); + if (!Child) + return Child.takeError(); + if (*Child == None) + continue; + auto ChildBuffer = (*Child)->getMemoryBufferRef(); + if (!ChildBuffer) + return ChildBuffer.takeError(); + ChildBufferInfos.insert( + {ChildBuffer->getBuffer(), ChildBuffer->getBufferIdentifier()}); + } + + for (auto ChildBufferInfo : ChildBufferInfos) { + MemoryBufferRef ChildBufferRef(ChildBufferInfo.first, + ChildBufferInfo.second); + + if (auto Err = L.add(JD, MemoryBuffer::getMemBuffer(ChildBufferRef, false))) + return Err; + } + + return Error::success(); +} + +StaticLibraryDefinitionGenerator::StaticLibraryDefinitionGenerator( + ObjectLayer &L, std::unique_ptr<MemoryBuffer> ArchiveBuffer, Error &Err) + : L(L), ArchiveBuffer(std::move(ArchiveBuffer)), + Archive(std::make_unique<object::Archive>(*this->ArchiveBuffer, Err)) {} + +} // End namespace orc. +} // End namespace llvm. diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/Orc/IRCompileLayer.cpp b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/IRCompileLayer.cpp new file mode 100644 index 0000000000..aadc437c80 --- /dev/null +++ b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/IRCompileLayer.cpp @@ -0,0 +1,48 @@ +//===--------------- IRCompileLayer.cpp - IR Compiling Layer --------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/IRCompileLayer.h" + +namespace llvm { +namespace orc { + +IRCompileLayer::IRCompiler::~IRCompiler() {} + +IRCompileLayer::IRCompileLayer(ExecutionSession &ES, ObjectLayer &BaseLayer, + std::unique_ptr<IRCompiler> Compile) + : IRLayer(ES, ManglingOpts), BaseLayer(BaseLayer), + Compile(std::move(Compile)) { + ManglingOpts = &this->Compile->getManglingOptions(); +} + +void IRCompileLayer::setNotifyCompiled(NotifyCompiledFunction NotifyCompiled) { + std::lock_guard<std::mutex> Lock(IRLayerMutex); + this->NotifyCompiled = std::move(NotifyCompiled); +} + +void IRCompileLayer::emit(std::unique_ptr<MaterializationResponsibility> R, + ThreadSafeModule TSM) { + assert(TSM && "Module must not be null"); + + if (auto Obj = TSM.withModuleDo(*Compile)) { + { + std::lock_guard<std::mutex> Lock(IRLayerMutex); + if (NotifyCompiled) + NotifyCompiled(*R, std::move(TSM)); + else + TSM = ThreadSafeModule(); + } + BaseLayer.emit(std::move(R), std::move(*Obj)); + } else { + R->failMaterialization(); + getExecutionSession().reportError(Obj.takeError()); + } +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/Orc/IRTransformLayer.cpp b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/IRTransformLayer.cpp new file mode 100644 index 0000000000..d5b1134927 --- /dev/null +++ b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/IRTransformLayer.cpp @@ -0,0 +1,33 @@ +//===-------------- IRTransformLayer.cpp - IR Transform Layer -------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/IRTransformLayer.h" +#include "llvm/Support/MemoryBuffer.h" + +namespace llvm { +namespace orc { + +IRTransformLayer::IRTransformLayer(ExecutionSession &ES, IRLayer &BaseLayer, + TransformFunction Transform) + : IRLayer(ES, BaseLayer.getManglingOptions()), BaseLayer(BaseLayer), + Transform(std::move(Transform)) {} + +void IRTransformLayer::emit(std::unique_ptr<MaterializationResponsibility> R, + ThreadSafeModule TSM) { + assert(TSM && "Module must not be null"); + + if (auto TransformedTSM = Transform(std::move(TSM), *R)) + BaseLayer.emit(std::move(R), std::move(*TransformedTSM)); + else { + R->failMaterialization(); + getExecutionSession().reportError(TransformedTSM.takeError()); + } +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/Orc/IndirectionUtils.cpp b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/IndirectionUtils.cpp new file mode 100644 index 0000000000..1cfcf8ae94 --- /dev/null +++ b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/IndirectionUtils.cpp @@ -0,0 +1,375 @@ +//===---- IndirectionUtils.cpp - Utilities for call indirection in Orc ----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/IndirectionUtils.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/Triple.h" +#include "llvm/ExecutionEngine/Orc/OrcABISupport.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/Support/Format.h" +#include "llvm/Transforms/Utils/Cloning.h" +#include <sstream> + +using namespace llvm; +using namespace llvm::orc; + +namespace { + +class CompileCallbackMaterializationUnit : public orc::MaterializationUnit { +public: + using CompileFunction = JITCompileCallbackManager::CompileFunction; + + CompileCallbackMaterializationUnit(SymbolStringPtr Name, + CompileFunction Compile) + : MaterializationUnit(SymbolFlagsMap({{Name, JITSymbolFlags::Exported}}), + nullptr), + Name(std::move(Name)), Compile(std::move(Compile)) {} + + StringRef getName() const override { return "<Compile Callbacks>"; } + +private: + void materialize(std::unique_ptr<MaterializationResponsibility> R) override { + SymbolMap Result; + Result[Name] = JITEvaluatedSymbol(Compile(), JITSymbolFlags::Exported); + // No dependencies, so these calls cannot fail. + cantFail(R->notifyResolved(Result)); + cantFail(R->notifyEmitted()); + } + + void discard(const JITDylib &JD, const SymbolStringPtr &Name) override { + llvm_unreachable("Discard should never occur on a LMU?"); + } + + SymbolStringPtr Name; + CompileFunction Compile; +}; + +} // namespace + +namespace llvm { +namespace orc { + +TrampolinePool::~TrampolinePool() {} +void IndirectStubsManager::anchor() {} + +Expected<JITTargetAddress> +JITCompileCallbackManager::getCompileCallback(CompileFunction Compile) { + if (auto TrampolineAddr = TP->getTrampoline()) { + auto CallbackName = + ES.intern(std::string("cc") + std::to_string(++NextCallbackId)); + + std::lock_guard<std::mutex> Lock(CCMgrMutex); + AddrToSymbol[*TrampolineAddr] = CallbackName; + cantFail( + CallbacksJD.define(std::make_unique<CompileCallbackMaterializationUnit>( + std::move(CallbackName), std::move(Compile)))); + return *TrampolineAddr; + } else + return TrampolineAddr.takeError(); +} + +JITTargetAddress JITCompileCallbackManager::executeCompileCallback( + JITTargetAddress TrampolineAddr) { + SymbolStringPtr Name; + + { + std::unique_lock<std::mutex> Lock(CCMgrMutex); + auto I = AddrToSymbol.find(TrampolineAddr); + + // If this address is not associated with a compile callback then report an + // error to the execution session and return ErrorHandlerAddress to the + // callee. + if (I == AddrToSymbol.end()) { + Lock.unlock(); + std::string ErrMsg; + { + raw_string_ostream ErrMsgStream(ErrMsg); + ErrMsgStream << "No compile callback for trampoline at " + << format("0x%016" PRIx64, TrampolineAddr); + } + ES.reportError( + make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode())); + return ErrorHandlerAddress; + } else + Name = I->second; + } + + if (auto Sym = + ES.lookup(makeJITDylibSearchOrder( + &CallbacksJD, JITDylibLookupFlags::MatchAllSymbols), + Name)) + return Sym->getAddress(); + else { + llvm::dbgs() << "Didn't find callback.\n"; + // If anything goes wrong materializing Sym then report it to the session + // and return the ErrorHandlerAddress; + ES.reportError(Sym.takeError()); + return ErrorHandlerAddress; + } +} + +Expected<std::unique_ptr<JITCompileCallbackManager>> +createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES, + JITTargetAddress ErrorHandlerAddress) { + switch (T.getArch()) { + default: + return make_error<StringError>( + std::string("No callback manager available for ") + T.str(), + inconvertibleErrorCode()); + case Triple::aarch64: + case Triple::aarch64_32: { + typedef orc::LocalJITCompileCallbackManager<orc::OrcAArch64> CCMgrT; + return CCMgrT::Create(ES, ErrorHandlerAddress); + } + + case Triple::x86: { + typedef orc::LocalJITCompileCallbackManager<orc::OrcI386> CCMgrT; + return CCMgrT::Create(ES, ErrorHandlerAddress); + } + + case Triple::mips: { + typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Be> CCMgrT; + return CCMgrT::Create(ES, ErrorHandlerAddress); + } + case Triple::mipsel: { + typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Le> CCMgrT; + return CCMgrT::Create(ES, ErrorHandlerAddress); + } + + case Triple::mips64: + case Triple::mips64el: { + typedef orc::LocalJITCompileCallbackManager<orc::OrcMips64> CCMgrT; + return CCMgrT::Create(ES, ErrorHandlerAddress); + } + + case Triple::x86_64: { + if (T.getOS() == Triple::OSType::Win32) { + typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_Win32> CCMgrT; + return CCMgrT::Create(ES, ErrorHandlerAddress); + } else { + typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_SysV> CCMgrT; + return CCMgrT::Create(ES, ErrorHandlerAddress); + } + } + + } +} + +std::function<std::unique_ptr<IndirectStubsManager>()> +createLocalIndirectStubsManagerBuilder(const Triple &T) { + switch (T.getArch()) { + default: + return [](){ + return std::make_unique< + orc::LocalIndirectStubsManager<orc::OrcGenericABI>>(); + }; + + case Triple::aarch64: + case Triple::aarch64_32: + return [](){ + return std::make_unique< + orc::LocalIndirectStubsManager<orc::OrcAArch64>>(); + }; + + case Triple::x86: + return [](){ + return std::make_unique< + orc::LocalIndirectStubsManager<orc::OrcI386>>(); + }; + + case Triple::mips: + return [](){ + return std::make_unique< + orc::LocalIndirectStubsManager<orc::OrcMips32Be>>(); + }; + + case Triple::mipsel: + return [](){ + return std::make_unique< + orc::LocalIndirectStubsManager<orc::OrcMips32Le>>(); + }; + + case Triple::mips64: + case Triple::mips64el: + return [](){ + return std::make_unique< + orc::LocalIndirectStubsManager<orc::OrcMips64>>(); + }; + + case Triple::x86_64: + if (T.getOS() == Triple::OSType::Win32) { + return [](){ + return std::make_unique< + orc::LocalIndirectStubsManager<orc::OrcX86_64_Win32>>(); + }; + } else { + return [](){ + return std::make_unique< + orc::LocalIndirectStubsManager<orc::OrcX86_64_SysV>>(); + }; + } + + } +} + +Constant* createIRTypedAddress(FunctionType &FT, JITTargetAddress Addr) { + Constant *AddrIntVal = + ConstantInt::get(Type::getInt64Ty(FT.getContext()), Addr); + Constant *AddrPtrVal = + ConstantExpr::getCast(Instruction::IntToPtr, AddrIntVal, + PointerType::get(&FT, 0)); + return AddrPtrVal; +} + +GlobalVariable* createImplPointer(PointerType &PT, Module &M, + const Twine &Name, Constant *Initializer) { + auto IP = new GlobalVariable(M, &PT, false, GlobalValue::ExternalLinkage, + Initializer, Name, nullptr, + GlobalValue::NotThreadLocal, 0, true); + IP->setVisibility(GlobalValue::HiddenVisibility); + return IP; +} + +void makeStub(Function &F, Value &ImplPointer) { + assert(F.isDeclaration() && "Can't turn a definition into a stub."); + assert(F.getParent() && "Function isn't in a module."); + Module &M = *F.getParent(); + BasicBlock *EntryBlock = BasicBlock::Create(M.getContext(), "entry", &F); + IRBuilder<> Builder(EntryBlock); + LoadInst *ImplAddr = Builder.CreateLoad(F.getType(), &ImplPointer); + std::vector<Value*> CallArgs; + for (auto &A : F.args()) + CallArgs.push_back(&A); + CallInst *Call = Builder.CreateCall(F.getFunctionType(), ImplAddr, CallArgs); + Call->setTailCall(); + Call->setAttributes(F.getAttributes()); + if (F.getReturnType()->isVoidTy()) + Builder.CreateRetVoid(); + else + Builder.CreateRet(Call); +} + +std::vector<GlobalValue *> SymbolLinkagePromoter::operator()(Module &M) { + std::vector<GlobalValue *> PromotedGlobals; + + for (auto &GV : M.global_values()) { + bool Promoted = true; + + // Rename if necessary. + if (!GV.hasName()) + GV.setName("__orc_anon." + Twine(NextId++)); + else if (GV.getName().startswith("\01L")) + GV.setName("__" + GV.getName().substr(1) + "." + Twine(NextId++)); + else if (GV.hasLocalLinkage()) + GV.setName("__orc_lcl." + GV.getName() + "." + Twine(NextId++)); + else + Promoted = false; + + if (GV.hasLocalLinkage()) { + GV.setLinkage(GlobalValue::ExternalLinkage); + GV.setVisibility(GlobalValue::HiddenVisibility); + Promoted = true; + } + GV.setUnnamedAddr(GlobalValue::UnnamedAddr::None); + + if (Promoted) + PromotedGlobals.push_back(&GV); + } + + return PromotedGlobals; +} + +Function* cloneFunctionDecl(Module &Dst, const Function &F, + ValueToValueMapTy *VMap) { + Function *NewF = + Function::Create(cast<FunctionType>(F.getValueType()), + F.getLinkage(), F.getName(), &Dst); + NewF->copyAttributesFrom(&F); + + if (VMap) { + (*VMap)[&F] = NewF; + auto NewArgI = NewF->arg_begin(); + for (auto ArgI = F.arg_begin(), ArgE = F.arg_end(); ArgI != ArgE; + ++ArgI, ++NewArgI) + (*VMap)[&*ArgI] = &*NewArgI; + } + + return NewF; +} + +void moveFunctionBody(Function &OrigF, ValueToValueMapTy &VMap, + ValueMaterializer *Materializer, + Function *NewF) { + assert(!OrigF.isDeclaration() && "Nothing to move"); + if (!NewF) + NewF = cast<Function>(VMap[&OrigF]); + else + assert(VMap[&OrigF] == NewF && "Incorrect function mapping in VMap."); + assert(NewF && "Function mapping missing from VMap."); + assert(NewF->getParent() != OrigF.getParent() && + "moveFunctionBody should only be used to move bodies between " + "modules."); + + SmallVector<ReturnInst *, 8> Returns; // Ignore returns cloned. + CloneFunctionInto(NewF, &OrigF, VMap, /*ModuleLevelChanges=*/true, Returns, + "", nullptr, nullptr, Materializer); + OrigF.deleteBody(); +} + +GlobalVariable* cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV, + ValueToValueMapTy *VMap) { + GlobalVariable *NewGV = new GlobalVariable( + Dst, GV.getValueType(), GV.isConstant(), + GV.getLinkage(), nullptr, GV.getName(), nullptr, + GV.getThreadLocalMode(), GV.getType()->getAddressSpace()); + NewGV->copyAttributesFrom(&GV); + if (VMap) + (*VMap)[&GV] = NewGV; + return NewGV; +} + +void moveGlobalVariableInitializer(GlobalVariable &OrigGV, + ValueToValueMapTy &VMap, + ValueMaterializer *Materializer, + GlobalVariable *NewGV) { + assert(OrigGV.hasInitializer() && "Nothing to move"); + if (!NewGV) + NewGV = cast<GlobalVariable>(VMap[&OrigGV]); + else + assert(VMap[&OrigGV] == NewGV && + "Incorrect global variable mapping in VMap."); + assert(NewGV->getParent() != OrigGV.getParent() && + "moveGlobalVariableInitializer should only be used to move " + "initializers between modules"); + + NewGV->setInitializer(MapValue(OrigGV.getInitializer(), VMap, RF_None, + nullptr, Materializer)); +} + +GlobalAlias* cloneGlobalAliasDecl(Module &Dst, const GlobalAlias &OrigA, + ValueToValueMapTy &VMap) { + assert(OrigA.getAliasee() && "Original alias doesn't have an aliasee?"); + auto *NewA = GlobalAlias::create(OrigA.getValueType(), + OrigA.getType()->getPointerAddressSpace(), + OrigA.getLinkage(), OrigA.getName(), &Dst); + NewA->copyAttributesFrom(&OrigA); + VMap[&OrigA] = NewA; + return NewA; +} + +void cloneModuleFlagsMetadata(Module &Dst, const Module &Src, + ValueToValueMapTy &VMap) { + auto *MFs = Src.getModuleFlagsMetadata(); + if (!MFs) + return; + for (auto *MF : MFs->operands()) + Dst.addModuleFlag(MapMetadata(MF, VMap)); +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp new file mode 100644 index 0000000000..8cf66c9e75 --- /dev/null +++ b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp @@ -0,0 +1,141 @@ +//===----- JITTargetMachineBuilder.cpp - Build TargetMachines for JIT -----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h" + +#include "llvm/Support/Host.h" +#include "llvm/Support/TargetRegistry.h" +#include "llvm/Support/raw_ostream.h" + +namespace llvm { +namespace orc { + +JITTargetMachineBuilder::JITTargetMachineBuilder(Triple TT) + : TT(std::move(TT)) { + Options.EmulatedTLS = true; + Options.ExplicitEmulatedTLS = true; +} + +Expected<JITTargetMachineBuilder> JITTargetMachineBuilder::detectHost() { + // FIXME: getProcessTriple is bogus. It returns the host LLVM was compiled on, + // rather than a valid triple for the current process. + JITTargetMachineBuilder TMBuilder((Triple(sys::getProcessTriple()))); + + // Retrieve host CPU name and sub-target features and add them to builder. + // Relocation model, code model and codegen opt level are kept to default + // values. + llvm::StringMap<bool> FeatureMap; + llvm::sys::getHostCPUFeatures(FeatureMap); + for (auto &Feature : FeatureMap) + TMBuilder.getFeatures().AddFeature(Feature.first(), Feature.second); + + TMBuilder.setCPU(std::string(llvm::sys::getHostCPUName())); + + return TMBuilder; +} + +Expected<std::unique_ptr<TargetMachine>> +JITTargetMachineBuilder::createTargetMachine() { + + std::string ErrMsg; + auto *TheTarget = TargetRegistry::lookupTarget(TT.getTriple(), ErrMsg); + if (!TheTarget) + return make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode()); + + auto *TM = + TheTarget->createTargetMachine(TT.getTriple(), CPU, Features.getString(), + Options, RM, CM, OptLevel, /*JIT*/ true); + if (!TM) + return make_error<StringError>("Could not allocate target machine", + inconvertibleErrorCode()); + + return std::unique_ptr<TargetMachine>(TM); +} + +JITTargetMachineBuilder &JITTargetMachineBuilder::addFeatures( + const std::vector<std::string> &FeatureVec) { + for (const auto &F : FeatureVec) + Features.AddFeature(F); + return *this; +} + +#ifndef NDEBUG +raw_ostream &operator<<(raw_ostream &OS, const JITTargetMachineBuilder &JTMB) { + OS << "{ Triple = \"" << JTMB.TT.str() << "\", CPU = \"" << JTMB.CPU + << "\", Options = <not-printable>, Relocation Model = "; + + if (JTMB.RM) { + switch (*JTMB.RM) { + case Reloc::Static: + OS << "Static"; + break; + case Reloc::PIC_: + OS << "PIC_"; + break; + case Reloc::DynamicNoPIC: + OS << "DynamicNoPIC"; + break; + case Reloc::ROPI: + OS << "ROPI"; + break; + case Reloc::RWPI: + OS << "RWPI"; + break; + case Reloc::ROPI_RWPI: + OS << "ROPI_RWPI"; + break; + } + } else + OS << "unspecified"; + + OS << ", Code Model = "; + + if (JTMB.CM) { + switch (*JTMB.CM) { + case CodeModel::Tiny: + OS << "Tiny"; + break; + case CodeModel::Small: + OS << "Small"; + break; + case CodeModel::Kernel: + OS << "Kernel"; + break; + case CodeModel::Medium: + OS << "Medium"; + break; + case CodeModel::Large: + OS << "Large"; + break; + } + } else + OS << "unspecified"; + + OS << ", Optimization Level = "; + switch (JTMB.OptLevel) { + case CodeGenOpt::None: + OS << "None"; + break; + case CodeGenOpt::Less: + OS << "Less"; + break; + case CodeGenOpt::Default: + OS << "Default"; + break; + case CodeGenOpt::Aggressive: + OS << "Aggressive"; + break; + } + + OS << " }"; + return OS; +} +#endif // NDEBUG + +} // End namespace orc. +} // End namespace llvm. diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/Orc/LLJIT.cpp b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/LLJIT.cpp new file mode 100644 index 0000000000..c368c1e371 --- /dev/null +++ b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/LLJIT.cpp @@ -0,0 +1,1230 @@ +//===--------- LLJIT.cpp - An ORC-based JIT for compiling LLVM IR ---------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/LLJIT.h" +#include "llvm/ExecutionEngine/JITLink/EHFrameSupport.h" +#include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h" +#include "llvm/ExecutionEngine/Orc/MachOPlatform.h" +#include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h" +#include "llvm/ExecutionEngine/Orc/ObjectTransformLayer.h" +#include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h" +#include "llvm/ExecutionEngine/Orc/Shared/OrcError.h" +#include "llvm/ExecutionEngine/Orc/TargetProcessControl.h" +#include "llvm/ExecutionEngine/SectionMemoryManager.h" +#include "llvm/IR/GlobalVariable.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/Mangler.h" +#include "llvm/IR/Module.h" +#include "llvm/Support/DynamicLibrary.h" + +#include <map> + +#define DEBUG_TYPE "orc" + +using namespace llvm; +using namespace llvm::orc; + +namespace { + +/// Adds helper function decls and wrapper functions that call the helper with +/// some additional prefix arguments. +/// +/// E.g. For wrapper "foo" with type i8(i8, i64), helper "bar", and prefix +/// args i32 4 and i16 12345, this function will add: +/// +/// declare i8 @bar(i32, i16, i8, i64) +/// +/// define i8 @foo(i8, i64) { +/// entry: +/// %2 = call i8 @bar(i32 4, i16 12345, i8 %0, i64 %1) +/// ret i8 %2 +/// } +/// +Function *addHelperAndWrapper(Module &M, StringRef WrapperName, + FunctionType *WrapperFnType, + GlobalValue::VisibilityTypes WrapperVisibility, + StringRef HelperName, + ArrayRef<Value *> HelperPrefixArgs) { + std::vector<Type *> HelperArgTypes; + for (auto *Arg : HelperPrefixArgs) + HelperArgTypes.push_back(Arg->getType()); + for (auto *T : WrapperFnType->params()) + HelperArgTypes.push_back(T); + auto *HelperFnType = + FunctionType::get(WrapperFnType->getReturnType(), HelperArgTypes, false); + auto *HelperFn = Function::Create(HelperFnType, GlobalValue::ExternalLinkage, + HelperName, M); + + auto *WrapperFn = Function::Create( + WrapperFnType, GlobalValue::ExternalLinkage, WrapperName, M); + WrapperFn->setVisibility(WrapperVisibility); + + auto *EntryBlock = BasicBlock::Create(M.getContext(), "entry", WrapperFn); + IRBuilder<> IB(EntryBlock); + + std::vector<Value *> HelperArgs; + for (auto *Arg : HelperPrefixArgs) + HelperArgs.push_back(Arg); + for (auto &Arg : WrapperFn->args()) + HelperArgs.push_back(&Arg); + auto *HelperResult = IB.CreateCall(HelperFn, HelperArgs); + if (HelperFn->getReturnType()->isVoidTy()) + IB.CreateRetVoid(); + else + IB.CreateRet(HelperResult); + + return WrapperFn; +} + +class GenericLLVMIRPlatformSupport; + +/// orc::Platform component of Generic LLVM IR Platform support. +/// Just forwards calls to the GenericLLVMIRPlatformSupport class below. +class GenericLLVMIRPlatform : public Platform { +public: + GenericLLVMIRPlatform(GenericLLVMIRPlatformSupport &S) : S(S) {} + Error setupJITDylib(JITDylib &JD) override; + Error notifyAdding(ResourceTracker &RT, + const MaterializationUnit &MU) override; + Error notifyRemoving(ResourceTracker &RT) override { + // Noop -- Nothing to do (yet). + return Error::success(); + } + +private: + GenericLLVMIRPlatformSupport &S; +}; + +/// This transform parses llvm.global_ctors to produce a single initialization +/// function for the module, records the function, then deletes +/// llvm.global_ctors. +class GlobalCtorDtorScraper { +public: + + GlobalCtorDtorScraper(GenericLLVMIRPlatformSupport &PS, + StringRef InitFunctionPrefix) + : PS(PS), InitFunctionPrefix(InitFunctionPrefix) {} + Expected<ThreadSafeModule> operator()(ThreadSafeModule TSM, + MaterializationResponsibility &R); + +private: + GenericLLVMIRPlatformSupport &PS; + StringRef InitFunctionPrefix; +}; + +/// Generic IR Platform Support +/// +/// Scrapes llvm.global_ctors and llvm.global_dtors and replaces them with +/// specially named 'init' and 'deinit'. Injects definitions / interposes for +/// some runtime API, including __cxa_atexit, dlopen, and dlclose. +class GenericLLVMIRPlatformSupport : public LLJIT::PlatformSupport { +public: + // GenericLLVMIRPlatform &P) : P(P) { + GenericLLVMIRPlatformSupport(LLJIT &J) + : J(J), InitFunctionPrefix(J.mangle("__orc_init_func.")) { + + getExecutionSession().setPlatform( + std::make_unique<GenericLLVMIRPlatform>(*this)); + + setInitTransform(J, GlobalCtorDtorScraper(*this, InitFunctionPrefix)); + + SymbolMap StdInterposes; + + StdInterposes[J.mangleAndIntern("__lljit.platform_support_instance")] = + JITEvaluatedSymbol(pointerToJITTargetAddress(this), + JITSymbolFlags::Exported); + StdInterposes[J.mangleAndIntern("__lljit.cxa_atexit_helper")] = + JITEvaluatedSymbol(pointerToJITTargetAddress(registerAtExitHelper), + JITSymbolFlags()); + + cantFail( + J.getMainJITDylib().define(absoluteSymbols(std::move(StdInterposes)))); + cantFail(setupJITDylib(J.getMainJITDylib())); + cantFail(J.addIRModule(J.getMainJITDylib(), createPlatformRuntimeModule())); + } + + ExecutionSession &getExecutionSession() { return J.getExecutionSession(); } + + /// Adds a module that defines the __dso_handle global. + Error setupJITDylib(JITDylib &JD) { + + // Add per-jitdylib standard interposes. + SymbolMap PerJDInterposes; + PerJDInterposes[J.mangleAndIntern("__lljit.run_atexits_helper")] = + JITEvaluatedSymbol(pointerToJITTargetAddress(runAtExitsHelper), + JITSymbolFlags()); + cantFail(JD.define(absoluteSymbols(std::move(PerJDInterposes)))); + + auto Ctx = std::make_unique<LLVMContext>(); + auto M = std::make_unique<Module>("__standard_lib", *Ctx); + M->setDataLayout(J.getDataLayout()); + + auto *Int64Ty = Type::getInt64Ty(*Ctx); + auto *DSOHandle = new GlobalVariable( + *M, Int64Ty, true, GlobalValue::ExternalLinkage, + ConstantInt::get(Int64Ty, reinterpret_cast<uintptr_t>(&JD)), + "__dso_handle"); + DSOHandle->setVisibility(GlobalValue::DefaultVisibility); + DSOHandle->setInitializer( + ConstantInt::get(Int64Ty, pointerToJITTargetAddress(&JD))); + + auto *GenericIRPlatformSupportTy = + StructType::create(*Ctx, "lljit.GenericLLJITIRPlatformSupport"); + + auto *PlatformInstanceDecl = new GlobalVariable( + *M, GenericIRPlatformSupportTy, true, GlobalValue::ExternalLinkage, + nullptr, "__lljit.platform_support_instance"); + + auto *VoidTy = Type::getVoidTy(*Ctx); + addHelperAndWrapper( + *M, "__lljit_run_atexits", FunctionType::get(VoidTy, {}, false), + GlobalValue::HiddenVisibility, "__lljit.run_atexits_helper", + {PlatformInstanceDecl, DSOHandle}); + + return J.addIRModule(JD, ThreadSafeModule(std::move(M), std::move(Ctx))); + } + + Error notifyAdding(ResourceTracker &RT, const MaterializationUnit &MU) { + auto &JD = RT.getJITDylib(); + if (auto &InitSym = MU.getInitializerSymbol()) + InitSymbols[&JD].add(InitSym, SymbolLookupFlags::WeaklyReferencedSymbol); + else { + // If there's no identified init symbol attached, but there is a symbol + // with the GenericIRPlatform::InitFunctionPrefix, then treat that as + // an init function. Add the symbol to both the InitSymbols map (which + // will trigger a lookup to materialize the module) and the InitFunctions + // map (which holds the names of the symbols to execute). + for (auto &KV : MU.getSymbols()) + if ((*KV.first).startswith(InitFunctionPrefix)) { + InitSymbols[&JD].add(KV.first, + SymbolLookupFlags::WeaklyReferencedSymbol); + InitFunctions[&JD].add(KV.first); + } + } + return Error::success(); + } + + Error initialize(JITDylib &JD) override { + LLVM_DEBUG({ + dbgs() << "GenericLLVMIRPlatformSupport getting initializers to run\n"; + }); + if (auto Initializers = getInitializers(JD)) { + LLVM_DEBUG( + { dbgs() << "GenericLLVMIRPlatformSupport running initializers\n"; }); + for (auto InitFnAddr : *Initializers) { + LLVM_DEBUG({ + dbgs() << " Running init " << formatv("{0:x16}", InitFnAddr) + << "...\n"; + }); + auto *InitFn = jitTargetAddressToFunction<void (*)()>(InitFnAddr); + InitFn(); + } + } else + return Initializers.takeError(); + return Error::success(); + } + + Error deinitialize(JITDylib &JD) override { + LLVM_DEBUG({ + dbgs() << "GenericLLVMIRPlatformSupport getting deinitializers to run\n"; + }); + if (auto Deinitializers = getDeinitializers(JD)) { + LLVM_DEBUG({ + dbgs() << "GenericLLVMIRPlatformSupport running deinitializers\n"; + }); + for (auto DeinitFnAddr : *Deinitializers) { + LLVM_DEBUG({ + dbgs() << " Running deinit " << formatv("{0:x16}", DeinitFnAddr) + << "...\n"; + }); + auto *DeinitFn = jitTargetAddressToFunction<void (*)()>(DeinitFnAddr); + DeinitFn(); + } + } else + return Deinitializers.takeError(); + + return Error::success(); + } + + void registerInitFunc(JITDylib &JD, SymbolStringPtr InitName) { + getExecutionSession().runSessionLocked([&]() { + InitFunctions[&JD].add(InitName); + }); + } + +private: + + Expected<std::vector<JITTargetAddress>> getInitializers(JITDylib &JD) { + if (auto Err = issueInitLookups(JD)) + return std::move(Err); + + DenseMap<JITDylib *, SymbolLookupSet> LookupSymbols; + std::vector<JITDylibSP> DFSLinkOrder; + + getExecutionSession().runSessionLocked([&]() { + DFSLinkOrder = JD.getDFSLinkOrder(); + + for (auto &NextJD : DFSLinkOrder) { + auto IFItr = InitFunctions.find(NextJD.get()); + if (IFItr != InitFunctions.end()) { + LookupSymbols[NextJD.get()] = std::move(IFItr->second); + InitFunctions.erase(IFItr); + } + } + }); + + LLVM_DEBUG({ + dbgs() << "JITDylib init order is [ "; + for (auto &JD : llvm::reverse(DFSLinkOrder)) + dbgs() << "\"" << JD->getName() << "\" "; + dbgs() << "]\n"; + dbgs() << "Looking up init functions:\n"; + for (auto &KV : LookupSymbols) + dbgs() << " \"" << KV.first->getName() << "\": " << KV.second << "\n"; + }); + + auto &ES = getExecutionSession(); + auto LookupResult = Platform::lookupInitSymbols(ES, LookupSymbols); + + if (!LookupResult) + return LookupResult.takeError(); + + std::vector<JITTargetAddress> Initializers; + while (!DFSLinkOrder.empty()) { + auto &NextJD = *DFSLinkOrder.back(); + DFSLinkOrder.pop_back(); + auto InitsItr = LookupResult->find(&NextJD); + if (InitsItr == LookupResult->end()) + continue; + for (auto &KV : InitsItr->second) + Initializers.push_back(KV.second.getAddress()); + } + + return Initializers; + } + + Expected<std::vector<JITTargetAddress>> getDeinitializers(JITDylib &JD) { + auto &ES = getExecutionSession(); + + auto LLJITRunAtExits = J.mangleAndIntern("__lljit_run_atexits"); + + DenseMap<JITDylib *, SymbolLookupSet> LookupSymbols; + std::vector<JITDylibSP> DFSLinkOrder; + + ES.runSessionLocked([&]() { + DFSLinkOrder = JD.getDFSLinkOrder(); + + for (auto &NextJD : DFSLinkOrder) { + auto &JDLookupSymbols = LookupSymbols[NextJD.get()]; + auto DIFItr = DeInitFunctions.find(NextJD.get()); + if (DIFItr != DeInitFunctions.end()) { + LookupSymbols[NextJD.get()] = std::move(DIFItr->second); + DeInitFunctions.erase(DIFItr); + } + JDLookupSymbols.add(LLJITRunAtExits, + SymbolLookupFlags::WeaklyReferencedSymbol); + } + }); + + LLVM_DEBUG({ + dbgs() << "JITDylib deinit order is [ "; + for (auto &JD : DFSLinkOrder) + dbgs() << "\"" << JD->getName() << "\" "; + dbgs() << "]\n"; + dbgs() << "Looking up deinit functions:\n"; + for (auto &KV : LookupSymbols) + dbgs() << " \"" << KV.first->getName() << "\": " << KV.second << "\n"; + }); + + auto LookupResult = Platform::lookupInitSymbols(ES, LookupSymbols); + + if (!LookupResult) + return LookupResult.takeError(); + + std::vector<JITTargetAddress> DeInitializers; + for (auto &NextJD : DFSLinkOrder) { + auto DeInitsItr = LookupResult->find(NextJD.get()); + assert(DeInitsItr != LookupResult->end() && + "Every JD should have at least __lljit_run_atexits"); + + auto RunAtExitsItr = DeInitsItr->second.find(LLJITRunAtExits); + if (RunAtExitsItr != DeInitsItr->second.end()) + DeInitializers.push_back(RunAtExitsItr->second.getAddress()); + + for (auto &KV : DeInitsItr->second) + if (KV.first != LLJITRunAtExits) + DeInitializers.push_back(KV.second.getAddress()); + } + + return DeInitializers; + } + + /// Issue lookups for all init symbols required to initialize JD (and any + /// JITDylibs that it depends on). + Error issueInitLookups(JITDylib &JD) { + DenseMap<JITDylib *, SymbolLookupSet> RequiredInitSymbols; + std::vector<JITDylibSP> DFSLinkOrder; + + getExecutionSession().runSessionLocked([&]() { + DFSLinkOrder = JD.getDFSLinkOrder(); + + for (auto &NextJD : DFSLinkOrder) { + auto ISItr = InitSymbols.find(NextJD.get()); + if (ISItr != InitSymbols.end()) { + RequiredInitSymbols[NextJD.get()] = std::move(ISItr->second); + InitSymbols.erase(ISItr); + } + } + }); + + return Platform::lookupInitSymbols(getExecutionSession(), + RequiredInitSymbols) + .takeError(); + } + + static void registerAtExitHelper(void *Self, void (*F)(void *), void *Ctx, + void *DSOHandle) { + LLVM_DEBUG({ + dbgs() << "Registering atexit function " << (void *)F << " for JD " + << (*static_cast<JITDylib **>(DSOHandle))->getName() << "\n"; + }); + static_cast<GenericLLVMIRPlatformSupport *>(Self)->AtExitMgr.registerAtExit( + F, Ctx, DSOHandle); + } + + static void runAtExitsHelper(void *Self, void *DSOHandle) { + LLVM_DEBUG({ + dbgs() << "Running atexit functions for JD " + << (*static_cast<JITDylib **>(DSOHandle))->getName() << "\n"; + }); + static_cast<GenericLLVMIRPlatformSupport *>(Self)->AtExitMgr.runAtExits( + DSOHandle); + } + + // Constructs an LLVM IR module containing platform runtime globals, + // functions, and interposes. + ThreadSafeModule createPlatformRuntimeModule() { + auto Ctx = std::make_unique<LLVMContext>(); + auto M = std::make_unique<Module>("__standard_lib", *Ctx); + M->setDataLayout(J.getDataLayout()); + + auto *GenericIRPlatformSupportTy = + StructType::create(*Ctx, "lljit.GenericLLJITIRPlatformSupport"); + + auto *PlatformInstanceDecl = new GlobalVariable( + *M, GenericIRPlatformSupportTy, true, GlobalValue::ExternalLinkage, + nullptr, "__lljit.platform_support_instance"); + + auto *Int8Ty = Type::getInt8Ty(*Ctx); + auto *IntTy = Type::getIntNTy(*Ctx, sizeof(int) * CHAR_BIT); + auto *VoidTy = Type::getVoidTy(*Ctx); + auto *BytePtrTy = PointerType::getUnqual(Int8Ty); + auto *AtExitCallbackTy = FunctionType::get(VoidTy, {BytePtrTy}, false); + auto *AtExitCallbackPtrTy = PointerType::getUnqual(AtExitCallbackTy); + + addHelperAndWrapper( + *M, "__cxa_atexit", + FunctionType::get(IntTy, {AtExitCallbackPtrTy, BytePtrTy, BytePtrTy}, + false), + GlobalValue::DefaultVisibility, "__lljit.cxa_atexit_helper", + {PlatformInstanceDecl}); + + return ThreadSafeModule(std::move(M), std::move(Ctx)); + } + + LLJIT &J; + std::string InitFunctionPrefix; + DenseMap<JITDylib *, SymbolLookupSet> InitSymbols; + DenseMap<JITDylib *, SymbolLookupSet> InitFunctions; + DenseMap<JITDylib *, SymbolLookupSet> DeInitFunctions; + ItaniumCXAAtExitSupport AtExitMgr; +}; + +Error GenericLLVMIRPlatform::setupJITDylib(JITDylib &JD) { + return S.setupJITDylib(JD); +} + +Error GenericLLVMIRPlatform::notifyAdding(ResourceTracker &RT, + const MaterializationUnit &MU) { + return S.notifyAdding(RT, MU); +} + +Expected<ThreadSafeModule> +GlobalCtorDtorScraper::operator()(ThreadSafeModule TSM, + MaterializationResponsibility &R) { + auto Err = TSM.withModuleDo([&](Module &M) -> Error { + auto &Ctx = M.getContext(); + auto *GlobalCtors = M.getNamedGlobal("llvm.global_ctors"); + + // If there's no llvm.global_ctors or it's just a decl then skip. + if (!GlobalCtors || GlobalCtors->isDeclaration()) + return Error::success(); + + std::string InitFunctionName; + raw_string_ostream(InitFunctionName) + << InitFunctionPrefix << M.getModuleIdentifier(); + + MangleAndInterner Mangle(PS.getExecutionSession(), M.getDataLayout()); + auto InternedName = Mangle(InitFunctionName); + if (auto Err = + R.defineMaterializing({{InternedName, JITSymbolFlags::Callable}})) + return Err; + + auto *InitFunc = + Function::Create(FunctionType::get(Type::getVoidTy(Ctx), {}, false), + GlobalValue::ExternalLinkage, InitFunctionName, &M); + InitFunc->setVisibility(GlobalValue::HiddenVisibility); + std::vector<std::pair<Function *, unsigned>> Inits; + for (auto E : getConstructors(M)) + Inits.push_back(std::make_pair(E.Func, E.Priority)); + llvm::sort(Inits, [](const std::pair<Function *, unsigned> &LHS, + const std::pair<Function *, unsigned> &RHS) { + return LHS.first < RHS.first; + }); + auto *EntryBlock = BasicBlock::Create(Ctx, "entry", InitFunc); + IRBuilder<> IB(EntryBlock); + for (auto &KV : Inits) + IB.CreateCall(KV.first); + IB.CreateRetVoid(); + + PS.registerInitFunc(R.getTargetJITDylib(), InternedName); + GlobalCtors->eraseFromParent(); + return Error::success(); + }); + + if (Err) + return std::move(Err); + + return std::move(TSM); +} + +class MachOPlatformSupport : public LLJIT::PlatformSupport { +public: + using DLOpenType = void *(*)(const char *Name, int Mode); + using DLCloseType = int (*)(void *Handle); + using DLSymType = void *(*)(void *Handle, const char *Name); + using DLErrorType = const char *(*)(); + + struct DlFcnValues { + Optional<void *> RTLDDefault; + DLOpenType dlopen = nullptr; + DLCloseType dlclose = nullptr; + DLSymType dlsym = nullptr; + DLErrorType dlerror = nullptr; + }; + + static Expected<std::unique_ptr<MachOPlatformSupport>> + Create(LLJIT &J, JITDylib &PlatformJITDylib) { + + // Make process symbols visible. + { + std::string ErrMsg; + auto Lib = sys::DynamicLibrary::getPermanentLibrary(nullptr, &ErrMsg); + if (!Lib.isValid()) + return make_error<StringError>(std::move(ErrMsg), + inconvertibleErrorCode()); + } + + DlFcnValues DlFcn; + + // Add support for RTLDDefault on known platforms. +#ifdef __APPLE__ + DlFcn.RTLDDefault = reinterpret_cast<void *>(-2); +#endif // __APPLE__ + + if (auto Err = hookUpFunction(DlFcn.dlopen, "dlopen")) + return std::move(Err); + if (auto Err = hookUpFunction(DlFcn.dlclose, "dlclose")) + return std::move(Err); + if (auto Err = hookUpFunction(DlFcn.dlsym, "dlsym")) + return std::move(Err); + if (auto Err = hookUpFunction(DlFcn.dlerror, "dlerror")) + return std::move(Err); + + std::unique_ptr<MachOPlatformSupport> MP( + new MachOPlatformSupport(J, PlatformJITDylib, DlFcn)); + return std::move(MP); + } + + Error initialize(JITDylib &JD) override { + LLVM_DEBUG({ + dbgs() << "MachOPlatformSupport initializing \"" << JD.getName() + << "\"\n"; + }); + + auto InitSeq = MP.getInitializerSequence(JD); + if (!InitSeq) + return InitSeq.takeError(); + + // If ObjC is not enabled but there are JIT'd ObjC inits then return + // an error. + if (!objCRegistrationEnabled()) + for (auto &KV : *InitSeq) { + if (!KV.second.getObjCSelRefsSections().empty() || + !KV.second.getObjCClassListSections().empty()) + return make_error<StringError>("JITDylib " + KV.first->getName() + + " contains objc metadata but objc" + " is not enabled", + inconvertibleErrorCode()); + } + + // Run the initializers. + for (auto &KV : *InitSeq) { + if (objCRegistrationEnabled()) { + KV.second.registerObjCSelectors(); + if (auto Err = KV.second.registerObjCClasses()) { + // FIXME: Roll back registrations on error? + return Err; + } + } + KV.second.runModInits(); + } + + return Error::success(); + } + + Error deinitialize(JITDylib &JD) override { + auto &ES = J.getExecutionSession(); + if (auto DeinitSeq = MP.getDeinitializerSequence(JD)) { + for (auto &KV : *DeinitSeq) { + auto DSOHandleName = ES.intern("___dso_handle"); + + // FIXME: Run DeInits here. + auto Result = ES.lookup( + {{KV.first, JITDylibLookupFlags::MatchAllSymbols}}, + SymbolLookupSet(DSOHandleName, + SymbolLookupFlags::WeaklyReferencedSymbol)); + if (!Result) + return Result.takeError(); + if (Result->empty()) + continue; + assert(Result->count(DSOHandleName) && + "Result does not contain __dso_handle"); + auto *DSOHandle = jitTargetAddressToPointer<void *>( + Result->begin()->second.getAddress()); + AtExitMgr.runAtExits(DSOHandle); + } + } else + return DeinitSeq.takeError(); + return Error::success(); + } + +private: + template <typename FunctionPtrTy> + static Error hookUpFunction(FunctionPtrTy &Fn, const char *Name) { + if (auto *FnAddr = sys::DynamicLibrary::SearchForAddressOfSymbol(Name)) { + Fn = reinterpret_cast<FunctionPtrTy>(Fn); + return Error::success(); + } + + return make_error<StringError>((Twine("Can not enable MachO JIT Platform: " + "missing function: ") + + Name) + .str(), + inconvertibleErrorCode()); + } + + MachOPlatformSupport(LLJIT &J, JITDylib &PlatformJITDylib, DlFcnValues DlFcn) + : J(J), MP(setupPlatform(J)), DlFcn(std::move(DlFcn)) { + + SymbolMap HelperSymbols; + + // platform and atexit helpers. + HelperSymbols[J.mangleAndIntern("__lljit.platform_support_instance")] = + JITEvaluatedSymbol(pointerToJITTargetAddress(this), JITSymbolFlags()); + HelperSymbols[J.mangleAndIntern("__lljit.cxa_atexit_helper")] = + JITEvaluatedSymbol(pointerToJITTargetAddress(registerAtExitHelper), + JITSymbolFlags()); + HelperSymbols[J.mangleAndIntern("__lljit.run_atexits_helper")] = + JITEvaluatedSymbol(pointerToJITTargetAddress(runAtExitsHelper), + JITSymbolFlags()); + + // dlfcn helpers. + HelperSymbols[J.mangleAndIntern("__lljit.dlopen_helper")] = + JITEvaluatedSymbol(pointerToJITTargetAddress(dlopenHelper), + JITSymbolFlags()); + HelperSymbols[J.mangleAndIntern("__lljit.dlclose_helper")] = + JITEvaluatedSymbol(pointerToJITTargetAddress(dlcloseHelper), + JITSymbolFlags()); + HelperSymbols[J.mangleAndIntern("__lljit.dlsym_helper")] = + JITEvaluatedSymbol(pointerToJITTargetAddress(dlsymHelper), + JITSymbolFlags()); + HelperSymbols[J.mangleAndIntern("__lljit.dlerror_helper")] = + JITEvaluatedSymbol(pointerToJITTargetAddress(dlerrorHelper), + JITSymbolFlags()); + + cantFail( + PlatformJITDylib.define(absoluteSymbols(std::move(HelperSymbols)))); + cantFail(MP.setupJITDylib(J.getMainJITDylib())); + cantFail(J.addIRModule(PlatformJITDylib, createPlatformRuntimeModule())); + } + + static MachOPlatform &setupPlatform(LLJIT &J) { + auto Tmp = std::make_unique<MachOPlatform>( + J.getExecutionSession(), + static_cast<ObjectLinkingLayer &>(J.getObjLinkingLayer()), + createStandardSymbolsObject(J)); + auto &MP = *Tmp; + J.getExecutionSession().setPlatform(std::move(Tmp)); + return MP; + } + + static std::unique_ptr<MemoryBuffer> createStandardSymbolsObject(LLJIT &J) { + LLVMContext Ctx; + Module M("__standard_symbols", Ctx); + M.setDataLayout(J.getDataLayout()); + + auto *Int64Ty = Type::getInt64Ty(Ctx); + + auto *DSOHandle = + new GlobalVariable(M, Int64Ty, true, GlobalValue::ExternalLinkage, + ConstantInt::get(Int64Ty, 0), "__dso_handle"); + DSOHandle->setVisibility(GlobalValue::DefaultVisibility); + + return cantFail(J.getIRCompileLayer().getCompiler()(M)); + } + + ThreadSafeModule createPlatformRuntimeModule() { + auto Ctx = std::make_unique<LLVMContext>(); + auto M = std::make_unique<Module>("__standard_lib", *Ctx); + M->setDataLayout(J.getDataLayout()); + + auto *MachOPlatformSupportTy = + StructType::create(*Ctx, "lljit.MachOPlatformSupport"); + + auto *PlatformInstanceDecl = new GlobalVariable( + *M, MachOPlatformSupportTy, true, GlobalValue::ExternalLinkage, nullptr, + "__lljit.platform_support_instance"); + + auto *Int8Ty = Type::getInt8Ty(*Ctx); + auto *IntTy = Type::getIntNTy(*Ctx, sizeof(int) * CHAR_BIT); + auto *VoidTy = Type::getVoidTy(*Ctx); + auto *BytePtrTy = PointerType::getUnqual(Int8Ty); + auto *AtExitCallbackTy = FunctionType::get(VoidTy, {BytePtrTy}, false); + auto *AtExitCallbackPtrTy = PointerType::getUnqual(AtExitCallbackTy); + + addHelperAndWrapper( + *M, "__cxa_atexit", + FunctionType::get(IntTy, {AtExitCallbackPtrTy, BytePtrTy, BytePtrTy}, + false), + GlobalValue::DefaultVisibility, "__lljit.cxa_atexit_helper", + {PlatformInstanceDecl}); + + addHelperAndWrapper(*M, "dlopen", + FunctionType::get(BytePtrTy, {BytePtrTy, IntTy}, false), + GlobalValue::DefaultVisibility, "__lljit.dlopen_helper", + {PlatformInstanceDecl}); + + addHelperAndWrapper(*M, "dlclose", + FunctionType::get(IntTy, {BytePtrTy}, false), + GlobalValue::DefaultVisibility, + "__lljit.dlclose_helper", {PlatformInstanceDecl}); + + addHelperAndWrapper( + *M, "dlsym", + FunctionType::get(BytePtrTy, {BytePtrTy, BytePtrTy}, false), + GlobalValue::DefaultVisibility, "__lljit.dlsym_helper", + {PlatformInstanceDecl}); + + addHelperAndWrapper(*M, "dlerror", FunctionType::get(BytePtrTy, {}, false), + GlobalValue::DefaultVisibility, + "__lljit.dlerror_helper", {PlatformInstanceDecl}); + + return ThreadSafeModule(std::move(M), std::move(Ctx)); + } + + static void registerAtExitHelper(void *Self, void (*F)(void *), void *Ctx, + void *DSOHandle) { + static_cast<MachOPlatformSupport *>(Self)->AtExitMgr.registerAtExit( + F, Ctx, DSOHandle); + } + + static void runAtExitsHelper(void *Self, void *DSOHandle) { + static_cast<MachOPlatformSupport *>(Self)->AtExitMgr.runAtExits(DSOHandle); + } + + void *jit_dlopen(const char *Path, int Mode) { + JITDylib *JDToOpen = nullptr; + // FIXME: Do the right thing with Mode flags. + { + std::lock_guard<std::mutex> Lock(PlatformSupportMutex); + + // Clear any existing error messages. + dlErrorMsgs.erase(std::this_thread::get_id()); + + if (auto *JD = J.getExecutionSession().getJITDylibByName(Path)) { + auto I = JDRefCounts.find(JD); + if (I != JDRefCounts.end()) { + ++I->second; + return JD; + } + + JDRefCounts[JD] = 1; + JDToOpen = JD; + } + } + + if (JDToOpen) { + if (auto Err = initialize(*JDToOpen)) { + recordError(std::move(Err)); + return 0; + } + } + + // Fall through to dlopen if no JITDylib found for Path. + return DlFcn.dlopen(Path, Mode); + } + + static void *dlopenHelper(void *Self, const char *Path, int Mode) { + return static_cast<MachOPlatformSupport *>(Self)->jit_dlopen(Path, Mode); + } + + int jit_dlclose(void *Handle) { + JITDylib *JDToClose = nullptr; + + { + std::lock_guard<std::mutex> Lock(PlatformSupportMutex); + + // Clear any existing error messages. + dlErrorMsgs.erase(std::this_thread::get_id()); + + auto I = JDRefCounts.find(Handle); + if (I != JDRefCounts.end()) { + --I->second; + if (I->second == 0) { + JDRefCounts.erase(I); + JDToClose = static_cast<JITDylib *>(Handle); + } else + return 0; + } + } + + if (JDToClose) { + if (auto Err = deinitialize(*JDToClose)) { + recordError(std::move(Err)); + return -1; + } + return 0; + } + + // Fall through to dlclose if no JITDylib found for Path. + return DlFcn.dlclose(Handle); + } + + static int dlcloseHelper(void *Self, void *Handle) { + return static_cast<MachOPlatformSupport *>(Self)->jit_dlclose(Handle); + } + + void *jit_dlsym(void *Handle, const char *Name) { + JITDylibSearchOrder JITSymSearchOrder; + + // FIXME: RTLD_NEXT, RTLD_SELF not supported. + { + std::lock_guard<std::mutex> Lock(PlatformSupportMutex); + + // Clear any existing error messages. + dlErrorMsgs.erase(std::this_thread::get_id()); + + if (JDRefCounts.count(Handle)) { + JITSymSearchOrder.push_back( + {static_cast<JITDylib *>(Handle), + JITDylibLookupFlags::MatchExportedSymbolsOnly}); + } else if (Handle == DlFcn.RTLDDefault) { + for (auto &KV : JDRefCounts) + JITSymSearchOrder.push_back( + {static_cast<JITDylib *>(KV.first), + JITDylibLookupFlags::MatchExportedSymbolsOnly}); + } + } + + if (!JITSymSearchOrder.empty()) { + auto MangledName = J.mangleAndIntern(Name); + SymbolLookupSet Syms(MangledName, + SymbolLookupFlags::WeaklyReferencedSymbol); + if (auto Result = J.getExecutionSession().lookup(JITSymSearchOrder, Syms, + LookupKind::DLSym)) { + auto I = Result->find(MangledName); + if (I != Result->end()) + return jitTargetAddressToPointer<void *>(I->second.getAddress()); + } else { + recordError(Result.takeError()); + return 0; + } + } + + // Fall through to dlsym. + return DlFcn.dlsym(Handle, Name); + } + + static void *dlsymHelper(void *Self, void *Handle, const char *Name) { + return static_cast<MachOPlatformSupport *>(Self)->jit_dlsym(Handle, Name); + } + + const char *jit_dlerror() { + { + std::lock_guard<std::mutex> Lock(PlatformSupportMutex); + auto I = dlErrorMsgs.find(std::this_thread::get_id()); + if (I != dlErrorMsgs.end()) + return I->second->c_str(); + } + return DlFcn.dlerror(); + } + + static const char *dlerrorHelper(void *Self) { + return static_cast<MachOPlatformSupport *>(Self)->jit_dlerror(); + } + + void recordError(Error Err) { + std::lock_guard<std::mutex> Lock(PlatformSupportMutex); + dlErrorMsgs[std::this_thread::get_id()] = + std::make_unique<std::string>(toString(std::move(Err))); + } + + std::mutex PlatformSupportMutex; + LLJIT &J; + MachOPlatform &MP; + DlFcnValues DlFcn; + ItaniumCXAAtExitSupport AtExitMgr; + DenseMap<void *, unsigned> JDRefCounts; + std::map<std::thread::id, std::unique_ptr<std::string>> dlErrorMsgs; +}; + +} // end anonymous namespace + +namespace llvm { +namespace orc { + +void LLJIT::PlatformSupport::setInitTransform( + LLJIT &J, IRTransformLayer::TransformFunction T) { + J.InitHelperTransformLayer->setTransform(std::move(T)); +} + +LLJIT::PlatformSupport::~PlatformSupport() {} + +Error LLJITBuilderState::prepareForConstruction() { + + LLVM_DEBUG(dbgs() << "Preparing to create LLJIT instance...\n"); + + if (!JTMB) { + LLVM_DEBUG({ + dbgs() << " No explicitly set JITTargetMachineBuilder. " + "Detecting host...\n"; + }); + if (auto JTMBOrErr = JITTargetMachineBuilder::detectHost()) + JTMB = std::move(*JTMBOrErr); + else + return JTMBOrErr.takeError(); + } + + LLVM_DEBUG({ + dbgs() << " JITTargetMachineBuilder is " << JTMB << "\n" + << " Pre-constructed ExecutionSession: " << (ES ? "Yes" : "No") + << "\n" + << " DataLayout: "; + if (DL) + dbgs() << DL->getStringRepresentation() << "\n"; + else + dbgs() << "None (will be created by JITTargetMachineBuilder)\n"; + + dbgs() << " Custom object-linking-layer creator: " + << (CreateObjectLinkingLayer ? "Yes" : "No") << "\n" + << " Custom compile-function creator: " + << (CreateCompileFunction ? "Yes" : "No") << "\n" + << " Custom platform-setup function: " + << (SetUpPlatform ? "Yes" : "No") << "\n" + << " Number of compile threads: " << NumCompileThreads; + if (!NumCompileThreads) + dbgs() << " (code will be compiled on the execution thread)\n"; + else + dbgs() << "\n"; + }); + + // If the client didn't configure any linker options then auto-configure the + // JIT linker. + if (!CreateObjectLinkingLayer) { + auto &TT = JTMB->getTargetTriple(); + if (TT.isOSBinFormatMachO() && + (TT.getArch() == Triple::aarch64 || TT.getArch() == Triple::x86_64)) { + + JTMB->setRelocationModel(Reloc::PIC_); + JTMB->setCodeModel(CodeModel::Small); + CreateObjectLinkingLayer = + [TPC = this->TPC]( + ExecutionSession &ES, + const Triple &) -> Expected<std::unique_ptr<ObjectLayer>> { + std::unique_ptr<ObjectLinkingLayer> ObjLinkingLayer; + if (TPC) + ObjLinkingLayer = + std::make_unique<ObjectLinkingLayer>(ES, TPC->getMemMgr()); + else + ObjLinkingLayer = std::make_unique<ObjectLinkingLayer>( + ES, std::make_unique<jitlink::InProcessMemoryManager>()); + ObjLinkingLayer->addPlugin(std::make_unique<EHFrameRegistrationPlugin>( + ES, std::make_unique<jitlink::InProcessEHFrameRegistrar>())); + return std::move(ObjLinkingLayer); + }; + } + } + + return Error::success(); +} + +LLJIT::~LLJIT() { + if (CompileThreads) + CompileThreads->wait(); + if (auto Err = ES->endSession()) + ES->reportError(std::move(Err)); +} + +Error LLJIT::addIRModule(ResourceTrackerSP RT, ThreadSafeModule TSM) { + assert(TSM && "Can not add null module"); + + if (auto Err = + TSM.withModuleDo([&](Module &M) { return applyDataLayout(M); })) + return Err; + + return InitHelperTransformLayer->add(std::move(RT), std::move(TSM)); +} + +Error LLJIT::addIRModule(JITDylib &JD, ThreadSafeModule TSM) { + return addIRModule(JD.getDefaultResourceTracker(), std::move(TSM)); +} + +Error LLJIT::addObjectFile(ResourceTrackerSP RT, + std::unique_ptr<MemoryBuffer> Obj) { + assert(Obj && "Can not add null object"); + + return ObjTransformLayer->add(std::move(RT), std::move(Obj)); +} + +Error LLJIT::addObjectFile(JITDylib &JD, std::unique_ptr<MemoryBuffer> Obj) { + return addObjectFile(JD.getDefaultResourceTracker(), std::move(Obj)); +} + +Expected<JITEvaluatedSymbol> LLJIT::lookupLinkerMangled(JITDylib &JD, + SymbolStringPtr Name) { + return ES->lookup( + makeJITDylibSearchOrder(&JD, JITDylibLookupFlags::MatchAllSymbols), Name); +} + +Expected<std::unique_ptr<ObjectLayer>> +LLJIT::createObjectLinkingLayer(LLJITBuilderState &S, ExecutionSession &ES) { + + // If the config state provided an ObjectLinkingLayer factory then use it. + if (S.CreateObjectLinkingLayer) + return S.CreateObjectLinkingLayer(ES, S.JTMB->getTargetTriple()); + + // Otherwise default to creating an RTDyldObjectLinkingLayer that constructs + // a new SectionMemoryManager for each object. + auto GetMemMgr = []() { return std::make_unique<SectionMemoryManager>(); }; + auto ObjLinkingLayer = + std::make_unique<RTDyldObjectLinkingLayer>(ES, std::move(GetMemMgr)); + + if (S.JTMB->getTargetTriple().isOSBinFormatCOFF()) { + ObjLinkingLayer->setOverrideObjectFlagsWithResponsibilityFlags(true); + ObjLinkingLayer->setAutoClaimResponsibilityForObjectSymbols(true); + } + + // FIXME: Explicit conversion to std::unique_ptr<ObjectLayer> added to silence + // errors from some GCC / libstdc++ bots. Remove this conversion (i.e. + // just return ObjLinkingLayer) once those bots are upgraded. + return std::unique_ptr<ObjectLayer>(std::move(ObjLinkingLayer)); +} + +Expected<std::unique_ptr<IRCompileLayer::IRCompiler>> +LLJIT::createCompileFunction(LLJITBuilderState &S, + JITTargetMachineBuilder JTMB) { + + /// If there is a custom compile function creator set then use it. + if (S.CreateCompileFunction) + return S.CreateCompileFunction(std::move(JTMB)); + + // Otherwise default to creating a SimpleCompiler, or ConcurrentIRCompiler, + // depending on the number of threads requested. + if (S.NumCompileThreads > 0) + return std::make_unique<ConcurrentIRCompiler>(std::move(JTMB)); + + auto TM = JTMB.createTargetMachine(); + if (!TM) + return TM.takeError(); + + return std::make_unique<TMOwningSimpleCompiler>(std::move(*TM)); +} + +LLJIT::LLJIT(LLJITBuilderState &S, Error &Err) + : ES(S.ES ? std::move(S.ES) : std::make_unique<ExecutionSession>()), Main(), + DL(""), TT(S.JTMB->getTargetTriple()) { + + ErrorAsOutParameter _(&Err); + + if (auto MainOrErr = this->ES->createJITDylib("main")) + Main = &*MainOrErr; + else { + Err = MainOrErr.takeError(); + return; + } + + if (S.DL) + DL = std::move(*S.DL); + else if (auto DLOrErr = S.JTMB->getDefaultDataLayoutForTarget()) + DL = std::move(*DLOrErr); + else { + Err = DLOrErr.takeError(); + return; + } + + auto ObjLayer = createObjectLinkingLayer(S, *ES); + if (!ObjLayer) { + Err = ObjLayer.takeError(); + return; + } + ObjLinkingLayer = std::move(*ObjLayer); + ObjTransformLayer = + std::make_unique<ObjectTransformLayer>(*ES, *ObjLinkingLayer); + + { + auto CompileFunction = createCompileFunction(S, std::move(*S.JTMB)); + if (!CompileFunction) { + Err = CompileFunction.takeError(); + return; + } + CompileLayer = std::make_unique<IRCompileLayer>( + *ES, *ObjTransformLayer, std::move(*CompileFunction)); + TransformLayer = std::make_unique<IRTransformLayer>(*ES, *CompileLayer); + InitHelperTransformLayer = + std::make_unique<IRTransformLayer>(*ES, *TransformLayer); + } + + if (S.NumCompileThreads > 0) { + InitHelperTransformLayer->setCloneToNewContextOnEmit(true); + CompileThreads = + std::make_unique<ThreadPool>(hardware_concurrency(S.NumCompileThreads)); + ES->setDispatchMaterialization( + [this](std::unique_ptr<MaterializationUnit> MU, + std::unique_ptr<MaterializationResponsibility> MR) { + // FIXME: We should be able to use move-capture here, but ThreadPool's + // AsyncTaskTys are std::functions rather than unique_functions + // (because MSVC's std::packaged_tasks don't support move-only types). + // Fix this when all the above gets sorted out. + CompileThreads->async( + [UnownedMU = MU.release(), UnownedMR = MR.release()]() mutable { + std::unique_ptr<MaterializationUnit> MU(UnownedMU); + std::unique_ptr<MaterializationResponsibility> MR(UnownedMR); + MU->materialize(std::move(MR)); + }); + }); + } + + if (S.SetUpPlatform) + Err = S.SetUpPlatform(*this); + else + setUpGenericLLVMIRPlatform(*this); +} + +std::string LLJIT::mangle(StringRef UnmangledName) const { + std::string MangledName; + { + raw_string_ostream MangledNameStream(MangledName); + Mangler::getNameWithPrefix(MangledNameStream, UnmangledName, DL); + } + return MangledName; +} + +Error LLJIT::applyDataLayout(Module &M) { + if (M.getDataLayout().isDefault()) + M.setDataLayout(DL); + + if (M.getDataLayout() != DL) + return make_error<StringError>( + "Added modules have incompatible data layouts: " + + M.getDataLayout().getStringRepresentation() + " (module) vs " + + DL.getStringRepresentation() + " (jit)", + inconvertibleErrorCode()); + + return Error::success(); +} + +void setUpGenericLLVMIRPlatform(LLJIT &J) { + LLVM_DEBUG( + { dbgs() << "Setting up GenericLLVMIRPlatform support for LLJIT\n"; }); + J.setPlatformSupport(std::make_unique<GenericLLVMIRPlatformSupport>(J)); +} + +Error setUpMachOPlatform(LLJIT &J) { + LLVM_DEBUG({ dbgs() << "Setting up MachOPlatform support for LLJIT\n"; }); + auto MP = MachOPlatformSupport::Create(J, J.getMainJITDylib()); + if (!MP) + return MP.takeError(); + J.setPlatformSupport(std::move(*MP)); + return Error::success(); +} + +Error LLLazyJITBuilderState::prepareForConstruction() { + if (auto Err = LLJITBuilderState::prepareForConstruction()) + return Err; + TT = JTMB->getTargetTriple(); + return Error::success(); +} + +Error LLLazyJIT::addLazyIRModule(JITDylib &JD, ThreadSafeModule TSM) { + assert(TSM && "Can not add null module"); + + if (auto Err = TSM.withModuleDo( + [&](Module &M) -> Error { return applyDataLayout(M); })) + return Err; + + return CODLayer->add(JD, std::move(TSM)); +} + +LLLazyJIT::LLLazyJIT(LLLazyJITBuilderState &S, Error &Err) : LLJIT(S, Err) { + + // If LLJIT construction failed then bail out. + if (Err) + return; + + ErrorAsOutParameter _(&Err); + + /// Take/Create the lazy-compile callthrough manager. + if (S.LCTMgr) + LCTMgr = std::move(S.LCTMgr); + else { + if (auto LCTMgrOrErr = createLocalLazyCallThroughManager( + S.TT, *ES, S.LazyCompileFailureAddr)) + LCTMgr = std::move(*LCTMgrOrErr); + else { + Err = LCTMgrOrErr.takeError(); + return; + } + } + + // Take/Create the indirect stubs manager builder. + auto ISMBuilder = std::move(S.ISMBuilder); + + // If none was provided, try to build one. + if (!ISMBuilder) + ISMBuilder = createLocalIndirectStubsManagerBuilder(S.TT); + + // No luck. Bail out. + if (!ISMBuilder) { + Err = make_error<StringError>("Could not construct " + "IndirectStubsManagerBuilder for target " + + S.TT.str(), + inconvertibleErrorCode()); + return; + } + + // Create the COD layer. + CODLayer = std::make_unique<CompileOnDemandLayer>( + *ES, *InitHelperTransformLayer, *LCTMgr, std::move(ISMBuilder)); + + if (S.NumCompileThreads > 0) + CODLayer->setCloneToNewContextOnEmit(true); +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/Orc/Layer.cpp b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/Layer.cpp new file mode 100644 index 0000000000..5e27e343d2 --- /dev/null +++ b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/Layer.cpp @@ -0,0 +1,212 @@ +//===-------------------- Layer.cpp - Layer interfaces --------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/Layer.h" + +#include "llvm/ExecutionEngine/Orc/DebugUtils.h" +#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" +#include "llvm/IR/Constants.h" +#include "llvm/Object/MachO.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Support/Debug.h" + +#define DEBUG_TYPE "orc" + +namespace llvm { +namespace orc { + +IRLayer::~IRLayer() {} + +Error IRLayer::add(ResourceTrackerSP RT, ThreadSafeModule TSM) { + assert(RT && "RT can not be null"); + auto &JD = RT->getJITDylib(); + return JD.define(std::make_unique<BasicIRLayerMaterializationUnit>( + *this, *getManglingOptions(), std::move(TSM)), + std::move(RT)); +} + +IRMaterializationUnit::IRMaterializationUnit( + ExecutionSession &ES, const IRSymbolMapper::ManglingOptions &MO, + ThreadSafeModule TSM) + : MaterializationUnit(SymbolFlagsMap(), nullptr), TSM(std::move(TSM)) { + + assert(this->TSM && "Module must not be null"); + + MangleAndInterner Mangle(ES, this->TSM.getModuleUnlocked()->getDataLayout()); + this->TSM.withModuleDo([&](Module &M) { + for (auto &G : M.global_values()) { + // Skip globals that don't generate symbols. + + if (!G.hasName() || G.isDeclaration() || G.hasLocalLinkage() || + G.hasAvailableExternallyLinkage() || G.hasAppendingLinkage()) + continue; + + // thread locals generate different symbols depending on whether or not + // emulated TLS is enabled. + if (G.isThreadLocal() && MO.EmulatedTLS) { + auto &GV = cast<GlobalVariable>(G); + + auto Flags = JITSymbolFlags::fromGlobalValue(GV); + + auto EmuTLSV = Mangle(("__emutls_v." + GV.getName()).str()); + SymbolFlags[EmuTLSV] = Flags; + SymbolToDefinition[EmuTLSV] = &GV; + + // If this GV has a non-zero initializer we'll need to emit an + // __emutls.t symbol too. + if (GV.hasInitializer()) { + const auto *InitVal = GV.getInitializer(); + + // Skip zero-initializers. + if (isa<ConstantAggregateZero>(InitVal)) + continue; + const auto *InitIntValue = dyn_cast<ConstantInt>(InitVal); + if (InitIntValue && InitIntValue->isZero()) + continue; + + auto EmuTLST = Mangle(("__emutls_t." + GV.getName()).str()); + SymbolFlags[EmuTLST] = Flags; + } + continue; + } + + // Otherwise we just need a normal linker mangling. + auto MangledName = Mangle(G.getName()); + SymbolFlags[MangledName] = JITSymbolFlags::fromGlobalValue(G); + SymbolToDefinition[MangledName] = &G; + } + + // If we need an init symbol for this module then create one. + if (!llvm::empty(getStaticInitGVs(M))) { + size_t Counter = 0; + + do { + std::string InitSymbolName; + raw_string_ostream(InitSymbolName) + << "$." << M.getModuleIdentifier() << ".__inits." << Counter++; + InitSymbol = ES.intern(InitSymbolName); + } while (SymbolFlags.count(InitSymbol)); + + SymbolFlags[InitSymbol] = JITSymbolFlags::MaterializationSideEffectsOnly; + } + }); +} + +IRMaterializationUnit::IRMaterializationUnit( + ThreadSafeModule TSM, SymbolFlagsMap SymbolFlags, + SymbolStringPtr InitSymbol, SymbolNameToDefinitionMap SymbolToDefinition) + : MaterializationUnit(std::move(SymbolFlags), std::move(InitSymbol)), + TSM(std::move(TSM)), SymbolToDefinition(std::move(SymbolToDefinition)) {} + +StringRef IRMaterializationUnit::getName() const { + if (TSM) + return TSM.withModuleDo( + [](const Module &M) -> StringRef { return M.getModuleIdentifier(); }); + return "<null module>"; +} + +void IRMaterializationUnit::discard(const JITDylib &JD, + const SymbolStringPtr &Name) { + LLVM_DEBUG(JD.getExecutionSession().runSessionLocked([&]() { + dbgs() << "In " << JD.getName() << " discarding " << *Name << " from MU@" + << this << " (" << getName() << ")\n"; + });); + + auto I = SymbolToDefinition.find(Name); + assert(I != SymbolToDefinition.end() && + "Symbol not provided by this MU, or previously discarded"); + assert(!I->second->isDeclaration() && + "Discard should only apply to definitions"); + I->second->setLinkage(GlobalValue::AvailableExternallyLinkage); + SymbolToDefinition.erase(I); +} + +BasicIRLayerMaterializationUnit::BasicIRLayerMaterializationUnit( + IRLayer &L, const IRSymbolMapper::ManglingOptions &MO, ThreadSafeModule TSM) + : IRMaterializationUnit(L.getExecutionSession(), MO, std::move(TSM)), L(L) { +} + +void BasicIRLayerMaterializationUnit::materialize( + std::unique_ptr<MaterializationResponsibility> R) { + + // Throw away the SymbolToDefinition map: it's not usable after we hand + // off the module. + SymbolToDefinition.clear(); + + // If cloneToNewContextOnEmit is set, clone the module now. + if (L.getCloneToNewContextOnEmit()) + TSM = cloneToNewContext(TSM); + +#ifndef NDEBUG + auto &ES = R->getTargetJITDylib().getExecutionSession(); + auto &N = R->getTargetJITDylib().getName(); +#endif // NDEBUG + + LLVM_DEBUG(ES.runSessionLocked( + [&]() { dbgs() << "Emitting, for " << N << ", " << *this << "\n"; });); + L.emit(std::move(R), std::move(TSM)); + LLVM_DEBUG(ES.runSessionLocked([&]() { + dbgs() << "Finished emitting, for " << N << ", " << *this << "\n"; + });); +} + +ObjectLayer::ObjectLayer(ExecutionSession &ES) : ES(ES) {} + +ObjectLayer::~ObjectLayer() {} + +Error ObjectLayer::add(ResourceTrackerSP RT, std::unique_ptr<MemoryBuffer> O) { + assert(RT && "RT can not be null"); + auto ObjMU = BasicObjectLayerMaterializationUnit::Create(*this, std::move(O)); + if (!ObjMU) + return ObjMU.takeError(); + auto &JD = RT->getJITDylib(); + return JD.define(std::move(*ObjMU), std::move(RT)); +} + +Expected<std::unique_ptr<BasicObjectLayerMaterializationUnit>> +BasicObjectLayerMaterializationUnit::Create(ObjectLayer &L, + std::unique_ptr<MemoryBuffer> O) { + auto ObjSymInfo = + getObjectSymbolInfo(L.getExecutionSession(), O->getMemBufferRef()); + + if (!ObjSymInfo) + return ObjSymInfo.takeError(); + + auto &SymbolFlags = ObjSymInfo->first; + auto &InitSymbol = ObjSymInfo->second; + + return std::unique_ptr<BasicObjectLayerMaterializationUnit>( + new BasicObjectLayerMaterializationUnit( + L, std::move(O), std::move(SymbolFlags), std::move(InitSymbol))); +} + +BasicObjectLayerMaterializationUnit::BasicObjectLayerMaterializationUnit( + ObjectLayer &L, std::unique_ptr<MemoryBuffer> O, SymbolFlagsMap SymbolFlags, + SymbolStringPtr InitSymbol) + : MaterializationUnit(std::move(SymbolFlags), std::move(InitSymbol)), L(L), + O(std::move(O)) {} + +StringRef BasicObjectLayerMaterializationUnit::getName() const { + if (O) + return O->getBufferIdentifier(); + return "<null object>"; +} + +void BasicObjectLayerMaterializationUnit::materialize( + std::unique_ptr<MaterializationResponsibility> R) { + L.emit(std::move(R), std::move(O)); +} + +void BasicObjectLayerMaterializationUnit::discard(const JITDylib &JD, + const SymbolStringPtr &Name) { + // This is a no-op for object files: Having removed 'Name' from SymbolFlags + // the symbol will be dead-stripped by the JIT linker. +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/Orc/LazyReexports.cpp b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/LazyReexports.cpp new file mode 100644 index 0000000000..e1f494415e --- /dev/null +++ b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/LazyReexports.cpp @@ -0,0 +1,234 @@ +//===---------- LazyReexports.cpp - Utilities for lazy reexports ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/LazyReexports.h" + +#include "llvm/ADT/Triple.h" +#include "llvm/ExecutionEngine/Orc/OrcABISupport.h" + +#define DEBUG_TYPE "orc" + +namespace llvm { +namespace orc { + +LazyCallThroughManager::LazyCallThroughManager( + ExecutionSession &ES, JITTargetAddress ErrorHandlerAddr, TrampolinePool *TP) + : ES(ES), ErrorHandlerAddr(ErrorHandlerAddr), TP(TP) {} + +Expected<JITTargetAddress> LazyCallThroughManager::getCallThroughTrampoline( + JITDylib &SourceJD, SymbolStringPtr SymbolName, + NotifyResolvedFunction NotifyResolved) { + assert(TP && "TrampolinePool not set"); + + std::lock_guard<std::mutex> Lock(LCTMMutex); + auto Trampoline = TP->getTrampoline(); + + if (!Trampoline) + return Trampoline.takeError(); + + Reexports[*Trampoline] = ReexportsEntry{&SourceJD, std::move(SymbolName)}; + Notifiers[*Trampoline] = std::move(NotifyResolved); + return *Trampoline; +} + +JITTargetAddress LazyCallThroughManager::reportCallThroughError(Error Err) { + ES.reportError(std::move(Err)); + return ErrorHandlerAddr; +} + +Expected<LazyCallThroughManager::ReexportsEntry> +LazyCallThroughManager::findReexport(JITTargetAddress TrampolineAddr) { + std::lock_guard<std::mutex> Lock(LCTMMutex); + auto I = Reexports.find(TrampolineAddr); + if (I == Reexports.end()) + return createStringError(inconvertibleErrorCode(), + "Missing reexport for trampoline address %p", + TrampolineAddr); + return I->second; +} + +Error LazyCallThroughManager::notifyResolved(JITTargetAddress TrampolineAddr, + JITTargetAddress ResolvedAddr) { + NotifyResolvedFunction NotifyResolved; + { + std::lock_guard<std::mutex> Lock(LCTMMutex); + auto I = Notifiers.find(TrampolineAddr); + if (I != Notifiers.end()) { + NotifyResolved = std::move(I->second); + Notifiers.erase(I); + } + } + + return NotifyResolved ? NotifyResolved(ResolvedAddr) : Error::success(); +} + +void LazyCallThroughManager::resolveTrampolineLandingAddress( + JITTargetAddress TrampolineAddr, + NotifyLandingResolvedFunction NotifyLandingResolved) { + + auto Entry = findReexport(TrampolineAddr); + if (!Entry) + return NotifyLandingResolved(reportCallThroughError(Entry.takeError())); + + // Declaring SLS and the callback outside of the call to ES.lookup is a + // workaround to fix build failures on AIX and on z/OS platforms. + SymbolLookupSet SLS({Entry->SymbolName}); + auto Callback = [this, TrampolineAddr, SymbolName = Entry->SymbolName, + NotifyLandingResolved = std::move(NotifyLandingResolved)]( + Expected<SymbolMap> Result) mutable { + if (Result) { + assert(Result->size() == 1 && "Unexpected result size"); + assert(Result->count(SymbolName) && "Unexpected result value"); + JITTargetAddress LandingAddr = (*Result)[SymbolName].getAddress(); + + if (auto Err = notifyResolved(TrampolineAddr, LandingAddr)) + NotifyLandingResolved(reportCallThroughError(std::move(Err))); + else + NotifyLandingResolved(LandingAddr); + } else { + NotifyLandingResolved(reportCallThroughError(Result.takeError())); + } + }; + + ES.lookup(LookupKind::Static, + makeJITDylibSearchOrder(Entry->SourceJD, + JITDylibLookupFlags::MatchAllSymbols), + std::move(SLS), SymbolState::Ready, std::move(Callback), + NoDependenciesToRegister); +} + +Expected<std::unique_ptr<LazyCallThroughManager>> +createLocalLazyCallThroughManager(const Triple &T, ExecutionSession &ES, + JITTargetAddress ErrorHandlerAddr) { + switch (T.getArch()) { + default: + return make_error<StringError>( + std::string("No callback manager available for ") + T.str(), + inconvertibleErrorCode()); + + case Triple::aarch64: + case Triple::aarch64_32: + return LocalLazyCallThroughManager::Create<OrcAArch64>(ES, + ErrorHandlerAddr); + + case Triple::x86: + return LocalLazyCallThroughManager::Create<OrcI386>(ES, ErrorHandlerAddr); + + case Triple::mips: + return LocalLazyCallThroughManager::Create<OrcMips32Be>(ES, + ErrorHandlerAddr); + + case Triple::mipsel: + return LocalLazyCallThroughManager::Create<OrcMips32Le>(ES, + ErrorHandlerAddr); + + case Triple::mips64: + case Triple::mips64el: + return LocalLazyCallThroughManager::Create<OrcMips64>(ES, ErrorHandlerAddr); + + case Triple::x86_64: + if (T.getOS() == Triple::OSType::Win32) + return LocalLazyCallThroughManager::Create<OrcX86_64_Win32>( + ES, ErrorHandlerAddr); + else + return LocalLazyCallThroughManager::Create<OrcX86_64_SysV>( + ES, ErrorHandlerAddr); + } +} + +LazyReexportsMaterializationUnit::LazyReexportsMaterializationUnit( + LazyCallThroughManager &LCTManager, IndirectStubsManager &ISManager, + JITDylib &SourceJD, SymbolAliasMap CallableAliases, ImplSymbolMap *SrcJDLoc) + : MaterializationUnit(extractFlags(CallableAliases), nullptr), + LCTManager(LCTManager), ISManager(ISManager), SourceJD(SourceJD), + CallableAliases(std::move(CallableAliases)), AliaseeTable(SrcJDLoc) {} + +StringRef LazyReexportsMaterializationUnit::getName() const { + return "<Lazy Reexports>"; +} + +void LazyReexportsMaterializationUnit::materialize( + std::unique_ptr<MaterializationResponsibility> R) { + auto RequestedSymbols = R->getRequestedSymbols(); + + SymbolAliasMap RequestedAliases; + for (auto &RequestedSymbol : RequestedSymbols) { + auto I = CallableAliases.find(RequestedSymbol); + assert(I != CallableAliases.end() && "Symbol not found in alias map?"); + RequestedAliases[I->first] = std::move(I->second); + CallableAliases.erase(I); + } + + if (!CallableAliases.empty()) + if (auto Err = R->replace(lazyReexports(LCTManager, ISManager, SourceJD, + std::move(CallableAliases), + AliaseeTable))) { + R->getExecutionSession().reportError(std::move(Err)); + R->failMaterialization(); + return; + } + + IndirectStubsManager::StubInitsMap StubInits; + for (auto &Alias : RequestedAliases) { + + auto CallThroughTrampoline = LCTManager.getCallThroughTrampoline( + SourceJD, Alias.second.Aliasee, + [&ISManager = this->ISManager, + StubSym = Alias.first](JITTargetAddress ResolvedAddr) -> Error { + return ISManager.updatePointer(*StubSym, ResolvedAddr); + }); + + if (!CallThroughTrampoline) { + SourceJD.getExecutionSession().reportError( + CallThroughTrampoline.takeError()); + R->failMaterialization(); + return; + } + + StubInits[*Alias.first] = + std::make_pair(*CallThroughTrampoline, Alias.second.AliasFlags); + } + + if (AliaseeTable != nullptr && !RequestedAliases.empty()) + AliaseeTable->trackImpls(RequestedAliases, &SourceJD); + + if (auto Err = ISManager.createStubs(StubInits)) { + SourceJD.getExecutionSession().reportError(std::move(Err)); + R->failMaterialization(); + return; + } + + SymbolMap Stubs; + for (auto &Alias : RequestedAliases) + Stubs[Alias.first] = ISManager.findStub(*Alias.first, false); + + // No registered dependencies, so these calls cannot fail. + cantFail(R->notifyResolved(Stubs)); + cantFail(R->notifyEmitted()); +} + +void LazyReexportsMaterializationUnit::discard(const JITDylib &JD, + const SymbolStringPtr &Name) { + assert(CallableAliases.count(Name) && + "Symbol not covered by this MaterializationUnit"); + CallableAliases.erase(Name); +} + +SymbolFlagsMap +LazyReexportsMaterializationUnit::extractFlags(const SymbolAliasMap &Aliases) { + SymbolFlagsMap SymbolFlags; + for (auto &KV : Aliases) { + assert(KV.second.AliasFlags.isCallable() && + "Lazy re-exports must be callable symbols"); + SymbolFlags[KV.first] = KV.second.AliasFlags; + } + return SymbolFlags; +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/Orc/MachOPlatform.cpp b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/MachOPlatform.cpp new file mode 100644 index 0000000000..17b9465a05 --- /dev/null +++ b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/MachOPlatform.cpp @@ -0,0 +1,489 @@ +//===------ MachOPlatform.cpp - Utilities for executing MachO in Orc ------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/MachOPlatform.h" + +#include "llvm/BinaryFormat/MachO.h" +#include "llvm/ExecutionEngine/Orc/DebugUtils.h" +#include "llvm/Support/BinaryByteStream.h" +#include "llvm/Support/Debug.h" + +#define DEBUG_TYPE "orc" + +namespace { + +struct objc_class; +struct objc_image_info; +struct objc_object; +struct objc_selector; + +using Class = objc_class *; +using id = objc_object *; +using SEL = objc_selector *; + +using ObjCMsgSendTy = id (*)(id, SEL, ...); +using ObjCReadClassPairTy = Class (*)(Class, const objc_image_info *); +using SelRegisterNameTy = SEL (*)(const char *); + +enum class ObjCRegistrationAPI { Uninitialized, Unavailable, Initialized }; + +ObjCRegistrationAPI ObjCRegistrationAPIState = + ObjCRegistrationAPI::Uninitialized; +ObjCMsgSendTy objc_msgSend = nullptr; +ObjCReadClassPairTy objc_readClassPair = nullptr; +SelRegisterNameTy sel_registerName = nullptr; + +} // end anonymous namespace + +namespace llvm { +namespace orc { + +template <typename FnTy> +static Error setUpObjCRegAPIFunc(FnTy &Target, sys::DynamicLibrary &LibObjC, + const char *Name) { + if (void *Addr = LibObjC.getAddressOfSymbol(Name)) + Target = reinterpret_cast<FnTy>(Addr); + else + return make_error<StringError>( + (Twine("Could not find address for ") + Name).str(), + inconvertibleErrorCode()); + return Error::success(); +} + +Error enableObjCRegistration(const char *PathToLibObjC) { + // If we've already tried to initialize then just bail out. + if (ObjCRegistrationAPIState != ObjCRegistrationAPI::Uninitialized) + return Error::success(); + + ObjCRegistrationAPIState = ObjCRegistrationAPI::Unavailable; + + std::string ErrMsg; + auto LibObjC = + sys::DynamicLibrary::getPermanentLibrary(PathToLibObjC, &ErrMsg); + + if (!LibObjC.isValid()) + return make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode()); + + if (auto Err = setUpObjCRegAPIFunc(objc_msgSend, LibObjC, "objc_msgSend")) + return Err; + if (auto Err = setUpObjCRegAPIFunc(objc_readClassPair, LibObjC, + "objc_readClassPair")) + return Err; + if (auto Err = + setUpObjCRegAPIFunc(sel_registerName, LibObjC, "sel_registerName")) + return Err; + + ObjCRegistrationAPIState = ObjCRegistrationAPI::Initialized; + return Error::success(); +} + +bool objCRegistrationEnabled() { + return ObjCRegistrationAPIState == ObjCRegistrationAPI::Initialized; +} + +void MachOJITDylibInitializers::runModInits() const { + for (const auto &ModInit : ModInitSections) { + for (uint64_t I = 0; I != ModInit.NumPtrs; ++I) { + auto *InitializerAddr = jitTargetAddressToPointer<uintptr_t *>( + ModInit.Address + (I * sizeof(uintptr_t))); + auto *Initializer = + jitTargetAddressToFunction<void (*)()>(*InitializerAddr); + Initializer(); + } + } +} + +void MachOJITDylibInitializers::registerObjCSelectors() const { + assert(objCRegistrationEnabled() && "ObjC registration not enabled."); + + for (const auto &ObjCSelRefs : ObjCSelRefsSections) { + for (uint64_t I = 0; I != ObjCSelRefs.NumPtrs; ++I) { + auto SelEntryAddr = ObjCSelRefs.Address + (I * sizeof(uintptr_t)); + const auto *SelName = + *jitTargetAddressToPointer<const char **>(SelEntryAddr); + auto Sel = sel_registerName(SelName); + *jitTargetAddressToPointer<SEL *>(SelEntryAddr) = Sel; + } + } +} + +Error MachOJITDylibInitializers::registerObjCClasses() const { + assert(objCRegistrationEnabled() && "ObjC registration not enabled."); + + struct ObjCClassCompiled { + void *Metaclass; + void *Parent; + void *Cache1; + void *Cache2; + void *Data; + }; + + auto *ImageInfo = + jitTargetAddressToPointer<const objc_image_info *>(ObjCImageInfoAddr); + auto ClassSelector = sel_registerName("class"); + + for (const auto &ObjCClassList : ObjCClassListSections) { + for (uint64_t I = 0; I != ObjCClassList.NumPtrs; ++I) { + auto ClassPtrAddr = ObjCClassList.Address + (I * sizeof(uintptr_t)); + auto Cls = *jitTargetAddressToPointer<Class *>(ClassPtrAddr); + auto *ClassCompiled = + *jitTargetAddressToPointer<ObjCClassCompiled **>(ClassPtrAddr); + objc_msgSend(reinterpret_cast<id>(ClassCompiled->Parent), ClassSelector); + auto Registered = objc_readClassPair(Cls, ImageInfo); + + // FIXME: Improve diagnostic by reporting the failed class's name. + if (Registered != Cls) + return make_error<StringError>("Unable to register Objective-C class", + inconvertibleErrorCode()); + } + } + return Error::success(); +} + +MachOPlatform::MachOPlatform( + ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, + std::unique_ptr<MemoryBuffer> StandardSymbolsObject) + : ES(ES), ObjLinkingLayer(ObjLinkingLayer), + StandardSymbolsObject(std::move(StandardSymbolsObject)) { + ObjLinkingLayer.addPlugin(std::make_unique<InitScraperPlugin>(*this)); +} + +Error MachOPlatform::setupJITDylib(JITDylib &JD) { + auto ObjBuffer = MemoryBuffer::getMemBuffer( + StandardSymbolsObject->getMemBufferRef(), false); + return ObjLinkingLayer.add(JD, std::move(ObjBuffer)); +} + +Error MachOPlatform::notifyAdding(ResourceTracker &RT, + const MaterializationUnit &MU) { + auto &JD = RT.getJITDylib(); + const auto &InitSym = MU.getInitializerSymbol(); + if (!InitSym) + return Error::success(); + + RegisteredInitSymbols[&JD].add(InitSym, + SymbolLookupFlags::WeaklyReferencedSymbol); + LLVM_DEBUG({ + dbgs() << "MachOPlatform: Registered init symbol " << *InitSym << " for MU " + << MU.getName() << "\n"; + }); + return Error::success(); +} + +Error MachOPlatform::notifyRemoving(ResourceTracker &RT) { + llvm_unreachable("Not supported yet"); +} + +Expected<MachOPlatform::InitializerSequence> +MachOPlatform::getInitializerSequence(JITDylib &JD) { + + LLVM_DEBUG({ + dbgs() << "MachOPlatform: Building initializer sequence for " + << JD.getName() << "\n"; + }); + + std::vector<JITDylibSP> DFSLinkOrder; + + while (true) { + + DenseMap<JITDylib *, SymbolLookupSet> NewInitSymbols; + + ES.runSessionLocked([&]() { + DFSLinkOrder = JD.getDFSLinkOrder(); + + for (auto &InitJD : DFSLinkOrder) { + auto RISItr = RegisteredInitSymbols.find(InitJD.get()); + if (RISItr != RegisteredInitSymbols.end()) { + NewInitSymbols[InitJD.get()] = std::move(RISItr->second); + RegisteredInitSymbols.erase(RISItr); + } + } + }); + + if (NewInitSymbols.empty()) + break; + + LLVM_DEBUG({ + dbgs() << "MachOPlatform: Issuing lookups for new init symbols: " + "(lookup may require multiple rounds)\n"; + for (auto &KV : NewInitSymbols) + dbgs() << " \"" << KV.first->getName() << "\": " << KV.second << "\n"; + }); + + // Outside the lock, issue the lookup. + if (auto R = lookupInitSymbols(JD.getExecutionSession(), NewInitSymbols)) + ; // Nothing to do in the success case. + else + return R.takeError(); + } + + LLVM_DEBUG({ + dbgs() << "MachOPlatform: Init symbol lookup complete, building init " + "sequence\n"; + }); + + // Lock again to collect the initializers. + InitializerSequence FullInitSeq; + { + std::lock_guard<std::mutex> Lock(InitSeqsMutex); + for (auto &InitJD : reverse(DFSLinkOrder)) { + LLVM_DEBUG({ + dbgs() << "MachOPlatform: Appending inits for \"" << InitJD->getName() + << "\" to sequence\n"; + }); + auto ISItr = InitSeqs.find(InitJD.get()); + if (ISItr != InitSeqs.end()) { + FullInitSeq.emplace_back(InitJD.get(), std::move(ISItr->second)); + InitSeqs.erase(ISItr); + } + } + } + + return FullInitSeq; +} + +Expected<MachOPlatform::DeinitializerSequence> +MachOPlatform::getDeinitializerSequence(JITDylib &JD) { + std::vector<JITDylibSP> DFSLinkOrder = JD.getDFSLinkOrder(); + + DeinitializerSequence FullDeinitSeq; + { + std::lock_guard<std::mutex> Lock(InitSeqsMutex); + for (auto &DeinitJD : DFSLinkOrder) { + FullDeinitSeq.emplace_back(DeinitJD.get(), MachOJITDylibDeinitializers()); + } + } + + return FullDeinitSeq; +} + +void MachOPlatform::registerInitInfo( + JITDylib &JD, JITTargetAddress ObjCImageInfoAddr, + MachOJITDylibInitializers::SectionExtent ModInits, + MachOJITDylibInitializers::SectionExtent ObjCSelRefs, + MachOJITDylibInitializers::SectionExtent ObjCClassList) { + std::lock_guard<std::mutex> Lock(InitSeqsMutex); + + auto &InitSeq = InitSeqs[&JD]; + + InitSeq.setObjCImageInfoAddr(ObjCImageInfoAddr); + + if (ModInits.Address) + InitSeq.addModInitsSection(std::move(ModInits)); + + if (ObjCSelRefs.Address) + InitSeq.addObjCSelRefsSection(std::move(ObjCSelRefs)); + + if (ObjCClassList.Address) + InitSeq.addObjCClassListSection(std::move(ObjCClassList)); +} + +static Expected<MachOJITDylibInitializers::SectionExtent> +getSectionExtent(jitlink::LinkGraph &G, StringRef SectionName) { + auto *Sec = G.findSectionByName(SectionName); + if (!Sec) + return MachOJITDylibInitializers::SectionExtent(); + jitlink::SectionRange R(*Sec); + if (R.getSize() % G.getPointerSize() != 0) + return make_error<StringError>(SectionName + " section size is not a " + "multiple of the pointer size", + inconvertibleErrorCode()); + return MachOJITDylibInitializers::SectionExtent( + R.getStart(), R.getSize() / G.getPointerSize()); +} + +void MachOPlatform::InitScraperPlugin::modifyPassConfig( + MaterializationResponsibility &MR, const Triple &TT, + jitlink::PassConfiguration &Config) { + + if (!MR.getInitializerSymbol()) + return; + + Config.PrePrunePasses.push_back([this, &MR](jitlink::LinkGraph &G) -> Error { + JITLinkSymbolVector InitSectionSymbols; + preserveInitSectionIfPresent(InitSectionSymbols, G, "__mod_init_func"); + preserveInitSectionIfPresent(InitSectionSymbols, G, "__objc_selrefs"); + preserveInitSectionIfPresent(InitSectionSymbols, G, "__objc_classlist"); + + if (!InitSectionSymbols.empty()) { + std::lock_guard<std::mutex> Lock(InitScraperMutex); + InitSymbolDeps[&MR] = std::move(InitSectionSymbols); + } + + if (auto Err = processObjCImageInfo(G, MR)) + return Err; + + return Error::success(); + }); + + Config.PostFixupPasses.push_back([this, &JD = MR.getTargetJITDylib()]( + jitlink::LinkGraph &G) -> Error { + MachOJITDylibInitializers::SectionExtent ModInits, ObjCSelRefs, + ObjCClassList; + + JITTargetAddress ObjCImageInfoAddr = 0; + if (auto *ObjCImageInfoSec = G.findSectionByName("__objc_image_info")) { + if (auto Addr = jitlink::SectionRange(*ObjCImageInfoSec).getStart()) + ObjCImageInfoAddr = Addr; + } + + // Record __mod_init_func. + if (auto ModInitsOrErr = getSectionExtent(G, "__mod_init_func")) + ModInits = std::move(*ModInitsOrErr); + else + return ModInitsOrErr.takeError(); + + // Record __objc_selrefs. + if (auto ObjCSelRefsOrErr = getSectionExtent(G, "__objc_selrefs")) + ObjCSelRefs = std::move(*ObjCSelRefsOrErr); + else + return ObjCSelRefsOrErr.takeError(); + + // Record __objc_classlist. + if (auto ObjCClassListOrErr = getSectionExtent(G, "__objc_classlist")) + ObjCClassList = std::move(*ObjCClassListOrErr); + else + return ObjCClassListOrErr.takeError(); + + // Dump the scraped inits. + LLVM_DEBUG({ + dbgs() << "MachOPlatform: Scraped " << G.getName() << " init sections:\n"; + dbgs() << " __objc_selrefs: "; + if (ObjCSelRefs.NumPtrs) + dbgs() << ObjCSelRefs.NumPtrs << " pointer(s) at " + << formatv("{0:x16}", ObjCSelRefs.Address) << "\n"; + else + dbgs() << "none\n"; + + dbgs() << " __objc_classlist: "; + if (ObjCClassList.NumPtrs) + dbgs() << ObjCClassList.NumPtrs << " pointer(s) at " + << formatv("{0:x16}", ObjCClassList.Address) << "\n"; + else + dbgs() << "none\n"; + + dbgs() << " __mod_init_func: "; + if (ModInits.NumPtrs) + dbgs() << ModInits.NumPtrs << " pointer(s) at " + << formatv("{0:x16}", ModInits.Address) << "\n"; + else + dbgs() << "none\n"; + }); + + MP.registerInitInfo(JD, ObjCImageInfoAddr, std::move(ModInits), + std::move(ObjCSelRefs), std::move(ObjCClassList)); + + return Error::success(); + }); +} + +ObjectLinkingLayer::Plugin::LocalDependenciesMap +MachOPlatform::InitScraperPlugin::getSyntheticSymbolLocalDependencies( + MaterializationResponsibility &MR) { + std::lock_guard<std::mutex> Lock(InitScraperMutex); + auto I = InitSymbolDeps.find(&MR); + if (I != InitSymbolDeps.end()) { + LocalDependenciesMap Result; + Result[MR.getInitializerSymbol()] = std::move(I->second); + InitSymbolDeps.erase(&MR); + return Result; + } + return LocalDependenciesMap(); +} + +void MachOPlatform::InitScraperPlugin::preserveInitSectionIfPresent( + JITLinkSymbolVector &Symbols, jitlink::LinkGraph &G, + StringRef SectionName) { + if (auto *Sec = G.findSectionByName(SectionName)) { + auto SecBlocks = Sec->blocks(); + if (!llvm::empty(SecBlocks)) + Symbols.push_back( + &G.addAnonymousSymbol(**SecBlocks.begin(), 0, 0, false, true)); + } +} + +Error MachOPlatform::InitScraperPlugin::processObjCImageInfo( + jitlink::LinkGraph &G, MaterializationResponsibility &MR) { + + // If there's an ObjC imagine info then either + // (1) It's the first __objc_imageinfo we've seen in this JITDylib. In + // this case we name and record it. + // OR + // (2) We already have a recorded __objc_imageinfo for this JITDylib, + // in which case we just verify it. + auto *ObjCImageInfo = G.findSectionByName("__objc_imageinfo"); + if (!ObjCImageInfo) + return Error::success(); + + auto ObjCImageInfoBlocks = ObjCImageInfo->blocks(); + + // Check that the section is not empty if present. + if (llvm::empty(ObjCImageInfoBlocks)) + return make_error<StringError>("Empty __objc_imageinfo section in " + + G.getName(), + inconvertibleErrorCode()); + + // Check that there's only one block in the section. + if (std::next(ObjCImageInfoBlocks.begin()) != ObjCImageInfoBlocks.end()) + return make_error<StringError>("Multiple blocks in __objc_imageinfo " + "section in " + + G.getName(), + inconvertibleErrorCode()); + + // Check that the __objc_imageinfo section is unreferenced. + // FIXME: We could optimize this check if Symbols had a ref-count. + for (auto &Sec : G.sections()) { + if (&Sec != ObjCImageInfo) + for (auto *B : Sec.blocks()) + for (auto &E : B->edges()) + if (E.getTarget().isDefined() && + &E.getTarget().getBlock().getSection() == ObjCImageInfo) + return make_error<StringError>("__objc_imageinfo is referenced " + "within file " + + G.getName(), + inconvertibleErrorCode()); + } + + auto &ObjCImageInfoBlock = **ObjCImageInfoBlocks.begin(); + auto *ObjCImageInfoData = ObjCImageInfoBlock.getContent().data(); + auto Version = support::endian::read32(ObjCImageInfoData, G.getEndianness()); + auto Flags = + support::endian::read32(ObjCImageInfoData + 4, G.getEndianness()); + + // Lock the mutex while we verify / update the ObjCImageInfos map. + std::lock_guard<std::mutex> Lock(InitScraperMutex); + + auto ObjCImageInfoItr = ObjCImageInfos.find(&MR.getTargetJITDylib()); + if (ObjCImageInfoItr != ObjCImageInfos.end()) { + // We've already registered an __objc_imageinfo section. Verify the + // content of this new section matches, then delete it. + if (ObjCImageInfoItr->second.first != Version) + return make_error<StringError>( + "ObjC version in " + G.getName() + + " does not match first registered version", + inconvertibleErrorCode()); + if (ObjCImageInfoItr->second.second != Flags) + return make_error<StringError>("ObjC flags in " + G.getName() + + " do not match first registered flags", + inconvertibleErrorCode()); + + // __objc_imageinfo is valid. Delete the block. + for (auto *S : ObjCImageInfo->symbols()) + G.removeDefinedSymbol(*S); + G.removeBlock(ObjCImageInfoBlock); + } else { + // We haven't registered an __objc_imageinfo section yet. Register and + // move on. The section should already be marked no-dead-strip. + ObjCImageInfos[&MR.getTargetJITDylib()] = std::make_pair(Version, Flags); + } + + return Error::success(); +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/Orc/Mangling.cpp b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/Mangling.cpp new file mode 100644 index 0000000000..606304741c --- /dev/null +++ b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/Mangling.cpp @@ -0,0 +1,160 @@ +//===----------- Mangling.cpp -- Name Mangling Utilities for ORC ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/Mangling.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/Mangler.h" +#include "llvm/Object/MachO.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Support/Debug.h" + +#define DEBUG_TYPE "orc" + +namespace llvm { +namespace orc { + +MangleAndInterner::MangleAndInterner(ExecutionSession &ES, const DataLayout &DL) + : ES(ES), DL(DL) {} + +SymbolStringPtr MangleAndInterner::operator()(StringRef Name) { + std::string MangledName; + { + raw_string_ostream MangledNameStream(MangledName); + Mangler::getNameWithPrefix(MangledNameStream, Name, DL); + } + return ES.intern(MangledName); +} + +void IRSymbolMapper::add(ExecutionSession &ES, const ManglingOptions &MO, + ArrayRef<GlobalValue *> GVs, + SymbolFlagsMap &SymbolFlags, + SymbolNameToDefinitionMap *SymbolToDefinition) { + if (GVs.empty()) + return; + + MangleAndInterner Mangle(ES, GVs[0]->getParent()->getDataLayout()); + for (auto *G : GVs) { + assert(G && "GVs cannot contain null elements"); + if (!G->hasName() || G->isDeclaration() || G->hasLocalLinkage() || + G->hasAvailableExternallyLinkage() || G->hasAppendingLinkage()) + continue; + + if (G->isThreadLocal() && MO.EmulatedTLS) { + auto *GV = cast<GlobalVariable>(G); + + auto Flags = JITSymbolFlags::fromGlobalValue(*GV); + + auto EmuTLSV = Mangle(("__emutls_v." + GV->getName()).str()); + SymbolFlags[EmuTLSV] = Flags; + if (SymbolToDefinition) + (*SymbolToDefinition)[EmuTLSV] = GV; + + // If this GV has a non-zero initializer we'll need to emit an + // __emutls.t symbol too. + if (GV->hasInitializer()) { + const auto *InitVal = GV->getInitializer(); + + // Skip zero-initializers. + if (isa<ConstantAggregateZero>(InitVal)) + continue; + const auto *InitIntValue = dyn_cast<ConstantInt>(InitVal); + if (InitIntValue && InitIntValue->isZero()) + continue; + + auto EmuTLST = Mangle(("__emutls_t." + GV->getName()).str()); + SymbolFlags[EmuTLST] = Flags; + if (SymbolToDefinition) + (*SymbolToDefinition)[EmuTLST] = GV; + } + continue; + } + + // Otherwise we just need a normal linker mangling. + auto MangledName = Mangle(G->getName()); + SymbolFlags[MangledName] = JITSymbolFlags::fromGlobalValue(*G); + if (SymbolToDefinition) + (*SymbolToDefinition)[MangledName] = G; + } +} + +Expected<std::pair<SymbolFlagsMap, SymbolStringPtr>> +getObjectSymbolInfo(ExecutionSession &ES, MemoryBufferRef ObjBuffer) { + auto Obj = object::ObjectFile::createObjectFile(ObjBuffer); + + if (!Obj) + return Obj.takeError(); + + bool IsMachO = isa<object::MachOObjectFile>(Obj->get()); + + SymbolFlagsMap SymbolFlags; + for (auto &Sym : (*Obj)->symbols()) { + Expected<uint32_t> SymFlagsOrErr = Sym.getFlags(); + if (!SymFlagsOrErr) + // TODO: Test this error. + return SymFlagsOrErr.takeError(); + + // Skip symbols not defined in this object file. + if (*SymFlagsOrErr & object::BasicSymbolRef::SF_Undefined) + continue; + + // Skip symbols that are not global. + if (!(*SymFlagsOrErr & object::BasicSymbolRef::SF_Global)) + continue; + + // Skip symbols that have type SF_File. + if (auto SymType = Sym.getType()) { + if (*SymType == object::SymbolRef::ST_File) + continue; + } else + return SymType.takeError(); + + auto Name = Sym.getName(); + if (!Name) + return Name.takeError(); + auto InternedName = ES.intern(*Name); + auto SymFlags = JITSymbolFlags::fromObjectSymbol(Sym); + if (!SymFlags) + return SymFlags.takeError(); + + // Strip the 'exported' flag from MachO linker-private symbols. + if (IsMachO && Name->startswith("l")) + *SymFlags &= ~JITSymbolFlags::Exported; + + SymbolFlags[InternedName] = std::move(*SymFlags); + } + + SymbolStringPtr InitSymbol; + + if (IsMachO) { + auto &MachOObj = cast<object::MachOObjectFile>(*Obj->get()); + for (auto &Sec : MachOObj.sections()) { + auto SecType = MachOObj.getSectionType(Sec); + if ((SecType & MachO::SECTION_TYPE) == MachO::S_MOD_INIT_FUNC_POINTERS) { + size_t Counter = 0; + while (true) { + std::string InitSymString; + raw_string_ostream(InitSymString) + << "$." << ObjBuffer.getBufferIdentifier() << ".__inits." + << Counter++; + InitSymbol = ES.intern(InitSymString); + if (SymbolFlags.count(InitSymbol)) + continue; + SymbolFlags[InitSymbol] = + JITSymbolFlags::MaterializationSideEffectsOnly; + break; + } + break; + } + } + } + + return std::make_pair(std::move(SymbolFlags), std::move(InitSymbol)); +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp new file mode 100644 index 0000000000..26f77acd91 --- /dev/null +++ b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp @@ -0,0 +1,652 @@ +//===------- ObjectLinkingLayer.cpp - JITLink backed ORC ObjectLayer ------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h" + +#include "llvm/ADT/Optional.h" +#include "llvm/ExecutionEngine/JITLink/EHFrameSupport.h" + +#include <vector> + +#define DEBUG_TYPE "orc" + +using namespace llvm; +using namespace llvm::jitlink; +using namespace llvm::orc; + +namespace llvm { +namespace orc { + +class ObjectLinkingLayerJITLinkContext final : public JITLinkContext { +public: + ObjectLinkingLayerJITLinkContext( + ObjectLinkingLayer &Layer, + std::unique_ptr<MaterializationResponsibility> MR, + std::unique_ptr<MemoryBuffer> ObjBuffer) + : JITLinkContext(&MR->getTargetJITDylib()), Layer(Layer), + MR(std::move(MR)), ObjBuffer(std::move(ObjBuffer)) {} + + ~ObjectLinkingLayerJITLinkContext() { + // If there is an object buffer return function then use it to + // return ownership of the buffer. + if (Layer.ReturnObjectBuffer && ObjBuffer) + Layer.ReturnObjectBuffer(std::move(ObjBuffer)); + } + + JITLinkMemoryManager &getMemoryManager() override { return Layer.MemMgr; } + + void notifyFailed(Error Err) override { + for (auto &P : Layer.Plugins) + Err = joinErrors(std::move(Err), P->notifyFailed(*MR)); + Layer.getExecutionSession().reportError(std::move(Err)); + MR->failMaterialization(); + } + + void lookup(const LookupMap &Symbols, + std::unique_ptr<JITLinkAsyncLookupContinuation> LC) override { + + JITDylibSearchOrder LinkOrder; + MR->getTargetJITDylib().withLinkOrderDo( + [&](const JITDylibSearchOrder &LO) { LinkOrder = LO; }); + + auto &ES = Layer.getExecutionSession(); + + SymbolLookupSet LookupSet; + for (auto &KV : Symbols) { + orc::SymbolLookupFlags LookupFlags; + switch (KV.second) { + case jitlink::SymbolLookupFlags::RequiredSymbol: + LookupFlags = orc::SymbolLookupFlags::RequiredSymbol; + break; + case jitlink::SymbolLookupFlags::WeaklyReferencedSymbol: + LookupFlags = orc::SymbolLookupFlags::WeaklyReferencedSymbol; + break; + } + LookupSet.add(ES.intern(KV.first), LookupFlags); + } + + // OnResolve -- De-intern the symbols and pass the result to the linker. + auto OnResolve = [LookupContinuation = + std::move(LC)](Expected<SymbolMap> Result) mutable { + if (!Result) + LookupContinuation->run(Result.takeError()); + else { + AsyncLookupResult LR; + for (auto &KV : *Result) + LR[*KV.first] = KV.second; + LookupContinuation->run(std::move(LR)); + } + }; + + for (auto &KV : InternalNamedSymbolDeps) { + SymbolDependenceMap InternalDeps; + InternalDeps[&MR->getTargetJITDylib()] = std::move(KV.second); + MR->addDependencies(KV.first, InternalDeps); + } + + ES.lookup(LookupKind::Static, LinkOrder, std::move(LookupSet), + SymbolState::Resolved, std::move(OnResolve), + [this](const SymbolDependenceMap &Deps) { + registerDependencies(Deps); + }); + } + + Error notifyResolved(LinkGraph &G) override { + auto &ES = Layer.getExecutionSession(); + + SymbolFlagsMap ExtraSymbolsToClaim; + bool AutoClaim = Layer.AutoClaimObjectSymbols; + + SymbolMap InternedResult; + for (auto *Sym : G.defined_symbols()) + if (Sym->hasName() && Sym->getScope() != Scope::Local) { + auto InternedName = ES.intern(Sym->getName()); + JITSymbolFlags Flags; + + if (Sym->isCallable()) + Flags |= JITSymbolFlags::Callable; + if (Sym->getScope() == Scope::Default) + Flags |= JITSymbolFlags::Exported; + + InternedResult[InternedName] = + JITEvaluatedSymbol(Sym->getAddress(), Flags); + if (AutoClaim && !MR->getSymbols().count(InternedName)) { + assert(!ExtraSymbolsToClaim.count(InternedName) && + "Duplicate symbol to claim?"); + ExtraSymbolsToClaim[InternedName] = Flags; + } + } + + for (auto *Sym : G.absolute_symbols()) + if (Sym->hasName()) { + auto InternedName = ES.intern(Sym->getName()); + JITSymbolFlags Flags; + Flags |= JITSymbolFlags::Absolute; + if (Sym->isCallable()) + Flags |= JITSymbolFlags::Callable; + if (Sym->getLinkage() == Linkage::Weak) + Flags |= JITSymbolFlags::Weak; + InternedResult[InternedName] = + JITEvaluatedSymbol(Sym->getAddress(), Flags); + if (AutoClaim && !MR->getSymbols().count(InternedName)) { + assert(!ExtraSymbolsToClaim.count(InternedName) && + "Duplicate symbol to claim?"); + ExtraSymbolsToClaim[InternedName] = Flags; + } + } + + if (!ExtraSymbolsToClaim.empty()) + if (auto Err = MR->defineMaterializing(ExtraSymbolsToClaim)) + return Err; + + { + + // Check that InternedResult matches up with MR->getSymbols(). + // This guards against faulty transformations / compilers / object caches. + + // First check that there aren't any missing symbols. + size_t NumMaterializationSideEffectsOnlySymbols = 0; + SymbolNameVector ExtraSymbols; + SymbolNameVector MissingSymbols; + for (auto &KV : MR->getSymbols()) { + + // If this is a materialization-side-effects only symbol then bump + // the counter and make sure it's *not* defined, otherwise make + // sure that it is defined. + if (KV.second.hasMaterializationSideEffectsOnly()) { + ++NumMaterializationSideEffectsOnlySymbols; + if (InternedResult.count(KV.first)) + ExtraSymbols.push_back(KV.first); + continue; + } else if (!InternedResult.count(KV.first)) + MissingSymbols.push_back(KV.first); + } + + // If there were missing symbols then report the error. + if (!MissingSymbols.empty()) + return make_error<MissingSymbolDefinitions>(G.getName(), + std::move(MissingSymbols)); + + // If there are more definitions than expected, add them to the + // ExtraSymbols vector. + if (InternedResult.size() > + MR->getSymbols().size() - NumMaterializationSideEffectsOnlySymbols) { + for (auto &KV : InternedResult) + if (!MR->getSymbols().count(KV.first)) + ExtraSymbols.push_back(KV.first); + } + + // If there were extra definitions then report the error. + if (!ExtraSymbols.empty()) + return make_error<UnexpectedSymbolDefinitions>(G.getName(), + std::move(ExtraSymbols)); + } + + if (auto Err = MR->notifyResolved(InternedResult)) + return Err; + + Layer.notifyLoaded(*MR); + return Error::success(); + } + + void notifyFinalized( + std::unique_ptr<JITLinkMemoryManager::Allocation> A) override { + if (auto Err = Layer.notifyEmitted(*MR, std::move(A))) { + Layer.getExecutionSession().reportError(std::move(Err)); + MR->failMaterialization(); + return; + } + if (auto Err = MR->notifyEmitted()) { + Layer.getExecutionSession().reportError(std::move(Err)); + MR->failMaterialization(); + } + } + + LinkGraphPassFunction getMarkLivePass(const Triple &TT) const override { + return [this](LinkGraph &G) { return markResponsibilitySymbolsLive(G); }; + } + + Error modifyPassConfig(const Triple &TT, PassConfiguration &Config) override { + // Add passes to mark duplicate defs as should-discard, and to walk the + // link graph to build the symbol dependence graph. + Config.PrePrunePasses.push_back([this](LinkGraph &G) { + return claimOrExternalizeWeakAndCommonSymbols(G); + }); + + Layer.modifyPassConfig(*MR, TT, Config); + + Config.PostPrunePasses.push_back( + [this](LinkGraph &G) { return computeNamedSymbolDependencies(G); }); + + return Error::success(); + } + +private: + struct LocalSymbolNamedDependencies { + SymbolNameSet Internal, External; + }; + + using LocalSymbolNamedDependenciesMap = + DenseMap<const Symbol *, LocalSymbolNamedDependencies>; + + Error claimOrExternalizeWeakAndCommonSymbols(LinkGraph &G) { + auto &ES = Layer.getExecutionSession(); + + SymbolFlagsMap NewSymbolsToClaim; + std::vector<std::pair<SymbolStringPtr, Symbol *>> NameToSym; + + auto ProcessSymbol = [&](Symbol *Sym) { + if (Sym->hasName() && Sym->getLinkage() == Linkage::Weak) { + auto Name = ES.intern(Sym->getName()); + if (!MR->getSymbols().count(ES.intern(Sym->getName()))) { + JITSymbolFlags SF = JITSymbolFlags::Weak; + if (Sym->getScope() == Scope::Default) + SF |= JITSymbolFlags::Exported; + NewSymbolsToClaim[Name] = SF; + NameToSym.push_back(std::make_pair(std::move(Name), Sym)); + } + } + }; + + for (auto *Sym : G.defined_symbols()) + ProcessSymbol(Sym); + for (auto *Sym : G.absolute_symbols()) + ProcessSymbol(Sym); + + // Attempt to claim all weak defs that we're not already responsible for. + // This cannot fail -- any clashes will just result in rejection of our + // claim, at which point we'll externalize that symbol. + cantFail(MR->defineMaterializing(std::move(NewSymbolsToClaim))); + + for (auto &KV : NameToSym) + if (!MR->getSymbols().count(KV.first)) + G.makeExternal(*KV.second); + + return Error::success(); + } + + Error markResponsibilitySymbolsLive(LinkGraph &G) const { + auto &ES = Layer.getExecutionSession(); + for (auto *Sym : G.defined_symbols()) + if (Sym->hasName() && MR->getSymbols().count(ES.intern(Sym->getName()))) + Sym->setLive(true); + return Error::success(); + } + + Error computeNamedSymbolDependencies(LinkGraph &G) { + auto &ES = MR->getTargetJITDylib().getExecutionSession(); + auto LocalDeps = computeLocalDeps(G); + + // Compute dependencies for symbols defined in the JITLink graph. + for (auto *Sym : G.defined_symbols()) { + + // Skip local symbols: we do not track dependencies for these. + if (Sym->getScope() == Scope::Local) + continue; + assert(Sym->hasName() && + "Defined non-local jitlink::Symbol should have a name"); + + SymbolNameSet ExternalSymDeps, InternalSymDeps; + + // Find internal and external named symbol dependencies. + for (auto &E : Sym->getBlock().edges()) { + auto &TargetSym = E.getTarget(); + + if (TargetSym.getScope() != Scope::Local) { + if (TargetSym.isExternal()) + ExternalSymDeps.insert(ES.intern(TargetSym.getName())); + else if (&TargetSym != Sym) + InternalSymDeps.insert(ES.intern(TargetSym.getName())); + } else { + assert(TargetSym.isDefined() && + "local symbols must be defined"); + auto I = LocalDeps.find(&TargetSym); + if (I != LocalDeps.end()) { + for (auto &S : I->second.External) + ExternalSymDeps.insert(S); + for (auto &S : I->second.Internal) + InternalSymDeps.insert(S); + } + } + } + + if (ExternalSymDeps.empty() && InternalSymDeps.empty()) + continue; + + auto SymName = ES.intern(Sym->getName()); + if (!ExternalSymDeps.empty()) + ExternalNamedSymbolDeps[SymName] = std::move(ExternalSymDeps); + if (!InternalSymDeps.empty()) + InternalNamedSymbolDeps[SymName] = std::move(InternalSymDeps); + } + + for (auto &P : Layer.Plugins) { + auto SyntheticLocalDeps = P->getSyntheticSymbolLocalDependencies(*MR); + if (SyntheticLocalDeps.empty()) + continue; + + for (auto &KV : SyntheticLocalDeps) { + auto &Name = KV.first; + auto &LocalDepsForName = KV.second; + for (auto *Local : LocalDepsForName) { + assert(Local->getScope() == Scope::Local && + "Dependence on non-local symbol"); + auto LocalNamedDepsItr = LocalDeps.find(Local); + if (LocalNamedDepsItr == LocalDeps.end()) + continue; + for (auto &S : LocalNamedDepsItr->second.Internal) + InternalNamedSymbolDeps[Name].insert(S); + for (auto &S : LocalNamedDepsItr->second.External) + ExternalNamedSymbolDeps[Name].insert(S); + } + } + } + + return Error::success(); + } + + LocalSymbolNamedDependenciesMap computeLocalDeps(LinkGraph &G) { + DenseMap<jitlink::Symbol *, DenseSet<jitlink::Symbol *>> DepMap; + + // For all local symbols: + // (1) Add their named dependencies. + // (2) Add them to the worklist for further iteration if they have any + // depend on any other local symbols. + struct WorklistEntry { + WorklistEntry(Symbol *Sym, DenseSet<Symbol *> LocalDeps) + : Sym(Sym), LocalDeps(std::move(LocalDeps)) {} + + Symbol *Sym = nullptr; + DenseSet<Symbol *> LocalDeps; + }; + std::vector<WorklistEntry> Worklist; + for (auto *Sym : G.defined_symbols()) + if (Sym->getScope() == Scope::Local) { + auto &SymNamedDeps = DepMap[Sym]; + DenseSet<Symbol *> LocalDeps; + + for (auto &E : Sym->getBlock().edges()) { + auto &TargetSym = E.getTarget(); + if (TargetSym.getScope() != Scope::Local) + SymNamedDeps.insert(&TargetSym); + else { + assert(TargetSym.isDefined() && + "local symbols must be defined"); + LocalDeps.insert(&TargetSym); + } + } + + if (!LocalDeps.empty()) + Worklist.push_back(WorklistEntry(Sym, std::move(LocalDeps))); + } + + // Loop over all local symbols with local dependencies, propagating + // their respective non-local dependencies. Iterate until we hit a stable + // state. + bool Changed; + do { + Changed = false; + for (auto &WLEntry : Worklist) { + auto *Sym = WLEntry.Sym; + auto &NamedDeps = DepMap[Sym]; + auto &LocalDeps = WLEntry.LocalDeps; + + for (auto *TargetSym : LocalDeps) { + auto I = DepMap.find(TargetSym); + if (I != DepMap.end()) + for (const auto &S : I->second) + Changed |= NamedDeps.insert(S).second; + } + } + } while (Changed); + + // Intern the results to produce a mapping of jitlink::Symbol* to internal + // and external symbol names. + auto &ES = Layer.getExecutionSession(); + LocalSymbolNamedDependenciesMap Result; + for (auto &KV : DepMap) { + auto *Local = KV.first; + assert(Local->getScope() == Scope::Local && + "DepMap keys should all be local symbols"); + auto &LocalNamedDeps = Result[Local]; + for (auto *Named : KV.second) { + assert(Named->getScope() != Scope::Local && + "DepMap values should all be non-local symbol sets"); + if (Named->isExternal()) + LocalNamedDeps.External.insert(ES.intern(Named->getName())); + else + LocalNamedDeps.Internal.insert(ES.intern(Named->getName())); + } + } + + return Result; + } + + void registerDependencies(const SymbolDependenceMap &QueryDeps) { + for (auto &NamedDepsEntry : ExternalNamedSymbolDeps) { + auto &Name = NamedDepsEntry.first; + auto &NameDeps = NamedDepsEntry.second; + SymbolDependenceMap SymbolDeps; + + for (const auto &QueryDepsEntry : QueryDeps) { + JITDylib &SourceJD = *QueryDepsEntry.first; + const SymbolNameSet &Symbols = QueryDepsEntry.second; + auto &DepsForJD = SymbolDeps[&SourceJD]; + + for (const auto &S : Symbols) + if (NameDeps.count(S)) + DepsForJD.insert(S); + + if (DepsForJD.empty()) + SymbolDeps.erase(&SourceJD); + } + + MR->addDependencies(Name, SymbolDeps); + } + } + + ObjectLinkingLayer &Layer; + std::unique_ptr<MaterializationResponsibility> MR; + std::unique_ptr<MemoryBuffer> ObjBuffer; + DenseMap<SymbolStringPtr, SymbolNameSet> ExternalNamedSymbolDeps; + DenseMap<SymbolStringPtr, SymbolNameSet> InternalNamedSymbolDeps; +}; + +ObjectLinkingLayer::Plugin::~Plugin() {} + +ObjectLinkingLayer::ObjectLinkingLayer(ExecutionSession &ES, + JITLinkMemoryManager &MemMgr) + : ObjectLayer(ES), MemMgr(MemMgr) { + ES.registerResourceManager(*this); +} + +ObjectLinkingLayer::ObjectLinkingLayer( + ExecutionSession &ES, std::unique_ptr<JITLinkMemoryManager> MemMgr) + : ObjectLayer(ES), MemMgr(*MemMgr), MemMgrOwnership(std::move(MemMgr)) { + ES.registerResourceManager(*this); +} + +ObjectLinkingLayer::~ObjectLinkingLayer() { + assert(Allocs.empty() && "Layer destroyed with resources still attached"); + getExecutionSession().deregisterResourceManager(*this); +} + +void ObjectLinkingLayer::emit(std::unique_ptr<MaterializationResponsibility> R, + std::unique_ptr<MemoryBuffer> O) { + assert(O && "Object must not be null"); + auto ObjBuffer = O->getMemBufferRef(); + auto Ctx = std::make_unique<ObjectLinkingLayerJITLinkContext>( + *this, std::move(R), std::move(O)); + if (auto G = createLinkGraphFromObject(std::move(ObjBuffer))) + link(std::move(*G), std::move(Ctx)); + else + Ctx->notifyFailed(G.takeError()); +} + +void ObjectLinkingLayer::emit(std::unique_ptr<MaterializationResponsibility> R, + std::unique_ptr<LinkGraph> G) { + link(std::move(G), std::make_unique<ObjectLinkingLayerJITLinkContext>( + *this, std::move(R), nullptr)); +} + +void ObjectLinkingLayer::modifyPassConfig(MaterializationResponsibility &MR, + const Triple &TT, + PassConfiguration &PassConfig) { + for (auto &P : Plugins) + P->modifyPassConfig(MR, TT, PassConfig); +} + +void ObjectLinkingLayer::notifyLoaded(MaterializationResponsibility &MR) { + for (auto &P : Plugins) + P->notifyLoaded(MR); +} + +Error ObjectLinkingLayer::notifyEmitted(MaterializationResponsibility &MR, + AllocPtr Alloc) { + Error Err = Error::success(); + for (auto &P : Plugins) + Err = joinErrors(std::move(Err), P->notifyEmitted(MR)); + + if (Err) + return Err; + + return MR.withResourceKeyDo( + [&](ResourceKey K) { Allocs[K].push_back(std::move(Alloc)); }); +} + +Error ObjectLinkingLayer::handleRemoveResources(ResourceKey K) { + + Error Err = Error::success(); + + for (auto &P : Plugins) + Err = joinErrors(std::move(Err), P->notifyRemovingResources(K)); + + std::vector<AllocPtr> AllocsToRemove; + getExecutionSession().runSessionLocked([&] { + auto I = Allocs.find(K); + if (I != Allocs.end()) { + std::swap(AllocsToRemove, I->second); + Allocs.erase(I); + } + }); + + while (!AllocsToRemove.empty()) { + Err = joinErrors(std::move(Err), AllocsToRemove.back()->deallocate()); + AllocsToRemove.pop_back(); + } + + return Err; +} + +void ObjectLinkingLayer::handleTransferResources(ResourceKey DstKey, + ResourceKey SrcKey) { + auto I = Allocs.find(SrcKey); + if (I != Allocs.end()) { + auto &SrcAllocs = I->second; + auto &DstAllocs = Allocs[DstKey]; + DstAllocs.reserve(DstAllocs.size() + SrcAllocs.size()); + for (auto &Alloc : SrcAllocs) + DstAllocs.push_back(std::move(Alloc)); + + // Erase SrcKey entry using value rather than iterator I: I may have been + // invalidated when we looked up DstKey. + Allocs.erase(SrcKey); + } + + for (auto &P : Plugins) + P->notifyTransferringResources(DstKey, SrcKey); +} + +EHFrameRegistrationPlugin::EHFrameRegistrationPlugin( + ExecutionSession &ES, std::unique_ptr<EHFrameRegistrar> Registrar) + : ES(ES), Registrar(std::move(Registrar)) {} + +void EHFrameRegistrationPlugin::modifyPassConfig( + MaterializationResponsibility &MR, const Triple &TT, + PassConfiguration &PassConfig) { + + PassConfig.PostFixupPasses.push_back(createEHFrameRecorderPass( + TT, [this, &MR](JITTargetAddress Addr, size_t Size) { + if (Addr) { + std::lock_guard<std::mutex> Lock(EHFramePluginMutex); + assert(!InProcessLinks.count(&MR) && + "Link for MR already being tracked?"); + InProcessLinks[&MR] = {Addr, Size}; + } + })); +} + +Error EHFrameRegistrationPlugin::notifyEmitted( + MaterializationResponsibility &MR) { + + EHFrameRange EmittedRange; + { + std::lock_guard<std::mutex> Lock(EHFramePluginMutex); + + auto EHFrameRangeItr = InProcessLinks.find(&MR); + if (EHFrameRangeItr == InProcessLinks.end()) + return Error::success(); + + EmittedRange = EHFrameRangeItr->second; + assert(EmittedRange.Addr && "eh-frame addr to register can not be null"); + InProcessLinks.erase(EHFrameRangeItr); + } + + if (auto Err = MR.withResourceKeyDo( + [&](ResourceKey K) { EHFrameRanges[K].push_back(EmittedRange); })) + return Err; + + return Registrar->registerEHFrames(EmittedRange.Addr, EmittedRange.Size); +} + +Error EHFrameRegistrationPlugin::notifyFailed( + MaterializationResponsibility &MR) { + std::lock_guard<std::mutex> Lock(EHFramePluginMutex); + InProcessLinks.erase(&MR); + return Error::success(); +} + +Error EHFrameRegistrationPlugin::notifyRemovingResources(ResourceKey K) { + std::vector<EHFrameRange> RangesToRemove; + + ES.runSessionLocked([&] { + auto I = EHFrameRanges.find(K); + if (I != EHFrameRanges.end()) { + RangesToRemove = std::move(I->second); + EHFrameRanges.erase(I); + } + }); + + Error Err = Error::success(); + while (!RangesToRemove.empty()) { + auto RangeToRemove = RangesToRemove.back(); + RangesToRemove.pop_back(); + assert(RangeToRemove.Addr && "Untracked eh-frame range must not be null"); + Err = joinErrors( + std::move(Err), + Registrar->deregisterEHFrames(RangeToRemove.Addr, RangeToRemove.Size)); + } + + return Err; +} + +void EHFrameRegistrationPlugin::notifyTransferringResources( + ResourceKey DstKey, ResourceKey SrcKey) { + auto SI = EHFrameRanges.find(SrcKey); + if (SI != EHFrameRanges.end()) { + auto &SrcRanges = SI->second; + auto &DstRanges = EHFrameRanges[DstKey]; + DstRanges.reserve(DstRanges.size() + SrcRanges.size()); + for (auto &SrcRange : SrcRanges) + DstRanges.push_back(std::move(SrcRange)); + EHFrameRanges.erase(SI); + } +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/Orc/ObjectTransformLayer.cpp b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/ObjectTransformLayer.cpp new file mode 100644 index 0000000000..a57662e10a --- /dev/null +++ b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/ObjectTransformLayer.cpp @@ -0,0 +1,40 @@ +//===---------- ObjectTransformLayer.cpp - Object Transform Layer ---------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/ObjectTransformLayer.h" +#include "llvm/Support/MemoryBuffer.h" + +namespace llvm { +namespace orc { + +ObjectTransformLayer::ObjectTransformLayer(ExecutionSession &ES, + ObjectLayer &BaseLayer, + TransformFunction Transform) + : ObjectLayer(ES), BaseLayer(BaseLayer), Transform(std::move(Transform)) {} + +void ObjectTransformLayer::emit( + std::unique_ptr<MaterializationResponsibility> R, + std::unique_ptr<MemoryBuffer> O) { + assert(O && "Module must not be null"); + + // If there is a transform set then apply it. + if (Transform) { + if (auto TransformedObj = Transform(std::move(O))) + O = std::move(*TransformedObj); + else { + R->failMaterialization(); + getExecutionSession().reportError(TransformedObj.takeError()); + return; + } + } + + BaseLayer.emit(std::move(R), std::move(O)); +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/Orc/OrcABISupport.cpp b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/OrcABISupport.cpp new file mode 100644 index 0000000000..18b3c5e12b --- /dev/null +++ b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/OrcABISupport.cpp @@ -0,0 +1,910 @@ +//===------------- OrcABISupport.cpp - ABI specific support code ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/OrcABISupport.h" +#include "llvm/Support/FormatVariadic.h" +#include "llvm/Support/Process.h" +#include "llvm/Support/raw_ostream.h" + +#define DEBUG_TYPE "orc" + +using namespace llvm; + +template <typename ORCABI> +bool stubAndPointerRangesOk(JITTargetAddress StubBlockAddr, + JITTargetAddress PointerBlockAddr, + unsigned NumStubs) { + constexpr unsigned MaxDisp = ORCABI::StubToPointerMaxDisplacement; + JITTargetAddress FirstStub = StubBlockAddr; + JITTargetAddress LastStub = FirstStub + ((NumStubs - 1) * ORCABI::StubSize); + JITTargetAddress FirstPointer = PointerBlockAddr; + JITTargetAddress LastPointer = + FirstPointer + ((NumStubs - 1) * ORCABI::StubSize); + + if (FirstStub < FirstPointer) { + if (LastStub >= FirstPointer) + return false; // Ranges overlap. + return (FirstPointer - FirstStub <= MaxDisp) && + (LastPointer - LastStub <= MaxDisp); // out-of-range. + } + + if (LastPointer >= FirstStub) + return false; // Ranges overlap. + + return (FirstStub - FirstPointer <= MaxDisp) && + (LastStub - LastPointer <= MaxDisp); +} + +namespace llvm { +namespace orc { + +void OrcAArch64::writeResolverCode(char *ResolverWorkingMem, + JITTargetAddress ResolverTargetAddress, + JITTargetAddress ReentryFnAddr, + JITTargetAddress ReentryCtxAddr) { + + const uint32_t ResolverCode[] = { + // resolver_entry: + 0xa9bf47fd, // 0x000: stp x29, x17, [sp, #-16]! + 0x910003fd, // 0x004: mov x29, sp + 0xa9bf73fb, // 0x008: stp x27, x28, [sp, #-16]! + 0xa9bf6bf9, // 0x00c: stp x25, x26, [sp, #-16]! + 0xa9bf63f7, // 0x010: stp x23, x24, [sp, #-16]! + 0xa9bf5bf5, // 0x014: stp x21, x22, [sp, #-16]! + 0xa9bf53f3, // 0x018: stp x19, x20, [sp, #-16]! + 0xa9bf3fee, // 0x01c: stp x14, x15, [sp, #-16]! + 0xa9bf37ec, // 0x020: stp x12, x13, [sp, #-16]! + 0xa9bf2fea, // 0x024: stp x10, x11, [sp, #-16]! + 0xa9bf27e8, // 0x028: stp x8, x9, [sp, #-16]! + 0xa9bf1fe6, // 0x02c: stp x6, x7, [sp, #-16]! + 0xa9bf17e4, // 0x030: stp x4, x5, [sp, #-16]! + 0xa9bf0fe2, // 0x034: stp x2, x3, [sp, #-16]! + 0xa9bf07e0, // 0x038: stp x0, x1, [sp, #-16]! + 0xadbf7ffe, // 0x03c: stp q30, q31, [sp, #-32]! + 0xadbf77fc, // 0x040: stp q28, q29, [sp, #-32]! + 0xadbf6ffa, // 0x044: stp q26, q27, [sp, #-32]! + 0xadbf67f8, // 0x048: stp q24, q25, [sp, #-32]! + 0xadbf5ff6, // 0x04c: stp q22, q23, [sp, #-32]! + 0xadbf57f4, // 0x050: stp q20, q21, [sp, #-32]! + 0xadbf4ff2, // 0x054: stp q18, q19, [sp, #-32]! + 0xadbf47f0, // 0x058: stp q16, q17, [sp, #-32]! + 0xadbf3fee, // 0x05c: stp q14, q15, [sp, #-32]! + 0xadbf37ec, // 0x060: stp q12, q13, [sp, #-32]! + 0xadbf2fea, // 0x064: stp q10, q11, [sp, #-32]! + 0xadbf27e8, // 0x068: stp q8, q9, [sp, #-32]! + 0xadbf1fe6, // 0x06c: stp q6, q7, [sp, #-32]! + 0xadbf17e4, // 0x070: stp q4, q5, [sp, #-32]! + 0xadbf0fe2, // 0x074: stp q2, q3, [sp, #-32]! + 0xadbf07e0, // 0x078: stp q0, q1, [sp, #-32]! + 0x580004e0, // 0x07c: ldr x0, Lreentry_ctx_ptr + 0xaa1e03e1, // 0x080: mov x1, x30 + 0xd1003021, // 0x084: sub x1, x1, #12 + 0x58000442, // 0x088: ldr x2, Lreentry_fn_ptr + 0xd63f0040, // 0x08c: blr x2 + 0xaa0003f1, // 0x090: mov x17, x0 + 0xacc107e0, // 0x094: ldp q0, q1, [sp], #32 + 0xacc10fe2, // 0x098: ldp q2, q3, [sp], #32 + 0xacc117e4, // 0x09c: ldp q4, q5, [sp], #32 + 0xacc11fe6, // 0x0a0: ldp q6, q7, [sp], #32 + 0xacc127e8, // 0x0a4: ldp q8, q9, [sp], #32 + 0xacc12fea, // 0x0a8: ldp q10, q11, [sp], #32 + 0xacc137ec, // 0x0ac: ldp q12, q13, [sp], #32 + 0xacc13fee, // 0x0b0: ldp q14, q15, [sp], #32 + 0xacc147f0, // 0x0b4: ldp q16, q17, [sp], #32 + 0xacc14ff2, // 0x0b8: ldp q18, q19, [sp], #32 + 0xacc157f4, // 0x0bc: ldp q20, q21, [sp], #32 + 0xacc15ff6, // 0x0c0: ldp q22, q23, [sp], #32 + 0xacc167f8, // 0x0c4: ldp q24, q25, [sp], #32 + 0xacc16ffa, // 0x0c8: ldp q26, q27, [sp], #32 + 0xacc177fc, // 0x0cc: ldp q28, q29, [sp], #32 + 0xacc17ffe, // 0x0d0: ldp q30, q31, [sp], #32 + 0xa8c107e0, // 0x0d4: ldp x0, x1, [sp], #16 + 0xa8c10fe2, // 0x0d8: ldp x2, x3, [sp], #16 + 0xa8c117e4, // 0x0dc: ldp x4, x5, [sp], #16 + 0xa8c11fe6, // 0x0e0: ldp x6, x7, [sp], #16 + 0xa8c127e8, // 0x0e4: ldp x8, x9, [sp], #16 + 0xa8c12fea, // 0x0e8: ldp x10, x11, [sp], #16 + 0xa8c137ec, // 0x0ec: ldp x12, x13, [sp], #16 + 0xa8c13fee, // 0x0f0: ldp x14, x15, [sp], #16 + 0xa8c153f3, // 0x0f4: ldp x19, x20, [sp], #16 + 0xa8c15bf5, // 0x0f8: ldp x21, x22, [sp], #16 + 0xa8c163f7, // 0x0fc: ldp x23, x24, [sp], #16 + 0xa8c16bf9, // 0x100: ldp x25, x26, [sp], #16 + 0xa8c173fb, // 0x104: ldp x27, x28, [sp], #16 + 0xa8c17bfd, // 0x108: ldp x29, x30, [sp], #16 + 0xd65f0220, // 0x10c: ret x17 + 0x01234567, // 0x110: Lreentry_fn_ptr: + 0xdeadbeef, // 0x114: .quad 0 + 0x98765432, // 0x118: Lreentry_ctx_ptr: + 0xcafef00d // 0x11c: .quad 0 + }; + + const unsigned ReentryFnAddrOffset = 0x110; + const unsigned ReentryCtxAddrOffset = 0x118; + + memcpy(ResolverWorkingMem, ResolverCode, sizeof(ResolverCode)); + memcpy(ResolverWorkingMem + ReentryFnAddrOffset, &ReentryFnAddr, + sizeof(uint64_t)); + memcpy(ResolverWorkingMem + ReentryCtxAddrOffset, &ReentryCtxAddr, + sizeof(uint64_t)); +} + +void OrcAArch64::writeTrampolines(char *TrampolineBlockWorkingMem, + JITTargetAddress TrampolineBlockTargetAddress, + JITTargetAddress ResolverAddr, + unsigned NumTrampolines) { + + unsigned OffsetToPtr = alignTo(NumTrampolines * TrampolineSize, 8); + + memcpy(TrampolineBlockWorkingMem + OffsetToPtr, &ResolverAddr, + sizeof(uint64_t)); + + // OffsetToPtr is actually the offset from the PC for the 2nd instruction, so + // subtract 32-bits. + OffsetToPtr -= 4; + + uint32_t *Trampolines = + reinterpret_cast<uint32_t *>(TrampolineBlockWorkingMem); + + for (unsigned I = 0; I < NumTrampolines; ++I, OffsetToPtr -= TrampolineSize) { + Trampolines[3 * I + 0] = 0xaa1e03f1; // mov x17, x30 + Trampolines[3 * I + 1] = 0x58000010 | (OffsetToPtr << 3); // adr x16, Lptr + Trampolines[3 * I + 2] = 0xd63f0200; // blr x16 + } +} + +void OrcAArch64::writeIndirectStubsBlock( + char *StubsBlockWorkingMem, JITTargetAddress StubsBlockTargetAddress, + JITTargetAddress PointersBlockTargetAddress, unsigned NumStubs) { + // Stub format is: + // + // .section __orc_stubs + // stub1: + // ldr x0, ptr1 ; PC-rel load of ptr1 + // br x0 ; Jump to resolver + // stub2: + // ldr x0, ptr2 ; PC-rel load of ptr2 + // br x0 ; Jump to resolver + // + // ... + // + // .section __orc_ptrs + // ptr1: + // .quad 0x0 + // ptr2: + // .quad 0x0 + // + // ... + + static_assert(StubSize == PointerSize, + "Pointer and stub size must match for algorithm below"); + assert(stubAndPointerRangesOk<OrcAArch64>( + StubsBlockTargetAddress, PointersBlockTargetAddress, NumStubs) && + "PointersBlock is out of range"); + uint64_t PtrDisplacement = + PointersBlockTargetAddress - StubsBlockTargetAddress; + uint64_t *Stub = reinterpret_cast<uint64_t *>(StubsBlockWorkingMem); + uint64_t PtrOffsetField = PtrDisplacement << 3; + + for (unsigned I = 0; I < NumStubs; ++I) + Stub[I] = 0xd61f020058000010 | PtrOffsetField; +} + +void OrcX86_64_Base::writeTrampolines( + char *TrampolineBlockWorkingMem, + JITTargetAddress TrampolineBlockTargetAddress, + JITTargetAddress ResolverAddr, unsigned NumTrampolines) { + + unsigned OffsetToPtr = NumTrampolines * TrampolineSize; + + memcpy(TrampolineBlockWorkingMem + OffsetToPtr, &ResolverAddr, + sizeof(uint64_t)); + + uint64_t *Trampolines = + reinterpret_cast<uint64_t *>(TrampolineBlockWorkingMem); + uint64_t CallIndirPCRel = 0xf1c40000000015ff; + + for (unsigned I = 0; I < NumTrampolines; ++I, OffsetToPtr -= TrampolineSize) + Trampolines[I] = CallIndirPCRel | ((OffsetToPtr - 6) << 16); +} + +void OrcX86_64_Base::writeIndirectStubsBlock( + char *StubsBlockWorkingMem, JITTargetAddress StubsBlockTargetAddress, + JITTargetAddress PointersBlockTargetAddress, unsigned NumStubs) { + // Stub format is: + // + // .section __orc_stubs + // stub1: + // jmpq *ptr1(%rip) + // .byte 0xC4 ; <- Invalid opcode padding. + // .byte 0xF1 + // stub2: + // jmpq *ptr2(%rip) + // + // ... + // + // .section __orc_ptrs + // ptr1: + // .quad 0x0 + // ptr2: + // .quad 0x0 + // + // ... + + // Populate the stubs page stubs and mark it executable. + static_assert(StubSize == PointerSize, + "Pointer and stub size must match for algorithm below"); + assert(stubAndPointerRangesOk<OrcX86_64_Base>( + StubsBlockTargetAddress, PointersBlockTargetAddress, NumStubs) && + "PointersBlock is out of range"); + uint64_t *Stub = reinterpret_cast<uint64_t *>(StubsBlockWorkingMem); + uint64_t PtrOffsetField = + (PointersBlockTargetAddress - StubsBlockTargetAddress - 6) << 16; + for (unsigned I = 0; I < NumStubs; ++I) + Stub[I] = 0xF1C40000000025ff | PtrOffsetField; +} + +void OrcX86_64_SysV::writeResolverCode(char *ResolverWorkingMem, + JITTargetAddress ResolverTargetAddress, + JITTargetAddress ReentryFnAddr, + JITTargetAddress ReentryCtxAddr) { + + LLVM_DEBUG({ + dbgs() << "Writing resolver code to " + << formatv("{0:x16}", ResolverTargetAddress) << "\n"; + }); + + const uint8_t ResolverCode[] = { + // resolver_entry: + 0x55, // 0x00: pushq %rbp + 0x48, 0x89, 0xe5, // 0x01: movq %rsp, %rbp + 0x50, // 0x04: pushq %rax + 0x53, // 0x05: pushq %rbx + 0x51, // 0x06: pushq %rcx + 0x52, // 0x07: pushq %rdx + 0x56, // 0x08: pushq %rsi + 0x57, // 0x09: pushq %rdi + 0x41, 0x50, // 0x0a: pushq %r8 + 0x41, 0x51, // 0x0c: pushq %r9 + 0x41, 0x52, // 0x0e: pushq %r10 + 0x41, 0x53, // 0x10: pushq %r11 + 0x41, 0x54, // 0x12: pushq %r12 + 0x41, 0x55, // 0x14: pushq %r13 + 0x41, 0x56, // 0x16: pushq %r14 + 0x41, 0x57, // 0x18: pushq %r15 + 0x48, 0x81, 0xec, 0x08, 0x02, 0x00, 0x00, // 0x1a: subq 0x208, %rsp + 0x48, 0x0f, 0xae, 0x04, 0x24, // 0x21: fxsave64 (%rsp) + 0x48, 0xbf, // 0x26: movabsq <CBMgr>, %rdi + + // 0x28: JIT re-entry ctx addr. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + + 0x48, 0x8b, 0x75, 0x08, // 0x30: movq 8(%rbp), %rsi + 0x48, 0x83, 0xee, 0x06, // 0x34: subq $6, %rsi + 0x48, 0xb8, // 0x38: movabsq <REntry>, %rax + + // 0x3a: JIT re-entry fn addr: + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + + 0xff, 0xd0, // 0x42: callq *%rax + 0x48, 0x89, 0x45, 0x08, // 0x44: movq %rax, 8(%rbp) + 0x48, 0x0f, 0xae, 0x0c, 0x24, // 0x48: fxrstor64 (%rsp) + 0x48, 0x81, 0xc4, 0x08, 0x02, 0x00, 0x00, // 0x4d: addq 0x208, %rsp + 0x41, 0x5f, // 0x54: popq %r15 + 0x41, 0x5e, // 0x56: popq %r14 + 0x41, 0x5d, // 0x58: popq %r13 + 0x41, 0x5c, // 0x5a: popq %r12 + 0x41, 0x5b, // 0x5c: popq %r11 + 0x41, 0x5a, // 0x5e: popq %r10 + 0x41, 0x59, // 0x60: popq %r9 + 0x41, 0x58, // 0x62: popq %r8 + 0x5f, // 0x64: popq %rdi + 0x5e, // 0x65: popq %rsi + 0x5a, // 0x66: popq %rdx + 0x59, // 0x67: popq %rcx + 0x5b, // 0x68: popq %rbx + 0x58, // 0x69: popq %rax + 0x5d, // 0x6a: popq %rbp + 0xc3, // 0x6b: retq + }; + + const unsigned ReentryFnAddrOffset = 0x3a; + const unsigned ReentryCtxAddrOffset = 0x28; + + memcpy(ResolverWorkingMem, ResolverCode, sizeof(ResolverCode)); + memcpy(ResolverWorkingMem + ReentryFnAddrOffset, &ReentryFnAddr, + sizeof(uint64_t)); + memcpy(ResolverWorkingMem + ReentryCtxAddrOffset, &ReentryCtxAddr, + sizeof(uint64_t)); +} + +void OrcX86_64_Win32::writeResolverCode(char *ResolverWorkingMem, + JITTargetAddress ResolverTargetAddress, + JITTargetAddress ReentryFnAddr, + JITTargetAddress ReentryCtxAddr) { + + // resolverCode is similar to OrcX86_64 with differences specific to windows + // x64 calling convention: arguments go into rcx, rdx and come in reverse + // order, shadow space allocation on stack + const uint8_t ResolverCode[] = { + // resolver_entry: + 0x55, // 0x00: pushq %rbp + 0x48, 0x89, 0xe5, // 0x01: movq %rsp, %rbp + 0x50, // 0x04: pushq %rax + 0x53, // 0x05: pushq %rbx + 0x51, // 0x06: pushq %rcx + 0x52, // 0x07: pushq %rdx + 0x56, // 0x08: pushq %rsi + 0x57, // 0x09: pushq %rdi + 0x41, 0x50, // 0x0a: pushq %r8 + 0x41, 0x51, // 0x0c: pushq %r9 + 0x41, 0x52, // 0x0e: pushq %r10 + 0x41, 0x53, // 0x10: pushq %r11 + 0x41, 0x54, // 0x12: pushq %r12 + 0x41, 0x55, // 0x14: pushq %r13 + 0x41, 0x56, // 0x16: pushq %r14 + 0x41, 0x57, // 0x18: pushq %r15 + 0x48, 0x81, 0xec, 0x08, 0x02, 0x00, 0x00, // 0x1a: subq 0x208, %rsp + 0x48, 0x0f, 0xae, 0x04, 0x24, // 0x21: fxsave64 (%rsp) + + 0x48, 0xb9, // 0x26: movabsq <CBMgr>, %rcx + // 0x28: JIT re-entry ctx addr. + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + + 0x48, 0x8B, 0x55, 0x08, // 0x30: mov rdx, [rbp+0x8] + 0x48, 0x83, 0xea, 0x06, // 0x34: sub rdx, 0x6 + + 0x48, 0xb8, // 0x38: movabsq <REntry>, %rax + // 0x3a: JIT re-entry fn addr: + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + + // 0x42: sub rsp, 0x20 (Allocate shadow space) + 0x48, 0x83, 0xEC, 0x20, + 0xff, 0xd0, // 0x46: callq *%rax + + // 0x48: add rsp, 0x20 (Free shadow space) + 0x48, 0x83, 0xC4, 0x20, + + 0x48, 0x89, 0x45, 0x08, // 0x4C: movq %rax, 8(%rbp) + 0x48, 0x0f, 0xae, 0x0c, 0x24, // 0x50: fxrstor64 (%rsp) + 0x48, 0x81, 0xc4, 0x08, 0x02, 0x00, 0x00, // 0x55: addq 0x208, %rsp + 0x41, 0x5f, // 0x5C: popq %r15 + 0x41, 0x5e, // 0x5E: popq %r14 + 0x41, 0x5d, // 0x60: popq %r13 + 0x41, 0x5c, // 0x62: popq %r12 + 0x41, 0x5b, // 0x64: popq %r11 + 0x41, 0x5a, // 0x66: popq %r10 + 0x41, 0x59, // 0x68: popq %r9 + 0x41, 0x58, // 0x6a: popq %r8 + 0x5f, // 0x6c: popq %rdi + 0x5e, // 0x6d: popq %rsi + 0x5a, // 0x6e: popq %rdx + 0x59, // 0x6f: popq %rcx + 0x5b, // 0x70: popq %rbx + 0x58, // 0x71: popq %rax + 0x5d, // 0x72: popq %rbp + 0xc3, // 0x73: retq + }; + + const unsigned ReentryFnAddrOffset = 0x3a; + const unsigned ReentryCtxAddrOffset = 0x28; + + memcpy(ResolverWorkingMem, ResolverCode, sizeof(ResolverCode)); + memcpy(ResolverWorkingMem + ReentryFnAddrOffset, &ReentryFnAddr, + sizeof(uint64_t)); + memcpy(ResolverWorkingMem + ReentryCtxAddrOffset, &ReentryCtxAddr, + sizeof(uint64_t)); +} + +void OrcI386::writeResolverCode(char *ResolverWorkingMem, + JITTargetAddress ResolverTargetAddress, + JITTargetAddress ReentryFnAddr, + JITTargetAddress ReentryCtxAddr) { + + assert((ReentryFnAddr >> 32) == 0 && "ReentryFnAddr out of range"); + assert((ReentryCtxAddr >> 32) == 0 && "ReentryCtxAddr out of range"); + + const uint8_t ResolverCode[] = { + // resolver_entry: + 0x55, // 0x00: pushl %ebp + 0x89, 0xe5, // 0x01: movl %esp, %ebp + 0x54, // 0x03: pushl %esp + 0x83, 0xe4, 0xf0, // 0x04: andl $-0x10, %esp + 0x50, // 0x07: pushl %eax + 0x53, // 0x08: pushl %ebx + 0x51, // 0x09: pushl %ecx + 0x52, // 0x0a: pushl %edx + 0x56, // 0x0b: pushl %esi + 0x57, // 0x0c: pushl %edi + 0x81, 0xec, 0x18, 0x02, 0x00, 0x00, // 0x0d: subl $0x218, %esp + 0x0f, 0xae, 0x44, 0x24, 0x10, // 0x13: fxsave 0x10(%esp) + 0x8b, 0x75, 0x04, // 0x18: movl 0x4(%ebp), %esi + 0x83, 0xee, 0x05, // 0x1b: subl $0x5, %esi + 0x89, 0x74, 0x24, 0x04, // 0x1e: movl %esi, 0x4(%esp) + 0xc7, 0x04, 0x24, 0x00, 0x00, 0x00, + 0x00, // 0x22: movl <cbmgr>, (%esp) + 0xb8, 0x00, 0x00, 0x00, 0x00, // 0x29: movl <reentry>, %eax + 0xff, 0xd0, // 0x2e: calll *%eax + 0x89, 0x45, 0x04, // 0x30: movl %eax, 0x4(%ebp) + 0x0f, 0xae, 0x4c, 0x24, 0x10, // 0x33: fxrstor 0x10(%esp) + 0x81, 0xc4, 0x18, 0x02, 0x00, 0x00, // 0x38: addl $0x218, %esp + 0x5f, // 0x3e: popl %edi + 0x5e, // 0x3f: popl %esi + 0x5a, // 0x40: popl %edx + 0x59, // 0x41: popl %ecx + 0x5b, // 0x42: popl %ebx + 0x58, // 0x43: popl %eax + 0x8b, 0x65, 0xfc, // 0x44: movl -0x4(%ebp), %esp + 0x5d, // 0x48: popl %ebp + 0xc3 // 0x49: retl + }; + + const unsigned ReentryFnAddrOffset = 0x2a; + const unsigned ReentryCtxAddrOffset = 0x25; + + memcpy(ResolverWorkingMem, ResolverCode, sizeof(ResolverCode)); + memcpy(ResolverWorkingMem + ReentryFnAddrOffset, &ReentryFnAddr, + sizeof(uint32_t)); + memcpy(ResolverWorkingMem + ReentryCtxAddrOffset, &ReentryCtxAddr, + sizeof(uint32_t)); +} + +void OrcI386::writeTrampolines(char *TrampolineWorkingMem, + JITTargetAddress TrampolineBlockTargetAddress, + JITTargetAddress ResolverAddr, + unsigned NumTrampolines) { + assert((ResolverAddr >> 32) == 0 && "ResolverAddr out of range"); + + uint64_t CallRelImm = 0xF1C4C400000000e8; + uint64_t ResolverRel = ResolverAddr - TrampolineBlockTargetAddress - 5; + + uint64_t *Trampolines = reinterpret_cast<uint64_t *>(TrampolineWorkingMem); + for (unsigned I = 0; I < NumTrampolines; ++I, ResolverRel -= TrampolineSize) + Trampolines[I] = CallRelImm | (ResolverRel << 8); +} + +void OrcI386::writeIndirectStubsBlock( + char *StubsBlockWorkingMem, JITTargetAddress StubsBlockTargetAddress, + JITTargetAddress PointersBlockTargetAddress, unsigned NumStubs) { + assert((StubsBlockTargetAddress >> 32) == 0 && + "StubsBlockTargetAddress is out of range"); + assert((PointersBlockTargetAddress >> 32) == 0 && + "PointersBlockTargetAddress is out of range"); + + // Stub format is: + // + // .section __orc_stubs + // stub1: + // jmpq *ptr1 + // .byte 0xC4 ; <- Invalid opcode padding. + // .byte 0xF1 + // stub2: + // jmpq *ptr2 + // + // ... + // + // .section __orc_ptrs + // ptr1: + // .quad 0x0 + // ptr2: + // .quad 0x0 + // + // ... + + assert(stubAndPointerRangesOk<OrcI386>( + StubsBlockTargetAddress, PointersBlockTargetAddress, NumStubs) && + "PointersBlock is out of range"); + + uint64_t *Stub = reinterpret_cast<uint64_t *>(StubsBlockWorkingMem); + uint64_t PtrAddr = PointersBlockTargetAddress; + for (unsigned I = 0; I < NumStubs; ++I, PtrAddr += 4) + Stub[I] = 0xF1C40000000025ff | (PtrAddr << 16); +} + +void OrcMips32_Base::writeResolverCode(char *ResolverWorkingMem, + JITTargetAddress ResolverTargetAddress, + JITTargetAddress ReentryFnAddr, + JITTargetAddress ReentryCtxAddr, + bool isBigEndian) { + + const uint32_t ResolverCode[] = { + // resolver_entry: + 0x27bdff98, // 0x00: addiu $sp,$sp,-104 + 0xafa20000, // 0x04: sw $v0,0($sp) + 0xafa30004, // 0x08: sw $v1,4($sp) + 0xafa40008, // 0x0c: sw $a0,8($sp) + 0xafa5000c, // 0x10: sw $a1,12($sp) + 0xafa60010, // 0x14: sw $a2,16($sp) + 0xafa70014, // 0x18: sw $a3,20($sp) + 0xafb00018, // 0x1c: sw $s0,24($sp) + 0xafb1001c, // 0x20: sw $s1,28($sp) + 0xafb20020, // 0x24: sw $s2,32($sp) + 0xafb30024, // 0x28: sw $s3,36($sp) + 0xafb40028, // 0x2c: sw $s4,40($sp) + 0xafb5002c, // 0x30: sw $s5,44($sp) + 0xafb60030, // 0x34: sw $s6,48($sp) + 0xafb70034, // 0x38: sw $s7,52($sp) + 0xafa80038, // 0x3c: sw $t0,56($sp) + 0xafa9003c, // 0x40: sw $t1,60($sp) + 0xafaa0040, // 0x44: sw $t2,64($sp) + 0xafab0044, // 0x48: sw $t3,68($sp) + 0xafac0048, // 0x4c: sw $t4,72($sp) + 0xafad004c, // 0x50: sw $t5,76($sp) + 0xafae0050, // 0x54: sw $t6,80($sp) + 0xafaf0054, // 0x58: sw $t7,84($sp) + 0xafb80058, // 0x5c: sw $t8,88($sp) + 0xafb9005c, // 0x60: sw $t9,92($sp) + 0xafbe0060, // 0x64: sw $fp,96($sp) + 0xafbf0064, // 0x68: sw $ra,100($sp) + + // JIT re-entry ctx addr. + 0x00000000, // 0x6c: lui $a0,ctx + 0x00000000, // 0x70: addiu $a0,$a0,ctx + + 0x03e02825, // 0x74: move $a1, $ra + 0x24a5ffec, // 0x78: addiu $a1,$a1,-20 + + // JIT re-entry fn addr: + 0x00000000, // 0x7c: lui $t9,reentry + 0x00000000, // 0x80: addiu $t9,$t9,reentry + + 0x0320f809, // 0x84: jalr $t9 + 0x00000000, // 0x88: nop + 0x8fbf0064, // 0x8c: lw $ra,100($sp) + 0x8fbe0060, // 0x90: lw $fp,96($sp) + 0x8fb9005c, // 0x94: lw $t9,92($sp) + 0x8fb80058, // 0x98: lw $t8,88($sp) + 0x8faf0054, // 0x9c: lw $t7,84($sp) + 0x8fae0050, // 0xa0: lw $t6,80($sp) + 0x8fad004c, // 0xa4: lw $t5,76($sp) + 0x8fac0048, // 0xa8: lw $t4,72($sp) + 0x8fab0044, // 0xac: lw $t3,68($sp) + 0x8faa0040, // 0xb0: lw $t2,64($sp) + 0x8fa9003c, // 0xb4: lw $t1,60($sp) + 0x8fa80038, // 0xb8: lw $t0,56($sp) + 0x8fb70034, // 0xbc: lw $s7,52($sp) + 0x8fb60030, // 0xc0: lw $s6,48($sp) + 0x8fb5002c, // 0xc4: lw $s5,44($sp) + 0x8fb40028, // 0xc8: lw $s4,40($sp) + 0x8fb30024, // 0xcc: lw $s3,36($sp) + 0x8fb20020, // 0xd0: lw $s2,32($sp) + 0x8fb1001c, // 0xd4: lw $s1,28($sp) + 0x8fb00018, // 0xd8: lw $s0,24($sp) + 0x8fa70014, // 0xdc: lw $a3,20($sp) + 0x8fa60010, // 0xe0: lw $a2,16($sp) + 0x8fa5000c, // 0xe4: lw $a1,12($sp) + 0x8fa40008, // 0xe8: lw $a0,8($sp) + 0x27bd0068, // 0xec: addiu $sp,$sp,104 + 0x0300f825, // 0xf0: move $ra, $t8 + 0x03200008, // 0xf4: jr $t9 + 0x00000000, // 0xf8: move $t9, $v0/v1 + }; + + const unsigned ReentryFnAddrOffset = 0x7c; // JIT re-entry fn addr lui + const unsigned ReentryCtxAddrOffset = 0x6c; // JIT re-entry context addr lui + const unsigned Offsett = 0xf8; + + memcpy(ResolverWorkingMem, ResolverCode, sizeof(ResolverCode)); + + // Depending on endian return value will be in v0 or v1. + uint32_t MoveVxT9 = isBigEndian ? 0x0060c825 : 0x0040c825; + memcpy(ResolverWorkingMem + Offsett, &MoveVxT9, sizeof(MoveVxT9)); + + uint32_t ReentryCtxLUi = + 0x3c040000 | (((ReentryCtxAddr + 0x8000) >> 16) & 0xFFFF); + uint32_t ReentryCtxADDiu = 0x24840000 | ((ReentryCtxAddr)&0xFFFF); + memcpy(ResolverWorkingMem + ReentryCtxAddrOffset, &ReentryCtxLUi, + sizeof(ReentryCtxLUi)); + memcpy(ResolverWorkingMem + ReentryCtxAddrOffset + 4, &ReentryCtxADDiu, + sizeof(ReentryCtxADDiu)); + + uint32_t ReentryFnLUi = + 0x3c190000 | (((ReentryFnAddr + 0x8000) >> 16) & 0xFFFF); + uint32_t ReentryFnADDiu = 0x27390000 | ((ReentryFnAddr)&0xFFFF); + memcpy(ResolverWorkingMem + ReentryFnAddrOffset, &ReentryFnLUi, + sizeof(ReentryFnLUi)); + memcpy(ResolverWorkingMem + ReentryFnAddrOffset + 4, &ReentryFnADDiu, + sizeof(ReentryFnADDiu)); +} + +void OrcMips32_Base::writeTrampolines( + char *TrampolineBlockWorkingMem, + JITTargetAddress TrampolineBlockTargetAddress, + JITTargetAddress ResolverAddr, unsigned NumTrampolines) { + + assert((ResolverAddr >> 32) == 0 && "ResolverAddr out of range"); + + uint32_t *Trampolines = + reinterpret_cast<uint32_t *>(TrampolineBlockWorkingMem); + uint32_t RHiAddr = ((ResolverAddr + 0x8000) >> 16); + + for (unsigned I = 0; I < NumTrampolines; ++I) { + // move $t8,$ra + // lui $t9,ResolverAddr + // addiu $t9,$t9,ResolverAddr + // jalr $t9 + // nop + Trampolines[5 * I + 0] = 0x03e0c025; + Trampolines[5 * I + 1] = 0x3c190000 | (RHiAddr & 0xFFFF); + Trampolines[5 * I + 2] = 0x27390000 | (ResolverAddr & 0xFFFF); + Trampolines[5 * I + 3] = 0x0320f809; + Trampolines[5 * I + 4] = 0x00000000; + } +} + +void OrcMips32_Base::writeIndirectStubsBlock( + char *StubsBlockWorkingMem, JITTargetAddress StubsBlockTargetAddress, + JITTargetAddress PointersBlockTargetAddress, unsigned NumStubs) { + assert((StubsBlockTargetAddress >> 32) == 0 && + "InitialPtrVal is out of range"); + + // Stub format is: + // + // .section __orc_stubs + // stub1: + // lui $t9, ptr1 + // lw $t9, %lo(ptr1)($t9) + // jr $t9 + // stub2: + // lui $t9, ptr2 + // lw $t9,%lo(ptr1)($t9) + // jr $t9 + // + // ... + // + // .section __orc_ptrs + // ptr1: + // .word 0x0 + // ptr2: + // .word 0x0 + // + // i.. + + assert(stubAndPointerRangesOk<OrcAArch64>( + StubsBlockTargetAddress, PointersBlockTargetAddress, NumStubs) && + "PointersBlock is out of range"); + + // Populate the stubs page stubs and mark it executable. + uint32_t *Stub = reinterpret_cast<uint32_t *>(StubsBlockWorkingMem); + uint64_t PtrAddr = PointersBlockTargetAddress; + + for (unsigned I = 0; I < NumStubs; ++I) { + uint32_t HiAddr = ((PtrAddr + 0x8000) >> 16); + Stub[4 * I + 0] = 0x3c190000 | (HiAddr & 0xFFFF); // lui $t9,ptr1 + Stub[4 * I + 1] = 0x8f390000 | (PtrAddr & 0xFFFF); // lw $t9,%lo(ptr1)($t9) + Stub[4 * I + 2] = 0x03200008; // jr $t9 + Stub[4 * I + 3] = 0x00000000; // nop + PtrAddr += 4; + } +} + +void OrcMips64::writeResolverCode(char *ResolverWorkingMem, + JITTargetAddress ResolverTargetAddress, + JITTargetAddress ReentryFnAddr, + JITTargetAddress ReentryCtxAddr) { + + const uint32_t ResolverCode[] = { + //resolver_entry: + 0x67bdff30, // 0x00: daddiu $sp,$sp,-208 + 0xffa20000, // 0x04: sd v0,0(sp) + 0xffa30008, // 0x08: sd v1,8(sp) + 0xffa40010, // 0x0c: sd a0,16(sp) + 0xffa50018, // 0x10: sd a1,24(sp) + 0xffa60020, // 0x14: sd a2,32(sp) + 0xffa70028, // 0x18: sd a3,40(sp) + 0xffa80030, // 0x1c: sd a4,48(sp) + 0xffa90038, // 0x20: sd a5,56(sp) + 0xffaa0040, // 0x24: sd a6,64(sp) + 0xffab0048, // 0x28: sd a7,72(sp) + 0xffac0050, // 0x2c: sd t0,80(sp) + 0xffad0058, // 0x30: sd t1,88(sp) + 0xffae0060, // 0x34: sd t2,96(sp) + 0xffaf0068, // 0x38: sd t3,104(sp) + 0xffb00070, // 0x3c: sd s0,112(sp) + 0xffb10078, // 0x40: sd s1,120(sp) + 0xffb20080, // 0x44: sd s2,128(sp) + 0xffb30088, // 0x48: sd s3,136(sp) + 0xffb40090, // 0x4c: sd s4,144(sp) + 0xffb50098, // 0x50: sd s5,152(sp) + 0xffb600a0, // 0x54: sd s6,160(sp) + 0xffb700a8, // 0x58: sd s7,168(sp) + 0xffb800b0, // 0x5c: sd t8,176(sp) + 0xffb900b8, // 0x60: sd t9,184(sp) + 0xffbe00c0, // 0x64: sd fp,192(sp) + 0xffbf00c8, // 0x68: sd ra,200(sp) + + // JIT re-entry ctx addr. + 0x00000000, // 0x6c: lui $a0,heighest(ctx) + 0x00000000, // 0x70: daddiu $a0,$a0,heigher(ctx) + 0x00000000, // 0x74: dsll $a0,$a0,16 + 0x00000000, // 0x78: daddiu $a0,$a0,hi(ctx) + 0x00000000, // 0x7c: dsll $a0,$a0,16 + 0x00000000, // 0x80: daddiu $a0,$a0,lo(ctx) + + 0x03e02825, // 0x84: move $a1, $ra + 0x64a5ffdc, // 0x88: daddiu $a1,$a1,-36 + + // JIT re-entry fn addr: + 0x00000000, // 0x8c: lui $t9,reentry + 0x00000000, // 0x90: daddiu $t9,$t9,reentry + 0x00000000, // 0x94: dsll $t9,$t9, + 0x00000000, // 0x98: daddiu $t9,$t9, + 0x00000000, // 0x9c: dsll $t9,$t9, + 0x00000000, // 0xa0: daddiu $t9,$t9, + 0x0320f809, // 0xa4: jalr $t9 + 0x00000000, // 0xa8: nop + 0xdfbf00c8, // 0xac: ld ra, 200(sp) + 0xdfbe00c0, // 0xb0: ld fp, 192(sp) + 0xdfb900b8, // 0xb4: ld t9, 184(sp) + 0xdfb800b0, // 0xb8: ld t8, 176(sp) + 0xdfb700a8, // 0xbc: ld s7, 168(sp) + 0xdfb600a0, // 0xc0: ld s6, 160(sp) + 0xdfb50098, // 0xc4: ld s5, 152(sp) + 0xdfb40090, // 0xc8: ld s4, 144(sp) + 0xdfb30088, // 0xcc: ld s3, 136(sp) + 0xdfb20080, // 0xd0: ld s2, 128(sp) + 0xdfb10078, // 0xd4: ld s1, 120(sp) + 0xdfb00070, // 0xd8: ld s0, 112(sp) + 0xdfaf0068, // 0xdc: ld t3, 104(sp) + 0xdfae0060, // 0xe0: ld t2, 96(sp) + 0xdfad0058, // 0xe4: ld t1, 88(sp) + 0xdfac0050, // 0xe8: ld t0, 80(sp) + 0xdfab0048, // 0xec: ld a7, 72(sp) + 0xdfaa0040, // 0xf0: ld a6, 64(sp) + 0xdfa90038, // 0xf4: ld a5, 56(sp) + 0xdfa80030, // 0xf8: ld a4, 48(sp) + 0xdfa70028, // 0xfc: ld a3, 40(sp) + 0xdfa60020, // 0x100: ld a2, 32(sp) + 0xdfa50018, // 0x104: ld a1, 24(sp) + 0xdfa40010, // 0x108: ld a0, 16(sp) + 0xdfa30008, // 0x10c: ld v1, 8(sp) + 0x67bd00d0, // 0x110: daddiu $sp,$sp,208 + 0x0300f825, // 0x114: move $ra, $t8 + 0x03200008, // 0x118: jr $t9 + 0x0040c825, // 0x11c: move $t9, $v0 + }; + + const unsigned ReentryFnAddrOffset = 0x8c; // JIT re-entry fn addr lui + const unsigned ReentryCtxAddrOffset = 0x6c; // JIT re-entry ctx addr lui + + memcpy(ResolverWorkingMem, ResolverCode, sizeof(ResolverCode)); + + uint32_t ReentryCtxLUi = + 0x3c040000 | (((ReentryCtxAddr + 0x800080008000) >> 48) & 0xFFFF); + uint32_t ReentryCtxDADDiu = + 0x64840000 | (((ReentryCtxAddr + 0x80008000) >> 32) & 0xFFFF); + uint32_t ReentryCtxDSLL = 0x00042438; + uint32_t ReentryCtxDADDiu2 = + 0x64840000 | ((((ReentryCtxAddr + 0x8000) >> 16) & 0xFFFF)); + uint32_t ReentryCtxDSLL2 = 0x00042438; + uint32_t ReentryCtxDADDiu3 = 0x64840000 | ((ReentryCtxAddr)&0xFFFF); + + memcpy(ResolverWorkingMem + ReentryCtxAddrOffset, &ReentryCtxLUi, + sizeof(ReentryCtxLUi)); + memcpy(ResolverWorkingMem + (ReentryCtxAddrOffset + 4), &ReentryCtxDADDiu, + sizeof(ReentryCtxDADDiu)); + memcpy(ResolverWorkingMem + (ReentryCtxAddrOffset + 8), &ReentryCtxDSLL, + sizeof(ReentryCtxDSLL)); + memcpy(ResolverWorkingMem + (ReentryCtxAddrOffset + 12), &ReentryCtxDADDiu2, + sizeof(ReentryCtxDADDiu2)); + memcpy(ResolverWorkingMem + (ReentryCtxAddrOffset + 16), &ReentryCtxDSLL2, + sizeof(ReentryCtxDSLL2)); + memcpy(ResolverWorkingMem + (ReentryCtxAddrOffset + 20), &ReentryCtxDADDiu3, + sizeof(ReentryCtxDADDiu3)); + + uint32_t ReentryFnLUi = + 0x3c190000 | (((ReentryFnAddr + 0x800080008000) >> 48) & 0xFFFF); + + uint32_t ReentryFnDADDiu = + 0x67390000 | (((ReentryFnAddr + 0x80008000) >> 32) & 0xFFFF); + + uint32_t ReentryFnDSLL = 0x0019cc38; + + uint32_t ReentryFnDADDiu2 = + 0x67390000 | (((ReentryFnAddr + 0x8000) >> 16) & 0xFFFF); + + uint32_t ReentryFnDSLL2 = 0x0019cc38; + + uint32_t ReentryFnDADDiu3 = 0x67390000 | ((ReentryFnAddr)&0xFFFF); + + memcpy(ResolverWorkingMem + ReentryFnAddrOffset, &ReentryFnLUi, + sizeof(ReentryFnLUi)); + memcpy(ResolverWorkingMem + (ReentryFnAddrOffset + 4), &ReentryFnDADDiu, + sizeof(ReentryFnDADDiu)); + memcpy(ResolverWorkingMem + (ReentryFnAddrOffset + 8), &ReentryFnDSLL, + sizeof(ReentryFnDSLL)); + memcpy(ResolverWorkingMem + (ReentryFnAddrOffset + 12), &ReentryFnDADDiu2, + sizeof(ReentryFnDADDiu2)); + memcpy(ResolverWorkingMem + (ReentryFnAddrOffset + 16), &ReentryFnDSLL2, + sizeof(ReentryFnDSLL2)); + memcpy(ResolverWorkingMem + (ReentryFnAddrOffset + 20), &ReentryFnDADDiu3, + sizeof(ReentryFnDADDiu3)); +} + +void OrcMips64::writeTrampolines(char *TrampolineBlockWorkingMem, + JITTargetAddress TrampolineBlockTargetAddress, + JITTargetAddress ResolverAddr, + unsigned NumTrampolines) { + + uint32_t *Trampolines = + reinterpret_cast<uint32_t *>(TrampolineBlockWorkingMem); + + uint64_t HeighestAddr = ((ResolverAddr + 0x800080008000) >> 48); + uint64_t HeigherAddr = ((ResolverAddr + 0x80008000) >> 32); + uint64_t HiAddr = ((ResolverAddr + 0x8000) >> 16); + + for (unsigned I = 0; I < NumTrampolines; ++I) { + Trampolines[10 * I + 0] = 0x03e0c025; // move $t8,$ra + Trampolines[10 * I + 1] = 0x3c190000 | (HeighestAddr & 0xFFFF); // lui $t9,resolveAddr + Trampolines[10 * I + 2] = 0x67390000 | (HeigherAddr & 0xFFFF); // daddiu $t9,$t9,%higher(resolveAddr) + Trampolines[10 * I + 3] = 0x0019cc38; // dsll $t9,$t9,16 + Trampolines[10 * I + 4] = 0x67390000 | (HiAddr & 0xFFFF); // daddiu $t9,$t9,%hi(ptr) + Trampolines[10 * I + 5] = 0x0019cc38; // dsll $t9,$t9,16 + Trampolines[10 * I + 6] = + 0x67390000 | (ResolverAddr & 0xFFFF); // daddiu $t9,$t9,%lo(ptr) + Trampolines[10 * I + 7] = 0x0320f809; // jalr $t9 + Trampolines[10 * I + 8] = 0x00000000; // nop + Trampolines[10 * I + 9] = 0x00000000; // nop + } +} + +void OrcMips64::writeIndirectStubsBlock( + char *StubsBlockWorkingMem, JITTargetAddress StubsBlockTargetAddress, + JITTargetAddress PointersBlockTargetAddress, unsigned NumStubs) { + // Stub format is: + // + // .section __orc_stubs + // stub1: + // lui $t9,ptr1 + // dsll $t9,$t9,16 + // daddiu $t9,$t9,%hi(ptr) + // dsll $t9,$t9,16 + // ld $t9,%lo(ptr) + // jr $t9 + // stub2: + // lui $t9,ptr1 + // dsll $t9,$t9,16 + // daddiu $t9,$t9,%hi(ptr) + // dsll $t9,$t9,16 + // ld $t9,%lo(ptr) + // jr $t9 + // + // ... + // + // .section __orc_ptrs + // ptr1: + // .dword 0x0 + // ptr2: + // .dword 0x0 + // + // ... + + assert(stubAndPointerRangesOk<OrcAArch64>( + StubsBlockTargetAddress, PointersBlockTargetAddress, NumStubs) && + "PointersBlock is out of range"); + + // Populate the stubs page stubs and mark it executable. + uint32_t *Stub = reinterpret_cast<uint32_t *>(StubsBlockWorkingMem); + uint64_t PtrAddr = PointersBlockTargetAddress; + + for (unsigned I = 0; I < NumStubs; ++I, PtrAddr += 8) { + uint64_t HeighestAddr = ((PtrAddr + 0x800080008000) >> 48); + uint64_t HeigherAddr = ((PtrAddr + 0x80008000) >> 32); + uint64_t HiAddr = ((PtrAddr + 0x8000) >> 16); + Stub[8 * I + 0] = 0x3c190000 | (HeighestAddr & 0xFFFF); // lui $t9,ptr1 + Stub[8 * I + 1] = 0x67390000 | (HeigherAddr & 0xFFFF); // daddiu $t9,$t9,%higher(ptr) + Stub[8 * I + 2] = 0x0019cc38; // dsll $t9,$t9,16 + Stub[8 * I + 3] = 0x67390000 | (HiAddr & 0xFFFF); // daddiu $t9,$t9,%hi(ptr) + Stub[8 * I + 4] = 0x0019cc38; // dsll $t9,$t9,16 + Stub[8 * I + 5] = 0xdf390000 | (PtrAddr & 0xFFFF); // ld $t9,%lo(ptr) + Stub[8 * I + 6] = 0x03200008; // jr $t9 + Stub[8 * I + 7] = 0x00000000; // nop + } +} +} // End namespace orc. +} // End namespace llvm. diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp new file mode 100644 index 0000000000..834d4cc8f5 --- /dev/null +++ b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp @@ -0,0 +1,529 @@ +//===--------------- OrcV2CBindings.cpp - C bindings OrcV2 APIs -----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm-c/LLJIT.h" +#include "llvm-c/Orc.h" +#include "llvm-c/OrcEE.h" +#include "llvm-c/TargetMachine.h" + +#include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h" +#include "llvm/ExecutionEngine/Orc/LLJIT.h" +#include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h" +#include "llvm/ExecutionEngine/SectionMemoryManager.h" + +using namespace llvm; +using namespace llvm::orc; + +namespace llvm { +namespace orc { + +class InProgressLookupState; + +class OrcV2CAPIHelper { +public: + using PoolEntry = SymbolStringPtr::PoolEntry; + using PoolEntryPtr = SymbolStringPtr::PoolEntryPtr; + + static PoolEntryPtr releaseSymbolStringPtr(SymbolStringPtr S) { + PoolEntryPtr Result = nullptr; + std::swap(Result, S.S); + return Result; + } + + static SymbolStringPtr retainSymbolStringPtr(PoolEntryPtr P) { + return SymbolStringPtr(P); + } + + static PoolEntryPtr getRawPoolEntryPtr(const SymbolStringPtr &S) { + return S.S; + } + + static void retainPoolEntry(PoolEntryPtr P) { + SymbolStringPtr S(P); + S.S = nullptr; + } + + static void releasePoolEntry(PoolEntryPtr P) { + SymbolStringPtr S; + S.S = P; + } + + static InProgressLookupState *extractLookupState(LookupState &LS) { + return LS.IPLS.release(); + } + + static void resetLookupState(LookupState &LS, InProgressLookupState *IPLS) { + return LS.reset(IPLS); + } +}; + +} // namespace orc +} // namespace llvm + +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(ExecutionSession, LLVMOrcExecutionSessionRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(SymbolStringPool, LLVMOrcSymbolStringPoolRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(OrcV2CAPIHelper::PoolEntry, + LLVMOrcSymbolStringPoolEntryRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(MaterializationUnit, + LLVMOrcMaterializationUnitRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(JITDylib, LLVMOrcJITDylibRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(ResourceTracker, LLVMOrcResourceTrackerRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DefinitionGenerator, + LLVMOrcDefinitionGeneratorRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(InProgressLookupState, LLVMOrcLookupStateRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(ThreadSafeContext, + LLVMOrcThreadSafeContextRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(ThreadSafeModule, LLVMOrcThreadSafeModuleRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(JITTargetMachineBuilder, + LLVMOrcJITTargetMachineBuilderRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(ObjectLayer, LLVMOrcObjectLayerRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(LLJITBuilder, LLVMOrcLLJITBuilderRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(LLJIT, LLVMOrcLLJITRef) + +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef) + +namespace llvm { +namespace orc { + +class CAPIDefinitionGenerator final : public DefinitionGenerator { +public: + CAPIDefinitionGenerator( + void *Ctx, + LLVMOrcCAPIDefinitionGeneratorTryToGenerateFunction TryToGenerate) + : Ctx(Ctx), TryToGenerate(TryToGenerate) {} + + Error tryToGenerate(LookupState &LS, LookupKind K, JITDylib &JD, + JITDylibLookupFlags JDLookupFlags, + const SymbolLookupSet &LookupSet) override { + + // Take the lookup state. + LLVMOrcLookupStateRef LSR = ::wrap(OrcV2CAPIHelper::extractLookupState(LS)); + + // Translate the lookup kind. + LLVMOrcLookupKind CLookupKind; + switch (K) { + case LookupKind::Static: + CLookupKind = LLVMOrcLookupKindStatic; + break; + case LookupKind::DLSym: + CLookupKind = LLVMOrcLookupKindDLSym; + break; + } + + // Translate the JITDylibSearchFlags. + LLVMOrcJITDylibLookupFlags CJDLookupFlags; + switch (JDLookupFlags) { + case JITDylibLookupFlags::MatchExportedSymbolsOnly: + CJDLookupFlags = LLVMOrcJITDylibLookupFlagsMatchExportedSymbolsOnly; + break; + case JITDylibLookupFlags::MatchAllSymbols: + CJDLookupFlags = LLVMOrcJITDylibLookupFlagsMatchAllSymbols; + break; + } + + // Translate the lookup set. + std::vector<LLVMOrcCLookupSetElement> CLookupSet; + CLookupSet.reserve(LookupSet.size()); + for (auto &KV : LookupSet) { + LLVMOrcSymbolLookupFlags SLF; + LLVMOrcSymbolStringPoolEntryRef Name = + ::wrap(OrcV2CAPIHelper::getRawPoolEntryPtr(KV.first)); + switch (KV.second) { + case SymbolLookupFlags::RequiredSymbol: + SLF = LLVMOrcSymbolLookupFlagsRequiredSymbol; + break; + case SymbolLookupFlags::WeaklyReferencedSymbol: + SLF = LLVMOrcSymbolLookupFlagsWeaklyReferencedSymbol; + break; + } + CLookupSet.push_back({Name, SLF}); + } + + // Run the C TryToGenerate function. + auto Err = unwrap(TryToGenerate(::wrap(this), Ctx, &LSR, CLookupKind, + ::wrap(&JD), CJDLookupFlags, + CLookupSet.data(), CLookupSet.size())); + + // Restore the lookup state. + OrcV2CAPIHelper::resetLookupState(LS, ::unwrap(LSR)); + + return Err; + } + +private: + void *Ctx; + LLVMOrcCAPIDefinitionGeneratorTryToGenerateFunction TryToGenerate; +}; + +} // end namespace orc +} // end namespace llvm + +void LLVMOrcExecutionSessionSetErrorReporter( + LLVMOrcExecutionSessionRef ES, LLVMOrcErrorReporterFunction ReportError, + void *Ctx) { + unwrap(ES)->setErrorReporter( + [=](Error Err) { ReportError(Ctx, wrap(std::move(Err))); }); +} + +LLVMOrcSymbolStringPoolRef +LLVMOrcExecutionSessionGetSymbolStringPool(LLVMOrcExecutionSessionRef ES) { + return wrap(unwrap(ES)->getSymbolStringPool().get()); +} + +void LLVMOrcSymbolStringPoolClearDeadEntries(LLVMOrcSymbolStringPoolRef SSP) { + unwrap(SSP)->clearDeadEntries(); +} + +LLVMOrcSymbolStringPoolEntryRef +LLVMOrcExecutionSessionIntern(LLVMOrcExecutionSessionRef ES, const char *Name) { + return wrap( + OrcV2CAPIHelper::releaseSymbolStringPtr(unwrap(ES)->intern(Name))); +} + +void LLVMOrcRetainSymbolStringPoolEntry(LLVMOrcSymbolStringPoolEntryRef S) { + OrcV2CAPIHelper::retainPoolEntry(unwrap(S)); +} + +void LLVMOrcReleaseSymbolStringPoolEntry(LLVMOrcSymbolStringPoolEntryRef S) { + OrcV2CAPIHelper::releasePoolEntry(unwrap(S)); +} + +const char *LLVMOrcSymbolStringPoolEntryStr(LLVMOrcSymbolStringPoolEntryRef S) { + return unwrap(S)->getKey().data(); +} + +LLVMOrcResourceTrackerRef +LLVMOrcJITDylibCreateResourceTracker(LLVMOrcJITDylibRef JD) { + auto RT = unwrap(JD)->createResourceTracker(); + // Retain the pointer for the C API client. + RT->Retain(); + return wrap(RT.get()); +} + +LLVMOrcResourceTrackerRef +LLVMOrcJITDylibGetDefaultResourceTracker(LLVMOrcJITDylibRef JD) { + auto RT = unwrap(JD)->getDefaultResourceTracker(); + // Retain the pointer for the C API client. + return wrap(RT.get()); +} + +void LLVMOrcReleaseResourceTracker(LLVMOrcResourceTrackerRef RT) { + ResourceTrackerSP TmpRT(unwrap(RT)); + TmpRT->Release(); +} + +void LLVMOrcResourceTrackerTransferTo(LLVMOrcResourceTrackerRef SrcRT, + LLVMOrcResourceTrackerRef DstRT) { + ResourceTrackerSP TmpRT(unwrap(SrcRT)); + TmpRT->transferTo(*unwrap(DstRT)); +} + +LLVMErrorRef LLVMOrcResourceTrackerRemove(LLVMOrcResourceTrackerRef RT) { + ResourceTrackerSP TmpRT(unwrap(RT)); + return wrap(TmpRT->remove()); +} + +void LLVMOrcDisposeDefinitionGenerator(LLVMOrcDefinitionGeneratorRef DG) { + std::unique_ptr<DefinitionGenerator> TmpDG(unwrap(DG)); +} + +void LLVMOrcDisposeMaterializationUnit(LLVMOrcMaterializationUnitRef MU) { + std::unique_ptr<MaterializationUnit> TmpMU(unwrap(MU)); +} + +LLVMOrcMaterializationUnitRef +LLVMOrcAbsoluteSymbols(LLVMOrcCSymbolMapPairs Syms, size_t NumPairs) { + SymbolMap SM; + for (size_t I = 0; I != NumPairs; ++I) { + JITSymbolFlags Flags; + + if (Syms[I].Sym.Flags.GenericFlags & LLVMJITSymbolGenericFlagsExported) + Flags |= JITSymbolFlags::Exported; + if (Syms[I].Sym.Flags.GenericFlags & LLVMJITSymbolGenericFlagsWeak) + Flags |= JITSymbolFlags::Weak; + + Flags.getTargetFlags() = Syms[I].Sym.Flags.TargetFlags; + + SM[OrcV2CAPIHelper::retainSymbolStringPtr(unwrap(Syms[I].Name))] = + JITEvaluatedSymbol(Syms[I].Sym.Address, Flags); + } + + return wrap(absoluteSymbols(std::move(SM)).release()); +} + +LLVMOrcJITDylibRef +LLVMOrcExecutionSessionCreateBareJITDylib(LLVMOrcExecutionSessionRef ES, + const char *Name) { + return wrap(&unwrap(ES)->createBareJITDylib(Name)); +} + +LLVMErrorRef +LLVMOrcExecutionSessionCreateJITDylib(LLVMOrcExecutionSessionRef ES, + LLVMOrcJITDylibRef *Result, + const char *Name) { + auto JD = unwrap(ES)->createJITDylib(Name); + if (!JD) + return wrap(JD.takeError()); + *Result = wrap(&*JD); + return LLVMErrorSuccess; +} + +LLVMOrcJITDylibRef +LLVMOrcExecutionSessionGetJITDylibByName(LLVMOrcExecutionSessionRef ES, + const char *Name) { + return wrap(unwrap(ES)->getJITDylibByName(Name)); +} + +LLVMErrorRef LLVMOrcJITDylibDefine(LLVMOrcJITDylibRef JD, + LLVMOrcMaterializationUnitRef MU) { + std::unique_ptr<MaterializationUnit> TmpMU(unwrap(MU)); + + if (auto Err = unwrap(JD)->define(TmpMU)) { + TmpMU.release(); + return wrap(std::move(Err)); + } + return LLVMErrorSuccess; +} + +LLVMErrorRef LLVMOrcJITDylibClear(LLVMOrcJITDylibRef JD) { + return wrap(unwrap(JD)->clear()); +} + +void LLVMOrcJITDylibAddGenerator(LLVMOrcJITDylibRef JD, + LLVMOrcDefinitionGeneratorRef DG) { + unwrap(JD)->addGenerator(std::unique_ptr<DefinitionGenerator>(unwrap(DG))); +} + +LLVMOrcDefinitionGeneratorRef LLVMOrcCreateCustomCAPIDefinitionGenerator( + LLVMOrcCAPIDefinitionGeneratorTryToGenerateFunction F, void *Ctx) { + auto DG = std::make_unique<CAPIDefinitionGenerator>(Ctx, F); + return wrap(DG.release()); +} + +LLVMErrorRef LLVMOrcCreateDynamicLibrarySearchGeneratorForProcess( + LLVMOrcDefinitionGeneratorRef *Result, char GlobalPrefix, + LLVMOrcSymbolPredicate Filter, void *FilterCtx) { + assert(Result && "Result can not be null"); + assert((Filter || !FilterCtx) && + "if Filter is null then FilterCtx must also be null"); + + DynamicLibrarySearchGenerator::SymbolPredicate Pred; + if (Filter) + Pred = [=](const SymbolStringPtr &Name) -> bool { + return Filter(FilterCtx, wrap(OrcV2CAPIHelper::getRawPoolEntryPtr(Name))); + }; + + auto ProcessSymsGenerator = + DynamicLibrarySearchGenerator::GetForCurrentProcess(GlobalPrefix, Pred); + + if (!ProcessSymsGenerator) { + *Result = 0; + return wrap(ProcessSymsGenerator.takeError()); + } + + *Result = wrap(ProcessSymsGenerator->release()); + return LLVMErrorSuccess; +} + +LLVMOrcThreadSafeContextRef LLVMOrcCreateNewThreadSafeContext(void) { + return wrap(new ThreadSafeContext(std::make_unique<LLVMContext>())); +} + +LLVMContextRef +LLVMOrcThreadSafeContextGetContext(LLVMOrcThreadSafeContextRef TSCtx) { + return wrap(unwrap(TSCtx)->getContext()); +} + +void LLVMOrcDisposeThreadSafeContext(LLVMOrcThreadSafeContextRef TSCtx) { + delete unwrap(TSCtx); +} + +LLVMOrcThreadSafeModuleRef +LLVMOrcCreateNewThreadSafeModule(LLVMModuleRef M, + LLVMOrcThreadSafeContextRef TSCtx) { + return wrap( + new ThreadSafeModule(std::unique_ptr<Module>(unwrap(M)), *unwrap(TSCtx))); +} + +void LLVMOrcDisposeThreadSafeModule(LLVMOrcThreadSafeModuleRef TSM) { + delete unwrap(TSM); +} + +LLVMErrorRef LLVMOrcJITTargetMachineBuilderDetectHost( + LLVMOrcJITTargetMachineBuilderRef *Result) { + assert(Result && "Result can not be null"); + + auto JTMB = JITTargetMachineBuilder::detectHost(); + if (!JTMB) { + Result = 0; + return wrap(JTMB.takeError()); + } + + *Result = wrap(new JITTargetMachineBuilder(std::move(*JTMB))); + return LLVMErrorSuccess; +} + +LLVMOrcJITTargetMachineBuilderRef +LLVMOrcJITTargetMachineBuilderCreateFromTargetMachine(LLVMTargetMachineRef TM) { + auto *TemplateTM = unwrap(TM); + + auto JTMB = + std::make_unique<JITTargetMachineBuilder>(TemplateTM->getTargetTriple()); + + (*JTMB) + .setCPU(TemplateTM->getTargetCPU().str()) + .setRelocationModel(TemplateTM->getRelocationModel()) + .setCodeModel(TemplateTM->getCodeModel()) + .setCodeGenOptLevel(TemplateTM->getOptLevel()) + .setFeatures(TemplateTM->getTargetFeatureString()) + .setOptions(TemplateTM->Options); + + LLVMDisposeTargetMachine(TM); + + return wrap(JTMB.release()); +} + +void LLVMOrcDisposeJITTargetMachineBuilder( + LLVMOrcJITTargetMachineBuilderRef JTMB) { + delete unwrap(JTMB); +} + +void LLVMOrcDisposeObjectLayer(LLVMOrcObjectLayerRef ObjLayer) { + delete unwrap(ObjLayer); +} + +LLVMOrcLLJITBuilderRef LLVMOrcCreateLLJITBuilder(void) { + return wrap(new LLJITBuilder()); +} + +void LLVMOrcDisposeLLJITBuilder(LLVMOrcLLJITBuilderRef Builder) { + delete unwrap(Builder); +} + +void LLVMOrcLLJITBuilderSetJITTargetMachineBuilder( + LLVMOrcLLJITBuilderRef Builder, LLVMOrcJITTargetMachineBuilderRef JTMB) { + unwrap(Builder)->setJITTargetMachineBuilder(*unwrap(JTMB)); +} + +void LLVMOrcLLJITBuilderSetObjectLinkingLayerCreator( + LLVMOrcLLJITBuilderRef Builder, + LLVMOrcLLJITBuilderObjectLinkingLayerCreatorFunction F, void *Ctx) { + unwrap(Builder)->setObjectLinkingLayerCreator( + [=](ExecutionSession &ES, const Triple &TT) { + auto TTStr = TT.str(); + return std::unique_ptr<ObjectLayer>( + unwrap(F(Ctx, wrap(&ES), TTStr.c_str()))); + }); +} + +LLVMErrorRef LLVMOrcCreateLLJIT(LLVMOrcLLJITRef *Result, + LLVMOrcLLJITBuilderRef Builder) { + assert(Result && "Result can not be null"); + + if (!Builder) + Builder = LLVMOrcCreateLLJITBuilder(); + + auto J = unwrap(Builder)->create(); + LLVMOrcDisposeLLJITBuilder(Builder); + + if (!J) { + Result = 0; + return wrap(J.takeError()); + } + + *Result = wrap(J->release()); + return LLVMErrorSuccess; +} + +LLVMErrorRef LLVMOrcDisposeLLJIT(LLVMOrcLLJITRef J) { + delete unwrap(J); + return LLVMErrorSuccess; +} + +LLVMOrcExecutionSessionRef LLVMOrcLLJITGetExecutionSession(LLVMOrcLLJITRef J) { + return wrap(&unwrap(J)->getExecutionSession()); +} + +LLVMOrcJITDylibRef LLVMOrcLLJITGetMainJITDylib(LLVMOrcLLJITRef J) { + return wrap(&unwrap(J)->getMainJITDylib()); +} + +const char *LLVMOrcLLJITGetTripleString(LLVMOrcLLJITRef J) { + return unwrap(J)->getTargetTriple().str().c_str(); +} + +char LLVMOrcLLJITGetGlobalPrefix(LLVMOrcLLJITRef J) { + return unwrap(J)->getDataLayout().getGlobalPrefix(); +} + +LLVMOrcSymbolStringPoolEntryRef +LLVMOrcLLJITMangleAndIntern(LLVMOrcLLJITRef J, const char *UnmangledName) { + return wrap(OrcV2CAPIHelper::releaseSymbolStringPtr( + unwrap(J)->mangleAndIntern(UnmangledName))); +} + +LLVMErrorRef LLVMOrcLLJITAddObjectFile(LLVMOrcLLJITRef J, LLVMOrcJITDylibRef JD, + LLVMMemoryBufferRef ObjBuffer) { + return wrap(unwrap(J)->addObjectFile( + *unwrap(JD), std::unique_ptr<MemoryBuffer>(unwrap(ObjBuffer)))); +} + +LLVMErrorRef LLVMOrcLLJITAddObjectFileWithRT(LLVMOrcLLJITRef J, + LLVMOrcResourceTrackerRef RT, + LLVMMemoryBufferRef ObjBuffer) { + return wrap(unwrap(J)->addObjectFile( + ResourceTrackerSP(unwrap(RT)), + std::unique_ptr<MemoryBuffer>(unwrap(ObjBuffer)))); +} + +LLVMErrorRef LLVMOrcLLJITAddLLVMIRModule(LLVMOrcLLJITRef J, + LLVMOrcJITDylibRef JD, + LLVMOrcThreadSafeModuleRef TSM) { + std::unique_ptr<ThreadSafeModule> TmpTSM(unwrap(TSM)); + return wrap(unwrap(J)->addIRModule(*unwrap(JD), std::move(*TmpTSM))); +} + +LLVMErrorRef LLVMOrcLLJITAddLLVMIRModuleWithRT(LLVMOrcLLJITRef J, + LLVMOrcResourceTrackerRef RT, + LLVMOrcThreadSafeModuleRef TSM) { + std::unique_ptr<ThreadSafeModule> TmpTSM(unwrap(TSM)); + return wrap(unwrap(J)->addIRModule(ResourceTrackerSP(unwrap(RT)), + std::move(*TmpTSM))); +} + +LLVMErrorRef LLVMOrcLLJITLookup(LLVMOrcLLJITRef J, + LLVMOrcJITTargetAddress *Result, + const char *Name) { + assert(Result && "Result can not be null"); + + auto Sym = unwrap(J)->lookup(Name); + if (!Sym) { + *Result = 0; + return wrap(Sym.takeError()); + } + + *Result = Sym->getAddress(); + return LLVMErrorSuccess; +} + +LLVMOrcObjectLayerRef +LLVMOrcCreateRTDyldObjectLinkingLayerWithSectionMemoryManager( + LLVMOrcExecutionSessionRef ES) { + assert(ES && "ES must not be null"); + return wrap(new RTDyldObjectLinkingLayer( + *unwrap(ES), [] { return std::make_unique<SectionMemoryManager>(); })); +} + +void LLVMOrcRTDyldObjectLinkingLayerRegisterJITEventListener( + LLVMOrcObjectLayerRef RTDyldObjLinkingLayer, + LLVMJITEventListenerRef Listener) { + assert(RTDyldObjLinkingLayer && "RTDyldObjLinkingLayer must not be null"); + assert(Listener && "Listener must not be null"); + reinterpret_cast<RTDyldObjectLinkingLayer *>(unwrap(RTDyldObjLinkingLayer)) + ->registerJITEventListener(*unwrap(Listener)); +} diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp new file mode 100644 index 0000000000..0ad666ebbe --- /dev/null +++ b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp @@ -0,0 +1,351 @@ +//===-- RTDyldObjectLinkingLayer.cpp - RuntimeDyld backed ORC ObjectLayer -===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h" +#include "llvm/Object/COFF.h" + +namespace { + +using namespace llvm; +using namespace llvm::orc; + +class JITDylibSearchOrderResolver : public JITSymbolResolver { +public: + JITDylibSearchOrderResolver(MaterializationResponsibility &MR) : MR(MR) {} + + void lookup(const LookupSet &Symbols, OnResolvedFunction OnResolved) override { + auto &ES = MR.getTargetJITDylib().getExecutionSession(); + SymbolLookupSet InternedSymbols; + + // Intern the requested symbols: lookup takes interned strings. + for (auto &S : Symbols) + InternedSymbols.add(ES.intern(S)); + + // Build an OnResolve callback to unwrap the interned strings and pass them + // to the OnResolved callback. + auto OnResolvedWithUnwrap = + [OnResolved = std::move(OnResolved)]( + Expected<SymbolMap> InternedResult) mutable { + if (!InternedResult) { + OnResolved(InternedResult.takeError()); + return; + } + + LookupResult Result; + for (auto &KV : *InternedResult) + Result[*KV.first] = std::move(KV.second); + OnResolved(Result); + }; + + // Register dependencies for all symbols contained in this set. + auto RegisterDependencies = [&](const SymbolDependenceMap &Deps) { + MR.addDependenciesForAll(Deps); + }; + + JITDylibSearchOrder LinkOrder; + MR.getTargetJITDylib().withLinkOrderDo( + [&](const JITDylibSearchOrder &LO) { LinkOrder = LO; }); + ES.lookup(LookupKind::Static, LinkOrder, InternedSymbols, + SymbolState::Resolved, std::move(OnResolvedWithUnwrap), + RegisterDependencies); + } + + Expected<LookupSet> getResponsibilitySet(const LookupSet &Symbols) override { + LookupSet Result; + + for (auto &KV : MR.getSymbols()) { + if (Symbols.count(*KV.first)) + Result.insert(*KV.first); + } + + return Result; + } + +private: + MaterializationResponsibility &MR; +}; + +} // end anonymous namespace + +namespace llvm { +namespace orc { + +RTDyldObjectLinkingLayer::RTDyldObjectLinkingLayer( + ExecutionSession &ES, GetMemoryManagerFunction GetMemoryManager) + : ObjectLayer(ES), GetMemoryManager(GetMemoryManager) { + ES.registerResourceManager(*this); +} + +RTDyldObjectLinkingLayer::~RTDyldObjectLinkingLayer() { + assert(MemMgrs.empty() && "Layer destroyed with resources still attached"); +} + +void RTDyldObjectLinkingLayer::emit( + std::unique_ptr<MaterializationResponsibility> R, + std::unique_ptr<MemoryBuffer> O) { + assert(O && "Object must not be null"); + + auto &ES = getExecutionSession(); + + auto Obj = object::ObjectFile::createObjectFile(*O); + + if (!Obj) { + getExecutionSession().reportError(Obj.takeError()); + R->failMaterialization(); + return; + } + + // Collect the internal symbols from the object file: We will need to + // filter these later. + auto InternalSymbols = std::make_shared<std::set<StringRef>>(); + { + for (auto &Sym : (*Obj)->symbols()) { + + // Skip file symbols. + if (auto SymType = Sym.getType()) { + if (*SymType == object::SymbolRef::ST_File) + continue; + } else { + ES.reportError(SymType.takeError()); + R->failMaterialization(); + return; + } + + Expected<uint32_t> SymFlagsOrErr = Sym.getFlags(); + if (!SymFlagsOrErr) { + // TODO: Test this error. + ES.reportError(SymFlagsOrErr.takeError()); + R->failMaterialization(); + return; + } + + // Don't include symbols that aren't global. + if (!(*SymFlagsOrErr & object::BasicSymbolRef::SF_Global)) { + if (auto SymName = Sym.getName()) + InternalSymbols->insert(*SymName); + else { + ES.reportError(SymName.takeError()); + R->failMaterialization(); + return; + } + } + } + } + + auto MemMgr = GetMemoryManager(); + auto &MemMgrRef = *MemMgr; + + // Switch to shared ownership of MR so that it can be captured by both + // lambdas below. + std::shared_ptr<MaterializationResponsibility> SharedR(std::move(R)); + + JITDylibSearchOrderResolver Resolver(*SharedR); + + jitLinkForORC( + object::OwningBinary<object::ObjectFile>(std::move(*Obj), std::move(O)), + MemMgrRef, Resolver, ProcessAllSections, + [this, SharedR, &MemMgrRef, InternalSymbols]( + const object::ObjectFile &Obj, + RuntimeDyld::LoadedObjectInfo &LoadedObjInfo, + std::map<StringRef, JITEvaluatedSymbol> ResolvedSymbols) { + return onObjLoad(*SharedR, Obj, MemMgrRef, LoadedObjInfo, + ResolvedSymbols, *InternalSymbols); + }, + [this, SharedR, MemMgr = std::move(MemMgr)]( + object::OwningBinary<object::ObjectFile> Obj, + std::unique_ptr<RuntimeDyld::LoadedObjectInfo> LoadedObjInfo, + Error Err) mutable { + onObjEmit(*SharedR, std::move(Obj), std::move(MemMgr), + std::move(LoadedObjInfo), std::move(Err)); + }); +} + +void RTDyldObjectLinkingLayer::registerJITEventListener(JITEventListener &L) { + std::lock_guard<std::mutex> Lock(RTDyldLayerMutex); + assert(!llvm::is_contained(EventListeners, &L) && + "Listener has already been registered"); + EventListeners.push_back(&L); +} + +void RTDyldObjectLinkingLayer::unregisterJITEventListener(JITEventListener &L) { + std::lock_guard<std::mutex> Lock(RTDyldLayerMutex); + auto I = llvm::find(EventListeners, &L); + assert(I != EventListeners.end() && "Listener not registered"); + EventListeners.erase(I); +} + +Error RTDyldObjectLinkingLayer::onObjLoad( + MaterializationResponsibility &R, const object::ObjectFile &Obj, + RuntimeDyld::MemoryManager &MemMgr, + RuntimeDyld::LoadedObjectInfo &LoadedObjInfo, + std::map<StringRef, JITEvaluatedSymbol> Resolved, + std::set<StringRef> &InternalSymbols) { + SymbolFlagsMap ExtraSymbolsToClaim; + SymbolMap Symbols; + + // Hack to support COFF constant pool comdats introduced during compilation: + // (See http://llvm.org/PR40074) + if (auto *COFFObj = dyn_cast<object::COFFObjectFile>(&Obj)) { + auto &ES = getExecutionSession(); + + // For all resolved symbols that are not already in the responsibilty set: + // check whether the symbol is in a comdat section and if so mark it as + // weak. + for (auto &Sym : COFFObj->symbols()) { + // getFlags() on COFF symbols can't fail. + uint32_t SymFlags = cantFail(Sym.getFlags()); + if (SymFlags & object::BasicSymbolRef::SF_Undefined) + continue; + auto Name = Sym.getName(); + if (!Name) + return Name.takeError(); + auto I = Resolved.find(*Name); + + // Skip unresolved symbols, internal symbols, and symbols that are + // already in the responsibility set. + if (I == Resolved.end() || InternalSymbols.count(*Name) || + R.getSymbols().count(ES.intern(*Name))) + continue; + auto Sec = Sym.getSection(); + if (!Sec) + return Sec.takeError(); + if (*Sec == COFFObj->section_end()) + continue; + auto &COFFSec = *COFFObj->getCOFFSection(**Sec); + if (COFFSec.Characteristics & COFF::IMAGE_SCN_LNK_COMDAT) + I->second.setFlags(I->second.getFlags() | JITSymbolFlags::Weak); + } + } + + for (auto &KV : Resolved) { + // Scan the symbols and add them to the Symbols map for resolution. + + // We never claim internal symbols. + if (InternalSymbols.count(KV.first)) + continue; + + auto InternedName = getExecutionSession().intern(KV.first); + auto Flags = KV.second.getFlags(); + + // Override object flags and claim responsibility for symbols if + // requested. + if (OverrideObjectFlags || AutoClaimObjectSymbols) { + auto I = R.getSymbols().find(InternedName); + + if (OverrideObjectFlags && I != R.getSymbols().end()) + Flags = I->second; + else if (AutoClaimObjectSymbols && I == R.getSymbols().end()) + ExtraSymbolsToClaim[InternedName] = Flags; + } + + Symbols[InternedName] = JITEvaluatedSymbol(KV.second.getAddress(), Flags); + } + + if (!ExtraSymbolsToClaim.empty()) { + if (auto Err = R.defineMaterializing(ExtraSymbolsToClaim)) + return Err; + + // If we claimed responsibility for any weak symbols but were rejected then + // we need to remove them from the resolved set. + for (auto &KV : ExtraSymbolsToClaim) + if (KV.second.isWeak() && !R.getSymbols().count(KV.first)) + Symbols.erase(KV.first); + } + + if (auto Err = R.notifyResolved(Symbols)) { + R.failMaterialization(); + return Err; + } + + if (NotifyLoaded) + NotifyLoaded(R, Obj, LoadedObjInfo); + + return Error::success(); +} + +void RTDyldObjectLinkingLayer::onObjEmit( + MaterializationResponsibility &R, + object::OwningBinary<object::ObjectFile> O, + std::unique_ptr<RuntimeDyld::MemoryManager> MemMgr, + std::unique_ptr<RuntimeDyld::LoadedObjectInfo> LoadedObjInfo, Error Err) { + if (Err) { + getExecutionSession().reportError(std::move(Err)); + R.failMaterialization(); + return; + } + + if (auto Err = R.notifyEmitted()) { + getExecutionSession().reportError(std::move(Err)); + R.failMaterialization(); + return; + } + + std::unique_ptr<object::ObjectFile> Obj; + std::unique_ptr<MemoryBuffer> ObjBuffer; + std::tie(Obj, ObjBuffer) = O.takeBinary(); + + // Run EventListener notifyLoaded callbacks. + { + std::lock_guard<std::mutex> Lock(RTDyldLayerMutex); + for (auto *L : EventListeners) + L->notifyObjectLoaded(pointerToJITTargetAddress(MemMgr.get()), *Obj, + *LoadedObjInfo); + } + + if (NotifyEmitted) + NotifyEmitted(R, std::move(ObjBuffer)); + + if (auto Err = R.withResourceKeyDo( + [&](ResourceKey K) { MemMgrs[K].push_back(std::move(MemMgr)); })) { + getExecutionSession().reportError(std::move(Err)); + R.failMaterialization(); + } +} + +Error RTDyldObjectLinkingLayer::handleRemoveResources(ResourceKey K) { + + std::vector<MemoryManagerUP> MemMgrsToRemove; + + getExecutionSession().runSessionLocked([&] { + auto I = MemMgrs.find(K); + if (I != MemMgrs.end()) { + std::swap(MemMgrsToRemove, I->second); + MemMgrs.erase(I); + } + }); + + { + std::lock_guard<std::mutex> Lock(RTDyldLayerMutex); + for (auto &MemMgr : MemMgrsToRemove) { + for (auto *L : EventListeners) + L->notifyFreeingObject(pointerToJITTargetAddress(MemMgr.get())); + MemMgr->deregisterEHFrames(); + } + } + + return Error::success(); +} + +void RTDyldObjectLinkingLayer::handleTransferResources(ResourceKey DstKey, + ResourceKey SrcKey) { + auto I = MemMgrs.find(SrcKey); + if (I != MemMgrs.end()) { + auto &SrcMemMgrs = I->second; + auto &DstMemMgrs = MemMgrs[DstKey]; + DstMemMgrs.reserve(DstMemMgrs.size() + SrcMemMgrs.size()); + for (auto &MemMgr : SrcMemMgrs) + DstMemMgrs.push_back(std::move(MemMgr)); + + // Erase SrcKey entry using value rather than iterator I: I may have been + // invalidated when we looked up DstKey. + MemMgrs.erase(SrcKey); + } +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/Orc/Shared/OrcError.cpp b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/Shared/OrcError.cpp new file mode 100644 index 0000000000..fdad90cbcf --- /dev/null +++ b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/Shared/OrcError.cpp @@ -0,0 +1,120 @@ +//===---------------- OrcError.cpp - Error codes for ORC ------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Error codes for ORC. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/Shared/OrcError.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/ManagedStatic.h" + +#include <type_traits> + +using namespace llvm; +using namespace llvm::orc; + +namespace { + +// FIXME: This class is only here to support the transition to llvm::Error. It +// will be removed once this transition is complete. Clients should prefer to +// deal with the Error value directly, rather than converting to error_code. +class OrcErrorCategory : public std::error_category { +public: + const char *name() const noexcept override { return "orc"; } + + std::string message(int condition) const override { + switch (static_cast<OrcErrorCode>(condition)) { + case OrcErrorCode::UnknownORCError: + return "Unknown ORC error"; + case OrcErrorCode::DuplicateDefinition: + return "Duplicate symbol definition"; + case OrcErrorCode::JITSymbolNotFound: + return "JIT symbol not found"; + case OrcErrorCode::RemoteAllocatorDoesNotExist: + return "Remote allocator does not exist"; + case OrcErrorCode::RemoteAllocatorIdAlreadyInUse: + return "Remote allocator Id already in use"; + case OrcErrorCode::RemoteMProtectAddrUnrecognized: + return "Remote mprotect call references unallocated memory"; + case OrcErrorCode::RemoteIndirectStubsOwnerDoesNotExist: + return "Remote indirect stubs owner does not exist"; + case OrcErrorCode::RemoteIndirectStubsOwnerIdAlreadyInUse: + return "Remote indirect stubs owner Id already in use"; + case OrcErrorCode::RPCConnectionClosed: + return "RPC connection closed"; + case OrcErrorCode::RPCCouldNotNegotiateFunction: + return "Could not negotiate RPC function"; + case OrcErrorCode::RPCResponseAbandoned: + return "RPC response abandoned"; + case OrcErrorCode::UnexpectedRPCCall: + return "Unexpected RPC call"; + case OrcErrorCode::UnexpectedRPCResponse: + return "Unexpected RPC response"; + case OrcErrorCode::UnknownErrorCodeFromRemote: + return "Unknown error returned from remote RPC function " + "(Use StringError to get error message)"; + case OrcErrorCode::UnknownResourceHandle: + return "Unknown resource handle"; + case OrcErrorCode::MissingSymbolDefinitions: + return "MissingSymbolsDefinitions"; + case OrcErrorCode::UnexpectedSymbolDefinitions: + return "UnexpectedSymbolDefinitions"; + } + llvm_unreachable("Unhandled error code"); + } +}; + +static ManagedStatic<OrcErrorCategory> OrcErrCat; +} // namespace + +namespace llvm { +namespace orc { + +char DuplicateDefinition::ID = 0; +char JITSymbolNotFound::ID = 0; + +std::error_code orcError(OrcErrorCode ErrCode) { + typedef std::underlying_type<OrcErrorCode>::type UT; + return std::error_code(static_cast<UT>(ErrCode), *OrcErrCat); +} + +DuplicateDefinition::DuplicateDefinition(std::string SymbolName) + : SymbolName(std::move(SymbolName)) {} + +std::error_code DuplicateDefinition::convertToErrorCode() const { + return orcError(OrcErrorCode::DuplicateDefinition); +} + +void DuplicateDefinition::log(raw_ostream &OS) const { + OS << "Duplicate definition of symbol '" << SymbolName << "'"; +} + +const std::string &DuplicateDefinition::getSymbolName() const { + return SymbolName; +} + +JITSymbolNotFound::JITSymbolNotFound(std::string SymbolName) + : SymbolName(std::move(SymbolName)) {} + +std::error_code JITSymbolNotFound::convertToErrorCode() const { + typedef std::underlying_type<OrcErrorCode>::type UT; + return std::error_code(static_cast<UT>(OrcErrorCode::JITSymbolNotFound), + *OrcErrCat); +} + +void JITSymbolNotFound::log(raw_ostream &OS) const { + OS << "Could not find symbol '" << SymbolName << "'"; +} + +const std::string &JITSymbolNotFound::getSymbolName() const { + return SymbolName; +} + +} // namespace orc +} // namespace llvm diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/Orc/Shared/RPCError.cpp b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/Shared/RPCError.cpp new file mode 100644 index 0000000000..a55cb220f2 --- /dev/null +++ b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/Shared/RPCError.cpp @@ -0,0 +1,58 @@ +//===--------------- RPCError.cpp - RPCERror implementation ---------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// RPC Error type implmentations. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/Shared/RPCUtils.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/raw_ostream.h" + +#include <string> +#include <system_error> + +char llvm::orc::shared::RPCFatalError::ID = 0; +char llvm::orc::shared::ConnectionClosed::ID = 0; +char llvm::orc::shared::ResponseAbandoned::ID = 0; +char llvm::orc::shared::CouldNotNegotiate::ID = 0; + +namespace llvm { +namespace orc { +namespace shared { + +std::error_code ConnectionClosed::convertToErrorCode() const { + return orcError(OrcErrorCode::RPCConnectionClosed); +} + +void ConnectionClosed::log(raw_ostream &OS) const { + OS << "RPC connection already closed"; +} + +std::error_code ResponseAbandoned::convertToErrorCode() const { + return orcError(OrcErrorCode::RPCResponseAbandoned); +} + +void ResponseAbandoned::log(raw_ostream &OS) const { + OS << "RPC response abandoned"; +} + +CouldNotNegotiate::CouldNotNegotiate(std::string Signature) + : Signature(std::move(Signature)) {} + +std::error_code CouldNotNegotiate::convertToErrorCode() const { + return orcError(OrcErrorCode::RPCCouldNotNegotiateFunction); +} + +void CouldNotNegotiate::log(raw_ostream &OS) const { + OS << "Could not negotiate RPC function " << Signature; +} + +} // end namespace shared +} // end namespace orc +} // end namespace llvm diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.cpp b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.cpp new file mode 100644 index 0000000000..52d11f0741 --- /dev/null +++ b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.cpp @@ -0,0 +1,44 @@ +//===---------- TargetProcessControlTypes.cpp - Shared TPC types ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// TargetProcessControl types. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h" + +namespace llvm { +namespace orc { +namespace tpctypes { + +WrapperFunctionResult WrapperFunctionResult::from(StringRef S) { + CWrapperFunctionResult R; + zeroInit(R); + R.Size = S.size(); + if (R.Size > sizeof(uint64_t)) { + R.Data.ValuePtr = new uint8_t[R.Size]; + memcpy(R.Data.ValuePtr, S.data(), R.Size); + R.Destroy = destroyWithDeleteArray; + } else + memcpy(R.Data.Value, S.data(), R.Size); + return R; +} + +void WrapperFunctionResult::destroyWithFree(CWrapperFunctionResultData Data, + uint64_t Size) { + free(Data.ValuePtr); +} + +void WrapperFunctionResult::destroyWithDeleteArray( + CWrapperFunctionResultData Data, uint64_t Size) { + delete[] Data.ValuePtr; +} + +} // end namespace tpctypes +} // end namespace orc +} // end namespace llvm diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/Orc/SpeculateAnalyses.cpp b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/SpeculateAnalyses.cpp new file mode 100644 index 0000000000..c2fa4466ea --- /dev/null +++ b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/SpeculateAnalyses.cpp @@ -0,0 +1,306 @@ +//===-- SpeculateAnalyses.cpp --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/SpeculateAnalyses.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Analysis/BlockFrequencyInfo.h" +#include "llvm/Analysis/BranchProbabilityInfo.h" +#include "llvm/Analysis/CFG.h" +#include "llvm/IR/PassManager.h" +#include "llvm/Passes/PassBuilder.h" +#include "llvm/Support/ErrorHandling.h" + +#include <algorithm> + +namespace { +using namespace llvm; +SmallVector<const BasicBlock *, 8> findBBwithCalls(const Function &F, + bool IndirectCall = false) { + SmallVector<const BasicBlock *, 8> BBs; + + auto findCallInst = [&IndirectCall](const Instruction &I) { + if (auto Call = dyn_cast<CallBase>(&I)) + return Call->isIndirectCall() ? IndirectCall : true; + else + return false; + }; + for (auto &BB : F) + if (findCallInst(*BB.getTerminator()) || + llvm::any_of(BB.instructionsWithoutDebug(), findCallInst)) + BBs.emplace_back(&BB); + + return BBs; +} +} // namespace + +// Implementations of Queries shouldn't need to lock the resources +// such as LLVMContext, each argument (function) has a non-shared LLVMContext +// Plus, if Queries contain states necessary locking scheme should be provided. +namespace llvm { +namespace orc { + +// Collect direct calls only +void SpeculateQuery::findCalles(const BasicBlock *BB, + DenseSet<StringRef> &CallesNames) { + assert(BB != nullptr && "Traversing Null BB to find calls?"); + + auto getCalledFunction = [&CallesNames](const CallBase *Call) { + auto CalledValue = Call->getCalledOperand()->stripPointerCasts(); + if (auto DirectCall = dyn_cast<Function>(CalledValue)) + CallesNames.insert(DirectCall->getName()); + }; + for (auto &I : BB->instructionsWithoutDebug()) + if (auto CI = dyn_cast<CallInst>(&I)) + getCalledFunction(CI); + + if (auto II = dyn_cast<InvokeInst>(BB->getTerminator())) + getCalledFunction(II); +} + +bool SpeculateQuery::isStraightLine(const Function &F) { + return llvm::all_of(F.getBasicBlockList(), [](const BasicBlock &BB) { + return BB.getSingleSuccessor() != nullptr; + }); +} + +// BlockFreqQuery Implementations + +size_t BlockFreqQuery::numBBToGet(size_t numBB) { + // small CFG + if (numBB < 4) + return numBB; + // mid-size CFG + else if (numBB < 20) + return (numBB / 2); + else + return (numBB / 2) + (numBB / 4); +} + +BlockFreqQuery::ResultTy BlockFreqQuery::operator()(Function &F) { + DenseMap<StringRef, DenseSet<StringRef>> CallerAndCalles; + DenseSet<StringRef> Calles; + SmallVector<std::pair<const BasicBlock *, uint64_t>, 8> BBFreqs; + + PassBuilder PB; + FunctionAnalysisManager FAM; + PB.registerFunctionAnalyses(FAM); + + auto IBBs = findBBwithCalls(F); + + if (IBBs.empty()) + return None; + + auto &BFI = FAM.getResult<BlockFrequencyAnalysis>(F); + + for (const auto I : IBBs) + BBFreqs.push_back({I, BFI.getBlockFreq(I).getFrequency()}); + + assert(IBBs.size() == BBFreqs.size() && "BB Count Mismatch"); + + llvm::sort(BBFreqs, [](decltype(BBFreqs)::const_reference BBF, + decltype(BBFreqs)::const_reference BBS) { + return BBF.second > BBS.second ? true : false; + }); + + // ignoring number of direct calls in a BB + auto Topk = numBBToGet(BBFreqs.size()); + + for (size_t i = 0; i < Topk; i++) + findCalles(BBFreqs[i].first, Calles); + + assert(!Calles.empty() && "Running Analysis on Function with no calls?"); + + CallerAndCalles.insert({F.getName(), std::move(Calles)}); + + return CallerAndCalles; +} + +// SequenceBBQuery Implementation +std::size_t SequenceBBQuery::getHottestBlocks(std::size_t TotalBlocks) { + if (TotalBlocks == 1) + return TotalBlocks; + return TotalBlocks / 2; +} + +// FIXME : find good implementation. +SequenceBBQuery::BlockListTy +SequenceBBQuery::rearrangeBB(const Function &F, const BlockListTy &BBList) { + BlockListTy RearrangedBBSet; + + for (auto &Block : F.getBasicBlockList()) + if (llvm::is_contained(BBList, &Block)) + RearrangedBBSet.push_back(&Block); + + assert(RearrangedBBSet.size() == BBList.size() && + "BasicBlock missing while rearranging?"); + return RearrangedBBSet; +} + +void SequenceBBQuery::traverseToEntryBlock(const BasicBlock *AtBB, + const BlockListTy &CallerBlocks, + const BackEdgesInfoTy &BackEdgesInfo, + const BranchProbabilityInfo *BPI, + VisitedBlocksInfoTy &VisitedBlocks) { + auto Itr = VisitedBlocks.find(AtBB); + if (Itr != VisitedBlocks.end()) { // already visited. + if (!Itr->second.Upward) + return; + Itr->second.Upward = false; + } else { + // Create hint for newly discoverd blocks. + WalkDirection BlockHint; + BlockHint.Upward = false; + // FIXME: Expensive Check + if (llvm::is_contained(CallerBlocks, AtBB)) + BlockHint.CallerBlock = true; + VisitedBlocks.insert(std::make_pair(AtBB, BlockHint)); + } + + const_pred_iterator PIt = pred_begin(AtBB), EIt = pred_end(AtBB); + // Move this check to top, when we have code setup to launch speculative + // compiles for function in entry BB, this triggers the speculative compiles + // before running the program. + if (PIt == EIt) // No Preds. + return; + + DenseSet<const BasicBlock *> PredSkipNodes; + + // Since we are checking for predecessor's backedges, this Block + // occurs in second position. + for (auto &I : BackEdgesInfo) + if (I.second == AtBB) + PredSkipNodes.insert(I.first); + + // Skip predecessors which source of back-edges. + for (; PIt != EIt; ++PIt) + // checking EdgeHotness is cheaper + if (BPI->isEdgeHot(*PIt, AtBB) && !PredSkipNodes.count(*PIt)) + traverseToEntryBlock(*PIt, CallerBlocks, BackEdgesInfo, BPI, + VisitedBlocks); +} + +void SequenceBBQuery::traverseToExitBlock(const BasicBlock *AtBB, + const BlockListTy &CallerBlocks, + const BackEdgesInfoTy &BackEdgesInfo, + const BranchProbabilityInfo *BPI, + VisitedBlocksInfoTy &VisitedBlocks) { + auto Itr = VisitedBlocks.find(AtBB); + if (Itr != VisitedBlocks.end()) { // already visited. + if (!Itr->second.Downward) + return; + Itr->second.Downward = false; + } else { + // Create hint for newly discoverd blocks. + WalkDirection BlockHint; + BlockHint.Downward = false; + // FIXME: Expensive Check + if (llvm::is_contained(CallerBlocks, AtBB)) + BlockHint.CallerBlock = true; + VisitedBlocks.insert(std::make_pair(AtBB, BlockHint)); + } + + const_succ_iterator PIt = succ_begin(AtBB), EIt = succ_end(AtBB); + if (PIt == EIt) // No succs. + return; + + // If there are hot edges, then compute SuccSkipNodes. + DenseSet<const BasicBlock *> SuccSkipNodes; + + // Since we are checking for successor's backedges, this Block + // occurs in first position. + for (auto &I : BackEdgesInfo) + if (I.first == AtBB) + SuccSkipNodes.insert(I.second); + + for (; PIt != EIt; ++PIt) + if (BPI->isEdgeHot(AtBB, *PIt) && !SuccSkipNodes.count(*PIt)) + traverseToExitBlock(*PIt, CallerBlocks, BackEdgesInfo, BPI, + VisitedBlocks); +} + +// Get Block frequencies for blocks and take most frquently executed block, +// walk towards the entry block from those blocks and discover the basic blocks +// with call. +SequenceBBQuery::BlockListTy +SequenceBBQuery::queryCFG(Function &F, const BlockListTy &CallerBlocks) { + + BlockFreqInfoTy BBFreqs; + VisitedBlocksInfoTy VisitedBlocks; + BackEdgesInfoTy BackEdgesInfo; + + PassBuilder PB; + FunctionAnalysisManager FAM; + PB.registerFunctionAnalyses(FAM); + + auto &BFI = FAM.getResult<BlockFrequencyAnalysis>(F); + + llvm::FindFunctionBackedges(F, BackEdgesInfo); + + for (const auto I : CallerBlocks) + BBFreqs.push_back({I, BFI.getBlockFreq(I).getFrequency()}); + + llvm::sort(BBFreqs, [](decltype(BBFreqs)::const_reference Bbf, + decltype(BBFreqs)::const_reference Bbs) { + return Bbf.second > Bbs.second; + }); + + ArrayRef<std::pair<const BasicBlock *, uint64_t>> HotBlocksRef(BBFreqs); + HotBlocksRef = + HotBlocksRef.drop_back(BBFreqs.size() - getHottestBlocks(BBFreqs.size())); + + BranchProbabilityInfo *BPI = + FAM.getCachedResult<BranchProbabilityAnalysis>(F); + + // visit NHotBlocks, + // traverse upwards to entry + // traverse downwards to end. + + for (auto I : HotBlocksRef) { + traverseToEntryBlock(I.first, CallerBlocks, BackEdgesInfo, BPI, + VisitedBlocks); + traverseToExitBlock(I.first, CallerBlocks, BackEdgesInfo, BPI, + VisitedBlocks); + } + + BlockListTy MinCallerBlocks; + for (auto &I : VisitedBlocks) + if (I.second.CallerBlock) + MinCallerBlocks.push_back(std::move(I.first)); + + return rearrangeBB(F, MinCallerBlocks); +} + +SpeculateQuery::ResultTy SequenceBBQuery::operator()(Function &F) { + // reduce the number of lists! + DenseMap<StringRef, DenseSet<StringRef>> CallerAndCalles; + DenseSet<StringRef> Calles; + BlockListTy SequencedBlocks; + BlockListTy CallerBlocks; + + CallerBlocks = findBBwithCalls(F); + if (CallerBlocks.empty()) + return None; + + if (isStraightLine(F)) + SequencedBlocks = rearrangeBB(F, CallerBlocks); + else + SequencedBlocks = queryCFG(F, CallerBlocks); + + for (auto BB : SequencedBlocks) + findCalles(BB, Calles); + + CallerAndCalles.insert({F.getName(), std::move(Calles)}); + return CallerAndCalles; +} + +} // namespace orc +} // namespace llvm diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/Orc/Speculation.cpp b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/Speculation.cpp new file mode 100644 index 0000000000..0b4755fe23 --- /dev/null +++ b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/Speculation.cpp @@ -0,0 +1,143 @@ +//===---------- speculation.cpp - Utilities for Speculation ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/Speculation.h" +#include "llvm/IR/BasicBlock.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/Instruction.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/Module.h" +#include "llvm/IR/Type.h" +#include "llvm/IR/Verifier.h" + +namespace llvm { + +namespace orc { + +// ImplSymbolMap methods +void ImplSymbolMap::trackImpls(SymbolAliasMap ImplMaps, JITDylib *SrcJD) { + assert(SrcJD && "Tracking on Null Source .impl dylib"); + std::lock_guard<std::mutex> Lockit(ConcurrentAccess); + for (auto &I : ImplMaps) { + auto It = Maps.insert({I.first, {I.second.Aliasee, SrcJD}}); + // check rationale when independent dylibs have same symbol name? + assert(It.second && "ImplSymbols are already tracked for this Symbol?"); + (void)(It); + } +} + +// Trigger Speculative Compiles. +void Speculator::speculateForEntryPoint(Speculator *Ptr, uint64_t StubId) { + assert(Ptr && " Null Address Received in orc_speculate_for "); + Ptr->speculateFor(StubId); +} + +Error Speculator::addSpeculationRuntime(JITDylib &JD, + MangleAndInterner &Mangle) { + JITEvaluatedSymbol ThisPtr(pointerToJITTargetAddress(this), + JITSymbolFlags::Exported); + JITEvaluatedSymbol SpeculateForEntryPtr( + pointerToJITTargetAddress(&speculateForEntryPoint), + JITSymbolFlags::Exported); + return JD.define(absoluteSymbols({ + {Mangle("__orc_speculator"), ThisPtr}, // Data Symbol + {Mangle("__orc_speculate_for"), SpeculateForEntryPtr} // Callable Symbol + })); +} + +// If two modules, share the same LLVMContext, different threads must +// not access them concurrently without locking the associated LLVMContext +// this implementation follows this contract. +void IRSpeculationLayer::emit(std::unique_ptr<MaterializationResponsibility> R, + ThreadSafeModule TSM) { + + assert(TSM && "Speculation Layer received Null Module ?"); + assert(TSM.getContext().getContext() != nullptr && + "Module with null LLVMContext?"); + + // Instrumentation of runtime calls, lock the Module + TSM.withModuleDo([this, &R](Module &M) { + auto &MContext = M.getContext(); + auto SpeculatorVTy = StructType::create(MContext, "Class.Speculator"); + auto RuntimeCallTy = FunctionType::get( + Type::getVoidTy(MContext), + {SpeculatorVTy->getPointerTo(), Type::getInt64Ty(MContext)}, false); + auto RuntimeCall = + Function::Create(RuntimeCallTy, Function::LinkageTypes::ExternalLinkage, + "__orc_speculate_for", &M); + auto SpeclAddr = new GlobalVariable( + M, SpeculatorVTy, false, GlobalValue::LinkageTypes::ExternalLinkage, + nullptr, "__orc_speculator"); + + IRBuilder<> Mutator(MContext); + + // QueryAnalysis allowed to transform the IR source, one such example is + // Simplify CFG helps the static branch prediction heuristics! + for (auto &Fn : M.getFunctionList()) { + if (!Fn.isDeclaration()) { + + auto IRNames = QueryAnalysis(Fn); + // Instrument and register if Query has result + if (IRNames.hasValue()) { + + // Emit globals for each function. + auto LoadValueTy = Type::getInt8Ty(MContext); + auto SpeculatorGuard = new GlobalVariable( + M, LoadValueTy, false, GlobalValue::LinkageTypes::InternalLinkage, + ConstantInt::get(LoadValueTy, 0), + "__orc_speculate.guard.for." + Fn.getName()); + SpeculatorGuard->setAlignment(Align(1)); + SpeculatorGuard->setUnnamedAddr(GlobalValue::UnnamedAddr::Local); + + BasicBlock &ProgramEntry = Fn.getEntryBlock(); + // Create BasicBlocks before the program's entry basicblock + BasicBlock *SpeculateBlock = BasicBlock::Create( + MContext, "__orc_speculate.block", &Fn, &ProgramEntry); + BasicBlock *SpeculateDecisionBlock = BasicBlock::Create( + MContext, "__orc_speculate.decision.block", &Fn, SpeculateBlock); + + assert(SpeculateDecisionBlock == &Fn.getEntryBlock() && + "SpeculateDecisionBlock not updated?"); + Mutator.SetInsertPoint(SpeculateDecisionBlock); + + auto LoadGuard = + Mutator.CreateLoad(LoadValueTy, SpeculatorGuard, "guard.value"); + // if just loaded value equal to 0,return true. + auto CanSpeculate = + Mutator.CreateICmpEQ(LoadGuard, ConstantInt::get(LoadValueTy, 0), + "compare.to.speculate"); + Mutator.CreateCondBr(CanSpeculate, SpeculateBlock, &ProgramEntry); + + Mutator.SetInsertPoint(SpeculateBlock); + auto ImplAddrToUint = + Mutator.CreatePtrToInt(&Fn, Type::getInt64Ty(MContext)); + Mutator.CreateCall(RuntimeCallTy, RuntimeCall, + {SpeclAddr, ImplAddrToUint}); + Mutator.CreateStore(ConstantInt::get(LoadValueTy, 1), + SpeculatorGuard); + Mutator.CreateBr(&ProgramEntry); + + assert(Mutator.GetInsertBlock()->getParent() == &Fn && + "IR builder association mismatch?"); + S.registerSymbols(internToJITSymbols(IRNames.getValue()), + &R->getTargetJITDylib()); + } + } + } + }); + + assert(!TSM.withModuleDo([](const Module &M) { return verifyModule(M); }) && + "Speculation Instrumentation breaks IR?"); + + NextLayer.emit(std::move(R), std::move(TSM)); +} + +} // namespace orc +} // namespace llvm diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/Orc/TPCDynamicLibrarySearchGenerator.cpp b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/TPCDynamicLibrarySearchGenerator.cpp new file mode 100644 index 0000000000..bbf3ada1d4 --- /dev/null +++ b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/TPCDynamicLibrarySearchGenerator.cpp @@ -0,0 +1,70 @@ +//===---------------- TPCDynamicLibrarySearchGenerator.cpp ----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/TPCDynamicLibrarySearchGenerator.h" + +namespace llvm { +namespace orc { + +Expected<std::unique_ptr<TPCDynamicLibrarySearchGenerator>> +TPCDynamicLibrarySearchGenerator::Load(TargetProcessControl &TPC, + const char *LibraryPath, + SymbolPredicate Allow) { + auto Handle = TPC.loadDylib(LibraryPath); + if (!Handle) + return Handle.takeError(); + + return std::make_unique<TPCDynamicLibrarySearchGenerator>(TPC, *Handle, + std::move(Allow)); +} + +Error TPCDynamicLibrarySearchGenerator::tryToGenerate( + LookupState &LS, LookupKind K, JITDylib &JD, + JITDylibLookupFlags JDLookupFlags, const SymbolLookupSet &Symbols) { + + if (Symbols.empty()) + return Error::success(); + + SymbolLookupSet LookupSymbols; + + for (auto &KV : Symbols) { + // Skip symbols that don't match the filter. + if (Allow && !Allow(KV.first)) + continue; + LookupSymbols.add(KV.first, SymbolLookupFlags::WeaklyReferencedSymbol); + } + + SymbolMap NewSymbols; + + TargetProcessControl::LookupRequest Request(H, LookupSymbols); + auto Result = TPC.lookupSymbols(Request); + if (!Result) + return Result.takeError(); + + assert(Result->size() == 1 && "Results for more than one library returned"); + assert(Result->front().size() == LookupSymbols.size() && + "Result has incorrect number of elements"); + + auto ResultI = Result->front().begin(); + for (auto &KV : LookupSymbols) { + if (*ResultI) + NewSymbols[KV.first] = + JITEvaluatedSymbol(*ResultI, JITSymbolFlags::Exported); + ++ResultI; + } + + // If there were no resolved symbols bail out. + if (NewSymbols.empty()) + return Error::success(); + + // Define resolved symbols. + return JD.define(absoluteSymbols(std::move(NewSymbols))); +} + +} // end namespace orc +} // end namespace llvm diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/Orc/TPCEHFrameRegistrar.cpp b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/TPCEHFrameRegistrar.cpp new file mode 100644 index 0000000000..4f901ce6d4 --- /dev/null +++ b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/TPCEHFrameRegistrar.cpp @@ -0,0 +1,80 @@ +//===------ TPCEHFrameRegistrar.cpp - TPC-based eh-frame registration -----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/TPCEHFrameRegistrar.h" +#include "llvm/Support/BinaryStreamWriter.h" + +namespace llvm { +namespace orc { + +Expected<std::unique_ptr<TPCEHFrameRegistrar>> +TPCEHFrameRegistrar::Create(TargetProcessControl &TPC) { + // FIXME: Proper mangling here -- we really need to decouple linker mangling + // from DataLayout. + + // Find the addresses of the registration/deregistration functions in the + // target process. + auto ProcessHandle = TPC.loadDylib(nullptr); + if (!ProcessHandle) + return ProcessHandle.takeError(); + + std::string RegisterWrapperName, DeregisterWrapperName; + if (TPC.getTargetTriple().isOSBinFormatMachO()) { + RegisterWrapperName += '_'; + DeregisterWrapperName += '_'; + } + RegisterWrapperName += "llvm_orc_registerEHFrameSectionWrapper"; + DeregisterWrapperName += "llvm_orc_deregisterEHFrameSectionWrapper"; + + SymbolLookupSet RegistrationSymbols; + RegistrationSymbols.add(TPC.intern(RegisterWrapperName)); + RegistrationSymbols.add(TPC.intern(DeregisterWrapperName)); + + auto Result = TPC.lookupSymbols({{*ProcessHandle, RegistrationSymbols}}); + if (!Result) + return Result.takeError(); + + assert(Result->size() == 1 && "Unexpected number of dylibs in result"); + assert((*Result)[0].size() == 2 && + "Unexpected number of addresses in result"); + + auto RegisterEHFrameWrapperFnAddr = (*Result)[0][0]; + auto DeregisterEHFrameWrapperFnAddr = (*Result)[0][1]; + + return std::make_unique<TPCEHFrameRegistrar>( + TPC, RegisterEHFrameWrapperFnAddr, DeregisterEHFrameWrapperFnAddr); +} + +Error TPCEHFrameRegistrar::registerEHFrames(JITTargetAddress EHFrameSectionAddr, + size_t EHFrameSectionSize) { + constexpr size_t ArgBufferSize = sizeof(uint64_t) + sizeof(uint64_t); + uint8_t ArgBuffer[ArgBufferSize]; + BinaryStreamWriter ArgWriter( + MutableArrayRef<uint8_t>(ArgBuffer, ArgBufferSize), + support::endianness::big); + cantFail(ArgWriter.writeInteger(static_cast<uint64_t>(EHFrameSectionAddr))); + cantFail(ArgWriter.writeInteger(static_cast<uint64_t>(EHFrameSectionSize))); + + return TPC.runWrapper(RegisterEHFrameWrapperFnAddr, ArgBuffer).takeError(); +} + +Error TPCEHFrameRegistrar::deregisterEHFrames( + JITTargetAddress EHFrameSectionAddr, size_t EHFrameSectionSize) { + constexpr size_t ArgBufferSize = sizeof(uint64_t) + sizeof(uint64_t); + uint8_t ArgBuffer[ArgBufferSize]; + BinaryStreamWriter ArgWriter( + MutableArrayRef<uint8_t>(ArgBuffer, ArgBufferSize), + support::endianness::big); + cantFail(ArgWriter.writeInteger(static_cast<uint64_t>(EHFrameSectionAddr))); + cantFail(ArgWriter.writeInteger(static_cast<uint64_t>(EHFrameSectionSize))); + + return TPC.runWrapper(DeregisterEHFrameWrapperFnAddr, ArgBuffer).takeError(); +} + +} // end namespace orc +} // end namespace llvm diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/Orc/TPCIndirectionUtils.cpp b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/TPCIndirectionUtils.cpp new file mode 100644 index 0000000000..7989ec4195 --- /dev/null +++ b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/TPCIndirectionUtils.cpp @@ -0,0 +1,423 @@ +//===------ TargetProcessControl.cpp -- Target process control APIs -------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/TPCIndirectionUtils.h" + +#include "llvm/ExecutionEngine/Orc/TargetProcessControl.h" +#include "llvm/Support/MathExtras.h" + +#include <future> + +using namespace llvm; +using namespace llvm::orc; + +namespace llvm { +namespace orc { + +class TPCIndirectionUtilsAccess { +public: + using IndirectStubInfo = TPCIndirectionUtils::IndirectStubInfo; + using IndirectStubInfoVector = TPCIndirectionUtils::IndirectStubInfoVector; + + static Expected<IndirectStubInfoVector> + getIndirectStubs(TPCIndirectionUtils &TPCIU, unsigned NumStubs) { + return TPCIU.getIndirectStubs(NumStubs); + }; +}; + +} // end namespace orc +} // end namespace llvm + +namespace { + +class TPCTrampolinePool : public TrampolinePool { +public: + TPCTrampolinePool(TPCIndirectionUtils &TPCIU); + Error deallocatePool(); + +protected: + Error grow() override; + + using Allocation = jitlink::JITLinkMemoryManager::Allocation; + + TPCIndirectionUtils &TPCIU; + unsigned TrampolineSize = 0; + unsigned TrampolinesPerPage = 0; + std::vector<std::unique_ptr<Allocation>> TrampolineBlocks; +}; + +class TPCIndirectStubsManager : public IndirectStubsManager, + private TPCIndirectionUtilsAccess { +public: + TPCIndirectStubsManager(TPCIndirectionUtils &TPCIU) : TPCIU(TPCIU) {} + + Error deallocateStubs(); + + Error createStub(StringRef StubName, JITTargetAddress StubAddr, + JITSymbolFlags StubFlags) override; + + Error createStubs(const StubInitsMap &StubInits) override; + + JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly) override; + + JITEvaluatedSymbol findPointer(StringRef Name) override; + + Error updatePointer(StringRef Name, JITTargetAddress NewAddr) override; + +private: + using StubInfo = std::pair<IndirectStubInfo, JITSymbolFlags>; + + std::mutex ISMMutex; + TPCIndirectionUtils &TPCIU; + StringMap<StubInfo> StubInfos; +}; + +TPCTrampolinePool::TPCTrampolinePool(TPCIndirectionUtils &TPCIU) + : TPCIU(TPCIU) { + auto &TPC = TPCIU.getTargetProcessControl(); + auto &ABI = TPCIU.getABISupport(); + + TrampolineSize = ABI.getTrampolineSize(); + TrampolinesPerPage = + (TPC.getPageSize() - ABI.getPointerSize()) / TrampolineSize; +} + +Error TPCTrampolinePool::deallocatePool() { + Error Err = Error::success(); + for (auto &Alloc : TrampolineBlocks) + Err = joinErrors(std::move(Err), Alloc->deallocate()); + return Err; +} + +Error TPCTrampolinePool::grow() { + assert(AvailableTrampolines.empty() && + "Grow called with trampolines still available"); + + auto ResolverAddress = TPCIU.getResolverBlockAddress(); + assert(ResolverAddress && "Resolver address can not be null"); + + auto &TPC = TPCIU.getTargetProcessControl(); + constexpr auto TrampolinePagePermissions = + static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | + sys::Memory::MF_EXEC); + auto PageSize = TPC.getPageSize(); + jitlink::JITLinkMemoryManager::SegmentsRequestMap Request; + Request[TrampolinePagePermissions] = {PageSize, static_cast<size_t>(PageSize), + 0}; + auto Alloc = TPC.getMemMgr().allocate(nullptr, Request); + + if (!Alloc) + return Alloc.takeError(); + + unsigned NumTrampolines = TrampolinesPerPage; + + auto WorkingMemory = (*Alloc)->getWorkingMemory(TrampolinePagePermissions); + auto TargetAddress = (*Alloc)->getTargetMemory(TrampolinePagePermissions); + + TPCIU.getABISupport().writeTrampolines(WorkingMemory.data(), TargetAddress, + ResolverAddress, NumTrampolines); + + auto TargetAddr = (*Alloc)->getTargetMemory(TrampolinePagePermissions); + for (unsigned I = 0; I < NumTrampolines; ++I) + AvailableTrampolines.push_back(TargetAddr + (I * TrampolineSize)); + + if (auto Err = (*Alloc)->finalize()) + return Err; + + TrampolineBlocks.push_back(std::move(*Alloc)); + + return Error::success(); +} + +Error TPCIndirectStubsManager::createStub(StringRef StubName, + JITTargetAddress StubAddr, + JITSymbolFlags StubFlags) { + StubInitsMap SIM; + SIM[StubName] = std::make_pair(StubAddr, StubFlags); + return createStubs(SIM); +} + +Error TPCIndirectStubsManager::createStubs(const StubInitsMap &StubInits) { + auto AvailableStubInfos = getIndirectStubs(TPCIU, StubInits.size()); + if (!AvailableStubInfos) + return AvailableStubInfos.takeError(); + + { + std::lock_guard<std::mutex> Lock(ISMMutex); + unsigned ASIdx = 0; + for (auto &SI : StubInits) { + auto &A = (*AvailableStubInfos)[ASIdx++]; + StubInfos[SI.first()] = std::make_pair(A, SI.second.second); + } + } + + auto &MemAccess = TPCIU.getTargetProcessControl().getMemoryAccess(); + switch (TPCIU.getABISupport().getPointerSize()) { + case 4: { + unsigned ASIdx = 0; + std::vector<tpctypes::UInt32Write> PtrUpdates; + for (auto &SI : StubInits) + PtrUpdates.push_back({(*AvailableStubInfos)[ASIdx++].PointerAddress, + static_cast<uint32_t>(SI.second.first)}); + return MemAccess.writeUInt32s(PtrUpdates); + } + case 8: { + unsigned ASIdx = 0; + std::vector<tpctypes::UInt64Write> PtrUpdates; + for (auto &SI : StubInits) + PtrUpdates.push_back({(*AvailableStubInfos)[ASIdx++].PointerAddress, + static_cast<uint64_t>(SI.second.first)}); + return MemAccess.writeUInt64s(PtrUpdates); + } + default: + return make_error<StringError>("Unsupported pointer size", + inconvertibleErrorCode()); + } +} + +JITEvaluatedSymbol TPCIndirectStubsManager::findStub(StringRef Name, + bool ExportedStubsOnly) { + std::lock_guard<std::mutex> Lock(ISMMutex); + auto I = StubInfos.find(Name); + if (I == StubInfos.end()) + return nullptr; + return {I->second.first.StubAddress, I->second.second}; +} + +JITEvaluatedSymbol TPCIndirectStubsManager::findPointer(StringRef Name) { + std::lock_guard<std::mutex> Lock(ISMMutex); + auto I = StubInfos.find(Name); + if (I == StubInfos.end()) + return nullptr; + return {I->second.first.PointerAddress, I->second.second}; +} + +Error TPCIndirectStubsManager::updatePointer(StringRef Name, + JITTargetAddress NewAddr) { + + JITTargetAddress PtrAddr = 0; + { + std::lock_guard<std::mutex> Lock(ISMMutex); + auto I = StubInfos.find(Name); + if (I == StubInfos.end()) + return make_error<StringError>("Unknown stub name", + inconvertibleErrorCode()); + PtrAddr = I->second.first.PointerAddress; + } + + auto &MemAccess = TPCIU.getTargetProcessControl().getMemoryAccess(); + switch (TPCIU.getABISupport().getPointerSize()) { + case 4: { + tpctypes::UInt32Write PUpdate(PtrAddr, NewAddr); + return MemAccess.writeUInt32s(PUpdate); + } + case 8: { + tpctypes::UInt64Write PUpdate(PtrAddr, NewAddr); + return MemAccess.writeUInt64s(PUpdate); + } + default: + return make_error<StringError>("Unsupported pointer size", + inconvertibleErrorCode()); + } +} + +} // end anonymous namespace. + +namespace llvm { +namespace orc { + +TPCIndirectionUtils::ABISupport::~ABISupport() {} + +Expected<std::unique_ptr<TPCIndirectionUtils>> +TPCIndirectionUtils::Create(TargetProcessControl &TPC) { + const auto &TT = TPC.getTargetTriple(); + switch (TT.getArch()) { + default: + return make_error<StringError>( + std::string("No TPCIndirectionUtils available for ") + TT.str(), + inconvertibleErrorCode()); + case Triple::aarch64: + case Triple::aarch64_32: + return CreateWithABI<OrcAArch64>(TPC); + + case Triple::x86: + return CreateWithABI<OrcI386>(TPC); + + case Triple::mips: + return CreateWithABI<OrcMips32Be>(TPC); + + case Triple::mipsel: + return CreateWithABI<OrcMips32Le>(TPC); + + case Triple::mips64: + case Triple::mips64el: + return CreateWithABI<OrcMips64>(TPC); + + case Triple::x86_64: + if (TT.getOS() == Triple::OSType::Win32) + return CreateWithABI<OrcX86_64_Win32>(TPC); + else + return CreateWithABI<OrcX86_64_SysV>(TPC); + } +} + +Error TPCIndirectionUtils::cleanup() { + Error Err = Error::success(); + + for (auto &A : IndirectStubAllocs) + Err = joinErrors(std::move(Err), A->deallocate()); + + if (TP) + Err = joinErrors(std::move(Err), + static_cast<TPCTrampolinePool &>(*TP).deallocatePool()); + + if (ResolverBlock) + Err = joinErrors(std::move(Err), ResolverBlock->deallocate()); + + return Err; +} + +Expected<JITTargetAddress> +TPCIndirectionUtils::writeResolverBlock(JITTargetAddress ReentryFnAddr, + JITTargetAddress ReentryCtxAddr) { + assert(ABI && "ABI can not be null"); + constexpr auto ResolverBlockPermissions = + static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | + sys::Memory::MF_EXEC); + auto ResolverSize = ABI->getResolverCodeSize(); + + jitlink::JITLinkMemoryManager::SegmentsRequestMap Request; + Request[ResolverBlockPermissions] = {TPC.getPageSize(), + static_cast<size_t>(ResolverSize), 0}; + auto Alloc = TPC.getMemMgr().allocate(nullptr, Request); + if (!Alloc) + return Alloc.takeError(); + + auto WorkingMemory = (*Alloc)->getWorkingMemory(ResolverBlockPermissions); + ResolverBlockAddr = (*Alloc)->getTargetMemory(ResolverBlockPermissions); + ABI->writeResolverCode(WorkingMemory.data(), ResolverBlockAddr, ReentryFnAddr, + ReentryCtxAddr); + + if (auto Err = (*Alloc)->finalize()) + return std::move(Err); + + ResolverBlock = std::move(*Alloc); + return ResolverBlockAddr; +} + +std::unique_ptr<IndirectStubsManager> +TPCIndirectionUtils::createIndirectStubsManager() { + return std::make_unique<TPCIndirectStubsManager>(*this); +} + +TrampolinePool &TPCIndirectionUtils::getTrampolinePool() { + if (!TP) + TP = std::make_unique<TPCTrampolinePool>(*this); + return *TP; +} + +LazyCallThroughManager &TPCIndirectionUtils::createLazyCallThroughManager( + ExecutionSession &ES, JITTargetAddress ErrorHandlerAddr) { + assert(!LCTM && + "createLazyCallThroughManager can not have been called before"); + LCTM = std::make_unique<LazyCallThroughManager>(ES, ErrorHandlerAddr, + &getTrampolinePool()); + return *LCTM; +} + +TPCIndirectionUtils::TPCIndirectionUtils(TargetProcessControl &TPC, + std::unique_ptr<ABISupport> ABI) + : TPC(TPC), ABI(std::move(ABI)) { + assert(this->ABI && "ABI can not be null"); + + assert(TPC.getPageSize() > getABISupport().getStubSize() && + "Stubs larger than one page are not supported"); +} + +Expected<TPCIndirectionUtils::IndirectStubInfoVector> +TPCIndirectionUtils::getIndirectStubs(unsigned NumStubs) { + + std::lock_guard<std::mutex> Lock(TPCUIMutex); + + // If there aren't enough stubs available then allocate some more. + if (NumStubs > AvailableIndirectStubs.size()) { + auto NumStubsToAllocate = NumStubs; + auto PageSize = TPC.getPageSize(); + auto StubBytes = alignTo(NumStubsToAllocate * ABI->getStubSize(), PageSize); + NumStubsToAllocate = StubBytes / ABI->getStubSize(); + auto PointerBytes = + alignTo(NumStubsToAllocate * ABI->getPointerSize(), PageSize); + + constexpr auto StubPagePermissions = + static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | + sys::Memory::MF_EXEC); + constexpr auto PointerPagePermissions = + static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | + sys::Memory::MF_WRITE); + + jitlink::JITLinkMemoryManager::SegmentsRequestMap Request; + Request[StubPagePermissions] = {PageSize, static_cast<size_t>(StubBytes), + 0}; + Request[PointerPagePermissions] = {PageSize, 0, PointerBytes}; + auto Alloc = TPC.getMemMgr().allocate(nullptr, Request); + if (!Alloc) + return Alloc.takeError(); + + auto StubTargetAddr = (*Alloc)->getTargetMemory(StubPagePermissions); + auto PointerTargetAddr = (*Alloc)->getTargetMemory(PointerPagePermissions); + + ABI->writeIndirectStubsBlock( + (*Alloc)->getWorkingMemory(StubPagePermissions).data(), StubTargetAddr, + PointerTargetAddr, NumStubsToAllocate); + + if (auto Err = (*Alloc)->finalize()) + return std::move(Err); + + for (unsigned I = 0; I != NumStubsToAllocate; ++I) { + AvailableIndirectStubs.push_back( + IndirectStubInfo(StubTargetAddr, PointerTargetAddr)); + StubTargetAddr += ABI->getStubSize(); + PointerTargetAddr += ABI->getPointerSize(); + } + + IndirectStubAllocs.push_back(std::move(*Alloc)); + } + + assert(NumStubs <= AvailableIndirectStubs.size() && + "Sufficient stubs should have been allocated above"); + + IndirectStubInfoVector Result; + while (NumStubs--) { + Result.push_back(AvailableIndirectStubs.back()); + AvailableIndirectStubs.pop_back(); + } + + return std::move(Result); +} + +static JITTargetAddress reentry(JITTargetAddress LCTMAddr, + JITTargetAddress TrampolineAddr) { + auto &LCTM = *jitTargetAddressToPointer<LazyCallThroughManager *>(LCTMAddr); + std::promise<JITTargetAddress> LandingAddrP; + auto LandingAddrF = LandingAddrP.get_future(); + LCTM.resolveTrampolineLandingAddress( + TrampolineAddr, + [&](JITTargetAddress Addr) { LandingAddrP.set_value(Addr); }); + return LandingAddrF.get(); +} + +Error setUpInProcessLCTMReentryViaTPCIU(TPCIndirectionUtils &TPCIU) { + auto &LCTM = TPCIU.getLazyCallThroughManager(); + return TPCIU + .writeResolverBlock(pointerToJITTargetAddress(&reentry), + pointerToJITTargetAddress(&LCTM)) + .takeError(); +} + +} // end namespace orc +} // end namespace llvm diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.cpp b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.cpp new file mode 100644 index 0000000000..aff7296cb6 --- /dev/null +++ b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.cpp @@ -0,0 +1,208 @@ +//===--------- RegisterEHFrames.cpp - Register EH frame sections ----------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h" + +#include "llvm/Config/config.h" +#include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/Support/BinaryStreamReader.h" +#include "llvm/Support/Compiler.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/DynamicLibrary.h" +#include "llvm/Support/raw_ostream.h" + +#include "llvm/Support/FormatVariadic.h" + +#define DEBUG_TYPE "orc" + +using namespace llvm; +using namespace llvm::orc; +using namespace llvm::orc::tpctypes; + +namespace llvm { +namespace orc { + +#if defined(HAVE_REGISTER_FRAME) && defined(HAVE_DEREGISTER_FRAME) && \ + !defined(__SEH__) && !defined(__USING_SJLJ_EXCEPTIONS__) + +extern "C" void __register_frame(const void *); +extern "C" void __deregister_frame(const void *); + +Error registerFrameWrapper(const void *P) { + __register_frame(P); + return Error::success(); +} + +Error deregisterFrameWrapper(const void *P) { + __deregister_frame(P); + return Error::success(); +} + +#else + +// The building compiler does not have __(de)register_frame but +// it may be found at runtime in a dynamically-loaded library. +// For example, this happens when building LLVM with Visual C++ +// but using the MingW runtime. +static Error registerFrameWrapper(const void *P) { + static void((*RegisterFrame)(const void *)) = 0; + + if (!RegisterFrame) + *(void **)&RegisterFrame = + llvm::sys::DynamicLibrary::SearchForAddressOfSymbol("__register_frame"); + + if (RegisterFrame) { + RegisterFrame(P); + return Error::success(); + } + + return make_error<StringError>("could not register eh-frame: " + "__register_frame function not found", + inconvertibleErrorCode()); +} + +static Error deregisterFrameWrapper(const void *P) { + static void((*DeregisterFrame)(const void *)) = 0; + + if (!DeregisterFrame) + *(void **)&DeregisterFrame = + llvm::sys::DynamicLibrary::SearchForAddressOfSymbol( + "__deregister_frame"); + + if (DeregisterFrame) { + DeregisterFrame(P); + return Error::success(); + } + + return make_error<StringError>("could not deregister eh-frame: " + "__deregister_frame function not found", + inconvertibleErrorCode()); +} +#endif + +#ifdef __APPLE__ + +template <typename HandleFDEFn> +Error walkAppleEHFrameSection(const char *const SectionStart, + size_t SectionSize, HandleFDEFn HandleFDE) { + const char *CurCFIRecord = SectionStart; + const char *End = SectionStart + SectionSize; + uint64_t Size = *reinterpret_cast<const uint32_t *>(CurCFIRecord); + + while (CurCFIRecord != End && Size != 0) { + const char *OffsetField = CurCFIRecord + (Size == 0xffffffff ? 12 : 4); + if (Size == 0xffffffff) + Size = *reinterpret_cast<const uint64_t *>(CurCFIRecord + 4) + 12; + else + Size += 4; + uint32_t Offset = *reinterpret_cast<const uint32_t *>(OffsetField); + + LLVM_DEBUG({ + dbgs() << "Registering eh-frame section:\n"; + dbgs() << "Processing " << (Offset ? "FDE" : "CIE") << " @" + << (void *)CurCFIRecord << ": ["; + for (unsigned I = 0; I < Size; ++I) + dbgs() << format(" 0x%02" PRIx8, *(CurCFIRecord + I)); + dbgs() << " ]\n"; + }); + + if (Offset != 0) + if (auto Err = HandleFDE(CurCFIRecord)) + return Err; + + CurCFIRecord += Size; + + Size = *reinterpret_cast<const uint32_t *>(CurCFIRecord); + } + + return Error::success(); +} + +#endif // __APPLE__ + +Error registerEHFrameSection(const void *EHFrameSectionAddr, + size_t EHFrameSectionSize) { +#ifdef __APPLE__ + // On Darwin __register_frame has to be called for each FDE entry. + return walkAppleEHFrameSection(static_cast<const char *>(EHFrameSectionAddr), + EHFrameSectionSize, registerFrameWrapper); +#else + // On Linux __register_frame takes a single argument: + // a pointer to the start of the .eh_frame section. + + // How can it find the end? Because crtendS.o is linked + // in and it has an .eh_frame section with four zero chars. + return registerFrameWrapper(EHFrameSectionAddr); +#endif +} + +Error deregisterEHFrameSection(const void *EHFrameSectionAddr, + size_t EHFrameSectionSize) { +#ifdef __APPLE__ + return walkAppleEHFrameSection(static_cast<const char *>(EHFrameSectionAddr), + EHFrameSectionSize, deregisterFrameWrapper); +#else + return deregisterFrameWrapper(EHFrameSectionAddr); +#endif +} + +} // end namespace orc +} // end namespace llvm + +extern "C" CWrapperFunctionResult +llvm_orc_registerEHFrameSectionWrapper(uint8_t *Data, uint64_t Size) { + if (Size != sizeof(uint64_t) + sizeof(uint64_t)) + return WrapperFunctionResult::from( + "Invalid arguments to llvm_orc_registerEHFrameSectionWrapper") + .release(); + + uint64_t EHFrameSectionAddr; + uint64_t EHFrameSectionSize; + + { + BinaryStreamReader ArgReader(ArrayRef<uint8_t>(Data, Size), + support::endianness::big); + cantFail(ArgReader.readInteger(EHFrameSectionAddr)); + cantFail(ArgReader.readInteger(EHFrameSectionSize)); + } + + if (auto Err = registerEHFrameSection( + jitTargetAddressToPointer<void *>(EHFrameSectionAddr), + EHFrameSectionSize)) { + auto ErrMsg = toString(std::move(Err)); + return WrapperFunctionResult::from(ErrMsg).release(); + } + return WrapperFunctionResult().release(); +} + +extern "C" CWrapperFunctionResult +llvm_orc_deregisterEHFrameSectionWrapper(uint8_t *Data, uint64_t Size) { + if (Size != sizeof(uint64_t) + sizeof(uint64_t)) + return WrapperFunctionResult::from( + "Invalid arguments to llvm_orc_registerEHFrameSectionWrapper") + .release(); + + uint64_t EHFrameSectionAddr; + uint64_t EHFrameSectionSize; + + { + BinaryStreamReader ArgReader(ArrayRef<uint8_t>(Data, Size), + support::endianness::big); + cantFail(ArgReader.readInteger(EHFrameSectionAddr)); + cantFail(ArgReader.readInteger(EHFrameSectionSize)); + } + + if (auto Err = deregisterEHFrameSection( + jitTargetAddressToPointer<void *>(EHFrameSectionAddr), + EHFrameSectionSize)) { + auto ErrMsg = toString(std::move(Err)); + return WrapperFunctionResult::from(ErrMsg).release(); + } + return WrapperFunctionResult().release(); +} diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/Orc/TargetProcess/TargetExecutionUtils.cpp b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/TargetProcess/TargetExecutionUtils.cpp new file mode 100644 index 0000000000..a8e6c049cf --- /dev/null +++ b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/TargetProcess/TargetExecutionUtils.cpp @@ -0,0 +1,43 @@ +//===--- TargetExecutionUtils.cpp - Execution utils for target processes --===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/TargetProcess/TargetExecutionUtils.h" + +#include <vector> + +namespace llvm { +namespace orc { + +int runAsMain(int (*Main)(int, char *[]), ArrayRef<std::string> Args, + Optional<StringRef> ProgramName) { + std::vector<std::unique_ptr<char[]>> ArgVStorage; + std::vector<char *> ArgV; + + ArgVStorage.reserve(Args.size() + (ProgramName ? 1 : 0)); + ArgV.reserve(Args.size() + 1 + (ProgramName ? 1 : 0)); + + if (ProgramName) { + ArgVStorage.push_back(std::make_unique<char[]>(ProgramName->size() + 1)); + llvm::copy(*ProgramName, &ArgVStorage.back()[0]); + ArgVStorage.back()[ProgramName->size()] = '\0'; + ArgV.push_back(ArgVStorage.back().get()); + } + + for (const auto &Arg : Args) { + ArgVStorage.push_back(std::make_unique<char[]>(Arg.size() + 1)); + llvm::copy(Arg, &ArgVStorage.back()[0]); + ArgVStorage.back()[Arg.size()] = '\0'; + ArgV.push_back(ArgVStorage.back().get()); + } + ArgV.push_back(nullptr); + + return Main(Args.size() + !!ProgramName, ArgV.data()); +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/Orc/TargetProcessControl.cpp b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/TargetProcessControl.cpp new file mode 100644 index 0000000000..7bf874e88c --- /dev/null +++ b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/TargetProcessControl.cpp @@ -0,0 +1,153 @@ +//===------ TargetProcessControl.cpp -- Target process control APIs -------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/TargetProcessControl.h" + +#include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/TargetProcess/TargetExecutionUtils.h" +#include "llvm/Support/Host.h" +#include "llvm/Support/Process.h" + +#include <mutex> + +namespace llvm { +namespace orc { + +TargetProcessControl::MemoryAccess::~MemoryAccess() {} + +TargetProcessControl::~TargetProcessControl() {} + +SelfTargetProcessControl::SelfTargetProcessControl( + std::shared_ptr<SymbolStringPool> SSP, Triple TargetTriple, + unsigned PageSize, std::unique_ptr<jitlink::JITLinkMemoryManager> MemMgr) + : TargetProcessControl(std::move(SSP)) { + + OwnedMemMgr = std::move(MemMgr); + if (!OwnedMemMgr) + OwnedMemMgr = std::make_unique<jitlink::InProcessMemoryManager>(); + + this->TargetTriple = std::move(TargetTriple); + this->PageSize = PageSize; + this->MemMgr = OwnedMemMgr.get(); + this->MemAccess = this; + if (this->TargetTriple.isOSBinFormatMachO()) + GlobalManglingPrefix = '_'; +} + +Expected<std::unique_ptr<SelfTargetProcessControl>> +SelfTargetProcessControl::Create( + std::shared_ptr<SymbolStringPool> SSP, + std::unique_ptr<jitlink::JITLinkMemoryManager> MemMgr) { + auto PageSize = sys::Process::getPageSize(); + if (!PageSize) + return PageSize.takeError(); + + Triple TT(sys::getProcessTriple()); + + return std::make_unique<SelfTargetProcessControl>( + std::move(SSP), std::move(TT), *PageSize, std::move(MemMgr)); +} + +Expected<tpctypes::DylibHandle> +SelfTargetProcessControl::loadDylib(const char *DylibPath) { + std::string ErrMsg; + auto Dylib = std::make_unique<sys::DynamicLibrary>( + sys::DynamicLibrary::getPermanentLibrary(DylibPath, &ErrMsg)); + if (!Dylib->isValid()) + return make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode()); + DynamicLibraries.push_back(std::move(Dylib)); + return pointerToJITTargetAddress(DynamicLibraries.back().get()); +} + +Expected<std::vector<tpctypes::LookupResult>> +SelfTargetProcessControl::lookupSymbols(ArrayRef<LookupRequest> Request) { + std::vector<tpctypes::LookupResult> R; + + for (auto &Elem : Request) { + auto *Dylib = jitTargetAddressToPointer<sys::DynamicLibrary *>(Elem.Handle); + assert(llvm::any_of(DynamicLibraries, + [=](const std::unique_ptr<sys::DynamicLibrary> &DL) { + return DL.get() == Dylib; + }) && + "Invalid handle"); + + R.push_back(std::vector<JITTargetAddress>()); + for (auto &KV : Elem.Symbols) { + auto &Sym = KV.first; + std::string Tmp((*Sym).data() + !!GlobalManglingPrefix, + (*Sym).size() - !!GlobalManglingPrefix); + void *Addr = Dylib->getAddressOfSymbol(Tmp.c_str()); + if (!Addr && KV.second == SymbolLookupFlags::RequiredSymbol) { + // FIXME: Collect all failing symbols before erroring out. + SymbolNameVector MissingSymbols; + MissingSymbols.push_back(Sym); + return make_error<SymbolsNotFound>(std::move(MissingSymbols)); + } + R.back().push_back(pointerToJITTargetAddress(Addr)); + } + } + + return R; +} + +Expected<int32_t> +SelfTargetProcessControl::runAsMain(JITTargetAddress MainFnAddr, + ArrayRef<std::string> Args) { + using MainTy = int (*)(int, char *[]); + return orc::runAsMain(jitTargetAddressToFunction<MainTy>(MainFnAddr), Args); +} + +Expected<tpctypes::WrapperFunctionResult> +SelfTargetProcessControl::runWrapper(JITTargetAddress WrapperFnAddr, + ArrayRef<uint8_t> ArgBuffer) { + using WrapperFnTy = + tpctypes::CWrapperFunctionResult (*)(const uint8_t *Data, uint64_t Size); + auto *WrapperFn = jitTargetAddressToFunction<WrapperFnTy>(WrapperFnAddr); + return WrapperFn(ArgBuffer.data(), ArgBuffer.size()); +} + +Error SelfTargetProcessControl::disconnect() { return Error::success(); } + +void SelfTargetProcessControl::writeUInt8s(ArrayRef<tpctypes::UInt8Write> Ws, + WriteResultFn OnWriteComplete) { + for (auto &W : Ws) + *jitTargetAddressToPointer<uint8_t *>(W.Address) = W.Value; + OnWriteComplete(Error::success()); +} + +void SelfTargetProcessControl::writeUInt16s(ArrayRef<tpctypes::UInt16Write> Ws, + WriteResultFn OnWriteComplete) { + for (auto &W : Ws) + *jitTargetAddressToPointer<uint16_t *>(W.Address) = W.Value; + OnWriteComplete(Error::success()); +} + +void SelfTargetProcessControl::writeUInt32s(ArrayRef<tpctypes::UInt32Write> Ws, + WriteResultFn OnWriteComplete) { + for (auto &W : Ws) + *jitTargetAddressToPointer<uint32_t *>(W.Address) = W.Value; + OnWriteComplete(Error::success()); +} + +void SelfTargetProcessControl::writeUInt64s(ArrayRef<tpctypes::UInt64Write> Ws, + WriteResultFn OnWriteComplete) { + for (auto &W : Ws) + *jitTargetAddressToPointer<uint64_t *>(W.Address) = W.Value; + OnWriteComplete(Error::success()); +} + +void SelfTargetProcessControl::writeBuffers(ArrayRef<tpctypes::BufferWrite> Ws, + WriteResultFn OnWriteComplete) { + for (auto &W : Ws) + memcpy(jitTargetAddressToPointer<char *>(W.Address), W.Buffer.data(), + W.Buffer.size()); + OnWriteComplete(Error::success()); +} + +} // end namespace orc +} // end namespace llvm diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/Orc/ThreadSafeModule.cpp b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/ThreadSafeModule.cpp new file mode 100644 index 0000000000..2e128dd237 --- /dev/null +++ b/contrib/libs/llvm12/lib/ExecutionEngine/Orc/ThreadSafeModule.cpp @@ -0,0 +1,64 @@ +//===-- ThreadSafeModule.cpp - Thread safe Module, Context, and Utilities +//h-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/ThreadSafeModule.h" +#include "llvm/Bitcode/BitcodeReader.h" +#include "llvm/Bitcode/BitcodeWriter.h" +#include "llvm/Transforms/Utils/Cloning.h" + +namespace llvm { +namespace orc { + +ThreadSafeModule cloneToNewContext(const ThreadSafeModule &TSM, + GVPredicate ShouldCloneDef, + GVModifier UpdateClonedDefSource) { + assert(TSM && "Can not clone null module"); + + if (!ShouldCloneDef) + ShouldCloneDef = [](const GlobalValue &) { return true; }; + + return TSM.withModuleDo([&](Module &M) { + SmallVector<char, 1> ClonedModuleBuffer; + + { + std::set<GlobalValue *> ClonedDefsInSrc; + ValueToValueMapTy VMap; + auto Tmp = CloneModule(M, VMap, [&](const GlobalValue *GV) { + if (ShouldCloneDef(*GV)) { + ClonedDefsInSrc.insert(const_cast<GlobalValue *>(GV)); + return true; + } + return false; + }); + + if (UpdateClonedDefSource) + for (auto *GV : ClonedDefsInSrc) + UpdateClonedDefSource(*GV); + + BitcodeWriter BCWriter(ClonedModuleBuffer); + + BCWriter.writeModule(*Tmp); + BCWriter.writeSymtab(); + BCWriter.writeStrtab(); + } + + MemoryBufferRef ClonedModuleBufferRef( + StringRef(ClonedModuleBuffer.data(), ClonedModuleBuffer.size()), + "cloned module buffer"); + ThreadSafeContext NewTSCtx(std::make_unique<LLVMContext>()); + + auto ClonedModule = cantFail( + parseBitcodeFile(ClonedModuleBufferRef, *NewTSCtx.getContext())); + ClonedModule->setModuleIdentifier(M.getName()); + return ThreadSafeModule(std::move(ClonedModule), std::move(NewTSCtx)); + }); +} + +} // end namespace orc +} // end namespace llvm diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/PerfJITEvents/.yandex_meta/licenses.list.txt b/contrib/libs/llvm12/lib/ExecutionEngine/PerfJITEvents/.yandex_meta/licenses.list.txt deleted file mode 100644 index c62d353021..0000000000 --- a/contrib/libs/llvm12/lib/ExecutionEngine/PerfJITEvents/.yandex_meta/licenses.list.txt +++ /dev/null @@ -1,7 +0,0 @@ -====================Apache-2.0 WITH LLVM-exception==================== -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. - - -====================Apache-2.0 WITH LLVM-exception==================== -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/PerfJITEvents/CMakeLists.txt b/contrib/libs/llvm12/lib/ExecutionEngine/PerfJITEvents/CMakeLists.txt index e8c98bfd99..c031ddb850 100644 --- a/contrib/libs/llvm12/lib/ExecutionEngine/PerfJITEvents/CMakeLists.txt +++ b/contrib/libs/llvm12/lib/ExecutionEngine/PerfJITEvents/CMakeLists.txt @@ -6,6 +6,6 @@ # original buildsystem will not be accepted. -if (UNIX AND NOT APPLE) +if (UNIX) include(CMakeLists.linux.txt) endif() diff --git a/contrib/libs/llvm12/lib/ExecutionEngine/RuntimeDyld/.yandex_meta/licenses.list.txt b/contrib/libs/llvm12/lib/ExecutionEngine/RuntimeDyld/.yandex_meta/licenses.list.txt deleted file mode 100644 index c62d353021..0000000000 --- a/contrib/libs/llvm12/lib/ExecutionEngine/RuntimeDyld/.yandex_meta/licenses.list.txt +++ /dev/null @@ -1,7 +0,0 @@ -====================Apache-2.0 WITH LLVM-exception==================== -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. - - -====================Apache-2.0 WITH LLVM-exception==================== -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |