diff options
author | vitalyisaev <vitalyisaev@yandex-team.com> | 2023-06-29 10:00:50 +0300 |
---|---|---|
committer | vitalyisaev <vitalyisaev@yandex-team.com> | 2023-06-29 10:00:50 +0300 |
commit | 6ffe9e53658409f212834330e13564e4952558f6 (patch) | |
tree | 85b1e00183517648b228aafa7c8fb07f5276f419 /contrib/libs/llvm14/include/llvm/ExecutionEngine | |
parent | 726057070f9c5a91fc10fde0d5024913d10f1ab9 (diff) | |
download | ydb-6ffe9e53658409f212834330e13564e4952558f6.tar.gz |
YQ Connector: support managed ClickHouse
Со стороны dqrun можно обратиться к инстансу коннектора, который работает на streaming стенде, и извлечь данные из облачного CH.
Diffstat (limited to 'contrib/libs/llvm14/include/llvm/ExecutionEngine')
80 files changed, 18060 insertions, 0 deletions
diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/ExecutionEngine.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/ExecutionEngine.h new file mode 100644 index 0000000000..75588e6490 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/ExecutionEngine.h @@ -0,0 +1,680 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===- ExecutionEngine.h - Abstract Execution Engine Interface --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines the abstract interface that implements execution support +// for LLVM. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_EXECUTIONENGINE_H +#define LLVM_EXECUTIONENGINE_EXECUTIONENGINE_H + +#include "llvm-c/ExecutionEngine.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/Module.h" +#include "llvm/Object/Binary.h" +#include "llvm/Support/CBindingWrapping.h" +#include "llvm/Support/CodeGen.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/Mutex.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetOptions.h" +#include <algorithm> +#include <cstdint> +#include <functional> +#include <map> +#include <memory> +#include <string> +#include <vector> + +namespace llvm { + +class Constant; +class Function; +struct GenericValue; +class GlobalValue; +class GlobalVariable; +class JITEventListener; +class MCJITMemoryManager; +class ObjectCache; +class RTDyldMemoryManager; +class Triple; +class Type; + +namespace object { + +class Archive; +class ObjectFile; + +} // end namespace object + +/// Helper class for helping synchronize access to the global address map +/// table. Access to this class should be serialized under a mutex. +class ExecutionEngineState { +public: + using GlobalAddressMapTy = StringMap<uint64_t>; + +private: + /// GlobalAddressMap - A mapping between LLVM global symbol names values and + /// their actualized version... + GlobalAddressMapTy GlobalAddressMap; + + /// GlobalAddressReverseMap - This is the reverse mapping of GlobalAddressMap, + /// used to convert raw addresses into the LLVM global value that is emitted + /// at the address. This map is not computed unless getGlobalValueAtAddress + /// is called at some point. + std::map<uint64_t, std::string> GlobalAddressReverseMap; + +public: + GlobalAddressMapTy &getGlobalAddressMap() { + return GlobalAddressMap; + } + + std::map<uint64_t, std::string> &getGlobalAddressReverseMap() { + return GlobalAddressReverseMap; + } + + /// Erase an entry from the mapping table. + /// + /// \returns The address that \p ToUnmap was happed to. + uint64_t RemoveMapping(StringRef Name); +}; + +using FunctionCreator = std::function<void *(const std::string &)>; + +/// Abstract interface for implementation execution of LLVM modules, +/// designed to support both interpreter and just-in-time (JIT) compiler +/// implementations. +class ExecutionEngine { + /// The state object holding the global address mapping, which must be + /// accessed synchronously. + // + // FIXME: There is no particular need the entire map needs to be + // synchronized. Wouldn't a reader-writer design be better here? + ExecutionEngineState EEState; + + /// The target data for the platform for which execution is being performed. + /// + /// Note: the DataLayout is LLVMContext specific because it has an + /// internal cache based on type pointers. It makes unsafe to reuse the + /// ExecutionEngine across context, we don't enforce this rule but undefined + /// behavior can occurs if the user tries to do it. + const DataLayout DL; + + /// Whether lazy JIT compilation is enabled. + bool CompilingLazily; + + /// Whether JIT compilation of external global variables is allowed. + bool GVCompilationDisabled; + + /// Whether the JIT should perform lookups of external symbols (e.g., + /// using dlsym). + bool SymbolSearchingDisabled; + + /// Whether the JIT should verify IR modules during compilation. + bool VerifyModules; + + friend class EngineBuilder; // To allow access to JITCtor and InterpCtor. + +protected: + /// The list of Modules that we are JIT'ing from. We use a SmallVector to + /// optimize for the case where there is only one module. + SmallVector<std::unique_ptr<Module>, 1> Modules; + + /// getMemoryforGV - Allocate memory for a global variable. + virtual char *getMemoryForGV(const GlobalVariable *GV); + + static ExecutionEngine *(*MCJITCtor)( + std::unique_ptr<Module> M, std::string *ErrorStr, + std::shared_ptr<MCJITMemoryManager> MM, + std::shared_ptr<LegacyJITSymbolResolver> SR, + std::unique_ptr<TargetMachine> TM); + + static ExecutionEngine *(*InterpCtor)(std::unique_ptr<Module> M, + std::string *ErrorStr); + + /// LazyFunctionCreator - If an unknown function is needed, this function + /// pointer is invoked to create it. If this returns null, the JIT will + /// abort. + FunctionCreator LazyFunctionCreator; + + /// getMangledName - Get mangled name. + std::string getMangledName(const GlobalValue *GV); + + std::string ErrMsg; + +public: + /// lock - This lock protects the ExecutionEngine and MCJIT classes. It must + /// be held while changing the internal state of any of those classes. + sys::Mutex lock; + + //===--------------------------------------------------------------------===// + // ExecutionEngine Startup + //===--------------------------------------------------------------------===// + + virtual ~ExecutionEngine(); + + /// Add a Module to the list of modules that we can JIT from. + virtual void addModule(std::unique_ptr<Module> M) { + Modules.push_back(std::move(M)); + } + + /// addObjectFile - Add an ObjectFile to the execution engine. + /// + /// This method is only supported by MCJIT. MCJIT will immediately load the + /// object into memory and adds its symbols to the list used to resolve + /// external symbols while preparing other objects for execution. + /// + /// Objects added using this function will not be made executable until + /// needed by another object. + /// + /// MCJIT will take ownership of the ObjectFile. + virtual void addObjectFile(std::unique_ptr<object::ObjectFile> O); + virtual void addObjectFile(object::OwningBinary<object::ObjectFile> O); + + /// addArchive - Add an Archive to the execution engine. + /// + /// This method is only supported by MCJIT. MCJIT will use the archive to + /// resolve external symbols in objects it is loading. If a symbol is found + /// in the Archive the contained object file will be extracted (in memory) + /// and loaded for possible execution. + virtual void addArchive(object::OwningBinary<object::Archive> A); + + //===--------------------------------------------------------------------===// + + const DataLayout &getDataLayout() const { return DL; } + + /// removeModule - Removes a Module from the list of modules, but does not + /// free the module's memory. Returns true if M is found, in which case the + /// caller assumes responsibility for deleting the module. + // + // FIXME: This stealth ownership transfer is horrible. This will probably be + // fixed by deleting ExecutionEngine. + virtual bool removeModule(Module *M); + + /// FindFunctionNamed - Search all of the active modules to find the function that + /// defines FnName. This is very slow operation and shouldn't be used for + /// general code. + virtual Function *FindFunctionNamed(StringRef FnName); + + /// FindGlobalVariableNamed - Search all of the active modules to find the global variable + /// that defines Name. This is very slow operation and shouldn't be used for + /// general code. + virtual GlobalVariable *FindGlobalVariableNamed(StringRef Name, bool AllowInternal = false); + + /// runFunction - Execute the specified function with the specified arguments, + /// and return the result. + /// + /// For MCJIT execution engines, clients are encouraged to use the + /// "GetFunctionAddress" method (rather than runFunction) and cast the + /// returned uint64_t to the desired function pointer type. However, for + /// backwards compatibility MCJIT's implementation can execute 'main-like' + /// function (i.e. those returning void or int, and taking either no + /// arguments or (int, char*[])). + virtual GenericValue runFunction(Function *F, + ArrayRef<GenericValue> ArgValues) = 0; + + /// getPointerToNamedFunction - This method returns the address of the + /// specified function by using the dlsym function call. As such it is only + /// useful for resolving library symbols, not code generated symbols. + /// + /// If AbortOnFailure is false and no function with the given name is + /// found, this function silently returns a null pointer. Otherwise, + /// it prints a message to stderr and aborts. + /// + /// This function is deprecated for the MCJIT execution engine. + virtual void *getPointerToNamedFunction(StringRef Name, + bool AbortOnFailure = true) = 0; + + /// mapSectionAddress - map a section to its target address space value. + /// Map the address of a JIT section as returned from the memory manager + /// to the address in the target process as the running code will see it. + /// This is the address which will be used for relocation resolution. + virtual void mapSectionAddress(const void *LocalAddress, + uint64_t TargetAddress) { + llvm_unreachable("Re-mapping of section addresses not supported with this " + "EE!"); + } + + /// generateCodeForModule - Run code generation for the specified module and + /// load it into memory. + /// + /// When this function has completed, all code and data for the specified + /// module, and any module on which this module depends, will be generated + /// and loaded into memory, but relocations will not yet have been applied + /// and all memory will be readable and writable but not executable. + /// + /// This function is primarily useful when generating code for an external + /// target, allowing the client an opportunity to remap section addresses + /// before relocations are applied. Clients that intend to execute code + /// locally can use the getFunctionAddress call, which will generate code + /// and apply final preparations all in one step. + /// + /// This method has no effect for the interpeter. + virtual void generateCodeForModule(Module *M) {} + + /// finalizeObject - ensure the module is fully processed and is usable. + /// + /// It is the user-level function for completing the process of making the + /// object usable for execution. It should be called after sections within an + /// object have been relocated using mapSectionAddress. When this method is + /// called the MCJIT execution engine will reapply relocations for a loaded + /// object. This method has no effect for the interpeter. + /// + /// Returns true on success, false on failure. Error messages can be retrieved + /// by calling getError(); + virtual void finalizeObject() {} + + /// Returns true if an error has been recorded. + bool hasError() const { return !ErrMsg.empty(); } + + /// Clear the error message. + void clearErrorMessage() { ErrMsg.clear(); } + + /// Returns the most recent error message. + const std::string &getErrorMessage() const { return ErrMsg; } + + /// runStaticConstructorsDestructors - This method is used to execute all of + /// the static constructors or destructors for a program. + /// + /// \param isDtors - Run the destructors instead of constructors. + virtual void runStaticConstructorsDestructors(bool isDtors); + + /// This method is used to execute all of the static constructors or + /// destructors for a particular module. + /// + /// \param isDtors - Run the destructors instead of constructors. + void runStaticConstructorsDestructors(Module &module, bool isDtors); + + + /// runFunctionAsMain - This is a helper function which wraps runFunction to + /// handle the common task of starting up main with the specified argc, argv, + /// and envp parameters. + int runFunctionAsMain(Function *Fn, const std::vector<std::string> &argv, + const char * const * envp); + + + /// addGlobalMapping - Tell the execution engine that the specified global is + /// at the specified location. This is used internally as functions are JIT'd + /// and as global variables are laid out in memory. It can and should also be + /// used by clients of the EE that want to have an LLVM global overlay + /// existing data in memory. Values to be mapped should be named, and have + /// external or weak linkage. Mappings are automatically removed when their + /// GlobalValue is destroyed. + void addGlobalMapping(const GlobalValue *GV, void *Addr); + void addGlobalMapping(StringRef Name, uint64_t Addr); + + /// clearAllGlobalMappings - Clear all global mappings and start over again, + /// for use in dynamic compilation scenarios to move globals. + void clearAllGlobalMappings(); + + /// clearGlobalMappingsFromModule - Clear all global mappings that came from a + /// particular module, because it has been removed from the JIT. + void clearGlobalMappingsFromModule(Module *M); + + /// updateGlobalMapping - Replace an existing mapping for GV with a new + /// address. This updates both maps as required. If "Addr" is null, the + /// entry for the global is removed from the mappings. This returns the old + /// value of the pointer, or null if it was not in the map. + uint64_t updateGlobalMapping(const GlobalValue *GV, void *Addr); + uint64_t updateGlobalMapping(StringRef Name, uint64_t Addr); + + /// getAddressToGlobalIfAvailable - This returns the address of the specified + /// global symbol. + uint64_t getAddressToGlobalIfAvailable(StringRef S); + + /// getPointerToGlobalIfAvailable - This returns the address of the specified + /// global value if it is has already been codegen'd, otherwise it returns + /// null. + void *getPointerToGlobalIfAvailable(StringRef S); + void *getPointerToGlobalIfAvailable(const GlobalValue *GV); + + /// getPointerToGlobal - This returns the address of the specified global + /// value. This may involve code generation if it's a function. + /// + /// This function is deprecated for the MCJIT execution engine. Use + /// getGlobalValueAddress instead. + void *getPointerToGlobal(const GlobalValue *GV); + + /// getPointerToFunction - The different EE's represent function bodies in + /// different ways. They should each implement this to say what a function + /// pointer should look like. When F is destroyed, the ExecutionEngine will + /// remove its global mapping and free any machine code. Be sure no threads + /// are running inside F when that happens. + /// + /// This function is deprecated for the MCJIT execution engine. Use + /// getFunctionAddress instead. + virtual void *getPointerToFunction(Function *F) = 0; + + /// getPointerToFunctionOrStub - If the specified function has been + /// code-gen'd, return a pointer to the function. If not, compile it, or use + /// a stub to implement lazy compilation if available. See + /// getPointerToFunction for the requirements on destroying F. + /// + /// This function is deprecated for the MCJIT execution engine. Use + /// getFunctionAddress instead. + virtual void *getPointerToFunctionOrStub(Function *F) { + // Default implementation, just codegen the function. + return getPointerToFunction(F); + } + + /// getGlobalValueAddress - Return the address of the specified global + /// value. This may involve code generation. + /// + /// This function should not be called with the interpreter engine. + virtual uint64_t getGlobalValueAddress(const std::string &Name) { + // Default implementation for the interpreter. MCJIT will override this. + // JIT and interpreter clients should use getPointerToGlobal instead. + return 0; + } + + /// getFunctionAddress - Return the address of the specified function. + /// This may involve code generation. + virtual uint64_t getFunctionAddress(const std::string &Name) { + // Default implementation for the interpreter. MCJIT will override this. + // Interpreter clients should use getPointerToFunction instead. + return 0; + } + + /// getGlobalValueAtAddress - Return the LLVM global value object that starts + /// at the specified address. + /// + const GlobalValue *getGlobalValueAtAddress(void *Addr); + + /// StoreValueToMemory - Stores the data in Val of type Ty at address Ptr. + /// Ptr is the address of the memory at which to store Val, cast to + /// GenericValue *. It is not a pointer to a GenericValue containing the + /// address at which to store Val. + void StoreValueToMemory(const GenericValue &Val, GenericValue *Ptr, + Type *Ty); + + void InitializeMemory(const Constant *Init, void *Addr); + + /// getOrEmitGlobalVariable - Return the address of the specified global + /// variable, possibly emitting it to memory if needed. This is used by the + /// Emitter. + /// + /// This function is deprecated for the MCJIT execution engine. Use + /// getGlobalValueAddress instead. + virtual void *getOrEmitGlobalVariable(const GlobalVariable *GV) { + return getPointerToGlobal((const GlobalValue *)GV); + } + + /// Registers a listener to be called back on various events within + /// the JIT. See JITEventListener.h for more details. Does not + /// take ownership of the argument. The argument may be NULL, in + /// which case these functions do nothing. + virtual void RegisterJITEventListener(JITEventListener *) {} + virtual void UnregisterJITEventListener(JITEventListener *) {} + + /// Sets the pre-compiled object cache. The ownership of the ObjectCache is + /// not changed. Supported by MCJIT but not the interpreter. + virtual void setObjectCache(ObjectCache *) { + llvm_unreachable("No support for an object cache"); + } + + /// setProcessAllSections (MCJIT Only): By default, only sections that are + /// "required for execution" are passed to the RTDyldMemoryManager, and other + /// sections are discarded. Passing 'true' to this method will cause + /// RuntimeDyld to pass all sections to its RTDyldMemoryManager regardless + /// of whether they are "required to execute" in the usual sense. + /// + /// Rationale: Some MCJIT clients want to be able to inspect metadata + /// sections (e.g. Dwarf, Stack-maps) to enable functionality or analyze + /// performance. Passing these sections to the memory manager allows the + /// client to make policy about the relevant sections, rather than having + /// MCJIT do it. + virtual void setProcessAllSections(bool ProcessAllSections) { + llvm_unreachable("No support for ProcessAllSections option"); + } + + /// Return the target machine (if available). + virtual TargetMachine *getTargetMachine() { return nullptr; } + + /// DisableLazyCompilation - When lazy compilation is off (the default), the + /// JIT will eagerly compile every function reachable from the argument to + /// getPointerToFunction. If lazy compilation is turned on, the JIT will only + /// compile the one function and emit stubs to compile the rest when they're + /// first called. If lazy compilation is turned off again while some lazy + /// stubs are still around, and one of those stubs is called, the program will + /// abort. + /// + /// In order to safely compile lazily in a threaded program, the user must + /// ensure that 1) only one thread at a time can call any particular lazy + /// stub, and 2) any thread modifying LLVM IR must hold the JIT's lock + /// (ExecutionEngine::lock) or otherwise ensure that no other thread calls a + /// lazy stub. See http://llvm.org/PR5184 for details. + void DisableLazyCompilation(bool Disabled = true) { + CompilingLazily = !Disabled; + } + bool isCompilingLazily() const { + return CompilingLazily; + } + + /// DisableGVCompilation - If called, the JIT will abort if it's asked to + /// allocate space and populate a GlobalVariable that is not internal to + /// the module. + void DisableGVCompilation(bool Disabled = true) { + GVCompilationDisabled = Disabled; + } + bool isGVCompilationDisabled() const { + return GVCompilationDisabled; + } + + /// DisableSymbolSearching - If called, the JIT will not try to lookup unknown + /// symbols with dlsym. A client can still use InstallLazyFunctionCreator to + /// resolve symbols in a custom way. + void DisableSymbolSearching(bool Disabled = true) { + SymbolSearchingDisabled = Disabled; + } + bool isSymbolSearchingDisabled() const { + return SymbolSearchingDisabled; + } + + /// Enable/Disable IR module verification. + /// + /// Note: Module verification is enabled by default in Debug builds, and + /// disabled by default in Release. Use this method to override the default. + void setVerifyModules(bool Verify) { + VerifyModules = Verify; + } + bool getVerifyModules() const { + return VerifyModules; + } + + /// InstallLazyFunctionCreator - If an unknown function is needed, the + /// specified function pointer is invoked to create it. If it returns null, + /// the JIT will abort. + void InstallLazyFunctionCreator(FunctionCreator C) { + LazyFunctionCreator = std::move(C); + } + +protected: + ExecutionEngine(DataLayout DL) : DL(std::move(DL)) {} + explicit ExecutionEngine(DataLayout DL, std::unique_ptr<Module> M); + explicit ExecutionEngine(std::unique_ptr<Module> M); + + void emitGlobals(); + + void emitGlobalVariable(const GlobalVariable *GV); + + GenericValue getConstantValue(const Constant *C); + void LoadValueFromMemory(GenericValue &Result, GenericValue *Ptr, + Type *Ty); + +private: + void Init(std::unique_ptr<Module> M); +}; + +namespace EngineKind { + + // These are actually bitmasks that get or-ed together. + enum Kind { + JIT = 0x1, + Interpreter = 0x2 + }; + const static Kind Either = (Kind)(JIT | Interpreter); + +} // end namespace EngineKind + +/// Builder class for ExecutionEngines. Use this by stack-allocating a builder, +/// chaining the various set* methods, and terminating it with a .create() +/// call. +class EngineBuilder { +private: + std::unique_ptr<Module> M; + EngineKind::Kind WhichEngine; + std::string *ErrorStr; + CodeGenOpt::Level OptLevel; + std::shared_ptr<MCJITMemoryManager> MemMgr; + std::shared_ptr<LegacyJITSymbolResolver> Resolver; + TargetOptions Options; + Optional<Reloc::Model> RelocModel; + Optional<CodeModel::Model> CMModel; + std::string MArch; + std::string MCPU; + SmallVector<std::string, 4> MAttrs; + bool VerifyModules; + bool EmulatedTLS = true; + +public: + /// Default constructor for EngineBuilder. + EngineBuilder(); + + /// Constructor for EngineBuilder. + EngineBuilder(std::unique_ptr<Module> M); + + // Out-of-line since we don't have the def'n of RTDyldMemoryManager here. + ~EngineBuilder(); + + /// setEngineKind - Controls whether the user wants the interpreter, the JIT, + /// or whichever engine works. This option defaults to EngineKind::Either. + EngineBuilder &setEngineKind(EngineKind::Kind w) { + WhichEngine = w; + return *this; + } + + /// setMCJITMemoryManager - Sets the MCJIT memory manager to use. This allows + /// clients to customize their memory allocation policies for the MCJIT. This + /// is only appropriate for the MCJIT; setting this and configuring the builder + /// to create anything other than MCJIT will cause a runtime error. If create() + /// is called and is successful, the created engine takes ownership of the + /// memory manager. This option defaults to NULL. + EngineBuilder &setMCJITMemoryManager(std::unique_ptr<RTDyldMemoryManager> mcjmm); + + EngineBuilder& + setMemoryManager(std::unique_ptr<MCJITMemoryManager> MM); + + EngineBuilder &setSymbolResolver(std::unique_ptr<LegacyJITSymbolResolver> SR); + + /// setErrorStr - Set the error string to write to on error. This option + /// defaults to NULL. + EngineBuilder &setErrorStr(std::string *e) { + ErrorStr = e; + return *this; + } + + /// setOptLevel - Set the optimization level for the JIT. This option + /// defaults to CodeGenOpt::Default. + EngineBuilder &setOptLevel(CodeGenOpt::Level l) { + OptLevel = l; + return *this; + } + + /// setTargetOptions - Set the target options that the ExecutionEngine + /// target is using. Defaults to TargetOptions(). + EngineBuilder &setTargetOptions(const TargetOptions &Opts) { + Options = Opts; + return *this; + } + + /// setRelocationModel - Set the relocation model that the ExecutionEngine + /// target is using. Defaults to target specific default "Reloc::Default". + EngineBuilder &setRelocationModel(Reloc::Model RM) { + RelocModel = RM; + return *this; + } + + /// setCodeModel - Set the CodeModel that the ExecutionEngine target + /// data is using. Defaults to target specific default + /// "CodeModel::JITDefault". + EngineBuilder &setCodeModel(CodeModel::Model M) { + CMModel = M; + return *this; + } + + /// setMArch - Override the architecture set by the Module's triple. + EngineBuilder &setMArch(StringRef march) { + MArch.assign(march.begin(), march.end()); + return *this; + } + + /// setMCPU - Target a specific cpu type. + EngineBuilder &setMCPU(StringRef mcpu) { + MCPU.assign(mcpu.begin(), mcpu.end()); + return *this; + } + + /// setVerifyModules - Set whether the JIT implementation should verify + /// IR modules during compilation. + EngineBuilder &setVerifyModules(bool Verify) { + VerifyModules = Verify; + return *this; + } + + /// setMAttrs - Set cpu-specific attributes. + template<typename StringSequence> + EngineBuilder &setMAttrs(const StringSequence &mattrs) { + MAttrs.clear(); + MAttrs.append(mattrs.begin(), mattrs.end()); + return *this; + } + + void setEmulatedTLS(bool EmulatedTLS) { + this->EmulatedTLS = EmulatedTLS; + } + + TargetMachine *selectTarget(); + + /// selectTarget - Pick a target either via -march or by guessing the native + /// arch. Add any CPU features specified via -mcpu or -mattr. + TargetMachine *selectTarget(const Triple &TargetTriple, + StringRef MArch, + StringRef MCPU, + const SmallVectorImpl<std::string>& MAttrs); + + ExecutionEngine *create() { + return create(selectTarget()); + } + + ExecutionEngine *create(TargetMachine *TM); +}; + +// Create wrappers for C Binding types (see CBindingWrapping.h). +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(ExecutionEngine, LLVMExecutionEngineRef) + +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_EXECUTIONENGINE_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/GenericValue.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/GenericValue.h new file mode 100644 index 0000000000..b3aa1f4957 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/GenericValue.h @@ -0,0 +1,65 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===- GenericValue.h - Represent any type of LLVM value --------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// The GenericValue class is used to represent an LLVM value of arbitrary type. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_GENERICVALUE_H +#define LLVM_EXECUTIONENGINE_GENERICVALUE_H + +#include "llvm/ADT/APInt.h" +#include <vector> + +namespace llvm { + +using PointerTy = void *; + +struct GenericValue { + struct IntPair { + unsigned int first; + unsigned int second; + }; + union { + double DoubleVal; + float FloatVal; + PointerTy PointerVal; + struct IntPair UIntPairVal; + unsigned char Untyped[8]; + }; + APInt IntVal; // also used for long doubles. + // For aggregate data types. + std::vector<GenericValue> AggregateVal; + + // to make code faster, set GenericValue to zero could be omitted, but it is + // potentially can cause problems, since GenericValue to store garbage + // instead of zero. + GenericValue() : IntVal(1, 0) { + UIntPairVal.first = 0; + UIntPairVal.second = 0; + } + explicit GenericValue(void *V) : PointerVal(V), IntVal(1, 0) {} +}; + +inline GenericValue PTOGV(void *P) { return GenericValue(P); } +inline void *GVTOP(const GenericValue &GV) { return GV.PointerVal; } + +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_GENERICVALUE_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Interpreter.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Interpreter.h new file mode 100644 index 0000000000..6e34141440 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Interpreter.h @@ -0,0 +1,38 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===-- Interpreter.h - Abstract Execution Engine Interface -----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file forces the interpreter to link in on certain operating systems. +// (Windows). +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_INTERPRETER_H +#define LLVM_EXECUTIONENGINE_INTERPRETER_H + +#include "llvm/ExecutionEngine/ExecutionEngine.h" + +extern "C" void LLVMLinkInInterpreter(); + +namespace { + struct ForceInterpreterLinking { + ForceInterpreterLinking() { LLVMLinkInInterpreter(); } + } ForceInterpreterLinking; +} + +#endif + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITEventListener.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITEventListener.h new file mode 100644 index 0000000000..1db57462ab --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITEventListener.h @@ -0,0 +1,126 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===- JITEventListener.h - Exposes events from JIT compilation -*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines the JITEventListener interface, which lets users get +// callbacks when significant events happen during the JIT compilation process. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_JITEVENTLISTENER_H +#define LLVM_EXECUTIONENGINE_JITEVENTLISTENER_H + +#include "llvm-c/ExecutionEngine.h" +#include "llvm/Config/llvm-config.h" +#include "llvm/ExecutionEngine/RuntimeDyld.h" +#include "llvm/IR/DebugLoc.h" +#include "llvm/Support/CBindingWrapping.h" +#include <cstdint> + +namespace llvm { + +class IntelJITEventsWrapper; +class OProfileWrapper; + +namespace object { + +class ObjectFile; + +} // end namespace object + +/// JITEventListener - Abstract interface for use by the JIT to notify clients +/// about significant events during compilation. For example, to notify +/// profilers and debuggers that need to know where functions have been emitted. +/// +/// The default implementation of each method does nothing. +class JITEventListener { +public: + using ObjectKey = uint64_t; + + JITEventListener() = default; + virtual ~JITEventListener() = default; + + /// notifyObjectLoaded - Called after an object has had its sections allocated + /// and addresses assigned to all symbols. Note: Section memory will not have + /// been relocated yet. notifyFunctionLoaded will not be called for + /// individual functions in the object. + /// + /// ELF-specific information + /// The ObjectImage contains the generated object image + /// with section headers updated to reflect the address at which sections + /// were loaded and with relocations performed in-place on debug sections. + virtual void notifyObjectLoaded(ObjectKey K, const object::ObjectFile &Obj, + const RuntimeDyld::LoadedObjectInfo &L) {} + + /// notifyFreeingObject - Called just before the memory associated with + /// a previously emitted object is released. + virtual void notifyFreeingObject(ObjectKey K) {} + + // Get a pointe to the GDB debugger registration listener. + static JITEventListener *createGDBRegistrationListener(); + +#if LLVM_USE_INTEL_JITEVENTS + // Construct an IntelJITEventListener + static JITEventListener *createIntelJITEventListener(); + + // Construct an IntelJITEventListener with a test Intel JIT API implementation + static JITEventListener *createIntelJITEventListener( + IntelJITEventsWrapper* AlternativeImpl); +#else + static JITEventListener *createIntelJITEventListener() { return nullptr; } + + static JITEventListener *createIntelJITEventListener( + IntelJITEventsWrapper* AlternativeImpl) { + return nullptr; + } +#endif // USE_INTEL_JITEVENTS + +#if LLVM_USE_OPROFILE + // Construct an OProfileJITEventListener + static JITEventListener *createOProfileJITEventListener(); + + // Construct an OProfileJITEventListener with a test opagent implementation + static JITEventListener *createOProfileJITEventListener( + OProfileWrapper* AlternativeImpl); +#else + static JITEventListener *createOProfileJITEventListener() { return nullptr; } + + static JITEventListener *createOProfileJITEventListener( + OProfileWrapper* AlternativeImpl) { + return nullptr; + } +#endif // USE_OPROFILE + +#if LLVM_USE_PERF + static JITEventListener *createPerfJITEventListener(); +#else + static JITEventListener *createPerfJITEventListener() + { + return nullptr; + } +#endif // USE_PERF + +private: + virtual void anchor(); +}; + +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(JITEventListener, LLVMJITEventListenerRef) + +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_JITEVENTLISTENER_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITLink/EHFrameSupport.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITLink/EHFrameSupport.h new file mode 100644 index 0000000000..52b05b3cd1 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITLink/EHFrameSupport.h @@ -0,0 +1,68 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===--------- EHFrameSupport.h - JITLink eh-frame utils --------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// EHFrame registration support for JITLink. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_JITLINK_EHFRAMESUPPORT_H +#define LLVM_EXECUTIONENGINE_JITLINK_EHFRAMESUPPORT_H + +#include "llvm/ADT/Triple.h" +#include "llvm/ExecutionEngine/JITLink/JITLink.h" +#include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/Support/Error.h" + +namespace llvm { +namespace jitlink { + +/// Supports registration/deregistration of EH-frames in a target process. +class EHFrameRegistrar { +public: + virtual ~EHFrameRegistrar(); + virtual Error registerEHFrames(orc::ExecutorAddrRange EHFrameSection) = 0; + virtual Error deregisterEHFrames(orc::ExecutorAddrRange EHFrameSection) = 0; +}; + +/// Registers / Deregisters EH-frames in the current process. +class InProcessEHFrameRegistrar final : public EHFrameRegistrar { +public: + Error registerEHFrames(orc::ExecutorAddrRange EHFrameSection) override; + + Error deregisterEHFrames(orc::ExecutorAddrRange EHFrameSection) override; +}; + +using StoreFrameRangeFunction = std::function<void( + orc::ExecutorAddr EHFrameSectionAddr, size_t EHFrameSectionSize)>; + +/// Creates a pass that records the address and size of the EH frame section. +/// If no eh-frame section is found then the address and size will both be given +/// as zero. +/// +/// Authors of JITLinkContexts can use this function to register a post-fixup +/// pass that records the range of the eh-frame section. This range can +/// be used after finalization to register and deregister the frame. +LinkGraphPassFunction +createEHFrameRecorderPass(const Triple &TT, + StoreFrameRangeFunction StoreFrameRange); + +} // end namespace jitlink +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_JITLINK_EHFRAMESUPPORT_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITLink/ELF.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITLink/ELF.h new file mode 100644 index 0000000000..c55da2cf80 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITLink/ELF.h @@ -0,0 +1,50 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===------- ELF.h - Generic JIT link function for ELF ------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Generic jit-link functions for ELF. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_JITLINK_ELF_H +#define LLVM_EXECUTIONENGINE_JITLINK_ELF_H + +#include "llvm/ExecutionEngine/JITLink/JITLink.h" + +namespace llvm { +namespace jitlink { + +/// Create a LinkGraph from an ELF relocatable object. +/// +/// Note: The graph does not take ownership of the underlying buffer, nor copy +/// its contents. The caller is responsible for ensuring that the object buffer +/// outlives the graph. +Expected<std::unique_ptr<LinkGraph>> +createLinkGraphFromELFObject(MemoryBufferRef ObjectBuffer); + +/// Link the given graph. +/// +/// Uses conservative defaults for GOT and stub handling based on the target +/// platform. +void link_ELF(std::unique_ptr<LinkGraph> G, + std::unique_ptr<JITLinkContext> Ctx); + +} // end namespace jitlink +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_JITLINK_ELF_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITLink/ELF_aarch64.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITLink/ELF_aarch64.h new file mode 100644 index 0000000000..c8d7c043a7 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITLink/ELF_aarch64.h @@ -0,0 +1,50 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===--- ELF_aarch64.h - JIT link functions for ELF/aarch64 --*- C++ -*----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +//===----------------------------------------------------------------------===// +// +// jit-link functions for ELF/aarch64. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_JITLINK_ELF_AARCH64_H +#define LLVM_EXECUTIONENGINE_JITLINK_ELF_AARCH64_H + +#include "llvm/ExecutionEngine/JITLink/JITLink.h" + +namespace llvm { +namespace jitlink { + +/// Create a LinkGraph from an ELF/aarch64 relocatable object +/// +/// Note: The graph does not take ownership of the underlying buffer, nor copy +/// its contents. The caller is responsible for ensuring that the object buffer +/// outlives the graph. +Expected<std::unique_ptr<LinkGraph>> +createLinkGraphFromELFObject_aarch64(MemoryBufferRef ObjectBuffer); + +/// jit-link the given object buffer, which must be a ELF aarch64 relocatable +/// object file. +void link_ELF_aarch64(std::unique_ptr<LinkGraph> G, + std::unique_ptr<JITLinkContext> Ctx); + +} // end namespace jitlink +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_JITLINK_ELF_AARCH64_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITLink/ELF_riscv.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITLink/ELF_riscv.h new file mode 100644 index 0000000000..9eaea5a2d1 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITLink/ELF_riscv.h @@ -0,0 +1,49 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===----- ELF_riscv.h - JIT link functions for ELF/riscv ----*- C++ -*----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +//===----------------------------------------------------------------------===// +// +// jit-link functions for ELF/riscv. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_JITLINK_ELF_RISCV_H +#define LLVM_EXECUTIONENGINE_JITLINK_ELF_RISCV_H + +#include "llvm/ExecutionEngine/JITLink/JITLink.h" + +namespace llvm { +namespace jitlink { + +/// Create a LinkGraph from an ELF/riscv relocatable object +/// +/// Note: The graph does not take ownership of the underlying buffer, nor copy +/// its contents. The caller is responsible for ensuring that the object buffer +/// outlives the graph. +Expected<std::unique_ptr<LinkGraph>> +createLinkGraphFromELFObject_riscv(MemoryBufferRef ObjectBuffer); + +/// jit-link the given object buffer, which must be a ELF riscv object file. +void link_ELF_riscv(std::unique_ptr<LinkGraph> G, + std::unique_ptr<JITLinkContext> Ctx); + +} // end namespace jitlink +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_JITLINK_ELF_RISCV_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITLink/ELF_x86_64.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITLink/ELF_x86_64.h new file mode 100644 index 0000000000..5ffae6fea9 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITLink/ELF_x86_64.h @@ -0,0 +1,67 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===--- ELF_x86_64.h - JIT link functions for ELF/x86-64 ---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// jit-link functions for ELF/x86-64. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_JITLINK_ELF_X86_64_H +#define LLVM_EXECUTIONENGINE_JITLINK_ELF_X86_64_H + +#include "llvm/ExecutionEngine/JITLink/JITLink.h" + +namespace llvm { +namespace jitlink { + +namespace ELF_x86_64_Edges { +enum ELFX86RelocationKind : Edge::Kind { + Branch32 = Edge::FirstRelocation, + Pointer32Signed, + Pointer64, + PCRel32, + PCRel32GOTLoad, + PCRel32GOTLoadRelaxable, + PCRel32REXGOTLoadRelaxable, + PCRel32TLV, + PCRel64GOT, + GOTOFF64, + GOT64, + Delta64, +}; + +} // end namespace ELF_x86_64_Edges + +/// Create a LinkGraph from an ELF/x86-64 relocatable object. +/// +/// Note: The graph does not take ownership of the underlying buffer, nor copy +/// its contents. The caller is responsible for ensuring that the object buffer +/// outlives the graph. +Expected<std::unique_ptr<LinkGraph>> +createLinkGraphFromELFObject_x86_64(MemoryBufferRef ObjectBuffer); + +/// jit-link the given object buffer, which must be a ELF x86-64 object file. +void link_ELF_x86_64(std::unique_ptr<LinkGraph> G, + std::unique_ptr<JITLinkContext> Ctx); + +/// Return the string name of the given ELF x86-64 edge kind. +const char *getELFX86RelocationKindName(Edge::Kind R); +} // end namespace jitlink +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_JITLINK_ELF_X86_64_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITLink/JITLink.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITLink/JITLink.h new file mode 100644 index 0000000000..07a0cc8a14 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITLink/JITLink.h @@ -0,0 +1,1791 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===------------ JITLink.h - JIT linker functionality ----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Contains generic JIT-linker types. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_JITLINK_JITLINK_H +#define LLVM_EXECUTIONENGINE_JITLINK_JITLINK_H + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/Triple.h" +#include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h" +#include "llvm/ExecutionEngine/JITLink/MemoryFlags.h" +#include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/Support/Allocator.h" +#include "llvm/Support/Endian.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/FormatVariadic.h" +#include "llvm/Support/MathExtras.h" +#include "llvm/Support/MemoryBuffer.h" + +#include <map> +#include <string> +#include <system_error> + +namespace llvm { +namespace jitlink { + +class LinkGraph; +class Symbol; +class Section; + +/// Base class for errors originating in JIT linker, e.g. missing relocation +/// support. +class JITLinkError : public ErrorInfo<JITLinkError> { +public: + static char ID; + + JITLinkError(Twine ErrMsg) : ErrMsg(ErrMsg.str()) {} + + void log(raw_ostream &OS) const override; + const std::string &getErrorMessage() const { return ErrMsg; } + std::error_code convertToErrorCode() const override; + +private: + std::string ErrMsg; +}; + +/// Represents fixups and constraints in the LinkGraph. +class Edge { +public: + using Kind = uint8_t; + + enum GenericEdgeKind : Kind { + Invalid, // Invalid edge value. + FirstKeepAlive, // Keeps target alive. Offset/addend zero. + KeepAlive = FirstKeepAlive, // Tag first edge kind that preserves liveness. + FirstRelocation // First architecture specific relocation. + }; + + using OffsetT = uint32_t; + using AddendT = int64_t; + + Edge(Kind K, OffsetT Offset, Symbol &Target, AddendT Addend) + : Target(&Target), Offset(Offset), Addend(Addend), K(K) {} + + OffsetT getOffset() const { return Offset; } + void setOffset(OffsetT Offset) { this->Offset = Offset; } + Kind getKind() const { return K; } + void setKind(Kind K) { this->K = K; } + bool isRelocation() const { return K >= FirstRelocation; } + Kind getRelocation() const { + assert(isRelocation() && "Not a relocation edge"); + return K - FirstRelocation; + } + bool isKeepAlive() const { return K >= FirstKeepAlive; } + Symbol &getTarget() const { return *Target; } + void setTarget(Symbol &Target) { this->Target = &Target; } + AddendT getAddend() const { return Addend; } + void setAddend(AddendT Addend) { this->Addend = Addend; } + +private: + Symbol *Target = nullptr; + OffsetT Offset = 0; + AddendT Addend = 0; + Kind K = 0; +}; + +/// Returns the string name of the given generic edge kind, or "unknown" +/// otherwise. Useful for debugging. +const char *getGenericEdgeKindName(Edge::Kind K); + +/// Base class for Addressable entities (externals, absolutes, blocks). +class Addressable { + friend class LinkGraph; + +protected: + Addressable(orc::ExecutorAddr Address, bool IsDefined) + : Address(Address), IsDefined(IsDefined), IsAbsolute(false) {} + + Addressable(orc::ExecutorAddr Address) + : Address(Address), IsDefined(false), IsAbsolute(true) { + assert(!(IsDefined && IsAbsolute) && + "Block cannot be both defined and absolute"); + } + +public: + Addressable(const Addressable &) = delete; + Addressable &operator=(const Addressable &) = default; + Addressable(Addressable &&) = delete; + Addressable &operator=(Addressable &&) = default; + + orc::ExecutorAddr getAddress() const { return Address; } + void setAddress(orc::ExecutorAddr Address) { this->Address = Address; } + + /// Returns true if this is a defined addressable, in which case you + /// can downcast this to a Block. + bool isDefined() const { return static_cast<bool>(IsDefined); } + bool isAbsolute() const { return static_cast<bool>(IsAbsolute); } + +private: + void setAbsolute(bool IsAbsolute) { + assert(!IsDefined && "Cannot change the Absolute flag on a defined block"); + this->IsAbsolute = IsAbsolute; + } + + orc::ExecutorAddr Address; + uint64_t IsDefined : 1; + uint64_t IsAbsolute : 1; + +protected: + // bitfields for Block, allocated here to improve packing. + uint64_t ContentMutable : 1; + uint64_t P2Align : 5; + uint64_t AlignmentOffset : 56; +}; + +using SectionOrdinal = unsigned; + +/// An Addressable with content and edges. +class Block : public Addressable { + friend class LinkGraph; + +private: + /// Create a zero-fill defined addressable. + Block(Section &Parent, orc::ExecutorAddrDiff Size, orc::ExecutorAddr Address, + uint64_t Alignment, uint64_t AlignmentOffset) + : Addressable(Address, true), Parent(&Parent), Size(Size) { + assert(isPowerOf2_64(Alignment) && "Alignment must be power of 2"); + assert(AlignmentOffset < Alignment && + "Alignment offset cannot exceed alignment"); + assert(AlignmentOffset <= MaxAlignmentOffset && + "Alignment offset exceeds maximum"); + ContentMutable = false; + P2Align = Alignment ? countTrailingZeros(Alignment) : 0; + this->AlignmentOffset = AlignmentOffset; + } + + /// Create a defined addressable for the given content. + /// The Content is assumed to be non-writable, and will be copied when + /// mutations are required. + Block(Section &Parent, ArrayRef<char> Content, orc::ExecutorAddr Address, + uint64_t Alignment, uint64_t AlignmentOffset) + : Addressable(Address, true), Parent(&Parent), Data(Content.data()), + Size(Content.size()) { + assert(isPowerOf2_64(Alignment) && "Alignment must be power of 2"); + assert(AlignmentOffset < Alignment && + "Alignment offset cannot exceed alignment"); + assert(AlignmentOffset <= MaxAlignmentOffset && + "Alignment offset exceeds maximum"); + ContentMutable = false; + P2Align = Alignment ? countTrailingZeros(Alignment) : 0; + this->AlignmentOffset = AlignmentOffset; + } + + /// Create a defined addressable for the given content. + /// The content is assumed to be writable, and the caller is responsible + /// for ensuring that it lives for the duration of the Block's lifetime. + /// The standard way to achieve this is to allocate it on the Graph's + /// allocator. + Block(Section &Parent, MutableArrayRef<char> Content, + orc::ExecutorAddr Address, uint64_t Alignment, uint64_t AlignmentOffset) + : Addressable(Address, true), Parent(&Parent), Data(Content.data()), + Size(Content.size()) { + assert(isPowerOf2_64(Alignment) && "Alignment must be power of 2"); + assert(AlignmentOffset < Alignment && + "Alignment offset cannot exceed alignment"); + assert(AlignmentOffset <= MaxAlignmentOffset && + "Alignment offset exceeds maximum"); + ContentMutable = true; + P2Align = Alignment ? countTrailingZeros(Alignment) : 0; + this->AlignmentOffset = AlignmentOffset; + } + +public: + using EdgeVector = std::vector<Edge>; + using edge_iterator = EdgeVector::iterator; + using const_edge_iterator = EdgeVector::const_iterator; + + Block(const Block &) = delete; + Block &operator=(const Block &) = delete; + Block(Block &&) = delete; + Block &operator=(Block &&) = delete; + + /// Return the parent section for this block. + Section &getSection() const { return *Parent; } + + /// Returns true if this is a zero-fill block. + /// + /// If true, getSize is callable but getContent is not (the content is + /// defined to be a sequence of zero bytes of length Size). + bool isZeroFill() const { return !Data; } + + /// Returns the size of this defined addressable. + size_t getSize() const { return Size; } + + /// Get the content for this block. Block must not be a zero-fill block. + ArrayRef<char> getContent() const { + assert(Data && "Block does not contain content"); + return ArrayRef<char>(Data, Size); + } + + /// Set the content for this block. + /// Caller is responsible for ensuring the underlying bytes are not + /// deallocated while pointed to by this block. + void setContent(ArrayRef<char> Content) { + assert(Content.data() && "Setting null content"); + Data = Content.data(); + Size = Content.size(); + ContentMutable = false; + } + + /// Get mutable content for this block. + /// + /// If this Block's content is not already mutable this will trigger a copy + /// of the existing immutable content to a new, mutable buffer allocated using + /// LinkGraph::allocateContent. + MutableArrayRef<char> getMutableContent(LinkGraph &G); + + /// Get mutable content for this block. + /// + /// This block's content must already be mutable. It is a programmatic error + /// to call this on a block with immutable content -- consider using + /// getMutableContent instead. + MutableArrayRef<char> getAlreadyMutableContent() { + assert(Data && "Block does not contain content"); + assert(ContentMutable && "Content is not mutable"); + return MutableArrayRef<char>(const_cast<char *>(Data), Size); + } + + /// Set mutable content for this block. + /// + /// The caller is responsible for ensuring that the memory pointed to by + /// MutableContent is not deallocated while pointed to by this block. + void setMutableContent(MutableArrayRef<char> MutableContent) { + assert(MutableContent.data() && "Setting null content"); + Data = MutableContent.data(); + Size = MutableContent.size(); + ContentMutable = true; + } + + /// Returns true if this block's content is mutable. + /// + /// This is primarily useful for asserting that a block is already in a + /// mutable state prior to modifying the content. E.g. when applying + /// fixups we expect the block to already be mutable as it should have been + /// copied to working memory. + bool isContentMutable() const { return ContentMutable; } + + /// Get the alignment for this content. + uint64_t getAlignment() const { return 1ull << P2Align; } + + /// Set the alignment for this content. + void setAlignment(uint64_t Alignment) { + assert(isPowerOf2_64(Alignment) && "Alignment must be a power of two"); + P2Align = Alignment ? countTrailingZeros(Alignment) : 0; + } + + /// Get the alignment offset for this content. + uint64_t getAlignmentOffset() const { return AlignmentOffset; } + + /// Set the alignment offset for this content. + void setAlignmentOffset(uint64_t AlignmentOffset) { + assert(AlignmentOffset < (1ull << P2Align) && + "Alignment offset can't exceed alignment"); + this->AlignmentOffset = AlignmentOffset; + } + + /// Add an edge to this block. + void addEdge(Edge::Kind K, Edge::OffsetT Offset, Symbol &Target, + Edge::AddendT Addend) { + assert(!isZeroFill() && "Adding edge to zero-fill block?"); + Edges.push_back(Edge(K, Offset, Target, Addend)); + } + + /// Add an edge by copying an existing one. This is typically used when + /// moving edges between blocks. + void addEdge(const Edge &E) { Edges.push_back(E); } + + /// Return the list of edges attached to this content. + iterator_range<edge_iterator> edges() { + return make_range(Edges.begin(), Edges.end()); + } + + /// Returns the list of edges attached to this content. + iterator_range<const_edge_iterator> edges() const { + return make_range(Edges.begin(), Edges.end()); + } + + /// Return the size of the edges list. + size_t edges_size() const { return Edges.size(); } + + /// Returns true if the list of edges is empty. + bool edges_empty() const { return Edges.empty(); } + + /// Remove the edge pointed to by the given iterator. + /// Returns an iterator to the new next element. + edge_iterator removeEdge(edge_iterator I) { return Edges.erase(I); } + + /// Returns the address of the fixup for the given edge, which is equal to + /// this block's address plus the edge's offset. + orc::ExecutorAddr getFixupAddress(const Edge &E) const { + return getAddress() + E.getOffset(); + } + +private: + static constexpr uint64_t MaxAlignmentOffset = (1ULL << 56) - 1; + + void setSection(Section &Parent) { this->Parent = &Parent; } + + Section *Parent; + const char *Data = nullptr; + size_t Size = 0; + std::vector<Edge> Edges; +}; + +// Align an address to conform with block alignment requirements. +inline uint64_t alignToBlock(uint64_t Addr, Block &B) { + uint64_t Delta = (B.getAlignmentOffset() - Addr) % B.getAlignment(); + return Addr + Delta; +} + +// Align a orc::ExecutorAddr to conform with block alignment requirements. +inline orc::ExecutorAddr alignToBlock(orc::ExecutorAddr Addr, Block &B) { + return orc::ExecutorAddr(alignToBlock(Addr.getValue(), B)); +} + +/// Describes symbol linkage. This can be used to make resolve definition +/// clashes. +enum class Linkage : uint8_t { + Strong, + Weak, +}; + +/// For errors and debugging output. +const char *getLinkageName(Linkage L); + +/// Defines the scope in which this symbol should be visible: +/// Default -- Visible in the public interface of the linkage unit. +/// Hidden -- Visible within the linkage unit, but not exported from it. +/// Local -- Visible only within the LinkGraph. +enum class Scope : uint8_t { + Default, + Hidden, + Local +}; + +/// For debugging output. +const char *getScopeName(Scope S); + +raw_ostream &operator<<(raw_ostream &OS, const Block &B); + +/// Symbol representation. +/// +/// Symbols represent locations within Addressable objects. +/// They can be either Named or Anonymous. +/// Anonymous symbols have neither linkage nor visibility, and must point at +/// ContentBlocks. +/// Named symbols may be in one of four states: +/// - Null: Default initialized. Assignable, but otherwise unusable. +/// - Defined: Has both linkage and visibility and points to a ContentBlock +/// - Common: Has both linkage and visibility, points to a null Addressable. +/// - External: Has neither linkage nor visibility, points to an external +/// Addressable. +/// +class Symbol { + friend class LinkGraph; + +private: + Symbol(Addressable &Base, orc::ExecutorAddrDiff Offset, StringRef Name, + orc::ExecutorAddrDiff Size, Linkage L, Scope S, bool IsLive, + bool IsCallable) + : Name(Name), Base(&Base), Offset(Offset), Size(Size) { + assert(Offset <= MaxOffset && "Offset out of range"); + setLinkage(L); + setScope(S); + setLive(IsLive); + setCallable(IsCallable); + } + + static Symbol &constructCommon(void *SymStorage, Block &Base, StringRef Name, + orc::ExecutorAddrDiff Size, Scope S, + bool IsLive) { + assert(SymStorage && "Storage cannot be null"); + assert(!Name.empty() && "Common symbol name cannot be empty"); + assert(Base.isDefined() && + "Cannot create common symbol from undefined block"); + assert(static_cast<Block &>(Base).getSize() == Size && + "Common symbol size should match underlying block size"); + auto *Sym = reinterpret_cast<Symbol *>(SymStorage); + new (Sym) Symbol(Base, 0, Name, Size, Linkage::Weak, S, IsLive, false); + return *Sym; + } + + static Symbol &constructExternal(void *SymStorage, Addressable &Base, + StringRef Name, orc::ExecutorAddrDiff Size, + Linkage L) { + assert(SymStorage && "Storage cannot be null"); + assert(!Base.isDefined() && + "Cannot create external symbol from defined block"); + assert(!Name.empty() && "External symbol name cannot be empty"); + auto *Sym = reinterpret_cast<Symbol *>(SymStorage); + new (Sym) Symbol(Base, 0, Name, Size, L, Scope::Default, false, false); + return *Sym; + } + + static Symbol &constructAbsolute(void *SymStorage, Addressable &Base, + StringRef Name, orc::ExecutorAddrDiff Size, + Linkage L, Scope S, bool IsLive) { + assert(SymStorage && "Storage cannot be null"); + assert(!Base.isDefined() && + "Cannot create absolute symbol from a defined block"); + auto *Sym = reinterpret_cast<Symbol *>(SymStorage); + new (Sym) Symbol(Base, 0, Name, Size, L, S, IsLive, false); + return *Sym; + } + + static Symbol &constructAnonDef(void *SymStorage, Block &Base, + orc::ExecutorAddrDiff Offset, + orc::ExecutorAddrDiff Size, bool IsCallable, + bool IsLive) { + assert(SymStorage && "Storage cannot be null"); + assert((Offset + Size) <= Base.getSize() && + "Symbol extends past end of block"); + auto *Sym = reinterpret_cast<Symbol *>(SymStorage); + new (Sym) Symbol(Base, Offset, StringRef(), Size, Linkage::Strong, + Scope::Local, IsLive, IsCallable); + return *Sym; + } + + static Symbol &constructNamedDef(void *SymStorage, Block &Base, + orc::ExecutorAddrDiff Offset, StringRef Name, + orc::ExecutorAddrDiff Size, Linkage L, + Scope S, bool IsLive, bool IsCallable) { + assert(SymStorage && "Storage cannot be null"); + assert((Offset + Size) <= Base.getSize() && + "Symbol extends past end of block"); + assert(!Name.empty() && "Name cannot be empty"); + auto *Sym = reinterpret_cast<Symbol *>(SymStorage); + new (Sym) Symbol(Base, Offset, Name, Size, L, S, IsLive, IsCallable); + return *Sym; + } + +public: + /// Create a null Symbol. This allows Symbols to be default initialized for + /// use in containers (e.g. as map values). Null symbols are only useful for + /// assigning to. + Symbol() = default; + + // Symbols are not movable or copyable. + Symbol(const Symbol &) = delete; + Symbol &operator=(const Symbol &) = delete; + Symbol(Symbol &&) = delete; + Symbol &operator=(Symbol &&) = delete; + + /// Returns true if this symbol has a name. + bool hasName() const { return !Name.empty(); } + + /// Returns the name of this symbol (empty if the symbol is anonymous). + StringRef getName() const { + assert((!Name.empty() || getScope() == Scope::Local) && + "Anonymous symbol has non-local scope"); + return Name; + } + + /// Rename this symbol. The client is responsible for updating scope and + /// linkage if this name-change requires it. + void setName(StringRef Name) { this->Name = Name; } + + /// Returns true if this Symbol has content (potentially) defined within this + /// object file (i.e. is anything but an external or absolute symbol). + bool isDefined() const { + assert(Base && "Attempt to access null symbol"); + return Base->isDefined(); + } + + /// Returns true if this symbol is live (i.e. should be treated as a root for + /// dead stripping). + bool isLive() const { + assert(Base && "Attempting to access null symbol"); + return IsLive; + } + + /// Set this symbol's live bit. + void setLive(bool IsLive) { this->IsLive = IsLive; } + + /// Returns true is this symbol is callable. + bool isCallable() const { return IsCallable; } + + /// Set this symbol's callable bit. + void setCallable(bool IsCallable) { this->IsCallable = IsCallable; } + + /// Returns true if the underlying addressable is an unresolved external. + bool isExternal() const { + assert(Base && "Attempt to access null symbol"); + return !Base->isDefined() && !Base->isAbsolute(); + } + + /// Returns true if the underlying addressable is an absolute symbol. + bool isAbsolute() const { + assert(Base && "Attempt to access null symbol"); + return Base->isAbsolute(); + } + + /// Return the addressable that this symbol points to. + Addressable &getAddressable() { + assert(Base && "Cannot get underlying addressable for null symbol"); + return *Base; + } + + /// Return the addressable that thsi symbol points to. + const Addressable &getAddressable() const { + assert(Base && "Cannot get underlying addressable for null symbol"); + return *Base; + } + + /// Return the Block for this Symbol (Symbol must be defined). + Block &getBlock() { + assert(Base && "Cannot get block for null symbol"); + assert(Base->isDefined() && "Not a defined symbol"); + return static_cast<Block &>(*Base); + } + + /// Return the Block for this Symbol (Symbol must be defined). + const Block &getBlock() const { + assert(Base && "Cannot get block for null symbol"); + assert(Base->isDefined() && "Not a defined symbol"); + return static_cast<const Block &>(*Base); + } + + /// Returns the offset for this symbol within the underlying addressable. + orc::ExecutorAddrDiff getOffset() const { return Offset; } + + /// Returns the address of this symbol. + orc::ExecutorAddr getAddress() const { return Base->getAddress() + Offset; } + + /// Returns the size of this symbol. + orc::ExecutorAddrDiff getSize() const { return Size; } + + /// Set the size of this symbol. + void setSize(orc::ExecutorAddrDiff Size) { + assert(Base && "Cannot set size for null Symbol"); + assert((Size == 0 || Base->isDefined()) && + "Non-zero size can only be set for defined symbols"); + assert((Offset + Size <= static_cast<const Block &>(*Base).getSize()) && + "Symbol size cannot extend past the end of its containing block"); + this->Size = Size; + } + + /// Returns true if this symbol is backed by a zero-fill block. + /// This method may only be called on defined symbols. + bool isSymbolZeroFill() const { return getBlock().isZeroFill(); } + + /// Returns the content in the underlying block covered by this symbol. + /// This method may only be called on defined non-zero-fill symbols. + ArrayRef<char> getSymbolContent() const { + return getBlock().getContent().slice(Offset, Size); + } + + /// Get the linkage for this Symbol. + Linkage getLinkage() const { return static_cast<Linkage>(L); } + + /// Set the linkage for this Symbol. + void setLinkage(Linkage L) { + assert((L == Linkage::Strong || (!Base->isAbsolute() && !Name.empty())) && + "Linkage can only be applied to defined named symbols"); + this->L = static_cast<uint8_t>(L); + } + + /// Get the visibility for this Symbol. + Scope getScope() const { return static_cast<Scope>(S); } + + /// Set the visibility for this Symbol. + void setScope(Scope S) { + assert((!Name.empty() || S == Scope::Local) && + "Can not set anonymous symbol to non-local scope"); + assert((S == Scope::Default || Base->isDefined() || Base->isAbsolute()) && + "Invalid visibility for symbol type"); + this->S = static_cast<uint8_t>(S); + } + +private: + void makeExternal(Addressable &A) { + assert(!A.isDefined() && !A.isAbsolute() && + "Attempting to make external with defined or absolute block"); + Base = &A; + Offset = 0; + setScope(Scope::Default); + IsLive = 0; + // note: Size, Linkage and IsCallable fields left unchanged. + } + + void makeAbsolute(Addressable &A) { + assert(!A.isDefined() && A.isAbsolute() && + "Attempting to make absolute with defined or external block"); + Base = &A; + Offset = 0; + } + + void setBlock(Block &B) { Base = &B; } + + void setOffset(orc::ExecutorAddrDiff NewOffset) { + assert(NewOffset <= MaxOffset && "Offset out of range"); + Offset = NewOffset; + } + + static constexpr uint64_t MaxOffset = (1ULL << 59) - 1; + + // FIXME: A char* or SymbolStringPtr may pack better. + StringRef Name; + Addressable *Base = nullptr; + uint64_t Offset : 59; + uint64_t L : 1; + uint64_t S : 2; + uint64_t IsLive : 1; + uint64_t IsCallable : 1; + orc::ExecutorAddrDiff Size = 0; +}; + +raw_ostream &operator<<(raw_ostream &OS, const Symbol &A); + +void printEdge(raw_ostream &OS, const Block &B, const Edge &E, + StringRef EdgeKindName); + +/// Represents an object file section. +class Section { + friend class LinkGraph; + +private: + Section(StringRef Name, MemProt Prot, SectionOrdinal SecOrdinal) + : Name(Name), Prot(Prot), SecOrdinal(SecOrdinal) {} + + using SymbolSet = DenseSet<Symbol *>; + using BlockSet = DenseSet<Block *>; + +public: + using symbol_iterator = SymbolSet::iterator; + using const_symbol_iterator = SymbolSet::const_iterator; + + using block_iterator = BlockSet::iterator; + using const_block_iterator = BlockSet::const_iterator; + + ~Section(); + + // Sections are not movable or copyable. + Section(const Section &) = delete; + Section &operator=(const Section &) = delete; + Section(Section &&) = delete; + Section &operator=(Section &&) = delete; + + /// Returns the name of this section. + StringRef getName() const { return Name; } + + /// Returns the protection flags for this section. + MemProt getMemProt() const { return Prot; } + + /// Set the protection flags for this section. + void setMemProt(MemProt Prot) { this->Prot = Prot; } + + /// Get the deallocation policy for this section. + MemDeallocPolicy getMemDeallocPolicy() const { return MDP; } + + /// Set the deallocation policy for this section. + void setMemDeallocPolicy(MemDeallocPolicy MDP) { this->MDP = MDP; } + + /// Returns the ordinal for this section. + SectionOrdinal getOrdinal() const { return SecOrdinal; } + + /// Returns an iterator over the blocks defined in this section. + iterator_range<block_iterator> blocks() { + return make_range(Blocks.begin(), Blocks.end()); + } + + /// Returns an iterator over the blocks defined in this section. + iterator_range<const_block_iterator> blocks() const { + return make_range(Blocks.begin(), Blocks.end()); + } + + /// Returns the number of blocks in this section. + BlockSet::size_type blocks_size() const { return Blocks.size(); } + + /// Returns an iterator over the symbols defined in this section. + iterator_range<symbol_iterator> symbols() { + return make_range(Symbols.begin(), Symbols.end()); + } + + /// Returns an iterator over the symbols defined in this section. + iterator_range<const_symbol_iterator> symbols() const { + return make_range(Symbols.begin(), Symbols.end()); + } + + /// Return the number of symbols in this section. + SymbolSet::size_type symbols_size() const { return Symbols.size(); } + +private: + void addSymbol(Symbol &Sym) { + assert(!Symbols.count(&Sym) && "Symbol is already in this section"); + Symbols.insert(&Sym); + } + + void removeSymbol(Symbol &Sym) { + assert(Symbols.count(&Sym) && "symbol is not in this section"); + Symbols.erase(&Sym); + } + + void addBlock(Block &B) { + assert(!Blocks.count(&B) && "Block is already in this section"); + Blocks.insert(&B); + } + + void removeBlock(Block &B) { + assert(Blocks.count(&B) && "Block is not in this section"); + Blocks.erase(&B); + } + + void transferContentTo(Section &DstSection) { + if (&DstSection == this) + return; + for (auto *S : Symbols) + DstSection.addSymbol(*S); + for (auto *B : Blocks) + DstSection.addBlock(*B); + Symbols.clear(); + Blocks.clear(); + } + + StringRef Name; + MemProt Prot; + MemDeallocPolicy MDP = MemDeallocPolicy::Standard; + SectionOrdinal SecOrdinal = 0; + BlockSet Blocks; + SymbolSet Symbols; +}; + +/// Represents a section address range via a pair of Block pointers +/// to the first and last Blocks in the section. +class SectionRange { +public: + SectionRange() = default; + SectionRange(const Section &Sec) { + if (llvm::empty(Sec.blocks())) + return; + First = Last = *Sec.blocks().begin(); + for (auto *B : Sec.blocks()) { + if (B->getAddress() < First->getAddress()) + First = B; + if (B->getAddress() > Last->getAddress()) + Last = B; + } + } + Block *getFirstBlock() const { + assert((!Last || First) && "First can not be null if end is non-null"); + return First; + } + Block *getLastBlock() const { + assert((First || !Last) && "Last can not be null if start is non-null"); + return Last; + } + bool empty() const { + assert((First || !Last) && "Last can not be null if start is non-null"); + return !First; + } + orc::ExecutorAddr getStart() const { + return First ? First->getAddress() : orc::ExecutorAddr(); + } + orc::ExecutorAddr getEnd() const { + return Last ? Last->getAddress() + Last->getSize() : orc::ExecutorAddr(); + } + orc::ExecutorAddrDiff getSize() const { return getEnd() - getStart(); } + + orc::ExecutorAddrRange getRange() const { + return orc::ExecutorAddrRange(getStart(), getEnd()); + } + +private: + Block *First = nullptr; + Block *Last = nullptr; +}; + +class LinkGraph { +private: + using SectionList = std::vector<std::unique_ptr<Section>>; + using ExternalSymbolSet = DenseSet<Symbol *>; + using BlockSet = DenseSet<Block *>; + + template <typename... ArgTs> + Addressable &createAddressable(ArgTs &&... Args) { + Addressable *A = + reinterpret_cast<Addressable *>(Allocator.Allocate<Addressable>()); + new (A) Addressable(std::forward<ArgTs>(Args)...); + return *A; + } + + void destroyAddressable(Addressable &A) { + A.~Addressable(); + Allocator.Deallocate(&A); + } + + template <typename... ArgTs> Block &createBlock(ArgTs &&... Args) { + Block *B = reinterpret_cast<Block *>(Allocator.Allocate<Block>()); + new (B) Block(std::forward<ArgTs>(Args)...); + B->getSection().addBlock(*B); + return *B; + } + + void destroyBlock(Block &B) { + B.~Block(); + Allocator.Deallocate(&B); + } + + void destroySymbol(Symbol &S) { + S.~Symbol(); + Allocator.Deallocate(&S); + } + + static iterator_range<Section::block_iterator> getSectionBlocks(Section &S) { + return S.blocks(); + } + + static iterator_range<Section::const_block_iterator> + getSectionConstBlocks(Section &S) { + return S.blocks(); + } + + static iterator_range<Section::symbol_iterator> + getSectionSymbols(Section &S) { + return S.symbols(); + } + + static iterator_range<Section::const_symbol_iterator> + getSectionConstSymbols(Section &S) { + return S.symbols(); + } + +public: + using external_symbol_iterator = ExternalSymbolSet::iterator; + + using section_iterator = pointee_iterator<SectionList::iterator>; + using const_section_iterator = pointee_iterator<SectionList::const_iterator>; + + template <typename OuterItrT, typename InnerItrT, typename T, + iterator_range<InnerItrT> getInnerRange( + typename OuterItrT::reference)> + class nested_collection_iterator + : public iterator_facade_base< + nested_collection_iterator<OuterItrT, InnerItrT, T, getInnerRange>, + std::forward_iterator_tag, T> { + public: + nested_collection_iterator() = default; + + nested_collection_iterator(OuterItrT OuterI, OuterItrT OuterE) + : OuterI(OuterI), OuterE(OuterE), + InnerI(getInnerBegin(OuterI, OuterE)) { + moveToNonEmptyInnerOrEnd(); + } + + bool operator==(const nested_collection_iterator &RHS) const { + return (OuterI == RHS.OuterI) && (InnerI == RHS.InnerI); + } + + T operator*() const { + assert(InnerI != getInnerRange(*OuterI).end() && "Dereferencing end?"); + return *InnerI; + } + + nested_collection_iterator operator++() { + ++InnerI; + moveToNonEmptyInnerOrEnd(); + return *this; + } + + private: + static InnerItrT getInnerBegin(OuterItrT OuterI, OuterItrT OuterE) { + return OuterI != OuterE ? getInnerRange(*OuterI).begin() : InnerItrT(); + } + + void moveToNonEmptyInnerOrEnd() { + while (OuterI != OuterE && InnerI == getInnerRange(*OuterI).end()) { + ++OuterI; + InnerI = getInnerBegin(OuterI, OuterE); + } + } + + OuterItrT OuterI, OuterE; + InnerItrT InnerI; + }; + + using defined_symbol_iterator = + nested_collection_iterator<const_section_iterator, + Section::symbol_iterator, Symbol *, + getSectionSymbols>; + + using const_defined_symbol_iterator = + nested_collection_iterator<const_section_iterator, + Section::const_symbol_iterator, const Symbol *, + getSectionConstSymbols>; + + using block_iterator = nested_collection_iterator<const_section_iterator, + Section::block_iterator, + Block *, getSectionBlocks>; + + using const_block_iterator = + nested_collection_iterator<const_section_iterator, + Section::const_block_iterator, const Block *, + getSectionConstBlocks>; + + using GetEdgeKindNameFunction = const char *(*)(Edge::Kind); + + LinkGraph(std::string Name, const Triple &TT, unsigned PointerSize, + support::endianness Endianness, + GetEdgeKindNameFunction GetEdgeKindName) + : Name(std::move(Name)), TT(TT), PointerSize(PointerSize), + Endianness(Endianness), GetEdgeKindName(std::move(GetEdgeKindName)) {} + + LinkGraph(const LinkGraph &) = delete; + LinkGraph &operator=(const LinkGraph &) = delete; + LinkGraph(LinkGraph &&) = delete; + LinkGraph &operator=(LinkGraph &&) = delete; + + /// Returns the name of this graph (usually the name of the original + /// underlying MemoryBuffer). + const std::string &getName() const { return Name; } + + /// Returns the target triple for this Graph. + const Triple &getTargetTriple() const { return TT; } + + /// Returns the pointer size for use in this graph. + unsigned getPointerSize() const { return PointerSize; } + + /// Returns the endianness of content in this graph. + support::endianness getEndianness() const { return Endianness; } + + const char *getEdgeKindName(Edge::Kind K) const { return GetEdgeKindName(K); } + + /// Allocate a mutable buffer of the given size using the LinkGraph's + /// allocator. + MutableArrayRef<char> allocateBuffer(size_t Size) { + return {Allocator.Allocate<char>(Size), Size}; + } + + /// Allocate a copy of the given string using the LinkGraph's allocator. + /// This can be useful when renaming symbols or adding new content to the + /// graph. + MutableArrayRef<char> allocateContent(ArrayRef<char> Source) { + auto *AllocatedBuffer = Allocator.Allocate<char>(Source.size()); + llvm::copy(Source, AllocatedBuffer); + return MutableArrayRef<char>(AllocatedBuffer, Source.size()); + } + + /// Allocate a copy of the given string using the LinkGraph's allocator. + /// This can be useful when renaming symbols or adding new content to the + /// graph. + /// + /// Note: This Twine-based overload requires an extra string copy and an + /// extra heap allocation for large strings. The ArrayRef<char> overload + /// should be preferred where possible. + MutableArrayRef<char> allocateString(Twine Source) { + SmallString<256> TmpBuffer; + auto SourceStr = Source.toStringRef(TmpBuffer); + auto *AllocatedBuffer = Allocator.Allocate<char>(SourceStr.size()); + llvm::copy(SourceStr, AllocatedBuffer); + return MutableArrayRef<char>(AllocatedBuffer, SourceStr.size()); + } + + /// Create a section with the given name, protection flags, and alignment. + Section &createSection(StringRef Name, MemProt Prot) { + assert(llvm::find_if(Sections, + [&](std::unique_ptr<Section> &Sec) { + return Sec->getName() == Name; + }) == Sections.end() && + "Duplicate section name"); + std::unique_ptr<Section> Sec(new Section(Name, Prot, Sections.size())); + Sections.push_back(std::move(Sec)); + return *Sections.back(); + } + + /// Create a content block. + Block &createContentBlock(Section &Parent, ArrayRef<char> Content, + orc::ExecutorAddr Address, uint64_t Alignment, + uint64_t AlignmentOffset) { + return createBlock(Parent, Content, Address, Alignment, AlignmentOffset); + } + + /// Create a content block with initially mutable data. + Block &createMutableContentBlock(Section &Parent, + MutableArrayRef<char> MutableContent, + orc::ExecutorAddr Address, + uint64_t Alignment, + uint64_t AlignmentOffset) { + return createBlock(Parent, MutableContent, Address, Alignment, + AlignmentOffset); + } + + /// Create a zero-fill block. + Block &createZeroFillBlock(Section &Parent, orc::ExecutorAddrDiff Size, + orc::ExecutorAddr Address, uint64_t Alignment, + uint64_t AlignmentOffset) { + return createBlock(Parent, Size, Address, Alignment, AlignmentOffset); + } + + /// Cache type for the splitBlock function. + using SplitBlockCache = Optional<SmallVector<Symbol *, 8>>; + + /// Splits block B at the given index which must be greater than zero. + /// If SplitIndex == B.getSize() then this function is a no-op and returns B. + /// If SplitIndex < B.getSize() then this function returns a new block + /// covering the range [ 0, SplitIndex ), and B is modified to cover the range + /// [ SplitIndex, B.size() ). + /// + /// The optional Cache parameter can be used to speed up repeated calls to + /// splitBlock for a single block. If the value is None the cache will be + /// treated as uninitialized and splitBlock will populate it. Otherwise it + /// is assumed to contain the list of Symbols pointing at B, sorted in + /// descending order of offset. + /// + /// Notes: + /// + /// 1. splitBlock must be used with care. Splitting a block may cause + /// incoming edges to become invalid if the edge target subexpression + /// points outside the bounds of the newly split target block (E.g. an + /// edge 'S + 10 : Pointer64' where S points to a newly split block + /// whose size is less than 10). No attempt is made to detect invalidation + /// of incoming edges, as in general this requires context that the + /// LinkGraph does not have. Clients are responsible for ensuring that + /// splitBlock is not used in a way that invalidates edges. + /// + /// 2. The newly introduced block will have a new ordinal which will be + /// higher than any other ordinals in the section. Clients are responsible + /// for re-assigning block ordinals to restore a compatible order if + /// needed. + /// + /// 3. The cache is not automatically updated if new symbols are introduced + /// between calls to splitBlock. Any newly introduced symbols may be + /// added to the cache manually (descending offset order must be + /// preserved), or the cache can be set to None and rebuilt by + /// splitBlock on the next call. + Block &splitBlock(Block &B, size_t SplitIndex, + SplitBlockCache *Cache = nullptr); + + /// Add an external symbol. + /// Some formats (e.g. ELF) allow Symbols to have sizes. For Symbols whose + /// size is not known, you should substitute '0'. + /// For external symbols Linkage determines whether the symbol must be + /// present during lookup: Externals with strong linkage must be found or + /// an error will be emitted. Externals with weak linkage are permitted to + /// be undefined, in which case they are assigned a value of 0. + Symbol &addExternalSymbol(StringRef Name, orc::ExecutorAddrDiff Size, + Linkage L) { + assert(llvm::count_if(ExternalSymbols, + [&](const Symbol *Sym) { + return Sym->getName() == Name; + }) == 0 && + "Duplicate external symbol"); + auto &Sym = Symbol::constructExternal( + Allocator.Allocate<Symbol>(), + createAddressable(orc::ExecutorAddr(), false), Name, Size, L); + ExternalSymbols.insert(&Sym); + return Sym; + } + + /// Add an absolute symbol. + Symbol &addAbsoluteSymbol(StringRef Name, orc::ExecutorAddr Address, + orc::ExecutorAddrDiff Size, Linkage L, Scope S, + bool IsLive) { + assert(llvm::count_if(AbsoluteSymbols, + [&](const Symbol *Sym) { + return Sym->getName() == Name; + }) == 0 && + "Duplicate absolute symbol"); + auto &Sym = Symbol::constructAbsolute(Allocator.Allocate<Symbol>(), + createAddressable(Address), Name, + Size, L, S, IsLive); + AbsoluteSymbols.insert(&Sym); + return Sym; + } + + /// Convenience method for adding a weak zero-fill symbol. + Symbol &addCommonSymbol(StringRef Name, Scope S, Section &Section, + orc::ExecutorAddr Address, orc::ExecutorAddrDiff Size, + uint64_t Alignment, bool IsLive) { + assert(llvm::count_if(defined_symbols(), + [&](const Symbol *Sym) { + return Sym->getName() == Name; + }) == 0 && + "Duplicate defined symbol"); + auto &Sym = Symbol::constructCommon( + Allocator.Allocate<Symbol>(), + createBlock(Section, Size, Address, Alignment, 0), Name, Size, S, + IsLive); + Section.addSymbol(Sym); + return Sym; + } + + /// Add an anonymous symbol. + Symbol &addAnonymousSymbol(Block &Content, orc::ExecutorAddrDiff Offset, + orc::ExecutorAddrDiff Size, bool IsCallable, + bool IsLive) { + auto &Sym = Symbol::constructAnonDef(Allocator.Allocate<Symbol>(), Content, + Offset, Size, IsCallable, IsLive); + Content.getSection().addSymbol(Sym); + return Sym; + } + + /// Add a named symbol. + Symbol &addDefinedSymbol(Block &Content, orc::ExecutorAddrDiff Offset, + StringRef Name, orc::ExecutorAddrDiff Size, + Linkage L, Scope S, bool IsCallable, bool IsLive) { + assert((S == Scope::Local || llvm::count_if(defined_symbols(), + [&](const Symbol *Sym) { + return Sym->getName() == Name; + }) == 0) && + "Duplicate defined symbol"); + auto &Sym = + Symbol::constructNamedDef(Allocator.Allocate<Symbol>(), Content, Offset, + Name, Size, L, S, IsLive, IsCallable); + Content.getSection().addSymbol(Sym); + return Sym; + } + + iterator_range<section_iterator> sections() { + return make_range(section_iterator(Sections.begin()), + section_iterator(Sections.end())); + } + + SectionList::size_type sections_size() const { return Sections.size(); } + + /// Returns the section with the given name if it exists, otherwise returns + /// null. + Section *findSectionByName(StringRef Name) { + for (auto &S : sections()) + if (S.getName() == Name) + return &S; + return nullptr; + } + + iterator_range<block_iterator> blocks() { + return make_range(block_iterator(Sections.begin(), Sections.end()), + block_iterator(Sections.end(), Sections.end())); + } + + iterator_range<const_block_iterator> blocks() const { + return make_range(const_block_iterator(Sections.begin(), Sections.end()), + const_block_iterator(Sections.end(), Sections.end())); + } + + iterator_range<external_symbol_iterator> external_symbols() { + return make_range(ExternalSymbols.begin(), ExternalSymbols.end()); + } + + iterator_range<external_symbol_iterator> absolute_symbols() { + return make_range(AbsoluteSymbols.begin(), AbsoluteSymbols.end()); + } + + iterator_range<defined_symbol_iterator> defined_symbols() { + return make_range(defined_symbol_iterator(Sections.begin(), Sections.end()), + defined_symbol_iterator(Sections.end(), Sections.end())); + } + + iterator_range<const_defined_symbol_iterator> defined_symbols() const { + return make_range( + const_defined_symbol_iterator(Sections.begin(), Sections.end()), + const_defined_symbol_iterator(Sections.end(), Sections.end())); + } + + /// Make the given symbol external (must not already be external). + /// + /// Symbol size, linkage and callability will be left unchanged. Symbol scope + /// will be set to Default, and offset will be reset to 0. + void makeExternal(Symbol &Sym) { + assert(!Sym.isExternal() && "Symbol is already external"); + if (Sym.isAbsolute()) { + assert(AbsoluteSymbols.count(&Sym) && + "Sym is not in the absolute symbols set"); + assert(Sym.getOffset() == 0 && "Absolute not at offset 0"); + AbsoluteSymbols.erase(&Sym); + Sym.getAddressable().setAbsolute(false); + } else { + assert(Sym.isDefined() && "Sym is not a defined symbol"); + Section &Sec = Sym.getBlock().getSection(); + Sec.removeSymbol(Sym); + Sym.makeExternal(createAddressable(orc::ExecutorAddr(), false)); + } + ExternalSymbols.insert(&Sym); + } + + /// Make the given symbol an absolute with the given address (must not already + /// be absolute). + /// + /// Symbol size, linkage, scope, and callability, and liveness will be left + /// unchanged. Symbol offset will be reset to 0. + void makeAbsolute(Symbol &Sym, orc::ExecutorAddr Address) { + assert(!Sym.isAbsolute() && "Symbol is already absolute"); + if (Sym.isExternal()) { + assert(ExternalSymbols.count(&Sym) && + "Sym is not in the absolute symbols set"); + assert(Sym.getOffset() == 0 && "External is not at offset 0"); + ExternalSymbols.erase(&Sym); + Sym.getAddressable().setAbsolute(true); + } else { + assert(Sym.isDefined() && "Sym is not a defined symbol"); + Section &Sec = Sym.getBlock().getSection(); + Sec.removeSymbol(Sym); + Sym.makeAbsolute(createAddressable(Address)); + } + AbsoluteSymbols.insert(&Sym); + } + + /// Turn an absolute or external symbol into a defined one by attaching it to + /// a block. Symbol must not already be defined. + void makeDefined(Symbol &Sym, Block &Content, orc::ExecutorAddrDiff Offset, + orc::ExecutorAddrDiff Size, Linkage L, Scope S, + bool IsLive) { + assert(!Sym.isDefined() && "Sym is already a defined symbol"); + if (Sym.isAbsolute()) { + assert(AbsoluteSymbols.count(&Sym) && + "Symbol is not in the absolutes set"); + AbsoluteSymbols.erase(&Sym); + } else { + assert(ExternalSymbols.count(&Sym) && + "Symbol is not in the externals set"); + ExternalSymbols.erase(&Sym); + } + Addressable &OldBase = *Sym.Base; + Sym.setBlock(Content); + Sym.setOffset(Offset); + Sym.setSize(Size); + Sym.setLinkage(L); + Sym.setScope(S); + Sym.setLive(IsLive); + Content.getSection().addSymbol(Sym); + destroyAddressable(OldBase); + } + + /// Transfer a defined symbol from one block to another. + /// + /// The symbol's offset within DestBlock is set to NewOffset. + /// + /// If ExplicitNewSize is given as None then the size of the symbol will be + /// checked and auto-truncated to at most the size of the remainder (from the + /// given offset) of the size of the new block. + /// + /// All other symbol attributes are unchanged. + void transferDefinedSymbol(Symbol &Sym, Block &DestBlock, + orc::ExecutorAddrDiff NewOffset, + Optional<orc::ExecutorAddrDiff> ExplicitNewSize) { + auto &OldSection = Sym.getBlock().getSection(); + Sym.setBlock(DestBlock); + Sym.setOffset(NewOffset); + if (ExplicitNewSize) + Sym.setSize(*ExplicitNewSize); + else { + auto RemainingBlockSize = DestBlock.getSize() - NewOffset; + if (Sym.getSize() > RemainingBlockSize) + Sym.setSize(RemainingBlockSize); + } + if (&DestBlock.getSection() != &OldSection) { + OldSection.removeSymbol(Sym); + DestBlock.getSection().addSymbol(Sym); + } + } + + /// Transfers the given Block and all Symbols pointing to it to the given + /// Section. + /// + /// No attempt is made to check compatibility of the source and destination + /// sections. Blocks may be moved between sections with incompatible + /// permissions (e.g. from data to text). The client is responsible for + /// ensuring that this is safe. + void transferBlock(Block &B, Section &NewSection) { + auto &OldSection = B.getSection(); + if (&OldSection == &NewSection) + return; + SmallVector<Symbol *> AttachedSymbols; + for (auto *S : OldSection.symbols()) + if (&S->getBlock() == &B) + AttachedSymbols.push_back(S); + for (auto *S : AttachedSymbols) { + OldSection.removeSymbol(*S); + NewSection.addSymbol(*S); + } + OldSection.removeBlock(B); + NewSection.addBlock(B); + } + + /// Move all blocks and symbols from the source section to the destination + /// section. + /// + /// If PreserveSrcSection is true (or SrcSection and DstSection are the same) + /// then SrcSection is preserved, otherwise it is removed (the default). + void mergeSections(Section &DstSection, Section &SrcSection, + bool PreserveSrcSection = false) { + if (&DstSection == &SrcSection) + return; + for (auto *B : SrcSection.blocks()) + B->setSection(DstSection); + SrcSection.transferContentTo(DstSection); + if (!PreserveSrcSection) + removeSection(SrcSection); + } + + /// Removes an external symbol. Also removes the underlying Addressable. + void removeExternalSymbol(Symbol &Sym) { + assert(!Sym.isDefined() && !Sym.isAbsolute() && + "Sym is not an external symbol"); + assert(ExternalSymbols.count(&Sym) && "Symbol is not in the externals set"); + ExternalSymbols.erase(&Sym); + Addressable &Base = *Sym.Base; + assert(llvm::find_if(ExternalSymbols, + [&](Symbol *AS) { return AS->Base == &Base; }) == + ExternalSymbols.end() && + "Base addressable still in use"); + destroySymbol(Sym); + destroyAddressable(Base); + } + + /// Remove an absolute symbol. Also removes the underlying Addressable. + void removeAbsoluteSymbol(Symbol &Sym) { + assert(!Sym.isDefined() && Sym.isAbsolute() && + "Sym is not an absolute symbol"); + assert(AbsoluteSymbols.count(&Sym) && + "Symbol is not in the absolute symbols set"); + AbsoluteSymbols.erase(&Sym); + Addressable &Base = *Sym.Base; + assert(llvm::find_if(ExternalSymbols, + [&](Symbol *AS) { return AS->Base == &Base; }) == + ExternalSymbols.end() && + "Base addressable still in use"); + destroySymbol(Sym); + destroyAddressable(Base); + } + + /// Removes defined symbols. Does not remove the underlying block. + void removeDefinedSymbol(Symbol &Sym) { + assert(Sym.isDefined() && "Sym is not a defined symbol"); + Sym.getBlock().getSection().removeSymbol(Sym); + destroySymbol(Sym); + } + + /// Remove a block. The block reference is defunct after calling this + /// function and should no longer be used. + void removeBlock(Block &B) { + assert(llvm::none_of(B.getSection().symbols(), + [&](const Symbol *Sym) { + return &Sym->getBlock() == &B; + }) && + "Block still has symbols attached"); + B.getSection().removeBlock(B); + destroyBlock(B); + } + + /// Remove a section. The section reference is defunct after calling this + /// function and should no longer be used. + void removeSection(Section &Sec) { + auto I = llvm::find_if(Sections, [&Sec](const std::unique_ptr<Section> &S) { + return S.get() == &Sec; + }); + assert(I != Sections.end() && "Section does not appear in this graph"); + Sections.erase(I); + } + + /// Accessor for the AllocActions object for this graph. This can be used to + /// register allocation action calls prior to finalization. + /// + /// Accessing this object after finalization will result in undefined + /// behavior. + orc::shared::AllocActions &allocActions() { return AAs; } + + /// Dump the graph. + void dump(raw_ostream &OS); + +private: + // Put the BumpPtrAllocator first so that we don't free any of the underlying + // memory until the Symbol/Addressable destructors have been run. + BumpPtrAllocator Allocator; + + std::string Name; + Triple TT; + unsigned PointerSize; + support::endianness Endianness; + GetEdgeKindNameFunction GetEdgeKindName = nullptr; + SectionList Sections; + ExternalSymbolSet ExternalSymbols; + ExternalSymbolSet AbsoluteSymbols; + orc::shared::AllocActions AAs; +}; + +inline MutableArrayRef<char> Block::getMutableContent(LinkGraph &G) { + if (!ContentMutable) + setMutableContent(G.allocateContent({Data, Size})); + return MutableArrayRef<char>(const_cast<char *>(Data), Size); +} + +/// Enables easy lookup of blocks by addresses. +class BlockAddressMap { +public: + using AddrToBlockMap = std::map<orc::ExecutorAddr, Block *>; + using const_iterator = AddrToBlockMap::const_iterator; + + /// A block predicate that always adds all blocks. + static bool includeAllBlocks(const Block &B) { return true; } + + /// A block predicate that always includes blocks with non-null addresses. + static bool includeNonNull(const Block &B) { return !!B.getAddress(); } + + BlockAddressMap() = default; + + /// Add a block to the map. Returns an error if the block overlaps with any + /// existing block. + template <typename PredFn = decltype(includeAllBlocks)> + Error addBlock(Block &B, PredFn Pred = includeAllBlocks) { + if (!Pred(B)) + return Error::success(); + + auto I = AddrToBlock.upper_bound(B.getAddress()); + + // If we're not at the end of the map, check for overlap with the next + // element. + if (I != AddrToBlock.end()) { + if (B.getAddress() + B.getSize() > I->second->getAddress()) + return overlapError(B, *I->second); + } + + // If we're not at the start of the map, check for overlap with the previous + // element. + if (I != AddrToBlock.begin()) { + auto &PrevBlock = *std::prev(I)->second; + if (PrevBlock.getAddress() + PrevBlock.getSize() > B.getAddress()) + return overlapError(B, PrevBlock); + } + + AddrToBlock.insert(I, std::make_pair(B.getAddress(), &B)); + return Error::success(); + } + + /// Add a block to the map without checking for overlap with existing blocks. + /// The client is responsible for ensuring that the block added does not + /// overlap with any existing block. + void addBlockWithoutChecking(Block &B) { AddrToBlock[B.getAddress()] = &B; } + + /// Add a range of blocks to the map. Returns an error if any block in the + /// range overlaps with any other block in the range, or with any existing + /// block in the map. + template <typename BlockPtrRange, + typename PredFn = decltype(includeAllBlocks)> + Error addBlocks(BlockPtrRange &&Blocks, PredFn Pred = includeAllBlocks) { + for (auto *B : Blocks) + if (auto Err = addBlock(*B, Pred)) + return Err; + return Error::success(); + } + + /// Add a range of blocks to the map without checking for overlap with + /// existing blocks. The client is responsible for ensuring that the block + /// added does not overlap with any existing block. + template <typename BlockPtrRange> + void addBlocksWithoutChecking(BlockPtrRange &&Blocks) { + for (auto *B : Blocks) + addBlockWithoutChecking(*B); + } + + /// Iterates over (Address, Block*) pairs in ascending order of address. + const_iterator begin() const { return AddrToBlock.begin(); } + const_iterator end() const { return AddrToBlock.end(); } + + /// Returns the block starting at the given address, or nullptr if no such + /// block exists. + Block *getBlockAt(orc::ExecutorAddr Addr) const { + auto I = AddrToBlock.find(Addr); + if (I == AddrToBlock.end()) + return nullptr; + return I->second; + } + + /// Returns the block covering the given address, or nullptr if no such block + /// exists. + Block *getBlockCovering(orc::ExecutorAddr Addr) const { + auto I = AddrToBlock.upper_bound(Addr); + if (I == AddrToBlock.begin()) + return nullptr; + auto *B = std::prev(I)->second; + if (Addr < B->getAddress() + B->getSize()) + return B; + return nullptr; + } + +private: + Error overlapError(Block &NewBlock, Block &ExistingBlock) { + auto NewBlockEnd = NewBlock.getAddress() + NewBlock.getSize(); + auto ExistingBlockEnd = + ExistingBlock.getAddress() + ExistingBlock.getSize(); + return make_error<JITLinkError>( + "Block at " + + formatv("{0:x16} -- {1:x16}", NewBlock.getAddress().getValue(), + NewBlockEnd.getValue()) + + " overlaps " + + formatv("{0:x16} -- {1:x16}", ExistingBlock.getAddress().getValue(), + ExistingBlockEnd.getValue())); + } + + AddrToBlockMap AddrToBlock; +}; + +/// A map of addresses to Symbols. +class SymbolAddressMap { +public: + using SymbolVector = SmallVector<Symbol *, 1>; + + /// Add a symbol to the SymbolAddressMap. + void addSymbol(Symbol &Sym) { + AddrToSymbols[Sym.getAddress()].push_back(&Sym); + } + + /// Add all symbols in a given range to the SymbolAddressMap. + template <typename SymbolPtrCollection> + void addSymbols(SymbolPtrCollection &&Symbols) { + for (auto *Sym : Symbols) + addSymbol(*Sym); + } + + /// Returns the list of symbols that start at the given address, or nullptr if + /// no such symbols exist. + const SymbolVector *getSymbolsAt(orc::ExecutorAddr Addr) const { + auto I = AddrToSymbols.find(Addr); + if (I == AddrToSymbols.end()) + return nullptr; + return &I->second; + } + +private: + std::map<orc::ExecutorAddr, SymbolVector> AddrToSymbols; +}; + +/// A function for mutating LinkGraphs. +using LinkGraphPassFunction = std::function<Error(LinkGraph &)>; + +/// A list of LinkGraph passes. +using LinkGraphPassList = std::vector<LinkGraphPassFunction>; + +/// An LinkGraph pass configuration, consisting of a list of pre-prune, +/// post-prune, and post-fixup passes. +struct PassConfiguration { + + /// Pre-prune passes. + /// + /// These passes are called on the graph after it is built, and before any + /// symbols have been pruned. Graph nodes still have their original vmaddrs. + /// + /// Notable use cases: Marking symbols live or should-discard. + LinkGraphPassList PrePrunePasses; + + /// Post-prune passes. + /// + /// These passes are called on the graph after dead stripping, but before + /// memory is allocated or nodes assigned their final addresses. + /// + /// Notable use cases: Building GOT, stub, and TLV symbols. + LinkGraphPassList PostPrunePasses; + + /// Post-allocation passes. + /// + /// These passes are called on the graph after memory has been allocated and + /// defined nodes have been assigned their final addresses, but before the + /// context has been notified of these addresses. At this point externals + /// have not been resolved, and symbol content has not yet been copied into + /// working memory. + /// + /// Notable use cases: Setting up data structures associated with addresses + /// of defined symbols (e.g. a mapping of __dso_handle to JITDylib* for the + /// JIT runtime) -- using a PostAllocationPass for this ensures that the + /// data structures are in-place before any query for resolved symbols + /// can complete. + LinkGraphPassList PostAllocationPasses; + + /// Pre-fixup passes. + /// + /// These passes are called on the graph after memory has been allocated, + /// content copied into working memory, and all nodes (including externals) + /// have been assigned their final addresses, but before any fixups have been + /// applied. + /// + /// Notable use cases: Late link-time optimizations like GOT and stub + /// elimination. + LinkGraphPassList PreFixupPasses; + + /// Post-fixup passes. + /// + /// These passes are called on the graph after block contents has been copied + /// to working memory, and fixups applied. Blocks have been updated to point + /// to their fixed up content. + /// + /// Notable use cases: Testing and validation. + LinkGraphPassList PostFixupPasses; +}; + +/// Flags for symbol lookup. +/// +/// FIXME: These basically duplicate orc::SymbolLookupFlags -- We should merge +/// the two types once we have an OrcSupport library. +enum class SymbolLookupFlags { RequiredSymbol, WeaklyReferencedSymbol }; + +raw_ostream &operator<<(raw_ostream &OS, const SymbolLookupFlags &LF); + +/// A map of symbol names to resolved addresses. +using AsyncLookupResult = DenseMap<StringRef, JITEvaluatedSymbol>; + +/// A function object to call with a resolved symbol map (See AsyncLookupResult) +/// or an error if resolution failed. +class JITLinkAsyncLookupContinuation { +public: + virtual ~JITLinkAsyncLookupContinuation() = default; + virtual void run(Expected<AsyncLookupResult> LR) = 0; + +private: + virtual void anchor(); +}; + +/// Create a lookup continuation from a function object. +template <typename Continuation> +std::unique_ptr<JITLinkAsyncLookupContinuation> +createLookupContinuation(Continuation Cont) { + + class Impl final : public JITLinkAsyncLookupContinuation { + public: + Impl(Continuation C) : C(std::move(C)) {} + void run(Expected<AsyncLookupResult> LR) override { C(std::move(LR)); } + + private: + Continuation C; + }; + + return std::make_unique<Impl>(std::move(Cont)); +} + +/// Holds context for a single jitLink invocation. +class JITLinkContext { +public: + using LookupMap = DenseMap<StringRef, SymbolLookupFlags>; + + /// Create a JITLinkContext. + JITLinkContext(const JITLinkDylib *JD) : JD(JD) {} + + /// Destroy a JITLinkContext. + virtual ~JITLinkContext(); + + /// Return the JITLinkDylib that this link is targeting, if any. + const JITLinkDylib *getJITLinkDylib() const { return JD; } + + /// Return the MemoryManager to be used for this link. + virtual JITLinkMemoryManager &getMemoryManager() = 0; + + /// Notify this context that linking failed. + /// Called by JITLink if linking cannot be completed. + virtual void notifyFailed(Error Err) = 0; + + /// Called by JITLink to resolve external symbols. This method is passed a + /// lookup continutation which it must call with a result to continue the + /// linking process. + virtual void lookup(const LookupMap &Symbols, + std::unique_ptr<JITLinkAsyncLookupContinuation> LC) = 0; + + /// Called by JITLink once all defined symbols in the graph have been assigned + /// their final memory locations in the target process. At this point the + /// LinkGraph can be inspected to build a symbol table, however the block + /// content will not generally have been copied to the target location yet. + /// + /// If the client detects an error in the LinkGraph state (e.g. unexpected or + /// missing symbols) they may return an error here. The error will be + /// propagated to notifyFailed and the linker will bail out. + virtual Error notifyResolved(LinkGraph &G) = 0; + + /// Called by JITLink to notify the context that the object has been + /// finalized (i.e. emitted to memory and memory permissions set). If all of + /// this objects dependencies have also been finalized then the code is ready + /// to run. + virtual void notifyFinalized(JITLinkMemoryManager::FinalizedAlloc Alloc) = 0; + + /// Called by JITLink prior to linking to determine whether default passes for + /// the target should be added. The default implementation returns true. + /// If subclasses override this method to return false for any target then + /// they are required to fully configure the pass pipeline for that target. + virtual bool shouldAddDefaultTargetPasses(const Triple &TT) const; + + /// Returns the mark-live pass to be used for this link. If no pass is + /// returned (the default) then the target-specific linker implementation will + /// choose a conservative default (usually marking all symbols live). + /// This function is only called if shouldAddDefaultTargetPasses returns true, + /// otherwise the JITContext is responsible for adding a mark-live pass in + /// modifyPassConfig. + virtual LinkGraphPassFunction getMarkLivePass(const Triple &TT) const; + + /// Called by JITLink to modify the pass pipeline prior to linking. + /// The default version performs no modification. + virtual Error modifyPassConfig(LinkGraph &G, PassConfiguration &Config); + +private: + const JITLinkDylib *JD = nullptr; +}; + +/// Marks all symbols in a graph live. This can be used as a default, +/// conservative mark-live implementation. +Error markAllSymbolsLive(LinkGraph &G); + +/// Create an out of range error for the given edge in the given block. +Error makeTargetOutOfRangeError(const LinkGraph &G, const Block &B, + const Edge &E); + +/// Base case for edge-visitors where the visitor-list is empty. +inline void visitEdge(LinkGraph &G, Block *B, Edge &E) {} + +/// Applies the first visitor in the list to the given edge. If the visitor's +/// visitEdge method returns true then we return immediately, otherwise we +/// apply the next visitor. +template <typename VisitorT, typename... VisitorTs> +void visitEdge(LinkGraph &G, Block *B, Edge &E, VisitorT &&V, + VisitorTs &&...Vs) { + if (!V.visitEdge(G, B, E)) + visitEdge(G, B, E, std::forward<VisitorTs>(Vs)...); +} + +/// For each edge in the given graph, apply a list of visitors to the edge, +/// stopping when the first visitor's visitEdge method returns true. +/// +/// Only visits edges that were in the graph at call time: if any visitor +/// adds new edges those will not be visited. Visitors are not allowed to +/// remove edges (though they can change their kind, target, and addend). +template <typename... VisitorTs> +void visitExistingEdges(LinkGraph &G, VisitorTs &&...Vs) { + // We may add new blocks during this process, but we don't want to iterate + // over them, so build a worklist. + std::vector<Block *> Worklist(G.blocks().begin(), G.blocks().end()); + + for (auto *B : Worklist) + for (auto &E : B->edges()) + visitEdge(G, B, E, std::forward<VisitorTs>(Vs)...); +} + +/// Create a LinkGraph from the given object buffer. +/// +/// Note: The graph does not take ownership of the underlying buffer, nor copy +/// its contents. The caller is responsible for ensuring that the object buffer +/// outlives the graph. +Expected<std::unique_ptr<LinkGraph>> +createLinkGraphFromObject(MemoryBufferRef ObjectBuffer); + +/// Link the given graph. +void link(std::unique_ptr<LinkGraph> G, std::unique_ptr<JITLinkContext> Ctx); + +} // end namespace jitlink +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_JITLINK_JITLINK_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITLink/JITLinkDylib.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITLink/JITLinkDylib.h new file mode 100644 index 0000000000..b6dff48a62 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITLink/JITLinkDylib.h @@ -0,0 +1,46 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===-- JITLinkDylib.h - JITLink Dylib type ---------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Defines the JITLinkDylib API. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_JITLINK_JITLINKDYLIB_H +#define LLVM_EXECUTIONENGINE_JITLINK_JITLINKDYLIB_H + +#include <string> + +namespace llvm { +namespace jitlink { + +class JITLinkDylib { +public: + JITLinkDylib(std::string Name) : Name(std::move(Name)) {} + + /// Get the name for this JITLinkDylib. + const std::string &getName() const { return Name; } + +private: + std::string Name; +}; + +} // end namespace jitlink +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_JITLINK_JITLINKDYLIB_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h new file mode 100644 index 0000000000..1e48c46fa0 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h @@ -0,0 +1,407 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===-- JITLinkMemoryManager.h - JITLink mem manager interface --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Contains the JITLinkMemoryManager interface. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_JITLINK_JITLINKMEMORYMANAGER_H +#define LLVM_EXECUTIONENGINE_JITLINK_JITLINKMEMORYMANAGER_H + +#include "llvm/ADT/FunctionExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ExecutionEngine/JITLink/JITLinkDylib.h" +#include "llvm/ExecutionEngine/JITLink/MemoryFlags.h" +#include "llvm/ExecutionEngine/Orc/Shared/AllocationActions.h" +#include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h" +#include "llvm/Support/Allocator.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/MSVCErrorWorkarounds.h" +#include "llvm/Support/Memory.h" +#include "llvm/Support/RecyclingAllocator.h" + +#include <cstdint> +#include <future> +#include <mutex> + +namespace llvm { +namespace jitlink { + +class Block; +class LinkGraph; +class Section; + +/// Manages allocations of JIT memory. +/// +/// Instances of this class may be accessed concurrently from multiple threads +/// and their implemetations should include any necessary synchronization. +class JITLinkMemoryManager { +public: + + /// Represents a finalized allocation. + /// + /// Finalized allocations must be passed to the + /// JITLinkMemoryManager:deallocate method prior to being destroyed. + /// + /// The interpretation of the Address associated with the finalized allocation + /// is up to the memory manager implementation. Common options are using the + /// base address of the allocation, or the address of a memory management + /// object that tracks the allocation. + class FinalizedAlloc { + friend class JITLinkMemoryManager; + + static constexpr auto InvalidAddr = ~uint64_t(0); + + public: + FinalizedAlloc() = default; + explicit FinalizedAlloc(orc::ExecutorAddr A) : A(A) { + assert(A.getValue() != InvalidAddr && + "Explicitly creating an invalid allocation?"); + } + FinalizedAlloc(const FinalizedAlloc &) = delete; + FinalizedAlloc(FinalizedAlloc &&Other) : A(Other.A) { + Other.A.setValue(InvalidAddr); + } + FinalizedAlloc &operator=(const FinalizedAlloc &) = delete; + FinalizedAlloc &operator=(FinalizedAlloc &&Other) { + assert(A.getValue() == InvalidAddr && + "Cannot overwrite active finalized allocation"); + std::swap(A, Other.A); + return *this; + } + ~FinalizedAlloc() { + assert(A.getValue() == InvalidAddr && + "Finalized allocation was not deallocated"); + } + + /// FinalizedAllocs convert to false for default-constructed, and + /// true otherwise. Default-constructed allocs need not be deallocated. + explicit operator bool() const { return A.getValue() != InvalidAddr; } + + /// Returns the address associated with this finalized allocation. + /// The allocation is unmodified. + orc::ExecutorAddr getAddress() const { return A; } + + /// Returns the address associated with this finalized allocation and + /// resets this object to the default state. + /// This should only be used by allocators when deallocating memory. + orc::ExecutorAddr release() { + orc::ExecutorAddr Tmp = A; + A.setValue(InvalidAddr); + return Tmp; + } + + private: + orc::ExecutorAddr A{InvalidAddr}; + }; + + /// Represents an allocation which has not been finalized yet. + /// + /// InFlightAllocs manage both executor memory allocations and working + /// memory allocations. + /// + /// On finalization, the InFlightAlloc should transfer the content of + /// working memory into executor memory, apply memory protections, and + /// run any finalization functions. + /// + /// Working memory should be kept alive at least until one of the following + /// happens: (1) the InFlightAlloc instance is destroyed, (2) the + /// InFlightAlloc is abandoned, (3) finalized target memory is destroyed. + /// + /// If abandon is called then working memory and executor memory should both + /// be freed. + class InFlightAlloc { + public: + using OnFinalizedFunction = unique_function<void(Expected<FinalizedAlloc>)>; + using OnAbandonedFunction = unique_function<void(Error)>; + + virtual ~InFlightAlloc(); + + /// Called prior to finalization if the allocation should be abandoned. + virtual void abandon(OnAbandonedFunction OnAbandoned) = 0; + + /// Called to transfer working memory to the target and apply finalization. + virtual void finalize(OnFinalizedFunction OnFinalized) = 0; + + /// Synchronous convenience version of finalize. + Expected<FinalizedAlloc> finalize() { + std::promise<MSVCPExpected<FinalizedAlloc>> FinalizeResultP; + auto FinalizeResultF = FinalizeResultP.get_future(); + finalize([&](Expected<FinalizedAlloc> Result) { + FinalizeResultP.set_value(std::move(Result)); + }); + return FinalizeResultF.get(); + } + }; + + /// Typedef for the argument to be passed to OnAllocatedFunction. + using AllocResult = Expected<std::unique_ptr<InFlightAlloc>>; + + /// Called when allocation has been completed. + using OnAllocatedFunction = unique_function<void(AllocResult)>; + + /// Called when deallocation has completed. + using OnDeallocatedFunction = unique_function<void(Error)>; + + virtual ~JITLinkMemoryManager(); + + /// Start the allocation process. + /// + /// If the initial allocation is successful then the OnAllocated function will + /// be called with a std::unique_ptr<InFlightAlloc> value. If the assocation + /// is unsuccessful then the OnAllocated function will be called with an + /// Error. + virtual void allocate(const JITLinkDylib *JD, LinkGraph &G, + OnAllocatedFunction OnAllocated) = 0; + + /// Convenience function for blocking allocation. + AllocResult allocate(const JITLinkDylib *JD, LinkGraph &G) { + std::promise<MSVCPExpected<std::unique_ptr<InFlightAlloc>>> AllocResultP; + auto AllocResultF = AllocResultP.get_future(); + allocate(JD, G, [&](AllocResult Alloc) { + AllocResultP.set_value(std::move(Alloc)); + }); + return AllocResultF.get(); + } + + /// Deallocate a list of allocation objects. + /// + /// Dealloc actions will be run in reverse order (from the end of the vector + /// to the start). + virtual void deallocate(std::vector<FinalizedAlloc> Allocs, + OnDeallocatedFunction OnDeallocated) = 0; + + /// Convenience function for deallocation of a single alloc. + void deallocate(FinalizedAlloc Alloc, OnDeallocatedFunction OnDeallocated) { + std::vector<FinalizedAlloc> Allocs; + Allocs.push_back(std::move(Alloc)); + deallocate(std::move(Allocs), std::move(OnDeallocated)); + } + + /// Convenience function for blocking deallocation. + Error deallocate(std::vector<FinalizedAlloc> Allocs) { + std::promise<MSVCPError> DeallocResultP; + auto DeallocResultF = DeallocResultP.get_future(); + deallocate(std::move(Allocs), + [&](Error Err) { DeallocResultP.set_value(std::move(Err)); }); + return DeallocResultF.get(); + } + + /// Convenience function for blocking deallocation of a single alloc. + Error deallocate(FinalizedAlloc Alloc) { + std::vector<FinalizedAlloc> Allocs; + Allocs.push_back(std::move(Alloc)); + return deallocate(std::move(Allocs)); + } +}; + +/// BasicLayout simplifies the implementation of JITLinkMemoryManagers. +/// +/// BasicLayout groups Sections into Segments based on their memory protection +/// and deallocation policies. JITLinkMemoryManagers can construct a BasicLayout +/// from a Graph, and then assign working memory and addresses to each of the +/// Segments. These addreses will be mapped back onto the Graph blocks in +/// the apply method. +class BasicLayout { +public: + /// The Alignment, ContentSize and ZeroFillSize of each segment will be + /// pre-filled from the Graph. Clients must set the Addr and WorkingMem fields + /// prior to calling apply. + // + // FIXME: The C++98 initializer is an attempt to work around compile failures + // due to http://www.open-std.org/jtc1/sc22/wg21/docs/cwg_defects.html#1397. + // We should be able to switch this back to member initialization once that + // issue is fixed. + class Segment { + friend class BasicLayout; + + public: + Segment() + : ContentSize(0), ZeroFillSize(0), Addr(0), WorkingMem(nullptr), + NextWorkingMemOffset(0) {} + Align Alignment; + size_t ContentSize; + uint64_t ZeroFillSize; + orc::ExecutorAddr Addr; + char *WorkingMem = nullptr; + + private: + size_t NextWorkingMemOffset; + std::vector<Block *> ContentBlocks, ZeroFillBlocks; + }; + + /// A convenience class that further groups segments based on memory + /// deallocation policy. This allows clients to make two slab allocations: + /// one for all standard segments, and one for all finalize segments. + struct ContiguousPageBasedLayoutSizes { + uint64_t StandardSegs = 0; + uint64_t FinalizeSegs = 0; + + uint64_t total() const { return StandardSegs + FinalizeSegs; } + }; + +private: + using SegmentMap = AllocGroupSmallMap<Segment>; + +public: + BasicLayout(LinkGraph &G); + + /// Return a reference to the graph this allocation was created from. + LinkGraph &getGraph() { return G; } + + /// Returns the total number of required to allocate all segments (with each + /// segment padded out to page size) for all standard segments, and all + /// finalize segments. + /// + /// This is a convenience function for the common case where the segments will + /// be allocated contiguously. + /// + /// This function will return an error if any segment has an alignment that + /// is higher than a page. + Expected<ContiguousPageBasedLayoutSizes> + getContiguousPageBasedLayoutSizes(uint64_t PageSize); + + /// Returns an iterator over the segments of the layout. + iterator_range<SegmentMap::iterator> segments() { + return {Segments.begin(), Segments.end()}; + } + + /// Apply the layout to the graph. + Error apply(); + + /// Returns a reference to the AllocActions in the graph. + /// This convenience function saves callers from having to #include + /// LinkGraph.h if all they need are allocation actions. + orc::shared::AllocActions &graphAllocActions(); + +private: + LinkGraph &G; + SegmentMap Segments; +}; + +/// A utility class for making simple allocations using JITLinkMemoryManager. +/// +/// SimpleSegementAlloc takes a mapping of AllocGroups to Segments and uses +/// this to create a LinkGraph with one Section (containing one Block) per +/// Segment. Clients can obtain a pointer to the working memory and executor +/// address of that block using the Segment's AllocGroup. Once memory has been +/// populated, clients can call finalize to finalize the memory. +class SimpleSegmentAlloc { +public: + /// Describes a segment to be allocated. + struct Segment { + Segment() = default; + Segment(size_t ContentSize, Align ContentAlign) + : ContentSize(ContentSize), ContentAlign(ContentAlign) {} + + size_t ContentSize = 0; + Align ContentAlign; + }; + + /// Describes the segment working memory and executor address. + struct SegmentInfo { + orc::ExecutorAddr Addr; + MutableArrayRef<char> WorkingMem; + }; + + using SegmentMap = AllocGroupSmallMap<Segment>; + + using OnCreatedFunction = unique_function<void(Expected<SimpleSegmentAlloc>)>; + + using OnFinalizedFunction = + JITLinkMemoryManager::InFlightAlloc::OnFinalizedFunction; + + static void Create(JITLinkMemoryManager &MemMgr, const JITLinkDylib *JD, + SegmentMap Segments, OnCreatedFunction OnCreated); + + static Expected<SimpleSegmentAlloc> Create(JITLinkMemoryManager &MemMgr, + const JITLinkDylib *JD, + SegmentMap Segments); + + SimpleSegmentAlloc(SimpleSegmentAlloc &&); + SimpleSegmentAlloc &operator=(SimpleSegmentAlloc &&); + ~SimpleSegmentAlloc(); + + /// Returns the SegmentInfo for the given group. + SegmentInfo getSegInfo(AllocGroup AG); + + /// Finalize all groups (async version). + void finalize(OnFinalizedFunction OnFinalized) { + Alloc->finalize(std::move(OnFinalized)); + } + + /// Finalize all groups. + Expected<JITLinkMemoryManager::FinalizedAlloc> finalize() { + return Alloc->finalize(); + } + +private: + SimpleSegmentAlloc( + std::unique_ptr<LinkGraph> G, AllocGroupSmallMap<Block *> ContentBlocks, + std::unique_ptr<JITLinkMemoryManager::InFlightAlloc> Alloc); + + std::unique_ptr<LinkGraph> G; + AllocGroupSmallMap<Block *> ContentBlocks; + std::unique_ptr<JITLinkMemoryManager::InFlightAlloc> Alloc; +}; + +/// A JITLinkMemoryManager that allocates in-process memory. +class InProcessMemoryManager : public JITLinkMemoryManager { +public: + class IPInFlightAlloc; + + /// Attempts to auto-detect the host page size. + static Expected<std::unique_ptr<InProcessMemoryManager>> Create(); + + /// Create an instance using the given page size. + InProcessMemoryManager(uint64_t PageSize) : PageSize(PageSize) {} + + void allocate(const JITLinkDylib *JD, LinkGraph &G, + OnAllocatedFunction OnAllocated) override; + + // Use overloads from base class. + using JITLinkMemoryManager::allocate; + + void deallocate(std::vector<FinalizedAlloc> Alloc, + OnDeallocatedFunction OnDeallocated) override; + + // Use overloads from base class. + using JITLinkMemoryManager::deallocate; + +private: + // FIXME: Use an in-place array instead of a vector for DeallocActions. + // There shouldn't need to be a heap alloc for this. + struct FinalizedAllocInfo { + sys::MemoryBlock StandardSegments; + std::vector<orc::shared::WrapperFunctionCall> DeallocActions; + }; + + FinalizedAlloc createFinalizedAlloc( + sys::MemoryBlock StandardSegments, + std::vector<orc::shared::WrapperFunctionCall> DeallocActions); + + uint64_t PageSize; + std::mutex FinalizedAllocsMutex; + RecyclingAllocator<BumpPtrAllocator, FinalizedAllocInfo> FinalizedAllocInfos; +}; + +} // end namespace jitlink +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_JITLINK_JITLINKMEMORYMANAGER_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITLink/MachO.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITLink/MachO.h new file mode 100644 index 0000000000..c73d7b9111 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITLink/MachO.h @@ -0,0 +1,50 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===------- MachO.h - Generic JIT link function for MachO ------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Generic jit-link functions for MachO. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_JITLINK_MACHO_H +#define LLVM_EXECUTIONENGINE_JITLINK_MACHO_H + +#include "llvm/ExecutionEngine/JITLink/JITLink.h" + +namespace llvm { +namespace jitlink { + +/// Create a LinkGraph from a MachO relocatable object. +/// +/// Note: The graph does not take ownership of the underlying buffer, nor copy +/// its contents. The caller is responsible for ensuring that the object buffer +/// outlives the graph. +Expected<std::unique_ptr<LinkGraph>> +createLinkGraphFromMachOObject(MemoryBufferRef ObjectBuffer); + +/// jit-link the given ObjBuffer, which must be a MachO object file. +/// +/// Uses conservative defaults for GOT and stub handling based on the target +/// platform. +void link_MachO(std::unique_ptr<LinkGraph> G, + std::unique_ptr<JITLinkContext> Ctx); + +} // end namespace jitlink +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_JITLINK_MACHO_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITLink/MachO_arm64.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITLink/MachO_arm64.h new file mode 100644 index 0000000000..d930ff648b --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITLink/MachO_arm64.h @@ -0,0 +1,82 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===---- MachO_arm64.h - JIT link functions for MachO/arm64 ----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// jit-link functions for MachO/arm64. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_JITLINK_MACHO_ARM64_H +#define LLVM_EXECUTIONENGINE_JITLINK_MACHO_ARM64_H + +#include "llvm/ExecutionEngine/JITLink/JITLink.h" + +namespace llvm { +namespace jitlink { + +namespace MachO_arm64_Edges { + +enum MachOARM64RelocationKind : Edge::Kind { + Branch26 = Edge::FirstRelocation, + Pointer32, + Pointer64, + Pointer64Anon, + Page21, + PageOffset12, + GOTPage21, + GOTPageOffset12, + TLVPage21, + TLVPageOffset12, + PointerToGOT, + PairedAddend, + LDRLiteral19, + Delta32, + Delta64, + NegDelta32, + NegDelta64, +}; + +} // namespace MachO_arm64_Edges + +/// Create a LinkGraph from a MachO/arm64 relocatable object. +/// +/// Note: The graph does not take ownership of the underlying buffer, nor copy +/// its contents. The caller is responsible for ensuring that the object buffer +/// outlives the graph. +Expected<std::unique_ptr<LinkGraph>> +createLinkGraphFromMachOObject_arm64(MemoryBufferRef ObjectBuffer); + +/// jit-link the given object buffer, which must be a MachO arm64 object file. +/// +/// If PrePrunePasses is empty then a default mark-live pass will be inserted +/// that will mark all exported atoms live. If PrePrunePasses is not empty, the +/// caller is responsible for including a pass to mark atoms as live. +/// +/// If PostPrunePasses is empty then a default GOT-and-stubs insertion pass will +/// be inserted. If PostPrunePasses is not empty then the caller is responsible +/// for including a pass to insert GOT and stub edges. +void link_MachO_arm64(std::unique_ptr<LinkGraph> G, + std::unique_ptr<JITLinkContext> Ctx); + +/// Return the string name of the given MachO arm64 edge kind. +const char *getMachOARM64RelocationKindName(Edge::Kind R); + +} // end namespace jitlink +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_JITLINK_MACHO_ARM64_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITLink/MachO_x86_64.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITLink/MachO_x86_64.h new file mode 100644 index 0000000000..5d07662bd7 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITLink/MachO_x86_64.h @@ -0,0 +1,63 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===--- MachO_x86_64.h - JIT link functions for MachO/x86-64 ---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// jit-link functions for MachO/x86-64. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_JITLINK_MACHO_X86_64_H +#define LLVM_EXECUTIONENGINE_JITLINK_MACHO_X86_64_H + +#include "llvm/ExecutionEngine/JITLink/JITLink.h" + +namespace llvm { +namespace jitlink { + +/// Create a LinkGraph from a MachO/x86-64 relocatable object. +/// +/// Note: The graph does not take ownership of the underlying buffer, nor copy +/// its contents. The caller is responsible for ensuring that the object buffer +/// outlives the graph. +Expected<std::unique_ptr<LinkGraph>> +createLinkGraphFromMachOObject_x86_64(MemoryBufferRef ObjectBuffer); + +/// jit-link the given LinkGraph. +/// +/// If PrePrunePasses is empty then a default mark-live pass will be inserted +/// that will mark all exported atoms live. If PrePrunePasses is not empty, the +/// caller is responsible for including a pass to mark atoms as live. +/// +/// If PostPrunePasses is empty then a default GOT-and-stubs insertion pass will +/// be inserted. If PostPrunePasses is not empty then the caller is responsible +/// for including a pass to insert GOT and stub edges. +void link_MachO_x86_64(std::unique_ptr<LinkGraph> G, + std::unique_ptr<JITLinkContext> Ctx); + +/// Returns a pass suitable for splitting __eh_frame sections in MachO/x86-64 +/// objects. +LinkGraphPassFunction createEHFrameSplitterPass_MachO_x86_64(); + +/// Returns a pass suitable for fixing missing edges in an __eh_frame section +/// in a MachO/x86-64 object. +LinkGraphPassFunction createEHFrameEdgeFixerPass_MachO_x86_64(); + +} // end namespace jitlink +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_JITLINK_MACHO_X86_64_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITLink/MemoryFlags.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITLink/MemoryFlags.h new file mode 100644 index 0000000000..e276eb5a13 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITLink/MemoryFlags.h @@ -0,0 +1,237 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===-------- MemoryFlags.h - Memory allocation flags -----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Defines types and operations related to memory protection and allocation +// lifetimes. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_JITLINK_MEMORYFLAGS_H +#define LLVM_EXECUTIONENGINE_JITLINK_MEMORYFLAGS_H + +#include "llvm/ADT/BitmaskEnum.h" +#include "llvm/ADT/DenseMapInfo.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/Support/Memory.h" +#include "llvm/Support/raw_ostream.h" + +namespace llvm { +namespace jitlink { + +/// Describes Read/Write/Exec permissions for memory. +enum class MemProt { + None = 0, + Read = 1U << 0, + Write = 1U << 1, + Exec = 1U << 2, + LLVM_MARK_AS_BITMASK_ENUM(/* LargestValue = */ Exec) +}; + +/// Print a MemProt as an RWX triple. +raw_ostream &operator<<(raw_ostream &OS, MemProt MP); + +/// Convert a MemProt value to a corresponding sys::Memory::ProtectionFlags +/// value. +inline sys::Memory::ProtectionFlags toSysMemoryProtectionFlags(MemProt MP) { + std::underlying_type_t<sys::Memory::ProtectionFlags> PF = 0; + if ((MP & MemProt::Read) != MemProt::None) + PF |= sys::Memory::MF_READ; + if ((MP & MemProt::Write) != MemProt::None) + PF |= sys::Memory::MF_WRITE; + if ((MP & MemProt::Exec) != MemProt::None) + PF |= sys::Memory::MF_EXEC; + return static_cast<sys::Memory::ProtectionFlags>(PF); +} + +/// Convert a sys::Memory::ProtectionFlags value to a corresponding MemProt +/// value. +inline MemProt fromSysMemoryProtectionFlags(sys::Memory::ProtectionFlags PF) { + MemProt MP = MemProt::None; + if (PF & sys::Memory::MF_READ) + MP |= MemProt::Read; + if (PF & sys::Memory::MF_WRITE) + MP |= MemProt::Write; + if (PF & sys::Memory::MF_EXEC) + MP |= MemProt::None; + return MP; +} + +/// Describes a memory deallocation policy for memory to be allocated by a +/// JITLinkMemoryManager. +/// +/// All memory allocated by a call to JITLinkMemoryManager::allocate should be +/// deallocated if a call is made to +/// JITLinkMemoryManager::InFlightAllocation::abandon. The policies below apply +/// to finalized allocations. +enum class MemDeallocPolicy { + /// Standard memory should be deallocated when the deallocate method is called + /// for the finalized allocation. + Standard, + + /// Finalize memory should be overwritten and then deallocated after all + /// finalization functions have been run. + Finalize +}; + +/// Print a MemDeallocPolicy. +raw_ostream &operator<<(raw_ostream &OS, MemDeallocPolicy MDP); + +/// A pair of memory protections and allocation policies. +/// +/// Optimized for use as a small map key. +class AllocGroup { + friend struct llvm::DenseMapInfo<AllocGroup>; + + using underlying_type = uint8_t; + static constexpr unsigned BitsForProt = 3; + static constexpr unsigned BitsForDeallocPolicy = 1; + static constexpr unsigned MaxIdentifiers = + 1U << (BitsForProt + BitsForDeallocPolicy); + +public: + static constexpr unsigned NumGroups = MaxIdentifiers; + + /// Create a default AllocGroup. No memory protections, standard + /// deallocation policy. + AllocGroup() = default; + + /// Create an AllocGroup from a MemProt only -- uses + /// MemoryDeallocationPolicy::Standard. + AllocGroup(MemProt MP) : Id(static_cast<underlying_type>(MP)) {} + + /// Create an AllocGroup from a MemProt and a MemoryDeallocationPolicy. + AllocGroup(MemProt MP, MemDeallocPolicy MDP) + : Id(static_cast<underlying_type>(MP) | + (static_cast<underlying_type>(MDP) << BitsForProt)) {} + + /// Returns the MemProt for this group. + MemProt getMemProt() const { + return static_cast<MemProt>(Id & ((1U << BitsForProt) - 1)); + } + + /// Returns the MemoryDeallocationPolicy for this group. + MemDeallocPolicy getMemDeallocPolicy() const { + return static_cast<MemDeallocPolicy>(Id >> BitsForProt); + } + + friend bool operator==(const AllocGroup &LHS, const AllocGroup &RHS) { + return LHS.Id == RHS.Id; + } + + friend bool operator!=(const AllocGroup &LHS, const AllocGroup &RHS) { + return !(LHS == RHS); + } + + friend bool operator<(const AllocGroup &LHS, const AllocGroup &RHS) { + return LHS.Id < RHS.Id; + } + +private: + AllocGroup(underlying_type RawId) : Id(RawId) {} + underlying_type Id = 0; +}; + +/// A specialized small-map for AllocGroups. +/// +/// Iteration order is guaranteed to match key ordering. +template <typename T> class AllocGroupSmallMap { +private: + using ElemT = std::pair<AllocGroup, T>; + using VectorTy = SmallVector<ElemT, 4>; + + static bool compareKey(const ElemT &E, const AllocGroup &G) { + return E.first < G; + } + +public: + using iterator = typename VectorTy::iterator; + + AllocGroupSmallMap() = default; + AllocGroupSmallMap(std::initializer_list<std::pair<AllocGroup, T>> Inits) { + Elems.reserve(Inits.size()); + for (const auto &E : Inits) + Elems.push_back(E); + llvm::sort(Elems, [](const ElemT &LHS, const ElemT &RHS) { + return LHS.first < RHS.first; + }); + } + + iterator begin() { return Elems.begin(); } + iterator end() { return Elems.end(); } + iterator find(AllocGroup G) { + auto I = lower_bound(Elems, G, compareKey); + return (I->first == G) ? I : end(); + } + + bool empty() const { return Elems.empty(); } + size_t size() const { return Elems.size(); } + + T &operator[](AllocGroup G) { + auto I = lower_bound(Elems, G, compareKey); + if (I == Elems.end() || I->first != G) + I = Elems.insert(I, std::make_pair(G, T())); + return I->second; + } + +private: + VectorTy Elems; +}; + +/// Print an AllocGroup. +raw_ostream &operator<<(raw_ostream &OS, AllocGroup AG); + +} // end namespace jitlink + +template <> struct DenseMapInfo<jitlink::MemProt> { + static inline jitlink::MemProt getEmptyKey() { + return jitlink::MemProt(~uint8_t(0)); + } + static inline jitlink::MemProt getTombstoneKey() { + return jitlink::MemProt(~uint8_t(0) - 1); + } + static unsigned getHashValue(const jitlink::MemProt &Val) { + using UT = std::underlying_type_t<jitlink::MemProt>; + return DenseMapInfo<UT>::getHashValue(static_cast<UT>(Val)); + } + static bool isEqual(const jitlink::MemProt &LHS, + const jitlink::MemProt &RHS) { + return LHS == RHS; + } +}; + +template <> struct DenseMapInfo<jitlink::AllocGroup> { + static inline jitlink::AllocGroup getEmptyKey() { + return jitlink::AllocGroup(~uint8_t(0)); + } + static inline jitlink::AllocGroup getTombstoneKey() { + return jitlink::AllocGroup(~uint8_t(0) - 1); + } + static unsigned getHashValue(const jitlink::AllocGroup &Val) { + return DenseMapInfo<jitlink::AllocGroup::underlying_type>::getHashValue( + Val.Id); + } + static bool isEqual(const jitlink::AllocGroup &LHS, + const jitlink::AllocGroup &RHS) { + return LHS == RHS; + } +}; + +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_JITLINK_MEMORYFLAGS_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITLink/TableManager.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITLink/TableManager.h new file mode 100644 index 0000000000..d071ada6f0 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITLink/TableManager.h @@ -0,0 +1,74 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===---------------------- TableManager.h ----------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Fix edge for edge that needs an entry to reference the target symbol +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_JITLINK_TABLEMANAGER_H +#define LLVM_EXECUTIONENGINE_JITLINK_TABLEMANAGER_H + +#include "llvm/ExecutionEngine/JITLink/JITLink.h" +#include "llvm/Support/Debug.h" + +namespace llvm { +namespace jitlink { + +/// A CRTP base for tables that are built on demand, e.g. Global Offset Tables +/// and Procedure Linkage Tables. +/// The getEntyrForTarget function returns the table entry corresponding to the +/// given target, calling down to the implementation class to build an entry if +/// one does not already exist. +template <typename TableManagerImplT> class TableManager { +public: + /// Return the constructed entry + /// + /// Use parameter G to construct the entry for target symbol + Symbol &getEntryForTarget(LinkGraph &G, Symbol &Target) { + assert(Target.hasName() && "Edge cannot point to anonymous target"); + + auto EntryI = Entries.find(Target.getName()); + + // Build the entry if it doesn't exist. + if (EntryI == Entries.end()) { + auto &Entry = impl().createEntry(G, Target); + DEBUG_WITH_TYPE("jitlink", { + dbgs() << " Created" << impl().getSectionName() << "entry for " + << Target.getName() << ": " << Entry << "\n"; + }); + EntryI = Entries.insert(std::make_pair(Target.getName(), &Entry)).first; + } + + assert(EntryI != Entries.end() && "Could not get entry symbol"); + DEBUG_WITH_TYPE("jitlink", { + dbgs() << " Using " << impl().getSectionName() << " entry " + << *EntryI->second << "\n"; + }); + return *EntryI->second; + } + +private: + TableManagerImplT &impl() { return static_cast<TableManagerImplT &>(*this); } + DenseMap<StringRef, Symbol *> Entries; +}; + +} // namespace jitlink +} // namespace llvm + +#endif + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITLink/aarch64.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITLink/aarch64.h new file mode 100644 index 0000000000..b7b83bc588 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITLink/aarch64.h @@ -0,0 +1,49 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//=== aarch64.h - Generic JITLink aarch64 edge kinds, utilities -*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Generic utilities for graphs representing aarch64 objects. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_JITLINK_AARCH64_H +#define LLVM_EXECUTIONENGINE_JITLINK_AARCH64_H + +#include "llvm/ExecutionEngine/JITLink/JITLink.h" + +namespace llvm { +namespace jitlink { +namespace aarch64 { + +/// Represets aarch64 fixups +enum EdgeKind_aarch64 : Edge::Kind { + + /// Set a CALL immediate field to bits [27:2] of X = Target - Fixup + Addend + R_AARCH64_CALL26 = Edge::FirstRelocation, + +}; + +/// Returns a string name for the given aarch64 edge. For debugging purposes +/// only +const char *getEdgeKindName(Edge::Kind K); + +} // namespace aarch64 +} // namespace jitlink +} // namespace llvm + +#endif // LLVM_EXECUTIONENGINE_JITLINK_AARCH64_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITLink/riscv.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITLink/riscv.h new file mode 100644 index 0000000000..f103496914 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITLink/riscv.h @@ -0,0 +1,191 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===-- riscv.h - Generic JITLink riscv edge kinds, utilities -*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Generic utilities for graphs representing riscv objects. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_JITLINK_RISCV_H +#define LLVM_EXECUTIONENGINE_JITLINK_RISCV_H + +#include "llvm/ExecutionEngine/JITLink/JITLink.h" + +namespace llvm { +namespace jitlink { +namespace riscv { + +/// Represets riscv fixups +enum EdgeKind_riscv : Edge::Kind { + + // TODO: Capture and replace to generic fixups + /// A plain 32-bit pointer value relocation + /// + /// Fixup expression: + /// Fixup <= Target + Addend : uint32 + /// + R_RISCV_32 = Edge::FirstRelocation, + + /// A plain 64-bit pointer value relocation + /// + /// Fixup expression: + /// Fixup <- Target + Addend : uint32 + /// + R_RISCV_64, + + /// Low 12 bits of PC-relative branch pointer value relocation + /// + /// Fixup expression: + /// Fixup <- (Target - Fixup + Addend) & 0xFFF + /// + R_RISCV_BRANCH, + + /// High 20 bits of 32-bit pointer value relocation + /// + /// Fixup expression + /// Fixup <- (Target + Addend + 0x800) >> 12 + R_RISCV_HI20, + + /// Low 12 bits of 32-bit pointer value relocation + /// + /// Fixup expression + /// Fixup <- (Target + Addend) & 0xFFF + R_RISCV_LO12_I, + /// High 20 bits of PC relative relocation + /// + /// Fixup expression: + /// Fixup <- (Target - Fixup + Addend + 0x800) >> 12 + R_RISCV_PCREL_HI20, + + /// Low 12 bits of PC relative relocation, used by I type instruction format + /// + /// Fixup expression: + /// Fixup <- (Target - Fixup + Addend) & 0xFFF + R_RISCV_PCREL_LO12_I, + + /// Low 12 bits of PC relative relocation, used by S type instruction format + /// + /// Fixup expression: + /// Fixup <- (Target - Fixup + Addend) & 0xFFF + R_RISCV_PCREL_LO12_S, + + /// PC relative call + /// + /// Fixup expression: + /// Fixup <- (Target - Fixup + Addend) + R_RISCV_CALL, + + /// 32 bits PC relative relocation + /// + /// Fixup expression: + /// Fixup <- (Target - Fixup + Addend) + R_RISCV_32_PCREL, + + /// PC relative GOT offset + /// + /// Fixup expression: + /// Fixup <- (GOT - Fixup + Addend) >> 12 + R_RISCV_GOT_HI20, + + /// PC relative call by PLT + /// + /// Fixup expression: + /// Fixup <- (Target - Fixup + Addend) + R_RISCV_CALL_PLT, + + /// 64 bits label addition + /// + /// Fixup expression: + /// Fixup <- (Target - *{8}Fixup + Addend) + R_RISCV_ADD64, + + /// 32 bits label addition + /// + /// Fixup expression: + /// Fixup <- (Target - *{4}Fixup + Addend) + R_RISCV_ADD32, + + /// 16 bits label addition + /// + /// Fixup expression + /// Fixup <- (Target - *{2}Fixup + Addend) + R_RISCV_ADD16, + + /// 8 bits label addition + /// + /// Fixup expression + /// Fixup <- (Target - *{1}Fixup + Addend) + R_RISCV_ADD8, + + /// 64 bits label subtraction + /// + /// Fixup expression + /// Fixup <- (Target - *{8}Fixup - Addend) + R_RISCV_SUB64, + + /// 32 bits label subtraction + /// + /// Fixup expression + /// Fixup <- (Target - *{4}Fixup - Addend) + R_RISCV_SUB32, + + /// 16 bits label subtraction + /// + /// Fixup expression + /// Fixup <- (Target - *{2}Fixup - Addend) + R_RISCV_SUB16, + + /// 8 bits label subtraction + /// + /// Fixup expression + /// Fixup <- (Target - *{1}Fixup - Addend) + R_RISCV_SUB8, + + /// Local label assignment + /// + /// Fixup expression: + /// Fixup <- (Target + Addend) + R_RISCV_SET6, + + /// Local label assignment + /// + /// Fixup expression: + /// Fixup <- (Target + Addend) + R_RISCV_SET8, + + /// Local label assignment + /// + /// Fixup expression: + /// Fixup <- (Target + Addend) + R_RISCV_SET16, + + /// Local label assignment + /// + /// Fixup expression: + /// Fixup <- (Target + Addend) + R_RISCV_SET32, +}; + +/// Returns a string name for the given riscv edge. For debugging purposes +/// only +const char *getEdgeKindName(Edge::Kind K); +} // namespace riscv +} // namespace jitlink +} // namespace llvm + +#endif + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITLink/x86_64.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITLink/x86_64.h new file mode 100644 index 0000000000..64550e20ad --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITLink/x86_64.h @@ -0,0 +1,640 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===-- x86_64.h - Generic JITLink x86-64 edge kinds, utilities -*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Generic utilities for graphs representing x86-64 objects. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_JITLINK_X86_64_H +#define LLVM_EXECUTIONENGINE_JITLINK_X86_64_H + +#include "llvm/ExecutionEngine/JITLink/JITLink.h" +#include "llvm/ExecutionEngine/JITLink/TableManager.h" + +#include <limits> + +namespace llvm { +namespace jitlink { +namespace x86_64 { + +/// Represents x86-64 fixups and other x86-64-specific edge kinds. +enum EdgeKind_x86_64 : Edge::Kind { + + /// A plain 64-bit pointer value relocation. + /// + /// Fixup expression: + /// Fixup <- Target + Addend : uint64 + /// + Pointer64 = Edge::FirstRelocation, + + /// A plain 32-bit pointer value relocation. + /// + /// Fixup expression: + /// Fixup <- Target + Addend : uint32 + /// + /// Errors: + /// - The target must reside in the low 32-bits of the address space, + /// otherwise an out-of-range error will be returned. + /// + Pointer32, + + /// A signed 32-bit pointer value relocation + /// + /// Fixup expression: + /// Fixup <- Target + Addend : int32 + /// + /// Errors: + /// - The target must reside in the signed 32-bits([-2**31, 2**32 - 1]) of + /// the address space, otherwise an out-of-range error will be returned. + Pointer32Signed, + + /// A 64-bit delta. + /// + /// Delta from the fixup to the target. + /// + /// Fixup expression: + /// Fixup <- Target - Fixup + Addend : int64 + /// + Delta64, + + /// A 32-bit delta. + /// + /// Delta from the fixup to the target. + /// + /// Fixup expression: + /// Fixup <- Target - Fixup + Addend : int64 + /// + /// Errors: + /// - The result of the fixup expression must fit into an int32, otherwise + /// an out-of-range error will be returned. + /// + Delta32, + + /// A 64-bit negative delta. + /// + /// Delta from target back to the fixup. + /// + /// Fixup expression: + /// Fixup <- Fixup - Target + Addend : int64 + /// + NegDelta64, + + /// A 32-bit negative delta. + /// + /// Delta from the target back to the fixup. + /// + /// Fixup expression: + /// Fixup <- Fixup - Target + Addend : int32 + /// + /// Errors: + /// - The result of the fixup expression must fit into an int32, otherwise + /// an out-of-range error will be returned. + NegDelta32, + + /// A 64-bit GOT delta. + /// + /// Delta from the global offset table to the target + /// + /// Fixup expression: + /// Fixup <- Target - GOTSymbol + Addend : int64 + /// + /// Errors: + /// - *ASSERTION* Failure to a null pointer GOTSymbol, which the GOT section + /// symbol was not been defined. + Delta64FromGOT, + + /// A 32-bit PC-relative branch. + /// + /// Represents a PC-relative call or branch to a target. This can be used to + /// identify, record, and/or patch call sites. + /// + /// The fixup expression for this kind includes an implicit offset to account + /// for the PC (unlike the Delta edges) so that a Branch32PCRel with a target + /// T and addend zero is a call/branch to the start (offset zero) of T. + /// + /// Fixup expression: + /// Fixup <- Target - (Fixup + 4) + Addend : int32 + /// + /// Errors: + /// - The result of the fixup expression must fit into an int32, otherwise + /// an out-of-range error will be returned. + /// + BranchPCRel32, + + /// A 32-bit PC-relative branch to a pointer jump stub. + /// + /// The target of this relocation should be a pointer jump stub of the form: + /// + /// \code{.s} + /// .text + /// jmpq *tgtptr(%rip) + /// ; ... + /// + /// .data + /// tgtptr: + /// .quad 0 + /// \endcode + /// + /// This edge kind has the same fixup expression as BranchPCRel32, but further + /// identifies the call/branch as being to a pointer jump stub. For edges of + /// this kind the jump stub should not be bypassed (use + /// BranchPCRel32ToPtrJumpStubBypassable for that), but the pointer location + /// target may be recorded to allow manipulation at runtime. + /// + /// Fixup expression: + /// Fixup <- Target - Fixup + Addend - 4 : int32 + /// + /// Errors: + /// - The result of the fixup expression must fit into an int32, otherwise + /// an out-of-range error will be returned. + /// + BranchPCRel32ToPtrJumpStub, + + /// A relaxable version of BranchPCRel32ToPtrJumpStub. + /// + /// The edge kind has the same fixup expression as BranchPCRel32ToPtrJumpStub, + /// but identifies the call/branch as being to a pointer jump stub that may be + /// bypassed with a direct jump to the ultimate target if the ultimate target + /// is within range of the fixup location. + /// + /// Fixup expression: + /// Fixup <- Target - Fixup + Addend - 4: int32 + /// + /// Errors: + /// - The result of the fixup expression must fit into an int32, otherwise + /// an out-of-range error will be returned. + /// + BranchPCRel32ToPtrJumpStubBypassable, + + /// A GOT entry getter/constructor, transformed to Delta32 pointing at the GOT + /// entry for the original target. + /// + /// Indicates that this edge should be transformed into a Delta32 targeting + /// the GOT entry for the edge's current target, maintaining the same addend. + /// A GOT entry for the target should be created if one does not already + /// exist. + /// + /// Edges of this kind are usually handled by a GOT builder pass inserted by + /// default. + /// + /// Fixup expression: + /// NONE + /// + /// Errors: + /// - *ASSERTION* Failure to handle edges of this kind prior to the fixup + /// phase will result in an assert/unreachable during the fixup phase. + /// + RequestGOTAndTransformToDelta32, + + /// A GOT entry getter/constructor, transformed to Delta64 pointing at the GOT + /// entry for the original target. + /// + /// Indicates that this edge should be transformed into a Delta64 targeting + /// the GOT entry for the edge's current target, maintaining the same addend. + /// A GOT entry for the target should be created if one does not already + /// exist. + /// + /// Edges of this kind are usually handled by a GOT builder pass inserted by + /// default. + /// + /// Fixup expression: + /// NONE + /// + /// Errors: + /// - *ASSERTION* Failure to handle edges of this kind prior to the fixup + /// phase will result in an assert/unreachable during the fixup phase. + /// + RequestGOTAndTransformToDelta64, + + /// A GOT entry offset within GOT getter/constructor, transformed to + /// Delta64FromGOT + /// pointing at the GOT entry for the original target + /// + /// Indicates that this edge should be transformed into a Delta64FromGOT + /// targeting + /// the GOT entry for the edge's current target, maintaining the same addend. + /// A GOT entry for the target should be created if one does not already + /// exist. + /// + /// Edges of this kind are usually handled by a GOT builder pass inserted by + /// default + /// + /// Fixup expression: + /// NONE + /// + /// Errors: + /// - *ASSERTION* Failure to handle edges of this kind prior to the fixup + /// phase will result in an assert/unreachable during the fixup phase + RequestGOTAndTransformToDelta64FromGOT, + + /// A PC-relative load of a GOT entry, relaxable if GOT entry target is + /// in-range of the fixup + /// + /// TODO: Explain the optimization + /// + /// Fixup expression + /// Fixup <- Target - (Fixup + 4) + Addend : int32 + /// + /// Errors: + /// - The result of the fixup expression must fit into an int32, otherwise + /// an out-of-range error will be returned. + // + PCRel32GOTLoadRelaxable, + + /// A PC-relative REX load of a GOT entry, relaxable if GOT entry target + /// is in-range of the fixup. + /// + /// If the GOT entry target is in-range of the fixup then the load from the + /// GOT may be replaced with a direct memory address calculation. + /// + /// Fixup expression: + /// Fixup <- Target - (Fixup + 4) + Addend : int32 + /// + /// Errors: + /// - The result of the fixup expression must fit into an int32, otherwise + /// an out-of-range error will be returned. + /// + PCRel32GOTLoadREXRelaxable, + + /// A GOT entry getter/constructor, transformed to + /// PCRel32ToGOTLoadREXRelaxable pointing at the GOT entry for the original + /// target. + /// + /// Indicates that this edge should be lowered to a PC32ToGOTLoadREXRelaxable + /// targeting the GOT entry for the edge's current target, maintaining the + /// same addend. A GOT entry for the target should be created if one does not + /// already exist. + /// + /// Edges of this kind are usually lowered by a GOT builder pass inserted by + /// default. + /// + /// Fixup expression: + /// NONE + /// + /// Errors: + /// - *ASSERTION* Failure to handle edges of this kind prior to the fixup + /// phase will result in an assert/unreachable during the fixup phase. + /// + RequestGOTAndTransformToPCRel32GOTLoadREXRelaxable, + + /// A GOT entry getter/constructor, transformed to + /// PCRel32ToGOTLoadRelaxable pointing at the GOT entry for the original + /// target. + /// + /// Indicates that this edge should be lowered to a PC32ToGOTLoadRelaxable + /// targeting the GOT entry for the edge's current target, maintaining the + /// same addend. A GOT entry for the target should be created if one does not + /// already exist. + /// + /// Edges of this kind are usually lowered by a GOT builder pass inserted by + /// default. + /// + /// Fixup expression: + /// NONE + /// + /// Errors: + /// - *ASSERTION* Failure to handle edges of this kind prior to the fixup + /// phase will result in an assert/unreachable during the fixup phase. + /// + RequestGOTAndTransformToPCRel32GOTLoadRelaxable, + + /// A PC-relative REX load of a Thread Local Variable Pointer (TLVP) entry, + /// relaxable if the TLVP entry target is in-range of the fixup. + /// + /// If the TLVP entry target is in-range of the fixup then the load from the + /// TLVP may be replaced with a direct memory address calculation. + /// + /// The target of this edge must be a thread local variable entry of the form + /// .quad <tlv getter thunk> + /// .quad <tlv key> + /// .quad <tlv initializer> + /// + /// Fixup expression: + /// Fixup <- Target - (Fixup + 4) + Addend : int32 + /// + /// Errors: + /// - The result of the fixup expression must fit into an int32, otherwise + /// an out-of-range error will be returned. + /// - The target must be either external, or a TLV entry of the required + /// form, otherwise a malformed TLV entry error will be returned. + /// + PCRel32TLVPLoadREXRelaxable, + + /// TODO: Explain the generic edge kind + RequestTLSDescInGOTAndTransformToDelta32, + + /// A TLVP entry getter/constructor, transformed to + /// Delta32ToTLVPLoadREXRelaxable. + /// + /// Indicates that this edge should be transformed into a + /// Delta32ToTLVPLoadREXRelaxable targeting the TLVP entry for the edge's + /// current target. A TLVP entry for the target should be created if one does + /// not already exist. + /// + /// Fixup expression: + /// NONE + /// + /// Errors: + /// - *ASSERTION* Failure to handle edges of this kind prior to the fixup + /// phase will result in an assert/unreachable during the fixup phase. + /// + RequestTLVPAndTransformToPCRel32TLVPLoadREXRelaxable +}; + +/// Returns a string name for the given x86-64 edge. For debugging purposes +/// only. +const char *getEdgeKindName(Edge::Kind K); + +/// Returns true if the given uint64_t value is in range for a uint32_t. +inline bool isInRangeForImmU32(uint64_t Value) { + return Value <= std::numeric_limits<uint32_t>::max(); +} + +/// Returns true if the given int64_t value is in range for an int32_t. +inline bool isInRangeForImmS32(int64_t Value) { + return (Value >= std::numeric_limits<int32_t>::min() && + Value <= std::numeric_limits<int32_t>::max()); +} + +/// Apply fixup expression for edge to block content. +inline Error applyFixup(LinkGraph &G, Block &B, const Edge &E, + const Symbol *GOTSymbol) { + using namespace support; + + char *BlockWorkingMem = B.getAlreadyMutableContent().data(); + char *FixupPtr = BlockWorkingMem + E.getOffset(); + auto FixupAddress = B.getAddress() + E.getOffset(); + + switch (E.getKind()) { + + case Pointer64: { + uint64_t Value = E.getTarget().getAddress().getValue() + E.getAddend(); + *(ulittle64_t *)FixupPtr = Value; + break; + } + + case Pointer32: { + uint64_t Value = E.getTarget().getAddress().getValue() + E.getAddend(); + if (LLVM_LIKELY(isInRangeForImmU32(Value))) + *(ulittle32_t *)FixupPtr = Value; + else + return makeTargetOutOfRangeError(G, B, E); + break; + } + case Pointer32Signed: { + int64_t Value = E.getTarget().getAddress().getValue() + E.getAddend(); + if (LLVM_LIKELY(isInRangeForImmS32(Value))) + *(little32_t *)FixupPtr = Value; + else + return makeTargetOutOfRangeError(G, B, E); + break; + } + + case BranchPCRel32: + case BranchPCRel32ToPtrJumpStub: + case BranchPCRel32ToPtrJumpStubBypassable: + case PCRel32GOTLoadRelaxable: + case PCRel32GOTLoadREXRelaxable: + case PCRel32TLVPLoadREXRelaxable: { + int64_t Value = + E.getTarget().getAddress() - (FixupAddress + 4) + E.getAddend(); + if (LLVM_LIKELY(isInRangeForImmS32(Value))) + *(little32_t *)FixupPtr = Value; + else + return makeTargetOutOfRangeError(G, B, E); + break; + } + + case Delta64: { + int64_t Value = E.getTarget().getAddress() - FixupAddress + E.getAddend(); + *(little64_t *)FixupPtr = Value; + break; + } + + case Delta32: { + int64_t Value = E.getTarget().getAddress() - FixupAddress + E.getAddend(); + if (LLVM_LIKELY(isInRangeForImmS32(Value))) + *(little32_t *)FixupPtr = Value; + else + return makeTargetOutOfRangeError(G, B, E); + break; + } + + case NegDelta64: { + int64_t Value = FixupAddress - E.getTarget().getAddress() + E.getAddend(); + *(little64_t *)FixupPtr = Value; + break; + } + + case NegDelta32: { + int64_t Value = FixupAddress - E.getTarget().getAddress() + E.getAddend(); + if (LLVM_LIKELY(isInRangeForImmS32(Value))) + *(little32_t *)FixupPtr = Value; + else + return makeTargetOutOfRangeError(G, B, E); + break; + } + case Delta64FromGOT: { + assert(GOTSymbol && "No GOT section symbol"); + int64_t Value = + E.getTarget().getAddress() - GOTSymbol->getAddress() + E.getAddend(); + *(little64_t *)FixupPtr = Value; + break; + } + + default: { + // If you hit this you should check that *constructor and other non-fixup + // edges have been removed prior to applying fixups. + llvm_unreachable("Graph contains edge kind with no fixup expression"); + } + } + + return Error::success(); +} + +/// x86_64 pointer size. +constexpr uint64_t PointerSize = 8; + +/// x86-64 null pointer content. +extern const char NullPointerContent[PointerSize]; + +/// x86-64 pointer jump stub content. +/// +/// Contains the instruction sequence for an indirect jump via an in-memory +/// pointer: +/// jmpq *ptr(%rip) +extern const char PointerJumpStubContent[6]; + +/// Creates a new pointer block in the given section and returns an anonymous +/// symbol pointing to it. +/// +/// If InitialTarget is given then an Pointer64 relocation will be added to the +/// block pointing at InitialTarget. +/// +/// The pointer block will have the following default values: +/// alignment: 64-bit +/// alignment-offset: 0 +/// address: highest allowable (~7U) +inline Symbol &createAnonymousPointer(LinkGraph &G, Section &PointerSection, + Symbol *InitialTarget = nullptr, + uint64_t InitialAddend = 0) { + auto &B = G.createContentBlock(PointerSection, NullPointerContent, + orc::ExecutorAddr(~uint64_t(7)), 8, 0); + if (InitialTarget) + B.addEdge(Pointer64, 0, *InitialTarget, InitialAddend); + return G.addAnonymousSymbol(B, 0, 8, false, false); +} + +/// Create a jump stub block that jumps via the pointer at the given symbol. +/// +/// The stub block will have the following default values: +/// alignment: 8-bit +/// alignment-offset: 0 +/// address: highest allowable: (~5U) +inline Block &createPointerJumpStubBlock(LinkGraph &G, Section &StubSection, + Symbol &PointerSymbol) { + auto &B = G.createContentBlock(StubSection, PointerJumpStubContent, + orc::ExecutorAddr(~uint64_t(5)), 1, 0); + B.addEdge(Delta32, 2, PointerSymbol, -4); + return B; +} + +/// Create a jump stub that jumps via the pointer at the given symbol and +/// an anonymous symbol pointing to it. Return the anonymous symbol. +/// +/// The stub block will be created by createPointerJumpStubBlock. +inline Symbol &createAnonymousPointerJumpStub(LinkGraph &G, + Section &StubSection, + Symbol &PointerSymbol) { + return G.addAnonymousSymbol( + createPointerJumpStubBlock(G, StubSection, PointerSymbol), 0, 6, true, + false); +} + +/// Global Offset Table Builder. +class GOTTableManager : public TableManager<GOTTableManager> { +public: + static StringRef getSectionName() { return "$__GOT"; } + + bool visitEdge(LinkGraph &G, Block *B, Edge &E) { + Edge::Kind KindToSet = Edge::Invalid; + switch (E.getKind()) { + case x86_64::Delta64FromGOT: { + // we need to make sure that the GOT section exists, but don't otherwise + // need to fix up this edge + getGOTSection(G); + return false; + } + case x86_64::RequestGOTAndTransformToPCRel32GOTLoadREXRelaxable: + KindToSet = x86_64::PCRel32GOTLoadREXRelaxable; + break; + case x86_64::RequestGOTAndTransformToPCRel32GOTLoadRelaxable: + KindToSet = x86_64::PCRel32GOTLoadRelaxable; + break; + case x86_64::RequestGOTAndTransformToDelta64: + KindToSet = x86_64::Delta64; + break; + case x86_64::RequestGOTAndTransformToDelta64FromGOT: + KindToSet = x86_64::Delta64FromGOT; + break; + case x86_64::RequestGOTAndTransformToDelta32: + KindToSet = x86_64::Delta32; + break; + default: + return false; + } + assert(KindToSet != Edge::Invalid && + "Fell through switch, but no new kind to set"); + DEBUG_WITH_TYPE("jitlink", { + dbgs() << " Fixing " << G.getEdgeKindName(E.getKind()) << " edge at " + << B->getFixupAddress(E) << " (" << B->getAddress() << " + " + << formatv("{0:x}", E.getOffset()) << ")\n"; + }); + E.setKind(KindToSet); + E.setTarget(getEntryForTarget(G, E.getTarget())); + return true; + } + + Symbol &createEntry(LinkGraph &G, Symbol &Target) { + return createAnonymousPointer(G, getGOTSection(G), &Target); + } + +private: + Section &getGOTSection(LinkGraph &G) { + if (!GOTSection) + GOTSection = &G.createSection(getSectionName(), MemProt::Read); + return *GOTSection; + } + + Section *GOTSection = nullptr; +}; + +/// Procedure Linkage Table Builder. +class PLTTableManager : public TableManager<PLTTableManager> { +public: + PLTTableManager(GOTTableManager &GOT) : GOT(GOT) {} + + static StringRef getSectionName() { return "$__STUBS"; } + + bool visitEdge(LinkGraph &G, Block *B, Edge &E) { + if (E.getKind() == x86_64::BranchPCRel32 && !E.getTarget().isDefined()) { + DEBUG_WITH_TYPE("jitlink", { + dbgs() << " Fixing " << G.getEdgeKindName(E.getKind()) << " edge at " + << B->getFixupAddress(E) << " (" << B->getAddress() << " + " + << formatv("{0:x}", E.getOffset()) << ")\n"; + }); + // Set the edge kind to Branch32ToPtrJumpStubBypassable to enable it to + // be optimized when the target is in-range. + E.setKind(x86_64::BranchPCRel32ToPtrJumpStubBypassable); + E.setTarget(getEntryForTarget(G, E.getTarget())); + return true; + } + return false; + } + + Symbol &createEntry(LinkGraph &G, Symbol &Target) { + return createAnonymousPointerJumpStub(G, getStubsSection(G), + GOT.getEntryForTarget(G, Target)); + } + +public: + Section &getStubsSection(LinkGraph &G) { + if (!PLTSection) + PLTSection = + &G.createSection(getSectionName(), MemProt::Read | MemProt::Exec); + return *PLTSection; + } + + GOTTableManager &GOT; + Section *PLTSection = nullptr; +}; + +/// Optimize the GOT and Stub relocations if the edge target address is in range +/// 1. PCRel32GOTLoadRelaxable. For this edge kind, if the target is in range, +/// then replace GOT load with lea +/// 2. BranchPCRel32ToPtrJumpStubRelaxable. For this edge kind, if the target is +/// in range, replace a indirect jump by plt stub with a direct jump to the +/// target +Error optimizeGOTAndStubAccesses(LinkGraph &G); + +} // namespace x86_64 +} // end namespace jitlink +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_JITLINK_X86_64_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITSymbol.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITSymbol.h new file mode 100644 index 0000000000..43e2fabc3a --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/JITSymbol.h @@ -0,0 +1,451 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===- JITSymbol.h - JIT symbol abstraction ---------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Abstraction for target process addresses. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_JITSYMBOL_H +#define LLVM_EXECUTIONENGINE_JITSYMBOL_H + +#include <algorithm> +#include <cassert> +#include <cstddef> +#include <cstdint> +#include <functional> +#include <map> +#include <set> +#include <string> + +#include "llvm/ADT/BitmaskEnum.h" +#include "llvm/ADT/FunctionExtras.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Error.h" + +namespace llvm { + +class GlobalValue; +class GlobalValueSummary; + +namespace object { + +class SymbolRef; + +} // end namespace object + +/// Represents an address in the target process's address space. +using JITTargetAddress = uint64_t; + +/// Convert a JITTargetAddress to a pointer. +/// +/// Note: This is a raw cast of the address bit pattern to the given pointer +/// type. When casting to a function pointer in order to execute JIT'd code +/// jitTargetAddressToFunction should be preferred, as it will also perform +/// pointer signing on targets that require it. +template <typename T> T jitTargetAddressToPointer(JITTargetAddress Addr) { + static_assert(std::is_pointer<T>::value, "T must be a pointer type"); + uintptr_t IntPtr = static_cast<uintptr_t>(Addr); + assert(IntPtr == Addr && "JITTargetAddress value out of range for uintptr_t"); + return reinterpret_cast<T>(IntPtr); +} + +/// Convert a JITTargetAddress to a callable function pointer. +/// +/// Casts the given address to a callable function pointer. This operation +/// will perform pointer signing for platforms that require it (e.g. arm64e). +template <typename T> T jitTargetAddressToFunction(JITTargetAddress Addr) { + static_assert(std::is_pointer<T>::value && + std::is_function<std::remove_pointer_t<T>>::value, + "T must be a function pointer type"); + return jitTargetAddressToPointer<T>(Addr); +} + +/// Convert a pointer to a JITTargetAddress. +template <typename T> JITTargetAddress pointerToJITTargetAddress(T *Ptr) { + return static_cast<JITTargetAddress>(reinterpret_cast<uintptr_t>(Ptr)); +} + +/// Flags for symbols in the JIT. +class JITSymbolFlags { +public: + using UnderlyingType = uint8_t; + using TargetFlagsType = uint8_t; + + enum FlagNames : UnderlyingType { + None = 0, + HasError = 1U << 0, + Weak = 1U << 1, + Common = 1U << 2, + Absolute = 1U << 3, + Exported = 1U << 4, + Callable = 1U << 5, + MaterializationSideEffectsOnly = 1U << 6, + LLVM_MARK_AS_BITMASK_ENUM( // LargestValue = + MaterializationSideEffectsOnly) + }; + + /// Default-construct a JITSymbolFlags instance. + JITSymbolFlags() = default; + + /// Construct a JITSymbolFlags instance from the given flags. + JITSymbolFlags(FlagNames Flags) : Flags(Flags) {} + + /// Construct a JITSymbolFlags instance from the given flags and target + /// flags. + JITSymbolFlags(FlagNames Flags, TargetFlagsType TargetFlags) + : TargetFlags(TargetFlags), Flags(Flags) {} + + /// Implicitly convert to bool. Returs true if any flag is set. + explicit operator bool() const { return Flags != None || TargetFlags != 0; } + + /// Compare for equality. + bool operator==(const JITSymbolFlags &RHS) const { + return Flags == RHS.Flags && TargetFlags == RHS.TargetFlags; + } + + /// Bitwise AND-assignment for FlagNames. + JITSymbolFlags &operator&=(const FlagNames &RHS) { + Flags &= RHS; + return *this; + } + + /// Bitwise OR-assignment for FlagNames. + JITSymbolFlags &operator|=(const FlagNames &RHS) { + Flags |= RHS; + return *this; + } + + /// Return true if there was an error retrieving this symbol. + bool hasError() const { + return (Flags & HasError) == HasError; + } + + /// Returns true if the Weak flag is set. + bool isWeak() const { + return (Flags & Weak) == Weak; + } + + /// Returns true if the Common flag is set. + bool isCommon() const { + return (Flags & Common) == Common; + } + + /// Returns true if the symbol isn't weak or common. + bool isStrong() const { + return !isWeak() && !isCommon(); + } + + /// Returns true if the Exported flag is set. + bool isExported() const { + return (Flags & Exported) == Exported; + } + + /// Returns true if the given symbol is known to be callable. + bool isCallable() const { return (Flags & Callable) == Callable; } + + /// Returns true if this symbol is a materialization-side-effects-only + /// symbol. Such symbols do not have a real address. They exist to trigger + /// and support synchronization of materialization side effects, e.g. for + /// collecting initialization information. These symbols will vanish from + /// the symbol table immediately upon reaching the ready state, and will + /// appear to queries as if they were never defined (except that query + /// callback execution will be delayed until they reach the ready state). + /// MaterializationSideEffectOnly symbols should only be queried using the + /// SymbolLookupFlags::WeaklyReferencedSymbol flag (see + /// llvm/include/llvm/ExecutionEngine/Orc/Core.h). + bool hasMaterializationSideEffectsOnly() const { + return (Flags & MaterializationSideEffectsOnly) == + MaterializationSideEffectsOnly; + } + + /// Get the underlying flags value as an integer. + UnderlyingType getRawFlagsValue() const { + return static_cast<UnderlyingType>(Flags); + } + + /// Return a reference to the target-specific flags. + TargetFlagsType& getTargetFlags() { return TargetFlags; } + + /// Return a reference to the target-specific flags. + const TargetFlagsType& getTargetFlags() const { return TargetFlags; } + + /// Construct a JITSymbolFlags value based on the flags of the given global + /// value. + static JITSymbolFlags fromGlobalValue(const GlobalValue &GV); + + /// Construct a JITSymbolFlags value based on the flags of the given global + /// value summary. + static JITSymbolFlags fromSummary(GlobalValueSummary *S); + + /// Construct a JITSymbolFlags value based on the flags of the given libobject + /// symbol. + static Expected<JITSymbolFlags> + fromObjectSymbol(const object::SymbolRef &Symbol); + +private: + TargetFlagsType TargetFlags = 0; + FlagNames Flags = None; +}; + +inline JITSymbolFlags operator&(const JITSymbolFlags &LHS, + const JITSymbolFlags::FlagNames &RHS) { + JITSymbolFlags Tmp = LHS; + Tmp &= RHS; + return Tmp; +} + +inline JITSymbolFlags operator|(const JITSymbolFlags &LHS, + const JITSymbolFlags::FlagNames &RHS) { + JITSymbolFlags Tmp = LHS; + Tmp |= RHS; + return Tmp; +} + +/// ARM-specific JIT symbol flags. +/// FIXME: This should be moved into a target-specific header. +class ARMJITSymbolFlags { +public: + ARMJITSymbolFlags() = default; + + enum FlagNames { + None = 0, + Thumb = 1 << 0 + }; + + operator JITSymbolFlags::TargetFlagsType&() { return Flags; } + + static ARMJITSymbolFlags fromObjectSymbol(const object::SymbolRef &Symbol); + +private: + JITSymbolFlags::TargetFlagsType Flags = 0; +}; + +/// Represents a symbol that has been evaluated to an address already. +class JITEvaluatedSymbol { +public: + JITEvaluatedSymbol() = default; + + /// Create a 'null' symbol. + JITEvaluatedSymbol(std::nullptr_t) {} + + /// Create a symbol for the given address and flags. + JITEvaluatedSymbol(JITTargetAddress Address, JITSymbolFlags Flags) + : Address(Address), Flags(Flags) {} + + /// Create a symbol from the given pointer with the given flags. + template <typename T> + static JITEvaluatedSymbol + fromPointer(T *P, JITSymbolFlags Flags = JITSymbolFlags::Exported) { + return JITEvaluatedSymbol(pointerToJITTargetAddress(P), Flags); + } + + /// An evaluated symbol converts to 'true' if its address is non-zero. + explicit operator bool() const { return Address != 0; } + + /// Return the address of this symbol. + JITTargetAddress getAddress() const { return Address; } + + /// Return the flags for this symbol. + JITSymbolFlags getFlags() const { return Flags; } + + /// Set the flags for this symbol. + void setFlags(JITSymbolFlags Flags) { this->Flags = std::move(Flags); } + +private: + JITTargetAddress Address = 0; + JITSymbolFlags Flags; +}; + +/// Represents a symbol in the JIT. +class JITSymbol { +public: + using GetAddressFtor = unique_function<Expected<JITTargetAddress>()>; + + /// Create a 'null' symbol, used to represent a "symbol not found" + /// result from a successful (non-erroneous) lookup. + JITSymbol(std::nullptr_t) + : CachedAddr(0) {} + + /// Create a JITSymbol representing an error in the symbol lookup + /// process (e.g. a network failure during a remote lookup). + JITSymbol(Error Err) + : Err(std::move(Err)), Flags(JITSymbolFlags::HasError) {} + + /// Create a symbol for a definition with a known address. + JITSymbol(JITTargetAddress Addr, JITSymbolFlags Flags) + : CachedAddr(Addr), Flags(Flags) {} + + /// Construct a JITSymbol from a JITEvaluatedSymbol. + JITSymbol(JITEvaluatedSymbol Sym) + : CachedAddr(Sym.getAddress()), Flags(Sym.getFlags()) {} + + /// Create a symbol for a definition that doesn't have a known address + /// yet. + /// @param GetAddress A functor to materialize a definition (fixing the + /// address) on demand. + /// + /// This constructor allows a JIT layer to provide a reference to a symbol + /// definition without actually materializing the definition up front. The + /// user can materialize the definition at any time by calling the getAddress + /// method. + JITSymbol(GetAddressFtor GetAddress, JITSymbolFlags Flags) + : GetAddress(std::move(GetAddress)), CachedAddr(0), Flags(Flags) {} + + JITSymbol(const JITSymbol&) = delete; + JITSymbol& operator=(const JITSymbol&) = delete; + + JITSymbol(JITSymbol &&Other) + : GetAddress(std::move(Other.GetAddress)), Flags(std::move(Other.Flags)) { + if (Flags.hasError()) + Err = std::move(Other.Err); + else + CachedAddr = std::move(Other.CachedAddr); + } + + JITSymbol& operator=(JITSymbol &&Other) { + GetAddress = std::move(Other.GetAddress); + Flags = std::move(Other.Flags); + if (Flags.hasError()) + Err = std::move(Other.Err); + else + CachedAddr = std::move(Other.CachedAddr); + return *this; + } + + ~JITSymbol() { + if (Flags.hasError()) + Err.~Error(); + else + CachedAddr.~JITTargetAddress(); + } + + /// Returns true if the symbol exists, false otherwise. + explicit operator bool() const { + return !Flags.hasError() && (CachedAddr || GetAddress); + } + + /// Move the error field value out of this JITSymbol. + Error takeError() { + if (Flags.hasError()) + return std::move(Err); + return Error::success(); + } + + /// Get the address of the symbol in the target address space. Returns + /// '0' if the symbol does not exist. + Expected<JITTargetAddress> getAddress() { + assert(!Flags.hasError() && "getAddress called on error value"); + if (GetAddress) { + if (auto CachedAddrOrErr = GetAddress()) { + GetAddress = nullptr; + CachedAddr = *CachedAddrOrErr; + assert(CachedAddr && "Symbol could not be materialized."); + } else + return CachedAddrOrErr.takeError(); + } + return CachedAddr; + } + + JITSymbolFlags getFlags() const { return Flags; } + +private: + GetAddressFtor GetAddress; + union { + JITTargetAddress CachedAddr; + Error Err; + }; + JITSymbolFlags Flags; +}; + +/// Symbol resolution interface. +/// +/// Allows symbol flags and addresses to be looked up by name. +/// Symbol queries are done in bulk (i.e. you request resolution of a set of +/// symbols, rather than a single one) to reduce IPC overhead in the case of +/// remote JITing, and expose opportunities for parallel compilation. +class JITSymbolResolver { +public: + using LookupSet = std::set<StringRef>; + using LookupResult = std::map<StringRef, JITEvaluatedSymbol>; + using OnResolvedFunction = unique_function<void(Expected<LookupResult>)>; + + virtual ~JITSymbolResolver() = default; + + /// Returns the fully resolved address and flags for each of the given + /// symbols. + /// + /// This method will return an error if any of the given symbols can not be + /// resolved, or if the resolution process itself triggers an error. + virtual void lookup(const LookupSet &Symbols, + OnResolvedFunction OnResolved) = 0; + + /// Returns the subset of the given symbols that should be materialized by + /// the caller. Only weak/common symbols should be looked up, as strong + /// definitions are implicitly always part of the caller's responsibility. + virtual Expected<LookupSet> + getResponsibilitySet(const LookupSet &Symbols) = 0; + + /// Specify if this resolver can return valid symbols with zero value. + virtual bool allowsZeroSymbols() { return false; } + +private: + virtual void anchor(); +}; + +/// Legacy symbol resolution interface. +class LegacyJITSymbolResolver : public JITSymbolResolver { +public: + /// Performs lookup by, for each symbol, first calling + /// findSymbolInLogicalDylib and if that fails calling + /// findSymbol. + void lookup(const LookupSet &Symbols, OnResolvedFunction OnResolved) final; + + /// Performs flags lookup by calling findSymbolInLogicalDylib and + /// returning the flags value for that symbol. + Expected<LookupSet> getResponsibilitySet(const LookupSet &Symbols) final; + + /// This method returns the address of the specified symbol if it exists + /// within the logical dynamic library represented by this JITSymbolResolver. + /// Unlike findSymbol, queries through this interface should return addresses + /// for hidden symbols. + /// + /// This is of particular importance for the Orc JIT APIs, which support lazy + /// compilation by breaking up modules: Each of those broken out modules + /// must be able to resolve hidden symbols provided by the others. Clients + /// writing memory managers for MCJIT can usually ignore this method. + /// + /// This method will be queried by RuntimeDyld when checking for previous + /// definitions of common symbols. + virtual JITSymbol findSymbolInLogicalDylib(const std::string &Name) = 0; + + /// This method returns the address of the specified function or variable. + /// It is used to resolve symbols during module linking. + /// + /// If the returned symbol's address is equal to ~0ULL then RuntimeDyld will + /// skip all relocations for that symbol, and the client will be responsible + /// for handling them manually. + virtual JITSymbol findSymbol(const std::string &Name) = 0; + +private: + void anchor() override; +}; + +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_JITSYMBOL_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/MCJIT.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/MCJIT.h new file mode 100644 index 0000000000..961d4ceee3 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/MCJIT.h @@ -0,0 +1,51 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===-- MCJIT.h - MC-Based Just-In-Time Execution Engine --------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file forces the MCJIT to link in on certain operating systems. +// (Windows). +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_MCJIT_H +#define LLVM_EXECUTIONENGINE_MCJIT_H + +#include "llvm/ExecutionEngine/ExecutionEngine.h" +#include <cstdlib> + +extern "C" void LLVMLinkInMCJIT(); + +namespace { + struct ForceMCJITLinking { + ForceMCJITLinking() { + // We must reference MCJIT in such a way that compilers will not + // delete it all as dead code, even with whole program optimization, + // yet is effectively a NO-OP. As the compiler isn't smart enough + // to know that getenv() never returns -1, this will do the job. + // This is so that globals in the translation units where these functions + // are defined are forced to be initialized, populating various + // registries. + if (std::getenv("bar") != (char*) -1) + return; + + LLVMLinkInMCJIT(); + } + } ForceMCJITLinking; +} + +#endif + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/ObjectCache.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/ObjectCache.h new file mode 100644 index 0000000000..b493bf7122 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/ObjectCache.h @@ -0,0 +1,53 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===-- ObjectCache.h - Class definition for the ObjectCache ----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_OBJECTCACHE_H +#define LLVM_EXECUTIONENGINE_OBJECTCACHE_H + +#include <memory> + +namespace llvm { + +class MemoryBuffer; +class MemoryBufferRef; +class Module; + +/// This is the base ObjectCache type which can be provided to an +/// ExecutionEngine for the purpose of avoiding compilation for Modules that +/// have already been compiled and an object file is available. +class ObjectCache { + virtual void anchor(); + +public: + ObjectCache() = default; + + virtual ~ObjectCache() = default; + + /// notifyObjectCompiled - Provides a pointer to compiled code for Module M. + virtual void notifyObjectCompiled(const Module *M, MemoryBufferRef Obj) = 0; + + /// Returns a pointer to a newly allocated MemoryBuffer that contains the + /// object which corresponds with Module M, or 0 if an object is not + /// available. + virtual std::unique_ptr<MemoryBuffer> getObject(const Module* M) = 0; +}; + +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_OBJECTCACHE_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h new file mode 100644 index 0000000000..5ab0de497a --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h @@ -0,0 +1,145 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===- CompileOnDemandLayer.h - Compile each function on demand -*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// JIT layer for breaking up modules and inserting callbacks to allow +// individual functions to be compiled on demand. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_COMPILEONDEMANDLAYER_H +#define LLVM_EXECUTIONENGINE_ORC_COMPILEONDEMANDLAYER_H + +#include "llvm/ADT/APInt.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/ExecutionEngine/Orc/IndirectionUtils.h" +#include "llvm/ExecutionEngine/Orc/Layer.h" +#include "llvm/ExecutionEngine/Orc/LazyReexports.h" +#include "llvm/ExecutionEngine/Orc/Speculation.h" +#include "llvm/ExecutionEngine/Orc/Shared/OrcError.h" +#include "llvm/ExecutionEngine/RuntimeDyld.h" +#include "llvm/IR/Attributes.h" +#include "llvm/IR/Constant.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/GlobalAlias.h" +#include "llvm/IR/GlobalValue.h" +#include "llvm/IR/GlobalVariable.h" +#include "llvm/IR/Instruction.h" +#include "llvm/IR/Mangler.h" +#include "llvm/IR/Module.h" +#include "llvm/IR/Type.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/Transforms/Utils/ValueMapper.h" +#include <algorithm> +#include <cassert> +#include <functional> +#include <iterator> +#include <list> +#include <memory> +#include <set> +#include <utility> +#include <vector> + +namespace llvm { +namespace orc { + +class CompileOnDemandLayer : public IRLayer { + friend class PartitioningIRMaterializationUnit; + +public: + /// Builder for IndirectStubsManagers. + using IndirectStubsManagerBuilder = + std::function<std::unique_ptr<IndirectStubsManager>()>; + + using GlobalValueSet = std::set<const GlobalValue *>; + + /// Partitioning function. + using PartitionFunction = + std::function<Optional<GlobalValueSet>(GlobalValueSet Requested)>; + + /// Off-the-shelf partitioning which compiles all requested symbols (usually + /// a single function at a time). + static Optional<GlobalValueSet> compileRequested(GlobalValueSet Requested); + + /// Off-the-shelf partitioning which compiles whole modules whenever any + /// symbol in them is requested. + static Optional<GlobalValueSet> compileWholeModule(GlobalValueSet Requested); + + /// Construct a CompileOnDemandLayer. + CompileOnDemandLayer(ExecutionSession &ES, IRLayer &BaseLayer, + LazyCallThroughManager &LCTMgr, + IndirectStubsManagerBuilder BuildIndirectStubsManager); + + /// Sets the partition function. + void setPartitionFunction(PartitionFunction Partition); + + /// Sets the ImplSymbolMap + void setImplMap(ImplSymbolMap *Imp); + + /// Emits the given module. This should not be called by clients: it will be + /// called by the JIT when a definition added via the add method is requested. + void emit(std::unique_ptr<MaterializationResponsibility> R, + ThreadSafeModule TSM) override; + +private: + struct PerDylibResources { + public: + PerDylibResources(JITDylib &ImplD, + std::unique_ptr<IndirectStubsManager> ISMgr) + : ImplD(ImplD), ISMgr(std::move(ISMgr)) {} + JITDylib &getImplDylib() { return ImplD; } + IndirectStubsManager &getISManager() { return *ISMgr; } + + private: + JITDylib &ImplD; + std::unique_ptr<IndirectStubsManager> ISMgr; + }; + + using PerDylibResourcesMap = std::map<const JITDylib *, PerDylibResources>; + + PerDylibResources &getPerDylibResources(JITDylib &TargetD); + + void cleanUpModule(Module &M); + + void expandPartition(GlobalValueSet &Partition); + + void emitPartition(std::unique_ptr<MaterializationResponsibility> R, + ThreadSafeModule TSM, + IRMaterializationUnit::SymbolNameToDefinitionMap Defs); + + mutable std::mutex CODLayerMutex; + + IRLayer &BaseLayer; + LazyCallThroughManager &LCTMgr; + IndirectStubsManagerBuilder BuildIndirectStubsManager; + PerDylibResourcesMap DylibResources; + PartitionFunction Partition = compileRequested; + SymbolLinkagePromoter PromoteSymbols; + ImplSymbolMap *AliaseeImpls = nullptr; +}; + +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_COMPILEONDEMANDLAYER_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/CompileUtils.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/CompileUtils.h new file mode 100644 index 0000000000..45006bae91 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/CompileUtils.h @@ -0,0 +1,111 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===- CompileUtils.h - Utilities for compiling IR in the JIT ---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Contains utilities for compiling IR to object files. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_COMPILEUTILS_H +#define LLVM_EXECUTIONENGINE_ORC_COMPILEUTILS_H + +#include "llvm/ExecutionEngine/Orc/IRCompileLayer.h" +#include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h" +#include "llvm/ExecutionEngine/Orc/Layer.h" +#include <memory> + +namespace llvm { + +class MemoryBuffer; +class Module; +class ObjectCache; +class TargetMachine; + +namespace orc { + +IRSymbolMapper::ManglingOptions +irManglingOptionsFromTargetOptions(const TargetOptions &Opts); + +/// Simple compile functor: Takes a single IR module and returns an ObjectFile. +/// This compiler supports a single compilation thread and LLVMContext only. +/// For multithreaded compilation, use ConcurrentIRCompiler below. +class SimpleCompiler : public IRCompileLayer::IRCompiler { +public: + using CompileResult = std::unique_ptr<MemoryBuffer>; + + /// Construct a simple compile functor with the given target. + SimpleCompiler(TargetMachine &TM, ObjectCache *ObjCache = nullptr) + : IRCompiler(irManglingOptionsFromTargetOptions(TM.Options)), TM(TM), + ObjCache(ObjCache) {} + + /// Set an ObjectCache to query before compiling. + void setObjectCache(ObjectCache *NewCache) { ObjCache = NewCache; } + + /// Compile a Module to an ObjectFile. + Expected<CompileResult> operator()(Module &M) override; + +private: + IRSymbolMapper::ManglingOptions + manglingOptionsForTargetMachine(const TargetMachine &TM); + + CompileResult tryToLoadFromObjectCache(const Module &M); + void notifyObjectCompiled(const Module &M, const MemoryBuffer &ObjBuffer); + + TargetMachine &TM; + ObjectCache *ObjCache = nullptr; +}; + +/// A SimpleCompiler that owns its TargetMachine. +/// +/// This convenient for clients who don't want to own their TargetMachines, +/// e.g. LLJIT. +class TMOwningSimpleCompiler : public SimpleCompiler { +public: + TMOwningSimpleCompiler(std::unique_ptr<TargetMachine> TM, + ObjectCache *ObjCache = nullptr) + : SimpleCompiler(*TM, ObjCache), TM(std::move(TM)) {} + +private: + // FIXME: shared because std::functions (and consequently + // IRCompileLayer::CompileFunction) are not moveable. + std::shared_ptr<llvm::TargetMachine> TM; +}; + +/// A thread-safe version of SimpleCompiler. +/// +/// This class creates a new TargetMachine and SimpleCompiler instance for each +/// compile. +class ConcurrentIRCompiler : public IRCompileLayer::IRCompiler { +public: + ConcurrentIRCompiler(JITTargetMachineBuilder JTMB, + ObjectCache *ObjCache = nullptr); + + void setObjectCache(ObjectCache *ObjCache) { this->ObjCache = ObjCache; } + + Expected<std::unique_ptr<MemoryBuffer>> operator()(Module &M) override; + +private: + JITTargetMachineBuilder JTMB; + ObjectCache *ObjCache = nullptr; +}; + +} // end namespace orc + +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_COMPILEUTILS_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/Core.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/Core.h new file mode 100644 index 0000000000..9fbe99f5be --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/Core.h @@ -0,0 +1,1969 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===------ Core.h -- Core ORC APIs (Layer, JITDylib, etc.) -----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Contains core ORC APIs. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_CORE_H +#define LLVM_EXECUTIONENGINE_ORC_CORE_H + +#include "llvm/ADT/BitmaskEnum.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/FunctionExtras.h" +#include "llvm/ADT/IntrusiveRefCntPtr.h" +#include "llvm/ExecutionEngine/JITLink/JITLinkDylib.h" +#include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h" +#include "llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h" +#include "llvm/ExecutionEngine/Orc/TaskDispatch.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/ExtensibleRTTI.h" + +#include <atomic> +#include <future> +#include <memory> +#include <vector> + +namespace llvm { +namespace orc { + +// Forward declare some classes. +class AsynchronousSymbolQuery; +class ExecutionSession; +class MaterializationUnit; +class MaterializationResponsibility; +class JITDylib; +class ResourceTracker; +class InProgressLookupState; + +enum class SymbolState : uint8_t; + +using ResourceTrackerSP = IntrusiveRefCntPtr<ResourceTracker>; +using JITDylibSP = IntrusiveRefCntPtr<JITDylib>; + +using ResourceKey = uintptr_t; + +/// API to remove / transfer ownership of JIT resources. +class ResourceTracker : public ThreadSafeRefCountedBase<ResourceTracker> { +private: + friend class ExecutionSession; + friend class JITDylib; + friend class MaterializationResponsibility; + +public: + ResourceTracker(const ResourceTracker &) = delete; + ResourceTracker &operator=(const ResourceTracker &) = delete; + ResourceTracker(ResourceTracker &&) = delete; + ResourceTracker &operator=(ResourceTracker &&) = delete; + + ~ResourceTracker(); + + /// Return the JITDylib targeted by this tracker. + JITDylib &getJITDylib() const { + return *reinterpret_cast<JITDylib *>(JDAndFlag.load() & + ~static_cast<uintptr_t>(1)); + } + + /// Remove all resources associated with this key. + Error remove(); + + /// Transfer all resources associated with this key to the given + /// tracker, which must target the same JITDylib as this one. + void transferTo(ResourceTracker &DstRT); + + /// Return true if this tracker has become defunct. + bool isDefunct() const { return JDAndFlag.load() & 0x1; } + + /// Returns the key associated with this tracker. + /// This method should not be used except for debug logging: there is no + /// guarantee that the returned value will remain valid. + ResourceKey getKeyUnsafe() const { return reinterpret_cast<uintptr_t>(this); } + +private: + ResourceTracker(JITDylibSP JD); + + void makeDefunct(); + + std::atomic_uintptr_t JDAndFlag; +}; + +/// Listens for ResourceTracker operations. +class ResourceManager { +public: + virtual ~ResourceManager(); + virtual Error handleRemoveResources(ResourceKey K) = 0; + virtual void handleTransferResources(ResourceKey DstK, ResourceKey SrcK) = 0; +}; + +/// A set of symbol names (represented by SymbolStringPtrs for +// efficiency). +using SymbolNameSet = DenseSet<SymbolStringPtr>; + +/// A vector of symbol names. +using SymbolNameVector = std::vector<SymbolStringPtr>; + +/// A map from symbol names (as SymbolStringPtrs) to JITSymbols +/// (address/flags pairs). +using SymbolMap = DenseMap<SymbolStringPtr, JITEvaluatedSymbol>; + +/// A map from symbol names (as SymbolStringPtrs) to JITSymbolFlags. +using SymbolFlagsMap = DenseMap<SymbolStringPtr, JITSymbolFlags>; + +/// A map from JITDylibs to sets of symbols. +using SymbolDependenceMap = DenseMap<JITDylib *, SymbolNameSet>; + +/// Lookup flags that apply to each dylib in the search order for a lookup. +/// +/// If MatchHiddenSymbolsOnly is used (the default) for a given dylib, then +/// only symbols in that Dylib's interface will be searched. If +/// MatchHiddenSymbols is used then symbols with hidden visibility will match +/// as well. +enum class JITDylibLookupFlags { MatchExportedSymbolsOnly, MatchAllSymbols }; + +/// Lookup flags that apply to each symbol in a lookup. +/// +/// If RequiredSymbol is used (the default) for a given symbol then that symbol +/// must be found during the lookup or the lookup will fail returning a +/// SymbolNotFound error. If WeaklyReferencedSymbol is used and the given +/// symbol is not found then the query will continue, and no result for the +/// missing symbol will be present in the result (assuming the rest of the +/// lookup succeeds). +enum class SymbolLookupFlags { RequiredSymbol, WeaklyReferencedSymbol }; + +/// Describes the kind of lookup being performed. The lookup kind is passed to +/// symbol generators (if they're invoked) to help them determine what +/// definitions to generate. +/// +/// Static -- Lookup is being performed as-if at static link time (e.g. +/// generators representing static archives should pull in new +/// definitions). +/// +/// DLSym -- Lookup is being performed as-if at runtime (e.g. generators +/// representing static archives should not pull in new definitions). +enum class LookupKind { Static, DLSym }; + +/// A list of (JITDylib*, JITDylibLookupFlags) pairs to be used as a search +/// order during symbol lookup. +using JITDylibSearchOrder = + std::vector<std::pair<JITDylib *, JITDylibLookupFlags>>; + +/// Convenience function for creating a search order from an ArrayRef of +/// JITDylib*, all with the same flags. +inline JITDylibSearchOrder makeJITDylibSearchOrder( + ArrayRef<JITDylib *> JDs, + JITDylibLookupFlags Flags = JITDylibLookupFlags::MatchExportedSymbolsOnly) { + JITDylibSearchOrder O; + O.reserve(JDs.size()); + for (auto *JD : JDs) + O.push_back(std::make_pair(JD, Flags)); + return O; +} + +/// A set of symbols to look up, each associated with a SymbolLookupFlags +/// value. +/// +/// This class is backed by a vector and optimized for fast insertion, +/// deletion and iteration. It does not guarantee a stable order between +/// operations, and will not automatically detect duplicate elements (they +/// can be manually checked by calling the validate method). +class SymbolLookupSet { +public: + using value_type = std::pair<SymbolStringPtr, SymbolLookupFlags>; + using UnderlyingVector = std::vector<value_type>; + using iterator = UnderlyingVector::iterator; + using const_iterator = UnderlyingVector::const_iterator; + + SymbolLookupSet() = default; + + explicit SymbolLookupSet( + SymbolStringPtr Name, + SymbolLookupFlags Flags = SymbolLookupFlags::RequiredSymbol) { + add(std::move(Name), Flags); + } + + /// Construct a SymbolLookupSet from an initializer list of SymbolStringPtrs. + explicit SymbolLookupSet( + std::initializer_list<SymbolStringPtr> Names, + SymbolLookupFlags Flags = SymbolLookupFlags::RequiredSymbol) { + Symbols.reserve(Names.size()); + for (auto &Name : Names) + add(std::move(Name), Flags); + } + + /// Construct a SymbolLookupSet from a SymbolNameSet with the given + /// Flags used for each value. + explicit SymbolLookupSet( + const SymbolNameSet &Names, + SymbolLookupFlags Flags = SymbolLookupFlags::RequiredSymbol) { + Symbols.reserve(Names.size()); + for (const auto &Name : Names) + add(Name, Flags); + } + + /// Construct a SymbolLookupSet from a vector of symbols with the given Flags + /// used for each value. + /// If the ArrayRef contains duplicates it is up to the client to remove these + /// before using this instance for lookup. + explicit SymbolLookupSet( + ArrayRef<SymbolStringPtr> Names, + SymbolLookupFlags Flags = SymbolLookupFlags::RequiredSymbol) { + Symbols.reserve(Names.size()); + for (const auto &Name : Names) + add(Name, Flags); + } + + /// Construct a SymbolLookupSet from DenseMap keys. + template <typename KeyT> + static SymbolLookupSet + fromMapKeys(const DenseMap<SymbolStringPtr, KeyT> &M, + SymbolLookupFlags Flags = SymbolLookupFlags::RequiredSymbol) { + SymbolLookupSet Result; + Result.Symbols.reserve(M.size()); + for (const auto &KV : M) + Result.add(KV.first, Flags); + return Result; + } + + /// Add an element to the set. The client is responsible for checking that + /// duplicates are not added. + SymbolLookupSet & + add(SymbolStringPtr Name, + SymbolLookupFlags Flags = SymbolLookupFlags::RequiredSymbol) { + Symbols.push_back(std::make_pair(std::move(Name), Flags)); + return *this; + } + + /// Quickly append one lookup set to another. + SymbolLookupSet &append(SymbolLookupSet Other) { + Symbols.reserve(Symbols.size() + Other.size()); + for (auto &KV : Other) + Symbols.push_back(std::move(KV)); + return *this; + } + + bool empty() const { return Symbols.empty(); } + UnderlyingVector::size_type size() const { return Symbols.size(); } + iterator begin() { return Symbols.begin(); } + iterator end() { return Symbols.end(); } + const_iterator begin() const { return Symbols.begin(); } + const_iterator end() const { return Symbols.end(); } + + /// Removes the Ith element of the vector, replacing it with the last element. + void remove(UnderlyingVector::size_type I) { + std::swap(Symbols[I], Symbols.back()); + Symbols.pop_back(); + } + + /// Removes the element pointed to by the given iterator. This iterator and + /// all subsequent ones (including end()) are invalidated. + void remove(iterator I) { remove(I - begin()); } + + /// Removes all elements matching the given predicate, which must be callable + /// as bool(const SymbolStringPtr &, SymbolLookupFlags Flags). + template <typename PredFn> void remove_if(PredFn &&Pred) { + UnderlyingVector::size_type I = 0; + while (I != Symbols.size()) { + const auto &Name = Symbols[I].first; + auto Flags = Symbols[I].second; + if (Pred(Name, Flags)) + remove(I); + else + ++I; + } + } + + /// Loop over the elements of this SymbolLookupSet, applying the Body function + /// to each one. Body must be callable as + /// bool(const SymbolStringPtr &, SymbolLookupFlags). + /// If Body returns true then the element just passed in is removed from the + /// set. If Body returns false then the element is retained. + template <typename BodyFn> + auto forEachWithRemoval(BodyFn &&Body) -> std::enable_if_t< + std::is_same<decltype(Body(std::declval<const SymbolStringPtr &>(), + std::declval<SymbolLookupFlags>())), + bool>::value> { + UnderlyingVector::size_type I = 0; + while (I != Symbols.size()) { + const auto &Name = Symbols[I].first; + auto Flags = Symbols[I].second; + if (Body(Name, Flags)) + remove(I); + else + ++I; + } + } + + /// Loop over the elements of this SymbolLookupSet, applying the Body function + /// to each one. Body must be callable as + /// Expected<bool>(const SymbolStringPtr &, SymbolLookupFlags). + /// If Body returns a failure value, the loop exits immediately. If Body + /// returns true then the element just passed in is removed from the set. If + /// Body returns false then the element is retained. + template <typename BodyFn> + auto forEachWithRemoval(BodyFn &&Body) -> std::enable_if_t< + std::is_same<decltype(Body(std::declval<const SymbolStringPtr &>(), + std::declval<SymbolLookupFlags>())), + Expected<bool>>::value, + Error> { + UnderlyingVector::size_type I = 0; + while (I != Symbols.size()) { + const auto &Name = Symbols[I].first; + auto Flags = Symbols[I].second; + auto Remove = Body(Name, Flags); + if (!Remove) + return Remove.takeError(); + if (*Remove) + remove(I); + else + ++I; + } + return Error::success(); + } + + /// Construct a SymbolNameVector from this instance by dropping the Flags + /// values. + SymbolNameVector getSymbolNames() const { + SymbolNameVector Names; + Names.reserve(Symbols.size()); + for (auto &KV : Symbols) + Names.push_back(KV.first); + return Names; + } + + /// Sort the lookup set by pointer value. This sort is fast but sensitive to + /// allocation order and so should not be used where a consistent order is + /// required. + void sortByAddress() { + llvm::sort(Symbols, [](const value_type &LHS, const value_type &RHS) { + return LHS.first < RHS.first; + }); + } + + /// Sort the lookup set lexicographically. This sort is slow but the order + /// is unaffected by allocation order. + void sortByName() { + llvm::sort(Symbols, [](const value_type &LHS, const value_type &RHS) { + return *LHS.first < *RHS.first; + }); + } + + /// Remove any duplicate elements. If a SymbolLookupSet is not duplicate-free + /// by construction, this method can be used to turn it into a proper set. + void removeDuplicates() { + sortByAddress(); + auto LastI = std::unique(Symbols.begin(), Symbols.end()); + Symbols.erase(LastI, Symbols.end()); + } + +#ifndef NDEBUG + /// Returns true if this set contains any duplicates. This should only be used + /// in assertions. + bool containsDuplicates() { + if (Symbols.size() < 2) + return false; + sortByAddress(); + for (UnderlyingVector::size_type I = 1; I != Symbols.size(); ++I) + if (Symbols[I].first == Symbols[I - 1].first) + return true; + return false; + } +#endif + +private: + UnderlyingVector Symbols; +}; + +struct SymbolAliasMapEntry { + SymbolAliasMapEntry() = default; + SymbolAliasMapEntry(SymbolStringPtr Aliasee, JITSymbolFlags AliasFlags) + : Aliasee(std::move(Aliasee)), AliasFlags(AliasFlags) {} + + SymbolStringPtr Aliasee; + JITSymbolFlags AliasFlags; +}; + +/// A map of Symbols to (Symbol, Flags) pairs. +using SymbolAliasMap = DenseMap<SymbolStringPtr, SymbolAliasMapEntry>; + +/// Callback to notify client that symbols have been resolved. +using SymbolsResolvedCallback = unique_function<void(Expected<SymbolMap>)>; + +/// Callback to register the dependencies for a given query. +using RegisterDependenciesFunction = + std::function<void(const SymbolDependenceMap &)>; + +/// This can be used as the value for a RegisterDependenciesFunction if there +/// are no dependants to register with. +extern RegisterDependenciesFunction NoDependenciesToRegister; + +class ResourceTrackerDefunct : public ErrorInfo<ResourceTrackerDefunct> { +public: + static char ID; + + ResourceTrackerDefunct(ResourceTrackerSP RT); + std::error_code convertToErrorCode() const override; + void log(raw_ostream &OS) const override; + +private: + ResourceTrackerSP RT; +}; + +/// Used to notify a JITDylib that the given set of symbols failed to +/// materialize. +class FailedToMaterialize : public ErrorInfo<FailedToMaterialize> { +public: + static char ID; + + FailedToMaterialize(std::shared_ptr<SymbolDependenceMap> Symbols); + std::error_code convertToErrorCode() const override; + void log(raw_ostream &OS) const override; + const SymbolDependenceMap &getSymbols() const { return *Symbols; } + +private: + std::shared_ptr<SymbolDependenceMap> Symbols; +}; + +/// Used to notify clients when symbols can not be found during a lookup. +class SymbolsNotFound : public ErrorInfo<SymbolsNotFound> { +public: + static char ID; + + SymbolsNotFound(std::shared_ptr<SymbolStringPool> SSP, SymbolNameSet Symbols); + SymbolsNotFound(std::shared_ptr<SymbolStringPool> SSP, + SymbolNameVector Symbols); + std::error_code convertToErrorCode() const override; + void log(raw_ostream &OS) const override; + std::shared_ptr<SymbolStringPool> getSymbolStringPool() { return SSP; } + const SymbolNameVector &getSymbols() const { return Symbols; } + +private: + std::shared_ptr<SymbolStringPool> SSP; + SymbolNameVector Symbols; +}; + +/// Used to notify clients that a set of symbols could not be removed. +class SymbolsCouldNotBeRemoved : public ErrorInfo<SymbolsCouldNotBeRemoved> { +public: + static char ID; + + SymbolsCouldNotBeRemoved(std::shared_ptr<SymbolStringPool> SSP, + SymbolNameSet Symbols); + std::error_code convertToErrorCode() const override; + void log(raw_ostream &OS) const override; + std::shared_ptr<SymbolStringPool> getSymbolStringPool() { return SSP; } + const SymbolNameSet &getSymbols() const { return Symbols; } + +private: + std::shared_ptr<SymbolStringPool> SSP; + SymbolNameSet Symbols; +}; + +/// Errors of this type should be returned if a module fails to include +/// definitions that are claimed by the module's associated +/// MaterializationResponsibility. If this error is returned it is indicative of +/// a broken transformation / compiler / object cache. +class MissingSymbolDefinitions : public ErrorInfo<MissingSymbolDefinitions> { +public: + static char ID; + + MissingSymbolDefinitions(std::shared_ptr<SymbolStringPool> SSP, + std::string ModuleName, SymbolNameVector Symbols) + : SSP(std::move(SSP)), ModuleName(std::move(ModuleName)), + Symbols(std::move(Symbols)) {} + std::error_code convertToErrorCode() const override; + void log(raw_ostream &OS) const override; + std::shared_ptr<SymbolStringPool> getSymbolStringPool() { return SSP; } + const std::string &getModuleName() const { return ModuleName; } + const SymbolNameVector &getSymbols() const { return Symbols; } +private: + std::shared_ptr<SymbolStringPool> SSP; + std::string ModuleName; + SymbolNameVector Symbols; +}; + +/// Errors of this type should be returned if a module contains definitions for +/// symbols that are not claimed by the module's associated +/// MaterializationResponsibility. If this error is returned it is indicative of +/// a broken transformation / compiler / object cache. +class UnexpectedSymbolDefinitions : public ErrorInfo<UnexpectedSymbolDefinitions> { +public: + static char ID; + + UnexpectedSymbolDefinitions(std::shared_ptr<SymbolStringPool> SSP, + std::string ModuleName, SymbolNameVector Symbols) + : SSP(std::move(SSP)), ModuleName(std::move(ModuleName)), + Symbols(std::move(Symbols)) {} + std::error_code convertToErrorCode() const override; + void log(raw_ostream &OS) const override; + std::shared_ptr<SymbolStringPool> getSymbolStringPool() { return SSP; } + const std::string &getModuleName() const { return ModuleName; } + const SymbolNameVector &getSymbols() const { return Symbols; } +private: + std::shared_ptr<SymbolStringPool> SSP; + std::string ModuleName; + SymbolNameVector Symbols; +}; + +/// Tracks responsibility for materialization, and mediates interactions between +/// MaterializationUnits and JDs. +/// +/// An instance of this class is passed to MaterializationUnits when their +/// materialize method is called. It allows MaterializationUnits to resolve and +/// emit symbols, or abandon materialization by notifying any unmaterialized +/// symbols of an error. +class MaterializationResponsibility { + friend class ExecutionSession; + friend class JITDylib; + +public: + MaterializationResponsibility(MaterializationResponsibility &&) = delete; + MaterializationResponsibility & + operator=(MaterializationResponsibility &&) = delete; + + /// Destruct a MaterializationResponsibility instance. In debug mode + /// this asserts that all symbols being tracked have been either + /// emitted or notified of an error. + ~MaterializationResponsibility(); + + /// Returns the ResourceTracker for this instance. + template <typename Func> Error withResourceKeyDo(Func &&F) const; + + /// Returns the target JITDylib that these symbols are being materialized + /// into. + JITDylib &getTargetJITDylib() const { return JD; } + + /// Returns the ExecutionSession for this instance. + ExecutionSession &getExecutionSession() const; + + /// Returns the symbol flags map for this responsibility instance. + /// Note: The returned flags may have transient flags (Lazy, Materializing) + /// set. These should be stripped with JITSymbolFlags::stripTransientFlags + /// before using. + const SymbolFlagsMap &getSymbols() const { return SymbolFlags; } + + /// Returns the initialization pseudo-symbol, if any. This symbol will also + /// be present in the SymbolFlagsMap for this MaterializationResponsibility + /// object. + const SymbolStringPtr &getInitializerSymbol() const { return InitSymbol; } + + /// Returns the names of any symbols covered by this + /// MaterializationResponsibility object that have queries pending. This + /// information can be used to return responsibility for unrequested symbols + /// back to the JITDylib via the delegate method. + SymbolNameSet getRequestedSymbols() const; + + /// Notifies the target JITDylib that the given symbols have been resolved. + /// This will update the given symbols' addresses in the JITDylib, and notify + /// any pending queries on the given symbols of their resolution. The given + /// symbols must be ones covered by this MaterializationResponsibility + /// instance. Individual calls to this method may resolve a subset of the + /// symbols, but all symbols must have been resolved prior to calling emit. + /// + /// This method will return an error if any symbols being resolved have been + /// moved to the error state due to the failure of a dependency. If this + /// method returns an error then clients should log it and call + /// failMaterialize. If no dependencies have been registered for the + /// symbols covered by this MaterializationResponsibiility then this method + /// is guaranteed to return Error::success() and can be wrapped with cantFail. + Error notifyResolved(const SymbolMap &Symbols); + + /// Notifies the target JITDylib (and any pending queries on that JITDylib) + /// that all symbols covered by this MaterializationResponsibility instance + /// have been emitted. + /// + /// This method will return an error if any symbols being resolved have been + /// moved to the error state due to the failure of a dependency. If this + /// method returns an error then clients should log it and call + /// failMaterialize. If no dependencies have been registered for the + /// symbols covered by this MaterializationResponsibiility then this method + /// is guaranteed to return Error::success() and can be wrapped with cantFail. + Error notifyEmitted(); + + /// Attempt to claim responsibility for new definitions. This method can be + /// used to claim responsibility for symbols that are added to a + /// materialization unit during the compilation process (e.g. literal pool + /// symbols). Symbol linkage rules are the same as for symbols that are + /// defined up front: duplicate strong definitions will result in errors. + /// Duplicate weak definitions will be discarded (in which case they will + /// not be added to this responsibility instance). + /// + /// This method can be used by materialization units that want to add + /// additional symbols at materialization time (e.g. stubs, compile + /// callbacks, metadata). + Error defineMaterializing(SymbolFlagsMap SymbolFlags); + + /// Define the given symbols as non-existent, removing it from the symbol + /// table and notifying any pending queries. Queries that lookup up the + /// symbol using the SymbolLookupFlags::WeaklyReferencedSymbol flag will + /// behave as if the symbol had not been matched in the first place. Queries + /// that required this symbol will fail with a missing symbol definition + /// error. + /// + /// This method is intended to support cleanup of special symbols like + /// initializer symbols: Queries using + /// SymbolLookupFlags::WeaklyReferencedSymbol can be used to trigger their + /// emission, and this method can be used to remove them from the JITDylib + /// once materialization is complete. + void defineNonExistent(ArrayRef<SymbolStringPtr> Symbols); + + /// Notify all not-yet-emitted covered by this MaterializationResponsibility + /// instance that an error has occurred. + /// This will remove all symbols covered by this MaterializationResponsibilty + /// from the target JITDylib, and send an error to any queries waiting on + /// these symbols. + void failMaterialization(); + + /// Transfers responsibility to the given MaterializationUnit for all + /// symbols defined by that MaterializationUnit. This allows + /// materializers to break up work based on run-time information (e.g. + /// by introspecting which symbols have actually been looked up and + /// materializing only those). + Error replace(std::unique_ptr<MaterializationUnit> MU); + + /// Delegates responsibility for the given symbols to the returned + /// materialization responsibility. Useful for breaking up work between + /// threads, or different kinds of materialization processes. + Expected<std::unique_ptr<MaterializationResponsibility>> + delegate(const SymbolNameSet &Symbols); + + void addDependencies(const SymbolStringPtr &Name, + const SymbolDependenceMap &Dependencies); + + /// Add dependencies that apply to all symbols covered by this instance. + void addDependenciesForAll(const SymbolDependenceMap &Dependencies); + +private: + /// Create a MaterializationResponsibility for the given JITDylib and + /// initial symbols. + MaterializationResponsibility(ResourceTrackerSP RT, + SymbolFlagsMap SymbolFlags, + SymbolStringPtr InitSymbol) + : JD(RT->getJITDylib()), RT(std::move(RT)), + SymbolFlags(std::move(SymbolFlags)), InitSymbol(std::move(InitSymbol)) { + assert(!this->SymbolFlags.empty() && "Materializing nothing?"); + } + + JITDylib &JD; + ResourceTrackerSP RT; + SymbolFlagsMap SymbolFlags; + SymbolStringPtr InitSymbol; +}; + +/// A MaterializationUnit represents a set of symbol definitions that can +/// be materialized as a group, or individually discarded (when +/// overriding definitions are encountered). +/// +/// MaterializationUnits are used when providing lazy definitions of symbols to +/// JITDylibs. The JITDylib will call materialize when the address of a symbol +/// is requested via the lookup method. The JITDylib will call discard if a +/// stronger definition is added or already present. +class MaterializationUnit { + friend class ExecutionSession; + friend class JITDylib; + +public: + static char ID; + + struct Interface { + Interface() = default; + Interface(SymbolFlagsMap InitalSymbolFlags, SymbolStringPtr InitSymbol) + : SymbolFlags(std::move(InitalSymbolFlags)), + InitSymbol(std::move(InitSymbol)) { + assert((!this->InitSymbol || this->SymbolFlags.count(this->InitSymbol)) && + "If set, InitSymbol should appear in InitialSymbolFlags map"); + } + + SymbolFlagsMap SymbolFlags; + SymbolStringPtr InitSymbol; + }; + + MaterializationUnit(Interface I) + : SymbolFlags(std::move(I.SymbolFlags)), + InitSymbol(std::move(I.InitSymbol)) {} + virtual ~MaterializationUnit() = default; + + /// Return the name of this materialization unit. Useful for debugging + /// output. + virtual StringRef getName() const = 0; + + /// Return the set of symbols that this source provides. + const SymbolFlagsMap &getSymbols() const { return SymbolFlags; } + + /// Returns the initialization symbol for this MaterializationUnit (if any). + const SymbolStringPtr &getInitializerSymbol() const { return InitSymbol; } + + /// Implementations of this method should materialize all symbols + /// in the materialzation unit, except for those that have been + /// previously discarded. + virtual void + materialize(std::unique_ptr<MaterializationResponsibility> R) = 0; + + /// Called by JITDylibs to notify MaterializationUnits that the given symbol + /// has been overridden. + void doDiscard(const JITDylib &JD, const SymbolStringPtr &Name) { + SymbolFlags.erase(Name); + discard(JD, std::move(Name)); + } + +protected: + SymbolFlagsMap SymbolFlags; + SymbolStringPtr InitSymbol; + +private: + virtual void anchor(); + + /// Implementations of this method should discard the given symbol + /// from the source (e.g. if the source is an LLVM IR Module and the + /// symbol is a function, delete the function body or mark it available + /// externally). + virtual void discard(const JITDylib &JD, const SymbolStringPtr &Name) = 0; +}; + +/// A MaterializationUnit implementation for pre-existing absolute symbols. +/// +/// All symbols will be resolved and marked ready as soon as the unit is +/// materialized. +class AbsoluteSymbolsMaterializationUnit : public MaterializationUnit { +public: + AbsoluteSymbolsMaterializationUnit(SymbolMap Symbols); + + StringRef getName() const override; + +private: + void materialize(std::unique_ptr<MaterializationResponsibility> R) override; + void discard(const JITDylib &JD, const SymbolStringPtr &Name) override; + static MaterializationUnit::Interface extractFlags(const SymbolMap &Symbols); + + SymbolMap Symbols; +}; + +/// Create an AbsoluteSymbolsMaterializationUnit with the given symbols. +/// Useful for inserting absolute symbols into a JITDylib. E.g.: +/// \code{.cpp} +/// JITDylib &JD = ...; +/// SymbolStringPtr Foo = ...; +/// JITEvaluatedSymbol FooSym = ...; +/// if (auto Err = JD.define(absoluteSymbols({{Foo, FooSym}}))) +/// return Err; +/// \endcode +/// +inline std::unique_ptr<AbsoluteSymbolsMaterializationUnit> +absoluteSymbols(SymbolMap Symbols) { + return std::make_unique<AbsoluteSymbolsMaterializationUnit>( + std::move(Symbols)); +} + +/// A materialization unit for symbol aliases. Allows existing symbols to be +/// aliased with alternate flags. +class ReExportsMaterializationUnit : public MaterializationUnit { +public: + /// SourceJD is allowed to be nullptr, in which case the source JITDylib is + /// taken to be whatever JITDylib these definitions are materialized in (and + /// MatchNonExported has no effect). This is useful for defining aliases + /// within a JITDylib. + /// + /// Note: Care must be taken that no sets of aliases form a cycle, as such + /// a cycle will result in a deadlock when any symbol in the cycle is + /// resolved. + ReExportsMaterializationUnit(JITDylib *SourceJD, + JITDylibLookupFlags SourceJDLookupFlags, + SymbolAliasMap Aliases); + + StringRef getName() const override; + +private: + void materialize(std::unique_ptr<MaterializationResponsibility> R) override; + void discard(const JITDylib &JD, const SymbolStringPtr &Name) override; + static MaterializationUnit::Interface + extractFlags(const SymbolAliasMap &Aliases); + + JITDylib *SourceJD = nullptr; + JITDylibLookupFlags SourceJDLookupFlags; + SymbolAliasMap Aliases; +}; + +/// Create a ReExportsMaterializationUnit with the given aliases. +/// Useful for defining symbol aliases.: E.g., given a JITDylib JD containing +/// symbols "foo" and "bar", we can define aliases "baz" (for "foo") and "qux" +/// (for "bar") with: \code{.cpp} +/// SymbolStringPtr Baz = ...; +/// SymbolStringPtr Qux = ...; +/// if (auto Err = JD.define(symbolAliases({ +/// {Baz, { Foo, JITSymbolFlags::Exported }}, +/// {Qux, { Bar, JITSymbolFlags::Weak }}})) +/// return Err; +/// \endcode +inline std::unique_ptr<ReExportsMaterializationUnit> +symbolAliases(SymbolAliasMap Aliases) { + return std::make_unique<ReExportsMaterializationUnit>( + nullptr, JITDylibLookupFlags::MatchAllSymbols, std::move(Aliases)); +} + +/// Create a materialization unit for re-exporting symbols from another JITDylib +/// with alternative names/flags. +/// SourceJD will be searched using the given JITDylibLookupFlags. +inline std::unique_ptr<ReExportsMaterializationUnit> +reexports(JITDylib &SourceJD, SymbolAliasMap Aliases, + JITDylibLookupFlags SourceJDLookupFlags = + JITDylibLookupFlags::MatchExportedSymbolsOnly) { + return std::make_unique<ReExportsMaterializationUnit>( + &SourceJD, SourceJDLookupFlags, std::move(Aliases)); +} + +/// Build a SymbolAliasMap for the common case where you want to re-export +/// symbols from another JITDylib with the same linkage/flags. +Expected<SymbolAliasMap> +buildSimpleReexportsAliasMap(JITDylib &SourceJD, const SymbolNameSet &Symbols); + +/// Represents the state that a symbol has reached during materialization. +enum class SymbolState : uint8_t { + Invalid, /// No symbol should be in this state. + NeverSearched, /// Added to the symbol table, never queried. + Materializing, /// Queried, materialization begun. + Resolved, /// Assigned address, still materializing. + Emitted, /// Emitted to memory, but waiting on transitive dependencies. + Ready = 0x3f /// Ready and safe for clients to access. +}; + +/// A symbol query that returns results via a callback when results are +/// ready. +/// +/// makes a callback when all symbols are available. +class AsynchronousSymbolQuery { + friend class ExecutionSession; + friend class InProgressFullLookupState; + friend class JITDylib; + friend class JITSymbolResolverAdapter; + friend class MaterializationResponsibility; + +public: + /// Create a query for the given symbols. The NotifyComplete + /// callback will be called once all queried symbols reach the given + /// minimum state. + AsynchronousSymbolQuery(const SymbolLookupSet &Symbols, + SymbolState RequiredState, + SymbolsResolvedCallback NotifyComplete); + + /// Notify the query that a requested symbol has reached the required state. + void notifySymbolMetRequiredState(const SymbolStringPtr &Name, + JITEvaluatedSymbol Sym); + + /// Returns true if all symbols covered by this query have been + /// resolved. + bool isComplete() const { return OutstandingSymbolsCount == 0; } + + +private: + void handleComplete(ExecutionSession &ES); + + SymbolState getRequiredState() { return RequiredState; } + + void addQueryDependence(JITDylib &JD, SymbolStringPtr Name); + + void removeQueryDependence(JITDylib &JD, const SymbolStringPtr &Name); + + void dropSymbol(const SymbolStringPtr &Name); + + void handleFailed(Error Err); + + void detach(); + + SymbolsResolvedCallback NotifyComplete; + SymbolDependenceMap QueryRegistrations; + SymbolMap ResolvedSymbols; + size_t OutstandingSymbolsCount; + SymbolState RequiredState; +}; + +/// Wraps state for a lookup-in-progress. +/// DefinitionGenerators can optionally take ownership of a LookupState object +/// to suspend a lookup-in-progress while they search for definitions. +class LookupState { + friend class OrcV2CAPIHelper; + friend class ExecutionSession; + +public: + LookupState(); + LookupState(LookupState &&); + LookupState &operator=(LookupState &&); + ~LookupState(); + + /// Continue the lookup. This can be called by DefinitionGenerators + /// to re-start a captured query-application operation. + void continueLookup(Error Err); + +private: + LookupState(std::unique_ptr<InProgressLookupState> IPLS); + + // For C API. + void reset(InProgressLookupState *IPLS); + + std::unique_ptr<InProgressLookupState> IPLS; +}; + +/// Definition generators can be attached to JITDylibs to generate new +/// definitions for otherwise unresolved symbols during lookup. +class DefinitionGenerator { +public: + virtual ~DefinitionGenerator(); + + /// DefinitionGenerators should override this method to insert new + /// definitions into the parent JITDylib. K specifies the kind of this + /// lookup. JD specifies the target JITDylib being searched, and + /// JDLookupFlags specifies whether the search should match against + /// hidden symbols. Finally, Symbols describes the set of unresolved + /// symbols and their associated lookup flags. + virtual Error tryToGenerate(LookupState &LS, LookupKind K, JITDylib &JD, + JITDylibLookupFlags JDLookupFlags, + const SymbolLookupSet &LookupSet) = 0; +}; + +/// Represents a JIT'd dynamic library. +/// +/// This class aims to mimic the behavior of a regular dylib or shared object, +/// but without requiring the contained program representations to be compiled +/// up-front. The JITDylib's content is defined by adding MaterializationUnits, +/// and contained MaterializationUnits will typically rely on the JITDylib's +/// links-against order to resolve external references (similar to a regular +/// dylib). +/// +/// The JITDylib object is a thin wrapper that references state held by the +/// ExecutionSession. JITDylibs can be removed, clearing this underlying state +/// and leaving the JITDylib object in a defunct state. In this state the +/// JITDylib's name is guaranteed to remain accessible. If the ExecutionSession +/// is still alive then other operations are callable but will return an Error +/// or null result (depending on the API). It is illegal to call any operation +/// other than getName on a JITDylib after the ExecutionSession has been torn +/// down. +/// +/// JITDylibs cannot be moved or copied. Their address is stable, and useful as +/// a key in some JIT data structures. +class JITDylib : public ThreadSafeRefCountedBase<JITDylib>, + public jitlink::JITLinkDylib { + friend class AsynchronousSymbolQuery; + friend class ExecutionSession; + friend class Platform; + friend class MaterializationResponsibility; +public: + + JITDylib(const JITDylib &) = delete; + JITDylib &operator=(const JITDylib &) = delete; + JITDylib(JITDylib &&) = delete; + JITDylib &operator=(JITDylib &&) = delete; + ~JITDylib(); + + /// Get a reference to the ExecutionSession for this JITDylib. + /// + /// It is legal to call this method on a defunct JITDylib, however the result + /// will only usable if the ExecutionSession is still alive. If this JITDylib + /// is held by an error that may have torn down the JIT then the result + /// should not be used. + ExecutionSession &getExecutionSession() const { return ES; } + + /// Dump current JITDylib state to OS. + /// + /// It is legal to call this method on a defunct JITDylib. + void dump(raw_ostream &OS); + + /// Calls remove on all trackers currently associated with this JITDylib. + /// Does not run static deinits. + /// + /// Note that removal happens outside the session lock, so new code may be + /// added concurrently while the clear is underway, and the newly added + /// code will *not* be cleared. Adding new code concurrently with a clear + /// is usually a bug and should be avoided. + /// + /// It is illegal to call this method on a defunct JITDylib and the client + /// is responsible for ensuring that they do not do so. + Error clear(); + + /// Get the default resource tracker for this JITDylib. + /// + /// It is illegal to call this method on a defunct JITDylib and the client + /// is responsible for ensuring that they do not do so. + ResourceTrackerSP getDefaultResourceTracker(); + + /// Create a resource tracker for this JITDylib. + /// + /// It is illegal to call this method on a defunct JITDylib and the client + /// is responsible for ensuring that they do not do so. + ResourceTrackerSP createResourceTracker(); + + /// Adds a definition generator to this JITDylib and returns a referenece to + /// it. + /// + /// When JITDylibs are searched during lookup, if no existing definition of + /// a symbol is found, then any generators that have been added are run (in + /// the order that they were added) to potentially generate a definition. + /// + /// It is illegal to call this method on a defunct JITDylib and the client + /// is responsible for ensuring that they do not do so. + template <typename GeneratorT> + GeneratorT &addGenerator(std::unique_ptr<GeneratorT> DefGenerator); + + /// Remove a definition generator from this JITDylib. + /// + /// The given generator must exist in this JITDylib's generators list (i.e. + /// have been added and not yet removed). + /// + /// It is illegal to call this method on a defunct JITDylib and the client + /// is responsible for ensuring that they do not do so. + void removeGenerator(DefinitionGenerator &G); + + /// Set the link order to be used when fixing up definitions in JITDylib. + /// This will replace the previous link order, and apply to any symbol + /// resolutions made for definitions in this JITDylib after the call to + /// setLinkOrder (even if the definition itself was added before the + /// call). + /// + /// If LinkAgainstThisJITDylibFirst is true (the default) then this JITDylib + /// will add itself to the beginning of the LinkOrder (Clients should not + /// put this JITDylib in the list in this case, to avoid redundant lookups). + /// + /// If LinkAgainstThisJITDylibFirst is false then the link order will be used + /// as-is. The primary motivation for this feature is to support deliberate + /// shadowing of symbols in this JITDylib by a facade JITDylib. For example, + /// the facade may resolve function names to stubs, and the stubs may compile + /// lazily by looking up symbols in this dylib. Adding the facade dylib + /// as the first in the link order (instead of this dylib) ensures that + /// definitions within this dylib resolve to the lazy-compiling stubs, + /// rather than immediately materializing the definitions in this dylib. + /// + /// It is illegal to call this method on a defunct JITDylib and the client + /// is responsible for ensuring that they do not do so. + void setLinkOrder(JITDylibSearchOrder NewSearchOrder, + bool LinkAgainstThisJITDylibFirst = true); + + /// Add the given JITDylib to the link order for definitions in this + /// JITDylib. + /// + /// It is illegal to call this method on a defunct JITDylib and the client + /// is responsible for ensuring that they do not do so. + void addToLinkOrder(JITDylib &JD, + JITDylibLookupFlags JDLookupFlags = + JITDylibLookupFlags::MatchExportedSymbolsOnly); + + /// Replace OldJD with NewJD in the link order if OldJD is present. + /// Otherwise this operation is a no-op. + /// + /// It is illegal to call this method on a defunct JITDylib and the client + /// is responsible for ensuring that they do not do so. + void replaceInLinkOrder(JITDylib &OldJD, JITDylib &NewJD, + JITDylibLookupFlags JDLookupFlags = + JITDylibLookupFlags::MatchExportedSymbolsOnly); + + /// Remove the given JITDylib from the link order for this JITDylib if it is + /// present. Otherwise this operation is a no-op. + /// + /// It is illegal to call this method on a defunct JITDylib and the client + /// is responsible for ensuring that they do not do so. + void removeFromLinkOrder(JITDylib &JD); + + /// Do something with the link order (run under the session lock). + /// + /// It is illegal to call this method on a defunct JITDylib and the client + /// is responsible for ensuring that they do not do so. + template <typename Func> + auto withLinkOrderDo(Func &&F) + -> decltype(F(std::declval<const JITDylibSearchOrder &>())); + + /// Define all symbols provided by the materialization unit to be part of this + /// JITDylib. + /// + /// If RT is not specified then the default resource tracker will be used. + /// + /// This overload always takes ownership of the MaterializationUnit. If any + /// errors occur, the MaterializationUnit consumed. + /// + /// It is illegal to call this method on a defunct JITDylib and the client + /// is responsible for ensuring that they do not do so. + template <typename MaterializationUnitType> + Error define(std::unique_ptr<MaterializationUnitType> &&MU, + ResourceTrackerSP RT = nullptr); + + /// Define all symbols provided by the materialization unit to be part of this + /// JITDylib. + /// + /// This overload only takes ownership of the MaterializationUnit no error is + /// generated. If an error occurs, ownership remains with the caller. This + /// may allow the caller to modify the MaterializationUnit to correct the + /// issue, then re-call define. + /// + /// It is illegal to call this method on a defunct JITDylib and the client + /// is responsible for ensuring that they do not do so. + template <typename MaterializationUnitType> + Error define(std::unique_ptr<MaterializationUnitType> &MU, + ResourceTrackerSP RT = nullptr); + + /// Tries to remove the given symbols. + /// + /// If any symbols are not defined in this JITDylib this method will return + /// a SymbolsNotFound error covering the missing symbols. + /// + /// If all symbols are found but some symbols are in the process of being + /// materialized this method will return a SymbolsCouldNotBeRemoved error. + /// + /// On success, all symbols are removed. On failure, the JITDylib state is + /// left unmodified (no symbols are removed). + /// + /// It is illegal to call this method on a defunct JITDylib and the client + /// is responsible for ensuring that they do not do so. + Error remove(const SymbolNameSet &Names); + + /// Returns the given JITDylibs and all of their transitive dependencies in + /// DFS order (based on linkage relationships). Each JITDylib will appear + /// only once. + /// + /// If any JITDylib in the order is defunct then this method will return an + /// error, otherwise returns the order. + static Expected<std::vector<JITDylibSP>> + getDFSLinkOrder(ArrayRef<JITDylibSP> JDs); + + /// Returns the given JITDylibs and all of their transitive dependencies in + /// reverse DFS order (based on linkage relationships). Each JITDylib will + /// appear only once. + /// + /// If any JITDylib in the order is defunct then this method will return an + /// error, otherwise returns the order. + static Expected<std::vector<JITDylibSP>> + getReverseDFSLinkOrder(ArrayRef<JITDylibSP> JDs); + + /// Return this JITDylib and its transitive dependencies in DFS order + /// based on linkage relationships. + /// + /// If any JITDylib in the order is defunct then this method will return an + /// error, otherwise returns the order. + Expected<std::vector<JITDylibSP>> getDFSLinkOrder(); + + /// Rteurn this JITDylib and its transitive dependencies in reverse DFS order + /// based on linkage relationships. + /// + /// If any JITDylib in the order is defunct then this method will return an + /// error, otherwise returns the order. + Expected<std::vector<JITDylibSP>> getReverseDFSLinkOrder(); + +private: + using AsynchronousSymbolQuerySet = + std::set<std::shared_ptr<AsynchronousSymbolQuery>>; + + using AsynchronousSymbolQueryList = + std::vector<std::shared_ptr<AsynchronousSymbolQuery>>; + + struct UnmaterializedInfo { + UnmaterializedInfo(std::unique_ptr<MaterializationUnit> MU, + ResourceTracker *RT) + : MU(std::move(MU)), RT(RT) {} + + std::unique_ptr<MaterializationUnit> MU; + ResourceTracker *RT; + }; + + using UnmaterializedInfosMap = + DenseMap<SymbolStringPtr, std::shared_ptr<UnmaterializedInfo>>; + + using UnmaterializedInfosList = + std::vector<std::shared_ptr<UnmaterializedInfo>>; + + struct MaterializingInfo { + SymbolDependenceMap Dependants; + SymbolDependenceMap UnemittedDependencies; + + void addQuery(std::shared_ptr<AsynchronousSymbolQuery> Q); + void removeQuery(const AsynchronousSymbolQuery &Q); + AsynchronousSymbolQueryList takeQueriesMeeting(SymbolState RequiredState); + AsynchronousSymbolQueryList takeAllPendingQueries() { + return std::move(PendingQueries); + } + bool hasQueriesPending() const { return !PendingQueries.empty(); } + const AsynchronousSymbolQueryList &pendingQueries() const { + return PendingQueries; + } + private: + AsynchronousSymbolQueryList PendingQueries; + }; + + using MaterializingInfosMap = DenseMap<SymbolStringPtr, MaterializingInfo>; + + class SymbolTableEntry { + public: + SymbolTableEntry() = default; + SymbolTableEntry(JITSymbolFlags Flags) + : Flags(Flags), State(static_cast<uint8_t>(SymbolState::NeverSearched)), + MaterializerAttached(false), PendingRemoval(false) {} + + JITTargetAddress getAddress() const { return Addr; } + JITSymbolFlags getFlags() const { return Flags; } + SymbolState getState() const { return static_cast<SymbolState>(State); } + + bool hasMaterializerAttached() const { return MaterializerAttached; } + bool isPendingRemoval() const { return PendingRemoval; } + + void setAddress(JITTargetAddress Addr) { this->Addr = Addr; } + void setFlags(JITSymbolFlags Flags) { this->Flags = Flags; } + void setState(SymbolState State) { + assert(static_cast<uint8_t>(State) < (1 << 6) && + "State does not fit in bitfield"); + this->State = static_cast<uint8_t>(State); + } + + void setMaterializerAttached(bool MaterializerAttached) { + this->MaterializerAttached = MaterializerAttached; + } + + void setPendingRemoval(bool PendingRemoval) { + this->PendingRemoval = PendingRemoval; + } + + JITEvaluatedSymbol getSymbol() const { + return JITEvaluatedSymbol(Addr, Flags); + } + + private: + JITTargetAddress Addr = 0; + JITSymbolFlags Flags; + uint8_t State : 6; + uint8_t MaterializerAttached : 1; + uint8_t PendingRemoval : 1; + }; + + using SymbolTable = DenseMap<SymbolStringPtr, SymbolTableEntry>; + + JITDylib(ExecutionSession &ES, std::string Name); + + std::pair<AsynchronousSymbolQuerySet, std::shared_ptr<SymbolDependenceMap>> + removeTracker(ResourceTracker &RT); + + void transferTracker(ResourceTracker &DstRT, ResourceTracker &SrcRT); + + Error defineImpl(MaterializationUnit &MU); + + void installMaterializationUnit(std::unique_ptr<MaterializationUnit> MU, + ResourceTracker &RT); + + void detachQueryHelper(AsynchronousSymbolQuery &Q, + const SymbolNameSet &QuerySymbols); + + void transferEmittedNodeDependencies(MaterializingInfo &DependantMI, + const SymbolStringPtr &DependantName, + MaterializingInfo &EmittedMI); + + Expected<SymbolFlagsMap> defineMaterializing(SymbolFlagsMap SymbolFlags); + + Error replace(MaterializationResponsibility &FromMR, + std::unique_ptr<MaterializationUnit> MU); + + Expected<std::unique_ptr<MaterializationResponsibility>> + delegate(MaterializationResponsibility &FromMR, SymbolFlagsMap SymbolFlags, + SymbolStringPtr InitSymbol); + + SymbolNameSet getRequestedSymbols(const SymbolFlagsMap &SymbolFlags) const; + + void addDependencies(const SymbolStringPtr &Name, + const SymbolDependenceMap &Dependants); + + Error resolve(MaterializationResponsibility &MR, const SymbolMap &Resolved); + + Error emit(MaterializationResponsibility &MR, const SymbolFlagsMap &Emitted); + + void unlinkMaterializationResponsibility(MaterializationResponsibility &MR); + + using FailedSymbolsWorklist = + std::vector<std::pair<JITDylib *, SymbolStringPtr>>; + + static std::pair<AsynchronousSymbolQuerySet, + std::shared_ptr<SymbolDependenceMap>> + failSymbols(FailedSymbolsWorklist); + + ExecutionSession &ES; + enum { Open, Closing, Closed } State = Open; + std::mutex GeneratorsMutex; + SymbolTable Symbols; + UnmaterializedInfosMap UnmaterializedInfos; + MaterializingInfosMap MaterializingInfos; + std::vector<std::shared_ptr<DefinitionGenerator>> DefGenerators; + JITDylibSearchOrder LinkOrder; + ResourceTrackerSP DefaultTracker; + + // Map trackers to sets of symbols tracked. + DenseMap<ResourceTracker *, SymbolNameVector> TrackerSymbols; + DenseMap<ResourceTracker *, DenseSet<MaterializationResponsibility *>> + TrackerMRs; +}; + +/// Platforms set up standard symbols and mediate interactions between dynamic +/// initializers (e.g. C++ static constructors) and ExecutionSession state. +/// Note that Platforms do not automatically run initializers: clients are still +/// responsible for doing this. +class Platform { +public: + virtual ~Platform(); + + /// This method will be called outside the session lock each time a JITDylib + /// is created (unless it is created with EmptyJITDylib set) to allow the + /// Platform to install any JITDylib specific standard symbols (e.g + /// __dso_handle). + virtual Error setupJITDylib(JITDylib &JD) = 0; + + /// This method will be called outside the session lock each time a JITDylib + /// is removed to allow the Platform to remove any JITDylib-specific data. + virtual Error teardownJITDylib(JITDylib &JD) = 0; + + /// This method will be called under the ExecutionSession lock each time a + /// MaterializationUnit is added to a JITDylib. + virtual Error notifyAdding(ResourceTracker &RT, + const MaterializationUnit &MU) = 0; + + /// This method will be called under the ExecutionSession lock when a + /// ResourceTracker is removed. + virtual Error notifyRemoving(ResourceTracker &RT) = 0; + + /// A utility function for looking up initializer symbols. Performs a blocking + /// lookup for the given symbols in each of the given JITDylibs. + /// + /// Note: This function is deprecated and will be removed in the near future. + static Expected<DenseMap<JITDylib *, SymbolMap>> + lookupInitSymbols(ExecutionSession &ES, + const DenseMap<JITDylib *, SymbolLookupSet> &InitSyms); + + /// Performs an async lookup for the the given symbols in each of the given + /// JITDylibs, calling the given handler once all lookups have completed. + static void + lookupInitSymbolsAsync(unique_function<void(Error)> OnComplete, + ExecutionSession &ES, + const DenseMap<JITDylib *, SymbolLookupSet> &InitSyms); +}; + +/// A materialization task. +class MaterializationTask : public RTTIExtends<MaterializationTask, Task> { +public: + static char ID; + + MaterializationTask(std::unique_ptr<MaterializationUnit> MU, + std::unique_ptr<MaterializationResponsibility> MR) + : MU(std::move(MU)), MR(std::move(MR)) {} + void printDescription(raw_ostream &OS) override; + void run() override; + +private: + std::unique_ptr<MaterializationUnit> MU; + std::unique_ptr<MaterializationResponsibility> MR; +}; + +/// An ExecutionSession represents a running JIT program. +class ExecutionSession { + friend class InProgressLookupFlagsState; + friend class InProgressFullLookupState; + friend class JITDylib; + friend class LookupState; + friend class MaterializationResponsibility; + friend class ResourceTracker; + +public: + /// For reporting errors. + using ErrorReporter = std::function<void(Error)>; + + /// Send a result to the remote. + using SendResultFunction = unique_function<void(shared::WrapperFunctionResult)>; + + /// For dispatching ORC tasks (typically materialization tasks). + using DispatchTaskFunction = unique_function<void(std::unique_ptr<Task> T)>; + + /// An asynchronous wrapper-function callable from the executor via + /// jit-dispatch. + using JITDispatchHandlerFunction = unique_function<void( + SendResultFunction SendResult, + const char *ArgData, size_t ArgSize)>; + + /// A map associating tag names with asynchronous wrapper function + /// implementations in the JIT. + using JITDispatchHandlerAssociationMap = + DenseMap<SymbolStringPtr, JITDispatchHandlerFunction>; + + /// Construct an ExecutionSession with the given ExecutorProcessControl + /// object. + ExecutionSession(std::unique_ptr<ExecutorProcessControl> EPC); + + /// End the session. Closes all JITDylibs and disconnects from the + /// executor. + Error endSession(); + + /// Get the ExecutorProcessControl object associated with this + /// ExecutionSession. + ExecutorProcessControl &getExecutorProcessControl() { return *EPC; } + + /// Get the SymbolStringPool for this instance. + std::shared_ptr<SymbolStringPool> getSymbolStringPool() { + return EPC->getSymbolStringPool(); + } + + /// Add a symbol name to the SymbolStringPool and return a pointer to it. + SymbolStringPtr intern(StringRef SymName) { return EPC->intern(SymName); } + + /// Set the Platform for this ExecutionSession. + void setPlatform(std::unique_ptr<Platform> P) { this->P = std::move(P); } + + /// Get the Platform for this session. + /// Will return null if no Platform has been set for this ExecutionSession. + Platform *getPlatform() { return P.get(); } + + /// Run the given lambda with the session mutex locked. + template <typename Func> decltype(auto) runSessionLocked(Func &&F) { + std::lock_guard<std::recursive_mutex> Lock(SessionMutex); + return F(); + } + + /// Register the given ResourceManager with this ExecutionSession. + /// Managers will be notified of events in reverse order of registration. + void registerResourceManager(ResourceManager &RM); + + /// Deregister the given ResourceManager with this ExecutionSession. + /// Manager must have been previously registered. + void deregisterResourceManager(ResourceManager &RM); + + /// Return a pointer to the "name" JITDylib. + /// Ownership of JITDylib remains within Execution Session + JITDylib *getJITDylibByName(StringRef Name); + + /// Add a new bare JITDylib to this ExecutionSession. + /// + /// The JITDylib Name is required to be unique. Clients should verify that + /// names are not being re-used (E.g. by calling getJITDylibByName) if names + /// are based on user input. + /// + /// This call does not install any library code or symbols into the newly + /// created JITDylib. The client is responsible for all configuration. + JITDylib &createBareJITDylib(std::string Name); + + /// Add a new JITDylib to this ExecutionSession. + /// + /// The JITDylib Name is required to be unique. Clients should verify that + /// names are not being re-used (e.g. by calling getJITDylibByName) if names + /// are based on user input. + /// + /// If a Platform is attached then Platform::setupJITDylib will be called to + /// install standard platform symbols (e.g. standard library interposes). + /// If no Platform is attached this call is equivalent to createBareJITDylib. + Expected<JITDylib &> createJITDylib(std::string Name); + + /// Closes the given JITDylib. + /// + /// This method clears all resources held for the JITDylib, puts it in the + /// closed state, and clears all references held by the ExecutionSession and + /// other JITDylibs. No further code can be added to the JITDylib, and the + /// object will be freed once any remaining JITDylibSPs to it are destroyed. + /// + /// This method does *not* run static destructors. + /// + /// This method can only be called once for each JITDylib. + Error removeJITDylib(JITDylib &JD); + + /// Set the error reporter function. + ExecutionSession &setErrorReporter(ErrorReporter ReportError) { + this->ReportError = std::move(ReportError); + return *this; + } + + /// Report a error for this execution session. + /// + /// Unhandled errors can be sent here to log them. + void reportError(Error Err) { ReportError(std::move(Err)); } + + /// Set the task dispatch function. + ExecutionSession &setDispatchTask(DispatchTaskFunction DispatchTask) { + this->DispatchTask = std::move(DispatchTask); + return *this; + } + + /// Search the given JITDylibs to find the flags associated with each of the + /// given symbols. + void lookupFlags(LookupKind K, JITDylibSearchOrder SearchOrder, + SymbolLookupSet Symbols, + unique_function<void(Expected<SymbolFlagsMap>)> OnComplete); + + /// Blocking version of lookupFlags. + Expected<SymbolFlagsMap> lookupFlags(LookupKind K, + JITDylibSearchOrder SearchOrder, + SymbolLookupSet Symbols); + + /// Search the given JITDylibs for the given symbols. + /// + /// SearchOrder lists the JITDylibs to search. For each dylib, the associated + /// boolean indicates whether the search should match against non-exported + /// (hidden visibility) symbols in that dylib (true means match against + /// non-exported symbols, false means do not match). + /// + /// The NotifyComplete callback will be called once all requested symbols + /// reach the required state. + /// + /// If all symbols are found, the RegisterDependencies function will be called + /// while the session lock is held. This gives clients a chance to register + /// dependencies for on the queried symbols for any symbols they are + /// materializing (if a MaterializationResponsibility instance is present, + /// this can be implemented by calling + /// MaterializationResponsibility::addDependencies). If there are no + /// dependenant symbols for this query (e.g. it is being made by a top level + /// client to get an address to call) then the value NoDependenciesToRegister + /// can be used. + void lookup(LookupKind K, const JITDylibSearchOrder &SearchOrder, + SymbolLookupSet Symbols, SymbolState RequiredState, + SymbolsResolvedCallback NotifyComplete, + RegisterDependenciesFunction RegisterDependencies); + + /// Blocking version of lookup above. Returns the resolved symbol map. + /// If WaitUntilReady is true (the default), will not return until all + /// requested symbols are ready (or an error occurs). If WaitUntilReady is + /// false, will return as soon as all requested symbols are resolved, + /// or an error occurs. If WaitUntilReady is false and an error occurs + /// after resolution, the function will return a success value, but the + /// error will be reported via reportErrors. + Expected<SymbolMap> lookup(const JITDylibSearchOrder &SearchOrder, + const SymbolLookupSet &Symbols, + LookupKind K = LookupKind::Static, + SymbolState RequiredState = SymbolState::Ready, + RegisterDependenciesFunction RegisterDependencies = + NoDependenciesToRegister); + + /// Convenience version of blocking lookup. + /// Searches each of the JITDylibs in the search order in turn for the given + /// symbol. + Expected<JITEvaluatedSymbol> + lookup(const JITDylibSearchOrder &SearchOrder, SymbolStringPtr Symbol, + SymbolState RequiredState = SymbolState::Ready); + + /// Convenience version of blocking lookup. + /// Searches each of the JITDylibs in the search order in turn for the given + /// symbol. The search will not find non-exported symbols. + Expected<JITEvaluatedSymbol> + lookup(ArrayRef<JITDylib *> SearchOrder, SymbolStringPtr Symbol, + SymbolState RequiredState = SymbolState::Ready); + + /// Convenience version of blocking lookup. + /// Searches each of the JITDylibs in the search order in turn for the given + /// symbol. The search will not find non-exported symbols. + Expected<JITEvaluatedSymbol> + lookup(ArrayRef<JITDylib *> SearchOrder, StringRef Symbol, + SymbolState RequiredState = SymbolState::Ready); + + /// Materialize the given unit. + void dispatchTask(std::unique_ptr<Task> T) { + assert(T && "T must be non-null"); + DEBUG_WITH_TYPE("orc", dumpDispatchInfo(*T)); + DispatchTask(std::move(T)); + } + + /// Run a wrapper function in the executor. + /// + /// The wrapper function should be callable as: + /// + /// \code{.cpp} + /// CWrapperFunctionResult fn(uint8_t *Data, uint64_t Size); + /// \endcode{.cpp} + /// + /// The given OnComplete function will be called to return the result. + template <typename... ArgTs> + void callWrapperAsync(ArgTs &&... Args) { + EPC->callWrapperAsync(std::forward<ArgTs>(Args)...); + } + + /// Run a wrapper function in the executor. The wrapper function should be + /// callable as: + /// + /// \code{.cpp} + /// CWrapperFunctionResult fn(uint8_t *Data, uint64_t Size); + /// \endcode{.cpp} + shared::WrapperFunctionResult callWrapper(ExecutorAddr WrapperFnAddr, + ArrayRef<char> ArgBuffer) { + return EPC->callWrapper(WrapperFnAddr, ArgBuffer); + } + + /// Run a wrapper function using SPS to serialize the arguments and + /// deserialize the results. + template <typename SPSSignature, typename SendResultT, typename... ArgTs> + void callSPSWrapperAsync(ExecutorAddr WrapperFnAddr, SendResultT &&SendResult, + const ArgTs &...Args) { + EPC->callSPSWrapperAsync<SPSSignature, SendResultT, ArgTs...>( + WrapperFnAddr, std::forward<SendResultT>(SendResult), Args...); + } + + /// Run a wrapper function using SPS to serialize the arguments and + /// deserialize the results. + /// + /// If SPSSignature is a non-void function signature then the second argument + /// (the first in the Args list) should be a reference to a return value. + template <typename SPSSignature, typename... WrapperCallArgTs> + Error callSPSWrapper(ExecutorAddr WrapperFnAddr, + WrapperCallArgTs &&...WrapperCallArgs) { + return EPC->callSPSWrapper<SPSSignature, WrapperCallArgTs...>( + WrapperFnAddr, std::forward<WrapperCallArgTs>(WrapperCallArgs)...); + } + + /// Wrap a handler that takes concrete argument types (and a sender for a + /// concrete return type) to produce an AsyncHandlerWrapperFunction. Uses SPS + /// to unpack the arguments and pack the result. + /// + /// This function is intended to support easy construction of + /// AsyncHandlerWrapperFunctions that can be associated with a tag + /// (using registerJITDispatchHandler) and called from the executor. + template <typename SPSSignature, typename HandlerT> + static JITDispatchHandlerFunction wrapAsyncWithSPS(HandlerT &&H) { + return [H = std::forward<HandlerT>(H)]( + SendResultFunction SendResult, + const char *ArgData, size_t ArgSize) mutable { + shared::WrapperFunction<SPSSignature>::handleAsync(ArgData, ArgSize, H, + std::move(SendResult)); + }; + } + + /// Wrap a class method that takes concrete argument types (and a sender for + /// a concrete return type) to produce an AsyncHandlerWrapperFunction. Uses + /// SPS to unpack teh arguments and pack the result. + /// + /// This function is intended to support easy construction of + /// AsyncHandlerWrapperFunctions that can be associated with a tag + /// (using registerJITDispatchHandler) and called from the executor. + template <typename SPSSignature, typename ClassT, typename... MethodArgTs> + static JITDispatchHandlerFunction + wrapAsyncWithSPS(ClassT *Instance, void (ClassT::*Method)(MethodArgTs...)) { + return wrapAsyncWithSPS<SPSSignature>( + [Instance, Method](MethodArgTs &&...MethodArgs) { + (Instance->*Method)(std::forward<MethodArgTs>(MethodArgs)...); + }); + } + + /// For each tag symbol name, associate the corresponding + /// AsyncHandlerWrapperFunction with the address of that symbol. The + /// handler becomes callable from the executor using the ORC runtime + /// __orc_rt_jit_dispatch function and the given tag. + /// + /// Tag symbols will be looked up in JD using LookupKind::Static, + /// JITDylibLookupFlags::MatchAllSymbols (hidden tags will be found), and + /// LookupFlags::WeaklyReferencedSymbol. Missing tag definitions will not + /// cause an error, the handler will simply be dropped. + Error registerJITDispatchHandlers(JITDylib &JD, + JITDispatchHandlerAssociationMap WFs); + + /// Run a registered jit-side wrapper function. + /// This should be called by the ExecutorProcessControl instance in response + /// to incoming jit-dispatch requests from the executor. + void + runJITDispatchHandler(SendResultFunction SendResult, + JITTargetAddress HandlerFnTagAddr, + ArrayRef<char> ArgBuffer); + + /// Dump the state of all the JITDylibs in this session. + void dump(raw_ostream &OS); + +private: + static void logErrorsToStdErr(Error Err) { + logAllUnhandledErrors(std::move(Err), errs(), "JIT session error: "); + } + + static void runOnCurrentThread(std::unique_ptr<Task> T) { T->run(); } + + void dispatchOutstandingMUs(); + + static std::unique_ptr<MaterializationResponsibility> + createMaterializationResponsibility(ResourceTracker &RT, + SymbolFlagsMap Symbols, + SymbolStringPtr InitSymbol) { + auto &JD = RT.getJITDylib(); + std::unique_ptr<MaterializationResponsibility> MR( + new MaterializationResponsibility(&RT, std::move(Symbols), + std::move(InitSymbol))); + JD.TrackerMRs[&RT].insert(MR.get()); + return MR; + } + + Error removeResourceTracker(ResourceTracker &RT); + void transferResourceTracker(ResourceTracker &DstRT, ResourceTracker &SrcRT); + void destroyResourceTracker(ResourceTracker &RT); + + // State machine functions for query application.. + + /// IL_updateCandidatesFor is called to remove already-defined symbols that + /// match a given query from the set of candidate symbols to generate + /// definitions for (no need to generate a definition if one already exists). + Error IL_updateCandidatesFor(JITDylib &JD, JITDylibLookupFlags JDLookupFlags, + SymbolLookupSet &Candidates, + SymbolLookupSet *NonCandidates); + + /// OL_applyQueryPhase1 is an optionally re-startable loop for triggering + /// definition generation. It is called when a lookup is performed, and again + /// each time that LookupState::continueLookup is called. + void OL_applyQueryPhase1(std::unique_ptr<InProgressLookupState> IPLS, + Error Err); + + /// OL_completeLookup is run once phase 1 successfully completes for a lookup + /// call. It attempts to attach the symbol to all symbol table entries and + /// collect all MaterializationUnits to dispatch. If this method fails then + /// all MaterializationUnits will be left un-materialized. + void OL_completeLookup(std::unique_ptr<InProgressLookupState> IPLS, + std::shared_ptr<AsynchronousSymbolQuery> Q, + RegisterDependenciesFunction RegisterDependencies); + + /// OL_completeLookupFlags is run once phase 1 successfully completes for a + /// lookupFlags call. + void OL_completeLookupFlags( + std::unique_ptr<InProgressLookupState> IPLS, + unique_function<void(Expected<SymbolFlagsMap>)> OnComplete); + + // State machine functions for MaterializationResponsibility. + void OL_destroyMaterializationResponsibility( + MaterializationResponsibility &MR); + SymbolNameSet OL_getRequestedSymbols(const MaterializationResponsibility &MR); + Error OL_notifyResolved(MaterializationResponsibility &MR, + const SymbolMap &Symbols); + Error OL_notifyEmitted(MaterializationResponsibility &MR); + Error OL_defineMaterializing(MaterializationResponsibility &MR, + SymbolFlagsMap SymbolFlags); + void OL_notifyFailed(MaterializationResponsibility &MR); + Error OL_replace(MaterializationResponsibility &MR, + std::unique_ptr<MaterializationUnit> MU); + Expected<std::unique_ptr<MaterializationResponsibility>> + OL_delegate(MaterializationResponsibility &MR, const SymbolNameSet &Symbols); + void OL_addDependencies(MaterializationResponsibility &MR, + const SymbolStringPtr &Name, + const SymbolDependenceMap &Dependencies); + void OL_addDependenciesForAll(MaterializationResponsibility &MR, + const SymbolDependenceMap &Dependencies); + +#ifndef NDEBUG + void dumpDispatchInfo(Task &T); +#endif // NDEBUG + + mutable std::recursive_mutex SessionMutex; + bool SessionOpen = true; + std::unique_ptr<ExecutorProcessControl> EPC; + std::unique_ptr<Platform> P; + ErrorReporter ReportError = logErrorsToStdErr; + DispatchTaskFunction DispatchTask = runOnCurrentThread; + + std::vector<ResourceManager *> ResourceManagers; + + std::vector<JITDylibSP> JDs; + + // FIXME: Remove this (and runOutstandingMUs) once the linking layer works + // with callbacks from asynchronous queries. + mutable std::recursive_mutex OutstandingMUsMutex; + std::vector<std::pair<std::unique_ptr<MaterializationUnit>, + std::unique_ptr<MaterializationResponsibility>>> + OutstandingMUs; + + mutable std::mutex JITDispatchHandlersMutex; + DenseMap<JITTargetAddress, std::shared_ptr<JITDispatchHandlerFunction>> + JITDispatchHandlers; +}; + +inline ExecutionSession & +MaterializationResponsibility::getExecutionSession() const { + return JD.getExecutionSession(); +} + +template <typename Func> +Error MaterializationResponsibility::withResourceKeyDo(Func &&F) const { + return JD.getExecutionSession().runSessionLocked([&]() -> Error { + if (RT->isDefunct()) + return make_error<ResourceTrackerDefunct>(RT); + F(RT->getKeyUnsafe()); + return Error::success(); + }); +} + +template <typename GeneratorT> +GeneratorT &JITDylib::addGenerator(std::unique_ptr<GeneratorT> DefGenerator) { + auto &G = *DefGenerator; + ES.runSessionLocked([&] { + assert(State == Open && "Cannot add generator to closed JITDylib"); + DefGenerators.push_back(std::move(DefGenerator)); + }); + return G; +} + +template <typename Func> +auto JITDylib::withLinkOrderDo(Func &&F) + -> decltype(F(std::declval<const JITDylibSearchOrder &>())) { + assert(State == Open && "Cannot use link order of closed JITDylib"); + return ES.runSessionLocked([&]() { return F(LinkOrder); }); +} + +template <typename MaterializationUnitType> +Error JITDylib::define(std::unique_ptr<MaterializationUnitType> &&MU, + ResourceTrackerSP RT) { + assert(MU && "Can not define with a null MU"); + + if (MU->getSymbols().empty()) { + // Empty MUs are allowable but pathological, so issue a warning. + DEBUG_WITH_TYPE("orc", { + dbgs() << "Warning: Discarding empty MU " << MU->getName() << " for " + << getName() << "\n"; + }); + return Error::success(); + } else + DEBUG_WITH_TYPE("orc", { + dbgs() << "Defining MU " << MU->getName() << " for " << getName() + << " (tracker: "; + if (RT == getDefaultResourceTracker()) + dbgs() << "default)"; + else if (RT) + dbgs() << RT.get() << ")\n"; + else + dbgs() << "0x0, default will be used)\n"; + }); + + return ES.runSessionLocked([&, this]() -> Error { + assert(State == Open && "JD is defunct"); + + if (auto Err = defineImpl(*MU)) + return Err; + + if (!RT) + RT = getDefaultResourceTracker(); + + if (auto *P = ES.getPlatform()) { + if (auto Err = P->notifyAdding(*RT, *MU)) + return Err; + } + + installMaterializationUnit(std::move(MU), *RT); + return Error::success(); + }); +} + +template <typename MaterializationUnitType> +Error JITDylib::define(std::unique_ptr<MaterializationUnitType> &MU, + ResourceTrackerSP RT) { + assert(MU && "Can not define with a null MU"); + + if (MU->getSymbols().empty()) { + // Empty MUs are allowable but pathological, so issue a warning. + DEBUG_WITH_TYPE("orc", { + dbgs() << "Warning: Discarding empty MU " << MU->getName() << getName() + << "\n"; + }); + return Error::success(); + } else + DEBUG_WITH_TYPE("orc", { + dbgs() << "Defining MU " << MU->getName() << " for " << getName() + << " (tracker: "; + if (RT == getDefaultResourceTracker()) + dbgs() << "default)"; + else if (RT) + dbgs() << RT.get() << ")\n"; + else + dbgs() << "0x0, default will be used)\n"; + }); + + return ES.runSessionLocked([&, this]() -> Error { + assert(State == Open && "JD is defunct"); + + if (auto Err = defineImpl(*MU)) + return Err; + + if (!RT) + RT = getDefaultResourceTracker(); + + if (auto *P = ES.getPlatform()) { + if (auto Err = P->notifyAdding(*RT, *MU)) + return Err; + } + + installMaterializationUnit(std::move(MU), *RT); + return Error::success(); + }); +} + +/// ReexportsGenerator can be used with JITDylib::addGenerator to automatically +/// re-export a subset of the source JITDylib's symbols in the target. +class ReexportsGenerator : public DefinitionGenerator { +public: + using SymbolPredicate = std::function<bool(SymbolStringPtr)>; + + /// Create a reexports generator. If an Allow predicate is passed, only + /// symbols for which the predicate returns true will be reexported. If no + /// Allow predicate is passed, all symbols will be exported. + ReexportsGenerator(JITDylib &SourceJD, + JITDylibLookupFlags SourceJDLookupFlags, + SymbolPredicate Allow = SymbolPredicate()); + + Error tryToGenerate(LookupState &LS, LookupKind K, JITDylib &JD, + JITDylibLookupFlags JDLookupFlags, + const SymbolLookupSet &LookupSet) override; + +private: + JITDylib &SourceJD; + JITDylibLookupFlags SourceJDLookupFlags; + SymbolPredicate Allow; +}; + +// --------------- IMPLEMENTATION -------------- +// Implementations for inline functions/methods. +// --------------------------------------------- + +inline MaterializationResponsibility::~MaterializationResponsibility() { + getExecutionSession().OL_destroyMaterializationResponsibility(*this); +} + +inline SymbolNameSet MaterializationResponsibility::getRequestedSymbols() const { + return getExecutionSession().OL_getRequestedSymbols(*this); +} + +inline Error MaterializationResponsibility::notifyResolved( + const SymbolMap &Symbols) { + return getExecutionSession().OL_notifyResolved(*this, Symbols); +} + +inline Error MaterializationResponsibility::notifyEmitted() { + return getExecutionSession().OL_notifyEmitted(*this); +} + +inline Error MaterializationResponsibility::defineMaterializing( + SymbolFlagsMap SymbolFlags) { + return getExecutionSession().OL_defineMaterializing(*this, + std::move(SymbolFlags)); +} + +inline void MaterializationResponsibility::failMaterialization() { + getExecutionSession().OL_notifyFailed(*this); +} + +inline Error MaterializationResponsibility::replace( + std::unique_ptr<MaterializationUnit> MU) { + return getExecutionSession().OL_replace(*this, std::move(MU)); +} + +inline Expected<std::unique_ptr<MaterializationResponsibility>> +MaterializationResponsibility::delegate(const SymbolNameSet &Symbols) { + return getExecutionSession().OL_delegate(*this, Symbols); +} + +inline void MaterializationResponsibility::addDependencies( + const SymbolStringPtr &Name, const SymbolDependenceMap &Dependencies) { + getExecutionSession().OL_addDependencies(*this, Name, Dependencies); +} + +inline void MaterializationResponsibility::addDependenciesForAll( + const SymbolDependenceMap &Dependencies) { + getExecutionSession().OL_addDependenciesForAll(*this, Dependencies); +} + +} // End namespace orc +} // End namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_CORE_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/DebugObjectManagerPlugin.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/DebugObjectManagerPlugin.h new file mode 100644 index 0000000000..356113bba4 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/DebugObjectManagerPlugin.h @@ -0,0 +1,96 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===---- DebugObjectManagerPlugin.h - JITLink debug objects ---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// ObjectLinkingLayer plugin for emitting debug objects. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_DEBUGOBJECTMANAGERPLUGIN_H +#define LLVM_EXECUTIONENGINE_ORC_DEBUGOBJECTMANAGERPLUGIN_H + +#include "llvm/ADT/Triple.h" +#include "llvm/ExecutionEngine/JITLink/JITLink.h" +#include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/EPCDebugObjectRegistrar.h" +#include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/Memory.h" +#include "llvm/Support/MemoryBufferRef.h" + +#include <functional> +#include <map> +#include <memory> +#include <mutex> + +namespace llvm { +namespace orc { + +class DebugObject; + +/// Creates and manages DebugObjects for JITLink artifacts. +/// +/// DebugObjects are created when linking for a MaterializationResponsibility +/// starts. They are pending as long as materialization is in progress. +/// +/// There can only be one pending DebugObject per MaterializationResponsibility. +/// If materialization fails, pending DebugObjects are discarded. +/// +/// Once executable code for the MaterializationResponsibility is emitted, the +/// corresponding DebugObject is finalized to target memory and the provided +/// DebugObjectRegistrar is notified. Ownership of DebugObjects remains with the +/// plugin. +/// +class DebugObjectManagerPlugin : public ObjectLinkingLayer::Plugin { +public: + DebugObjectManagerPlugin(ExecutionSession &ES, + std::unique_ptr<DebugObjectRegistrar> Target); + ~DebugObjectManagerPlugin(); + + void notifyMaterializing(MaterializationResponsibility &MR, + jitlink::LinkGraph &G, jitlink::JITLinkContext &Ctx, + MemoryBufferRef InputObject) override; + + Error notifyEmitted(MaterializationResponsibility &MR) override; + Error notifyFailed(MaterializationResponsibility &MR) override; + Error notifyRemovingResources(ResourceKey K) override; + + void notifyTransferringResources(ResourceKey DstKey, + ResourceKey SrcKey) override; + + void modifyPassConfig(MaterializationResponsibility &MR, + jitlink::LinkGraph &LG, + jitlink::PassConfiguration &PassConfig) override; + +private: + ExecutionSession &ES; + + using OwnedDebugObject = std::unique_ptr<DebugObject>; + std::map<MaterializationResponsibility *, OwnedDebugObject> PendingObjs; + std::map<ResourceKey, std::vector<OwnedDebugObject>> RegisteredObjs; + + std::mutex PendingObjsLock; + std::mutex RegisteredObjsLock; + + std::unique_ptr<DebugObjectRegistrar> Target; +}; + +} // namespace orc +} // namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_DEBUGOBJECTMANAGERPLUGIN_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/DebugUtils.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/DebugUtils.h new file mode 100644 index 0000000000..761aa2a75c --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/DebugUtils.h @@ -0,0 +1,138 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===----- DebugUtils.h - Utilities for debugging ORC JITs ------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Utilities for debugging ORC-based JITs. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_DEBUGUTILS_H +#define LLVM_EXECUTIONENGINE_ORC_DEBUGUTILS_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/SymbolStringPool.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/raw_ostream.h" +#include <memory> +#include <string> + +namespace llvm { + +class MemoryBuffer; + +namespace orc { + +// --raw_ostream operators for ORC types-- + +/// Render a SymbolStringPtr. +raw_ostream &operator<<(raw_ostream &OS, const SymbolStringPtr &Sym); + +/// Render a SymbolNameSet. +raw_ostream &operator<<(raw_ostream &OS, const SymbolNameSet &Symbols); + +/// Render a SymbolNameVector. +raw_ostream &operator<<(raw_ostream &OS, const SymbolNameVector &Symbols); + +/// Render an array of SymbolStringPtrs. +raw_ostream &operator<<(raw_ostream &OS, ArrayRef<SymbolStringPtr> Symbols); + +/// Render JITSymbolFlags. +raw_ostream &operator<<(raw_ostream &OS, const JITSymbolFlags &Flags); + +/// Render a SymbolFlagsMap entry. +raw_ostream &operator<<(raw_ostream &OS, const SymbolFlagsMap::value_type &KV); + +/// Render a SymbolMap entry. +raw_ostream &operator<<(raw_ostream &OS, const SymbolMap::value_type &KV); + +/// Render a SymbolFlagsMap. +raw_ostream &operator<<(raw_ostream &OS, const SymbolFlagsMap &SymbolFlags); + +/// Render a SymbolMap. +raw_ostream &operator<<(raw_ostream &OS, const SymbolMap &Symbols); + +/// Render a SymbolDependenceMap entry. +raw_ostream &operator<<(raw_ostream &OS, + const SymbolDependenceMap::value_type &KV); + +/// Render a SymbolDependendeMap. +raw_ostream &operator<<(raw_ostream &OS, const SymbolDependenceMap &Deps); + +/// Render a MaterializationUnit. +raw_ostream &operator<<(raw_ostream &OS, const MaterializationUnit &MU); + +//// Render a JITDylibLookupFlags instance. +raw_ostream &operator<<(raw_ostream &OS, + const JITDylibLookupFlags &JDLookupFlags); + +/// Rendar a SymbolLookupFlags instance. +raw_ostream &operator<<(raw_ostream &OS, const SymbolLookupFlags &LookupFlags); + +/// Render a SymbolLookupSet entry. +raw_ostream &operator<<(raw_ostream &OS, const SymbolLookupSet::value_type &KV); + +/// Render a SymbolLookupSet. +raw_ostream &operator<<(raw_ostream &OS, const SymbolLookupSet &LookupSet); + +/// Render a JITDylibSearchOrder. +raw_ostream &operator<<(raw_ostream &OS, + const JITDylibSearchOrder &SearchOrder); + +/// Render a SymbolAliasMap. +raw_ostream &operator<<(raw_ostream &OS, const SymbolAliasMap &Aliases); + +/// Render a SymbolState. +raw_ostream &operator<<(raw_ostream &OS, const SymbolState &S); + +/// Render a LookupKind. +raw_ostream &operator<<(raw_ostream &OS, const LookupKind &K); + +/// A function object that can be used as an ObjectTransformLayer transform +/// to dump object files to disk at a specified path. +class DumpObjects { +public: + /// Construct a DumpObjects transform that will dump objects to disk. + /// + /// @param DumpDir specifies the path to write dumped objects to. DumpDir may + /// be empty, in which case files will be dumped to the working directory. If + /// DumpDir is non-empty then any trailing separators will be discarded. + /// + /// @param IdentifierOverride specifies a file name stem to use when dumping + /// objects. If empty, each MemoryBuffer's identifier will be used (with a .o + /// suffix added if not already present). If an identifier override is + /// supplied it will be used instead (since all buffers will use the same + /// identifier, the resulting files will be named <ident>.o, <ident>.2.o, + /// <ident>.3.o, and so on). IdentifierOverride should not contain an + /// extension, as a .o suffix will be added by DumpObjects. + DumpObjects(std::string DumpDir = "", std::string IdentifierOverride = ""); + + /// Dumps the given buffer to disk. + Expected<std::unique_ptr<MemoryBuffer>> + operator()(std::unique_ptr<MemoryBuffer> Obj); + +private: + StringRef getBufferIdentifier(MemoryBuffer &B); + std::string DumpDir; + std::string IdentifierOverride; +}; + +} // End namespace orc +} // End namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_DEBUGUTILS_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/DebuggerSupportPlugin.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/DebuggerSupportPlugin.h new file mode 100644 index 0000000000..a83951242c --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/DebuggerSupportPlugin.h @@ -0,0 +1,75 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===--- DebugerSupportPlugin.h -- Utils for debugger support ---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Generates debug objects and registers them using the jit-loader-gdb protocol. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_DEBUGGERSUPPORTPLUGIN_H +#define LLVM_EXECUTIONENGINE_ORC_DEBUGGERSUPPORTPLUGIN_H + +#include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/EPCDebugObjectRegistrar.h" +#include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h" + +namespace llvm { +namespace orc { + +/// For each object containing debug info, installs JITLink passes to synthesize +/// a debug object and then register it via the GDB JIT-registration interface. +/// +/// Currently MachO only. For ELF use DebugObjectManagerPlugin. These two +/// plugins will be merged in the near future. +class GDBJITDebugInfoRegistrationPlugin : public ObjectLinkingLayer::Plugin { +public: + class DebugSectionSynthesizer { + public: + virtual ~DebugSectionSynthesizer() = default; + virtual Error startSynthesis() = 0; + virtual Error completeSynthesisAndRegister() = 0; + }; + + static Expected<std::unique_ptr<GDBJITDebugInfoRegistrationPlugin>> + Create(ExecutionSession &ES, JITDylib &ProcessJD, const Triple &TT); + + GDBJITDebugInfoRegistrationPlugin(ExecutorAddr RegisterActionAddr) + : RegisterActionAddr(RegisterActionAddr) {} + + Error notifyFailed(MaterializationResponsibility &MR) override; + Error notifyRemovingResources(ResourceKey K) override; + + void notifyTransferringResources(ResourceKey DstKey, + ResourceKey SrcKey) override; + + void modifyPassConfig(MaterializationResponsibility &MR, + jitlink::LinkGraph &LG, + jitlink::PassConfiguration &PassConfig) override; + +private: + void modifyPassConfigForMachO(MaterializationResponsibility &MR, + jitlink::LinkGraph &LG, + jitlink::PassConfiguration &PassConfig); + + ExecutorAddr RegisterActionAddr; +}; + +} // namespace orc +} // namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_DEBUGGERSUPPORTPLUGIN_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/ELFNixPlatform.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/ELFNixPlatform.h new file mode 100644 index 0000000000..8aa8ef1ae6 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/ELFNixPlatform.h @@ -0,0 +1,342 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===-- ELFNixPlatform.h -- Utilities for executing ELF in Orc --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Linux/BSD support for executing JIT'd ELF in Orc. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_ELFNIXPLATFORM_H +#define LLVM_EXECUTIONENGINE_ORC_ELFNIXPLATFORM_H + +#include "llvm/ADT/StringRef.h" +#include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h" +#include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h" +#include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h" + +#include <future> +#include <thread> +#include <vector> + +namespace llvm { +namespace orc { + +struct ELFPerObjectSectionsToRegister { + ExecutorAddrRange EHFrameSection; + ExecutorAddrRange ThreadDataSection; +}; + +struct ELFNixJITDylibInitializers { + using SectionList = std::vector<ExecutorAddrRange>; + + ELFNixJITDylibInitializers(std::string Name, ExecutorAddr DSOHandleAddress) + : Name(std::move(Name)), DSOHandleAddress(std::move(DSOHandleAddress)) {} + + std::string Name; + ExecutorAddr DSOHandleAddress; + + StringMap<SectionList> InitSections; +}; + +class ELFNixJITDylibDeinitializers {}; + +using ELFNixJITDylibInitializerSequence = + std::vector<ELFNixJITDylibInitializers>; + +using ELFNixJITDylibDeinitializerSequence = + std::vector<ELFNixJITDylibDeinitializers>; + +/// Mediates between ELFNix initialization and ExecutionSession state. +class ELFNixPlatform : public Platform { +public: + /// Try to create a ELFNixPlatform instance, adding the ORC runtime to the + /// given JITDylib. + /// + /// The ORC runtime requires access to a number of symbols in + /// libc++. It is up to the caller to ensure that the requried + /// symbols can be referenced by code added to PlatformJD. The + /// standard way to achieve this is to first attach dynamic library + /// search generators for either the given process, or for the + /// specific required libraries, to PlatformJD, then to create the + /// platform instance: + /// + /// \code{.cpp} + /// auto &PlatformJD = ES.createBareJITDylib("stdlib"); + /// PlatformJD.addGenerator( + /// ExitOnErr(EPCDynamicLibrarySearchGenerator + /// ::GetForTargetProcess(EPC))); + /// ES.setPlatform( + /// ExitOnErr(ELFNixPlatform::Create(ES, ObjLayer, EPC, PlatformJD, + /// "/path/to/orc/runtime"))); + /// \endcode + /// + /// Alternatively, these symbols could be added to another JITDylib that + /// PlatformJD links against. + /// + /// Clients are also responsible for ensuring that any JIT'd code that + /// depends on runtime functions (including any code using TLV or static + /// destructors) can reference the runtime symbols. This is usually achieved + /// by linking any JITDylibs containing regular code against + /// PlatformJD. + /// + /// By default, ELFNixPlatform will add the set of aliases returned by the + /// standardPlatformAliases function. This includes both required aliases + /// (e.g. __cxa_atexit -> __orc_rt_elf_cxa_atexit for static destructor + /// support), and optional aliases that provide JIT versions of common + /// functions (e.g. dlopen -> __orc_rt_elf_jit_dlopen). Clients can + /// override these defaults by passing a non-None value for the + /// RuntimeAliases function, in which case the client is responsible for + /// setting up all aliases (including the required ones). + static Expected<std::unique_ptr<ELFNixPlatform>> + Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, + JITDylib &PlatformJD, const char *OrcRuntimePath, + Optional<SymbolAliasMap> RuntimeAliases = None); + + ExecutionSession &getExecutionSession() const { return ES; } + ObjectLinkingLayer &getObjectLinkingLayer() const { return ObjLinkingLayer; } + + Error setupJITDylib(JITDylib &JD) override; + Error teardownJITDylib(JITDylib &JD) override; + Error notifyAdding(ResourceTracker &RT, + const MaterializationUnit &MU) override; + Error notifyRemoving(ResourceTracker &RT) override; + + /// Returns an AliasMap containing the default aliases for the ELFNixPlatform. + /// This can be modified by clients when constructing the platform to add + /// or remove aliases. + static SymbolAliasMap standardPlatformAliases(ExecutionSession &ES); + + /// Returns the array of required CXX aliases. + static ArrayRef<std::pair<const char *, const char *>> requiredCXXAliases(); + + /// Returns the array of standard runtime utility aliases for ELF. + static ArrayRef<std::pair<const char *, const char *>> + standardRuntimeUtilityAliases(); + + /// Returns true if the given section name is an initializer section. + static bool isInitializerSection(StringRef SecName); + +private: + // The ELFNixPlatformPlugin scans/modifies LinkGraphs to support ELF + // platform features including initializers, exceptions, TLV, and language + // runtime registration. + class ELFNixPlatformPlugin : public ObjectLinkingLayer::Plugin { + public: + ELFNixPlatformPlugin(ELFNixPlatform &MP) : MP(MP) {} + + void modifyPassConfig(MaterializationResponsibility &MR, + jitlink::LinkGraph &G, + jitlink::PassConfiguration &Config) override; + + SyntheticSymbolDependenciesMap + getSyntheticSymbolDependencies(MaterializationResponsibility &MR) override; + + // FIXME: We should be tentatively tracking scraped sections and discarding + // if the MR fails. + Error notifyFailed(MaterializationResponsibility &MR) override { + return Error::success(); + } + + Error notifyRemovingResources(ResourceKey K) override { + return Error::success(); + } + + void notifyTransferringResources(ResourceKey DstKey, + ResourceKey SrcKey) override {} + + private: + using InitSymbolDepMap = + DenseMap<MaterializationResponsibility *, JITLinkSymbolSet>; + + void addInitializerSupportPasses(MaterializationResponsibility &MR, + jitlink::PassConfiguration &Config); + + void addDSOHandleSupportPasses(MaterializationResponsibility &MR, + jitlink::PassConfiguration &Config); + + void addEHAndTLVSupportPasses(MaterializationResponsibility &MR, + jitlink::PassConfiguration &Config); + + Error preserveInitSections(jitlink::LinkGraph &G, + MaterializationResponsibility &MR); + + Error registerInitSections(jitlink::LinkGraph &G, JITDylib &JD); + + Error fixTLVSectionsAndEdges(jitlink::LinkGraph &G, JITDylib &JD); + + std::mutex PluginMutex; + ELFNixPlatform &MP; + InitSymbolDepMap InitSymbolDeps; + }; + + using SendInitializerSequenceFn = + unique_function<void(Expected<ELFNixJITDylibInitializerSequence>)>; + + using SendDeinitializerSequenceFn = + unique_function<void(Expected<ELFNixJITDylibDeinitializerSequence>)>; + + using SendSymbolAddressFn = unique_function<void(Expected<ExecutorAddr>)>; + + static bool supportedTarget(const Triple &TT); + + ELFNixPlatform(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, + JITDylib &PlatformJD, + std::unique_ptr<DefinitionGenerator> OrcRuntimeGenerator, + Error &Err); + + // Associate ELFNixPlatform JIT-side runtime support functions with handlers. + Error associateRuntimeSupportFunctions(JITDylib &PlatformJD); + + void getInitializersBuildSequencePhase(SendInitializerSequenceFn SendResult, + JITDylib &JD, + std::vector<JITDylibSP> DFSLinkOrder); + + void getInitializersLookupPhase(SendInitializerSequenceFn SendResult, + JITDylib &JD); + + void rt_getInitializers(SendInitializerSequenceFn SendResult, + StringRef JDName); + + void rt_getDeinitializers(SendDeinitializerSequenceFn SendResult, + ExecutorAddr Handle); + + void rt_lookupSymbol(SendSymbolAddressFn SendResult, ExecutorAddr Handle, + StringRef SymbolName); + + // Records the addresses of runtime symbols used by the platform. + Error bootstrapELFNixRuntime(JITDylib &PlatformJD); + + Error registerInitInfo(JITDylib &JD, + ArrayRef<jitlink::Section *> InitSections); + + Error registerPerObjectSections(const ELFPerObjectSectionsToRegister &POSR); + + Expected<uint64_t> createPThreadKey(); + + ExecutionSession &ES; + ObjectLinkingLayer &ObjLinkingLayer; + + SymbolStringPtr DSOHandleSymbol; + std::atomic<bool> RuntimeBootstrapped{false}; + + ExecutorAddr orc_rt_elfnix_platform_bootstrap; + ExecutorAddr orc_rt_elfnix_platform_shutdown; + ExecutorAddr orc_rt_elfnix_register_object_sections; + ExecutorAddr orc_rt_elfnix_create_pthread_key; + + DenseMap<JITDylib *, SymbolLookupSet> RegisteredInitSymbols; + + // InitSeqs gets its own mutex to avoid locking the whole session when + // aggregating data from the jitlink. + std::mutex PlatformMutex; + DenseMap<JITDylib *, ELFNixJITDylibInitializers> InitSeqs; + std::vector<ELFPerObjectSectionsToRegister> BootstrapPOSRs; + + DenseMap<ExecutorAddr, JITDylib *> HandleAddrToJITDylib; + DenseMap<JITDylib *, uint64_t> JITDylibToPThreadKey; +}; + +namespace shared { + +using SPSELFPerObjectSectionsToRegister = + SPSTuple<SPSExecutorAddrRange, SPSExecutorAddrRange>; + +template <> +class SPSSerializationTraits<SPSELFPerObjectSectionsToRegister, + ELFPerObjectSectionsToRegister> { + +public: + static size_t size(const ELFPerObjectSectionsToRegister &MOPOSR) { + return SPSELFPerObjectSectionsToRegister::AsArgList::size( + MOPOSR.EHFrameSection, MOPOSR.ThreadDataSection); + } + + static bool serialize(SPSOutputBuffer &OB, + const ELFPerObjectSectionsToRegister &MOPOSR) { + return SPSELFPerObjectSectionsToRegister::AsArgList::serialize( + OB, MOPOSR.EHFrameSection, MOPOSR.ThreadDataSection); + } + + static bool deserialize(SPSInputBuffer &IB, + ELFPerObjectSectionsToRegister &MOPOSR) { + return SPSELFPerObjectSectionsToRegister::AsArgList::deserialize( + IB, MOPOSR.EHFrameSection, MOPOSR.ThreadDataSection); + } +}; + +using SPSNamedExecutorAddrRangeSequenceMap = + SPSSequence<SPSTuple<SPSString, SPSExecutorAddrRangeSequence>>; + +using SPSELFNixJITDylibInitializers = + SPSTuple<SPSString, SPSExecutorAddr, SPSNamedExecutorAddrRangeSequenceMap>; + +using SPSELFNixJITDylibInitializerSequence = + SPSSequence<SPSELFNixJITDylibInitializers>; + +/// Serialization traits for ELFNixJITDylibInitializers. +template <> +class SPSSerializationTraits<SPSELFNixJITDylibInitializers, + ELFNixJITDylibInitializers> { +public: + static size_t size(const ELFNixJITDylibInitializers &MOJDIs) { + return SPSELFNixJITDylibInitializers::AsArgList::size( + MOJDIs.Name, MOJDIs.DSOHandleAddress, MOJDIs.InitSections); + } + + static bool serialize(SPSOutputBuffer &OB, + const ELFNixJITDylibInitializers &MOJDIs) { + return SPSELFNixJITDylibInitializers::AsArgList::serialize( + OB, MOJDIs.Name, MOJDIs.DSOHandleAddress, MOJDIs.InitSections); + } + + static bool deserialize(SPSInputBuffer &IB, + ELFNixJITDylibInitializers &MOJDIs) { + return SPSELFNixJITDylibInitializers::AsArgList::deserialize( + IB, MOJDIs.Name, MOJDIs.DSOHandleAddress, MOJDIs.InitSections); + } +}; + +using SPSELFJITDylibDeinitializers = SPSEmpty; + +using SPSELFJITDylibDeinitializerSequence = + SPSSequence<SPSELFJITDylibDeinitializers>; + +template <> +class SPSSerializationTraits<SPSELFJITDylibDeinitializers, + ELFNixJITDylibDeinitializers> { +public: + static size_t size(const ELFNixJITDylibDeinitializers &MOJDDs) { return 0; } + + static bool serialize(SPSOutputBuffer &OB, + const ELFNixJITDylibDeinitializers &MOJDDs) { + return true; + } + + static bool deserialize(SPSInputBuffer &IB, + ELFNixJITDylibDeinitializers &MOJDDs) { + MOJDDs = ELFNixJITDylibDeinitializers(); + return true; + } +}; + +} // end namespace shared +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_ELFNIXPLATFORM_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/EPCDebugObjectRegistrar.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/EPCDebugObjectRegistrar.h new file mode 100644 index 0000000000..37067cc170 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/EPCDebugObjectRegistrar.h @@ -0,0 +1,73 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===- EPCDebugObjectRegistrar.h - EPC-based debug registration -*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// ExecutorProcessControl based registration of debug objects. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_EPCDEBUGOBJECTREGISTRAR_H +#define LLVM_EXECUTIONENGINE_ORC_EPCDEBUGOBJECTREGISTRAR_H + +#include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h" +#include "llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/Memory.h" + +#include <cstdint> +#include <memory> +#include <vector> + +using namespace llvm::orc::shared; + +namespace llvm { +namespace orc { + +class ExecutionSession; + +/// Abstract interface for registering debug objects in the executor process. +class DebugObjectRegistrar { +public: + virtual Error registerDebugObject(ExecutorAddrRange TargetMem) = 0; + virtual ~DebugObjectRegistrar() = default; +}; + +/// Use ExecutorProcessControl to register debug objects locally or in a remote +/// executor process. +class EPCDebugObjectRegistrar : public DebugObjectRegistrar { +public: + EPCDebugObjectRegistrar(ExecutionSession &ES, ExecutorAddr RegisterFn) + : ES(ES), RegisterFn(RegisterFn) {} + + Error registerDebugObject(ExecutorAddrRange TargetMem) override; + +private: + ExecutionSession &ES; + ExecutorAddr RegisterFn; +}; + +/// Create a ExecutorProcessControl-based DebugObjectRegistrar that emits debug +/// objects to the GDB JIT interface. +Expected<std::unique_ptr<EPCDebugObjectRegistrar>> +createJITLoaderGDBRegistrar(ExecutionSession &ES); + +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_EPCDEBUGOBJECTREGISTRAR_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h new file mode 100644 index 0000000000..4dd97e0d44 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h @@ -0,0 +1,79 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===------------ EPCDynamicLibrarySearchGenerator.h ------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Support loading and searching of dynamic libraries in an executor process +// via the ExecutorProcessControl class. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_EPCDYNAMICLIBRARYSEARCHGENERATOR_H +#define LLVM_EXECUTIONENGINE_ORC_EPCDYNAMICLIBRARYSEARCHGENERATOR_H + +#include "llvm/ADT/FunctionExtras.h" +#include "llvm/ExecutionEngine/Orc/Core.h" + +namespace llvm { +namespace orc { + +class ExecutorProcessControl; + +class EPCDynamicLibrarySearchGenerator : public DefinitionGenerator { +public: + using SymbolPredicate = unique_function<bool(const SymbolStringPtr &)>; + + /// Create a DynamicLibrarySearchGenerator that searches for symbols in the + /// library with the given handle. + /// + /// If the Allow predicate is given then only symbols matching the predicate + /// will be searched for. If the predicate is not given then all symbols will + /// be searched for. + EPCDynamicLibrarySearchGenerator(ExecutionSession &ES, + tpctypes::DylibHandle H, + SymbolPredicate Allow = SymbolPredicate()) + : EPC(ES.getExecutorProcessControl()), H(H), Allow(std::move(Allow)) {} + + /// Permanently loads the library at the given path and, on success, returns + /// a DynamicLibrarySearchGenerator that will search it for symbol definitions + /// in the library. On failure returns the reason the library failed to load. + static Expected<std::unique_ptr<EPCDynamicLibrarySearchGenerator>> + Load(ExecutionSession &ES, const char *LibraryPath, + SymbolPredicate Allow = SymbolPredicate()); + + /// Creates a EPCDynamicLibrarySearchGenerator that searches for symbols in + /// the target process. + static Expected<std::unique_ptr<EPCDynamicLibrarySearchGenerator>> + GetForTargetProcess(ExecutionSession &ES, + SymbolPredicate Allow = SymbolPredicate()) { + return Load(ES, nullptr, std::move(Allow)); + } + + Error tryToGenerate(LookupState &LS, LookupKind K, JITDylib &JD, + JITDylibLookupFlags JDLookupFlags, + const SymbolLookupSet &Symbols) override; + +private: + ExecutorProcessControl &EPC; + tpctypes::DylibHandle H; + SymbolPredicate Allow; +}; + +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_EPCDYNAMICLIBRARYSEARCHGENERATOR_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/EPCEHFrameRegistrar.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/EPCEHFrameRegistrar.h new file mode 100644 index 0000000000..9567f74dce --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/EPCEHFrameRegistrar.h @@ -0,0 +1,65 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===-- EPCEHFrameRegistrar.h - EPC based eh-frame registration -*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// ExecutorProcessControl based eh-frame registration. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_EPCEHFRAMEREGISTRAR_H +#define LLVM_EXECUTIONENGINE_ORC_EPCEHFRAMEREGISTRAR_H + +#include "llvm/ExecutionEngine/JITLink/EHFrameSupport.h" +#include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h" + +namespace llvm { +namespace orc { + +class ExecutionSession; + +/// Register/Deregisters EH frames in a remote process via a +/// ExecutorProcessControl instance. +class EPCEHFrameRegistrar : public jitlink::EHFrameRegistrar { +public: + /// Create from a ExecutorProcessControl instance alone. This will use + /// the EPC's lookupSymbols method to find the registration/deregistration + /// funciton addresses by name. + static Expected<std::unique_ptr<EPCEHFrameRegistrar>> + Create(ExecutionSession &ES); + + /// Create a EPCEHFrameRegistrar with the given ExecutorProcessControl + /// object and registration/deregistration function addresses. + EPCEHFrameRegistrar(ExecutionSession &ES, + ExecutorAddr RegisterEHFrameWrapperFnAddr, + ExecutorAddr DeregisterEHFRameWrapperFnAddr) + : ES(ES), RegisterEHFrameWrapperFnAddr(RegisterEHFrameWrapperFnAddr), + DeregisterEHFrameWrapperFnAddr(DeregisterEHFRameWrapperFnAddr) {} + + Error registerEHFrames(ExecutorAddrRange EHFrameSection) override; + Error deregisterEHFrames(ExecutorAddrRange EHFrameSection) override; + +private: + ExecutionSession &ES; + ExecutorAddr RegisterEHFrameWrapperFnAddr; + ExecutorAddr DeregisterEHFrameWrapperFnAddr; +}; + +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_EPCEHFRAMEREGISTRAR_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/EPCGenericDylibManager.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/EPCGenericDylibManager.h new file mode 100644 index 0000000000..6111dc16fb --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/EPCGenericDylibManager.h @@ -0,0 +1,78 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===- EPCGenericDylibManager.h -- Generic EPC Dylib management -*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Implements dylib loading and searching by making calls to +// ExecutorProcessControl::callWrapper. +// +// This simplifies the implementaton of new ExecutorProcessControl instances, +// as this implementation will always work (at the cost of some performance +// overhead for the calls). +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_EPCGENERICDYLIBMANAGER_H +#define LLVM_EXECUTIONENGINE_ORC_EPCGENERICDYLIBMANAGER_H + +#include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h" +#include "llvm/ExecutionEngine/Orc/Shared/SimpleRemoteEPCUtils.h" + +namespace llvm { +namespace orc { + +class SymbolLookupSet; + +class EPCGenericDylibManager { +public: + /// Function addresses for memory access. + struct SymbolAddrs { + ExecutorAddr Instance; + ExecutorAddr Open; + ExecutorAddr Lookup; + }; + + /// Create an EPCGenericMemoryAccess instance from a given set of + /// function addrs. + static Expected<EPCGenericDylibManager> + CreateWithDefaultBootstrapSymbols(ExecutorProcessControl &EPC); + + /// Create an EPCGenericMemoryAccess instance from a given set of + /// function addrs. + EPCGenericDylibManager(ExecutorProcessControl &EPC, SymbolAddrs SAs) + : EPC(EPC), SAs(SAs) {} + + /// Loads the dylib with the given name. + Expected<tpctypes::DylibHandle> open(StringRef Path, uint64_t Mode); + + /// Looks up symbols within the given dylib. + Expected<std::vector<ExecutorAddr>> lookup(tpctypes::DylibHandle H, + const SymbolLookupSet &Lookup); + + /// Looks up symbols within the given dylib. + Expected<std::vector<ExecutorAddr>> + lookup(tpctypes::DylibHandle H, const RemoteSymbolLookupSet &Lookup); + +private: + ExecutorProcessControl &EPC; + SymbolAddrs SAs; +}; + +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_EPCGENERICDYLIBMANAGER_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.h new file mode 100644 index 0000000000..19bcb49265 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.h @@ -0,0 +1,108 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===- EPCGenericJITLinkMemoryManager.h - EPC-based mem manager -*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Implements JITLinkMemoryManager by making remove calls via +// ExecutorProcessControl::callWrapperAsync. +// +// This simplifies the implementaton of new ExecutorProcessControl instances, +// as this implementation will always work (at the cost of some performance +// overhead for the calls). +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_EPCGENERICJITLINKMEMORYMANAGER_H +#define LLVM_EXECUTIONENGINE_ORC_EPCGENERICJITLINKMEMORYMANAGER_H + +#include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h" +#include "llvm/ExecutionEngine/Orc/Core.h" + +namespace llvm { +namespace orc { + +class EPCGenericJITLinkMemoryManager : public jitlink::JITLinkMemoryManager { +public: + /// Function addresses for memory access. + struct SymbolAddrs { + ExecutorAddr Allocator; + ExecutorAddr Reserve; + ExecutorAddr Finalize; + ExecutorAddr Deallocate; + }; + + /// Create an EPCGenericJITLinkMemoryManager instance from a given set of + /// function addrs. + EPCGenericJITLinkMemoryManager(ExecutorProcessControl &EPC, SymbolAddrs SAs) + : EPC(EPC), SAs(SAs) {} + + void allocate(const jitlink::JITLinkDylib *JD, jitlink::LinkGraph &G, + OnAllocatedFunction OnAllocated) override; + + // Use overloads from base class. + using JITLinkMemoryManager::allocate; + + void deallocate(std::vector<FinalizedAlloc> Allocs, + OnDeallocatedFunction OnDeallocated) override; + + // Use overloads from base class. + using JITLinkMemoryManager::deallocate; + +private: + class InFlightAlloc; + + void completeAllocation(ExecutorAddr AllocAddr, jitlink::BasicLayout BL, + OnAllocatedFunction OnAllocated); + + ExecutorProcessControl &EPC; + SymbolAddrs SAs; +}; + +namespace shared { + +/// FIXME: This specialization should be moved into TargetProcessControlTypes.h +/// (or whereever those types get merged to) once ORC depends on JITLink. +template <> +class SPSSerializationTraits<SPSExecutorAddr, + jitlink::JITLinkMemoryManager::FinalizedAlloc> { +public: + static size_t size(const jitlink::JITLinkMemoryManager::FinalizedAlloc &FA) { + return SPSArgList<SPSExecutorAddr>::size(ExecutorAddr(FA.getAddress())); + } + + static bool + serialize(SPSOutputBuffer &OB, + const jitlink::JITLinkMemoryManager::FinalizedAlloc &FA) { + return SPSArgList<SPSExecutorAddr>::serialize( + OB, ExecutorAddr(FA.getAddress())); + } + + static bool deserialize(SPSInputBuffer &IB, + jitlink::JITLinkMemoryManager::FinalizedAlloc &FA) { + ExecutorAddr A; + if (!SPSArgList<SPSExecutorAddr>::deserialize(IB, A)) + return false; + FA = jitlink::JITLinkMemoryManager::FinalizedAlloc(A); + return true; + } +}; + +} // end namespace shared +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_EPCGENERICJITLINKMEMORYMANAGER_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/EPCGenericMemoryAccess.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/EPCGenericMemoryAccess.h new file mode 100644 index 0000000000..8c89b91c24 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/EPCGenericMemoryAccess.h @@ -0,0 +1,96 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===- EPCGenericMemoryAccess.h - Generic EPC MemoryAccess impl -*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Implements ExecutorProcessControl::MemoryAccess by making calls to +// ExecutorProcessControl::callWrapperAsync. +// +// This simplifies the implementaton of new ExecutorProcessControl instances, +// as this implementation will always work (at the cost of some performance +// overhead for the calls). +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_EPCGENERICMEMORYACCESS_H +#define LLVM_EXECUTIONENGINE_ORC_EPCGENERICMEMORYACCESS_H + +#include "llvm/ExecutionEngine/Orc/Core.h" + +namespace llvm { +namespace orc { + +class EPCGenericMemoryAccess : public ExecutorProcessControl::MemoryAccess { +public: + /// Function addresses for memory access. + struct FuncAddrs { + ExecutorAddr WriteUInt8s; + ExecutorAddr WriteUInt16s; + ExecutorAddr WriteUInt32s; + ExecutorAddr WriteUInt64s; + ExecutorAddr WriteBuffers; + }; + + /// Create an EPCGenericMemoryAccess instance from a given set of + /// function addrs. + EPCGenericMemoryAccess(ExecutorProcessControl &EPC, FuncAddrs FAs) + : EPC(EPC), FAs(FAs) {} + + void writeUInt8sAsync(ArrayRef<tpctypes::UInt8Write> Ws, + WriteResultFn OnWriteComplete) override { + using namespace shared; + EPC.callSPSWrapperAsync<void(SPSSequence<SPSMemoryAccessUInt8Write>)>( + FAs.WriteUInt8s, std::move(OnWriteComplete), Ws); + } + + void writeUInt16sAsync(ArrayRef<tpctypes::UInt16Write> Ws, + WriteResultFn OnWriteComplete) override { + using namespace shared; + EPC.callSPSWrapperAsync<void(SPSSequence<SPSMemoryAccessUInt16Write>)>( + FAs.WriteUInt16s, std::move(OnWriteComplete), Ws); + } + + void writeUInt32sAsync(ArrayRef<tpctypes::UInt32Write> Ws, + WriteResultFn OnWriteComplete) override { + using namespace shared; + EPC.callSPSWrapperAsync<void(SPSSequence<SPSMemoryAccessUInt32Write>)>( + FAs.WriteUInt32s, std::move(OnWriteComplete), Ws); + } + + void writeUInt64sAsync(ArrayRef<tpctypes::UInt64Write> Ws, + WriteResultFn OnWriteComplete) override { + using namespace shared; + EPC.callSPSWrapperAsync<void(SPSSequence<SPSMemoryAccessUInt64Write>)>( + FAs.WriteUInt64s, std::move(OnWriteComplete), Ws); + } + + void writeBuffersAsync(ArrayRef<tpctypes::BufferWrite> Ws, + WriteResultFn OnWriteComplete) override { + using namespace shared; + EPC.callSPSWrapperAsync<void(SPSSequence<SPSMemoryAccessBufferWrite>)>( + FAs.WriteBuffers, std::move(OnWriteComplete), Ws); + } + +private: + ExecutorProcessControl &EPC; + FuncAddrs FAs; +}; + +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_EPCGENERICMEMORYACCESS_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.h new file mode 100644 index 0000000000..53d97205b3 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.h @@ -0,0 +1,139 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===---- EPCGenericRTDyldMemoryManager.h - EPC-based MemMgr ----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Defines a RuntimeDyld::MemoryManager that uses EPC and the ORC runtime +// bootstrap functions. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_EPCGENERICRTDYLDMEMORYMANAGER_H +#define LLVM_EXECUTIONENGINE_ORC_EPCGENERICRTDYLDMEMORYMANAGER_H + +#include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h" +#include "llvm/ExecutionEngine/RuntimeDyld.h" + +#define DEBUG_TYPE "orc" + +namespace llvm { +namespace orc { + +/// Remote-mapped RuntimeDyld-compatible memory manager. +class EPCGenericRTDyldMemoryManager : public RuntimeDyld::MemoryManager { +public: + /// Symbol addresses for memory access. + struct SymbolAddrs { + ExecutorAddr Instance; + ExecutorAddr Reserve; + ExecutorAddr Finalize; + ExecutorAddr Deallocate; + ExecutorAddr RegisterEHFrame; + ExecutorAddr DeregisterEHFrame; + }; + + /// Create an EPCGenericRTDyldMemoryManager using the given EPC, looking up + /// the default symbol names in the bootstrap symbol set. + static Expected<std::unique_ptr<EPCGenericRTDyldMemoryManager>> + CreateWithDefaultBootstrapSymbols(ExecutorProcessControl &EPC); + + /// Create an EPCGenericRTDyldMemoryManager using the given EPC and symbol + /// addrs. + EPCGenericRTDyldMemoryManager(ExecutorProcessControl &EPC, SymbolAddrs SAs); + + EPCGenericRTDyldMemoryManager(const EPCGenericRTDyldMemoryManager &) = delete; + EPCGenericRTDyldMemoryManager & + operator=(const EPCGenericRTDyldMemoryManager &) = delete; + EPCGenericRTDyldMemoryManager(EPCGenericRTDyldMemoryManager &&) = delete; + EPCGenericRTDyldMemoryManager & + operator=(EPCGenericRTDyldMemoryManager &&) = delete; + ~EPCGenericRTDyldMemoryManager(); + + uint8_t *allocateCodeSection(uintptr_t Size, unsigned Alignment, + unsigned SectionID, + StringRef SectionName) override; + + uint8_t *allocateDataSection(uintptr_t Size, unsigned Alignment, + unsigned SectionID, StringRef SectionName, + bool IsReadOnly) override; + + void reserveAllocationSpace(uintptr_t CodeSize, uint32_t CodeAlign, + uintptr_t RODataSize, uint32_t RODataAlign, + uintptr_t RWDataSize, + uint32_t RWDataAlign) override; + + bool needsToReserveAllocationSpace() override; + + void registerEHFrames(uint8_t *Addr, uint64_t LoadAddr, size_t Size) override; + + void deregisterEHFrames() override; + + void notifyObjectLoaded(RuntimeDyld &Dyld, + const object::ObjectFile &Obj) override; + + bool finalizeMemory(std::string *ErrMsg = nullptr) override; + +private: + struct Alloc { + public: + Alloc(uint64_t Size, unsigned Align) + : Size(Size), Align(Align), + Contents(std::make_unique<uint8_t[]>(Size + Align - 1)) {} + + uint64_t Size; + unsigned Align; + std::unique_ptr<uint8_t[]> Contents; + ExecutorAddr RemoteAddr; + }; + + // Group of section allocations to be allocated together in the executor. The + // RemoteCodeAddr will stand in as the id of the group for deallocation + // purposes. + struct AllocGroup { + AllocGroup() = default; + AllocGroup(const AllocGroup &) = delete; + AllocGroup &operator=(const AllocGroup &) = delete; + AllocGroup(AllocGroup &&) = default; + AllocGroup &operator=(AllocGroup &&) = default; + + ExecutorAddrRange RemoteCode; + ExecutorAddrRange RemoteROData; + ExecutorAddrRange RemoteRWData; + std::vector<ExecutorAddrRange> UnfinalizedEHFrames; + std::vector<Alloc> CodeAllocs, RODataAllocs, RWDataAllocs; + }; + + // Maps all allocations in Allocs to aligned blocks + void mapAllocsToRemoteAddrs(RuntimeDyld &Dyld, std::vector<Alloc> &Allocs, + ExecutorAddr NextAddr); + + ExecutorProcessControl &EPC; + SymbolAddrs SAs; + + std::mutex M; + std::vector<AllocGroup> Unmapped; + std::vector<AllocGroup> Unfinalized; + std::vector<ExecutorAddr> FinalizedAllocs; + std::string ErrMsg; +}; + +} // end namespace orc +} // end namespace llvm + +#undef DEBUG_TYPE + +#endif // LLVM_EXECUTIONENGINE_ORC_EPCGENERICRTDYLDMEMORYMANAGER_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/EPCIndirectionUtils.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/EPCIndirectionUtils.h new file mode 100644 index 0000000000..e78a575561 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/EPCIndirectionUtils.h @@ -0,0 +1,233 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===--- EPCIndirectionUtils.h - EPC based indirection utils ----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Indirection utilities (stubs, trampolines, lazy call-throughs) that use the +// ExecutorProcessControl API to interact with the executor process. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_EPCINDIRECTIONUTILS_H +#define LLVM_EXECUTIONENGINE_ORC_EPCINDIRECTIONUTILS_H + +#include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h" +#include "llvm/ExecutionEngine/Orc/IndirectionUtils.h" +#include "llvm/ExecutionEngine/Orc/LazyReexports.h" + +#include <mutex> + +namespace llvm { +namespace orc { + +class ExecutorProcessControl; + +/// Provides ExecutorProcessControl based indirect stubs, trampoline pool and +/// lazy call through manager. +class EPCIndirectionUtils { + friend class EPCIndirectionUtilsAccess; + +public: + /// ABI support base class. Used to write resolver, stub, and trampoline + /// blocks. + class ABISupport { + protected: + ABISupport(unsigned PointerSize, unsigned TrampolineSize, unsigned StubSize, + unsigned StubToPointerMaxDisplacement, unsigned ResolverCodeSize) + : PointerSize(PointerSize), TrampolineSize(TrampolineSize), + StubSize(StubSize), + StubToPointerMaxDisplacement(StubToPointerMaxDisplacement), + ResolverCodeSize(ResolverCodeSize) {} + + public: + virtual ~ABISupport(); + + unsigned getPointerSize() const { return PointerSize; } + unsigned getTrampolineSize() const { return TrampolineSize; } + unsigned getStubSize() const { return StubSize; } + unsigned getStubToPointerMaxDisplacement() const { + return StubToPointerMaxDisplacement; + } + unsigned getResolverCodeSize() const { return ResolverCodeSize; } + + virtual void writeResolverCode(char *ResolverWorkingMem, + JITTargetAddress ResolverTargetAddr, + JITTargetAddress ReentryFnAddr, + JITTargetAddress ReentryCtxAddr) const = 0; + + virtual void writeTrampolines(char *TrampolineBlockWorkingMem, + JITTargetAddress TrampolineBlockTragetAddr, + JITTargetAddress ResolverAddr, + unsigned NumTrampolines) const = 0; + + virtual void + writeIndirectStubsBlock(char *StubsBlockWorkingMem, + JITTargetAddress StubsBlockTargetAddress, + JITTargetAddress PointersBlockTargetAddress, + unsigned NumStubs) const = 0; + + private: + unsigned PointerSize = 0; + unsigned TrampolineSize = 0; + unsigned StubSize = 0; + unsigned StubToPointerMaxDisplacement = 0; + unsigned ResolverCodeSize = 0; + }; + + /// Create using the given ABI class. + template <typename ORCABI> + static std::unique_ptr<EPCIndirectionUtils> + CreateWithABI(ExecutorProcessControl &EPC); + + /// Create based on the ExecutorProcessControl triple. + static Expected<std::unique_ptr<EPCIndirectionUtils>> + Create(ExecutorProcessControl &EPC); + + /// Return a reference to the ExecutorProcessControl object. + ExecutorProcessControl &getExecutorProcessControl() const { return EPC; } + + /// Return a reference to the ABISupport object for this instance. + ABISupport &getABISupport() const { return *ABI; } + + /// Release memory for resources held by this instance. This *must* be called + /// prior to destruction of the class. + Error cleanup(); + + /// Write resolver code to the executor process and return its address. + /// This must be called before any call to createTrampolinePool or + /// createLazyCallThroughManager. + Expected<JITTargetAddress> + writeResolverBlock(JITTargetAddress ReentryFnAddr, + JITTargetAddress ReentryCtxAddr); + + /// Returns the address of the Resolver block. Returns zero if the + /// writeResolverBlock method has not previously been called. + JITTargetAddress getResolverBlockAddress() const { return ResolverBlockAddr; } + + /// Create an IndirectStubsManager for the executor process. + std::unique_ptr<IndirectStubsManager> createIndirectStubsManager(); + + /// Create a TrampolinePool for the executor process. + TrampolinePool &getTrampolinePool(); + + /// Create a LazyCallThroughManager. + /// This function should only be called once. + LazyCallThroughManager & + createLazyCallThroughManager(ExecutionSession &ES, + JITTargetAddress ErrorHandlerAddr); + + /// Create a LazyCallThroughManager for the executor process. + LazyCallThroughManager &getLazyCallThroughManager() { + assert(LCTM && "createLazyCallThroughManager must be called first"); + return *LCTM; + } + +private: + using FinalizedAlloc = jitlink::JITLinkMemoryManager::FinalizedAlloc; + + struct IndirectStubInfo { + IndirectStubInfo() = default; + IndirectStubInfo(JITTargetAddress StubAddress, + JITTargetAddress PointerAddress) + : StubAddress(StubAddress), PointerAddress(PointerAddress) {} + JITTargetAddress StubAddress = 0; + JITTargetAddress PointerAddress = 0; + }; + + using IndirectStubInfoVector = std::vector<IndirectStubInfo>; + + /// Create an EPCIndirectionUtils instance. + EPCIndirectionUtils(ExecutorProcessControl &EPC, + std::unique_ptr<ABISupport> ABI); + + Expected<IndirectStubInfoVector> getIndirectStubs(unsigned NumStubs); + + std::mutex EPCUIMutex; + ExecutorProcessControl &EPC; + std::unique_ptr<ABISupport> ABI; + JITTargetAddress ResolverBlockAddr = 0; + FinalizedAlloc ResolverBlock; + std::unique_ptr<TrampolinePool> TP; + std::unique_ptr<LazyCallThroughManager> LCTM; + + std::vector<IndirectStubInfo> AvailableIndirectStubs; + std::vector<FinalizedAlloc> IndirectStubAllocs; +}; + +/// This will call writeResolver on the given EPCIndirectionUtils instance +/// to set up re-entry via a function that will directly return the trampoline +/// landing address. +/// +/// The EPCIndirectionUtils' LazyCallThroughManager must have been previously +/// created via EPCIndirectionUtils::createLazyCallThroughManager. +/// +/// The EPCIndirectionUtils' writeResolver method must not have been previously +/// called. +/// +/// This function is experimental and likely subject to revision. +Error setUpInProcessLCTMReentryViaEPCIU(EPCIndirectionUtils &EPCIU); + +namespace detail { + +template <typename ORCABI> +class ABISupportImpl : public EPCIndirectionUtils::ABISupport { +public: + ABISupportImpl() + : ABISupport(ORCABI::PointerSize, ORCABI::TrampolineSize, + ORCABI::StubSize, ORCABI::StubToPointerMaxDisplacement, + ORCABI::ResolverCodeSize) {} + + void writeResolverCode(char *ResolverWorkingMem, + JITTargetAddress ResolverTargetAddr, + JITTargetAddress ReentryFnAddr, + JITTargetAddress ReentryCtxAddr) const override { + ORCABI::writeResolverCode(ResolverWorkingMem, ResolverTargetAddr, + ReentryFnAddr, ReentryCtxAddr); + } + + void writeTrampolines(char *TrampolineBlockWorkingMem, + JITTargetAddress TrampolineBlockTargetAddr, + JITTargetAddress ResolverAddr, + unsigned NumTrampolines) const override { + ORCABI::writeTrampolines(TrampolineBlockWorkingMem, + TrampolineBlockTargetAddr, ResolverAddr, + NumTrampolines); + } + + void writeIndirectStubsBlock(char *StubsBlockWorkingMem, + JITTargetAddress StubsBlockTargetAddress, + JITTargetAddress PointersBlockTargetAddress, + unsigned NumStubs) const override { + ORCABI::writeIndirectStubsBlock(StubsBlockWorkingMem, + StubsBlockTargetAddress, + PointersBlockTargetAddress, NumStubs); + } +}; + +} // end namespace detail + +template <typename ORCABI> +std::unique_ptr<EPCIndirectionUtils> +EPCIndirectionUtils::CreateWithABI(ExecutorProcessControl &EPC) { + return std::unique_ptr<EPCIndirectionUtils>(new EPCIndirectionUtils( + EPC, std::make_unique<detail::ABISupportImpl<ORCABI>>())); +} + +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_EPCINDIRECTIONUTILS_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h new file mode 100644 index 0000000000..885b9cd085 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h @@ -0,0 +1,320 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===- ExecutionUtils.h - Utilities for executing code in Orc ---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Contains utilities for executing code in Orc. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_EXECUTIONUTILS_H +#define LLVM_EXECUTIONENGINE_ORC_EXECUTIONUTILS_H + +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/iterator_range.h" +#include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/Mangling.h" +#include "llvm/ExecutionEngine/Orc/Shared/OrcError.h" +#include "llvm/ExecutionEngine/RuntimeDyld.h" +#include "llvm/Object/Archive.h" +#include "llvm/Support/DynamicLibrary.h" +#include <algorithm> +#include <cstdint> +#include <utility> +#include <vector> + +namespace llvm { + +class ConstantArray; +class GlobalVariable; +class Function; +class Module; +class Value; + +namespace orc { + +class ObjectLayer; + +/// This iterator provides a convenient way to iterate over the elements +/// of an llvm.global_ctors/llvm.global_dtors instance. +/// +/// The easiest way to get hold of instances of this class is to use the +/// getConstructors/getDestructors functions. +class CtorDtorIterator { +public: + /// Accessor for an element of the global_ctors/global_dtors array. + /// + /// This class provides a read-only view of the element with any casts on + /// the function stripped away. + struct Element { + Element(unsigned Priority, Function *Func, Value *Data) + : Priority(Priority), Func(Func), Data(Data) {} + + unsigned Priority; + Function *Func; + Value *Data; + }; + + /// Construct an iterator instance. If End is true then this iterator + /// acts as the end of the range, otherwise it is the beginning. + CtorDtorIterator(const GlobalVariable *GV, bool End); + + /// Test iterators for equality. + bool operator==(const CtorDtorIterator &Other) const; + + /// Test iterators for inequality. + bool operator!=(const CtorDtorIterator &Other) const; + + /// Pre-increment iterator. + CtorDtorIterator& operator++(); + + /// Post-increment iterator. + CtorDtorIterator operator++(int); + + /// Dereference iterator. The resulting value provides a read-only view + /// of this element of the global_ctors/global_dtors list. + Element operator*() const; + +private: + const ConstantArray *InitList; + unsigned I; +}; + +/// Create an iterator range over the entries of the llvm.global_ctors +/// array. +iterator_range<CtorDtorIterator> getConstructors(const Module &M); + +/// Create an iterator range over the entries of the llvm.global_ctors +/// array. +iterator_range<CtorDtorIterator> getDestructors(const Module &M); + +/// This iterator provides a convenient way to iterate over GlobalValues that +/// have initialization effects. +class StaticInitGVIterator { +public: + StaticInitGVIterator() = default; + + StaticInitGVIterator(Module &M) + : I(M.global_values().begin()), E(M.global_values().end()), + ObjFmt(Triple(M.getTargetTriple()).getObjectFormat()) { + if (I != E) { + if (!isStaticInitGlobal(*I)) + moveToNextStaticInitGlobal(); + } else + I = E = Module::global_value_iterator(); + } + + bool operator==(const StaticInitGVIterator &O) const { return I == O.I; } + bool operator!=(const StaticInitGVIterator &O) const { return I != O.I; } + + StaticInitGVIterator &operator++() { + assert(I != E && "Increment past end of range"); + moveToNextStaticInitGlobal(); + return *this; + } + + GlobalValue &operator*() { return *I; } + +private: + bool isStaticInitGlobal(GlobalValue &GV); + void moveToNextStaticInitGlobal() { + ++I; + while (I != E && !isStaticInitGlobal(*I)) + ++I; + if (I == E) + I = E = Module::global_value_iterator(); + } + + Module::global_value_iterator I, E; + Triple::ObjectFormatType ObjFmt; +}; + +/// Create an iterator range over the GlobalValues that contribute to static +/// initialization. +inline iterator_range<StaticInitGVIterator> getStaticInitGVs(Module &M) { + return make_range(StaticInitGVIterator(M), StaticInitGVIterator()); +} + +class CtorDtorRunner { +public: + CtorDtorRunner(JITDylib &JD) : JD(JD) {} + void add(iterator_range<CtorDtorIterator> CtorDtors); + Error run(); + +private: + using CtorDtorList = std::vector<SymbolStringPtr>; + using CtorDtorPriorityMap = std::map<unsigned, CtorDtorList>; + + JITDylib &JD; + CtorDtorPriorityMap CtorDtorsByPriority; +}; + +/// Support class for static dtor execution. For hosted (in-process) JITs +/// only! +/// +/// If a __cxa_atexit function isn't found C++ programs that use static +/// destructors will fail to link. However, we don't want to use the host +/// process's __cxa_atexit, because it will schedule JIT'd destructors to run +/// after the JIT has been torn down, which is no good. This class makes it easy +/// to override __cxa_atexit (and the related __dso_handle). +/// +/// To use, clients should manually call searchOverrides from their symbol +/// resolver. This should generally be done after attempting symbol resolution +/// inside the JIT, but before searching the host process's symbol table. When +/// the client determines that destructors should be run (generally at JIT +/// teardown or after a return from main), the runDestructors method should be +/// called. +class LocalCXXRuntimeOverridesBase { +public: + /// Run any destructors recorded by the overriden __cxa_atexit function + /// (CXAAtExitOverride). + void runDestructors(); + +protected: + template <typename PtrTy> JITTargetAddress toTargetAddress(PtrTy *P) { + return static_cast<JITTargetAddress>(reinterpret_cast<uintptr_t>(P)); + } + + using DestructorPtr = void (*)(void *); + using CXXDestructorDataPair = std::pair<DestructorPtr, void *>; + using CXXDestructorDataPairList = std::vector<CXXDestructorDataPair>; + CXXDestructorDataPairList DSOHandleOverride; + static int CXAAtExitOverride(DestructorPtr Destructor, void *Arg, + void *DSOHandle); +}; + +class LocalCXXRuntimeOverrides : public LocalCXXRuntimeOverridesBase { +public: + Error enable(JITDylib &JD, MangleAndInterner &Mangler); +}; + +/// An interface for Itanium __cxa_atexit interposer implementations. +class ItaniumCXAAtExitSupport { +public: + struct AtExitRecord { + void (*F)(void *); + void *Ctx; + }; + + void registerAtExit(void (*F)(void *), void *Ctx, void *DSOHandle); + void runAtExits(void *DSOHandle); + +private: + std::mutex AtExitsMutex; + DenseMap<void *, std::vector<AtExitRecord>> AtExitRecords; +}; + +/// A utility class to expose symbols found via dlsym to the JIT. +/// +/// If an instance of this class is attached to a JITDylib as a fallback +/// definition generator, then any symbol found in the given DynamicLibrary that +/// passes the 'Allow' predicate will be added to the JITDylib. +class DynamicLibrarySearchGenerator : public DefinitionGenerator { +public: + using SymbolPredicate = std::function<bool(const SymbolStringPtr &)>; + + /// Create a DynamicLibrarySearchGenerator that searches for symbols in the + /// given sys::DynamicLibrary. + /// + /// If the Allow predicate is given then only symbols matching the predicate + /// will be searched for. If the predicate is not given then all symbols will + /// be searched for. + DynamicLibrarySearchGenerator(sys::DynamicLibrary Dylib, char GlobalPrefix, + SymbolPredicate Allow = SymbolPredicate()); + + /// Permanently loads the library at the given path and, on success, returns + /// a DynamicLibrarySearchGenerator that will search it for symbol definitions + /// in the library. On failure returns the reason the library failed to load. + static Expected<std::unique_ptr<DynamicLibrarySearchGenerator>> + Load(const char *FileName, char GlobalPrefix, + SymbolPredicate Allow = SymbolPredicate()); + + /// Creates a DynamicLibrarySearchGenerator that searches for symbols in + /// the current process. + static Expected<std::unique_ptr<DynamicLibrarySearchGenerator>> + GetForCurrentProcess(char GlobalPrefix, + SymbolPredicate Allow = SymbolPredicate()) { + return Load(nullptr, GlobalPrefix, std::move(Allow)); + } + + Error tryToGenerate(LookupState &LS, LookupKind K, JITDylib &JD, + JITDylibLookupFlags JDLookupFlags, + const SymbolLookupSet &Symbols) override; + +private: + sys::DynamicLibrary Dylib; + SymbolPredicate Allow; + char GlobalPrefix; +}; + +/// A utility class to expose symbols from a static library. +/// +/// If an instance of this class is attached to a JITDylib as a fallback +/// definition generator, then any symbol found in the archive will result in +/// the containing object being added to the JITDylib. +class StaticLibraryDefinitionGenerator : public DefinitionGenerator { +public: + // Interface builder function for objects loaded from this archive. + using GetObjectFileInterface = + unique_function<Expected<MaterializationUnit::Interface>( + ExecutionSession &ES, MemoryBufferRef ObjBuffer)>; + + /// Try to create a StaticLibraryDefinitionGenerator from the given path. + /// + /// This call will succeed if the file at the given path is a static library + /// is a valid archive, otherwise it will return an error. + static Expected<std::unique_ptr<StaticLibraryDefinitionGenerator>> + Load(ObjectLayer &L, const char *FileName, + GetObjectFileInterface GetObjFileInterface = GetObjectFileInterface()); + + /// Try to create a StaticLibraryDefinitionGenerator from the given path. + /// + /// This call will succeed if the file at the given path is a static library + /// or a MachO universal binary containing a static library that is compatible + /// with the given triple. Otherwise it will return an error. + static Expected<std::unique_ptr<StaticLibraryDefinitionGenerator>> + Load(ObjectLayer &L, const char *FileName, const Triple &TT, + GetObjectFileInterface GetObjFileInterface = GetObjectFileInterface()); + + /// Try to create a StaticLibrarySearchGenerator from the given memory buffer. + /// This call will succeed if the buffer contains a valid archive, otherwise + /// it will return an error. + static Expected<std::unique_ptr<StaticLibraryDefinitionGenerator>> + Create(ObjectLayer &L, std::unique_ptr<MemoryBuffer> ArchiveBuffer, + GetObjectFileInterface GetObjFileInterface = GetObjectFileInterface()); + + Error tryToGenerate(LookupState &LS, LookupKind K, JITDylib &JD, + JITDylibLookupFlags JDLookupFlags, + const SymbolLookupSet &Symbols) override; + +private: + StaticLibraryDefinitionGenerator(ObjectLayer &L, + std::unique_ptr<MemoryBuffer> ArchiveBuffer, + GetObjectFileInterface GetObjFileInterface, + Error &Err); + + ObjectLayer &L; + GetObjectFileInterface GetObjFileInterface; + std::unique_ptr<MemoryBuffer> ArchiveBuffer; + std::unique_ptr<object::Archive> Archive; +}; + +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_EXECUTIONUTILS_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/ExecutorProcessControl.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/ExecutorProcessControl.h new file mode 100644 index 0000000000..ba6f8c4a65 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/ExecutorProcessControl.h @@ -0,0 +1,482 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===- ExecutorProcessControl.h - Executor process control APIs -*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Utilities for interacting with the executor processes. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_EXECUTORPROCESSCONTROL_H +#define LLVM_EXECUTIONENGINE_ORC_EXECUTORPROCESSCONTROL_H + +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/Triple.h" +#include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h" +#include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h" +#include "llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h" +#include "llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h" +#include "llvm/ExecutionEngine/Orc/SymbolStringPool.h" +#include "llvm/ExecutionEngine/Orc/TaskDispatch.h" +#include "llvm/Support/DynamicLibrary.h" +#include "llvm/Support/MSVCErrorWorkarounds.h" + +#include <future> +#include <mutex> +#include <vector> + +namespace llvm { +namespace orc { + +class ExecutionSession; +class SymbolLookupSet; + +/// ExecutorProcessControl supports interaction with a JIT target process. +class ExecutorProcessControl { + friend class ExecutionSession; +public: + + /// A handler or incoming WrapperFunctionResults -- either return values from + /// callWrapper* calls, or incoming JIT-dispatch requests. + /// + /// IncomingWFRHandlers are constructible from + /// unique_function<void(shared::WrapperFunctionResult)>s using the + /// runInPlace function or a RunWithDispatch object. + class IncomingWFRHandler { + friend class ExecutorProcessControl; + public: + IncomingWFRHandler() = default; + explicit operator bool() const { return !!H; } + void operator()(shared::WrapperFunctionResult WFR) { H(std::move(WFR)); } + private: + template <typename FnT> IncomingWFRHandler(FnT &&Fn) + : H(std::forward<FnT>(Fn)) {} + + unique_function<void(shared::WrapperFunctionResult)> H; + }; + + /// Constructs an IncomingWFRHandler from a function object that is callable + /// as void(shared::WrapperFunctionResult). The function object will be called + /// directly. This should be used with care as it may block listener threads + /// in remote EPCs. It is only suitable for simple tasks (e.g. setting a + /// future), or for performing some quick analysis before dispatching "real" + /// work as a Task. + class RunInPlace { + public: + template <typename FnT> + IncomingWFRHandler operator()(FnT &&Fn) { + return IncomingWFRHandler(std::forward<FnT>(Fn)); + } + }; + + /// Constructs an IncomingWFRHandler from a function object by creating a new + /// function object that dispatches the original using a TaskDispatcher, + /// wrapping the original as a GenericNamedTask. + /// + /// This is the default approach for running WFR handlers. + class RunAsTask { + public: + RunAsTask(TaskDispatcher &D) : D(D) {} + + template <typename FnT> + IncomingWFRHandler operator()(FnT &&Fn) { + return IncomingWFRHandler( + [&D = this->D, Fn = std::move(Fn)] + (shared::WrapperFunctionResult WFR) mutable { + D.dispatch( + makeGenericNamedTask( + [Fn = std::move(Fn), WFR = std::move(WFR)]() mutable { + Fn(std::move(WFR)); + }, "WFR handler task")); + }); + } + private: + TaskDispatcher &D; + }; + + /// APIs for manipulating memory in the target process. + class MemoryAccess { + public: + /// Callback function for asynchronous writes. + using WriteResultFn = unique_function<void(Error)>; + + virtual ~MemoryAccess(); + + virtual void writeUInt8sAsync(ArrayRef<tpctypes::UInt8Write> Ws, + WriteResultFn OnWriteComplete) = 0; + + virtual void writeUInt16sAsync(ArrayRef<tpctypes::UInt16Write> Ws, + WriteResultFn OnWriteComplete) = 0; + + virtual void writeUInt32sAsync(ArrayRef<tpctypes::UInt32Write> Ws, + WriteResultFn OnWriteComplete) = 0; + + virtual void writeUInt64sAsync(ArrayRef<tpctypes::UInt64Write> Ws, + WriteResultFn OnWriteComplete) = 0; + + virtual void writeBuffersAsync(ArrayRef<tpctypes::BufferWrite> Ws, + WriteResultFn OnWriteComplete) = 0; + + Error writeUInt8s(ArrayRef<tpctypes::UInt8Write> Ws) { + std::promise<MSVCPError> ResultP; + auto ResultF = ResultP.get_future(); + writeUInt8sAsync(Ws, + [&](Error Err) { ResultP.set_value(std::move(Err)); }); + return ResultF.get(); + } + + Error writeUInt16s(ArrayRef<tpctypes::UInt16Write> Ws) { + std::promise<MSVCPError> ResultP; + auto ResultF = ResultP.get_future(); + writeUInt16sAsync(Ws, + [&](Error Err) { ResultP.set_value(std::move(Err)); }); + return ResultF.get(); + } + + Error writeUInt32s(ArrayRef<tpctypes::UInt32Write> Ws) { + std::promise<MSVCPError> ResultP; + auto ResultF = ResultP.get_future(); + writeUInt32sAsync(Ws, + [&](Error Err) { ResultP.set_value(std::move(Err)); }); + return ResultF.get(); + } + + Error writeUInt64s(ArrayRef<tpctypes::UInt64Write> Ws) { + std::promise<MSVCPError> ResultP; + auto ResultF = ResultP.get_future(); + writeUInt64sAsync(Ws, + [&](Error Err) { ResultP.set_value(std::move(Err)); }); + return ResultF.get(); + } + + Error writeBuffers(ArrayRef<tpctypes::BufferWrite> Ws) { + std::promise<MSVCPError> ResultP; + auto ResultF = ResultP.get_future(); + writeBuffersAsync(Ws, + [&](Error Err) { ResultP.set_value(std::move(Err)); }); + return ResultF.get(); + } + }; + + /// A pair of a dylib and a set of symbols to be looked up. + struct LookupRequest { + LookupRequest(tpctypes::DylibHandle Handle, const SymbolLookupSet &Symbols) + : Handle(Handle), Symbols(Symbols) {} + tpctypes::DylibHandle Handle; + const SymbolLookupSet &Symbols; + }; + + /// Contains the address of the dispatch function and context that the ORC + /// runtime can use to call functions in the JIT. + struct JITDispatchInfo { + ExecutorAddr JITDispatchFunction; + ExecutorAddr JITDispatchContext; + }; + + ExecutorProcessControl(std::shared_ptr<SymbolStringPool> SSP, + std::unique_ptr<TaskDispatcher> D) + : SSP(std::move(SSP)), D(std::move(D)) {} + + virtual ~ExecutorProcessControl(); + + /// Return the ExecutionSession associated with this instance. + /// Not callable until the ExecutionSession has been associated. + ExecutionSession &getExecutionSession() { + assert(ES && "No ExecutionSession associated yet"); + return *ES; + } + + /// Intern a symbol name in the SymbolStringPool. + SymbolStringPtr intern(StringRef SymName) { return SSP->intern(SymName); } + + /// Return a shared pointer to the SymbolStringPool for this instance. + std::shared_ptr<SymbolStringPool> getSymbolStringPool() const { return SSP; } + + TaskDispatcher &getDispatcher() { return *D; } + + /// Return the Triple for the target process. + const Triple &getTargetTriple() const { return TargetTriple; } + + /// Get the page size for the target process. + unsigned getPageSize() const { return PageSize; } + + /// Get the JIT dispatch function and context address for the executor. + const JITDispatchInfo &getJITDispatchInfo() const { return JDI; } + + /// Return a MemoryAccess object for the target process. + MemoryAccess &getMemoryAccess() const { + assert(MemAccess && "No MemAccess object set."); + return *MemAccess; + } + + /// Return a JITLinkMemoryManager for the target process. + jitlink::JITLinkMemoryManager &getMemMgr() const { + assert(MemMgr && "No MemMgr object set"); + return *MemMgr; + } + + /// Returns the bootstrap symbol map. + const StringMap<ExecutorAddr> &getBootstrapSymbolsMap() const { + return BootstrapSymbols; + } + + /// For each (ExecutorAddr&, StringRef) pair, looks up the string in the + /// bootstrap symbols map and writes its address to the ExecutorAddr if + /// found. If any symbol is not found then the function returns an error. + Error getBootstrapSymbols( + ArrayRef<std::pair<ExecutorAddr &, StringRef>> Pairs) const { + for (auto &KV : Pairs) { + auto I = BootstrapSymbols.find(KV.second); + if (I == BootstrapSymbols.end()) + return make_error<StringError>("Symbol \"" + KV.second + + "\" not found " + "in bootstrap symbols map", + inconvertibleErrorCode()); + + KV.first = I->second; + } + return Error::success(); + } + + /// Load the dynamic library at the given path and return a handle to it. + /// If LibraryPath is null this function will return the global handle for + /// the target process. + virtual Expected<tpctypes::DylibHandle> loadDylib(const char *DylibPath) = 0; + + /// Search for symbols in the target process. + /// + /// The result of the lookup is a 2-dimentional array of target addresses + /// that correspond to the lookup order. If a required symbol is not + /// found then this method will return an error. If a weakly referenced + /// symbol is not found then it be assigned a '0' value. + virtual Expected<std::vector<tpctypes::LookupResult>> + lookupSymbols(ArrayRef<LookupRequest> Request) = 0; + + /// Run function with a main-like signature. + virtual Expected<int32_t> runAsMain(ExecutorAddr MainFnAddr, + ArrayRef<std::string> Args) = 0; + + /// Run a wrapper function in the executor. The given WFRHandler will be + /// called on the result when it is returned. + /// + /// The wrapper function should be callable as: + /// + /// \code{.cpp} + /// CWrapperFunctionResult fn(uint8_t *Data, uint64_t Size); + /// \endcode{.cpp} + virtual void callWrapperAsync(ExecutorAddr WrapperFnAddr, + IncomingWFRHandler OnComplete, + ArrayRef<char> ArgBuffer) = 0; + + /// Run a wrapper function in the executor using the given Runner to dispatch + /// OnComplete when the result is ready. + template <typename RunPolicyT, typename FnT> + void callWrapperAsync(RunPolicyT &&Runner, ExecutorAddr WrapperFnAddr, + FnT &&OnComplete, ArrayRef<char> ArgBuffer) { + callWrapperAsync( + WrapperFnAddr, Runner(std::forward<FnT>(OnComplete)), ArgBuffer); + } + + /// Run a wrapper function in the executor. OnComplete will be dispatched + /// as a GenericNamedTask using this instance's TaskDispatch object. + template <typename FnT> + void callWrapperAsync(ExecutorAddr WrapperFnAddr, FnT &&OnComplete, + ArrayRef<char> ArgBuffer) { + callWrapperAsync(RunAsTask(*D), WrapperFnAddr, + std::forward<FnT>(OnComplete), ArgBuffer); + } + + /// Run a wrapper function in the executor. The wrapper function should be + /// callable as: + /// + /// \code{.cpp} + /// CWrapperFunctionResult fn(uint8_t *Data, uint64_t Size); + /// \endcode{.cpp} + shared::WrapperFunctionResult callWrapper(ExecutorAddr WrapperFnAddr, + ArrayRef<char> ArgBuffer) { + std::promise<shared::WrapperFunctionResult> RP; + auto RF = RP.get_future(); + callWrapperAsync( + RunInPlace(), WrapperFnAddr, + [&](shared::WrapperFunctionResult R) { + RP.set_value(std::move(R)); + }, ArgBuffer); + return RF.get(); + } + + /// Run a wrapper function using SPS to serialize the arguments and + /// deserialize the results. + template <typename SPSSignature, typename RunPolicyT, typename SendResultT, + typename... ArgTs> + void callSPSWrapperAsync(RunPolicyT &&Runner, ExecutorAddr WrapperFnAddr, + SendResultT &&SendResult, const ArgTs &...Args) { + shared::WrapperFunction<SPSSignature>::callAsync( + [this, WrapperFnAddr, Runner = std::move(Runner)] + (auto &&SendResult, const char *ArgData, size_t ArgSize) mutable { + this->callWrapperAsync(std::move(Runner), WrapperFnAddr, + std::move(SendResult), + ArrayRef<char>(ArgData, ArgSize)); + }, + std::forward<SendResultT>(SendResult), Args...); + } + + /// Run a wrapper function using SPS to serialize the arguments and + /// deserialize the results. + template <typename SPSSignature, typename SendResultT, typename... ArgTs> + void callSPSWrapperAsync(ExecutorAddr WrapperFnAddr, SendResultT &&SendResult, + const ArgTs &...Args) { + callSPSWrapperAsync<SPSSignature>(RunAsTask(*D), WrapperFnAddr, + std::forward<SendResultT>(SendResult), + Args...); + } + + /// Run a wrapper function using SPS to serialize the arguments and + /// deserialize the results. + /// + /// If SPSSignature is a non-void function signature then the second argument + /// (the first in the Args list) should be a reference to a return value. + template <typename SPSSignature, typename... WrapperCallArgTs> + Error callSPSWrapper(ExecutorAddr WrapperFnAddr, + WrapperCallArgTs &&...WrapperCallArgs) { + return shared::WrapperFunction<SPSSignature>::call( + [this, WrapperFnAddr](const char *ArgData, size_t ArgSize) { + return callWrapper(WrapperFnAddr, ArrayRef<char>(ArgData, ArgSize)); + }, + std::forward<WrapperCallArgTs>(WrapperCallArgs)...); + } + + /// Disconnect from the target process. + /// + /// This should be called after the JIT session is shut down. + virtual Error disconnect() = 0; + +protected: + + std::shared_ptr<SymbolStringPool> SSP; + std::unique_ptr<TaskDispatcher> D; + ExecutionSession *ES = nullptr; + Triple TargetTriple; + unsigned PageSize = 0; + JITDispatchInfo JDI; + MemoryAccess *MemAccess = nullptr; + jitlink::JITLinkMemoryManager *MemMgr = nullptr; + StringMap<ExecutorAddr> BootstrapSymbols; +}; + +/// A ExecutorProcessControl instance that asserts if any of its methods are +/// used. Suitable for use is unit tests, and by ORC clients who haven't moved +/// to ExecutorProcessControl-based APIs yet. +class UnsupportedExecutorProcessControl : public ExecutorProcessControl { +public: + UnsupportedExecutorProcessControl( + std::shared_ptr<SymbolStringPool> SSP = nullptr, + std::unique_ptr<TaskDispatcher> D = nullptr, + const std::string &TT = "", unsigned PageSize = 0) + : ExecutorProcessControl(SSP ? std::move(SSP) + : std::make_shared<SymbolStringPool>(), + D ? std::move(D) + : std::make_unique<InPlaceTaskDispatcher>()) { + this->TargetTriple = Triple(TT); + this->PageSize = PageSize; + } + + Expected<tpctypes::DylibHandle> loadDylib(const char *DylibPath) override { + llvm_unreachable("Unsupported"); + } + + Expected<std::vector<tpctypes::LookupResult>> + lookupSymbols(ArrayRef<LookupRequest> Request) override { + llvm_unreachable("Unsupported"); + } + + Expected<int32_t> runAsMain(ExecutorAddr MainFnAddr, + ArrayRef<std::string> Args) override { + llvm_unreachable("Unsupported"); + } + + void callWrapperAsync(ExecutorAddr WrapperFnAddr, + IncomingWFRHandler OnComplete, + ArrayRef<char> ArgBuffer) override { + llvm_unreachable("Unsupported"); + } + + Error disconnect() override { return Error::success(); } +}; + +/// A ExecutorProcessControl implementation targeting the current process. +class SelfExecutorProcessControl + : public ExecutorProcessControl, + private ExecutorProcessControl::MemoryAccess { +public: + SelfExecutorProcessControl( + std::shared_ptr<SymbolStringPool> SSP, std::unique_ptr<TaskDispatcher> D, + Triple TargetTriple, unsigned PageSize, + std::unique_ptr<jitlink::JITLinkMemoryManager> MemMgr); + + /// Create a SelfExecutorProcessControl with the given symbol string pool and + /// memory manager. + /// If no symbol string pool is given then one will be created. + /// If no memory manager is given a jitlink::InProcessMemoryManager will + /// be created and used by default. + static Expected<std::unique_ptr<SelfExecutorProcessControl>> + Create(std::shared_ptr<SymbolStringPool> SSP = nullptr, + std::unique_ptr<TaskDispatcher> D = nullptr, + std::unique_ptr<jitlink::JITLinkMemoryManager> MemMgr = nullptr); + + Expected<tpctypes::DylibHandle> loadDylib(const char *DylibPath) override; + + Expected<std::vector<tpctypes::LookupResult>> + lookupSymbols(ArrayRef<LookupRequest> Request) override; + + Expected<int32_t> runAsMain(ExecutorAddr MainFnAddr, + ArrayRef<std::string> Args) override; + + void callWrapperAsync(ExecutorAddr WrapperFnAddr, + IncomingWFRHandler OnComplete, + ArrayRef<char> ArgBuffer) override; + + Error disconnect() override; + +private: + void writeUInt8sAsync(ArrayRef<tpctypes::UInt8Write> Ws, + WriteResultFn OnWriteComplete) override; + + void writeUInt16sAsync(ArrayRef<tpctypes::UInt16Write> Ws, + WriteResultFn OnWriteComplete) override; + + void writeUInt32sAsync(ArrayRef<tpctypes::UInt32Write> Ws, + WriteResultFn OnWriteComplete) override; + + void writeUInt64sAsync(ArrayRef<tpctypes::UInt64Write> Ws, + WriteResultFn OnWriteComplete) override; + + void writeBuffersAsync(ArrayRef<tpctypes::BufferWrite> Ws, + WriteResultFn OnWriteComplete) override; + + static shared::CWrapperFunctionResult + jitDispatchViaWrapperFunctionManager(void *Ctx, const void *FnTag, + const char *Data, size_t Size); + + std::unique_ptr<jitlink::JITLinkMemoryManager> OwnedMemMgr; + char GlobalManglingPrefix = 0; + std::vector<std::unique_ptr<sys::DynamicLibrary>> DynamicLibraries; +}; + +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_EXECUTORPROCESSCONTROL_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/IRCompileLayer.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/IRCompileLayer.h new file mode 100644 index 0000000000..be74a2d36b --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/IRCompileLayer.h @@ -0,0 +1,84 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===- IRCompileLayer.h -- Eagerly compile IR for JIT -----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Contains the definition for a basic, eagerly compiling layer of the JIT. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_IRCOMPILELAYER_H +#define LLVM_EXECUTIONENGINE_ORC_IRCOMPILELAYER_H + +#include "llvm/ADT/STLExtras.h" +#include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/ExecutionEngine/Orc/Layer.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/MemoryBuffer.h" +#include <functional> +#include <memory> +#include <mutex> + +namespace llvm { + +class Module; + +namespace orc { + +class IRCompileLayer : public IRLayer { +public: + class IRCompiler { + public: + IRCompiler(IRSymbolMapper::ManglingOptions MO) : MO(std::move(MO)) {} + virtual ~IRCompiler(); + const IRSymbolMapper::ManglingOptions &getManglingOptions() const { + return MO; + } + virtual Expected<std::unique_ptr<MemoryBuffer>> operator()(Module &M) = 0; + + protected: + IRSymbolMapper::ManglingOptions &manglingOptions() { return MO; } + + private: + IRSymbolMapper::ManglingOptions MO; + }; + + using NotifyCompiledFunction = std::function<void( + MaterializationResponsibility &R, ThreadSafeModule TSM)>; + + IRCompileLayer(ExecutionSession &ES, ObjectLayer &BaseLayer, + std::unique_ptr<IRCompiler> Compile); + + IRCompiler &getCompiler() { return *Compile; } + + void setNotifyCompiled(NotifyCompiledFunction NotifyCompiled); + + void emit(std::unique_ptr<MaterializationResponsibility> R, + ThreadSafeModule TSM) override; + +private: + mutable std::mutex IRLayerMutex; + ObjectLayer &BaseLayer; + std::unique_ptr<IRCompiler> Compile; + const IRSymbolMapper::ManglingOptions *ManglingOpts; + NotifyCompiledFunction NotifyCompiled = NotifyCompiledFunction(); +}; + +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_IRCOMPILELAYER_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h new file mode 100644 index 0000000000..a8972c08b8 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h @@ -0,0 +1,66 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===- IRTransformLayer.h - Run all IR through a functor --------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Run all IR passed in through a user supplied functor. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_IRTRANSFORMLAYER_H +#define LLVM_EXECUTIONENGINE_ORC_IRTRANSFORMLAYER_H + +#include "llvm/ADT/FunctionExtras.h" +#include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/ExecutionEngine/Orc/Layer.h" +#include <memory> + +namespace llvm { +namespace orc { + +/// A layer that applies a transform to emitted modules. +/// The transform function is responsible for locking the ThreadSafeContext +/// before operating on the module. +class IRTransformLayer : public IRLayer { +public: + using TransformFunction = unique_function<Expected<ThreadSafeModule>( + ThreadSafeModule, MaterializationResponsibility &R)>; + + IRTransformLayer(ExecutionSession &ES, IRLayer &BaseLayer, + TransformFunction Transform = identityTransform); + + void setTransform(TransformFunction Transform) { + this->Transform = std::move(Transform); + } + + void emit(std::unique_ptr<MaterializationResponsibility> R, + ThreadSafeModule TSM) override; + + static ThreadSafeModule identityTransform(ThreadSafeModule TSM, + MaterializationResponsibility &R) { + return TSM; + } + +private: + IRLayer &BaseLayer; + TransformFunction Transform; +}; + +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_IRTRANSFORMLAYER_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h new file mode 100644 index 0000000000..170a38ac6e --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h @@ -0,0 +1,609 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===- IndirectionUtils.h - Utilities for adding indirections ---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Contains utilities for adding indirections and breaking up modules. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H +#define LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H + +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/OrcABISupport.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/Memory.h" +#include "llvm/Support/Process.h" +#include "llvm/Transforms/Utils/ValueMapper.h" +#include <algorithm> +#include <cassert> +#include <cstdint> +#include <functional> +#include <future> +#include <map> +#include <memory> +#include <system_error> +#include <utility> +#include <vector> + +namespace llvm { + +class Constant; +class Function; +class FunctionType; +class GlobalAlias; +class GlobalVariable; +class Module; +class PointerType; +class Triple; +class Twine; +class Value; +class MCDisassembler; +class MCInstrAnalysis; + +namespace jitlink { +class LinkGraph; +class Symbol; +} // namespace jitlink + +namespace orc { + +/// Base class for pools of compiler re-entry trampolines. +/// These trampolines are callable addresses that save all register state +/// before calling a supplied function to return the trampoline landing +/// address, then restore all state before jumping to that address. They +/// are used by various ORC APIs to support lazy compilation +class TrampolinePool { +public: + using NotifyLandingResolvedFunction = + unique_function<void(JITTargetAddress) const>; + + using ResolveLandingFunction = unique_function<void( + JITTargetAddress TrampolineAddr, + NotifyLandingResolvedFunction OnLandingResolved) const>; + + virtual ~TrampolinePool(); + + /// Get an available trampoline address. + /// Returns an error if no trampoline can be created. + Expected<JITTargetAddress> getTrampoline() { + std::lock_guard<std::mutex> Lock(TPMutex); + if (AvailableTrampolines.empty()) { + if (auto Err = grow()) + return std::move(Err); + } + assert(!AvailableTrampolines.empty() && "Failed to grow trampoline pool"); + auto TrampolineAddr = AvailableTrampolines.back(); + AvailableTrampolines.pop_back(); + return TrampolineAddr; + } + + /// Returns the given trampoline to the pool for re-use. + void releaseTrampoline(JITTargetAddress TrampolineAddr) { + std::lock_guard<std::mutex> Lock(TPMutex); + AvailableTrampolines.push_back(TrampolineAddr); + } + +protected: + virtual Error grow() = 0; + + std::mutex TPMutex; + std::vector<JITTargetAddress> AvailableTrampolines; +}; + +/// A trampoline pool for trampolines within the current process. +template <typename ORCABI> class LocalTrampolinePool : public TrampolinePool { +public: + /// Creates a LocalTrampolinePool with the given RunCallback function. + /// Returns an error if this function is unable to correctly allocate, write + /// and protect the resolver code block. + static Expected<std::unique_ptr<LocalTrampolinePool>> + Create(ResolveLandingFunction ResolveLanding) { + Error Err = Error::success(); + + auto LTP = std::unique_ptr<LocalTrampolinePool>( + new LocalTrampolinePool(std::move(ResolveLanding), Err)); + + if (Err) + return std::move(Err); + return std::move(LTP); + } + +private: + static JITTargetAddress reenter(void *TrampolinePoolPtr, void *TrampolineId) { + LocalTrampolinePool<ORCABI> *TrampolinePool = + static_cast<LocalTrampolinePool *>(TrampolinePoolPtr); + + std::promise<JITTargetAddress> LandingAddressP; + auto LandingAddressF = LandingAddressP.get_future(); + + TrampolinePool->ResolveLanding(pointerToJITTargetAddress(TrampolineId), + [&](JITTargetAddress LandingAddress) { + LandingAddressP.set_value(LandingAddress); + }); + return LandingAddressF.get(); + } + + LocalTrampolinePool(ResolveLandingFunction ResolveLanding, Error &Err) + : ResolveLanding(std::move(ResolveLanding)) { + + ErrorAsOutParameter _(&Err); + + /// Try to set up the resolver block. + std::error_code EC; + ResolverBlock = sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory( + ORCABI::ResolverCodeSize, nullptr, + sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC)); + if (EC) { + Err = errorCodeToError(EC); + return; + } + + ORCABI::writeResolverCode(static_cast<char *>(ResolverBlock.base()), + pointerToJITTargetAddress(ResolverBlock.base()), + pointerToJITTargetAddress(&reenter), + pointerToJITTargetAddress(this)); + + EC = sys::Memory::protectMappedMemory(ResolverBlock.getMemoryBlock(), + sys::Memory::MF_READ | + sys::Memory::MF_EXEC); + if (EC) { + Err = errorCodeToError(EC); + return; + } + } + + Error grow() override { + assert(AvailableTrampolines.empty() && "Growing prematurely?"); + + std::error_code EC; + auto TrampolineBlock = + sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory( + sys::Process::getPageSizeEstimate(), nullptr, + sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC)); + if (EC) + return errorCodeToError(EC); + + unsigned NumTrampolines = + (sys::Process::getPageSizeEstimate() - ORCABI::PointerSize) / + ORCABI::TrampolineSize; + + char *TrampolineMem = static_cast<char *>(TrampolineBlock.base()); + ORCABI::writeTrampolines( + TrampolineMem, pointerToJITTargetAddress(TrampolineMem), + pointerToJITTargetAddress(ResolverBlock.base()), NumTrampolines); + + for (unsigned I = 0; I < NumTrampolines; ++I) + AvailableTrampolines.push_back(pointerToJITTargetAddress( + TrampolineMem + (I * ORCABI::TrampolineSize))); + + if (auto EC = sys::Memory::protectMappedMemory( + TrampolineBlock.getMemoryBlock(), + sys::Memory::MF_READ | sys::Memory::MF_EXEC)) + return errorCodeToError(EC); + + TrampolineBlocks.push_back(std::move(TrampolineBlock)); + return Error::success(); + } + + ResolveLandingFunction ResolveLanding; + + sys::OwningMemoryBlock ResolverBlock; + std::vector<sys::OwningMemoryBlock> TrampolineBlocks; +}; + +/// Target-independent base class for compile callback management. +class JITCompileCallbackManager { +public: + using CompileFunction = std::function<JITTargetAddress()>; + + virtual ~JITCompileCallbackManager() = default; + + /// Reserve a compile callback. + Expected<JITTargetAddress> getCompileCallback(CompileFunction Compile); + + /// Execute the callback for the given trampoline id. Called by the JIT + /// to compile functions on demand. + JITTargetAddress executeCompileCallback(JITTargetAddress TrampolineAddr); + +protected: + /// Construct a JITCompileCallbackManager. + JITCompileCallbackManager(std::unique_ptr<TrampolinePool> TP, + ExecutionSession &ES, + JITTargetAddress ErrorHandlerAddress) + : TP(std::move(TP)), ES(ES), + CallbacksJD(ES.createBareJITDylib("<Callbacks>")), + ErrorHandlerAddress(ErrorHandlerAddress) {} + + void setTrampolinePool(std::unique_ptr<TrampolinePool> TP) { + this->TP = std::move(TP); + } + +private: + std::mutex CCMgrMutex; + std::unique_ptr<TrampolinePool> TP; + ExecutionSession &ES; + JITDylib &CallbacksJD; + JITTargetAddress ErrorHandlerAddress; + std::map<JITTargetAddress, SymbolStringPtr> AddrToSymbol; + size_t NextCallbackId = 0; +}; + +/// Manage compile callbacks for in-process JITs. +template <typename ORCABI> +class LocalJITCompileCallbackManager : public JITCompileCallbackManager { +public: + /// Create a new LocalJITCompileCallbackManager. + static Expected<std::unique_ptr<LocalJITCompileCallbackManager>> + Create(ExecutionSession &ES, JITTargetAddress ErrorHandlerAddress) { + Error Err = Error::success(); + auto CCMgr = std::unique_ptr<LocalJITCompileCallbackManager>( + new LocalJITCompileCallbackManager(ES, ErrorHandlerAddress, Err)); + if (Err) + return std::move(Err); + return std::move(CCMgr); + } + +private: + /// Construct a InProcessJITCompileCallbackManager. + /// @param ErrorHandlerAddress The address of an error handler in the target + /// process to be used if a compile callback fails. + LocalJITCompileCallbackManager(ExecutionSession &ES, + JITTargetAddress ErrorHandlerAddress, + Error &Err) + : JITCompileCallbackManager(nullptr, ES, ErrorHandlerAddress) { + using NotifyLandingResolvedFunction = + TrampolinePool::NotifyLandingResolvedFunction; + + ErrorAsOutParameter _(&Err); + auto TP = LocalTrampolinePool<ORCABI>::Create( + [this](JITTargetAddress TrampolineAddr, + NotifyLandingResolvedFunction NotifyLandingResolved) { + NotifyLandingResolved(executeCompileCallback(TrampolineAddr)); + }); + + if (!TP) { + Err = TP.takeError(); + return; + } + + setTrampolinePool(std::move(*TP)); + } +}; + +/// Base class for managing collections of named indirect stubs. +class IndirectStubsManager { +public: + /// Map type for initializing the manager. See init. + using StubInitsMap = StringMap<std::pair<JITTargetAddress, JITSymbolFlags>>; + + virtual ~IndirectStubsManager() = default; + + /// Create a single stub with the given name, target address and flags. + virtual Error createStub(StringRef StubName, JITTargetAddress StubAddr, + JITSymbolFlags StubFlags) = 0; + + /// Create StubInits.size() stubs with the given names, target + /// addresses, and flags. + virtual Error createStubs(const StubInitsMap &StubInits) = 0; + + /// Find the stub with the given name. If ExportedStubsOnly is true, + /// this will only return a result if the stub's flags indicate that it + /// is exported. + virtual JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly) = 0; + + /// Find the implementation-pointer for the stub. + virtual JITEvaluatedSymbol findPointer(StringRef Name) = 0; + + /// Change the value of the implementation pointer for the stub. + virtual Error updatePointer(StringRef Name, JITTargetAddress NewAddr) = 0; + +private: + virtual void anchor(); +}; + +template <typename ORCABI> class LocalIndirectStubsInfo { +public: + LocalIndirectStubsInfo(unsigned NumStubs, sys::OwningMemoryBlock StubsMem) + : NumStubs(NumStubs), StubsMem(std::move(StubsMem)) {} + + static Expected<LocalIndirectStubsInfo> create(unsigned MinStubs, + unsigned PageSize) { + auto ISAS = getIndirectStubsBlockSizes<ORCABI>(MinStubs, PageSize); + + assert((ISAS.StubBytes % PageSize == 0) && + "StubBytes is not a page size multiple"); + uint64_t PointerAlloc = alignTo(ISAS.PointerBytes, PageSize); + + // Allocate memory for stubs and pointers in one call. + std::error_code EC; + auto StubsAndPtrsMem = + sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory( + ISAS.StubBytes + PointerAlloc, nullptr, + sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC)); + if (EC) + return errorCodeToError(EC); + + sys::MemoryBlock StubsBlock(StubsAndPtrsMem.base(), ISAS.StubBytes); + auto StubsBlockMem = static_cast<char *>(StubsAndPtrsMem.base()); + auto PtrBlockAddress = + pointerToJITTargetAddress(StubsBlockMem) + ISAS.StubBytes; + + ORCABI::writeIndirectStubsBlock(StubsBlockMem, + pointerToJITTargetAddress(StubsBlockMem), + PtrBlockAddress, ISAS.NumStubs); + + if (auto EC = sys::Memory::protectMappedMemory( + StubsBlock, sys::Memory::MF_READ | sys::Memory::MF_EXEC)) + return errorCodeToError(EC); + + return LocalIndirectStubsInfo(ISAS.NumStubs, std::move(StubsAndPtrsMem)); + } + + unsigned getNumStubs() const { return NumStubs; } + + void *getStub(unsigned Idx) const { + return static_cast<char *>(StubsMem.base()) + Idx * ORCABI::StubSize; + } + + void **getPtr(unsigned Idx) const { + char *PtrsBase = + static_cast<char *>(StubsMem.base()) + NumStubs * ORCABI::StubSize; + return reinterpret_cast<void **>(PtrsBase) + Idx; + } + +private: + unsigned NumStubs = 0; + sys::OwningMemoryBlock StubsMem; +}; + +/// IndirectStubsManager implementation for the host architecture, e.g. +/// OrcX86_64. (See OrcArchitectureSupport.h). +template <typename TargetT> +class LocalIndirectStubsManager : public IndirectStubsManager { +public: + Error createStub(StringRef StubName, JITTargetAddress StubAddr, + JITSymbolFlags StubFlags) override { + std::lock_guard<std::mutex> Lock(StubsMutex); + if (auto Err = reserveStubs(1)) + return Err; + + createStubInternal(StubName, StubAddr, StubFlags); + + return Error::success(); + } + + Error createStubs(const StubInitsMap &StubInits) override { + std::lock_guard<std::mutex> Lock(StubsMutex); + if (auto Err = reserveStubs(StubInits.size())) + return Err; + + for (auto &Entry : StubInits) + createStubInternal(Entry.first(), Entry.second.first, + Entry.second.second); + + return Error::success(); + } + + JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly) override { + std::lock_guard<std::mutex> Lock(StubsMutex); + auto I = StubIndexes.find(Name); + if (I == StubIndexes.end()) + return nullptr; + auto Key = I->second.first; + void *StubAddr = IndirectStubsInfos[Key.first].getStub(Key.second); + assert(StubAddr && "Missing stub address"); + auto StubTargetAddr = + static_cast<JITTargetAddress>(reinterpret_cast<uintptr_t>(StubAddr)); + auto StubSymbol = JITEvaluatedSymbol(StubTargetAddr, I->second.second); + if (ExportedStubsOnly && !StubSymbol.getFlags().isExported()) + return nullptr; + return StubSymbol; + } + + JITEvaluatedSymbol findPointer(StringRef Name) override { + std::lock_guard<std::mutex> Lock(StubsMutex); + auto I = StubIndexes.find(Name); + if (I == StubIndexes.end()) + return nullptr; + auto Key = I->second.first; + void *PtrAddr = IndirectStubsInfos[Key.first].getPtr(Key.second); + assert(PtrAddr && "Missing pointer address"); + auto PtrTargetAddr = + static_cast<JITTargetAddress>(reinterpret_cast<uintptr_t>(PtrAddr)); + return JITEvaluatedSymbol(PtrTargetAddr, I->second.second); + } + + Error updatePointer(StringRef Name, JITTargetAddress NewAddr) override { + using AtomicIntPtr = std::atomic<uintptr_t>; + + std::lock_guard<std::mutex> Lock(StubsMutex); + auto I = StubIndexes.find(Name); + assert(I != StubIndexes.end() && "No stub pointer for symbol"); + auto Key = I->second.first; + AtomicIntPtr *AtomicStubPtr = reinterpret_cast<AtomicIntPtr *>( + IndirectStubsInfos[Key.first].getPtr(Key.second)); + *AtomicStubPtr = static_cast<uintptr_t>(NewAddr); + return Error::success(); + } + +private: + Error reserveStubs(unsigned NumStubs) { + if (NumStubs <= FreeStubs.size()) + return Error::success(); + + unsigned NewStubsRequired = NumStubs - FreeStubs.size(); + unsigned NewBlockId = IndirectStubsInfos.size(); + auto ISI = + LocalIndirectStubsInfo<TargetT>::create(NewStubsRequired, PageSize); + if (!ISI) + return ISI.takeError(); + for (unsigned I = 0; I < ISI->getNumStubs(); ++I) + FreeStubs.push_back(std::make_pair(NewBlockId, I)); + IndirectStubsInfos.push_back(std::move(*ISI)); + return Error::success(); + } + + void createStubInternal(StringRef StubName, JITTargetAddress InitAddr, + JITSymbolFlags StubFlags) { + auto Key = FreeStubs.back(); + FreeStubs.pop_back(); + *IndirectStubsInfos[Key.first].getPtr(Key.second) = + jitTargetAddressToPointer<void *>(InitAddr); + StubIndexes[StubName] = std::make_pair(Key, StubFlags); + } + + unsigned PageSize = sys::Process::getPageSizeEstimate(); + std::mutex StubsMutex; + std::vector<LocalIndirectStubsInfo<TargetT>> IndirectStubsInfos; + using StubKey = std::pair<uint16_t, uint16_t>; + std::vector<StubKey> FreeStubs; + StringMap<std::pair<StubKey, JITSymbolFlags>> StubIndexes; +}; + +/// Create a local compile callback manager. +/// +/// The given target triple will determine the ABI, and the given +/// ErrorHandlerAddress will be used by the resulting compile callback +/// manager if a compile callback fails. +Expected<std::unique_ptr<JITCompileCallbackManager>> +createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES, + JITTargetAddress ErrorHandlerAddress); + +/// Create a local indriect stubs manager builder. +/// +/// The given target triple will determine the ABI. +std::function<std::unique_ptr<IndirectStubsManager>()> +createLocalIndirectStubsManagerBuilder(const Triple &T); + +/// Build a function pointer of FunctionType with the given constant +/// address. +/// +/// Usage example: Turn a trampoline address into a function pointer constant +/// for use in a stub. +Constant *createIRTypedAddress(FunctionType &FT, JITTargetAddress Addr); + +/// Create a function pointer with the given type, name, and initializer +/// in the given Module. +GlobalVariable *createImplPointer(PointerType &PT, Module &M, const Twine &Name, + Constant *Initializer); + +/// Turn a function declaration into a stub function that makes an +/// indirect call using the given function pointer. +void makeStub(Function &F, Value &ImplPointer); + +/// Promotes private symbols to global hidden, and renames to prevent clashes +/// with other promoted symbols. The same SymbolPromoter instance should be +/// used for all symbols to be added to a single JITDylib. +class SymbolLinkagePromoter { +public: + /// Promote symbols in the given module. Returns the set of global values + /// that have been renamed/promoted. + std::vector<GlobalValue *> operator()(Module &M); + +private: + unsigned NextId = 0; +}; + +/// Clone a function declaration into a new module. +/// +/// This function can be used as the first step towards creating a callback +/// stub (see makeStub), or moving a function body (see moveFunctionBody). +/// +/// If the VMap argument is non-null, a mapping will be added between F and +/// the new declaration, and between each of F's arguments and the new +/// declaration's arguments. This map can then be passed in to moveFunction to +/// move the function body if required. Note: When moving functions between +/// modules with these utilities, all decls should be cloned (and added to a +/// single VMap) before any bodies are moved. This will ensure that references +/// between functions all refer to the versions in the new module. +Function *cloneFunctionDecl(Module &Dst, const Function &F, + ValueToValueMapTy *VMap = nullptr); + +/// Move the body of function 'F' to a cloned function declaration in a +/// different module (See related cloneFunctionDecl). +/// +/// If the target function declaration is not supplied via the NewF parameter +/// then it will be looked up via the VMap. +/// +/// This will delete the body of function 'F' from its original parent module, +/// but leave its declaration. +void moveFunctionBody(Function &OrigF, ValueToValueMapTy &VMap, + ValueMaterializer *Materializer = nullptr, + Function *NewF = nullptr); + +/// Clone a global variable declaration into a new module. +GlobalVariable *cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV, + ValueToValueMapTy *VMap = nullptr); + +/// Move global variable GV from its parent module to cloned global +/// declaration in a different module. +/// +/// If the target global declaration is not supplied via the NewGV parameter +/// then it will be looked up via the VMap. +/// +/// This will delete the initializer of GV from its original parent module, +/// but leave its declaration. +void moveGlobalVariableInitializer(GlobalVariable &OrigGV, + ValueToValueMapTy &VMap, + ValueMaterializer *Materializer = nullptr, + GlobalVariable *NewGV = nullptr); + +/// Clone a global alias declaration into a new module. +GlobalAlias *cloneGlobalAliasDecl(Module &Dst, const GlobalAlias &OrigA, + ValueToValueMapTy &VMap); + +/// Clone module flags metadata into the destination module. +void cloneModuleFlagsMetadata(Module &Dst, const Module &Src, + ValueToValueMapTy &VMap); + +/// Introduce relocations to \p Sym in its own definition if there are any +/// pointers formed via PC-relative address that do not already have a +/// relocation. +/// +/// This is useful when introducing indirection via a stub function at link time +/// without compiler support. If a function pointer is formed without a +/// relocation, e.g. in the definition of \c foo +/// +/// \code +/// _foo: +/// leaq -7(%rip), rax # form pointer to _foo without relocation +/// _bar: +/// leaq (%rip), %rax # uses X86_64_RELOC_SIGNED to '_foo' +/// \endcode +/// +/// the pointer to \c _foo computed by \c _foo and \c _bar may differ if we +/// introduce a stub for _foo. If the pointer is used as a key, this may be +/// observable to the program. This pass will attempt to introduce the missing +/// "self-relocation" on the leaq instruction. +/// +/// This is based on disassembly and should be considered "best effort". It may +/// silently fail to add relocations. +Error addFunctionPointerRelocationsToCurrentSymbol(jitlink::Symbol &Sym, + jitlink::LinkGraph &G, + MCDisassembler &Disassembler, + MCInstrAnalysis &MIA); + +} // end namespace orc + +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h new file mode 100644 index 0000000000..1bb7fae54f --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h @@ -0,0 +1,192 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===- JITTargetMachineBuilder.h - Build TargetMachines for JIT -*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// A utitily for building TargetMachines for JITs. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_JITTARGETMACHINEBUILDER_H +#define LLVM_EXECUTIONENGINE_ORC_JITTARGETMACHINEBUILDER_H + +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/Triple.h" +#include "llvm/MC/SubtargetFeature.h" +#include "llvm/Support/CodeGen.h" +#include "llvm/Support/Error.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetOptions.h" +#include <memory> +#include <string> +#include <vector> + +namespace llvm { + +class raw_ostream; + +namespace orc { + +/// A utility class for building TargetMachines for JITs. +class JITTargetMachineBuilder { +#ifndef NDEBUG + friend class JITTargetMachineBuilderPrinter; +#endif +public: + /// Create a JITTargetMachineBuilder based on the given triple. + /// + /// Note: TargetOptions is default-constructed, then EmulatedTLS and + /// ExplicitEmulatedTLS are set to true. If EmulatedTLS is not + /// required, these values should be reset before calling + /// createTargetMachine. + JITTargetMachineBuilder(Triple TT); + + /// Create a JITTargetMachineBuilder for the host system. + /// + /// Note: TargetOptions is default-constructed, then EmulatedTLS and + /// ExplicitEmulatedTLS are set to true. If EmulatedTLS is not + /// required, these values should be reset before calling + /// createTargetMachine. + static Expected<JITTargetMachineBuilder> detectHost(); + + /// Create a TargetMachine. + /// + /// This operation will fail if the requested target is not registered, + /// in which case see llvm/Support/TargetSelect.h. To JIT IR the Target and + /// the target's AsmPrinter must both be registered. To JIT assembly + /// (including inline and module level assembly) the target's AsmParser must + /// also be registered. + Expected<std::unique_ptr<TargetMachine>> createTargetMachine(); + + /// Get the default DataLayout for the target. + /// + /// Note: This is reasonably expensive, as it creates a temporary + /// TargetMachine instance under the hood. It is only suitable for use during + /// JIT setup. + Expected<DataLayout> getDefaultDataLayoutForTarget() { + auto TM = createTargetMachine(); + if (!TM) + return TM.takeError(); + return (*TM)->createDataLayout(); + } + + /// Set the CPU string. + JITTargetMachineBuilder &setCPU(std::string CPU) { + this->CPU = std::move(CPU); + return *this; + } + + /// Returns the CPU string. + const std::string &getCPU() const { return CPU; } + + /// Set the relocation model. + JITTargetMachineBuilder &setRelocationModel(Optional<Reloc::Model> RM) { + this->RM = std::move(RM); + return *this; + } + + /// Get the relocation model. + const Optional<Reloc::Model> &getRelocationModel() const { return RM; } + + /// Set the code model. + JITTargetMachineBuilder &setCodeModel(Optional<CodeModel::Model> CM) { + this->CM = std::move(CM); + return *this; + } + + /// Get the code model. + const Optional<CodeModel::Model> &getCodeModel() const { return CM; } + + /// Set the LLVM CodeGen optimization level. + JITTargetMachineBuilder &setCodeGenOptLevel(CodeGenOpt::Level OptLevel) { + this->OptLevel = OptLevel; + return *this; + } + + /// Set subtarget features. + JITTargetMachineBuilder &setFeatures(StringRef FeatureString) { + Features = SubtargetFeatures(FeatureString); + return *this; + } + + /// Add subtarget features. + JITTargetMachineBuilder & + addFeatures(const std::vector<std::string> &FeatureVec); + + /// Access subtarget features. + SubtargetFeatures &getFeatures() { return Features; } + + /// Access subtarget features. + const SubtargetFeatures &getFeatures() const { return Features; } + + /// Set TargetOptions. + /// + /// Note: This operation will overwrite any previously configured options, + /// including EmulatedTLS and ExplicitEmulatedTLS which + /// the JITTargetMachineBuilder sets by default. Clients are responsible + /// for re-enabling these overwritten options. + JITTargetMachineBuilder &setOptions(TargetOptions Options) { + this->Options = std::move(Options); + return *this; + } + + /// Access TargetOptions. + TargetOptions &getOptions() { return Options; } + + /// Access TargetOptions. + const TargetOptions &getOptions() const { return Options; } + + /// Access Triple. + Triple &getTargetTriple() { return TT; } + + /// Access Triple. + const Triple &getTargetTriple() const { return TT; } + +private: + Triple TT; + std::string CPU; + SubtargetFeatures Features; + TargetOptions Options; + Optional<Reloc::Model> RM; + Optional<CodeModel::Model> CM; + CodeGenOpt::Level OptLevel = CodeGenOpt::Default; +}; + +#ifndef NDEBUG +class JITTargetMachineBuilderPrinter { +public: + JITTargetMachineBuilderPrinter(JITTargetMachineBuilder &JTMB, + StringRef Indent) + : JTMB(JTMB), Indent(Indent) {} + void print(raw_ostream &OS) const; + + friend raw_ostream &operator<<(raw_ostream &OS, + const JITTargetMachineBuilderPrinter &JTMBP) { + JTMBP.print(OS); + return OS; + } + +private: + JITTargetMachineBuilder &JTMB; + StringRef Indent; +}; +#endif // NDEBUG + +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_JITTARGETMACHINEBUILDER_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/LLJIT.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/LLJIT.h new file mode 100644 index 0000000000..5a091f2557 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/LLJIT.h @@ -0,0 +1,476 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===----- LLJIT.h -- An ORC-based JIT for compiling LLVM IR ----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// An ORC-based JIT for compiling LLVM IR. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_LLJIT_H +#define LLVM_EXECUTIONENGINE_ORC_LLJIT_H + +#include "llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h" +#include "llvm/ExecutionEngine/Orc/CompileUtils.h" +#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" +#include "llvm/ExecutionEngine/Orc/IRCompileLayer.h" +#include "llvm/ExecutionEngine/Orc/IRTransformLayer.h" +#include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h" +#include "llvm/ExecutionEngine/Orc/ThreadSafeModule.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/ThreadPool.h" + +namespace llvm { +namespace orc { + +class LLJITBuilderState; +class LLLazyJITBuilderState; +class ObjectTransformLayer; +class ExecutorProcessControl; + +/// A pre-fabricated ORC JIT stack that can serve as an alternative to MCJIT. +/// +/// Create instances using LLJITBuilder. +class LLJIT { + template <typename, typename, typename> friend class LLJITBuilderSetters; + + friend void setUpGenericLLVMIRPlatform(LLJIT &J); + +public: + /// Initializer support for LLJIT. + class PlatformSupport { + public: + virtual ~PlatformSupport(); + + virtual Error initialize(JITDylib &JD) = 0; + + virtual Error deinitialize(JITDylib &JD) = 0; + + protected: + static void setInitTransform(LLJIT &J, + IRTransformLayer::TransformFunction T); + }; + + /// Destruct this instance. If a multi-threaded instance, waits for all + /// compile threads to complete. + ~LLJIT(); + + /// Returns the ExecutionSession for this instance. + ExecutionSession &getExecutionSession() { return *ES; } + + /// Returns a reference to the triple for this instance. + const Triple &getTargetTriple() const { return TT; } + + /// Returns a reference to the DataLayout for this instance. + const DataLayout &getDataLayout() const { return DL; } + + /// Returns a reference to the JITDylib representing the JIT'd main program. + JITDylib &getMainJITDylib() { return *Main; } + + /// Returns the JITDylib with the given name, or nullptr if no JITDylib with + /// that name exists. + JITDylib *getJITDylibByName(StringRef Name) { + return ES->getJITDylibByName(Name); + } + + /// Create a new JITDylib with the given name and return a reference to it. + /// + /// JITDylib names must be unique. If the given name is derived from user + /// input or elsewhere in the environment then the client should check + /// (e.g. by calling getJITDylibByName) that the given name is not already in + /// use. + Expected<JITDylib &> createJITDylib(std::string Name) { + return ES->createJITDylib(std::move(Name)); + } + + /// Adds an IR module with the given ResourceTracker. + Error addIRModule(ResourceTrackerSP RT, ThreadSafeModule TSM); + + /// Adds an IR module to the given JITDylib. + Error addIRModule(JITDylib &JD, ThreadSafeModule TSM); + + /// Adds an IR module to the Main JITDylib. + Error addIRModule(ThreadSafeModule TSM) { + return addIRModule(*Main, std::move(TSM)); + } + + /// Adds an object file to the given JITDylib. + Error addObjectFile(ResourceTrackerSP RT, std::unique_ptr<MemoryBuffer> Obj); + + /// Adds an object file to the given JITDylib. + Error addObjectFile(JITDylib &JD, std::unique_ptr<MemoryBuffer> Obj); + + /// Adds an object file to the given JITDylib. + Error addObjectFile(std::unique_ptr<MemoryBuffer> Obj) { + return addObjectFile(*Main, std::move(Obj)); + } + + /// Look up a symbol in JITDylib JD by the symbol's linker-mangled name (to + /// look up symbols based on their IR name use the lookup function instead). + Expected<JITEvaluatedSymbol> lookupLinkerMangled(JITDylib &JD, + SymbolStringPtr Name); + + /// Look up a symbol in JITDylib JD by the symbol's linker-mangled name (to + /// look up symbols based on their IR name use the lookup function instead). + Expected<JITEvaluatedSymbol> lookupLinkerMangled(JITDylib &JD, + StringRef Name) { + return lookupLinkerMangled(JD, ES->intern(Name)); + } + + /// Look up a symbol in the main JITDylib by the symbol's linker-mangled name + /// (to look up symbols based on their IR name use the lookup function + /// instead). + Expected<JITEvaluatedSymbol> lookupLinkerMangled(StringRef Name) { + return lookupLinkerMangled(*Main, Name); + } + + /// Look up a symbol in JITDylib JD based on its IR symbol name. + Expected<JITEvaluatedSymbol> lookup(JITDylib &JD, StringRef UnmangledName) { + return lookupLinkerMangled(JD, mangle(UnmangledName)); + } + + /// Look up a symbol in the main JITDylib based on its IR symbol name. + Expected<JITEvaluatedSymbol> lookup(StringRef UnmangledName) { + return lookup(*Main, UnmangledName); + } + + /// Set the PlatformSupport instance. + void setPlatformSupport(std::unique_ptr<PlatformSupport> PS) { + this->PS = std::move(PS); + } + + /// Get the PlatformSupport instance. + PlatformSupport *getPlatformSupport() { return PS.get(); } + + /// Run the initializers for the given JITDylib. + Error initialize(JITDylib &JD) { + DEBUG_WITH_TYPE("orc", { + dbgs() << "LLJIT running initializers for JITDylib \"" << JD.getName() + << "\"\n"; + }); + assert(PS && "PlatformSupport must be set to run initializers."); + return PS->initialize(JD); + } + + /// Run the deinitializers for the given JITDylib. + Error deinitialize(JITDylib &JD) { + DEBUG_WITH_TYPE("orc", { + dbgs() << "LLJIT running deinitializers for JITDylib \"" << JD.getName() + << "\"\n"; + }); + assert(PS && "PlatformSupport must be set to run initializers."); + return PS->deinitialize(JD); + } + + /// Returns a reference to the ObjLinkingLayer + ObjectLayer &getObjLinkingLayer() { return *ObjLinkingLayer; } + + /// Returns a reference to the object transform layer. + ObjectTransformLayer &getObjTransformLayer() { return *ObjTransformLayer; } + + /// Returns a reference to the IR transform layer. + IRTransformLayer &getIRTransformLayer() { return *TransformLayer; } + + /// Returns a reference to the IR compile layer. + IRCompileLayer &getIRCompileLayer() { return *CompileLayer; } + + /// Returns a linker-mangled version of UnmangledName. + std::string mangle(StringRef UnmangledName) const; + + /// Returns an interned, linker-mangled version of UnmangledName. + SymbolStringPtr mangleAndIntern(StringRef UnmangledName) const { + return ES->intern(mangle(UnmangledName)); + } + +protected: + static Expected<std::unique_ptr<ObjectLayer>> + createObjectLinkingLayer(LLJITBuilderState &S, ExecutionSession &ES); + + static Expected<std::unique_ptr<IRCompileLayer::IRCompiler>> + createCompileFunction(LLJITBuilderState &S, JITTargetMachineBuilder JTMB); + + /// Create an LLJIT instance with a single compile thread. + LLJIT(LLJITBuilderState &S, Error &Err); + + Error applyDataLayout(Module &M); + + void recordCtorDtors(Module &M); + + std::unique_ptr<ExecutionSession> ES; + std::unique_ptr<PlatformSupport> PS; + + JITDylib *Main = nullptr; + + DataLayout DL; + Triple TT; + std::unique_ptr<ThreadPool> CompileThreads; + + std::unique_ptr<ObjectLayer> ObjLinkingLayer; + std::unique_ptr<ObjectTransformLayer> ObjTransformLayer; + std::unique_ptr<IRCompileLayer> CompileLayer; + std::unique_ptr<IRTransformLayer> TransformLayer; + std::unique_ptr<IRTransformLayer> InitHelperTransformLayer; +}; + +/// An extended version of LLJIT that supports lazy function-at-a-time +/// compilation of LLVM IR. +class LLLazyJIT : public LLJIT { + template <typename, typename, typename> friend class LLJITBuilderSetters; + +public: + + /// Sets the partition function. + void + setPartitionFunction(CompileOnDemandLayer::PartitionFunction Partition) { + CODLayer->setPartitionFunction(std::move(Partition)); + } + + /// Returns a reference to the on-demand layer. + CompileOnDemandLayer &getCompileOnDemandLayer() { return *CODLayer; } + + /// Add a module to be lazily compiled to JITDylib JD. + Error addLazyIRModule(JITDylib &JD, ThreadSafeModule M); + + /// Add a module to be lazily compiled to the main JITDylib. + Error addLazyIRModule(ThreadSafeModule M) { + return addLazyIRModule(*Main, std::move(M)); + } + +private: + + // Create a single-threaded LLLazyJIT instance. + LLLazyJIT(LLLazyJITBuilderState &S, Error &Err); + + std::unique_ptr<LazyCallThroughManager> LCTMgr; + std::unique_ptr<CompileOnDemandLayer> CODLayer; +}; + +class LLJITBuilderState { +public: + using ObjectLinkingLayerCreator = + std::function<Expected<std::unique_ptr<ObjectLayer>>(ExecutionSession &, + const Triple &)>; + + using CompileFunctionCreator = + std::function<Expected<std::unique_ptr<IRCompileLayer::IRCompiler>>( + JITTargetMachineBuilder JTMB)>; + + using PlatformSetupFunction = std::function<Error(LLJIT &J)>; + + std::unique_ptr<ExecutorProcessControl> EPC; + std::unique_ptr<ExecutionSession> ES; + Optional<JITTargetMachineBuilder> JTMB; + Optional<DataLayout> DL; + ObjectLinkingLayerCreator CreateObjectLinkingLayer; + CompileFunctionCreator CreateCompileFunction; + PlatformSetupFunction SetUpPlatform; + unsigned NumCompileThreads = 0; + + /// Called prior to JIT class construcion to fix up defaults. + Error prepareForConstruction(); +}; + +template <typename JITType, typename SetterImpl, typename State> +class LLJITBuilderSetters { +public: + /// Set a ExecutorProcessControl for this instance. + /// This should not be called if ExecutionSession has already been set. + SetterImpl & + setExecutorProcessControl(std::unique_ptr<ExecutorProcessControl> EPC) { + assert( + !impl().ES && + "setExecutorProcessControl should not be called if an ExecutionSession " + "has already been set"); + impl().EPC = std::move(EPC); + return impl(); + } + + /// Set an ExecutionSession for this instance. + SetterImpl &setExecutionSession(std::unique_ptr<ExecutionSession> ES) { + impl().ES = std::move(ES); + return impl(); + } + + /// Set the JITTargetMachineBuilder for this instance. + /// + /// If this method is not called, JITTargetMachineBuilder::detectHost will be + /// used to construct a default target machine builder for the host platform. + SetterImpl &setJITTargetMachineBuilder(JITTargetMachineBuilder JTMB) { + impl().JTMB = std::move(JTMB); + return impl(); + } + + /// Return a reference to the JITTargetMachineBuilder. + /// + Optional<JITTargetMachineBuilder> &getJITTargetMachineBuilder() { + return impl().JTMB; + } + + /// Set a DataLayout for this instance. If no data layout is specified then + /// the target's default data layout will be used. + SetterImpl &setDataLayout(Optional<DataLayout> DL) { + impl().DL = std::move(DL); + return impl(); + } + + /// Set an ObjectLinkingLayer creation function. + /// + /// If this method is not called, a default creation function will be used + /// that will construct an RTDyldObjectLinkingLayer. + SetterImpl &setObjectLinkingLayerCreator( + LLJITBuilderState::ObjectLinkingLayerCreator CreateObjectLinkingLayer) { + impl().CreateObjectLinkingLayer = std::move(CreateObjectLinkingLayer); + return impl(); + } + + /// Set a CompileFunctionCreator. + /// + /// If this method is not called, a default creation function wil be used + /// that will construct a basic IR compile function that is compatible with + /// the selected number of threads (SimpleCompiler for '0' compile threads, + /// ConcurrentIRCompiler otherwise). + SetterImpl &setCompileFunctionCreator( + LLJITBuilderState::CompileFunctionCreator CreateCompileFunction) { + impl().CreateCompileFunction = std::move(CreateCompileFunction); + return impl(); + } + + /// Set up an PlatformSetupFunction. + /// + /// If this method is not called then setUpGenericLLVMIRPlatform + /// will be used to configure the JIT's platform support. + SetterImpl & + setPlatformSetUp(LLJITBuilderState::PlatformSetupFunction SetUpPlatform) { + impl().SetUpPlatform = std::move(SetUpPlatform); + return impl(); + } + + /// Set the number of compile threads to use. + /// + /// If set to zero, compilation will be performed on the execution thread when + /// JITing in-process. If set to any other number N, a thread pool of N + /// threads will be created for compilation. + /// + /// If this method is not called, behavior will be as if it were called with + /// a zero argument. + SetterImpl &setNumCompileThreads(unsigned NumCompileThreads) { + impl().NumCompileThreads = NumCompileThreads; + return impl(); + } + + /// Set an ExecutorProcessControl object. + /// + /// If the platform uses ObjectLinkingLayer by default and no + /// ObjectLinkingLayerCreator has been set then the ExecutorProcessControl + /// object will be used to supply the memory manager for the + /// ObjectLinkingLayer. + SetterImpl &setExecutorProcessControl(ExecutorProcessControl &EPC) { + impl().EPC = &EPC; + return impl(); + } + + /// Create an instance of the JIT. + Expected<std::unique_ptr<JITType>> create() { + if (auto Err = impl().prepareForConstruction()) + return std::move(Err); + + Error Err = Error::success(); + std::unique_ptr<JITType> J(new JITType(impl(), Err)); + if (Err) + return std::move(Err); + return std::move(J); + } + +protected: + SetterImpl &impl() { return static_cast<SetterImpl &>(*this); } +}; + +/// Constructs LLJIT instances. +class LLJITBuilder + : public LLJITBuilderState, + public LLJITBuilderSetters<LLJIT, LLJITBuilder, LLJITBuilderState> {}; + +class LLLazyJITBuilderState : public LLJITBuilderState { + friend class LLLazyJIT; + +public: + using IndirectStubsManagerBuilderFunction = + std::function<std::unique_ptr<IndirectStubsManager>()>; + + Triple TT; + JITTargetAddress LazyCompileFailureAddr = 0; + std::unique_ptr<LazyCallThroughManager> LCTMgr; + IndirectStubsManagerBuilderFunction ISMBuilder; + + Error prepareForConstruction(); +}; + +template <typename JITType, typename SetterImpl, typename State> +class LLLazyJITBuilderSetters + : public LLJITBuilderSetters<JITType, SetterImpl, State> { +public: + /// Set the address in the target address to call if a lazy compile fails. + /// + /// If this method is not called then the value will default to 0. + SetterImpl &setLazyCompileFailureAddr(JITTargetAddress Addr) { + this->impl().LazyCompileFailureAddr = Addr; + return this->impl(); + } + + /// Set the lazy-callthrough manager. + /// + /// If this method is not called then a default, in-process lazy callthrough + /// manager for the host platform will be used. + SetterImpl & + setLazyCallthroughManager(std::unique_ptr<LazyCallThroughManager> LCTMgr) { + this->impl().LCTMgr = std::move(LCTMgr); + return this->impl(); + } + + /// Set the IndirectStubsManager builder function. + /// + /// If this method is not called then a default, in-process + /// IndirectStubsManager builder for the host platform will be used. + SetterImpl &setIndirectStubsManagerBuilder( + LLLazyJITBuilderState::IndirectStubsManagerBuilderFunction ISMBuilder) { + this->impl().ISMBuilder = std::move(ISMBuilder); + return this->impl(); + } +}; + +/// Constructs LLLazyJIT instances. +class LLLazyJITBuilder + : public LLLazyJITBuilderState, + public LLLazyJITBuilderSetters<LLLazyJIT, LLLazyJITBuilder, + LLLazyJITBuilderState> {}; + +/// Configure the LLJIT instance to scrape modules for llvm.global_ctors and +/// llvm.global_dtors variables and (if present) build initialization and +/// deinitialization functions. Platform specific initialization configurations +/// should be preferred where available. +void setUpGenericLLVMIRPlatform(LLJIT &J); + +/// Configure the LLJIT instance to disable platform support explicitly. This is +/// useful in two cases: for platforms that don't have such requirements and for +/// platforms, that we have no explicit support yet and that don't work well +/// with the generic IR platform. +Error setUpInactivePlatform(LLJIT &J); + +} // End namespace orc +} // End namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_LLJIT_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/Layer.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/Layer.h new file mode 100644 index 0000000000..4eba02cd42 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/Layer.h @@ -0,0 +1,210 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===---------------- Layer.h -- Layer interfaces --------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Layer interfaces. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_LAYER_H +#define LLVM_EXECUTIONENGINE_ORC_LAYER_H + +#include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/Mangling.h" +#include "llvm/ExecutionEngine/Orc/ThreadSafeModule.h" +#include "llvm/IR/Module.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/ExtensibleRTTI.h" +#include "llvm/Support/MemoryBuffer.h" + +namespace llvm { +namespace orc { + +/// IRMaterializationUnit is a convenient base class for MaterializationUnits +/// wrapping LLVM IR. Represents materialization responsibility for all symbols +/// in the given module. If symbols are overridden by other definitions, then +/// their linkage is changed to available-externally. +class IRMaterializationUnit : public MaterializationUnit { +public: + using SymbolNameToDefinitionMap = std::map<SymbolStringPtr, GlobalValue *>; + + /// Create an IRMaterializationLayer. Scans the module to build the + /// SymbolFlags and SymbolToDefinition maps. + IRMaterializationUnit(ExecutionSession &ES, + const IRSymbolMapper::ManglingOptions &MO, + ThreadSafeModule TSM); + + /// Create an IRMaterializationLayer from a module, and pre-existing + /// SymbolFlags and SymbolToDefinition maps. The maps must provide + /// entries for each definition in M. + /// This constructor is useful for delegating work from one + /// IRMaterializationUnit to another. + IRMaterializationUnit(ThreadSafeModule TSM, Interface I, + SymbolNameToDefinitionMap SymbolToDefinition); + + /// Return the ModuleIdentifier as the name for this MaterializationUnit. + StringRef getName() const override; + + /// Return a reference to the contained ThreadSafeModule. + const ThreadSafeModule &getModule() const { return TSM; } + +protected: + ThreadSafeModule TSM; + SymbolNameToDefinitionMap SymbolToDefinition; + +private: + static SymbolStringPtr getInitSymbol(ExecutionSession &ES, + const ThreadSafeModule &TSM); + + void discard(const JITDylib &JD, const SymbolStringPtr &Name) override; +}; + +/// Interface for layers that accept LLVM IR. +class IRLayer { +public: + IRLayer(ExecutionSession &ES, const IRSymbolMapper::ManglingOptions *&MO) + : ES(ES), MO(MO) {} + + virtual ~IRLayer(); + + /// Returns the ExecutionSession for this layer. + ExecutionSession &getExecutionSession() { return ES; } + + /// Get the mangling options for this layer. + const IRSymbolMapper::ManglingOptions *&getManglingOptions() const { + return MO; + } + + /// Sets the CloneToNewContextOnEmit flag (false by default). + /// + /// When set, IR modules added to this layer will be cloned on to a new + /// context before emit is called. This can be used by clients who want + /// to load all IR using one LLVMContext (to save memory via type and + /// constant uniquing), but want to move Modules to fresh contexts before + /// compiling them to enable concurrent compilation. + /// Single threaded clients, or clients who load every module on a new + /// context, need not set this. + void setCloneToNewContextOnEmit(bool CloneToNewContextOnEmit) { + this->CloneToNewContextOnEmit = CloneToNewContextOnEmit; + } + + /// Returns the current value of the CloneToNewContextOnEmit flag. + bool getCloneToNewContextOnEmit() const { return CloneToNewContextOnEmit; } + + /// Add a MaterializatinoUnit representing the given IR to the JITDylib + /// targeted by the given tracker. + virtual Error add(ResourceTrackerSP RT, ThreadSafeModule TSM); + + /// Adds a MaterializationUnit representing the given IR to the given + /// JITDylib. If RT is not specif + Error add(JITDylib &JD, ThreadSafeModule TSM) { + return add(JD.getDefaultResourceTracker(), std::move(TSM)); + } + + /// Emit should materialize the given IR. + virtual void emit(std::unique_ptr<MaterializationResponsibility> R, + ThreadSafeModule TSM) = 0; + +private: + bool CloneToNewContextOnEmit = false; + ExecutionSession &ES; + const IRSymbolMapper::ManglingOptions *&MO; +}; + +/// MaterializationUnit that materializes modules by calling the 'emit' method +/// on the given IRLayer. +class BasicIRLayerMaterializationUnit : public IRMaterializationUnit { +public: + BasicIRLayerMaterializationUnit(IRLayer &L, + const IRSymbolMapper::ManglingOptions &MO, + ThreadSafeModule TSM); + +private: + void materialize(std::unique_ptr<MaterializationResponsibility> R) override; + + IRLayer &L; +}; + +/// Interface for Layers that accept object files. +class ObjectLayer : public RTTIExtends<ObjectLayer, RTTIRoot> { +public: + static char ID; + + ObjectLayer(ExecutionSession &ES); + virtual ~ObjectLayer(); + + /// Returns the execution session for this layer. + ExecutionSession &getExecutionSession() { return ES; } + + /// Adds a MaterializationUnit for the object file in the given memory buffer + /// to the JITDylib for the given ResourceTracker. + virtual Error add(ResourceTrackerSP RT, std::unique_ptr<MemoryBuffer> O, + MaterializationUnit::Interface I); + + /// Adds a MaterializationUnit for the object file in the given memory buffer + /// to the JITDylib for the given ResourceTracker. The interface for the + /// object will be built using the default object interface builder. + Error add(ResourceTrackerSP RT, std::unique_ptr<MemoryBuffer> O); + + /// Adds a MaterializationUnit for the object file in the given memory buffer + /// to the given JITDylib. + Error add(JITDylib &JD, std::unique_ptr<MemoryBuffer> O, + MaterializationUnit::Interface I) { + return add(JD.getDefaultResourceTracker(), std::move(O), std::move(I)); + } + + /// Adds a MaterializationUnit for the object file in the given memory buffer + /// to the given JITDylib. The interface for the object will be built using + /// the default object interface builder. + Error add(JITDylib &JD, std::unique_ptr<MemoryBuffer> O); + + /// Emit should materialize the given IR. + virtual void emit(std::unique_ptr<MaterializationResponsibility> R, + std::unique_ptr<MemoryBuffer> O) = 0; + +private: + ExecutionSession &ES; +}; + +/// Materializes the given object file (represented by a MemoryBuffer +/// instance) by calling 'emit' on the given ObjectLayer. +class BasicObjectLayerMaterializationUnit : public MaterializationUnit { +public: + /// Create using the default object interface builder function. + static Expected<std::unique_ptr<BasicObjectLayerMaterializationUnit>> + Create(ObjectLayer &L, std::unique_ptr<MemoryBuffer> O); + + BasicObjectLayerMaterializationUnit(ObjectLayer &L, + std::unique_ptr<MemoryBuffer> O, + Interface I); + + /// Return the buffer's identifier as the name for this MaterializationUnit. + StringRef getName() const override; + +private: + void materialize(std::unique_ptr<MaterializationResponsibility> R) override; + void discard(const JITDylib &JD, const SymbolStringPtr &Name) override; + + ObjectLayer &L; + std::unique_ptr<MemoryBuffer> O; +}; + +} // End namespace orc +} // End namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_LAYER_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/LazyReexports.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/LazyReexports.h new file mode 100644 index 0000000000..f08d293423 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/LazyReexports.h @@ -0,0 +1,190 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===------ LazyReexports.h -- Utilities for lazy reexports -----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Lazy re-exports are similar to normal re-exports, except that for callable +// symbols the definitions are replaced with trampolines that will look up and +// call through to the re-exported symbol at runtime. This can be used to +// enable lazy compilation. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_LAZYREEXPORTS_H +#define LLVM_EXECUTIONENGINE_ORC_LAZYREEXPORTS_H + +#include "llvm/ADT/STLExtras.h" +#include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/IndirectionUtils.h" +#include "llvm/ExecutionEngine/Orc/Speculation.h" + +namespace llvm { + +class Triple; + +namespace orc { + +/// Manages a set of 'lazy call-through' trampolines. These are compiler +/// re-entry trampolines that are pre-bound to look up a given symbol in a given +/// JITDylib, then jump to that address. Since compilation of symbols is +/// triggered on first lookup, these call-through trampolines can be used to +/// implement lazy compilation. +/// +/// The easiest way to construct these call-throughs is using the lazyReexport +/// function. +class LazyCallThroughManager { +public: + using NotifyResolvedFunction = + unique_function<Error(JITTargetAddress ResolvedAddr)>; + + LazyCallThroughManager(ExecutionSession &ES, + JITTargetAddress ErrorHandlerAddr, TrampolinePool *TP); + + // Return a free call-through trampoline and bind it to look up and call + // through to the given symbol. + Expected<JITTargetAddress> + getCallThroughTrampoline(JITDylib &SourceJD, SymbolStringPtr SymbolName, + NotifyResolvedFunction NotifyResolved); + + void resolveTrampolineLandingAddress( + JITTargetAddress TrampolineAddr, + TrampolinePool::NotifyLandingResolvedFunction NotifyLandingResolved); + + virtual ~LazyCallThroughManager() = default; + +protected: + using NotifyLandingResolvedFunction = + TrampolinePool::NotifyLandingResolvedFunction; + + struct ReexportsEntry { + JITDylib *SourceJD; + SymbolStringPtr SymbolName; + }; + + JITTargetAddress reportCallThroughError(Error Err); + Expected<ReexportsEntry> findReexport(JITTargetAddress TrampolineAddr); + Error notifyResolved(JITTargetAddress TrampolineAddr, + JITTargetAddress ResolvedAddr); + void setTrampolinePool(TrampolinePool &TP) { this->TP = &TP; } + +private: + using ReexportsMap = std::map<JITTargetAddress, ReexportsEntry>; + + using NotifiersMap = std::map<JITTargetAddress, NotifyResolvedFunction>; + + std::mutex LCTMMutex; + ExecutionSession &ES; + JITTargetAddress ErrorHandlerAddr; + TrampolinePool *TP = nullptr; + ReexportsMap Reexports; + NotifiersMap Notifiers; +}; + +/// A lazy call-through manager that builds trampolines in the current process. +class LocalLazyCallThroughManager : public LazyCallThroughManager { +private: + using NotifyTargetResolved = unique_function<void(JITTargetAddress)>; + + LocalLazyCallThroughManager(ExecutionSession &ES, + JITTargetAddress ErrorHandlerAddr) + : LazyCallThroughManager(ES, ErrorHandlerAddr, nullptr) {} + + template <typename ORCABI> Error init() { + auto TP = LocalTrampolinePool<ORCABI>::Create( + [this](JITTargetAddress TrampolineAddr, + TrampolinePool::NotifyLandingResolvedFunction + NotifyLandingResolved) { + resolveTrampolineLandingAddress(TrampolineAddr, + std::move(NotifyLandingResolved)); + }); + + if (!TP) + return TP.takeError(); + + this->TP = std::move(*TP); + setTrampolinePool(*this->TP); + return Error::success(); + } + + std::unique_ptr<TrampolinePool> TP; + +public: + /// Create a LocalLazyCallThroughManager using the given ABI. See + /// createLocalLazyCallThroughManager. + template <typename ORCABI> + static Expected<std::unique_ptr<LocalLazyCallThroughManager>> + Create(ExecutionSession &ES, JITTargetAddress ErrorHandlerAddr) { + auto LLCTM = std::unique_ptr<LocalLazyCallThroughManager>( + new LocalLazyCallThroughManager(ES, ErrorHandlerAddr)); + + if (auto Err = LLCTM->init<ORCABI>()) + return std::move(Err); + + return std::move(LLCTM); + } +}; + +/// Create a LocalLazyCallThroughManager from the given triple and execution +/// session. +Expected<std::unique_ptr<LazyCallThroughManager>> +createLocalLazyCallThroughManager(const Triple &T, ExecutionSession &ES, + JITTargetAddress ErrorHandlerAddr); + +/// A materialization unit that builds lazy re-exports. These are callable +/// entry points that call through to the given symbols. +/// Unlike a 'true' re-export, the address of the lazy re-export will not +/// match the address of the re-exported symbol, but calling it will behave +/// the same as calling the re-exported symbol. +class LazyReexportsMaterializationUnit : public MaterializationUnit { +public: + LazyReexportsMaterializationUnit(LazyCallThroughManager &LCTManager, + IndirectStubsManager &ISManager, + JITDylib &SourceJD, + SymbolAliasMap CallableAliases, + ImplSymbolMap *SrcJDLoc); + + StringRef getName() const override; + +private: + void materialize(std::unique_ptr<MaterializationResponsibility> R) override; + void discard(const JITDylib &JD, const SymbolStringPtr &Name) override; + static MaterializationUnit::Interface + extractFlags(const SymbolAliasMap &Aliases); + + LazyCallThroughManager &LCTManager; + IndirectStubsManager &ISManager; + JITDylib &SourceJD; + SymbolAliasMap CallableAliases; + ImplSymbolMap *AliaseeTable; +}; + +/// Define lazy-reexports based on the given SymbolAliasMap. Each lazy re-export +/// is a callable symbol that will look up and dispatch to the given aliasee on +/// first call. All subsequent calls will go directly to the aliasee. +inline std::unique_ptr<LazyReexportsMaterializationUnit> +lazyReexports(LazyCallThroughManager &LCTManager, + IndirectStubsManager &ISManager, JITDylib &SourceJD, + SymbolAliasMap CallableAliases, + ImplSymbolMap *SrcJDLoc = nullptr) { + return std::make_unique<LazyReexportsMaterializationUnit>( + LCTManager, ISManager, SourceJD, std::move(CallableAliases), SrcJDLoc); +} + +} // End namespace orc +} // End namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_LAZYREEXPORTS_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/LookupAndRecordAddrs.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/LookupAndRecordAddrs.h new file mode 100644 index 0000000000..c014e260a9 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/LookupAndRecordAddrs.h @@ -0,0 +1,81 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===-- LookupAndRecordAddrs.h - Symbol lookup support utility --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Record the addresses of a set of symbols into ExecutorAddr objects. +// +// This can be used to avoid repeated lookup (via ExecutionSession::lookup) of +// the given symbols. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_LOOKUPANDRECORDADDRS_H +#define LLVM_EXECUTIONENGINE_ORC_LOOKUPANDRECORDADDRS_H + +#include "llvm/ADT/FunctionExtras.h" +#include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h" + +#include <vector> + +namespace llvm { +namespace orc { + +/// Record addresses of the given symbols in the given ExecutorAddrs. +/// +/// Useful for making permanent records of symbol addreses to call or +/// access in the executor (e.g. runtime support functions in Platform +/// subclasses). +/// +/// By default the symbols are looked up using +/// SymbolLookupFlags::RequiredSymbol, and an error will be generated if any of +/// the requested symbols are not defined. +/// +/// If SymbolLookupFlags::WeaklyReferencedSymbol is used then any missing +/// symbols will have their corresponding address objects set to zero, and +/// this function will never generate an error (the caller will need to check +/// addresses before using them). +/// +/// Asynchronous version. +void lookupAndRecordAddrs( + unique_function<void(Error)> OnRecorded, ExecutionSession &ES, LookupKind K, + const JITDylibSearchOrder &SearchOrder, + std::vector<std::pair<SymbolStringPtr, ExecutorAddr *>> Pairs, + SymbolLookupFlags LookupFlags = SymbolLookupFlags::RequiredSymbol); + +/// Record addresses of the given symbols in the given ExecutorAddrs. +/// +/// Blocking version. +Error lookupAndRecordAddrs( + ExecutionSession &ES, LookupKind K, const JITDylibSearchOrder &SearchOrder, + std::vector<std::pair<SymbolStringPtr, ExecutorAddr *>> Pairs, + SymbolLookupFlags LookupFlags = SymbolLookupFlags::RequiredSymbol); + +/// Record addresses of given symbols in the given ExecutorAddrs. +/// +/// ExecutorProcessControl lookup version. Lookups are always implicitly +/// weak. +Error lookupAndRecordAddrs( + ExecutorProcessControl &EPC, tpctypes::DylibHandle H, + std::vector<std::pair<SymbolStringPtr, ExecutorAddr *>> Pairs, + SymbolLookupFlags LookupFlags = SymbolLookupFlags::RequiredSymbol); + +} // End namespace orc +} // End namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_LOOKUPANDRECORDADDRS_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/MachOPlatform.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/MachOPlatform.h new file mode 100644 index 0000000000..77d057ba6c --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/MachOPlatform.h @@ -0,0 +1,323 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===-- MachOPlatform.h - Utilities for executing MachO in Orc --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Utilities for executing JIT'd MachO in Orc. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_MACHOPLATFORM_H +#define LLVM_EXECUTIONENGINE_ORC_MACHOPLATFORM_H + +#include "llvm/ADT/StringRef.h" +#include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h" +#include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h" +#include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h" + +#include <future> +#include <thread> +#include <vector> + +namespace llvm { +namespace orc { + +struct MachOJITDylibInitializers { + using SectionList = std::vector<ExecutorAddrRange>; + + MachOJITDylibInitializers(std::string Name, ExecutorAddr MachOHeaderAddress) + : Name(std::move(Name)), + MachOHeaderAddress(std::move(MachOHeaderAddress)) {} + + std::string Name; + ExecutorAddr MachOHeaderAddress; + ExecutorAddr ObjCImageInfoAddress; + + StringMap<SectionList> InitSections; +}; + +class MachOJITDylibDeinitializers {}; + +using MachOJITDylibInitializerSequence = std::vector<MachOJITDylibInitializers>; + +using MachOJITDylibDeinitializerSequence = + std::vector<MachOJITDylibDeinitializers>; + +/// Mediates between MachO initialization and ExecutionSession state. +class MachOPlatform : public Platform { +public: + /// Try to create a MachOPlatform instance, adding the ORC runtime to the + /// given JITDylib. + /// + /// The ORC runtime requires access to a number of symbols in libc++, and + /// requires access to symbols in libobjc, and libswiftCore to support + /// Objective-C and Swift code. It is up to the caller to ensure that the + /// requried symbols can be referenced by code added to PlatformJD. The + /// standard way to achieve this is to first attach dynamic library search + /// generators for either the given process, or for the specific required + /// libraries, to PlatformJD, then to create the platform instance: + /// + /// \code{.cpp} + /// auto &PlatformJD = ES.createBareJITDylib("stdlib"); + /// PlatformJD.addGenerator( + /// ExitOnErr(EPCDynamicLibrarySearchGenerator + /// ::GetForTargetProcess(EPC))); + /// ES.setPlatform( + /// ExitOnErr(MachOPlatform::Create(ES, ObjLayer, EPC, PlatformJD, + /// "/path/to/orc/runtime"))); + /// \endcode + /// + /// Alternatively, these symbols could be added to another JITDylib that + /// PlatformJD links against. + /// + /// Clients are also responsible for ensuring that any JIT'd code that + /// depends on runtime functions (including any code using TLV or static + /// destructors) can reference the runtime symbols. This is usually achieved + /// by linking any JITDylibs containing regular code against + /// PlatformJD. + /// + /// By default, MachOPlatform will add the set of aliases returned by the + /// standardPlatformAliases function. This includes both required aliases + /// (e.g. __cxa_atexit -> __orc_rt_macho_cxa_atexit for static destructor + /// support), and optional aliases that provide JIT versions of common + /// functions (e.g. dlopen -> __orc_rt_macho_jit_dlopen). Clients can + /// override these defaults by passing a non-None value for the + /// RuntimeAliases function, in which case the client is responsible for + /// setting up all aliases (including the required ones). + static Expected<std::unique_ptr<MachOPlatform>> + Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, + JITDylib &PlatformJD, const char *OrcRuntimePath, + Optional<SymbolAliasMap> RuntimeAliases = None); + + ExecutionSession &getExecutionSession() const { return ES; } + ObjectLinkingLayer &getObjectLinkingLayer() const { return ObjLinkingLayer; } + + Error setupJITDylib(JITDylib &JD) override; + Error teardownJITDylib(JITDylib &JD) override; + Error notifyAdding(ResourceTracker &RT, + const MaterializationUnit &MU) override; + Error notifyRemoving(ResourceTracker &RT) override; + + /// Returns an AliasMap containing the default aliases for the MachOPlatform. + /// This can be modified by clients when constructing the platform to add + /// or remove aliases. + static SymbolAliasMap standardPlatformAliases(ExecutionSession &ES); + + /// Returns the array of required CXX aliases. + static ArrayRef<std::pair<const char *, const char *>> requiredCXXAliases(); + + /// Returns the array of standard runtime utility aliases for MachO. + static ArrayRef<std::pair<const char *, const char *>> + standardRuntimeUtilityAliases(); + + /// Returns true if the given section name is an initializer section. + static bool isInitializerSection(StringRef SegName, StringRef SectName); + +private: + // The MachOPlatformPlugin scans/modifies LinkGraphs to support MachO + // platform features including initializers, exceptions, TLV, and language + // runtime registration. + class MachOPlatformPlugin : public ObjectLinkingLayer::Plugin { + public: + MachOPlatformPlugin(MachOPlatform &MP) : MP(MP) {} + + void modifyPassConfig(MaterializationResponsibility &MR, + jitlink::LinkGraph &G, + jitlink::PassConfiguration &Config) override; + + SyntheticSymbolDependenciesMap + getSyntheticSymbolDependencies(MaterializationResponsibility &MR) override; + + // FIXME: We should be tentatively tracking scraped sections and discarding + // if the MR fails. + Error notifyFailed(MaterializationResponsibility &MR) override { + return Error::success(); + } + + Error notifyRemovingResources(ResourceKey K) override { + return Error::success(); + } + + void notifyTransferringResources(ResourceKey DstKey, + ResourceKey SrcKey) override {} + + private: + using InitSymbolDepMap = + DenseMap<MaterializationResponsibility *, JITLinkSymbolSet>; + + void addEHAndTLVSupportPasses(MaterializationResponsibility &MR, + jitlink::PassConfiguration &Config); + + Error associateJITDylibHeaderSymbol(jitlink::LinkGraph &G, + MaterializationResponsibility &MR); + + Error preserveInitSections(jitlink::LinkGraph &G, + MaterializationResponsibility &MR); + + Error processObjCImageInfo(jitlink::LinkGraph &G, + MaterializationResponsibility &MR); + + Error registerInitSections(jitlink::LinkGraph &G, JITDylib &JD); + + Error fixTLVSectionsAndEdges(jitlink::LinkGraph &G, JITDylib &JD); + + Error registerEHAndTLVSections(jitlink::LinkGraph &G); + + Error registerEHSectionsPhase1(jitlink::LinkGraph &G); + + std::mutex PluginMutex; + MachOPlatform &MP; + DenseMap<JITDylib *, std::pair<uint32_t, uint32_t>> ObjCImageInfos; + InitSymbolDepMap InitSymbolDeps; + }; + + using SendInitializerSequenceFn = + unique_function<void(Expected<MachOJITDylibInitializerSequence>)>; + + using SendDeinitializerSequenceFn = + unique_function<void(Expected<MachOJITDylibDeinitializerSequence>)>; + + using SendSymbolAddressFn = unique_function<void(Expected<ExecutorAddr>)>; + + static bool supportedTarget(const Triple &TT); + + MachOPlatform(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, + JITDylib &PlatformJD, + std::unique_ptr<DefinitionGenerator> OrcRuntimeGenerator, + Error &Err); + + // Associate MachOPlatform JIT-side runtime support functions with handlers. + Error associateRuntimeSupportFunctions(JITDylib &PlatformJD); + + void getInitializersBuildSequencePhase(SendInitializerSequenceFn SendResult, + JITDylib &JD, + std::vector<JITDylibSP> DFSLinkOrder); + + void getInitializersLookupPhase(SendInitializerSequenceFn SendResult, + JITDylib &JD); + + void rt_getInitializers(SendInitializerSequenceFn SendResult, + StringRef JDName); + + void rt_getDeinitializers(SendDeinitializerSequenceFn SendResult, + ExecutorAddr Handle); + + void rt_lookupSymbol(SendSymbolAddressFn SendResult, ExecutorAddr Handle, + StringRef SymbolName); + + // Records the addresses of runtime symbols used by the platform. + Error bootstrapMachORuntime(JITDylib &PlatformJD); + + Error registerInitInfo(JITDylib &JD, ExecutorAddr ObjCImageInfoAddr, + ArrayRef<jitlink::Section *> InitSections); + + Expected<uint64_t> createPThreadKey(); + + enum PlatformState { BootstrapPhase1, BootstrapPhase2, Initialized }; + + ExecutionSession &ES; + ObjectLinkingLayer &ObjLinkingLayer; + + SymbolStringPtr MachOHeaderStartSymbol; + std::atomic<PlatformState> State{BootstrapPhase1}; + + ExecutorAddr orc_rt_macho_platform_bootstrap; + ExecutorAddr orc_rt_macho_platform_shutdown; + ExecutorAddr orc_rt_macho_register_ehframe_section; + ExecutorAddr orc_rt_macho_deregister_ehframe_section; + ExecutorAddr orc_rt_macho_register_thread_data_section; + ExecutorAddr orc_rt_macho_deregister_thread_data_section; + ExecutorAddr orc_rt_macho_create_pthread_key; + + DenseMap<JITDylib *, SymbolLookupSet> RegisteredInitSymbols; + + // InitSeqs gets its own mutex to avoid locking the whole session when + // aggregating data from the jitlink. + std::mutex PlatformMutex; + DenseMap<JITDylib *, MachOJITDylibInitializers> InitSeqs; + + DenseMap<ExecutorAddr, JITDylib *> HeaderAddrToJITDylib; + DenseMap<JITDylib *, uint64_t> JITDylibToPThreadKey; +}; + +namespace shared { + +using SPSNamedExecutorAddrRangeSequenceMap = + SPSSequence<SPSTuple<SPSString, SPSExecutorAddrRangeSequence>>; + +using SPSMachOJITDylibInitializers = + SPSTuple<SPSString, SPSExecutorAddr, SPSExecutorAddr, + SPSNamedExecutorAddrRangeSequenceMap>; + +using SPSMachOJITDylibInitializerSequence = + SPSSequence<SPSMachOJITDylibInitializers>; + +/// Serialization traits for MachOJITDylibInitializers. +template <> +class SPSSerializationTraits<SPSMachOJITDylibInitializers, + MachOJITDylibInitializers> { +public: + static size_t size(const MachOJITDylibInitializers &MOJDIs) { + return SPSMachOJITDylibInitializers::AsArgList::size( + MOJDIs.Name, MOJDIs.MachOHeaderAddress, MOJDIs.ObjCImageInfoAddress, + MOJDIs.InitSections); + } + + static bool serialize(SPSOutputBuffer &OB, + const MachOJITDylibInitializers &MOJDIs) { + return SPSMachOJITDylibInitializers::AsArgList::serialize( + OB, MOJDIs.Name, MOJDIs.MachOHeaderAddress, MOJDIs.ObjCImageInfoAddress, + MOJDIs.InitSections); + } + + static bool deserialize(SPSInputBuffer &IB, + MachOJITDylibInitializers &MOJDIs) { + return SPSMachOJITDylibInitializers::AsArgList::deserialize( + IB, MOJDIs.Name, MOJDIs.MachOHeaderAddress, MOJDIs.ObjCImageInfoAddress, + MOJDIs.InitSections); + } +}; + +using SPSMachOJITDylibDeinitializers = SPSEmpty; + +using SPSMachOJITDylibDeinitializerSequence = + SPSSequence<SPSMachOJITDylibDeinitializers>; + +template <> +class SPSSerializationTraits<SPSMachOJITDylibDeinitializers, + MachOJITDylibDeinitializers> { +public: + static size_t size(const MachOJITDylibDeinitializers &MOJDDs) { return 0; } + + static bool serialize(SPSOutputBuffer &OB, + const MachOJITDylibDeinitializers &MOJDDs) { + return true; + } + + static bool deserialize(SPSInputBuffer &IB, + MachOJITDylibDeinitializers &MOJDDs) { + MOJDDs = MachOJITDylibDeinitializers(); + return true; + } +}; + +} // end namespace shared +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_MACHOPLATFORM_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/Mangling.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/Mangling.h new file mode 100644 index 0000000000..0f7f4114a9 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/Mangling.h @@ -0,0 +1,72 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===------ Mangling.h -- Name Mangling Utilities for ORC -------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Name mangling utilities for ORC. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_MANGLING_H +#define LLVM_EXECUTIONENGINE_ORC_MANGLING_H + +#include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/ThreadSafeModule.h" +#include "llvm/IR/Module.h" +#include "llvm/Support/MemoryBuffer.h" + +namespace llvm { +namespace orc { + +/// Mangles symbol names then uniques them in the context of an +/// ExecutionSession. +class MangleAndInterner { +public: + MangleAndInterner(ExecutionSession &ES, const DataLayout &DL); + SymbolStringPtr operator()(StringRef Name); + +private: + ExecutionSession &ES; + const DataLayout &DL; +}; + +/// Maps IR global values to their linker symbol names / flags. +/// +/// This utility can be used when adding new IR globals in the JIT. +class IRSymbolMapper { +public: + struct ManglingOptions { + bool EmulatedTLS = false; + }; + + using SymbolNameToDefinitionMap = std::map<SymbolStringPtr, GlobalValue *>; + + /// Add mangled symbols for the given GlobalValues to SymbolFlags. + /// If a SymbolToDefinitionMap pointer is supplied then it will be populated + /// with Name-to-GlobalValue* mappings. Note that this mapping is not + /// necessarily one-to-one: thread-local GlobalValues, for example, may + /// produce more than one symbol, in which case the map will contain duplicate + /// values. + static void add(ExecutionSession &ES, const ManglingOptions &MO, + ArrayRef<GlobalValue *> GVs, SymbolFlagsMap &SymbolFlags, + SymbolNameToDefinitionMap *SymbolToDefinition = nullptr); +}; + +} // End namespace orc +} // End namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_MANGLING_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/ObjectFileInterface.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/ObjectFileInterface.h new file mode 100644 index 0000000000..4bab6decf0 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/ObjectFileInterface.h @@ -0,0 +1,49 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===-- ObjectFileInterface.h - MU interface utils for objects --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Utilities for building MaterializationUnit::Interface objects from +// object files. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_OBJECTFILEINTERFACE_H +#define LLVM_EXECUTIONENGINE_ORC_OBJECTFILEINTERFACE_H + +#include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/Support/MemoryBuffer.h" + +namespace llvm { +namespace orc { + +/// Adds an initializer symbol to the given MU interface. +/// The init symbol's name is guaranteed to be unique within I, and will be of +/// the form $.<ObjFileName>.__inits.<N>, where N is some integer. +void addInitSymbol(MaterializationUnit::Interface &I, ExecutionSession &ES, + StringRef ObjFileName); + +/// Returns a MaterializationUnit::Interface for the object file contained in +/// the given buffer, or an error if the buffer does not contain a valid object +/// file. +Expected<MaterializationUnit::Interface> +getObjectFileInterface(ExecutionSession &ES, MemoryBufferRef ObjBuffer); + +} // End namespace orc +} // End namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_OBJECTFILEINTERFACE_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h new file mode 100644 index 0000000000..2804a9c162 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h @@ -0,0 +1,240 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===-- ObjectLinkingLayer.h - JITLink-based jit linking layer --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Contains the definition for an JITLink-based, in-process object linking +// layer. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_OBJECTLINKINGLAYER_H +#define LLVM_EXECUTIONENGINE_ORC_OBJECTLINKINGLAYER_H + +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ExecutionEngine/JITLink/JITLink.h" +#include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/Layer.h" +#include "llvm/Support/Error.h" +#include <algorithm> +#include <cassert> +#include <functional> +#include <list> +#include <memory> +#include <utility> +#include <vector> + +namespace llvm { + +namespace jitlink { +class EHFrameRegistrar; +class LinkGraph; +class Symbol; +} // namespace jitlink + +namespace orc { + +class ObjectLinkingLayerJITLinkContext; + +/// An ObjectLayer implementation built on JITLink. +/// +/// Clients can use this class to add relocatable object files to an +/// ExecutionSession, and it typically serves as the base layer (underneath +/// a compiling layer like IRCompileLayer) for the rest of the JIT. +class ObjectLinkingLayer : public RTTIExtends<ObjectLinkingLayer, ObjectLayer>, + private ResourceManager { + friend class ObjectLinkingLayerJITLinkContext; + +public: + static char ID; + + /// Plugin instances can be added to the ObjectLinkingLayer to receive + /// callbacks when code is loaded or emitted, and when JITLink is being + /// configured. + class Plugin { + public: + using JITLinkSymbolSet = DenseSet<jitlink::Symbol *>; + using SyntheticSymbolDependenciesMap = + DenseMap<SymbolStringPtr, JITLinkSymbolSet>; + + virtual ~Plugin(); + virtual void modifyPassConfig(MaterializationResponsibility &MR, + jitlink::LinkGraph &G, + jitlink::PassConfiguration &Config) {} + + // Deprecated. Don't use this in new code. There will be a proper mechanism + // for capturing object buffers. + virtual void notifyMaterializing(MaterializationResponsibility &MR, + jitlink::LinkGraph &G, + jitlink::JITLinkContext &Ctx, + MemoryBufferRef InputObject) {} + + virtual void notifyLoaded(MaterializationResponsibility &MR) {} + virtual Error notifyEmitted(MaterializationResponsibility &MR) { + return Error::success(); + } + virtual Error notifyFailed(MaterializationResponsibility &MR) = 0; + virtual Error notifyRemovingResources(ResourceKey K) = 0; + virtual void notifyTransferringResources(ResourceKey DstKey, + ResourceKey SrcKey) = 0; + + /// Return any dependencies that synthetic symbols (e.g. init symbols) + /// have on symbols in the LinkGraph. + /// This is used by the ObjectLinkingLayer to update the dependencies for + /// the synthetic symbols. + virtual SyntheticSymbolDependenciesMap + getSyntheticSymbolDependencies(MaterializationResponsibility &MR) { + return SyntheticSymbolDependenciesMap(); + } + }; + + using ReturnObjectBufferFunction = + std::function<void(std::unique_ptr<MemoryBuffer>)>; + + /// Construct an ObjectLinkingLayer using the ExecutorProcessControl + /// instance's memory manager. + ObjectLinkingLayer(ExecutionSession &ES); + + /// Construct an ObjectLinkingLayer using a custom memory manager. + ObjectLinkingLayer(ExecutionSession &ES, + jitlink::JITLinkMemoryManager &MemMgr); + + /// Construct an ObjectLinkingLayer. Takes ownership of the given + /// JITLinkMemoryManager. This method is a temporary hack to simplify + /// co-existence with RTDyldObjectLinkingLayer (which also owns its + /// allocators). + ObjectLinkingLayer(ExecutionSession &ES, + std::unique_ptr<jitlink::JITLinkMemoryManager> MemMgr); + + /// Destruct an ObjectLinkingLayer. + ~ObjectLinkingLayer(); + + /// Set an object buffer return function. By default object buffers are + /// deleted once the JIT has linked them. If a return function is set then + /// it will be called to transfer ownership of the buffer instead. + void setReturnObjectBuffer(ReturnObjectBufferFunction ReturnObjectBuffer) { + this->ReturnObjectBuffer = std::move(ReturnObjectBuffer); + } + + /// Add a pass-config modifier. + ObjectLinkingLayer &addPlugin(std::unique_ptr<Plugin> P) { + std::lock_guard<std::mutex> Lock(LayerMutex); + Plugins.push_back(std::move(P)); + return *this; + } + + /// Add a LinkGraph to the JITDylib targeted by the given tracker. + Error add(ResourceTrackerSP, std::unique_ptr<jitlink::LinkGraph> G); + + /// Add a LinkGraph to the given JITDylib. + Error add(JITDylib &JD, std::unique_ptr<jitlink::LinkGraph> G) { + return add(JD.getDefaultResourceTracker(), std::move(G)); + } + + // Un-hide ObjectLayer add methods. + using ObjectLayer::add; + + /// Emit an object file. + void emit(std::unique_ptr<MaterializationResponsibility> R, + std::unique_ptr<MemoryBuffer> O) override; + + /// Emit a LinkGraph. + void emit(std::unique_ptr<MaterializationResponsibility> R, + std::unique_ptr<jitlink::LinkGraph> G); + + /// Instructs this ObjectLinkingLayer instance to override the symbol flags + /// found in the AtomGraph with the flags supplied by the + /// MaterializationResponsibility instance. This is a workaround to support + /// symbol visibility in COFF, which does not use the libObject's + /// SF_Exported flag. Use only when generating / adding COFF object files. + /// + /// FIXME: We should be able to remove this if/when COFF properly tracks + /// exported symbols. + ObjectLinkingLayer & + setOverrideObjectFlagsWithResponsibilityFlags(bool OverrideObjectFlags) { + this->OverrideObjectFlags = OverrideObjectFlags; + return *this; + } + + /// If set, this ObjectLinkingLayer instance will claim responsibility + /// for any symbols provided by a given object file that were not already in + /// the MaterializationResponsibility instance. Setting this flag allows + /// higher-level program representations (e.g. LLVM IR) to be added based on + /// only a subset of the symbols they provide, without having to write + /// intervening layers to scan and add the additional symbols. This trades + /// diagnostic quality for convenience however: If all symbols are enumerated + /// up-front then clashes can be detected and reported early (and usually + /// deterministically). If this option is set, clashes for the additional + /// symbols may not be detected until late, and detection may depend on + /// the flow of control through JIT'd code. Use with care. + ObjectLinkingLayer & + setAutoClaimResponsibilityForObjectSymbols(bool AutoClaimObjectSymbols) { + this->AutoClaimObjectSymbols = AutoClaimObjectSymbols; + return *this; + } + +private: + using FinalizedAlloc = jitlink::JITLinkMemoryManager::FinalizedAlloc; + + void modifyPassConfig(MaterializationResponsibility &MR, + jitlink::LinkGraph &G, + jitlink::PassConfiguration &PassConfig); + void notifyLoaded(MaterializationResponsibility &MR); + Error notifyEmitted(MaterializationResponsibility &MR, FinalizedAlloc FA); + + Error handleRemoveResources(ResourceKey K) override; + void handleTransferResources(ResourceKey DstKey, ResourceKey SrcKey) override; + + mutable std::mutex LayerMutex; + jitlink::JITLinkMemoryManager &MemMgr; + std::unique_ptr<jitlink::JITLinkMemoryManager> MemMgrOwnership; + bool OverrideObjectFlags = false; + bool AutoClaimObjectSymbols = false; + ReturnObjectBufferFunction ReturnObjectBuffer; + DenseMap<ResourceKey, std::vector<FinalizedAlloc>> Allocs; + std::vector<std::unique_ptr<Plugin>> Plugins; +}; + +class EHFrameRegistrationPlugin : public ObjectLinkingLayer::Plugin { +public: + EHFrameRegistrationPlugin( + ExecutionSession &ES, + std::unique_ptr<jitlink::EHFrameRegistrar> Registrar); + void modifyPassConfig(MaterializationResponsibility &MR, + jitlink::LinkGraph &G, + jitlink::PassConfiguration &PassConfig) override; + Error notifyEmitted(MaterializationResponsibility &MR) override; + Error notifyFailed(MaterializationResponsibility &MR) override; + Error notifyRemovingResources(ResourceKey K) override; + void notifyTransferringResources(ResourceKey DstKey, + ResourceKey SrcKey) override; + +private: + std::mutex EHFramePluginMutex; + ExecutionSession &ES; + std::unique_ptr<jitlink::EHFrameRegistrar> Registrar; + DenseMap<MaterializationResponsibility *, ExecutorAddrRange> InProcessLinks; + DenseMap<ResourceKey, std::vector<ExecutorAddrRange>> EHFrameRanges; +}; + +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_OBJECTLINKINGLAYER_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/ObjectTransformLayer.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/ObjectTransformLayer.h new file mode 100644 index 0000000000..1327dd8905 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/ObjectTransformLayer.h @@ -0,0 +1,62 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===- ObjectTransformLayer.h - Run all objects through functor -*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Run all objects passed in through a user supplied functor. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_OBJECTTRANSFORMLAYER_H +#define LLVM_EXECUTIONENGINE_ORC_OBJECTTRANSFORMLAYER_H + +#include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/ExecutionEngine/Orc/Layer.h" +#include <algorithm> +#include <memory> + +namespace llvm { +namespace orc { + +class ObjectTransformLayer + : public RTTIExtends<ObjectTransformLayer, ObjectLayer> { +public: + static char ID; + + using TransformFunction = + std::function<Expected<std::unique_ptr<MemoryBuffer>>( + std::unique_ptr<MemoryBuffer>)>; + + ObjectTransformLayer(ExecutionSession &ES, ObjectLayer &BaseLayer, + TransformFunction Transform = TransformFunction()); + + void emit(std::unique_ptr<MaterializationResponsibility> R, + std::unique_ptr<MemoryBuffer> O) override; + + void setTransform(TransformFunction Transform) { + this->Transform = std::move(Transform); + } + +private: + ObjectLayer &BaseLayer; + TransformFunction Transform; +}; + +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_OBJECTTRANSFORMLAYER_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/OrcABISupport.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/OrcABISupport.h new file mode 100644 index 0000000000..a998bfa9e0 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/OrcABISupport.h @@ -0,0 +1,347 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===- OrcABISupport.h - ABI support code -----------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// ABI specific code for Orc, e.g. callback assembly. +// +// ABI classes should be part of the JIT *target* process, not the host +// process (except where you're doing hosted JITing and the two are one and the +// same). +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_ORCABISUPPORT_H +#define LLVM_EXECUTIONENGINE_ORC_ORCABISUPPORT_H + +#include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/MathExtras.h" +#include <cstdint> + +namespace llvm { +namespace orc { + +struct IndirectStubsAllocationSizes { + uint64_t StubBytes = 0; + uint64_t PointerBytes = 0; + unsigned NumStubs = 0; +}; + +template <typename ORCABI> +IndirectStubsAllocationSizes +getIndirectStubsBlockSizes(unsigned MinStubs, unsigned RoundToMultipleOf = 0) { + assert( + (RoundToMultipleOf == 0 || (RoundToMultipleOf % ORCABI::StubSize == 0)) && + "RoundToMultipleOf is not a multiple of stub size"); + uint64_t StubBytes = MinStubs * ORCABI::StubSize; + if (RoundToMultipleOf) + StubBytes = alignTo(StubBytes, RoundToMultipleOf); + unsigned NumStubs = StubBytes / ORCABI::StubSize; + uint64_t PointerBytes = NumStubs * ORCABI::PointerSize; + return {StubBytes, PointerBytes, NumStubs}; +} + +/// Generic ORC ABI support. +/// +/// This class can be substituted as the target architecture support class for +/// ORC templates that require one (e.g. IndirectStubsManagers). It does not +/// support lazy JITing however, and any attempt to use that functionality +/// will result in execution of an llvm_unreachable. +class OrcGenericABI { +public: + static constexpr unsigned PointerSize = sizeof(uintptr_t); + static constexpr unsigned TrampolineSize = 1; + static constexpr unsigned StubSize = 1; + static constexpr unsigned StubToPointerMaxDisplacement = 1; + static constexpr unsigned ResolverCodeSize = 1; + + static void writeResolverCode(char *ResolveWorkingMem, + JITTargetAddress ResolverTargetAddr, + JITTargetAddress ReentryFnAddr, + JITTargetAddress ReentryCtxAddr) { + llvm_unreachable("writeResolverCode is not supported by the generic host " + "support class"); + } + + static void writeTrampolines(char *TrampolineBlockWorkingMem, + JITTargetAddress TrampolineBlockTargetAddr, + JITTargetAddress ResolverAddr, + unsigned NumTrampolines) { + llvm_unreachable("writeTrampolines is not supported by the generic host " + "support class"); + } + + static void writeIndirectStubsBlock( + char *StubsBlockWorkingMem, JITTargetAddress StubsBlockTargetAddress, + JITTargetAddress PointersBlockTargetAddress, unsigned NumStubs) { + llvm_unreachable( + "writeIndirectStubsBlock is not supported by the generic host " + "support class"); + } +}; + +class OrcAArch64 { +public: + static constexpr unsigned PointerSize = 8; + static constexpr unsigned TrampolineSize = 12; + static constexpr unsigned StubSize = 8; + static constexpr unsigned StubToPointerMaxDisplacement = 1U << 27; + static constexpr unsigned ResolverCodeSize = 0x120; + + /// Write the resolver code into the given memory. The user is + /// responsible for allocating the memory and setting permissions. + /// + /// ReentryFnAddr should be the address of a function whose signature matches + /// void* (*)(void *TrampolineAddr, void *ReentryCtxAddr). The ReentryCtxAddr + /// argument of writeResolverCode will be passed as the second argument to + /// the function at ReentryFnAddr. + static void writeResolverCode(char *ResolverWorkingMem, + JITTargetAddress ResolverTargetAddress, + JITTargetAddress ReentryFnAddr, + JITTargetAddress RentryCtxAddr); + + /// Write the requested number of trampolines into the given memory, + /// which must be big enough to hold 1 pointer, plus NumTrampolines + /// trampolines. + static void writeTrampolines(char *TrampolineBlockWorkingMem, + JITTargetAddress TrampolineBlockTargetAddress, + JITTargetAddress ResolverAddr, + unsigned NumTrampolines); + + /// Write NumStubs indirect stubs to working memory at StubsBlockWorkingMem. + /// Stubs will be written as if linked at StubsBlockTargetAddress, with the + /// Nth stub using the Nth pointer in memory starting at + /// PointersBlockTargetAddress. + static void writeIndirectStubsBlock( + char *StubsBlockWorkingMem, JITTargetAddress StubsBlockTargetAddress, + JITTargetAddress PointersBlockTargetAddress, unsigned MinStubs); +}; + +/// X86_64 code that's common to all ABIs. +/// +/// X86_64 supports lazy JITing. +class OrcX86_64_Base { +public: + static constexpr unsigned PointerSize = 8; + static constexpr unsigned TrampolineSize = 8; + static constexpr unsigned StubSize = 8; + static constexpr unsigned StubToPointerMaxDisplacement = 1 << 31; + + /// Write the requested number of trampolines into the given memory, + /// which must be big enough to hold 1 pointer, plus NumTrampolines + /// trampolines. + static void writeTrampolines(char *TrampolineBlockWorkingMem, + JITTargetAddress TrampolineBlockTargetAddress, + JITTargetAddress ResolverAddr, + unsigned NumTrampolines); + + /// Write NumStubs indirect stubs to working memory at StubsBlockWorkingMem. + /// Stubs will be written as if linked at StubsBlockTargetAddress, with the + /// Nth stub using the Nth pointer in memory starting at + /// PointersBlockTargetAddress. + static void writeIndirectStubsBlock( + char *StubsBlockWorkingMem, JITTargetAddress StubsBlockTargetAddress, + JITTargetAddress PointersBlockTargetAddress, unsigned NumStubs); +}; + +/// X86_64 support for SysV ABI (Linux, MacOSX). +/// +/// X86_64_SysV supports lazy JITing. +class OrcX86_64_SysV : public OrcX86_64_Base { +public: + static constexpr unsigned ResolverCodeSize = 0x6C; + + /// Write the resolver code into the given memory. The user is + /// responsible for allocating the memory and setting permissions. + /// + /// ReentryFnAddr should be the address of a function whose signature matches + /// void* (*)(void *TrampolineAddr, void *ReentryCtxAddr). The ReentryCtxAddr + /// argument of writeResolverCode will be passed as the second argument to + /// the function at ReentryFnAddr. + static void writeResolverCode(char *ResolverWorkingMem, + JITTargetAddress ResolverTargetAddress, + JITTargetAddress ReentryFnAddr, + JITTargetAddress ReentryCtxAddr); +}; + +/// X86_64 support for Win32. +/// +/// X86_64_Win32 supports lazy JITing. +class OrcX86_64_Win32 : public OrcX86_64_Base { +public: + static constexpr unsigned ResolverCodeSize = 0x74; + + /// Write the resolver code into the given memory. The user is + /// responsible for allocating the memory and setting permissions. + /// + /// ReentryFnAddr should be the address of a function whose signature matches + /// void* (*)(void *TrampolineAddr, void *ReentryCtxAddr). The ReentryCtxAddr + /// argument of writeResolverCode will be passed as the second argument to + /// the function at ReentryFnAddr. + static void writeResolverCode(char *ResolverWorkingMem, + JITTargetAddress ResolverTargetAddress, + JITTargetAddress ReentryFnAddr, + JITTargetAddress ReentryCtxAddr); +}; + +/// I386 support. +/// +/// I386 supports lazy JITing. +class OrcI386 { +public: + static constexpr unsigned PointerSize = 4; + static constexpr unsigned TrampolineSize = 8; + static constexpr unsigned StubSize = 8; + static constexpr unsigned StubToPointerMaxDisplacement = 1 << 31; + static constexpr unsigned ResolverCodeSize = 0x4a; + + /// Write the resolver code into the given memory. The user is + /// responsible for allocating the memory and setting permissions. + /// + /// ReentryFnAddr should be the address of a function whose signature matches + /// void* (*)(void *TrampolineAddr, void *ReentryCtxAddr). The ReentryCtxAddr + /// argument of writeResolverCode will be passed as the second argument to + /// the function at ReentryFnAddr. + static void writeResolverCode(char *ResolverWorkingMem, + JITTargetAddress ResolverTargetAddress, + JITTargetAddress ReentryFnAddr, + JITTargetAddress ReentryCtxAddr); + + /// Write the requested number of trampolines into the given memory, + /// which must be big enough to hold 1 pointer, plus NumTrampolines + /// trampolines. + static void writeTrampolines(char *TrampolineBlockWorkingMem, + JITTargetAddress TrampolineBlockTargetAddress, + JITTargetAddress ResolverAddr, + unsigned NumTrampolines); + + /// Write NumStubs indirect stubs to working memory at StubsBlockWorkingMem. + /// Stubs will be written as if linked at StubsBlockTargetAddress, with the + /// Nth stub using the Nth pointer in memory starting at + /// PointersBlockTargetAddress. + static void writeIndirectStubsBlock( + char *StubsBlockWorkingMem, JITTargetAddress StubsBlockTargetAddress, + JITTargetAddress PointersBlockTargetAddress, unsigned NumStubs); +}; + +// @brief Mips32 support. +// +// Mips32 supports lazy JITing. +class OrcMips32_Base { +public: + static constexpr unsigned PointerSize = 4; + static constexpr unsigned TrampolineSize = 20; + static constexpr unsigned StubSize = 8; + static constexpr unsigned StubToPointerMaxDisplacement = 1 << 31; + static constexpr unsigned ResolverCodeSize = 0xfc; + + /// Write the requested number of trampolines into the given memory, + /// which must be big enough to hold 1 pointer, plus NumTrampolines + /// trampolines. + static void writeTrampolines(char *TrampolineBlockWorkingMem, + JITTargetAddress TrampolineBlockTargetAddress, + JITTargetAddress ResolverAddr, + unsigned NumTrampolines); + + /// Write the resolver code into the given memory. The user is + /// responsible for allocating the memory and setting permissions. + /// + /// ReentryFnAddr should be the address of a function whose signature matches + /// void* (*)(void *TrampolineAddr, void *ReentryCtxAddr). The ReentryCtxAddr + /// argument of writeResolverCode will be passed as the second argument to + /// the function at ReentryFnAddr. + static void writeResolverCode(char *ResolverBlockWorkingMem, + JITTargetAddress ResolverBlockTargetAddress, + JITTargetAddress ReentryFnAddr, + JITTargetAddress ReentryCtxAddr, + bool isBigEndian); + /// Write NumStubs indirect stubs to working memory at StubsBlockWorkingMem. + /// Stubs will be written as if linked at StubsBlockTargetAddress, with the + /// Nth stub using the Nth pointer in memory starting at + /// PointersBlockTargetAddress. + static void writeIndirectStubsBlock( + char *StubsBlockWorkingMem, JITTargetAddress StubsBlockTargetAddress, + JITTargetAddress PointersBlockTargetAddress, unsigned NumStubs); +}; + +class OrcMips32Le : public OrcMips32_Base { +public: + static void writeResolverCode(char *ResolverWorkingMem, + JITTargetAddress ResolverTargetAddress, + JITTargetAddress ReentryFnAddr, + JITTargetAddress ReentryCtxAddr) { + OrcMips32_Base::writeResolverCode(ResolverWorkingMem, ResolverTargetAddress, + ReentryFnAddr, ReentryCtxAddr, false); + } +}; + +class OrcMips32Be : public OrcMips32_Base { +public: + static void writeResolverCode(char *ResolverWorkingMem, + JITTargetAddress ResolverTargetAddress, + JITTargetAddress ReentryFnAddr, + JITTargetAddress ReentryCtxAddr) { + OrcMips32_Base::writeResolverCode(ResolverWorkingMem, ResolverTargetAddress, + ReentryFnAddr, ReentryCtxAddr, true); + } +}; + +// @brief Mips64 support. +// +// Mips64 supports lazy JITing. +class OrcMips64 { +public: + static constexpr unsigned PointerSize = 8; + static constexpr unsigned TrampolineSize = 40; + static constexpr unsigned StubSize = 32; + static constexpr unsigned StubToPointerMaxDisplacement = 1 << 31; + static constexpr unsigned ResolverCodeSize = 0x120; + + /// Write the resolver code into the given memory. The user is + /// responsible for allocating the memory and setting permissions. + /// + /// ReentryFnAddr should be the address of a function whose signature matches + /// void* (*)(void *TrampolineAddr, void *ReentryCtxAddr). The ReentryCtxAddr + /// argument of writeResolverCode will be passed as the second argument to + /// the function at ReentryFnAddr. + static void writeResolverCode(char *ResolverWorkingMem, + JITTargetAddress ResolverTargetAddress, + JITTargetAddress ReentryFnAddr, + JITTargetAddress ReentryCtxAddr); + + /// Write the requested number of trampolines into the given memory, + /// which must be big enough to hold 1 pointer, plus NumTrampolines + /// trampolines. + static void writeTrampolines(char *TrampolineBlockWorkingMem, + JITTargetAddress TrampolineBlockTargetAddress, + JITTargetAddress ResolverFnAddr, + unsigned NumTrampolines); + /// Write NumStubs indirect stubs to working memory at StubsBlockWorkingMem. + /// Stubs will be written as if linked at StubsBlockTargetAddress, with the + /// Nth stub using the Nth pointer in memory starting at + /// PointersBlockTargetAddress. + static void writeIndirectStubsBlock( + char *StubsBlockWorkingMem, JITTargetAddress StubsBlockTargetAddress, + JITTargetAddress PointersBlockTargetAddress, unsigned NumStubs); +}; + +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_ORCABISUPPORT_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h new file mode 100644 index 0000000000..5693aa10ef --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h @@ -0,0 +1,171 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===- RTDyldObjectLinkingLayer.h - RTDyld-based jit linking ---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Contains the definition for an RTDyld-based, in-process object linking layer. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_RTDYLDOBJECTLINKINGLAYER_H +#define LLVM_EXECUTIONENGINE_ORC_RTDYLDOBJECTLINKINGLAYER_H + +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ExecutionEngine/JITEventListener.h" +#include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/Layer.h" +#include "llvm/ExecutionEngine/RuntimeDyld.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Support/Error.h" +#include <algorithm> +#include <cassert> +#include <functional> +#include <list> +#include <memory> +#include <utility> +#include <vector> + +namespace llvm { +namespace orc { + +class RTDyldObjectLinkingLayer + : public RTTIExtends<RTDyldObjectLinkingLayer, ObjectLayer>, + private ResourceManager { +public: + static char ID; + + /// Functor for receiving object-loaded notifications. + using NotifyLoadedFunction = std::function<void( + MaterializationResponsibility &R, const object::ObjectFile &Obj, + const RuntimeDyld::LoadedObjectInfo &)>; + + /// Functor for receiving finalization notifications. + using NotifyEmittedFunction = std::function<void( + MaterializationResponsibility &R, std::unique_ptr<MemoryBuffer>)>; + + using GetMemoryManagerFunction = + std::function<std::unique_ptr<RuntimeDyld::MemoryManager>()>; + + /// Construct an ObjectLinkingLayer with the given NotifyLoaded, + /// and NotifyEmitted functors. + RTDyldObjectLinkingLayer(ExecutionSession &ES, + GetMemoryManagerFunction GetMemoryManager); + + ~RTDyldObjectLinkingLayer(); + + /// Emit the object. + void emit(std::unique_ptr<MaterializationResponsibility> R, + std::unique_ptr<MemoryBuffer> O) override; + + /// Set the NotifyLoaded callback. + RTDyldObjectLinkingLayer &setNotifyLoaded(NotifyLoadedFunction NotifyLoaded) { + this->NotifyLoaded = std::move(NotifyLoaded); + return *this; + } + + /// Set the NotifyEmitted callback. + RTDyldObjectLinkingLayer & + setNotifyEmitted(NotifyEmittedFunction NotifyEmitted) { + this->NotifyEmitted = std::move(NotifyEmitted); + return *this; + } + + /// Set the 'ProcessAllSections' flag. + /// + /// If set to true, all sections in each object file will be allocated using + /// the memory manager, rather than just the sections required for execution. + /// + /// This is kludgy, and may be removed in the future. + RTDyldObjectLinkingLayer &setProcessAllSections(bool ProcessAllSections) { + this->ProcessAllSections = ProcessAllSections; + return *this; + } + + /// Instructs this RTDyldLinkingLayer2 instance to override the symbol flags + /// returned by RuntimeDyld for any given object file with the flags supplied + /// by the MaterializationResponsibility instance. This is a workaround to + /// support symbol visibility in COFF, which does not use the libObject's + /// SF_Exported flag. Use only when generating / adding COFF object files. + /// + /// FIXME: We should be able to remove this if/when COFF properly tracks + /// exported symbols. + RTDyldObjectLinkingLayer & + setOverrideObjectFlagsWithResponsibilityFlags(bool OverrideObjectFlags) { + this->OverrideObjectFlags = OverrideObjectFlags; + return *this; + } + + /// If set, this RTDyldObjectLinkingLayer instance will claim responsibility + /// for any symbols provided by a given object file that were not already in + /// the MaterializationResponsibility instance. Setting this flag allows + /// higher-level program representations (e.g. LLVM IR) to be added based on + /// only a subset of the symbols they provide, without having to write + /// intervening layers to scan and add the additional symbols. This trades + /// diagnostic quality for convenience however: If all symbols are enumerated + /// up-front then clashes can be detected and reported early (and usually + /// deterministically). If this option is set, clashes for the additional + /// symbols may not be detected until late, and detection may depend on + /// the flow of control through JIT'd code. Use with care. + RTDyldObjectLinkingLayer & + setAutoClaimResponsibilityForObjectSymbols(bool AutoClaimObjectSymbols) { + this->AutoClaimObjectSymbols = AutoClaimObjectSymbols; + return *this; + } + + /// Register a JITEventListener. + void registerJITEventListener(JITEventListener &L); + + /// Unregister a JITEventListener. + void unregisterJITEventListener(JITEventListener &L); + +private: + using MemoryManagerUP = std::unique_ptr<RuntimeDyld::MemoryManager>; + + Error onObjLoad(MaterializationResponsibility &R, + const object::ObjectFile &Obj, + RuntimeDyld::MemoryManager &MemMgr, + RuntimeDyld::LoadedObjectInfo &LoadedObjInfo, + std::map<StringRef, JITEvaluatedSymbol> Resolved, + std::set<StringRef> &InternalSymbols); + + void onObjEmit(MaterializationResponsibility &R, + object::OwningBinary<object::ObjectFile> O, + std::unique_ptr<RuntimeDyld::MemoryManager> MemMgr, + std::unique_ptr<RuntimeDyld::LoadedObjectInfo> LoadedObjInfo, + Error Err); + + Error handleRemoveResources(ResourceKey K) override; + void handleTransferResources(ResourceKey DstKey, ResourceKey SrcKey) override; + + mutable std::mutex RTDyldLayerMutex; + GetMemoryManagerFunction GetMemoryManager; + NotifyLoadedFunction NotifyLoaded; + NotifyEmittedFunction NotifyEmitted; + bool ProcessAllSections = false; + bool OverrideObjectFlags = false; + bool AutoClaimObjectSymbols = false; + DenseMap<ResourceKey, std::vector<MemoryManagerUP>> MemMgrs; + std::vector<JITEventListener *> EventListeners; +}; + +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_RTDYLDOBJECTLINKINGLAYER_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/Shared/AllocationActions.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/Shared/AllocationActions.h new file mode 100644 index 0000000000..4f849e30b0 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/Shared/AllocationActions.h @@ -0,0 +1,112 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===- AllocationActions.h -- JITLink allocation support calls -*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Structures for making memory allocation support calls. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_SHARED_ALLOCATIONACTIONS_H +#define LLVM_EXECUTIONENGINE_ORC_SHARED_ALLOCATIONACTIONS_H + +#include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h" +#include "llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h" +#include "llvm/Support/Memory.h" + +#include <vector> + +namespace llvm { +namespace orc { +namespace shared { + +/// A pair of WrapperFunctionCalls, one to be run at finalization time, one to +/// be run at deallocation time. +/// +/// AllocActionCallPairs should be constructed for paired operations (e.g. +/// __register_ehframe and __deregister_ehframe for eh-frame registration). +/// See comments for AllocActions for execution ordering. +/// +/// For unpaired operations one or the other member can be left unused, as +/// AllocationActionCalls with an FnAddr of zero will be skipped. +struct AllocActionCallPair { + WrapperFunctionCall Finalize; + WrapperFunctionCall Dealloc; +}; + +/// A vector of allocation actions to be run for this allocation. +/// +/// Finalize allocations will be run in order at finalize time. Dealloc +/// actions will be run in reverse order at deallocation time. +using AllocActions = std::vector<AllocActionCallPair>; + +/// Returns the number of deallocaton actions in the given AllocActions array. +/// +/// This can be useful if clients want to pre-allocate room for deallocation +/// actions with the rest of their memory. +inline size_t numDeallocActions(const AllocActions &AAs) { + return llvm::count_if( + AAs, [](const AllocActionCallPair &P) { return !!P.Dealloc; }); +} + +/// Run finalize actions. +/// +/// If any finalize action fails then the corresponding dealloc actions will be +/// run in reverse order (not including the deallocation action for the failed +/// finalize action), and the error for the failing action will be returned. +/// +/// If all finalize actions succeed then a vector of deallocation actions will +/// be returned. The dealloc actions should be run by calling +/// runDeallocationActions. If this function succeeds then the AA argument will +/// be cleared before the function returns. +Expected<std::vector<WrapperFunctionCall>> +runFinalizeActions(AllocActions &AAs); + +/// Run deallocation actions. +/// Dealloc actions will be run in reverse order (from last element of DAs to +/// first). +Error runDeallocActions(ArrayRef<WrapperFunctionCall> DAs); + +using SPSAllocActionCallPair = + SPSTuple<SPSWrapperFunctionCall, SPSWrapperFunctionCall>; + +template <> +class SPSSerializationTraits<SPSAllocActionCallPair, + AllocActionCallPair> { + using AL = SPSAllocActionCallPair::AsArgList; + +public: + static size_t size(const AllocActionCallPair &AAP) { + return AL::size(AAP.Finalize, AAP.Dealloc); + } + + static bool serialize(SPSOutputBuffer &OB, + const AllocActionCallPair &AAP) { + return AL::serialize(OB, AAP.Finalize, AAP.Dealloc); + } + + static bool deserialize(SPSInputBuffer &IB, + AllocActionCallPair &AAP) { + return AL::deserialize(IB, AAP.Finalize, AAP.Dealloc); + } +}; + +} // end namespace shared +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_SHARED_ALLOCATIONACTIONS_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h new file mode 100644 index 0000000000..1388f53aa6 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h @@ -0,0 +1,257 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===------ ExecutorAddress.h - Executing process address -------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Represents an address in the executing program. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_SHARED_EXECUTORADDRESS_H +#define LLVM_EXECUTIONENGINE_ORC_SHARED_EXECUTORADDRESS_H + +#include "llvm/ADT/DenseMapInfo.h" +#include "llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h" +#include "llvm/Support/FormatVariadic.h" +#include "llvm/Support/raw_ostream.h" + +#include <cassert> +#include <type_traits> + +namespace llvm { +namespace orc { + +using ExecutorAddrDiff = uint64_t; + +/// Represents an address in the executor process. +class ExecutorAddr { +public: + ExecutorAddr() = default; + + /// Create an ExecutorAddr from the given value. + explicit constexpr ExecutorAddr(uint64_t Addr) : Addr(Addr) {} + + /// Create an ExecutorAddr from the given pointer. + /// Warning: This should only be used when JITing in-process. + template <typename T> static ExecutorAddr fromPtr(T *Value) { + return ExecutorAddr( + static_cast<uint64_t>(reinterpret_cast<uintptr_t>(Value))); + } + + /// Cast this ExecutorAddr to a pointer of the given type. + /// Warning: This should only be used when JITing in-process. + template <typename T> T toPtr() const { + static_assert(std::is_pointer<T>::value, "T must be a pointer type"); + uintptr_t IntPtr = static_cast<uintptr_t>(Addr); + assert(IntPtr == Addr && "ExecutorAddr value out of range for uintptr_t"); + return reinterpret_cast<T>(IntPtr); + } + + uint64_t getValue() const { return Addr; } + void setValue(uint64_t Addr) { this->Addr = Addr; } + bool isNull() const { return Addr == 0; } + + explicit operator bool() const { return Addr != 0; } + + friend bool operator==(const ExecutorAddr &LHS, const ExecutorAddr &RHS) { + return LHS.Addr == RHS.Addr; + } + + friend bool operator!=(const ExecutorAddr &LHS, const ExecutorAddr &RHS) { + return LHS.Addr != RHS.Addr; + } + + friend bool operator<(const ExecutorAddr &LHS, const ExecutorAddr &RHS) { + return LHS.Addr < RHS.Addr; + } + + friend bool operator<=(const ExecutorAddr &LHS, const ExecutorAddr &RHS) { + return LHS.Addr <= RHS.Addr; + } + + friend bool operator>(const ExecutorAddr &LHS, const ExecutorAddr &RHS) { + return LHS.Addr > RHS.Addr; + } + + friend bool operator>=(const ExecutorAddr &LHS, const ExecutorAddr &RHS) { + return LHS.Addr >= RHS.Addr; + } + + ExecutorAddr &operator++() { + ++Addr; + return *this; + } + ExecutorAddr &operator--() { + --Addr; + return *this; + } + ExecutorAddr operator++(int) { return ExecutorAddr(Addr++); } + ExecutorAddr operator--(int) { return ExecutorAddr(Addr--); } + + ExecutorAddr &operator+=(const ExecutorAddrDiff &Delta) { + Addr += Delta; + return *this; + } + + ExecutorAddr &operator-=(const ExecutorAddrDiff &Delta) { + Addr -= Delta; + return *this; + } + +private: + uint64_t Addr = 0; +}; + +/// Subtracting two addresses yields an offset. +inline ExecutorAddrDiff operator-(const ExecutorAddr &LHS, + const ExecutorAddr &RHS) { + return ExecutorAddrDiff(LHS.getValue() - RHS.getValue()); +} + +/// Adding an offset and an address yields an address. +inline ExecutorAddr operator+(const ExecutorAddr &LHS, + const ExecutorAddrDiff &RHS) { + return ExecutorAddr(LHS.getValue() + RHS); +} + +/// Adding an address and an offset yields an address. +inline ExecutorAddr operator+(const ExecutorAddrDiff &LHS, + const ExecutorAddr &RHS) { + return ExecutorAddr(LHS + RHS.getValue()); +} + +/// Subtracting an offset from an address yields an address. +inline ExecutorAddr operator-(const ExecutorAddr &LHS, + const ExecutorAddrDiff &RHS) { + return ExecutorAddr(LHS.getValue() - RHS); +} + +/// Taking the modulus of an address and a diff yields a diff. +inline ExecutorAddrDiff operator%(const ExecutorAddr &LHS, + const ExecutorAddrDiff &RHS) { + return ExecutorAddrDiff(LHS.getValue() % RHS); +} + +/// Represents an address range in the exceutor process. +struct ExecutorAddrRange { + ExecutorAddrRange() = default; + ExecutorAddrRange(ExecutorAddr Start, ExecutorAddr End) + : Start(Start), End(End) {} + ExecutorAddrRange(ExecutorAddr Start, ExecutorAddrDiff Size) + : Start(Start), End(Start + Size) {} + + bool empty() const { return Start == End; } + ExecutorAddrDiff size() const { return End - Start; } + + friend bool operator==(const ExecutorAddrRange &LHS, + const ExecutorAddrRange &RHS) { + return LHS.Start == RHS.Start && LHS.End == RHS.End; + } + friend bool operator!=(const ExecutorAddrRange &LHS, + const ExecutorAddrRange &RHS) { + return !(LHS == RHS); + } + bool contains(ExecutorAddr Addr) const { return Start <= Addr && Addr < End; } + bool overlaps(const ExecutorAddrRange &Other) { + return !(Other.End <= Start || End <= Other.Start); + } + + ExecutorAddr Start; + ExecutorAddr End; +}; + +inline raw_ostream &operator<<(raw_ostream &OS, const ExecutorAddr &A) { + return OS << formatv("{0:x}", A.getValue()); +} + +inline raw_ostream &operator<<(raw_ostream &OS, const ExecutorAddrRange &R) { + return OS << formatv("{0:x} -- {1:x}", R.Start.getValue(), R.End.getValue()); +} + +namespace shared { + +class SPSExecutorAddr {}; + +/// SPS serializatior for ExecutorAddr. +template <> class SPSSerializationTraits<SPSExecutorAddr, ExecutorAddr> { +public: + static size_t size(const ExecutorAddr &EA) { + return SPSArgList<uint64_t>::size(EA.getValue()); + } + + static bool serialize(SPSOutputBuffer &BOB, const ExecutorAddr &EA) { + return SPSArgList<uint64_t>::serialize(BOB, EA.getValue()); + } + + static bool deserialize(SPSInputBuffer &BIB, ExecutorAddr &EA) { + uint64_t Tmp; + if (!SPSArgList<uint64_t>::deserialize(BIB, Tmp)) + return false; + EA = ExecutorAddr(Tmp); + return true; + } +}; + +using SPSExecutorAddrRange = SPSTuple<SPSExecutorAddr, SPSExecutorAddr>; + +/// Serialization traits for address ranges. +template <> +class SPSSerializationTraits<SPSExecutorAddrRange, ExecutorAddrRange> { +public: + static size_t size(const ExecutorAddrRange &Value) { + return SPSArgList<SPSExecutorAddr, SPSExecutorAddr>::size(Value.Start, + Value.End); + } + + static bool serialize(SPSOutputBuffer &BOB, const ExecutorAddrRange &Value) { + return SPSArgList<SPSExecutorAddr, SPSExecutorAddr>::serialize( + BOB, Value.Start, Value.End); + } + + static bool deserialize(SPSInputBuffer &BIB, ExecutorAddrRange &Value) { + return SPSArgList<SPSExecutorAddr, SPSExecutorAddr>::deserialize( + BIB, Value.Start, Value.End); + } +}; + +using SPSExecutorAddrRangeSequence = SPSSequence<SPSExecutorAddrRange>; + +} // End namespace shared. +} // End namespace orc. + +// Provide DenseMapInfo for ExecutorAddrs. +template <> struct DenseMapInfo<orc::ExecutorAddr> { + static inline orc::ExecutorAddr getEmptyKey() { + return orc::ExecutorAddr(DenseMapInfo<uint64_t>::getEmptyKey()); + } + static inline orc::ExecutorAddr getTombstoneKey() { + return orc::ExecutorAddr(DenseMapInfo<uint64_t>::getTombstoneKey()); + } + + static unsigned getHashValue(const orc::ExecutorAddr &Addr) { + return DenseMapInfo<uint64_t>::getHashValue(Addr.getValue()); + } + + static bool isEqual(const orc::ExecutorAddr &LHS, + const orc::ExecutorAddr &RHS) { + return DenseMapInfo<uint64_t>::isEqual(LHS.getValue(), RHS.getValue()); + } +}; + +} // End namespace llvm. + +#endif // LLVM_EXECUTIONENGINE_ORC_SHARED_EXECUTORADDRESS_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/Shared/OrcError.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/Shared/OrcError.h new file mode 100644 index 0000000000..487d473a02 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/Shared/OrcError.h @@ -0,0 +1,85 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===--------------- OrcError.h - Orc Error Types ---------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Define an error category, error codes, and helper utilities for Orc. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_SHARED_ORCERROR_H +#define LLVM_EXECUTIONENGINE_ORC_SHARED_ORCERROR_H + +#include "llvm/Support/Error.h" +#include "llvm/Support/raw_ostream.h" +#include <string> +#include <system_error> + +namespace llvm { +namespace orc { + +enum class OrcErrorCode : int { + // RPC Errors + UnknownORCError = 1, + DuplicateDefinition, + JITSymbolNotFound, + RemoteAllocatorDoesNotExist, + RemoteAllocatorIdAlreadyInUse, + RemoteMProtectAddrUnrecognized, + RemoteIndirectStubsOwnerDoesNotExist, + RemoteIndirectStubsOwnerIdAlreadyInUse, + RPCConnectionClosed, + RPCCouldNotNegotiateFunction, + RPCResponseAbandoned, + UnexpectedRPCCall, + UnexpectedRPCResponse, + UnknownErrorCodeFromRemote, + UnknownResourceHandle, + MissingSymbolDefinitions, + UnexpectedSymbolDefinitions, +}; + +std::error_code orcError(OrcErrorCode ErrCode); + +class DuplicateDefinition : public ErrorInfo<DuplicateDefinition> { +public: + static char ID; + + DuplicateDefinition(std::string SymbolName); + std::error_code convertToErrorCode() const override; + void log(raw_ostream &OS) const override; + const std::string &getSymbolName() const; +private: + std::string SymbolName; +}; + +class JITSymbolNotFound : public ErrorInfo<JITSymbolNotFound> { +public: + static char ID; + + JITSymbolNotFound(std::string SymbolName); + std::error_code convertToErrorCode() const override; + void log(raw_ostream &OS) const override; + const std::string &getSymbolName() const; +private: + std::string SymbolName; +}; + +} // End namespace orc. +} // End namespace llvm. + +#endif // LLVM_EXECUTIONENGINE_ORC_SHARED_ORCERROR_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h new file mode 100644 index 0000000000..a8295b7c57 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h @@ -0,0 +1,79 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===---- OrcRTBridge.h -- Utils for interacting with orc-rt ----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Declares types and symbol names provided by the ORC runtime. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_SHARED_ORCRTBRIDGE_H +#define LLVM_EXECUTIONENGINE_ORC_SHARED_ORCRTBRIDGE_H + +#include "llvm/ADT/StringMap.h" +#include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h" +#include "llvm/ExecutionEngine/Orc/Shared/SimpleRemoteEPCUtils.h" +#include "llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h" + +namespace llvm { +namespace orc { +namespace rt { + +extern const char *SimpleExecutorDylibManagerInstanceName; +extern const char *SimpleExecutorDylibManagerOpenWrapperName; +extern const char *SimpleExecutorDylibManagerLookupWrapperName; + +extern const char *SimpleExecutorMemoryManagerInstanceName; +extern const char *SimpleExecutorMemoryManagerReserveWrapperName; +extern const char *SimpleExecutorMemoryManagerFinalizeWrapperName; +extern const char *SimpleExecutorMemoryManagerDeallocateWrapperName; + +extern const char *MemoryWriteUInt8sWrapperName; +extern const char *MemoryWriteUInt16sWrapperName; +extern const char *MemoryWriteUInt32sWrapperName; +extern const char *MemoryWriteUInt64sWrapperName; +extern const char *MemoryWriteBuffersWrapperName; + +extern const char *RegisterEHFrameSectionWrapperName; +extern const char *DeregisterEHFrameSectionWrapperName; + +extern const char *RunAsMainWrapperName; + +using SPSSimpleExecutorDylibManagerOpenSignature = + shared::SPSExpected<uint64_t>(shared::SPSExecutorAddr, shared::SPSString, + uint64_t); + +using SPSSimpleExecutorDylibManagerLookupSignature = + shared::SPSExpected<shared::SPSSequence<shared::SPSExecutorAddr>>( + shared::SPSExecutorAddr, uint64_t, shared::SPSRemoteSymbolLookupSet); + +using SPSSimpleExecutorMemoryManagerReserveSignature = + shared::SPSExpected<shared::SPSExecutorAddr>(shared::SPSExecutorAddr, + uint64_t); +using SPSSimpleExecutorMemoryManagerFinalizeSignature = + shared::SPSError(shared::SPSExecutorAddr, shared::SPSFinalizeRequest); +using SPSSimpleExecutorMemoryManagerDeallocateSignature = shared::SPSError( + shared::SPSExecutorAddr, shared::SPSSequence<shared::SPSExecutorAddr>); + +using SPSRunAsMainSignature = int64_t(shared::SPSExecutorAddr, + shared::SPSSequence<shared::SPSString>); + +} // end namespace rt +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_SHARED_ORCRTBRIDGE_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h new file mode 100644 index 0000000000..788bac0659 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h @@ -0,0 +1,723 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===---- SimplePackedSerialization.h - simple serialization ----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// The behavior of the utilities in this header must be synchronized with the +// behavior of the utilities in +// compiler-rt/lib/orc/simple_packed_serialization.h. +// +// The Simple Packed Serialization (SPS) utilities are used to generate +// argument and return buffers for wrapper functions using the following +// serialization scheme: +// +// Primitives (signed types should be two's complement): +// bool, char, int8_t, uint8_t -- 8-bit (0=false, 1=true) +// int16_t, uint16_t -- 16-bit little endian +// int32_t, uint32_t -- 32-bit little endian +// int64_t, int64_t -- 64-bit little endian +// +// Sequence<T>: +// Serialized as the sequence length (as a uint64_t) followed by the +// serialization of each of the elements without padding. +// +// Tuple<T1, ..., TN>: +// Serialized as each of the element types from T1 to TN without padding. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_SHARED_SIMPLEPACKEDSERIALIZATION_H +#define LLVM_EXECUTIONENGINE_ORC_SHARED_SIMPLEPACKEDSERIALIZATION_H + +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/SwapByteOrder.h" + +#include <limits> +#include <string> +#include <tuple> +#include <type_traits> +#include <utility> +#include <vector> + +namespace llvm { +namespace orc { +namespace shared { + +/// Output char buffer with overflow check. +class SPSOutputBuffer { +public: + SPSOutputBuffer(char *Buffer, size_t Remaining) + : Buffer(Buffer), Remaining(Remaining) {} + bool write(const char *Data, size_t Size) { + if (Size > Remaining) + return false; + memcpy(Buffer, Data, Size); + Buffer += Size; + Remaining -= Size; + return true; + } + +private: + char *Buffer = nullptr; + size_t Remaining = 0; +}; + +/// Input char buffer with underflow check. +class SPSInputBuffer { +public: + SPSInputBuffer() = default; + SPSInputBuffer(const char *Buffer, size_t Remaining) + : Buffer(Buffer), Remaining(Remaining) {} + bool read(char *Data, size_t Size) { + if (Size > Remaining) + return false; + memcpy(Data, Buffer, Size); + Buffer += Size; + Remaining -= Size; + return true; + } + + const char *data() const { return Buffer; } + bool skip(size_t Size) { + if (Size > Remaining) + return false; + Buffer += Size; + Remaining -= Size; + return true; + } + +private: + const char *Buffer = nullptr; + size_t Remaining = 0; +}; + +/// Specialize to describe how to serialize/deserialize to/from the given +/// concrete type. +template <typename SPSTagT, typename ConcreteT, typename _ = void> +class SPSSerializationTraits; + +/// A utility class for serializing to a blob from a variadic list. +template <typename... ArgTs> class SPSArgList; + +// Empty list specialization for SPSArgList. +template <> class SPSArgList<> { +public: + static size_t size() { return 0; } + + static bool serialize(SPSOutputBuffer &OB) { return true; } + static bool deserialize(SPSInputBuffer &IB) { return true; } + + static bool serializeToSmallVector(SmallVectorImpl<char> &V) { return true; } + + static bool deserializeFromSmallVector(const SmallVectorImpl<char> &V) { + return true; + } +}; + +// Non-empty list specialization for SPSArgList. +template <typename SPSTagT, typename... SPSTagTs> +class SPSArgList<SPSTagT, SPSTagTs...> { +public: + // FIXME: This typedef is here to enable SPS arg serialization from + // JITLink. It can be removed once JITLink can access SPS directly. + using OutputBuffer = SPSOutputBuffer; + + template <typename ArgT, typename... ArgTs> + static size_t size(const ArgT &Arg, const ArgTs &...Args) { + return SPSSerializationTraits<SPSTagT, ArgT>::size(Arg) + + SPSArgList<SPSTagTs...>::size(Args...); + } + + template <typename ArgT, typename... ArgTs> + static bool serialize(SPSOutputBuffer &OB, const ArgT &Arg, + const ArgTs &...Args) { + return SPSSerializationTraits<SPSTagT, ArgT>::serialize(OB, Arg) && + SPSArgList<SPSTagTs...>::serialize(OB, Args...); + } + + template <typename ArgT, typename... ArgTs> + static bool deserialize(SPSInputBuffer &IB, ArgT &Arg, ArgTs &...Args) { + return SPSSerializationTraits<SPSTagT, ArgT>::deserialize(IB, Arg) && + SPSArgList<SPSTagTs...>::deserialize(IB, Args...); + } +}; + +/// SPS serialization for integral types, bool, and char. +template <typename SPSTagT> +class SPSSerializationTraits< + SPSTagT, SPSTagT, + std::enable_if_t<std::is_same<SPSTagT, bool>::value || + std::is_same<SPSTagT, char>::value || + std::is_same<SPSTagT, int8_t>::value || + std::is_same<SPSTagT, int16_t>::value || + std::is_same<SPSTagT, int32_t>::value || + std::is_same<SPSTagT, int64_t>::value || + std::is_same<SPSTagT, uint8_t>::value || + std::is_same<SPSTagT, uint16_t>::value || + std::is_same<SPSTagT, uint32_t>::value || + std::is_same<SPSTagT, uint64_t>::value>> { +public: + static size_t size(const SPSTagT &Value) { return sizeof(SPSTagT); } + + static bool serialize(SPSOutputBuffer &OB, const SPSTagT &Value) { + SPSTagT Tmp = Value; + if (sys::IsBigEndianHost) + sys::swapByteOrder(Tmp); + return OB.write(reinterpret_cast<const char *>(&Tmp), sizeof(Tmp)); + } + + static bool deserialize(SPSInputBuffer &IB, SPSTagT &Value) { + SPSTagT Tmp; + if (!IB.read(reinterpret_cast<char *>(&Tmp), sizeof(Tmp))) + return false; + if (sys::IsBigEndianHost) + sys::swapByteOrder(Tmp); + Value = Tmp; + return true; + } +}; + +// Any empty placeholder suitable as a substitute for void when deserializing +class SPSEmpty {}; + +/// SPS tag type for tuples. +/// +/// A blob tuple should be serialized by serializing each of the elements in +/// sequence. +template <typename... SPSTagTs> class SPSTuple { +public: + /// Convenience typedef of the corresponding arg list. + typedef SPSArgList<SPSTagTs...> AsArgList; +}; + +/// SPS tag type for sequences. +/// +/// SPSSequences should be serialized as a uint64_t sequence length, +/// followed by the serialization of each of the elements. +template <typename SPSElementTagT> class SPSSequence; + +/// SPS tag type for strings, which are equivalent to sequences of chars. +using SPSString = SPSSequence<char>; + +/// SPS tag type for maps. +/// +/// SPS maps are just sequences of (Key, Value) tuples. +template <typename SPSTagT1, typename SPSTagT2> +using SPSMap = SPSSequence<SPSTuple<SPSTagT1, SPSTagT2>>; + +/// Serialization for SPSEmpty type. +template <> class SPSSerializationTraits<SPSEmpty, SPSEmpty> { +public: + static size_t size(const SPSEmpty &EP) { return 0; } + static bool serialize(SPSOutputBuffer &OB, const SPSEmpty &BE) { + return true; + } + static bool deserialize(SPSInputBuffer &IB, SPSEmpty &BE) { return true; } +}; + +/// Specialize this to implement 'trivial' sequence serialization for +/// a concrete sequence type. +/// +/// Trivial sequence serialization uses the sequence's 'size' member to get the +/// length of the sequence, and uses a range-based for loop to iterate over the +/// elements. +/// +/// Specializing this template class means that you do not need to provide a +/// specialization of SPSSerializationTraits for your type. +template <typename SPSElementTagT, typename ConcreteSequenceT> +class TrivialSPSSequenceSerialization { +public: + static constexpr bool available = false; +}; + +/// Specialize this to implement 'trivial' sequence deserialization for +/// a concrete sequence type. +/// +/// Trivial deserialization calls a static 'reserve(SequenceT&)' method on your +/// specialization (you must implement this) to reserve space, and then calls +/// a static 'append(SequenceT&, ElementT&) method to append each of the +/// deserialized elements. +/// +/// Specializing this template class means that you do not need to provide a +/// specialization of SPSSerializationTraits for your type. +template <typename SPSElementTagT, typename ConcreteSequenceT> +class TrivialSPSSequenceDeserialization { +public: + static constexpr bool available = false; +}; + +/// Trivial std::string -> SPSSequence<char> serialization. +template <> class TrivialSPSSequenceSerialization<char, std::string> { +public: + static constexpr bool available = true; +}; + +/// Trivial SPSSequence<char> -> std::string deserialization. +template <> class TrivialSPSSequenceDeserialization<char, std::string> { +public: + static constexpr bool available = true; + + using element_type = char; + + static void reserve(std::string &S, uint64_t Size) { S.reserve(Size); } + static bool append(std::string &S, char C) { + S.push_back(C); + return true; + } +}; + +/// Trivial std::vector<T> -> SPSSequence<SPSElementTagT> serialization. +template <typename SPSElementTagT, typename T> +class TrivialSPSSequenceSerialization<SPSElementTagT, std::vector<T>> { +public: + static constexpr bool available = true; +}; + +/// Trivial SPSSequence<SPSElementTagT> -> std::vector<T> deserialization. +template <typename SPSElementTagT, typename T> +class TrivialSPSSequenceDeserialization<SPSElementTagT, std::vector<T>> { +public: + static constexpr bool available = true; + + using element_type = typename std::vector<T>::value_type; + + static void reserve(std::vector<T> &V, uint64_t Size) { V.reserve(Size); } + static bool append(std::vector<T> &V, T E) { + V.push_back(std::move(E)); + return true; + } +}; + +/// Trivial SmallVectorImpl<T> -> SPSSequence<char> serialization. +template <typename SPSElementTagT, typename T> +class TrivialSPSSequenceSerialization<SPSElementTagT, SmallVectorImpl<T>> { +public: + static constexpr bool available = true; +}; + +/// Trivial SPSSequence<SPSElementTagT> -> SmallVectorImpl<T> deserialization. +template <typename SPSElementTagT, typename T> +class TrivialSPSSequenceDeserialization<SPSElementTagT, SmallVectorImpl<T>> { +public: + static constexpr bool available = true; + + using element_type = typename SmallVectorImpl<T>::value_type; + + static void reserve(SmallVectorImpl<T> &V, uint64_t Size) { V.reserve(Size); } + static bool append(SmallVectorImpl<T> &V, T E) { + V.push_back(std::move(E)); + return true; + } +}; + +/// Trivial SmallVectorImpl<T> -> SPSSequence<char> serialization. +template <typename SPSElementTagT, typename T, unsigned N> +class TrivialSPSSequenceSerialization<SPSElementTagT, SmallVector<T, N>> + : public TrivialSPSSequenceSerialization<SPSElementTagT, + SmallVectorImpl<T>> {}; + +/// Trivial SPSSequence<SPSElementTagT> -> SmallVectorImpl<T> deserialization. +template <typename SPSElementTagT, typename T, unsigned N> +class TrivialSPSSequenceDeserialization<SPSElementTagT, SmallVector<T, N>> + : public TrivialSPSSequenceDeserialization<SPSElementTagT, + SmallVectorImpl<T>> {}; + +/// Trivial ArrayRef<T> -> SPSSequence<SPSElementTagT> serialization. +template <typename SPSElementTagT, typename T> +class TrivialSPSSequenceSerialization<SPSElementTagT, ArrayRef<T>> { +public: + static constexpr bool available = true; +}; + +/// Specialized SPSSequence<char> -> ArrayRef<char> serialization. +/// +/// On deserialize, points directly into the input buffer. +template <> class SPSSerializationTraits<SPSSequence<char>, ArrayRef<char>> { +public: + static size_t size(const ArrayRef<char> &A) { + return SPSArgList<uint64_t>::size(static_cast<uint64_t>(A.size())) + + A.size(); + } + + static bool serialize(SPSOutputBuffer &OB, const ArrayRef<char> &A) { + if (!SPSArgList<uint64_t>::serialize(OB, static_cast<uint64_t>(A.size()))) + return false; + return OB.write(A.data(), A.size()); + } + + static bool deserialize(SPSInputBuffer &IB, ArrayRef<char> &A) { + uint64_t Size; + if (!SPSArgList<uint64_t>::deserialize(IB, Size)) + return false; + if (Size > std::numeric_limits<size_t>::max()) + return false; + A = {IB.data(), static_cast<size_t>(Size)}; + return IB.skip(Size); + } +}; + +/// 'Trivial' sequence serialization: Sequence is serialized as a uint64_t size +/// followed by a for-earch loop over the elements of the sequence to serialize +/// each of them. +template <typename SPSElementTagT, typename SequenceT> +class SPSSerializationTraits<SPSSequence<SPSElementTagT>, SequenceT, + std::enable_if_t<TrivialSPSSequenceSerialization< + SPSElementTagT, SequenceT>::available>> { +public: + static size_t size(const SequenceT &S) { + size_t Size = SPSArgList<uint64_t>::size(static_cast<uint64_t>(S.size())); + for (const auto &E : S) + Size += SPSArgList<SPSElementTagT>::size(E); + return Size; + } + + static bool serialize(SPSOutputBuffer &OB, const SequenceT &S) { + if (!SPSArgList<uint64_t>::serialize(OB, static_cast<uint64_t>(S.size()))) + return false; + for (const auto &E : S) + if (!SPSArgList<SPSElementTagT>::serialize(OB, E)) + return false; + return true; + } + + static bool deserialize(SPSInputBuffer &IB, SequenceT &S) { + using TBSD = TrivialSPSSequenceDeserialization<SPSElementTagT, SequenceT>; + uint64_t Size; + if (!SPSArgList<uint64_t>::deserialize(IB, Size)) + return false; + TBSD::reserve(S, Size); + for (size_t I = 0; I != Size; ++I) { + typename TBSD::element_type E; + if (!SPSArgList<SPSElementTagT>::deserialize(IB, E)) + return false; + if (!TBSD::append(S, std::move(E))) + return false; + } + return true; + } +}; + +/// SPSTuple serialization for std::tuple. +template <typename... SPSTagTs, typename... Ts> +class SPSSerializationTraits<SPSTuple<SPSTagTs...>, std::tuple<Ts...>> { +private: + using TupleArgList = typename SPSTuple<SPSTagTs...>::AsArgList; + using ArgIndices = std::make_index_sequence<sizeof...(Ts)>; + + template <std::size_t... I> + static size_t size(const std::tuple<Ts...> &T, std::index_sequence<I...>) { + return TupleArgList::size(std::get<I>(T)...); + } + + template <std::size_t... I> + static bool serialize(SPSOutputBuffer &OB, const std::tuple<Ts...> &T, + std::index_sequence<I...>) { + return TupleArgList::serialize(OB, std::get<I>(T)...); + } + + template <std::size_t... I> + static bool deserialize(SPSInputBuffer &IB, std::tuple<Ts...> &T, + std::index_sequence<I...>) { + return TupleArgList::deserialize(IB, std::get<I>(T)...); + } + +public: + static size_t size(const std::tuple<Ts...> &T) { + return size(T, ArgIndices{}); + } + + static bool serialize(SPSOutputBuffer &OB, const std::tuple<Ts...> &T) { + return serialize(OB, T, ArgIndices{}); + } + + static bool deserialize(SPSInputBuffer &IB, std::tuple<Ts...> &T) { + return deserialize(IB, T, ArgIndices{}); + } +}; + +/// SPSTuple serialization for std::pair. +template <typename SPSTagT1, typename SPSTagT2, typename T1, typename T2> +class SPSSerializationTraits<SPSTuple<SPSTagT1, SPSTagT2>, std::pair<T1, T2>> { +public: + static size_t size(const std::pair<T1, T2> &P) { + return SPSArgList<SPSTagT1>::size(P.first) + + SPSArgList<SPSTagT2>::size(P.second); + } + + static bool serialize(SPSOutputBuffer &OB, const std::pair<T1, T2> &P) { + return SPSArgList<SPSTagT1>::serialize(OB, P.first) && + SPSArgList<SPSTagT2>::serialize(OB, P.second); + } + + static bool deserialize(SPSInputBuffer &IB, std::pair<T1, T2> &P) { + return SPSArgList<SPSTagT1>::deserialize(IB, P.first) && + SPSArgList<SPSTagT2>::deserialize(IB, P.second); + } +}; + +/// Serialization for StringRefs. +/// +/// Serialization is as for regular strings. Deserialization points directly +/// into the blob. +template <> class SPSSerializationTraits<SPSString, StringRef> { +public: + static size_t size(const StringRef &S) { + return SPSArgList<uint64_t>::size(static_cast<uint64_t>(S.size())) + + S.size(); + } + + static bool serialize(SPSOutputBuffer &OB, StringRef S) { + if (!SPSArgList<uint64_t>::serialize(OB, static_cast<uint64_t>(S.size()))) + return false; + return OB.write(S.data(), S.size()); + } + + static bool deserialize(SPSInputBuffer &IB, StringRef &S) { + const char *Data = nullptr; + uint64_t Size; + if (!SPSArgList<uint64_t>::deserialize(IB, Size)) + return false; + Data = IB.data(); + if (!IB.skip(Size)) + return false; + S = StringRef(Data, Size); + return true; + } +}; + +/// Serialization for StringMap<ValueT>s. +template <typename SPSValueT, typename ValueT> +class SPSSerializationTraits<SPSSequence<SPSTuple<SPSString, SPSValueT>>, + StringMap<ValueT>> { +public: + static size_t size(const StringMap<ValueT> &M) { + size_t Sz = SPSArgList<uint64_t>::size(static_cast<uint64_t>(M.size())); + for (auto &E : M) + Sz += SPSArgList<SPSString, SPSValueT>::size(E.first(), E.second); + return Sz; + } + + static bool serialize(SPSOutputBuffer &OB, const StringMap<ValueT> &M) { + if (!SPSArgList<uint64_t>::serialize(OB, static_cast<uint64_t>(M.size()))) + return false; + + for (auto &E : M) + if (!SPSArgList<SPSString, SPSValueT>::serialize(OB, E.first(), E.second)) + return false; + + return true; + } + + static bool deserialize(SPSInputBuffer &IB, StringMap<ValueT> &M) { + uint64_t Size; + assert(M.empty() && "M already contains elements"); + + if (!SPSArgList<uint64_t>::deserialize(IB, Size)) + return false; + + while (Size--) { + StringRef S; + ValueT V; + if (!SPSArgList<SPSString, SPSValueT>::deserialize(IB, S, V)) + return false; + if (!M.insert(std::make_pair(S, V)).second) + return false; + } + + return true; + } +}; + +/// SPS tag type for errors. +class SPSError; + +/// SPS tag type for expecteds, which are either a T or a string representing +/// an error. +template <typename SPSTagT> class SPSExpected; + +namespace detail { + +/// Helper type for serializing Errors. +/// +/// llvm::Errors are move-only, and not inspectable except by consuming them. +/// This makes them unsuitable for direct serialization via +/// SPSSerializationTraits, which needs to inspect values twice (once to +/// determine the amount of space to reserve, and then again to serialize). +/// +/// The SPSSerializableError type is a helper that can be +/// constructed from an llvm::Error, but inspected more than once. +struct SPSSerializableError { + bool HasError = false; + std::string ErrMsg; +}; + +/// Helper type for serializing Expected<T>s. +/// +/// See SPSSerializableError for more details. +/// +// FIXME: Use std::variant for storage once we have c++17. +template <typename T> struct SPSSerializableExpected { + bool HasValue = false; + T Value{}; + std::string ErrMsg; +}; + +inline SPSSerializableError toSPSSerializable(Error Err) { + if (Err) + return {true, toString(std::move(Err))}; + return {false, {}}; +} + +inline Error fromSPSSerializable(SPSSerializableError BSE) { + if (BSE.HasError) + return make_error<StringError>(BSE.ErrMsg, inconvertibleErrorCode()); + return Error::success(); +} + +template <typename T> +SPSSerializableExpected<T> toSPSSerializable(Expected<T> E) { + if (E) + return {true, std::move(*E), {}}; + else + return {false, {}, toString(E.takeError())}; +} + +template <typename T> +Expected<T> fromSPSSerializable(SPSSerializableExpected<T> BSE) { + if (BSE.HasValue) + return std::move(BSE.Value); + else + return make_error<StringError>(BSE.ErrMsg, inconvertibleErrorCode()); +} + +} // end namespace detail + +/// Serialize to a SPSError from a detail::SPSSerializableError. +template <> +class SPSSerializationTraits<SPSError, detail::SPSSerializableError> { +public: + static size_t size(const detail::SPSSerializableError &BSE) { + size_t Size = SPSArgList<bool>::size(BSE.HasError); + if (BSE.HasError) + Size += SPSArgList<SPSString>::size(BSE.ErrMsg); + return Size; + } + + static bool serialize(SPSOutputBuffer &OB, + const detail::SPSSerializableError &BSE) { + if (!SPSArgList<bool>::serialize(OB, BSE.HasError)) + return false; + if (BSE.HasError) + if (!SPSArgList<SPSString>::serialize(OB, BSE.ErrMsg)) + return false; + return true; + } + + static bool deserialize(SPSInputBuffer &IB, + detail::SPSSerializableError &BSE) { + if (!SPSArgList<bool>::deserialize(IB, BSE.HasError)) + return false; + + if (!BSE.HasError) + return true; + + return SPSArgList<SPSString>::deserialize(IB, BSE.ErrMsg); + } +}; + +/// Serialize to a SPSExpected<SPSTagT> from a +/// detail::SPSSerializableExpected<T>. +template <typename SPSTagT, typename T> +class SPSSerializationTraits<SPSExpected<SPSTagT>, + detail::SPSSerializableExpected<T>> { +public: + static size_t size(const detail::SPSSerializableExpected<T> &BSE) { + size_t Size = SPSArgList<bool>::size(BSE.HasValue); + if (BSE.HasValue) + Size += SPSArgList<SPSTagT>::size(BSE.Value); + else + Size += SPSArgList<SPSString>::size(BSE.ErrMsg); + return Size; + } + + static bool serialize(SPSOutputBuffer &OB, + const detail::SPSSerializableExpected<T> &BSE) { + if (!SPSArgList<bool>::serialize(OB, BSE.HasValue)) + return false; + + if (BSE.HasValue) + return SPSArgList<SPSTagT>::serialize(OB, BSE.Value); + + return SPSArgList<SPSString>::serialize(OB, BSE.ErrMsg); + } + + static bool deserialize(SPSInputBuffer &IB, + detail::SPSSerializableExpected<T> &BSE) { + if (!SPSArgList<bool>::deserialize(IB, BSE.HasValue)) + return false; + + if (BSE.HasValue) + return SPSArgList<SPSTagT>::deserialize(IB, BSE.Value); + + return SPSArgList<SPSString>::deserialize(IB, BSE.ErrMsg); + } +}; + +/// Serialize to a SPSExpected<SPSTagT> from a detail::SPSSerializableError. +template <typename SPSTagT> +class SPSSerializationTraits<SPSExpected<SPSTagT>, + detail::SPSSerializableError> { +public: + static size_t size(const detail::SPSSerializableError &BSE) { + assert(BSE.HasError && "Cannot serialize expected from a success value"); + return SPSArgList<bool>::size(false) + + SPSArgList<SPSString>::size(BSE.ErrMsg); + } + + static bool serialize(SPSOutputBuffer &OB, + const detail::SPSSerializableError &BSE) { + assert(BSE.HasError && "Cannot serialize expected from a success value"); + if (!SPSArgList<bool>::serialize(OB, false)) + return false; + return SPSArgList<SPSString>::serialize(OB, BSE.ErrMsg); + } +}; + +/// Serialize to a SPSExpected<SPSTagT> from a T. +template <typename SPSTagT, typename T> +class SPSSerializationTraits<SPSExpected<SPSTagT>, T> { +public: + static size_t size(const T &Value) { + return SPSArgList<bool>::size(true) + SPSArgList<SPSTagT>::size(Value); + } + + static bool serialize(SPSOutputBuffer &OB, const T &Value) { + if (!SPSArgList<bool>::serialize(OB, true)) + return false; + return SPSArgList<SPSTagT>::serialize(Value); + } +}; + +} // end namespace shared +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_SHARED_SIMPLEPACKEDSERIALIZATION_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/Shared/SimpleRemoteEPCUtils.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/Shared/SimpleRemoteEPCUtils.h new file mode 100644 index 0000000000..a6b69eeef6 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/Shared/SimpleRemoteEPCUtils.h @@ -0,0 +1,246 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===--- SimpleRemoteEPCUtils.h - Utils for Simple Remote EPC ---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Message definitions and other utilities for SimpleRemoteEPC and +// SimpleRemoteEPCServer. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_SHARED_SIMPLEREMOTEEPCUTILS_H +#define LLVM_EXECUTIONENGINE_ORC_SHARED_SIMPLEREMOTEEPCUTILS_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h" +#include "llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h" +#include "llvm/Support/Error.h" + +#include <atomic> +#include <mutex> +#include <string> +#include <thread> + +namespace llvm { +namespace orc { + +namespace SimpleRemoteEPCDefaultBootstrapSymbolNames { +extern const char *ExecutorSessionObjectName; +extern const char *DispatchFnName; +} // end namespace SimpleRemoteEPCDefaultBootstrapSymbolNames + +enum class SimpleRemoteEPCOpcode : uint8_t { + Setup, + Hangup, + Result, + CallWrapper, + LastOpC = CallWrapper +}; + +struct SimpleRemoteEPCExecutorInfo { + std::string TargetTriple; + uint64_t PageSize; + StringMap<ExecutorAddr> BootstrapSymbols; +}; + +using SimpleRemoteEPCArgBytesVector = SmallVector<char, 128>; + +class SimpleRemoteEPCTransportClient { +public: + enum HandleMessageAction { ContinueSession, EndSession }; + + virtual ~SimpleRemoteEPCTransportClient(); + + /// Handle receipt of a message. + /// + /// Returns an Error if the message cannot be handled, 'EndSession' if the + /// client will not accept any further messages, and 'ContinueSession' + /// otherwise. + virtual Expected<HandleMessageAction> + handleMessage(SimpleRemoteEPCOpcode OpC, uint64_t SeqNo, ExecutorAddr TagAddr, + SimpleRemoteEPCArgBytesVector ArgBytes) = 0; + + /// Handle a disconnection from the underlying transport. No further messages + /// should be sent to handleMessage after this is called. + /// Err may contain an Error value indicating unexpected disconnection. This + /// allows clients to log such errors, but no attempt should be made at + /// recovery (which should be handled inside the transport class, if it is + /// supported at all). + virtual void handleDisconnect(Error Err) = 0; +}; + +class SimpleRemoteEPCTransport { +public: + virtual ~SimpleRemoteEPCTransport(); + + /// Called during setup of the client to indicate that the client is ready + /// to receive messages. + /// + /// Transport objects should not access the client until this method is + /// called. + virtual Error start() = 0; + + /// Send a SimpleRemoteEPC message. + /// + /// This function may be called concurrently. Subclasses should implement + /// locking if required for the underlying transport. + virtual Error sendMessage(SimpleRemoteEPCOpcode OpC, uint64_t SeqNo, + ExecutorAddr TagAddr, ArrayRef<char> ArgBytes) = 0; + + /// Trigger disconnection from the transport. The implementation should + /// respond by calling handleDisconnect on the client once disconnection + /// is complete. May be called more than once and from different threads. + virtual void disconnect() = 0; +}; + +/// Uses read/write on FileDescriptors for transport. +class FDSimpleRemoteEPCTransport : public SimpleRemoteEPCTransport { +public: + /// Create a FDSimpleRemoteEPCTransport using the given FDs for + /// reading (InFD) and writing (OutFD). + static Expected<std::unique_ptr<FDSimpleRemoteEPCTransport>> + Create(SimpleRemoteEPCTransportClient &C, int InFD, int OutFD); + + /// Create a FDSimpleRemoteEPCTransport using the given FD for both + /// reading and writing. + static Expected<std::unique_ptr<FDSimpleRemoteEPCTransport>> + Create(SimpleRemoteEPCTransportClient &C, int FD) { + return Create(C, FD, FD); + } + + ~FDSimpleRemoteEPCTransport() override; + + Error start() override; + + Error sendMessage(SimpleRemoteEPCOpcode OpC, uint64_t SeqNo, + ExecutorAddr TagAddr, ArrayRef<char> ArgBytes) override; + + void disconnect() override; + +private: + FDSimpleRemoteEPCTransport(SimpleRemoteEPCTransportClient &C, int InFD, + int OutFD) + : C(C), InFD(InFD), OutFD(OutFD) {} + + Error readBytes(char *Dst, size_t Size, bool *IsEOF = nullptr); + int writeBytes(const char *Src, size_t Size); + void listenLoop(); + + std::mutex M; + SimpleRemoteEPCTransportClient &C; + std::thread ListenerThread; + int InFD, OutFD; + std::atomic<bool> Disconnected{false}; +}; + +struct RemoteSymbolLookupSetElement { + std::string Name; + bool Required; +}; + +using RemoteSymbolLookupSet = std::vector<RemoteSymbolLookupSetElement>; + +struct RemoteSymbolLookup { + uint64_t H; + RemoteSymbolLookupSet Symbols; +}; + +namespace shared { + +using SPSRemoteSymbolLookupSetElement = SPSTuple<SPSString, bool>; + +using SPSRemoteSymbolLookupSet = SPSSequence<SPSRemoteSymbolLookupSetElement>; + +using SPSRemoteSymbolLookup = SPSTuple<uint64_t, SPSRemoteSymbolLookupSet>; + +/// Tuple containing target triple, page size, and bootstrap symbols. +using SPSSimpleRemoteEPCExecutorInfo = + SPSTuple<SPSString, uint64_t, + SPSSequence<SPSTuple<SPSString, SPSExecutorAddr>>>; + +template <> +class SPSSerializationTraits<SPSRemoteSymbolLookupSetElement, + RemoteSymbolLookupSetElement> { +public: + static size_t size(const RemoteSymbolLookupSetElement &V) { + return SPSArgList<SPSString, bool>::size(V.Name, V.Required); + } + + static size_t serialize(SPSOutputBuffer &OB, + const RemoteSymbolLookupSetElement &V) { + return SPSArgList<SPSString, bool>::serialize(OB, V.Name, V.Required); + } + + static size_t deserialize(SPSInputBuffer &IB, + RemoteSymbolLookupSetElement &V) { + return SPSArgList<SPSString, bool>::deserialize(IB, V.Name, V.Required); + } +}; + +template <> +class SPSSerializationTraits<SPSRemoteSymbolLookup, RemoteSymbolLookup> { +public: + static size_t size(const RemoteSymbolLookup &V) { + return SPSArgList<uint64_t, SPSRemoteSymbolLookupSet>::size(V.H, V.Symbols); + } + + static size_t serialize(SPSOutputBuffer &OB, const RemoteSymbolLookup &V) { + return SPSArgList<uint64_t, SPSRemoteSymbolLookupSet>::serialize(OB, V.H, + V.Symbols); + } + + static size_t deserialize(SPSInputBuffer &IB, RemoteSymbolLookup &V) { + return SPSArgList<uint64_t, SPSRemoteSymbolLookupSet>::deserialize( + IB, V.H, V.Symbols); + } +}; + +template <> +class SPSSerializationTraits<SPSSimpleRemoteEPCExecutorInfo, + SimpleRemoteEPCExecutorInfo> { +public: + static size_t size(const SimpleRemoteEPCExecutorInfo &SI) { + return SPSSimpleRemoteEPCExecutorInfo::AsArgList ::size( + SI.TargetTriple, SI.PageSize, SI.BootstrapSymbols); + } + + static bool serialize(SPSOutputBuffer &OB, + const SimpleRemoteEPCExecutorInfo &SI) { + return SPSSimpleRemoteEPCExecutorInfo::AsArgList ::serialize( + OB, SI.TargetTriple, SI.PageSize, SI.BootstrapSymbols); + } + + static bool deserialize(SPSInputBuffer &IB, SimpleRemoteEPCExecutorInfo &SI) { + return SPSSimpleRemoteEPCExecutorInfo::AsArgList ::deserialize( + IB, SI.TargetTriple, SI.PageSize, SI.BootstrapSymbols); + } +}; + +using SPSLoadDylibSignature = SPSExpected<SPSExecutorAddr>(SPSExecutorAddr, + SPSString, uint64_t); + +using SPSLookupSymbolsSignature = + SPSExpected<SPSSequence<SPSSequence<SPSExecutorAddr>>>( + SPSExecutorAddr, SPSSequence<SPSRemoteSymbolLookup>); + +} // end namespace shared +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_SHARED_SIMPLEREMOTEEPCUTILS_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h new file mode 100644 index 0000000000..1ee04e0fdd --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h @@ -0,0 +1,263 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===--- TargetProcessControlTypes.h -- Shared Core/TPC types ---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// TargetProcessControl types that are used by both the Orc and +// OrcTargetProcess libraries. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_SHARED_TARGETPROCESSCONTROLTYPES_H +#define LLVM_EXECUTIONENGINE_ORC_SHARED_TARGETPROCESSCONTROLTYPES_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/ExecutionEngine/Orc/Shared/AllocationActions.h" +#include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h" +#include "llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h" +#include "llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h" +#include "llvm/Support/Memory.h" + +#include <vector> + +namespace llvm { +namespace orc { +namespace tpctypes { + +enum WireProtectionFlags : uint8_t { + WPF_None = 0, + WPF_Read = 1U << 0, + WPF_Write = 1U << 1, + WPF_Exec = 1U << 2, + LLVM_MARK_AS_BITMASK_ENUM(WPF_Exec) +}; + +/// Convert from sys::Memory::ProtectionFlags +inline WireProtectionFlags +toWireProtectionFlags(sys::Memory::ProtectionFlags PF) { + WireProtectionFlags WPF = WPF_None; + if (PF & sys::Memory::MF_READ) + WPF |= WPF_Read; + if (PF & sys::Memory::MF_WRITE) + WPF |= WPF_Write; + if (PF & sys::Memory::MF_EXEC) + WPF |= WPF_Exec; + return WPF; +} + +inline sys::Memory::ProtectionFlags +fromWireProtectionFlags(WireProtectionFlags WPF) { + int PF = 0; + if (WPF & WPF_Read) + PF |= sys::Memory::MF_READ; + if (WPF & WPF_Write) + PF |= sys::Memory::MF_WRITE; + if (WPF & WPF_Exec) + PF |= sys::Memory::MF_EXEC; + return static_cast<sys::Memory::ProtectionFlags>(PF); +} + +inline std::string getWireProtectionFlagsStr(WireProtectionFlags WPF) { + std::string Result; + Result += (WPF & WPF_Read) ? 'R' : '-'; + Result += (WPF & WPF_Write) ? 'W' : '-'; + Result += (WPF & WPF_Exec) ? 'X' : '-'; + return Result; +} + +struct SegFinalizeRequest { + WireProtectionFlags Prot; + ExecutorAddr Addr; + uint64_t Size; + ArrayRef<char> Content; +}; + +struct FinalizeRequest { + std::vector<SegFinalizeRequest> Segments; + shared::AllocActions Actions; +}; + +template <typename T> struct UIntWrite { + UIntWrite() = default; + UIntWrite(ExecutorAddr Addr, T Value) : Addr(Addr), Value(Value) {} + + ExecutorAddr Addr; + T Value = 0; +}; + +/// Describes a write to a uint8_t. +using UInt8Write = UIntWrite<uint8_t>; + +/// Describes a write to a uint16_t. +using UInt16Write = UIntWrite<uint16_t>; + +/// Describes a write to a uint32_t. +using UInt32Write = UIntWrite<uint32_t>; + +/// Describes a write to a uint64_t. +using UInt64Write = UIntWrite<uint64_t>; + +/// Describes a write to a buffer. +/// For use with TargetProcessControl::MemoryAccess objects. +struct BufferWrite { + BufferWrite() = default; + BufferWrite(ExecutorAddr Addr, StringRef Buffer) + : Addr(Addr), Buffer(Buffer) {} + + ExecutorAddr Addr; + StringRef Buffer; +}; + +/// A handle used to represent a loaded dylib in the target process. +using DylibHandle = JITTargetAddress; + +using LookupResult = std::vector<JITTargetAddress>; + +} // end namespace tpctypes + +namespace shared { + +class SPSMemoryProtectionFlags {}; + +using SPSSegFinalizeRequest = + SPSTuple<SPSMemoryProtectionFlags, SPSExecutorAddr, uint64_t, + SPSSequence<char>>; + +using SPSFinalizeRequest = SPSTuple<SPSSequence<SPSSegFinalizeRequest>, + SPSSequence<SPSAllocActionCallPair>>; + +template <typename T> +using SPSMemoryAccessUIntWrite = SPSTuple<SPSExecutorAddr, T>; + +using SPSMemoryAccessUInt8Write = SPSMemoryAccessUIntWrite<uint8_t>; +using SPSMemoryAccessUInt16Write = SPSMemoryAccessUIntWrite<uint16_t>; +using SPSMemoryAccessUInt32Write = SPSMemoryAccessUIntWrite<uint32_t>; +using SPSMemoryAccessUInt64Write = SPSMemoryAccessUIntWrite<uint64_t>; + +using SPSMemoryAccessBufferWrite = SPSTuple<SPSExecutorAddr, SPSSequence<char>>; + +template <> +class SPSSerializationTraits<SPSMemoryProtectionFlags, + tpctypes::WireProtectionFlags> { +public: + static size_t size(const tpctypes::WireProtectionFlags &WPF) { + return SPSArgList<uint8_t>::size(static_cast<uint8_t>(WPF)); + } + + static bool serialize(SPSOutputBuffer &OB, + const tpctypes::WireProtectionFlags &WPF) { + return SPSArgList<uint8_t>::serialize(OB, static_cast<uint8_t>(WPF)); + } + + static bool deserialize(SPSInputBuffer &IB, + tpctypes::WireProtectionFlags &WPF) { + uint8_t Val; + if (!SPSArgList<uint8_t>::deserialize(IB, Val)) + return false; + WPF = static_cast<tpctypes::WireProtectionFlags>(Val); + return true; + } +}; + +template <> +class SPSSerializationTraits<SPSSegFinalizeRequest, + tpctypes::SegFinalizeRequest> { + using SFRAL = SPSSegFinalizeRequest::AsArgList; + +public: + static size_t size(const tpctypes::SegFinalizeRequest &SFR) { + return SFRAL::size(SFR.Prot, SFR.Addr, SFR.Size, SFR.Content); + } + + static bool serialize(SPSOutputBuffer &OB, + const tpctypes::SegFinalizeRequest &SFR) { + return SFRAL::serialize(OB, SFR.Prot, SFR.Addr, SFR.Size, SFR.Content); + } + + static bool deserialize(SPSInputBuffer &IB, + tpctypes::SegFinalizeRequest &SFR) { + return SFRAL::deserialize(IB, SFR.Prot, SFR.Addr, SFR.Size, SFR.Content); + } +}; + +template <> +class SPSSerializationTraits<SPSFinalizeRequest, tpctypes::FinalizeRequest> { + using FRAL = SPSFinalizeRequest::AsArgList; + +public: + static size_t size(const tpctypes::FinalizeRequest &FR) { + return FRAL::size(FR.Segments, FR.Actions); + } + + static bool serialize(SPSOutputBuffer &OB, + const tpctypes::FinalizeRequest &FR) { + return FRAL::serialize(OB, FR.Segments, FR.Actions); + } + + static bool deserialize(SPSInputBuffer &IB, tpctypes::FinalizeRequest &FR) { + return FRAL::deserialize(IB, FR.Segments, FR.Actions); + } +}; + +template <typename T> +class SPSSerializationTraits<SPSMemoryAccessUIntWrite<T>, + tpctypes::UIntWrite<T>> { +public: + static size_t size(const tpctypes::UIntWrite<T> &W) { + return SPSTuple<SPSExecutorAddr, T>::AsArgList::size(W.Addr, W.Value); + } + + static bool serialize(SPSOutputBuffer &OB, const tpctypes::UIntWrite<T> &W) { + return SPSTuple<SPSExecutorAddr, T>::AsArgList::serialize(OB, W.Addr, + W.Value); + } + + static bool deserialize(SPSInputBuffer &IB, tpctypes::UIntWrite<T> &W) { + return SPSTuple<SPSExecutorAddr, T>::AsArgList::deserialize(IB, W.Addr, + W.Value); + } +}; + +template <> +class SPSSerializationTraits<SPSMemoryAccessBufferWrite, + tpctypes::BufferWrite> { +public: + static size_t size(const tpctypes::BufferWrite &W) { + return SPSTuple<SPSExecutorAddr, SPSSequence<char>>::AsArgList::size( + W.Addr, W.Buffer); + } + + static bool serialize(SPSOutputBuffer &OB, const tpctypes::BufferWrite &W) { + return SPSTuple<SPSExecutorAddr, SPSSequence<char>>::AsArgList ::serialize( + OB, W.Addr, W.Buffer); + } + + static bool deserialize(SPSInputBuffer &IB, tpctypes::BufferWrite &W) { + return SPSTuple<SPSExecutorAddr, + SPSSequence<char>>::AsArgList ::deserialize(IB, W.Addr, + W.Buffer); + } +}; + + +} // end namespace shared +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_SHARED_TARGETPROCESSCONTROLTYPES_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h new file mode 100644 index 0000000000..e3d9217438 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h @@ -0,0 +1,747 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===- WrapperFunctionUtils.h - Utilities for wrapper functions -*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// A buffer for serialized results. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_SHARED_WRAPPERFUNCTIONUTILS_H +#define LLVM_EXECUTIONENGINE_ORC_SHARED_WRAPPERFUNCTIONUTILS_H + +#include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h" +#include "llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h" +#include "llvm/Support/Error.h" + +#include <type_traits> + +namespace llvm { +namespace orc { +namespace shared { + +// Must be kept in-sync with compiler-rt/lib/orc/c-api.h. +union CWrapperFunctionResultDataUnion { + char *ValuePtr; + char Value[sizeof(ValuePtr)]; +}; + +// Must be kept in-sync with compiler-rt/lib/orc/c-api.h. +typedef struct { + CWrapperFunctionResultDataUnion Data; + size_t Size; +} CWrapperFunctionResult; + +/// C++ wrapper function result: Same as CWrapperFunctionResult but +/// auto-releases memory. +class WrapperFunctionResult { +public: + /// Create a default WrapperFunctionResult. + WrapperFunctionResult() { init(R); } + + /// Create a WrapperFunctionResult by taking ownership of a + /// CWrapperFunctionResult. + /// + /// Warning: This should only be used by clients writing wrapper-function + /// caller utilities (like TargetProcessControl). + WrapperFunctionResult(CWrapperFunctionResult R) : R(R) { + // Reset R. + init(R); + } + + WrapperFunctionResult(const WrapperFunctionResult &) = delete; + WrapperFunctionResult &operator=(const WrapperFunctionResult &) = delete; + + WrapperFunctionResult(WrapperFunctionResult &&Other) { + init(R); + std::swap(R, Other.R); + } + + WrapperFunctionResult &operator=(WrapperFunctionResult &&Other) { + WrapperFunctionResult Tmp(std::move(Other)); + std::swap(R, Tmp.R); + return *this; + } + + ~WrapperFunctionResult() { + if ((R.Size > sizeof(R.Data.Value)) || + (R.Size == 0 && R.Data.ValuePtr != nullptr)) + free(R.Data.ValuePtr); + } + + /// Release ownership of the contained CWrapperFunctionResult. + /// Warning: Do not use -- this method will be removed in the future. It only + /// exists to temporarily support some code that will eventually be moved to + /// the ORC runtime. + CWrapperFunctionResult release() { + CWrapperFunctionResult Tmp; + init(Tmp); + std::swap(R, Tmp); + return Tmp; + } + + /// Get a pointer to the data contained in this instance. + char *data() { + assert((R.Size != 0 || R.Data.ValuePtr == nullptr) && + "Cannot get data for out-of-band error value"); + return R.Size > sizeof(R.Data.Value) ? R.Data.ValuePtr : R.Data.Value; + } + + /// Get a const pointer to the data contained in this instance. + const char *data() const { + assert((R.Size != 0 || R.Data.ValuePtr == nullptr) && + "Cannot get data for out-of-band error value"); + return R.Size > sizeof(R.Data.Value) ? R.Data.ValuePtr : R.Data.Value; + } + + /// Returns the size of the data contained in this instance. + size_t size() const { + assert((R.Size != 0 || R.Data.ValuePtr == nullptr) && + "Cannot get data for out-of-band error value"); + return R.Size; + } + + /// Returns true if this value is equivalent to a default-constructed + /// WrapperFunctionResult. + bool empty() const { return R.Size == 0 && R.Data.ValuePtr == nullptr; } + + /// Create a WrapperFunctionResult with the given size and return a pointer + /// to the underlying memory. + static WrapperFunctionResult allocate(size_t Size) { + // Reset. + WrapperFunctionResult WFR; + WFR.R.Size = Size; + if (WFR.R.Size > sizeof(WFR.R.Data.Value)) + WFR.R.Data.ValuePtr = (char *)malloc(WFR.R.Size); + return WFR; + } + + /// Copy from the given char range. + static WrapperFunctionResult copyFrom(const char *Source, size_t Size) { + auto WFR = allocate(Size); + memcpy(WFR.data(), Source, Size); + return WFR; + } + + /// Copy from the given null-terminated string (includes the null-terminator). + static WrapperFunctionResult copyFrom(const char *Source) { + return copyFrom(Source, strlen(Source) + 1); + } + + /// Copy from the given std::string (includes the null terminator). + static WrapperFunctionResult copyFrom(const std::string &Source) { + return copyFrom(Source.c_str()); + } + + /// Create an out-of-band error by copying the given string. + static WrapperFunctionResult createOutOfBandError(const char *Msg) { + // Reset. + WrapperFunctionResult WFR; + char *Tmp = (char *)malloc(strlen(Msg) + 1); + strcpy(Tmp, Msg); + WFR.R.Data.ValuePtr = Tmp; + return WFR; + } + + /// Create an out-of-band error by copying the given string. + static WrapperFunctionResult createOutOfBandError(const std::string &Msg) { + return createOutOfBandError(Msg.c_str()); + } + + /// If this value is an out-of-band error then this returns the error message, + /// otherwise returns nullptr. + const char *getOutOfBandError() const { + return R.Size == 0 ? R.Data.ValuePtr : nullptr; + } + +private: + static void init(CWrapperFunctionResult &R) { + R.Data.ValuePtr = nullptr; + R.Size = 0; + } + + CWrapperFunctionResult R; +}; + +namespace detail { + +template <typename SPSArgListT, typename... ArgTs> +WrapperFunctionResult +serializeViaSPSToWrapperFunctionResult(const ArgTs &...Args) { + auto Result = WrapperFunctionResult::allocate(SPSArgListT::size(Args...)); + SPSOutputBuffer OB(Result.data(), Result.size()); + if (!SPSArgListT::serialize(OB, Args...)) + return WrapperFunctionResult::createOutOfBandError( + "Error serializing arguments to blob in call"); + return Result; +} + +template <typename RetT> class WrapperFunctionHandlerCaller { +public: + template <typename HandlerT, typename ArgTupleT, std::size_t... I> + static decltype(auto) call(HandlerT &&H, ArgTupleT &Args, + std::index_sequence<I...>) { + return std::forward<HandlerT>(H)(std::get<I>(Args)...); + } +}; + +template <> class WrapperFunctionHandlerCaller<void> { +public: + template <typename HandlerT, typename ArgTupleT, std::size_t... I> + static SPSEmpty call(HandlerT &&H, ArgTupleT &Args, + std::index_sequence<I...>) { + std::forward<HandlerT>(H)(std::get<I>(Args)...); + return SPSEmpty(); + } +}; + +template <typename WrapperFunctionImplT, + template <typename> class ResultSerializer, typename... SPSTagTs> +class WrapperFunctionHandlerHelper + : public WrapperFunctionHandlerHelper< + decltype(&std::remove_reference_t<WrapperFunctionImplT>::operator()), + ResultSerializer, SPSTagTs...> {}; + +template <typename RetT, typename... ArgTs, + template <typename> class ResultSerializer, typename... SPSTagTs> +class WrapperFunctionHandlerHelper<RetT(ArgTs...), ResultSerializer, + SPSTagTs...> { +public: + using ArgTuple = std::tuple<std::decay_t<ArgTs>...>; + using ArgIndices = std::make_index_sequence<std::tuple_size<ArgTuple>::value>; + + template <typename HandlerT> + static WrapperFunctionResult apply(HandlerT &&H, const char *ArgData, + size_t ArgSize) { + ArgTuple Args; + if (!deserialize(ArgData, ArgSize, Args, ArgIndices{})) + return WrapperFunctionResult::createOutOfBandError( + "Could not deserialize arguments for wrapper function call"); + + auto HandlerResult = WrapperFunctionHandlerCaller<RetT>::call( + std::forward<HandlerT>(H), Args, ArgIndices{}); + + return ResultSerializer<decltype(HandlerResult)>::serialize( + std::move(HandlerResult)); + } + +private: + template <std::size_t... I> + static bool deserialize(const char *ArgData, size_t ArgSize, ArgTuple &Args, + std::index_sequence<I...>) { + SPSInputBuffer IB(ArgData, ArgSize); + return SPSArgList<SPSTagTs...>::deserialize(IB, std::get<I>(Args)...); + } +}; + +// Map function pointers to function types. +template <typename RetT, typename... ArgTs, + template <typename> class ResultSerializer, typename... SPSTagTs> +class WrapperFunctionHandlerHelper<RetT (*)(ArgTs...), ResultSerializer, + SPSTagTs...> + : public WrapperFunctionHandlerHelper<RetT(ArgTs...), ResultSerializer, + SPSTagTs...> {}; + +// Map non-const member function types to function types. +template <typename ClassT, typename RetT, typename... ArgTs, + template <typename> class ResultSerializer, typename... SPSTagTs> +class WrapperFunctionHandlerHelper<RetT (ClassT::*)(ArgTs...), ResultSerializer, + SPSTagTs...> + : public WrapperFunctionHandlerHelper<RetT(ArgTs...), ResultSerializer, + SPSTagTs...> {}; + +// Map const member function types to function types. +template <typename ClassT, typename RetT, typename... ArgTs, + template <typename> class ResultSerializer, typename... SPSTagTs> +class WrapperFunctionHandlerHelper<RetT (ClassT::*)(ArgTs...) const, + ResultSerializer, SPSTagTs...> + : public WrapperFunctionHandlerHelper<RetT(ArgTs...), ResultSerializer, + SPSTagTs...> {}; + +template <typename WrapperFunctionImplT, + template <typename> class ResultSerializer, typename... SPSTagTs> +class WrapperFunctionAsyncHandlerHelper + : public WrapperFunctionAsyncHandlerHelper< + decltype(&std::remove_reference_t<WrapperFunctionImplT>::operator()), + ResultSerializer, SPSTagTs...> {}; + +template <typename RetT, typename SendResultT, typename... ArgTs, + template <typename> class ResultSerializer, typename... SPSTagTs> +class WrapperFunctionAsyncHandlerHelper<RetT(SendResultT, ArgTs...), + ResultSerializer, SPSTagTs...> { +public: + using ArgTuple = std::tuple<std::decay_t<ArgTs>...>; + using ArgIndices = std::make_index_sequence<std::tuple_size<ArgTuple>::value>; + + template <typename HandlerT, typename SendWrapperFunctionResultT> + static void applyAsync(HandlerT &&H, + SendWrapperFunctionResultT &&SendWrapperFunctionResult, + const char *ArgData, size_t ArgSize) { + ArgTuple Args; + if (!deserialize(ArgData, ArgSize, Args, ArgIndices{})) { + SendWrapperFunctionResult(WrapperFunctionResult::createOutOfBandError( + "Could not deserialize arguments for wrapper function call")); + return; + } + + auto SendResult = + [SendWFR = std::move(SendWrapperFunctionResult)](auto Result) mutable { + using ResultT = decltype(Result); + SendWFR(ResultSerializer<ResultT>::serialize(std::move(Result))); + }; + + callAsync(std::forward<HandlerT>(H), std::move(SendResult), std::move(Args), + ArgIndices{}); + } + +private: + template <std::size_t... I> + static bool deserialize(const char *ArgData, size_t ArgSize, ArgTuple &Args, + std::index_sequence<I...>) { + SPSInputBuffer IB(ArgData, ArgSize); + return SPSArgList<SPSTagTs...>::deserialize(IB, std::get<I>(Args)...); + } + + template <typename HandlerT, typename SerializeAndSendResultT, + typename ArgTupleT, std::size_t... I> + static void callAsync(HandlerT &&H, + SerializeAndSendResultT &&SerializeAndSendResult, + ArgTupleT Args, std::index_sequence<I...>) { + (void)Args; // Silence a buggy GCC warning. + return std::forward<HandlerT>(H)(std::move(SerializeAndSendResult), + std::move(std::get<I>(Args))...); + } +}; + +// Map function pointers to function types. +template <typename RetT, typename... ArgTs, + template <typename> class ResultSerializer, typename... SPSTagTs> +class WrapperFunctionAsyncHandlerHelper<RetT (*)(ArgTs...), ResultSerializer, + SPSTagTs...> + : public WrapperFunctionAsyncHandlerHelper<RetT(ArgTs...), ResultSerializer, + SPSTagTs...> {}; + +// Map non-const member function types to function types. +template <typename ClassT, typename RetT, typename... ArgTs, + template <typename> class ResultSerializer, typename... SPSTagTs> +class WrapperFunctionAsyncHandlerHelper<RetT (ClassT::*)(ArgTs...), + ResultSerializer, SPSTagTs...> + : public WrapperFunctionAsyncHandlerHelper<RetT(ArgTs...), ResultSerializer, + SPSTagTs...> {}; + +// Map const member function types to function types. +template <typename ClassT, typename RetT, typename... ArgTs, + template <typename> class ResultSerializer, typename... SPSTagTs> +class WrapperFunctionAsyncHandlerHelper<RetT (ClassT::*)(ArgTs...) const, + ResultSerializer, SPSTagTs...> + : public WrapperFunctionAsyncHandlerHelper<RetT(ArgTs...), ResultSerializer, + SPSTagTs...> {}; + +template <typename SPSRetTagT, typename RetT> class ResultSerializer { +public: + static WrapperFunctionResult serialize(RetT Result) { + return serializeViaSPSToWrapperFunctionResult<SPSArgList<SPSRetTagT>>( + Result); + } +}; + +template <typename SPSRetTagT> class ResultSerializer<SPSRetTagT, Error> { +public: + static WrapperFunctionResult serialize(Error Err) { + return serializeViaSPSToWrapperFunctionResult<SPSArgList<SPSRetTagT>>( + toSPSSerializable(std::move(Err))); + } +}; + +template <typename SPSRetTagT> +class ResultSerializer<SPSRetTagT, ErrorSuccess> { +public: + static WrapperFunctionResult serialize(ErrorSuccess Err) { + return serializeViaSPSToWrapperFunctionResult<SPSArgList<SPSRetTagT>>( + toSPSSerializable(std::move(Err))); + } +}; + +template <typename SPSRetTagT, typename T> +class ResultSerializer<SPSRetTagT, Expected<T>> { +public: + static WrapperFunctionResult serialize(Expected<T> E) { + return serializeViaSPSToWrapperFunctionResult<SPSArgList<SPSRetTagT>>( + toSPSSerializable(std::move(E))); + } +}; + +template <typename SPSRetTagT, typename RetT> class ResultDeserializer { +public: + static RetT makeValue() { return RetT(); } + static void makeSafe(RetT &Result) {} + + static Error deserialize(RetT &Result, const char *ArgData, size_t ArgSize) { + SPSInputBuffer IB(ArgData, ArgSize); + if (!SPSArgList<SPSRetTagT>::deserialize(IB, Result)) + return make_error<StringError>( + "Error deserializing return value from blob in call", + inconvertibleErrorCode()); + return Error::success(); + } +}; + +template <> class ResultDeserializer<SPSError, Error> { +public: + static Error makeValue() { return Error::success(); } + static void makeSafe(Error &Err) { cantFail(std::move(Err)); } + + static Error deserialize(Error &Err, const char *ArgData, size_t ArgSize) { + SPSInputBuffer IB(ArgData, ArgSize); + SPSSerializableError BSE; + if (!SPSArgList<SPSError>::deserialize(IB, BSE)) + return make_error<StringError>( + "Error deserializing return value from blob in call", + inconvertibleErrorCode()); + Err = fromSPSSerializable(std::move(BSE)); + return Error::success(); + } +}; + +template <typename SPSTagT, typename T> +class ResultDeserializer<SPSExpected<SPSTagT>, Expected<T>> { +public: + static Expected<T> makeValue() { return T(); } + static void makeSafe(Expected<T> &E) { cantFail(E.takeError()); } + + static Error deserialize(Expected<T> &E, const char *ArgData, + size_t ArgSize) { + SPSInputBuffer IB(ArgData, ArgSize); + SPSSerializableExpected<T> BSE; + if (!SPSArgList<SPSExpected<SPSTagT>>::deserialize(IB, BSE)) + return make_error<StringError>( + "Error deserializing return value from blob in call", + inconvertibleErrorCode()); + E = fromSPSSerializable(std::move(BSE)); + return Error::success(); + } +}; + +template <typename SPSRetTagT, typename RetT> class AsyncCallResultHelper { + // Did you forget to use Error / Expected in your handler? +}; + +} // end namespace detail + +template <typename SPSSignature> class WrapperFunction; + +template <typename SPSRetTagT, typename... SPSTagTs> +class WrapperFunction<SPSRetTagT(SPSTagTs...)> { +private: + template <typename RetT> + using ResultSerializer = detail::ResultSerializer<SPSRetTagT, RetT>; + +public: + /// Call a wrapper function. Caller should be callable as + /// WrapperFunctionResult Fn(const char *ArgData, size_t ArgSize); + template <typename CallerFn, typename RetT, typename... ArgTs> + static Error call(const CallerFn &Caller, RetT &Result, + const ArgTs &...Args) { + + // RetT might be an Error or Expected value. Set the checked flag now: + // we don't want the user to have to check the unused result if this + // operation fails. + detail::ResultDeserializer<SPSRetTagT, RetT>::makeSafe(Result); + + auto ArgBuffer = + detail::serializeViaSPSToWrapperFunctionResult<SPSArgList<SPSTagTs...>>( + Args...); + if (const char *ErrMsg = ArgBuffer.getOutOfBandError()) + return make_error<StringError>(ErrMsg, inconvertibleErrorCode()); + + WrapperFunctionResult ResultBuffer = + Caller(ArgBuffer.data(), ArgBuffer.size()); + if (auto ErrMsg = ResultBuffer.getOutOfBandError()) + return make_error<StringError>(ErrMsg, inconvertibleErrorCode()); + + return detail::ResultDeserializer<SPSRetTagT, RetT>::deserialize( + Result, ResultBuffer.data(), ResultBuffer.size()); + } + + /// Call an async wrapper function. + /// Caller should be callable as + /// void Fn(unique_function<void(WrapperFunctionResult)> SendResult, + /// WrapperFunctionResult ArgBuffer); + template <typename AsyncCallerFn, typename SendDeserializedResultFn, + typename... ArgTs> + static void callAsync(AsyncCallerFn &&Caller, + SendDeserializedResultFn &&SendDeserializedResult, + const ArgTs &...Args) { + using RetT = typename std::tuple_element< + 1, typename detail::WrapperFunctionHandlerHelper< + std::remove_reference_t<SendDeserializedResultFn>, + ResultSerializer, SPSRetTagT>::ArgTuple>::type; + + auto ArgBuffer = + detail::serializeViaSPSToWrapperFunctionResult<SPSArgList<SPSTagTs...>>( + Args...); + if (auto *ErrMsg = ArgBuffer.getOutOfBandError()) { + SendDeserializedResult( + make_error<StringError>(ErrMsg, inconvertibleErrorCode()), + detail::ResultDeserializer<SPSRetTagT, RetT>::makeValue()); + return; + } + + auto SendSerializedResult = [SDR = std::move(SendDeserializedResult)]( + WrapperFunctionResult R) mutable { + RetT RetVal = detail::ResultDeserializer<SPSRetTagT, RetT>::makeValue(); + detail::ResultDeserializer<SPSRetTagT, RetT>::makeSafe(RetVal); + + if (auto *ErrMsg = R.getOutOfBandError()) { + SDR(make_error<StringError>(ErrMsg, inconvertibleErrorCode()), + std::move(RetVal)); + return; + } + + SPSInputBuffer IB(R.data(), R.size()); + if (auto Err = detail::ResultDeserializer<SPSRetTagT, RetT>::deserialize( + RetVal, R.data(), R.size())) + SDR(std::move(Err), std::move(RetVal)); + + SDR(Error::success(), std::move(RetVal)); + }; + + Caller(std::move(SendSerializedResult), ArgBuffer.data(), ArgBuffer.size()); + } + + /// Handle a call to a wrapper function. + template <typename HandlerT> + static WrapperFunctionResult handle(const char *ArgData, size_t ArgSize, + HandlerT &&Handler) { + using WFHH = + detail::WrapperFunctionHandlerHelper<std::remove_reference_t<HandlerT>, + ResultSerializer, SPSTagTs...>; + return WFHH::apply(std::forward<HandlerT>(Handler), ArgData, ArgSize); + } + + /// Handle a call to an async wrapper function. + template <typename HandlerT, typename SendResultT> + static void handleAsync(const char *ArgData, size_t ArgSize, + HandlerT &&Handler, SendResultT &&SendResult) { + using WFAHH = detail::WrapperFunctionAsyncHandlerHelper< + std::remove_reference_t<HandlerT>, ResultSerializer, SPSTagTs...>; + WFAHH::applyAsync(std::forward<HandlerT>(Handler), + std::forward<SendResultT>(SendResult), ArgData, ArgSize); + } + +private: + template <typename T> static const T &makeSerializable(const T &Value) { + return Value; + } + + static detail::SPSSerializableError makeSerializable(Error Err) { + return detail::toSPSSerializable(std::move(Err)); + } + + template <typename T> + static detail::SPSSerializableExpected<T> makeSerializable(Expected<T> E) { + return detail::toSPSSerializable(std::move(E)); + } +}; + +template <typename... SPSTagTs> +class WrapperFunction<void(SPSTagTs...)> + : private WrapperFunction<SPSEmpty(SPSTagTs...)> { + +public: + template <typename CallerFn, typename... ArgTs> + static Error call(const CallerFn &Caller, const ArgTs &...Args) { + SPSEmpty BE; + return WrapperFunction<SPSEmpty(SPSTagTs...)>::call(Caller, BE, Args...); + } + + template <typename AsyncCallerFn, typename SendDeserializedResultFn, + typename... ArgTs> + static void callAsync(AsyncCallerFn &&Caller, + SendDeserializedResultFn &&SendDeserializedResult, + const ArgTs &...Args) { + WrapperFunction<SPSEmpty(SPSTagTs...)>::callAsync( + std::forward<AsyncCallerFn>(Caller), + [SDR = std::move(SendDeserializedResult)](Error SerializeErr, + SPSEmpty E) mutable { + SDR(std::move(SerializeErr)); + }, + Args...); + } + + using WrapperFunction<SPSEmpty(SPSTagTs...)>::handle; + using WrapperFunction<SPSEmpty(SPSTagTs...)>::handleAsync; +}; + +/// A function object that takes an ExecutorAddr as its first argument, +/// casts that address to a ClassT*, then calls the given method on that +/// pointer passing in the remaining function arguments. This utility +/// removes some of the boilerplate from writing wrappers for method calls. +/// +/// @code{.cpp} +/// class MyClass { +/// public: +/// void myMethod(uint32_t, bool) { ... } +/// }; +/// +/// // SPS Method signature -- note MyClass object address as first argument. +/// using SPSMyMethodWrapperSignature = +/// SPSTuple<SPSExecutorAddr, uint32_t, bool>; +/// +/// WrapperFunctionResult +/// myMethodCallWrapper(const char *ArgData, size_t ArgSize) { +/// return WrapperFunction<SPSMyMethodWrapperSignature>::handle( +/// ArgData, ArgSize, makeMethodWrapperHandler(&MyClass::myMethod)); +/// } +/// @endcode +/// +template <typename RetT, typename ClassT, typename... ArgTs> +class MethodWrapperHandler { +public: + using MethodT = RetT (ClassT::*)(ArgTs...); + MethodWrapperHandler(MethodT M) : M(M) {} + RetT operator()(ExecutorAddr ObjAddr, ArgTs &...Args) { + return (ObjAddr.toPtr<ClassT*>()->*M)(std::forward<ArgTs>(Args)...); + } + +private: + MethodT M; +}; + +/// Create a MethodWrapperHandler object from the given method pointer. +template <typename RetT, typename ClassT, typename... ArgTs> +MethodWrapperHandler<RetT, ClassT, ArgTs...> +makeMethodWrapperHandler(RetT (ClassT::*Method)(ArgTs...)) { + return MethodWrapperHandler<RetT, ClassT, ArgTs...>(Method); +} + +/// Represents a serialized wrapper function call. +/// Serializing calls themselves allows us to batch them: We can make one +/// "run-wrapper-functions" utility and send it a list of calls to run. +/// +/// The motivating use-case for this API is JITLink allocation actions, where +/// we want to run multiple functions to finalize linked memory without having +/// to make separate IPC calls for each one. +class WrapperFunctionCall { +public: + using ArgDataBufferType = SmallVector<char, 24>; + + /// Create a WrapperFunctionCall using the given SPS serializer to serialize + /// the arguments. + template <typename SPSSerializer, typename... ArgTs> + static Expected<WrapperFunctionCall> Create(ExecutorAddr FnAddr, + const ArgTs &...Args) { + ArgDataBufferType ArgData; + ArgData.resize(SPSSerializer::size(Args...)); + SPSOutputBuffer OB(&ArgData[0], ArgData.size()); + if (SPSSerializer::serialize(OB, Args...)) + return WrapperFunctionCall(FnAddr, std::move(ArgData)); + return make_error<StringError>("Cannot serialize arguments for " + "AllocActionCall", + inconvertibleErrorCode()); + } + + WrapperFunctionCall() = default; + + /// Create a WrapperFunctionCall from a target function and arg buffer. + WrapperFunctionCall(ExecutorAddr FnAddr, ArgDataBufferType ArgData) + : FnAddr(FnAddr), ArgData(std::move(ArgData)) {} + + /// Returns the address to be called. + const ExecutorAddr &getCallee() const { return FnAddr; } + + /// Returns the argument data. + const ArgDataBufferType &getArgData() const { return ArgData; } + + /// WrapperFunctionCalls convert to true if the callee is non-null. + explicit operator bool() const { return !!FnAddr; } + + /// Run call returning raw WrapperFunctionResult. + shared::WrapperFunctionResult run() const { + using FnTy = + shared::CWrapperFunctionResult(const char *ArgData, size_t ArgSize); + return shared::WrapperFunctionResult( + FnAddr.toPtr<FnTy *>()(ArgData.data(), ArgData.size())); + } + + /// Run call and deserialize result using SPS. + template <typename SPSRetT, typename RetT> + std::enable_if_t<!std::is_same<SPSRetT, void>::value, Error> + runWithSPSRet(RetT &RetVal) const { + auto WFR = run(); + if (const char *ErrMsg = WFR.getOutOfBandError()) + return make_error<StringError>(ErrMsg, inconvertibleErrorCode()); + shared::SPSInputBuffer IB(WFR.data(), WFR.size()); + if (!shared::SPSSerializationTraits<SPSRetT, RetT>::deserialize(IB, RetVal)) + return make_error<StringError>("Could not deserialize result from " + "serialized wrapper function call", + inconvertibleErrorCode()); + return Error::success(); + } + + /// Overload for SPS functions returning void. + template <typename SPSRetT> + std::enable_if_t<std::is_same<SPSRetT, void>::value, Error> + runWithSPSRet() const { + shared::SPSEmpty E; + return runWithSPSRet<shared::SPSEmpty>(E); + } + + /// Run call and deserialize an SPSError result. SPSError returns and + /// deserialization failures are merged into the returned error. + Error runWithSPSRetErrorMerged() const { + detail::SPSSerializableError RetErr; + if (auto Err = runWithSPSRet<SPSError>(RetErr)) + return Err; + return detail::fromSPSSerializable(std::move(RetErr)); + } + +private: + orc::ExecutorAddr FnAddr; + ArgDataBufferType ArgData; +}; + +using SPSWrapperFunctionCall = SPSTuple<SPSExecutorAddr, SPSSequence<char>>; + +template <> +class SPSSerializationTraits<SPSWrapperFunctionCall, WrapperFunctionCall> { +public: + static size_t size(const WrapperFunctionCall &WFC) { + return SPSWrapperFunctionCall::AsArgList::size(WFC.getCallee(), + WFC.getArgData()); + } + + static bool serialize(SPSOutputBuffer &OB, const WrapperFunctionCall &WFC) { + return SPSWrapperFunctionCall::AsArgList::serialize(OB, WFC.getCallee(), + WFC.getArgData()); + } + + static bool deserialize(SPSInputBuffer &IB, WrapperFunctionCall &WFC) { + ExecutorAddr FnAddr; + WrapperFunctionCall::ArgDataBufferType ArgData; + if (!SPSWrapperFunctionCall::AsArgList::deserialize(IB, FnAddr, ArgData)) + return false; + WFC = WrapperFunctionCall(FnAddr, std::move(ArgData)); + return true; + } +}; + +} // end namespace shared +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_SHARED_WRAPPERFUNCTIONUTILS_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/SimpleRemoteEPC.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/SimpleRemoteEPC.h new file mode 100644 index 0000000000..be2ef600c7 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/SimpleRemoteEPC.h @@ -0,0 +1,151 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===---- SimpleRemoteEPC.h - Simple remote executor control ----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Simple remote executor process control. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_SIMPLEREMOTEEPC_H +#define LLVM_EXECUTIONENGINE_ORC_SIMPLEREMOTEEPC_H + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/FunctionExtras.h" +#include "llvm/ExecutionEngine/Orc/EPCGenericDylibManager.h" +#include "llvm/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.h" +#include "llvm/ExecutionEngine/Orc/EPCGenericMemoryAccess.h" +#include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h" +#include "llvm/ExecutionEngine/Orc/Shared/SimpleRemoteEPCUtils.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/MSVCErrorWorkarounds.h" + +#include <future> + +namespace llvm { +namespace orc { + +class SimpleRemoteEPC : public ExecutorProcessControl, + public SimpleRemoteEPCTransportClient { +public: + /// A setup object containing callbacks to construct a memory manager and + /// memory access object. Both are optional. If not specified, + /// EPCGenericJITLinkMemoryManager and EPCGenericMemoryAccess will be used. + struct Setup { + using CreateMemoryManagerFn = + Expected<std::unique_ptr<jitlink::JITLinkMemoryManager>>( + SimpleRemoteEPC &); + using CreateMemoryAccessFn = + Expected<std::unique_ptr<MemoryAccess>>(SimpleRemoteEPC &); + + unique_function<CreateMemoryManagerFn> CreateMemoryManager; + unique_function<CreateMemoryAccessFn> CreateMemoryAccess; + }; + + /// Create a SimpleRemoteEPC using the given transport type and args. + template <typename TransportT, typename... TransportTCtorArgTs> + static Expected<std::unique_ptr<SimpleRemoteEPC>> + Create(std::unique_ptr<TaskDispatcher> D, Setup S, + TransportTCtorArgTs &&...TransportTCtorArgs) { + std::unique_ptr<SimpleRemoteEPC> SREPC( + new SimpleRemoteEPC(std::make_shared<SymbolStringPool>(), + std::move(D))); + auto T = TransportT::Create( + *SREPC, std::forward<TransportTCtorArgTs>(TransportTCtorArgs)...); + if (!T) + return T.takeError(); + SREPC->T = std::move(*T); + if (auto Err = SREPC->setup(std::move(S))) + return joinErrors(std::move(Err), SREPC->disconnect()); + return std::move(SREPC); + } + + SimpleRemoteEPC(const SimpleRemoteEPC &) = delete; + SimpleRemoteEPC &operator=(const SimpleRemoteEPC &) = delete; + SimpleRemoteEPC(SimpleRemoteEPC &&) = delete; + SimpleRemoteEPC &operator=(SimpleRemoteEPC &&) = delete; + ~SimpleRemoteEPC(); + + Expected<tpctypes::DylibHandle> loadDylib(const char *DylibPath) override; + + Expected<std::vector<tpctypes::LookupResult>> + lookupSymbols(ArrayRef<LookupRequest> Request) override; + + Expected<int32_t> runAsMain(ExecutorAddr MainFnAddr, + ArrayRef<std::string> Args) override; + + void callWrapperAsync(ExecutorAddr WrapperFnAddr, + IncomingWFRHandler OnComplete, + ArrayRef<char> ArgBuffer) override; + + Error disconnect() override; + + Expected<HandleMessageAction> + handleMessage(SimpleRemoteEPCOpcode OpC, uint64_t SeqNo, ExecutorAddr TagAddr, + SimpleRemoteEPCArgBytesVector ArgBytes) override; + + void handleDisconnect(Error Err) override; + +private: + SimpleRemoteEPC(std::shared_ptr<SymbolStringPool> SSP, + std::unique_ptr<TaskDispatcher> D) + : ExecutorProcessControl(std::move(SSP), std::move(D)) {} + + static Expected<std::unique_ptr<jitlink::JITLinkMemoryManager>> + createDefaultMemoryManager(SimpleRemoteEPC &SREPC); + static Expected<std::unique_ptr<MemoryAccess>> + createDefaultMemoryAccess(SimpleRemoteEPC &SREPC); + + Error sendMessage(SimpleRemoteEPCOpcode OpC, uint64_t SeqNo, + ExecutorAddr TagAddr, ArrayRef<char> ArgBytes); + + Error handleSetup(uint64_t SeqNo, ExecutorAddr TagAddr, + SimpleRemoteEPCArgBytesVector ArgBytes); + Error setup(Setup S); + + Error handleResult(uint64_t SeqNo, ExecutorAddr TagAddr, + SimpleRemoteEPCArgBytesVector ArgBytes); + void handleCallWrapper(uint64_t RemoteSeqNo, ExecutorAddr TagAddr, + SimpleRemoteEPCArgBytesVector ArgBytes); + Error handleHangup(SimpleRemoteEPCArgBytesVector ArgBytes); + + uint64_t getNextSeqNo() { return NextSeqNo++; } + void releaseSeqNo(uint64_t SeqNo) {} + + using PendingCallWrapperResultsMap = + DenseMap<uint64_t, IncomingWFRHandler>; + + std::mutex SimpleRemoteEPCMutex; + std::condition_variable DisconnectCV; + bool Disconnected = false; + Error DisconnectErr = Error::success(); + + std::unique_ptr<SimpleRemoteEPCTransport> T; + std::unique_ptr<jitlink::JITLinkMemoryManager> OwnedMemMgr; + std::unique_ptr<MemoryAccess> OwnedMemAccess; + + std::unique_ptr<EPCGenericDylibManager> DylibMgr; + ExecutorAddr RunAsMainAddr; + + uint64_t NextSeqNo = 0; + PendingCallWrapperResultsMap PendingCallWrapperResults; +}; + +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_SIMPLEREMOTEEPC_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/SpeculateAnalyses.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/SpeculateAnalyses.h new file mode 100644 index 0000000000..32080a61ce --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/SpeculateAnalyses.h @@ -0,0 +1,95 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===-- SpeculateAnalyses.h --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// \file +/// Contains the Analyses and Result Interpretation to select likely functions +/// to Speculatively compile before they are called. [Purely Experimentation] +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_SPECULATEANALYSES_H +#define LLVM_EXECUTIONENGINE_ORC_SPECULATEANALYSES_H + +#include "llvm/Analysis/BranchProbabilityInfo.h" +#include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/Speculation.h" + +#include <vector> + +namespace llvm { + +namespace orc { + +// Provides common code. +class SpeculateQuery { +protected: + void findCalles(const BasicBlock *, DenseSet<StringRef> &); + bool isStraightLine(const Function &F); + +public: + using ResultTy = Optional<DenseMap<StringRef, DenseSet<StringRef>>>; +}; + +// Direct calls in high frequency basic blocks are extracted. +class BlockFreqQuery : public SpeculateQuery { + size_t numBBToGet(size_t); + +public: + // Find likely next executables based on IR Block Frequency + ResultTy operator()(Function &F); +}; + +// This Query generates a sequence of basic blocks which follows the order of +// execution. +// A handful of BB with higher block frequencies are taken, then path to entry +// and end BB are discovered by traversing up & down the CFG. +class SequenceBBQuery : public SpeculateQuery { + struct WalkDirection { + bool Upward = true, Downward = true; + // the block associated contain a call + bool CallerBlock = false; + }; + +public: + using VisitedBlocksInfoTy = DenseMap<const BasicBlock *, WalkDirection>; + using BlockListTy = SmallVector<const BasicBlock *, 8>; + using BackEdgesInfoTy = + SmallVector<std::pair<const BasicBlock *, const BasicBlock *>, 8>; + using BlockFreqInfoTy = + SmallVector<std::pair<const BasicBlock *, uint64_t>, 8>; + +private: + std::size_t getHottestBlocks(std::size_t TotalBlocks); + BlockListTy rearrangeBB(const Function &, const BlockListTy &); + BlockListTy queryCFG(Function &, const BlockListTy &); + void traverseToEntryBlock(const BasicBlock *, const BlockListTy &, + const BackEdgesInfoTy &, + const BranchProbabilityInfo *, + VisitedBlocksInfoTy &); + void traverseToExitBlock(const BasicBlock *, const BlockListTy &, + const BackEdgesInfoTy &, + const BranchProbabilityInfo *, + VisitedBlocksInfoTy &); + +public: + ResultTy operator()(Function &F); +}; + +} // namespace orc +} // namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_SPECULATEANALYSES_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/Speculation.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/Speculation.h new file mode 100644 index 0000000000..f4193ff075 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/Speculation.h @@ -0,0 +1,221 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===-- Speculation.h - Speculative Compilation --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Contains the definition to support speculative compilation when laziness is +// enabled. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_SPECULATION_H +#define LLVM_EXECUTIONENGINE_ORC_SPECULATION_H + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/DebugUtils.h" +#include "llvm/ExecutionEngine/Orc/IRCompileLayer.h" +#include "llvm/Support/Debug.h" +#include <mutex> +#include <type_traits> +#include <utility> + +namespace llvm { +namespace orc { + +class Speculator; + +// Track the Impls (JITDylib,Symbols) of Symbols while lazy call through +// trampolines are created. Operations are guarded by locks tp ensure that Imap +// stays in consistent state after read/write + +class ImplSymbolMap { + friend class Speculator; + +public: + using AliaseeDetails = std::pair<SymbolStringPtr, JITDylib *>; + using Alias = SymbolStringPtr; + using ImapTy = DenseMap<Alias, AliaseeDetails>; + void trackImpls(SymbolAliasMap ImplMaps, JITDylib *SrcJD); + +private: + // FIX ME: find a right way to distinguish the pre-compile Symbols, and update + // the callsite + Optional<AliaseeDetails> getImplFor(const SymbolStringPtr &StubSymbol) { + std::lock_guard<std::mutex> Lockit(ConcurrentAccess); + auto Position = Maps.find(StubSymbol); + if (Position != Maps.end()) + return Position->getSecond(); + else + return None; + } + + std::mutex ConcurrentAccess; + ImapTy Maps; +}; + +// Defines Speculator Concept, +class Speculator { +public: + using TargetFAddr = JITTargetAddress; + using FunctionCandidatesMap = DenseMap<SymbolStringPtr, SymbolNameSet>; + using StubAddrLikelies = DenseMap<TargetFAddr, SymbolNameSet>; + +private: + void registerSymbolsWithAddr(TargetFAddr ImplAddr, + SymbolNameSet likelySymbols) { + std::lock_guard<std::mutex> Lockit(ConcurrentAccess); + GlobalSpecMap.insert({ImplAddr, std::move(likelySymbols)}); + } + + void launchCompile(JITTargetAddress FAddr) { + SymbolNameSet CandidateSet; + // Copy CandidateSet is necessary, to avoid unsynchronized access to + // the datastructure. + { + std::lock_guard<std::mutex> Lockit(ConcurrentAccess); + auto It = GlobalSpecMap.find(FAddr); + if (It == GlobalSpecMap.end()) + return; + CandidateSet = It->getSecond(); + } + + SymbolDependenceMap SpeculativeLookUpImpls; + + for (auto &Callee : CandidateSet) { + auto ImplSymbol = AliaseeImplTable.getImplFor(Callee); + // try to distinguish already compiled & library symbols + if (!ImplSymbol.hasValue()) + continue; + const auto &ImplSymbolName = ImplSymbol.getPointer()->first; + JITDylib *ImplJD = ImplSymbol.getPointer()->second; + auto &SymbolsInJD = SpeculativeLookUpImpls[ImplJD]; + SymbolsInJD.insert(ImplSymbolName); + } + + DEBUG_WITH_TYPE("orc", { + for (auto &I : SpeculativeLookUpImpls) { + llvm::dbgs() << "\n In " << I.first->getName() << " JITDylib "; + for (auto &N : I.second) + llvm::dbgs() << "\n Likely Symbol : " << N; + } + }); + + // for a given symbol, there may be no symbol qualified for speculatively + // compile try to fix this before jumping to this code if possible. + for (auto &LookupPair : SpeculativeLookUpImpls) + ES.lookup( + LookupKind::Static, + makeJITDylibSearchOrder(LookupPair.first, + JITDylibLookupFlags::MatchAllSymbols), + SymbolLookupSet(LookupPair.second), SymbolState::Ready, + [this](Expected<SymbolMap> Result) { + if (auto Err = Result.takeError()) + ES.reportError(std::move(Err)); + }, + NoDependenciesToRegister); + } + +public: + Speculator(ImplSymbolMap &Impl, ExecutionSession &ref) + : AliaseeImplTable(Impl), ES(ref), GlobalSpecMap(0) {} + Speculator(const Speculator &) = delete; + Speculator(Speculator &&) = delete; + Speculator &operator=(const Speculator &) = delete; + Speculator &operator=(Speculator &&) = delete; + + /// Define symbols for this Speculator object (__orc_speculator) and the + /// speculation runtime entry point symbol (__orc_speculate_for) in the + /// given JITDylib. + Error addSpeculationRuntime(JITDylib &JD, MangleAndInterner &Mangle); + + // Speculatively compile likely functions for the given Stub Address. + // destination of __orc_speculate_for jump + void speculateFor(TargetFAddr StubAddr) { launchCompile(StubAddr); } + + // FIXME : Register with Stub Address, after JITLink Fix. + void registerSymbols(FunctionCandidatesMap Candidates, JITDylib *JD) { + for (auto &SymPair : Candidates) { + auto Target = SymPair.first; + auto Likely = SymPair.second; + + auto OnReadyFixUp = [Likely, Target, + this](Expected<SymbolMap> ReadySymbol) { + if (ReadySymbol) { + auto RAddr = (*ReadySymbol)[Target].getAddress(); + registerSymbolsWithAddr(RAddr, std::move(Likely)); + } else + this->getES().reportError(ReadySymbol.takeError()); + }; + // Include non-exported symbols also. + ES.lookup( + LookupKind::Static, + makeJITDylibSearchOrder(JD, JITDylibLookupFlags::MatchAllSymbols), + SymbolLookupSet(Target, SymbolLookupFlags::WeaklyReferencedSymbol), + SymbolState::Ready, OnReadyFixUp, NoDependenciesToRegister); + } + } + + ExecutionSession &getES() { return ES; } + +private: + static void speculateForEntryPoint(Speculator *Ptr, uint64_t StubId); + std::mutex ConcurrentAccess; + ImplSymbolMap &AliaseeImplTable; + ExecutionSession &ES; + StubAddrLikelies GlobalSpecMap; +}; + +class IRSpeculationLayer : public IRLayer { +public: + using IRlikiesStrRef = Optional<DenseMap<StringRef, DenseSet<StringRef>>>; + using ResultEval = std::function<IRlikiesStrRef(Function &)>; + using TargetAndLikelies = DenseMap<SymbolStringPtr, SymbolNameSet>; + + IRSpeculationLayer(ExecutionSession &ES, IRCompileLayer &BaseLayer, + Speculator &Spec, MangleAndInterner &Mangle, + ResultEval Interpreter) + : IRLayer(ES, BaseLayer.getManglingOptions()), NextLayer(BaseLayer), + S(Spec), Mangle(Mangle), QueryAnalysis(Interpreter) {} + + void emit(std::unique_ptr<MaterializationResponsibility> R, + ThreadSafeModule TSM) override; + +private: + TargetAndLikelies + internToJITSymbols(DenseMap<StringRef, DenseSet<StringRef>> IRNames) { + assert(!IRNames.empty() && "No IRNames received to Intern?"); + TargetAndLikelies InternedNames; + for (auto &NamePair : IRNames) { + DenseSet<SymbolStringPtr> TargetJITNames; + for (auto &TargetNames : NamePair.second) + TargetJITNames.insert(Mangle(TargetNames)); + InternedNames[Mangle(NamePair.first)] = std::move(TargetJITNames); + } + return InternedNames; + } + + IRCompileLayer &NextLayer; + Speculator &S; + MangleAndInterner &Mangle; + ResultEval QueryAnalysis; +}; + +} // namespace orc +} // namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_SPECULATION_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h new file mode 100644 index 0000000000..01ca026fda --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h @@ -0,0 +1,219 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===- SymbolStringPool.h - Multi-threaded pool for JIT symbols -*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Contains a multi-threaded string pool suitable for use with ORC. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_SYMBOLSTRINGPOOL_H +#define LLVM_EXECUTIONENGINE_ORC_SYMBOLSTRINGPOOL_H + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/StringMap.h" +#include <atomic> +#include <mutex> + +namespace llvm { +namespace orc { + +class SymbolStringPtr; + +/// String pool for symbol names used by the JIT. +class SymbolStringPool { + friend class SymbolStringPtr; +public: + /// Destroy a SymbolStringPool. + ~SymbolStringPool(); + + /// Create a symbol string pointer from the given string. + SymbolStringPtr intern(StringRef S); + + /// Remove from the pool any entries that are no longer referenced. + void clearDeadEntries(); + + /// Returns true if the pool is empty. + bool empty() const; +private: + using RefCountType = std::atomic<size_t>; + using PoolMap = StringMap<RefCountType>; + using PoolMapEntry = StringMapEntry<RefCountType>; + mutable std::mutex PoolMutex; + PoolMap Pool; +}; + +/// Pointer to a pooled string representing a symbol name. +class SymbolStringPtr { + friend class OrcV2CAPIHelper; + friend class SymbolStringPool; + friend struct DenseMapInfo<SymbolStringPtr>; + +public: + SymbolStringPtr() = default; + SymbolStringPtr(std::nullptr_t) {} + SymbolStringPtr(const SymbolStringPtr &Other) + : S(Other.S) { + if (isRealPoolEntry(S)) + ++S->getValue(); + } + + SymbolStringPtr& operator=(const SymbolStringPtr &Other) { + if (isRealPoolEntry(S)) { + assert(S->getValue() && "Releasing SymbolStringPtr with zero ref count"); + --S->getValue(); + } + S = Other.S; + if (isRealPoolEntry(S)) + ++S->getValue(); + return *this; + } + + SymbolStringPtr(SymbolStringPtr &&Other) : S(nullptr) { + std::swap(S, Other.S); + } + + SymbolStringPtr& operator=(SymbolStringPtr &&Other) { + if (isRealPoolEntry(S)) { + assert(S->getValue() && "Releasing SymbolStringPtr with zero ref count"); + --S->getValue(); + } + S = nullptr; + std::swap(S, Other.S); + return *this; + } + + ~SymbolStringPtr() { + if (isRealPoolEntry(S)) { + assert(S->getValue() && "Releasing SymbolStringPtr with zero ref count"); + --S->getValue(); + } + } + + explicit operator bool() const { return S; } + + StringRef operator*() const { return S->first(); } + + friend bool operator==(const SymbolStringPtr &LHS, + const SymbolStringPtr &RHS) { + return LHS.S == RHS.S; + } + + friend bool operator!=(const SymbolStringPtr &LHS, + const SymbolStringPtr &RHS) { + return !(LHS == RHS); + } + + friend bool operator<(const SymbolStringPtr &LHS, + const SymbolStringPtr &RHS) { + return LHS.S < RHS.S; + } + +private: + using PoolEntry = SymbolStringPool::PoolMapEntry; + using PoolEntryPtr = PoolEntry *; + + SymbolStringPtr(SymbolStringPool::PoolMapEntry *S) + : S(S) { + if (isRealPoolEntry(S)) + ++S->getValue(); + } + + // Returns false for null, empty, and tombstone values, true otherwise. + bool isRealPoolEntry(PoolEntryPtr P) { + return ((reinterpret_cast<uintptr_t>(P) - 1) & InvalidPtrMask) != + InvalidPtrMask; + } + + static SymbolStringPtr getEmptyVal() { + return SymbolStringPtr(reinterpret_cast<PoolEntryPtr>(EmptyBitPattern)); + } + + static SymbolStringPtr getTombstoneVal() { + return SymbolStringPtr(reinterpret_cast<PoolEntryPtr>(TombstoneBitPattern)); + } + + constexpr static uintptr_t EmptyBitPattern = + std::numeric_limits<uintptr_t>::max() + << PointerLikeTypeTraits<PoolEntryPtr>::NumLowBitsAvailable; + + constexpr static uintptr_t TombstoneBitPattern = + (std::numeric_limits<uintptr_t>::max() - 1) + << PointerLikeTypeTraits<PoolEntryPtr>::NumLowBitsAvailable; + + constexpr static uintptr_t InvalidPtrMask = + (std::numeric_limits<uintptr_t>::max() - 3) + << PointerLikeTypeTraits<PoolEntryPtr>::NumLowBitsAvailable; + + PoolEntryPtr S = nullptr; +}; + +inline SymbolStringPool::~SymbolStringPool() { +#ifndef NDEBUG + clearDeadEntries(); + assert(Pool.empty() && "Dangling references at pool destruction time"); +#endif // NDEBUG +} + +inline SymbolStringPtr SymbolStringPool::intern(StringRef S) { + std::lock_guard<std::mutex> Lock(PoolMutex); + PoolMap::iterator I; + bool Added; + std::tie(I, Added) = Pool.try_emplace(S, 0); + return SymbolStringPtr(&*I); +} + +inline void SymbolStringPool::clearDeadEntries() { + std::lock_guard<std::mutex> Lock(PoolMutex); + for (auto I = Pool.begin(), E = Pool.end(); I != E;) { + auto Tmp = I++; + if (Tmp->second == 0) + Pool.erase(Tmp); + } +} + +inline bool SymbolStringPool::empty() const { + std::lock_guard<std::mutex> Lock(PoolMutex); + return Pool.empty(); +} + +} // end namespace orc + +template <> +struct DenseMapInfo<orc::SymbolStringPtr> { + + static orc::SymbolStringPtr getEmptyKey() { + return orc::SymbolStringPtr::getEmptyVal(); + } + + static orc::SymbolStringPtr getTombstoneKey() { + return orc::SymbolStringPtr::getTombstoneVal(); + } + + static unsigned getHashValue(const orc::SymbolStringPtr &V) { + return DenseMapInfo<orc::SymbolStringPtr::PoolEntryPtr>::getHashValue(V.S); + } + + static bool isEqual(const orc::SymbolStringPtr &LHS, + const orc::SymbolStringPtr &RHS) { + return LHS.S == RHS.S; + } +}; + +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_SYMBOLSTRINGPOOL_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/TargetProcess/ExecutorBootstrapService.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/TargetProcess/ExecutorBootstrapService.h new file mode 100644 index 0000000000..56e269d949 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/TargetProcess/ExecutorBootstrapService.h @@ -0,0 +1,47 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===- ExecutorService.h - Provide bootstrap symbols to session -*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Provides a service by supplying some set of bootstrap symbols. +// +// FIXME: The functionality in this file should be moved to the ORC runtime. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_EXECUTORBOOTSTRAPSERVICE_H +#define LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_EXECUTORBOOTSTRAPSERVICE_H + +#include "llvm/ADT/StringMap.h" +#include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h" + +namespace llvm { +namespace orc { + +class ExecutorBootstrapService { +public: + virtual ~ExecutorBootstrapService(); + + virtual void + addBootstrapSymbols(StringMap<ExecutorAddr> &BootstrapSymbols) = 0; + virtual Error shutdown() = 0; +}; + +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_EXECUTORBOOTSTRAPSERVICE_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.h new file mode 100644 index 0000000000..f1f5a8e4e3 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.h @@ -0,0 +1,33 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===- JITLoaderGDB.h - Register objects via GDB JIT interface -*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Register objects for access by debuggers via the GDB JIT interface. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_JITLOADERGDB_H +#define LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_JITLOADERGDB_H + +#include "llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h" +#include <cstdint> + +extern "C" llvm::orc::shared::CWrapperFunctionResult +llvm_orc_registerJITLoaderGDBWrapper(const char *Data, uint64_t Size); + +#endif // LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_JITLOADERGDB_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h new file mode 100644 index 0000000000..19a1cc985c --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h @@ -0,0 +1,53 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===----- RegisterEHFrames.h -- Register EH frame sections -----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Support for dynamically registering and deregistering eh-frame sections +// in-process via libunwind. +// +// FIXME: The functionality in this file should be moved to the ORC runtime. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_REGISTEREHFRAMES_H +#define LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_REGISTEREHFRAMES_H + +#include "llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h" +#include "llvm/Support/Error.h" + +namespace llvm { +namespace orc { + +/// Register frames in the given eh-frame section with libunwind. +Error registerEHFrameSection(const void *EHFrameSectionAddr, + size_t EHFrameSectionSize); + +/// Unregister frames in the given eh-frame section with libunwind. +Error deregisterEHFrameSection(const void *EHFrameSectionAddr, + size_t EHFrameSectionSize); + +} // end namespace orc +} // end namespace llvm + +extern "C" llvm::orc::shared::CWrapperFunctionResult +llvm_orc_registerEHFrameSectionWrapper(const char *Data, uint64_t Size); + +extern "C" llvm::orc::shared::CWrapperFunctionResult +llvm_orc_deregisterEHFrameSectionWrapper(const char *Data, uint64_t Size); + +#endif // LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_REGISTEREHFRAMES_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorDylibManager.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorDylibManager.h new file mode 100644 index 0000000000..8ea528833c --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorDylibManager.h @@ -0,0 +1,75 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===--------------- SimpleExecutorDylibManager.h ---------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// A simple dynamic library management class. Allows dynamic libraries to be +// loaded and searched. +// +// FIXME: The functionality in this file should be moved to the ORC runtime. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_SIMPLEEXECUTORDYLIBMANAGER_H +#define LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_SIMPLEEXECUTORDYLIBMANAGER_H + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h" +#include "llvm/ExecutionEngine/Orc/Shared/SimpleRemoteEPCUtils.h" +#include "llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h" +#include "llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h" +#include "llvm/ExecutionEngine/Orc/TargetProcess/ExecutorBootstrapService.h" +#include "llvm/Support/DynamicLibrary.h" +#include "llvm/Support/Error.h" + +#include <mutex> + +namespace llvm { +namespace orc { +namespace rt_bootstrap { + +/// Simple page-based allocator. +class SimpleExecutorDylibManager : public ExecutorBootstrapService { +public: + virtual ~SimpleExecutorDylibManager(); + + Expected<tpctypes::DylibHandle> open(const std::string &Path, uint64_t Mode); + Expected<std::vector<ExecutorAddr>> lookup(tpctypes::DylibHandle H, + const RemoteSymbolLookupSet &L); + + Error shutdown() override; + void addBootstrapSymbols(StringMap<ExecutorAddr> &M) override; + +private: + using DylibsMap = DenseMap<uint64_t, sys::DynamicLibrary>; + + static llvm::orc::shared::CWrapperFunctionResult + openWrapper(const char *ArgData, size_t ArgSize); + + static llvm::orc::shared::CWrapperFunctionResult + lookupWrapper(const char *ArgData, size_t ArgSize); + + std::mutex M; + uint64_t NextId = 0; + DylibsMap Dylibs; +}; + +} // end namespace rt_bootstrap +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_SIMPLEEXECUTORDYLIBMANAGER_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.h new file mode 100644 index 0000000000..1c40338cc9 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.h @@ -0,0 +1,81 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===---------------- SimpleExecutorMemoryManager.h -------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// A simple allocator class suitable for basic remote-JIT use. +// +// FIXME: The functionality in this file should be moved to the ORC runtime. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_SIMPLEEXECUTORMEMORYMANAGER_H +#define LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_SIMPLEEXECUTORMEMORYMANAGER_H + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h" +#include "llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h" +#include "llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h" +#include "llvm/ExecutionEngine/Orc/TargetProcess/ExecutorBootstrapService.h" +#include "llvm/Support/Error.h" + +#include <mutex> + +namespace llvm { +namespace orc { +namespace rt_bootstrap { + +/// Simple page-based allocator. +class SimpleExecutorMemoryManager : public ExecutorBootstrapService { +public: + virtual ~SimpleExecutorMemoryManager(); + + Expected<ExecutorAddr> allocate(uint64_t Size); + Error finalize(tpctypes::FinalizeRequest &FR); + Error deallocate(const std::vector<ExecutorAddr> &Bases); + + Error shutdown() override; + void addBootstrapSymbols(StringMap<ExecutorAddr> &M) override; + +private: + struct Allocation { + size_t Size = 0; + std::vector<shared::WrapperFunctionCall> DeallocationActions; + }; + + using AllocationsMap = DenseMap<void *, Allocation>; + + Error deallocateImpl(void *Base, Allocation &A); + + static llvm::orc::shared::CWrapperFunctionResult + reserveWrapper(const char *ArgData, size_t ArgSize); + + static llvm::orc::shared::CWrapperFunctionResult + finalizeWrapper(const char *ArgData, size_t ArgSize); + + static llvm::orc::shared::CWrapperFunctionResult + deallocateWrapper(const char *ArgData, size_t ArgSize); + + std::mutex M; + AllocationsMap Allocations; +}; + +} // end namespace rt_bootstrap +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_SIMPLEEXECUTORMEMORYMANAGER_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleRemoteEPCServer.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleRemoteEPCServer.h new file mode 100644 index 0000000000..73f5fb58c9 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleRemoteEPCServer.h @@ -0,0 +1,193 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===---- SimpleRemoteEPCServer.h - EPC over abstract channel ---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// EPC over simple abstract channel. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_SIMPLEREMOTEEPCSERVER_H +#define LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_SIMPLEREMOTEEPCSERVER_H + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/FunctionExtras.h" +#include "llvm/Config/llvm-config.h" +#include "llvm/ExecutionEngine/Orc/Shared/SimpleRemoteEPCUtils.h" +#include "llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h" +#include "llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h" +#include "llvm/ExecutionEngine/Orc/TargetProcess/ExecutorBootstrapService.h" +#include "llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorDylibManager.h" +#include "llvm/Support/DynamicLibrary.h" +#include "llvm/Support/Error.h" + +#include <condition_variable> +#include <future> +#include <memory> +#include <mutex> + +namespace llvm { +namespace orc { + +/// A simple EPC server implementation. +class SimpleRemoteEPCServer : public SimpleRemoteEPCTransportClient { +public: + using ReportErrorFunction = unique_function<void(Error)>; + + /// Dispatches calls to runWrapper. + class Dispatcher { + public: + virtual ~Dispatcher(); + virtual void dispatch(unique_function<void()> Work) = 0; + virtual void shutdown() = 0; + }; + +#if LLVM_ENABLE_THREADS + class ThreadDispatcher : public Dispatcher { + public: + void dispatch(unique_function<void()> Work) override; + void shutdown() override; + + private: + std::mutex DispatchMutex; + bool Running = true; + size_t Outstanding = 0; + std::condition_variable OutstandingCV; + }; +#endif + + class Setup { + friend class SimpleRemoteEPCServer; + + public: + SimpleRemoteEPCServer &server() { return S; } + StringMap<ExecutorAddr> &bootstrapSymbols() { return BootstrapSymbols; } + std::vector<std::unique_ptr<ExecutorBootstrapService>> &services() { + return Services; + } + void setDispatcher(std::unique_ptr<Dispatcher> D) { S.D = std::move(D); } + void setErrorReporter(unique_function<void(Error)> ReportError) { + S.ReportError = std::move(ReportError); + } + + private: + Setup(SimpleRemoteEPCServer &S) : S(S) {} + SimpleRemoteEPCServer &S; + StringMap<ExecutorAddr> BootstrapSymbols; + std::vector<std::unique_ptr<ExecutorBootstrapService>> Services; + }; + + static StringMap<ExecutorAddr> defaultBootstrapSymbols(); + + template <typename TransportT, typename... TransportTCtorArgTs> + static Expected<std::unique_ptr<SimpleRemoteEPCServer>> + Create(unique_function<Error(Setup &S)> SetupFunction, + TransportTCtorArgTs &&...TransportTCtorArgs) { + auto Server = std::make_unique<SimpleRemoteEPCServer>(); + Setup S(*Server); + if (auto Err = SetupFunction(S)) + return std::move(Err); + + // Set ReportError up-front so that it can be used if construction + // process fails. + if (!Server->ReportError) + Server->ReportError = [](Error Err) { + logAllUnhandledErrors(std::move(Err), errs(), "SimpleRemoteEPCServer "); + }; + + // Attempt to create transport. + auto T = TransportT::Create( + *Server, std::forward<TransportTCtorArgTs>(TransportTCtorArgs)...); + if (!T) + return T.takeError(); + Server->T = std::move(*T); + if (auto Err = Server->T->start()) + return std::move(Err); + + // If transport creation succeeds then start up services. + Server->Services = std::move(S.services()); + Server->Services.push_back( + std::make_unique<rt_bootstrap::SimpleExecutorDylibManager>()); + for (auto &Service : Server->Services) + Service->addBootstrapSymbols(S.bootstrapSymbols()); + + if (auto Err = Server->sendSetupMessage(std::move(S.BootstrapSymbols))) + return std::move(Err); + return std::move(Server); + } + + /// Set an error reporter for this server. + void setErrorReporter(ReportErrorFunction ReportError) { + this->ReportError = std::move(ReportError); + } + + /// Call to handle an incoming message. + /// + /// Returns 'Disconnect' if the message is a 'detach' message from the remote + /// otherwise returns 'Continue'. If the server has moved to an error state, + /// returns an error, which should be reported and treated as a 'Disconnect'. + Expected<HandleMessageAction> + handleMessage(SimpleRemoteEPCOpcode OpC, uint64_t SeqNo, ExecutorAddr TagAddr, + SimpleRemoteEPCArgBytesVector ArgBytes) override; + + Error waitForDisconnect(); + + void handleDisconnect(Error Err) override; + +private: + Error sendMessage(SimpleRemoteEPCOpcode OpC, uint64_t SeqNo, + ExecutorAddr TagAddr, ArrayRef<char> ArgBytes); + + Error sendSetupMessage(StringMap<ExecutorAddr> BootstrapSymbols); + + Error handleResult(uint64_t SeqNo, ExecutorAddr TagAddr, + SimpleRemoteEPCArgBytesVector ArgBytes); + void handleCallWrapper(uint64_t RemoteSeqNo, ExecutorAddr TagAddr, + SimpleRemoteEPCArgBytesVector ArgBytes); + + shared::WrapperFunctionResult + doJITDispatch(const void *FnTag, const char *ArgData, size_t ArgSize); + + static shared::CWrapperFunctionResult jitDispatchEntry(void *DispatchCtx, + const void *FnTag, + const char *ArgData, + size_t ArgSize); + + uint64_t getNextSeqNo() { return NextSeqNo++; } + void releaseSeqNo(uint64_t) {} + + using PendingJITDispatchResultsMap = + DenseMap<uint64_t, std::promise<shared::WrapperFunctionResult> *>; + + std::mutex ServerStateMutex; + std::condition_variable ShutdownCV; + enum { ServerRunning, ServerShuttingDown, ServerShutDown } RunState; + Error ShutdownErr = Error::success(); + std::unique_ptr<SimpleRemoteEPCTransport> T; + std::unique_ptr<Dispatcher> D; + std::vector<std::unique_ptr<ExecutorBootstrapService>> Services; + ReportErrorFunction ReportError; + + uint64_t NextSeqNo = 0; + PendingJITDispatchResultsMap PendingJITDispatchResults; + std::vector<sys::DynamicLibrary> Dylibs; +}; + +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_SIMPLEREMOTEEPCSERVER_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/TargetProcess/TargetExecutionUtils.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/TargetProcess/TargetExecutionUtils.h new file mode 100644 index 0000000000..15a354dd34 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/TargetProcess/TargetExecutionUtils.h @@ -0,0 +1,49 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===-- TargetExecutionUtils.h - Utils for execution in target --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Utilities for execution in the target process. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_TARGETEXECUTIONUTILS_H +#define LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_TARGETEXECUTIONUTILS_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/StringRef.h" +#include <string> + +namespace llvm { +namespace orc { + +/// Run a main function, returning the result. +/// +/// If the optional ProgramName argument is given then it will be inserted +/// before the strings in Args as the first argument to the called function. +/// +/// It is legal to have an empty argument list and no program name, however +/// many main functions will expect a name argument at least, and will fail +/// if none is provided. +int runAsMain(int (*Main)(int, char *[]), ArrayRef<std::string> Args, + Optional<StringRef> ProgramName = None); + +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_TARGETEXECUTIONUTILS_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/TaskDispatch.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/TaskDispatch.h new file mode 100644 index 0000000000..dae59b8d54 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/TaskDispatch.h @@ -0,0 +1,142 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===--------- TaskDispatch.h - ORC task dispatch utils ---------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Task and TaskDispatch classes. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_TASKDISPATCH_H +#define LLVM_EXECUTIONENGINE_ORC_TASKDISPATCH_H + +#include "llvm/Config/llvm-config.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/ExtensibleRTTI.h" +#include "llvm/Support/raw_ostream.h" + +#include <cassert> +#include <string> + +#if LLVM_ENABLE_THREADS +#include <condition_variable> +#include <mutex> +#include <thread> +#endif + +namespace llvm { +namespace orc { + +/// Represents an abstract task for ORC to run. +class Task : public RTTIExtends<Task, RTTIRoot> { +public: + static char ID; + + virtual ~Task() = default; + + /// Description of the task to be performed. Used for logging. + virtual void printDescription(raw_ostream &OS) = 0; + + /// Run the task. + virtual void run() = 0; + +private: + void anchor() override; +}; + +/// Base class for generic tasks. +class GenericNamedTask : public RTTIExtends<GenericNamedTask, Task> { +public: + static char ID; + static const char *DefaultDescription; +}; + +/// Generic task implementation. +template <typename FnT> class GenericNamedTaskImpl : public GenericNamedTask { +public: + GenericNamedTaskImpl(FnT &&Fn, std::string DescBuffer) + : Fn(std::forward<FnT>(Fn)), Desc(DescBuffer.c_str()), + DescBuffer(std::move(DescBuffer)) {} + GenericNamedTaskImpl(FnT &&Fn, const char *Desc) + : Fn(std::forward<FnT>(Fn)), Desc(Desc) { + assert(Desc && "Description cannot be null"); + } + void printDescription(raw_ostream &OS) override { OS << Desc; } + void run() override { Fn(); } + +private: + FnT Fn; + const char *Desc; + std::string DescBuffer; +}; + +/// Create a generic named task from a std::string description. +template <typename FnT> +std::unique_ptr<GenericNamedTask> makeGenericNamedTask(FnT &&Fn, + std::string Desc) { + return std::make_unique<GenericNamedTaskImpl<FnT>>(std::forward<FnT>(Fn), + std::move(Desc)); +} + +/// Create a generic named task from a const char * description. +template <typename FnT> +std::unique_ptr<GenericNamedTask> +makeGenericNamedTask(FnT &&Fn, const char *Desc = nullptr) { + if (!Desc) + Desc = GenericNamedTask::DefaultDescription; + return std::make_unique<GenericNamedTaskImpl<FnT>>(std::forward<FnT>(Fn), + Desc); +} + +/// Abstract base for classes that dispatch ORC Tasks. +class TaskDispatcher { +public: + virtual ~TaskDispatcher(); + + /// Run the given task. + virtual void dispatch(std::unique_ptr<Task> T) = 0; + + /// Called by ExecutionSession. Waits until all tasks have completed. + virtual void shutdown() = 0; +}; + +/// Runs all tasks on the current thread. +class InPlaceTaskDispatcher : public TaskDispatcher { +public: + void dispatch(std::unique_ptr<Task> T) override; + void shutdown() override; +}; + +#if LLVM_ENABLE_THREADS + +class DynamicThreadPoolTaskDispatcher : public TaskDispatcher { +public: + void dispatch(std::unique_ptr<Task> T) override; + void shutdown() override; +private: + std::mutex DispatchMutex; + bool Running = true; + size_t Outstanding = 0; + std::condition_variable OutstandingCV; +}; + +#endif // LLVM_ENABLE_THREADS + +} // End namespace orc +} // End namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_TASKDISPATCH_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/ThreadSafeModule.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/ThreadSafeModule.h new file mode 100644 index 0000000000..a196a0e5b5 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/Orc/ThreadSafeModule.h @@ -0,0 +1,183 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===----------- ThreadSafeModule.h -- Layer interfaces ---------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Thread safe wrappers and utilities for Module and LLVMContext. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_THREADSAFEMODULE_H +#define LLVM_EXECUTIONENGINE_ORC_THREADSAFEMODULE_H + +#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/Module.h" +#include "llvm/Support/Compiler.h" + +#include <functional> +#include <memory> +#include <mutex> + +namespace llvm { +namespace orc { + +/// An LLVMContext together with an associated mutex that can be used to lock +/// the context to prevent concurrent access by other threads. +class ThreadSafeContext { +private: + struct State { + State(std::unique_ptr<LLVMContext> Ctx) : Ctx(std::move(Ctx)) {} + + std::unique_ptr<LLVMContext> Ctx; + std::recursive_mutex Mutex; + }; + +public: + // RAII based lock for ThreadSafeContext. + class LLVM_NODISCARD Lock { + public: + Lock(std::shared_ptr<State> S) : S(std::move(S)), L(this->S->Mutex) {} + + private: + std::shared_ptr<State> S; + std::unique_lock<std::recursive_mutex> L; + }; + + /// Construct a null context. + ThreadSafeContext() = default; + + /// Construct a ThreadSafeContext from the given LLVMContext. + ThreadSafeContext(std::unique_ptr<LLVMContext> NewCtx) + : S(std::make_shared<State>(std::move(NewCtx))) { + assert(S->Ctx != nullptr && + "Can not construct a ThreadSafeContext from a nullptr"); + } + + /// Returns a pointer to the LLVMContext that was used to construct this + /// instance, or null if the instance was default constructed. + LLVMContext *getContext() { return S ? S->Ctx.get() : nullptr; } + + /// Returns a pointer to the LLVMContext that was used to construct this + /// instance, or null if the instance was default constructed. + const LLVMContext *getContext() const { return S ? S->Ctx.get() : nullptr; } + + Lock getLock() const { + assert(S && "Can not lock an empty ThreadSafeContext"); + return Lock(S); + } + +private: + std::shared_ptr<State> S; +}; + +/// An LLVM Module together with a shared ThreadSafeContext. +class ThreadSafeModule { +public: + /// Default construct a ThreadSafeModule. This results in a null module and + /// null context. + ThreadSafeModule() = default; + + ThreadSafeModule(ThreadSafeModule &&Other) = default; + + ThreadSafeModule &operator=(ThreadSafeModule &&Other) { + // We have to explicitly define this move operator to copy the fields in + // reverse order (i.e. module first) to ensure the dependencies are + // protected: The old module that is being overwritten must be destroyed + // *before* the context that it depends on. + // We also need to lock the context to make sure the module tear-down + // does not overlap any other work on the context. + if (M) { + auto L = TSCtx.getLock(); + M = nullptr; + } + M = std::move(Other.M); + TSCtx = std::move(Other.TSCtx); + return *this; + } + + /// Construct a ThreadSafeModule from a unique_ptr<Module> and a + /// unique_ptr<LLVMContext>. This creates a new ThreadSafeContext from the + /// given context. + ThreadSafeModule(std::unique_ptr<Module> M, std::unique_ptr<LLVMContext> Ctx) + : M(std::move(M)), TSCtx(std::move(Ctx)) {} + + /// Construct a ThreadSafeModule from a unique_ptr<Module> and an + /// existing ThreadSafeContext. + ThreadSafeModule(std::unique_ptr<Module> M, ThreadSafeContext TSCtx) + : M(std::move(M)), TSCtx(std::move(TSCtx)) {} + + ~ThreadSafeModule() { + // We need to lock the context while we destruct the module. + if (M) { + auto L = TSCtx.getLock(); + M = nullptr; + } + } + + /// Boolean conversion: This ThreadSafeModule will evaluate to true if it + /// wraps a non-null module. + explicit operator bool() const { + if (M) { + assert(TSCtx.getContext() && + "Non-null module must have non-null context"); + return true; + } + return false; + } + + /// Locks the associated ThreadSafeContext and calls the given function + /// on the contained Module. + template <typename Func> decltype(auto) withModuleDo(Func &&F) { + assert(M && "Can not call on null module"); + auto Lock = TSCtx.getLock(); + return F(*M); + } + + /// Locks the associated ThreadSafeContext and calls the given function + /// on the contained Module. + template <typename Func> decltype(auto) withModuleDo(Func &&F) const { + auto Lock = TSCtx.getLock(); + return F(*M); + } + + /// Get a raw pointer to the contained module without locking the context. + Module *getModuleUnlocked() { return M.get(); } + + /// Get a raw pointer to the contained module without locking the context. + const Module *getModuleUnlocked() const { return M.get(); } + + /// Returns the context for this ThreadSafeModule. + ThreadSafeContext getContext() const { return TSCtx; } + +private: + std::unique_ptr<Module> M; + ThreadSafeContext TSCtx; +}; + +using GVPredicate = std::function<bool(const GlobalValue &)>; +using GVModifier = std::function<void(GlobalValue &)>; + +/// Clones the given module on to a new context. +ThreadSafeModule +cloneToNewContext(const ThreadSafeModule &TSMW, + GVPredicate ShouldCloneDef = GVPredicate(), + GVModifier UpdateClonedDefSource = GVModifier()); + +} // End namespace orc +} // End namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_THREADSAFEMODULE_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/RTDyldMemoryManager.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/RTDyldMemoryManager.h new file mode 100644 index 0000000000..c47d8363e1 --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/RTDyldMemoryManager.h @@ -0,0 +1,169 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===-- RTDyldMemoryManager.cpp - Memory manager for MC-JIT -----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Interface of the runtime dynamic memory manager base class. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_RTDYLDMEMORYMANAGER_H +#define LLVM_EXECUTIONENGINE_RTDYLDMEMORYMANAGER_H + +#include "llvm-c/ExecutionEngine.h" +#include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/ExecutionEngine/RuntimeDyld.h" +#include "llvm/Support/CBindingWrapping.h" +#include <cstddef> +#include <cstdint> +#include <string> + +namespace llvm { + +class ExecutionEngine; + +namespace object { + class ObjectFile; +} // end namespace object + +class MCJITMemoryManager : public RuntimeDyld::MemoryManager { +public: + // Don't hide the notifyObjectLoaded method from RuntimeDyld::MemoryManager. + using RuntimeDyld::MemoryManager::notifyObjectLoaded; + + /// This method is called after an object has been loaded into memory but + /// before relocations are applied to the loaded sections. The object load + /// may have been initiated by MCJIT to resolve an external symbol for another + /// object that is being finalized. In that case, the object about which + /// the memory manager is being notified will be finalized immediately after + /// the memory manager returns from this call. + /// + /// Memory managers which are preparing code for execution in an external + /// address space can use this call to remap the section addresses for the + /// newly loaded object. + virtual void notifyObjectLoaded(ExecutionEngine *EE, + const object::ObjectFile &) {} + +private: + void anchor() override; +}; + +// RuntimeDyld clients often want to handle the memory management of +// what gets placed where. For JIT clients, this is the subset of +// JITMemoryManager required for dynamic loading of binaries. +// +// FIXME: As the RuntimeDyld fills out, additional routines will be needed +// for the varying types of objects to be allocated. +class RTDyldMemoryManager : public MCJITMemoryManager, + public LegacyJITSymbolResolver { +public: + RTDyldMemoryManager() = default; + RTDyldMemoryManager(const RTDyldMemoryManager&) = delete; + void operator=(const RTDyldMemoryManager&) = delete; + ~RTDyldMemoryManager() override; + + /// Register EH frames in the current process. + static void registerEHFramesInProcess(uint8_t *Addr, size_t Size); + + /// Deregister EH frames in the current proces. + static void deregisterEHFramesInProcess(uint8_t *Addr, size_t Size); + + void registerEHFrames(uint8_t *Addr, uint64_t LoadAddr, size_t Size) override; + void deregisterEHFrames() override; + + /// This method returns the address of the specified function or variable in + /// the current process. + static uint64_t getSymbolAddressInProcess(const std::string &Name); + + /// Legacy symbol lookup - DEPRECATED! Please override findSymbol instead. + /// + /// This method returns the address of the specified function or variable. + /// It is used to resolve symbols during module linking. + virtual uint64_t getSymbolAddress(const std::string &Name) { + return getSymbolAddressInProcess(Name); + } + + /// This method returns a RuntimeDyld::SymbolInfo for the specified function + /// or variable. It is used to resolve symbols during module linking. + /// + /// By default this falls back on the legacy lookup method: + /// 'getSymbolAddress'. The address returned by getSymbolAddress is treated as + /// a strong, exported symbol, consistent with historical treatment by + /// RuntimeDyld. + /// + /// Clients writing custom RTDyldMemoryManagers are encouraged to override + /// this method and return a SymbolInfo with the flags set correctly. This is + /// necessary for RuntimeDyld to correctly handle weak and non-exported symbols. + JITSymbol findSymbol(const std::string &Name) override { + return JITSymbol(getSymbolAddress(Name), JITSymbolFlags::Exported); + } + + /// Legacy symbol lookup -- DEPRECATED! Please override + /// findSymbolInLogicalDylib instead. + /// + /// Default to treating all modules as separate. + virtual uint64_t getSymbolAddressInLogicalDylib(const std::string &Name) { + return 0; + } + + /// Default to treating all modules as separate. + /// + /// By default this falls back on the legacy lookup method: + /// 'getSymbolAddressInLogicalDylib'. The address returned by + /// getSymbolAddressInLogicalDylib is treated as a strong, exported symbol, + /// consistent with historical treatment by RuntimeDyld. + /// + /// Clients writing custom RTDyldMemoryManagers are encouraged to override + /// this method and return a SymbolInfo with the flags set correctly. This is + /// necessary for RuntimeDyld to correctly handle weak and non-exported symbols. + JITSymbol + findSymbolInLogicalDylib(const std::string &Name) override { + return JITSymbol(getSymbolAddressInLogicalDylib(Name), + JITSymbolFlags::Exported); + } + + /// This method returns the address of the specified function. As such it is + /// only useful for resolving library symbols, not code generated symbols. + /// + /// If \p AbortOnFailure is false and no function with the given name is + /// found, this function returns a null pointer. Otherwise, it prints a + /// message to stderr and aborts. + /// + /// This function is deprecated for memory managers to be used with + /// MCJIT or RuntimeDyld. Use getSymbolAddress instead. + virtual void *getPointerToNamedFunction(const std::string &Name, + bool AbortOnFailure = true); + +protected: + struct EHFrame { + uint8_t *Addr; + size_t Size; + }; + typedef std::vector<EHFrame> EHFrameInfos; + EHFrameInfos EHFrames; + +private: + void anchor() override; +}; + +// Create wrappers for C Binding types (see CBindingWrapping.h). +DEFINE_SIMPLE_CONVERSION_FUNCTIONS( + RTDyldMemoryManager, LLVMMCJITMemoryManagerRef) + +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_RTDYLDMEMORYMANAGER_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/RuntimeDyld.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/RuntimeDyld.h new file mode 100644 index 0000000000..57dcc0d69e --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/RuntimeDyld.h @@ -0,0 +1,339 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===- RuntimeDyld.h - Run-time dynamic linker for MC-JIT -------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Interface for the runtime dynamic linker facilities of the MC-JIT. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_RUNTIMEDYLD_H +#define LLVM_EXECUTIONENGINE_RUNTIMEDYLD_H + +#include "llvm/ADT/FunctionExtras.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/DebugInfo/DIContext.h" +#include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Support/Error.h" +#include <algorithm> +#include <cassert> +#include <cstddef> +#include <cstdint> +#include <map> +#include <memory> +#include <string> +#include <system_error> + +namespace llvm { + +namespace object { + +template <typename T> class OwningBinary; + +} // end namespace object + +/// Base class for errors originating in RuntimeDyld, e.g. missing relocation +/// support. +class RuntimeDyldError : public ErrorInfo<RuntimeDyldError> { +public: + static char ID; + + RuntimeDyldError(std::string ErrMsg) : ErrMsg(std::move(ErrMsg)) {} + + void log(raw_ostream &OS) const override; + const std::string &getErrorMessage() const { return ErrMsg; } + std::error_code convertToErrorCode() const override; + +private: + std::string ErrMsg; +}; + +class RuntimeDyldImpl; + +class RuntimeDyld { +public: + // Change the address associated with a section when resolving relocations. + // Any relocations already associated with the symbol will be re-resolved. + void reassignSectionAddress(unsigned SectionID, uint64_t Addr); + + using NotifyStubEmittedFunction = std::function<void( + StringRef FileName, StringRef SectionName, StringRef SymbolName, + unsigned SectionID, uint32_t StubOffset)>; + + /// Information about the loaded object. + class LoadedObjectInfo : public llvm::LoadedObjectInfo { + friend class RuntimeDyldImpl; + + public: + using ObjSectionToIDMap = std::map<object::SectionRef, unsigned>; + + LoadedObjectInfo(RuntimeDyldImpl &RTDyld, ObjSectionToIDMap ObjSecToIDMap) + : RTDyld(RTDyld), ObjSecToIDMap(std::move(ObjSecToIDMap)) {} + + virtual object::OwningBinary<object::ObjectFile> + getObjectForDebug(const object::ObjectFile &Obj) const = 0; + + uint64_t + getSectionLoadAddress(const object::SectionRef &Sec) const override; + + protected: + virtual void anchor(); + + RuntimeDyldImpl &RTDyld; + ObjSectionToIDMap ObjSecToIDMap; + }; + + /// Memory Management. + class MemoryManager { + friend class RuntimeDyld; + + public: + MemoryManager() = default; + virtual ~MemoryManager() = default; + + /// Allocate a memory block of (at least) the given size suitable for + /// executable code. The SectionID is a unique identifier assigned by the + /// RuntimeDyld instance, and optionally recorded by the memory manager to + /// access a loaded section. + virtual uint8_t *allocateCodeSection(uintptr_t Size, unsigned Alignment, + unsigned SectionID, + StringRef SectionName) = 0; + + /// Allocate a memory block of (at least) the given size suitable for data. + /// The SectionID is a unique identifier assigned by the JIT engine, and + /// optionally recorded by the memory manager to access a loaded section. + virtual uint8_t *allocateDataSection(uintptr_t Size, unsigned Alignment, + unsigned SectionID, + StringRef SectionName, + bool IsReadOnly) = 0; + + /// An allocated TLS section + struct TLSSection { + /// The pointer to the initialization image + uint8_t *InitializationImage; + /// The TLS offset + intptr_t Offset; + }; + + /// Allocate a memory block of (at least) the given size to be used for + /// thread-local storage (TLS). + virtual TLSSection allocateTLSSection(uintptr_t Size, unsigned Alignment, + unsigned SectionID, + StringRef SectionName); + + /// Inform the memory manager about the total amount of memory required to + /// allocate all sections to be loaded: + /// \p CodeSize - the total size of all code sections + /// \p DataSizeRO - the total size of all read-only data sections + /// \p DataSizeRW - the total size of all read-write data sections + /// + /// Note that by default the callback is disabled. To enable it + /// redefine the method needsToReserveAllocationSpace to return true. + virtual void reserveAllocationSpace(uintptr_t CodeSize, uint32_t CodeAlign, + uintptr_t RODataSize, + uint32_t RODataAlign, + uintptr_t RWDataSize, + uint32_t RWDataAlign) {} + + /// Override to return true to enable the reserveAllocationSpace callback. + virtual bool needsToReserveAllocationSpace() { return false; } + + /// Override to return false to tell LLVM no stub space will be needed. + /// This requires some guarantees depending on architecuture, but when + /// you know what you are doing it saves allocated space. + virtual bool allowStubAllocation() const { return true; } + + /// Register the EH frames with the runtime so that c++ exceptions work. + /// + /// \p Addr parameter provides the local address of the EH frame section + /// data, while \p LoadAddr provides the address of the data in the target + /// address space. If the section has not been remapped (which will usually + /// be the case for local execution) these two values will be the same. + virtual void registerEHFrames(uint8_t *Addr, uint64_t LoadAddr, + size_t Size) = 0; + virtual void deregisterEHFrames() = 0; + + /// This method is called when object loading is complete and section page + /// permissions can be applied. It is up to the memory manager implementation + /// to decide whether or not to act on this method. The memory manager will + /// typically allocate all sections as read-write and then apply specific + /// permissions when this method is called. Code sections cannot be executed + /// until this function has been called. In addition, any cache coherency + /// operations needed to reliably use the memory are also performed. + /// + /// Returns true if an error occurred, false otherwise. + virtual bool finalizeMemory(std::string *ErrMsg = nullptr) = 0; + + /// This method is called after an object has been loaded into memory but + /// before relocations are applied to the loaded sections. + /// + /// Memory managers which are preparing code for execution in an external + /// address space can use this call to remap the section addresses for the + /// newly loaded object. + /// + /// For clients that do not need access to an ExecutionEngine instance this + /// method should be preferred to its cousin + /// MCJITMemoryManager::notifyObjectLoaded as this method is compatible with + /// ORC JIT stacks. + virtual void notifyObjectLoaded(RuntimeDyld &RTDyld, + const object::ObjectFile &Obj) {} + + private: + virtual void anchor(); + + bool FinalizationLocked = false; + }; + + /// Construct a RuntimeDyld instance. + RuntimeDyld(MemoryManager &MemMgr, JITSymbolResolver &Resolver); + RuntimeDyld(const RuntimeDyld &) = delete; + RuntimeDyld &operator=(const RuntimeDyld &) = delete; + ~RuntimeDyld(); + + /// Add the referenced object file to the list of objects to be loaded and + /// relocated. + std::unique_ptr<LoadedObjectInfo> loadObject(const object::ObjectFile &O); + + /// Get the address of our local copy of the symbol. This may or may not + /// be the address used for relocation (clients can copy the data around + /// and resolve relocatons based on where they put it). + void *getSymbolLocalAddress(StringRef Name) const; + + /// Get the section ID for the section containing the given symbol. + unsigned getSymbolSectionID(StringRef Name) const; + + /// Get the target address and flags for the named symbol. + /// This address is the one used for relocation. + JITEvaluatedSymbol getSymbol(StringRef Name) const; + + /// Returns a copy of the symbol table. This can be used by on-finalized + /// callbacks to extract the symbol table before throwing away the + /// RuntimeDyld instance. Because the map keys (StringRefs) are backed by + /// strings inside the RuntimeDyld instance, the map should be processed + /// before the RuntimeDyld instance is discarded. + std::map<StringRef, JITEvaluatedSymbol> getSymbolTable() const; + + /// Resolve the relocations for all symbols we currently know about. + void resolveRelocations(); + + /// Map a section to its target address space value. + /// Map the address of a JIT section as returned from the memory manager + /// to the address in the target process as the running code will see it. + /// This is the address which will be used for relocation resolution. + void mapSectionAddress(const void *LocalAddress, uint64_t TargetAddress); + + /// Returns the section's working memory. + StringRef getSectionContent(unsigned SectionID) const; + + /// If the section was loaded, return the section's load address, + /// otherwise return None. + uint64_t getSectionLoadAddress(unsigned SectionID) const; + + /// Set the NotifyStubEmitted callback. This is used for debugging + /// purposes. A callback is made for each stub that is generated. + void setNotifyStubEmitted(NotifyStubEmittedFunction NotifyStubEmitted) { + this->NotifyStubEmitted = std::move(NotifyStubEmitted); + } + + /// Register any EH frame sections that have been loaded but not previously + /// registered with the memory manager. Note, RuntimeDyld is responsible + /// for identifying the EH frame and calling the memory manager with the + /// EH frame section data. However, the memory manager itself will handle + /// the actual target-specific EH frame registration. + void registerEHFrames(); + + void deregisterEHFrames(); + + bool hasError(); + StringRef getErrorString(); + + /// By default, only sections that are "required for execution" are passed to + /// the RTDyldMemoryManager, and other sections are discarded. Passing 'true' + /// to this method will cause RuntimeDyld to pass all sections to its + /// memory manager regardless of whether they are "required to execute" in the + /// usual sense. This is useful for inspecting metadata sections that may not + /// contain relocations, E.g. Debug info, stackmaps. + /// + /// Must be called before the first object file is loaded. + void setProcessAllSections(bool ProcessAllSections) { + assert(!Dyld && "setProcessAllSections must be called before loadObject."); + this->ProcessAllSections = ProcessAllSections; + } + + /// Perform all actions needed to make the code owned by this RuntimeDyld + /// instance executable: + /// + /// 1) Apply relocations. + /// 2) Register EH frames. + /// 3) Update memory permissions*. + /// + /// * Finalization is potentially recursive**, and the 3rd step will only be + /// applied by the outermost call to finalize. This allows different + /// RuntimeDyld instances to share a memory manager without the innermost + /// finalization locking the memory and causing relocation fixup errors in + /// outer instances. + /// + /// ** Recursive finalization occurs when one RuntimeDyld instances needs the + /// address of a symbol owned by some other instance in order to apply + /// relocations. + /// + void finalizeWithMemoryManagerLocking(); + +private: + friend void jitLinkForORC( + object::OwningBinary<object::ObjectFile> O, + RuntimeDyld::MemoryManager &MemMgr, JITSymbolResolver &Resolver, + bool ProcessAllSections, + unique_function<Error(const object::ObjectFile &Obj, LoadedObjectInfo &, + std::map<StringRef, JITEvaluatedSymbol>)> + OnLoaded, + unique_function<void(object::OwningBinary<object::ObjectFile> O, + std::unique_ptr<LoadedObjectInfo>, Error)> + OnEmitted); + + // RuntimeDyldImpl is the actual class. RuntimeDyld is just the public + // interface. + std::unique_ptr<RuntimeDyldImpl> Dyld; + MemoryManager &MemMgr; + JITSymbolResolver &Resolver; + bool ProcessAllSections; + NotifyStubEmittedFunction NotifyStubEmitted; +}; + +// Asynchronous JIT link for ORC. +// +// Warning: This API is experimental and probably should not be used by anyone +// but ORC's RTDyldObjectLinkingLayer2. Internally it constructs a RuntimeDyld +// instance and uses continuation passing to perform the fix-up and finalize +// steps asynchronously. +void jitLinkForORC( + object::OwningBinary<object::ObjectFile> O, + RuntimeDyld::MemoryManager &MemMgr, JITSymbolResolver &Resolver, + bool ProcessAllSections, + unique_function<Error(const object::ObjectFile &Obj, + RuntimeDyld::LoadedObjectInfo &, + std::map<StringRef, JITEvaluatedSymbol>)> + OnLoaded, + unique_function<void(object::OwningBinary<object::ObjectFile>, + std::unique_ptr<RuntimeDyld::LoadedObjectInfo>, Error)> + OnEmitted); + +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_RUNTIMEDYLD_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/RuntimeDyldChecker.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/RuntimeDyldChecker.h new file mode 100644 index 0000000000..92f00e058f --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/RuntimeDyldChecker.h @@ -0,0 +1,195 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===---- RuntimeDyldChecker.h - RuntimeDyld tester framework -----*- C++ -*-=// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_RUNTIMEDYLDCHECKER_H +#define LLVM_EXECUTIONENGINE_RUNTIMEDYLDCHECKER_H + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/Support/Endian.h" + +#include <cstdint> +#include <memory> +#include <string> +#include <utility> + +namespace llvm { + +class StringRef; +class MCDisassembler; +class MemoryBuffer; +class MCInstPrinter; +class RuntimeDyld; +class RuntimeDyldCheckerImpl; +class raw_ostream; + +/// RuntimeDyld invariant checker for verifying that RuntimeDyld has +/// correctly applied relocations. +/// +/// The RuntimeDyldChecker class evaluates expressions against an attached +/// RuntimeDyld instance to verify that relocations have been applied +/// correctly. +/// +/// The expression language supports basic pointer arithmetic and bit-masking, +/// and has limited disassembler integration for accessing instruction +/// operands and the next PC (program counter) address for each instruction. +/// +/// The language syntax is: +/// +/// check = expr '=' expr +/// +/// expr = binary_expr +/// | sliceable_expr +/// +/// sliceable_expr = '*{' number '}' load_addr_expr [slice] +/// | '(' expr ')' [slice] +/// | ident_expr [slice] +/// | number [slice] +/// +/// slice = '[' high-bit-index ':' low-bit-index ']' +/// +/// load_addr_expr = symbol +/// | '(' symbol '+' number ')' +/// | '(' symbol '-' number ')' +/// +/// ident_expr = 'decode_operand' '(' symbol ',' operand-index ')' +/// | 'next_pc' '(' symbol ')' +/// | 'stub_addr' '(' stub-container-name ',' symbol ')' +/// | 'got_addr' '(' stub-container-name ',' symbol ')' +/// | symbol +/// +/// binary_expr = expr '+' expr +/// | expr '-' expr +/// | expr '&' expr +/// | expr '|' expr +/// | expr '<<' expr +/// | expr '>>' expr +/// +class RuntimeDyldChecker { +public: + class MemoryRegionInfo { + public: + MemoryRegionInfo() = default; + + /// Constructor for symbols/sections with content. + MemoryRegionInfo(ArrayRef<char> Content, JITTargetAddress TargetAddress) + : ContentPtr(Content.data()), Size(Content.size()), + TargetAddress(TargetAddress) {} + + /// Constructor for zero-fill symbols/sections. + MemoryRegionInfo(uint64_t Size, JITTargetAddress TargetAddress) + : Size(Size), TargetAddress(TargetAddress) {} + + /// Returns true if this is a zero-fill symbol/section. + bool isZeroFill() const { + assert(Size && "setContent/setZeroFill must be called first"); + return !ContentPtr; + } + + /// Set the content for this memory region. + void setContent(ArrayRef<char> Content) { + assert(!ContentPtr && !Size && "Content/zero-fill already set"); + ContentPtr = Content.data(); + Size = Content.size(); + } + + /// Set a zero-fill length for this memory region. + void setZeroFill(uint64_t Size) { + assert(!ContentPtr && !this->Size && "Content/zero-fill already set"); + this->Size = Size; + } + + /// Returns the content for this section if there is any. + ArrayRef<char> getContent() const { + assert(!isZeroFill() && "Can't get content for a zero-fill section"); + return {ContentPtr, static_cast<size_t>(Size)}; + } + + /// Returns the zero-fill length for this section. + uint64_t getZeroFillLength() const { + assert(isZeroFill() && "Can't get zero-fill length for content section"); + return Size; + } + + /// Set the target address for this region. + void setTargetAddress(JITTargetAddress TargetAddress) { + assert(!this->TargetAddress && "TargetAddress already set"); + this->TargetAddress = TargetAddress; + } + + /// Return the target address for this region. + JITTargetAddress getTargetAddress() const { return TargetAddress; } + + private: + const char *ContentPtr = nullptr; + uint64_t Size = 0; + JITTargetAddress TargetAddress = 0; + }; + + using IsSymbolValidFunction = std::function<bool(StringRef Symbol)>; + using GetSymbolInfoFunction = + std::function<Expected<MemoryRegionInfo>(StringRef SymbolName)>; + using GetSectionInfoFunction = std::function<Expected<MemoryRegionInfo>( + StringRef FileName, StringRef SectionName)>; + using GetStubInfoFunction = std::function<Expected<MemoryRegionInfo>( + StringRef StubContainer, StringRef TargetName)>; + using GetGOTInfoFunction = std::function<Expected<MemoryRegionInfo>( + StringRef GOTContainer, StringRef TargetName)>; + + RuntimeDyldChecker(IsSymbolValidFunction IsSymbolValid, + GetSymbolInfoFunction GetSymbolInfo, + GetSectionInfoFunction GetSectionInfo, + GetStubInfoFunction GetStubInfo, + GetGOTInfoFunction GetGOTInfo, + support::endianness Endianness, + MCDisassembler *Disassembler, MCInstPrinter *InstPrinter, + raw_ostream &ErrStream); + ~RuntimeDyldChecker(); + + /// Check a single expression against the attached RuntimeDyld + /// instance. + bool check(StringRef CheckExpr) const; + + /// Scan the given memory buffer for lines beginning with the string + /// in RulePrefix. The remainder of the line is passed to the check + /// method to be evaluated as an expression. + bool checkAllRulesInBuffer(StringRef RulePrefix, MemoryBuffer *MemBuf) const; + + /// Returns the address of the requested section (or an error message + /// in the second element of the pair if the address cannot be found). + /// + /// if 'LocalAddress' is true, this returns the address of the section + /// within the linker's memory. If 'LocalAddress' is false it returns the + /// address within the target process (i.e. the load address). + std::pair<uint64_t, std::string> getSectionAddr(StringRef FileName, + StringRef SectionName, + bool LocalAddress); + + /// If there is a section at the given local address, return its load + /// address, otherwise return none. + Optional<uint64_t> getSectionLoadAddress(void *LocalAddress) const; + +private: + std::unique_ptr<RuntimeDyldCheckerImpl> Impl; +}; + +} // end namespace llvm + +#endif + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif diff --git a/contrib/libs/llvm14/include/llvm/ExecutionEngine/SectionMemoryManager.h b/contrib/libs/llvm14/include/llvm/ExecutionEngine/SectionMemoryManager.h new file mode 100644 index 0000000000..f575133f5b --- /dev/null +++ b/contrib/libs/llvm14/include/llvm/ExecutionEngine/SectionMemoryManager.h @@ -0,0 +1,204 @@ +#pragma once + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +//===- SectionMemoryManager.h - Memory manager for MCJIT/RtDyld -*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains the declaration of a section-based memory manager used by +// the MCJIT execution engine and RuntimeDyld. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_SECTIONMEMORYMANAGER_H +#define LLVM_EXECUTIONENGINE_SECTIONMEMORYMANAGER_H + +#include "llvm/ADT/SmallVector.h" +#include "llvm/ExecutionEngine/RTDyldMemoryManager.h" +#include "llvm/Support/Memory.h" +#include <cstdint> +#include <string> +#include <system_error> + +namespace llvm { + +/// This is a simple memory manager which implements the methods called by +/// the RuntimeDyld class to allocate memory for section-based loading of +/// objects, usually those generated by the MCJIT execution engine. +/// +/// This memory manager allocates all section memory as read-write. The +/// RuntimeDyld will copy JITed section memory into these allocated blocks +/// and perform any necessary linking and relocations. +/// +/// Any client using this memory manager MUST ensure that section-specific +/// page permissions have been applied before attempting to execute functions +/// in the JITed object. Permissions can be applied either by calling +/// MCJIT::finalizeObject or by calling SectionMemoryManager::finalizeMemory +/// directly. Clients of MCJIT should call MCJIT::finalizeObject. +class SectionMemoryManager : public RTDyldMemoryManager { +public: + /// This enum describes the various reasons to allocate pages from + /// allocateMappedMemory. + enum class AllocationPurpose { + Code, + ROData, + RWData, + }; + + /// Implementations of this interface are used by SectionMemoryManager to + /// request pages from the operating system. + class MemoryMapper { + public: + /// This method attempts to allocate \p NumBytes bytes of virtual memory for + /// \p Purpose. \p NearBlock may point to an existing allocation, in which + /// case an attempt is made to allocate more memory near the existing block. + /// The actual allocated address is not guaranteed to be near the requested + /// address. \p Flags is used to set the initial protection flags for the + /// block of the memory. \p EC [out] returns an object describing any error + /// that occurs. + /// + /// This method may allocate more than the number of bytes requested. The + /// actual number of bytes allocated is indicated in the returned + /// MemoryBlock. + /// + /// The start of the allocated block must be aligned with the system + /// allocation granularity (64K on Windows, page size on Linux). If the + /// address following \p NearBlock is not so aligned, it will be rounded up + /// to the next allocation granularity boundary. + /// + /// \r a non-null MemoryBlock if the function was successful, otherwise a + /// null MemoryBlock with \p EC describing the error. + virtual sys::MemoryBlock + allocateMappedMemory(AllocationPurpose Purpose, size_t NumBytes, + const sys::MemoryBlock *const NearBlock, + unsigned Flags, std::error_code &EC) = 0; + + /// This method sets the protection flags for a block of memory to the state + /// specified by \p Flags. The behavior is not specified if the memory was + /// not allocated using the allocateMappedMemory method. + /// \p Block describes the memory block to be protected. + /// \p Flags specifies the new protection state to be assigned to the block. + /// + /// If \p Flags is MF_WRITE, the actual behavior varies with the operating + /// system (i.e. MF_READ | MF_WRITE on Windows) and the target architecture + /// (i.e. MF_WRITE -> MF_READ | MF_WRITE on i386). + /// + /// \r error_success if the function was successful, or an error_code + /// describing the failure if an error occurred. + virtual std::error_code protectMappedMemory(const sys::MemoryBlock &Block, + unsigned Flags) = 0; + + /// This method releases a block of memory that was allocated with the + /// allocateMappedMemory method. It should not be used to release any memory + /// block allocated any other way. + /// \p Block describes the memory to be released. + /// + /// \r error_success if the function was successful, or an error_code + /// describing the failure if an error occurred. + virtual std::error_code releaseMappedMemory(sys::MemoryBlock &M) = 0; + + virtual ~MemoryMapper(); + }; + + /// Creates a SectionMemoryManager instance with \p MM as the associated + /// memory mapper. If \p MM is nullptr then a default memory mapper is used + /// that directly calls into the operating system. + SectionMemoryManager(MemoryMapper *MM = nullptr); + SectionMemoryManager(const SectionMemoryManager &) = delete; + void operator=(const SectionMemoryManager &) = delete; + ~SectionMemoryManager() override; + + /// Allocates a memory block of (at least) the given size suitable for + /// executable code. + /// + /// The value of \p Alignment must be a power of two. If \p Alignment is zero + /// a default alignment of 16 will be used. + uint8_t *allocateCodeSection(uintptr_t Size, unsigned Alignment, + unsigned SectionID, + StringRef SectionName) override; + + /// Allocates a memory block of (at least) the given size suitable for + /// executable code. + /// + /// The value of \p Alignment must be a power of two. If \p Alignment is zero + /// a default alignment of 16 will be used. + uint8_t *allocateDataSection(uintptr_t Size, unsigned Alignment, + unsigned SectionID, StringRef SectionName, + bool isReadOnly) override; + + /// Update section-specific memory permissions and other attributes. + /// + /// This method is called when object loading is complete and section page + /// permissions can be applied. It is up to the memory manager implementation + /// to decide whether or not to act on this method. The memory manager will + /// typically allocate all sections as read-write and then apply specific + /// permissions when this method is called. Code sections cannot be executed + /// until this function has been called. In addition, any cache coherency + /// operations needed to reliably use the memory are also performed. + /// + /// \returns true if an error occurred, false otherwise. + bool finalizeMemory(std::string *ErrMsg = nullptr) override; + + /// Invalidate instruction cache for code sections. + /// + /// Some platforms with separate data cache and instruction cache require + /// explicit cache flush, otherwise JIT code manipulations (like resolved + /// relocations) will get to the data cache but not to the instruction cache. + /// + /// This method is called from finalizeMemory. + virtual void invalidateInstructionCache(); + +private: + struct FreeMemBlock { + // The actual block of free memory + sys::MemoryBlock Free; + // If there is a pending allocation from the same reservation right before + // this block, store it's index in PendingMem, to be able to update the + // pending region if part of this block is allocated, rather than having to + // create a new one + unsigned PendingPrefixIndex; + }; + + struct MemoryGroup { + // PendingMem contains all blocks of memory (subblocks of AllocatedMem) + // which have not yet had their permissions applied, but have been given + // out to the user. FreeMem contains all block of memory, which have + // neither had their permissions applied, nor been given out to the user. + SmallVector<sys::MemoryBlock, 16> PendingMem; + SmallVector<FreeMemBlock, 16> FreeMem; + + // All memory blocks that have been requested from the system + SmallVector<sys::MemoryBlock, 16> AllocatedMem; + + sys::MemoryBlock Near; + }; + + uint8_t *allocateSection(AllocationPurpose Purpose, uintptr_t Size, + unsigned Alignment); + + std::error_code applyMemoryGroupPermissions(MemoryGroup &MemGroup, + unsigned Permissions); + + void anchor() override; + + MemoryGroup CodeMem; + MemoryGroup RWDataMem; + MemoryGroup RODataMem; + MemoryMapper &MMapper; +}; + +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_SECTIONMEMORYMANAGER_H + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif |