1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
|
#include <stddef.h>
#include <stdint.h>
#include <memory>
#include <string>
#include <utility>
#include <vector>
#include "EmitContext.h"
#include "LLVMJITPrivate.h"
#include "WAVM/IR/Types.h"
#include "WAVM/Inline/Assert.h"
#include "WAVM/Inline/BasicTypes.h"
#include "WAVM/Inline/Hash.h"
#include "WAVM/Inline/HashMap.h"
#include "WAVM/LLVMJIT/LLVMJIT.h"
#include "WAVM/Logging/Logging.h"
#include "WAVM/Platform/Diagnostics.h"
#include "WAVM/Platform/RWMutex.h"
#include "WAVM/RuntimeABI/RuntimeABI.h"
PUSH_DISABLE_WARNINGS_FOR_LLVM_HEADERS
#include <llvm/ADT/SmallVector.h>
#include <llvm/ADT/StringRef.h>
#include <llvm/ADT/iterator_range.h>
#include <llvm/IR/Argument.h>
#include <llvm/IR/BasicBlock.h>
#include <llvm/IR/Constant.h>
#include <llvm/IR/Constants.h>
#include <llvm/IR/DerivedTypes.h>
#include <llvm/IR/Function.h>
#include <llvm/IR/GlobalValue.h>
#include <llvm/IR/IRBuilder.h>
#include <llvm/IR/Instructions.h>
#include <llvm/IR/Module.h>
#include <llvm/IR/Type.h>
POP_DISABLE_WARNINGS_FOR_LLVM_HEADERS
namespace llvm {
class Value;
}
using namespace WAVM;
using namespace WAVM::IR;
using namespace WAVM::LLVMJIT;
using namespace WAVM::Runtime;
// A global invoke thunk cache
struct InvokeThunkCache
{
Platform::RWMutex mutex;
HashMap<FunctionType, Runtime::Function*> typeToFunctionMap;
std::vector<std::unique_ptr<LLVMJIT::Module>> modules;
static InvokeThunkCache& get()
{
static InvokeThunkCache singleton;
return singleton;
}
private:
InvokeThunkCache() {}
};
InvokeThunkPointer LLVMJIT::getInvokeThunk(FunctionType functionType)
{
InvokeThunkCache& invokeThunkCache = InvokeThunkCache::get();
// First, take a shareable lock on the cache mutex, and check if the thunk is cached.
{
Platform::RWMutex::ShareableLock shareableLock(invokeThunkCache.mutex);
Runtime::Function** invokeThunkFunction
= invokeThunkCache.typeToFunctionMap.get(functionType);
if(invokeThunkFunction)
{
return reinterpret_cast<InvokeThunkPointer>(
const_cast<U8*>((*invokeThunkFunction)->code));
}
}
// If the thunk is not cached, take an exclusive lock on the cache mutex.
Platform::RWMutex::ExclusiveLock invokeThunkLock(invokeThunkCache.mutex);
// Since the cache is unlocked briefly while switching from the shareable to the exclusive lock,
// check again if the thunk is cached.
Runtime::Function*& invokeThunkFunction
= invokeThunkCache.typeToFunctionMap.getOrAdd(functionType, nullptr);
if(invokeThunkFunction)
{ return reinterpret_cast<InvokeThunkPointer>(const_cast<U8*>(invokeThunkFunction->code)); }
// Create a FunctionMutableData object for the thunk.
FunctionMutableData* functionMutableData
= new FunctionMutableData("thnk!C to WASM thunk!" + asString(functionType));
// Create a LLVM module and a LLVM function for the thunk.
LLVMContext llvmContext;
llvm::Module llvmModule("", llvmContext);
std::unique_ptr<llvm::TargetMachine> targetMachine = getTargetMachine(getHostTargetSpec());
llvmModule.setDataLayout(targetMachine->createDataLayout());
auto llvmFunctionType = llvm::FunctionType::get(llvmContext.i8PtrType,
{llvmContext.i8PtrType,
llvmContext.i8PtrType,
llvmContext.i8PtrType,
llvmContext.i8PtrType},
false);
#if LLVM_VERSION_MAJOR >= 7
llvm::Type* iptrType = getIptrType(llvmContext, targetMachine->getProgramPointerSize());
#else
llvm::Type* iptrType = getIptrType(llvmContext, targetMachine->getPointerSize());
#endif
auto function = llvm::Function::Create(
llvmFunctionType, llvm::Function::ExternalLinkage, "thunk", &llvmModule);
setRuntimeFunctionPrefix(llvmContext,
iptrType,
function,
emitLiteralIptr(reinterpret_cast<Uptr>(functionMutableData), iptrType),
emitLiteralIptr(UINTPTR_MAX, iptrType),
emitLiteralIptr(functionType.getEncoding().impl, iptrType));
setFunctionAttributes(targetMachine.get(), function);
llvm::Value* calleeFunction = &*(function->args().begin() + 0);
llvm::Value* contextPointer = &*(function->args().begin() + 1);
llvm::Value* argsArray = &*(function->args().begin() + 2);
llvm::Value* resultsArray = &*(function->args().begin() + 3);
EmitContext emitContext(llvmContext, {});
emitContext.irBuilder.SetInsertPoint(llvm::BasicBlock::Create(llvmContext, "entry", function));
emitContext.initContextVariables(contextPointer, iptrType);
// Load the function's arguments from the argument array.
std::vector<llvm::Value*> arguments;
for(Uptr argIndex = 0; argIndex < functionType.params().size(); ++argIndex)
{
const ValueType paramType = functionType.params()[argIndex];
llvm::Value* argOffset = emitLiteral(llvmContext, argIndex * sizeof(UntaggedValue));
llvm::Value* arg = emitContext.loadFromUntypedPointer(
emitContext.irBuilder.CreateInBoundsGEP(llvmContext.i8Type, argsArray, {argOffset}),
asLLVMType(llvmContext, paramType),
alignof(UntaggedValue));
arguments.push_back(arg);
}
// Call the function.
llvm::Value* functionCode = emitContext.irBuilder.CreateInBoundsGEP(
llvmContext.i8Type, calleeFunction, {emitLiteralIptr(offsetof(Runtime::Function, code), iptrType)});
ValueVector results = emitContext.emitCallOrInvoke(
emitContext.irBuilder.CreatePointerCast(
functionCode, asLLVMType(llvmContext, functionType)->getPointerTo()),
arguments,
functionType);
// Write the function's results to the results array.
WAVM_ASSERT(results.size() == functionType.results().size());
for(Uptr resultIndex = 0; resultIndex < results.size(); ++resultIndex)
{
llvm::Value* resultOffset = emitLiteral(llvmContext, resultIndex * sizeof(UntaggedValue));
llvm::Value* result = results[resultIndex];
emitContext.storeToUntypedPointer(
result,
emitContext.irBuilder.CreateInBoundsGEP(llvmContext.i8Type, resultsArray, {resultOffset}),
alignof(UntaggedValue));
}
// Return the new context pointer.
emitContext.irBuilder.CreateRet(
emitContext.irBuilder.CreateLoad(llvmContext.i8PtrType, emitContext.contextPointerVariable));
// Compile the LLVM IR to object code.
std::vector<U8> objectBytes
= compileLLVMModule(llvmContext, std::move(llvmModule), false, targetMachine.get());
// Load the object code.
auto importedSymbolMap = HashMap<std::string, Uptr>();
auto jitModule
= new LLVMJIT::Module(objectBytes, &importedSymbolMap, false, std::string(functionMutableData->debugName));
invokeThunkCache.modules.push_back(std::unique_ptr<LLVMJIT::Module>(jitModule));
invokeThunkFunction = jitModule->nameToFunctionMap[mangleSymbol("thunk")];
return reinterpret_cast<InvokeThunkPointer>(const_cast<U8*>(invokeThunkFunction->code));
}
|
