summaryrefslogtreecommitdiffstats
path: root/yql/essentials/minikql/computation/mkql_computation_node_codegen.cpp
diff options
context:
space:
mode:
authorvvvv <[email protected]>2024-11-07 04:19:26 +0300
committervvvv <[email protected]>2024-11-07 04:29:50 +0300
commit2661be00f3bc47590fda9218bf0386d6355c8c88 (patch)
tree3d316c07519191283d31c5f537efc6aabb42a2f0 /yql/essentials/minikql/computation/mkql_computation_node_codegen.cpp
parentcf2a23963ac10add28c50cc114fbf48953eca5aa (diff)
Moved yql/minikql YQL-19206
init [nodiff:caesar] commit_hash:d1182ef7d430ccf7e4d37ed933c7126d7bd5d6e4
Diffstat (limited to 'yql/essentials/minikql/computation/mkql_computation_node_codegen.cpp')
-rw-r--r--yql/essentials/minikql/computation/mkql_computation_node_codegen.cpp2670
1 files changed, 2670 insertions, 0 deletions
diff --git a/yql/essentials/minikql/computation/mkql_computation_node_codegen.cpp b/yql/essentials/minikql/computation/mkql_computation_node_codegen.cpp
new file mode 100644
index 00000000000..113e9da8dcc
--- /dev/null
+++ b/yql/essentials/minikql/computation/mkql_computation_node_codegen.cpp
@@ -0,0 +1,2670 @@
+#include "mkql_computation_node_codegen.h" // Y_IGNORE
+#include "mkql_computation_node_holders.h"
+
+#include <yql/essentials/minikql/codegen/codegen.h>
+#include <yql/essentials/public/decimal/yql_decimal.h>
+
+#include <util/string/cast.h>
+#include <util/folder/path.h>
+
+#ifndef MKQL_DISABLE_CODEGEN
+
+extern "C" void DeleteBoxed(NKikimr::NUdf::IBoxedValue *const boxed) {
+ delete boxed;
+}
+
+extern "C" void DeleteString(void* strData) {
+ auto& str = *(NKikimr::NUdf::TStringValue*)(&strData);
+ UdfFreeWithSize(strData, 16 + str.Capacity());
+}
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+constexpr bool EnableStaticRefcount = true;
+
+using namespace llvm;
+
+Type* GetCompContextType(LLVMContext &context) {
+ const auto ptrValueType = PointerType::getUnqual(Type::getInt128Ty(context));
+ const auto structPtrType = PointerType::getUnqual(StructType::get(context));
+ const auto stringRefType = StructType::get(context, {
+ Type::getInt8PtrTy(context),
+ Type::getInt32Ty(context),
+ Type::getInt32Ty(context)
+ });
+ const auto sourcePosType = StructType::get(context, {
+ Type::getInt32Ty(context),
+ Type::getInt32Ty(context),
+ stringRefType
+ });
+ return StructType::get(context, {
+ structPtrType, // factory
+ structPtrType, // stats
+ ptrValueType, // mutables
+ structPtrType, // builder
+ Type::getFloatTy(context), // adjustor
+ Type::getInt32Ty(context), // rsscounter
+ PointerType::getUnqual(sourcePosType)
+ });
+}
+
+Value* TCodegenContext::GetFactory() const {
+ if (!Factory) {
+ auto& context = Codegen.GetContext();
+ const auto indexType = Type::getInt32Ty(context);
+ const auto ptrType = PointerType::getUnqual(StructType::get(context));
+ if (Func->getEntryBlock().empty()) {
+ const auto ptr = GetElementPtrInst::CreateInBounds(GetCompContextType(context), Ctx, {ConstantInt::get(indexType, 0), ConstantInt::get(indexType, 0)}, "factory_ptr", &Func->getEntryBlock());
+ const_cast<Value*&>(Factory) = new LoadInst(ptrType, ptr, "factory", &Func->getEntryBlock());
+ } else {
+ const auto ptr = GetElementPtrInst::CreateInBounds(GetCompContextType(context), Ctx, {ConstantInt::get(indexType, 0), ConstantInt::get(indexType, 0)}, "factory_ptr", &Func->getEntryBlock().front());
+ const_cast<Value*&>(Factory) = new LoadInst(ptrType, ptr, "factory", &Func->getEntryBlock().back());
+ }
+ }
+ return Factory;
+}
+
+Value* TCodegenContext::GetStat() const {
+ if (!Stat) {
+ auto& context = Codegen.GetContext();
+ const auto indexType = Type::getInt32Ty(context);
+ const auto ptrType = PointerType::getUnqual(StructType::get(context));
+ if (Func->getEntryBlock().empty()) {
+ const auto ptr = GetElementPtrInst::CreateInBounds(GetCompContextType(context), Ctx, {ConstantInt::get(indexType, 0), ConstantInt::get(indexType, 1)}, "stat_ptr", &Func->getEntryBlock());
+ const_cast<Value*&>(Stat) = new LoadInst(ptrType, ptr, "stat", &Func->getEntryBlock());
+ } else {
+ const auto ptr = GetElementPtrInst::CreateInBounds(GetCompContextType(context), Ctx, {ConstantInt::get(indexType, 0), ConstantInt::get(indexType, 1)}, "stat_ptr", &Func->getEntryBlock().front());
+ const_cast<Value*&>(Stat) = new LoadInst(ptrType, ptr, "stat", &Func->getEntryBlock().back());
+ }
+ }
+ return Stat;
+}
+
+Value* TCodegenContext::GetMutables() const {
+ if (!Mutables) {
+ auto& context = Codegen.GetContext();
+ const auto indexType = Type::getInt32Ty(context);
+ const auto ptrType = PointerType::getUnqual(Type::getInt128Ty(context));
+ if (Func->getEntryBlock().empty()) {
+ const auto ptr = GetElementPtrInst::CreateInBounds(GetCompContextType(context), Ctx, {ConstantInt::get(indexType, 0), ConstantInt::get(indexType, 2)}, "mutables_ptr", &Func->getEntryBlock());
+ const_cast<Value*&>(Mutables) = new LoadInst(ptrType, ptr, "mutables", &Func->getEntryBlock());
+ } else {
+ const auto ptr = GetElementPtrInst::CreateInBounds(GetCompContextType(context), Ctx, {ConstantInt::get(indexType, 0), ConstantInt::get(indexType, 2)}, "mutables_ptr", &Func->getEntryBlock().front());
+ const_cast<Value*&>(Mutables) = new LoadInst(ptrType, ptr, "mutables", &Func->getEntryBlock().back());
+ }
+ }
+ return Mutables;
+}
+
+Value* TCodegenContext::GetBuilder() const {
+ if (!Builder) {
+ auto& context = Codegen.GetContext();
+ const auto indexType = Type::getInt32Ty(context);
+ const auto ptrType = PointerType::getUnqual(StructType::get(context));
+ if (Func->getEntryBlock().empty()) {
+ const auto ptr = GetElementPtrInst::CreateInBounds(GetCompContextType(context), Ctx, {ConstantInt::get(indexType, 0), ConstantInt::get(indexType, 3)}, "builder_ptr", &Func->getEntryBlock());
+ const_cast<Value*&>(Builder) = new LoadInst(ptrType, ptr, "builder", &Func->getEntryBlock());
+ } else {
+ const auto ptr = GetElementPtrInst::CreateInBounds(GetCompContextType(context), Ctx, {ConstantInt::get(indexType, 0), ConstantInt::get(indexType, 3)}, "builder_ptr", &Func->getEntryBlock().front());
+ const_cast<Value*&>(Builder) = new LoadInst(ptrType, ptr, "builder", &Func->getEntryBlock().back());
+ }
+ }
+ return Builder;
+}
+
+Function* GenerateCompareFunction(NYql::NCodegen::ICodegen& codegen, const TString& name, IComputationExternalNode* left,
+ IComputationExternalNode* right, IComputationNode* compare) {
+ auto& module = codegen.GetModule();
+ if (const auto f = module.getFunction(name.c_str()))
+ return f;
+
+ const auto codegenLeft = dynamic_cast<ICodegeneratorExternalNode*>(left);
+ const auto codegenRight = dynamic_cast<ICodegeneratorExternalNode*>(right);
+ MKQL_ENSURE(codegenLeft, "Left must be codegenerator node.");
+ MKQL_ENSURE(codegenRight, "Right must be codegenerator node.");
+
+ auto& context = codegen.GetContext();
+ const auto valueType = Type::getInt128Ty(context);
+ const auto ptrType = PointerType::getUnqual(valueType);
+ const auto returnType = Type::getInt1Ty(context);
+ const auto contextType = GetCompContextType(context);
+
+ const auto funcType = codegen.GetEffectiveTarget() != NYql::NCodegen::ETarget::Windows ?
+ FunctionType::get(returnType, {PointerType::getUnqual(contextType), valueType, valueType}, false):
+ FunctionType::get(returnType, {PointerType::getUnqual(contextType), ptrType, ptrType}, false);
+
+ TCodegenContext ctx(codegen);
+ ctx.AlwaysInline = true;
+ ctx.Func = cast<Function>(module.getOrInsertFunction(name.c_str(), funcType).getCallee());
+
+ DISubprogramAnnotator annotator(ctx, ctx.Func);
+
+
+ auto args = ctx.Func->arg_begin();
+
+ const auto main = BasicBlock::Create(context, "main", ctx.Func);
+ auto block = main;
+
+ ctx.Ctx = &*args;
+ ctx.Ctx->addAttr(Attribute::NonNull);
+
+ const auto lv = &*++args;
+ const auto rv = &*++args;
+
+ codegenLeft->SetValueBuilder([lv](const TCodegenContext&) { return lv; });
+ codegenRight->SetValueBuilder([rv](const TCodegenContext&) { return rv; });
+
+ codegenLeft->CreateInvalidate(ctx, block);
+ codegenRight->CreateInvalidate(ctx, block);
+
+ const auto res = GetNodeValue(compare, ctx, block);
+ const auto cast = CastInst::Create(Instruction::Trunc, res, returnType, "bool", block);
+ ReturnInst::Create(context, cast, block);
+
+ codegenLeft->SetValueBuilder({});
+ codegenRight->SetValueBuilder({});
+
+ return ctx.Func;
+}
+
+Value* GetterFor(NUdf::EDataSlot slot, Value* value, LLVMContext &context, BasicBlock* block) {
+ switch (slot) {
+ case NUdf::EDataSlot::Bool: return GetterFor<bool>(value, context, block);
+ case NUdf::EDataSlot::Decimal: return GetterForInt128(value, block);
+ case NUdf::EDataSlot::Float: return GetterFor<float>(value, context, block);
+ case NUdf::EDataSlot::Double: return GetterFor<double>(value, context, block);
+ default: break;
+ }
+
+ const auto trunc = CastInst::Create(Instruction::Trunc, value, IntegerType::get(context, NUdf::GetDataTypeInfo(slot).FixedSize << 3U), "trunc", block);
+ return trunc;
+}
+
+namespace {
+
+Value* GetMarkFromUnboxed(Value* value, const TCodegenContext& ctx, BasicBlock* block) {
+ auto& context = ctx.Codegen.GetContext();
+
+ const auto type8 = Type::getInt8Ty(context);
+ if (value->getType()->isPointerTy()) {
+ const auto type = StructType::get(context, {PointerType::getUnqual(StructType::get(context)), ArrayType::get(type8, 8U)});
+ const auto cast = CastInst::Create(Instruction::BitCast, value, PointerType::getUnqual(type), "cast", block);
+ const auto type32 = Type::getInt32Ty(context);
+ const auto metaptr = GetElementPtrInst::CreateInBounds(type, cast, {ConstantInt::get(type32, 0), ConstantInt::get(type32, 1), ConstantInt::get(type32, 7)}, "metaptr", block);
+ const auto meta = new LoadInst(type8, metaptr, "meta", block);
+ const auto mark = BinaryOperator::CreateAnd(meta, ConstantInt::get(meta->getType(), 3), "mark", block);
+ return mark;
+ } else {
+ const auto lshr = BinaryOperator::CreateLShr(value, ConstantInt::get(value->getType(), 120), "lshr", block);
+ const auto meta = CastInst::Create(Instruction::Trunc, lshr, type8, "meta", block);
+ const auto mark = BinaryOperator::CreateAnd(ConstantInt::get(meta->getType(), 3), meta, "mark", block);
+ return mark;
+ }
+
+}
+
+template<bool BoxedOrString>
+Value* GetPointerFromUnboxed(Value* value, const TCodegenContext& ctx, BasicBlock* block) {
+ auto& context = ctx.Codegen.GetContext();
+
+ const auto type32 = Type::getInt32Ty(context);
+ const auto type64 = Type::getInt64Ty(context);
+ const auto type = PointerType::getUnqual(BoxedOrString ?
+ StructType::get(context, {PointerType::getUnqual(StructType::get(context)), type32, Type::getInt16Ty(context)}):
+ StructType::get(context, {type32, type32, type32, type32})
+ );
+ if (value->getType()->isPointerTy()) {
+ const auto strType = StructType::get(context, {type, type64});
+ const auto cast = CastInst::Create(Instruction::BitCast, value, PointerType::getUnqual(strType), "cast", block);
+ const auto ptr = GetElementPtrInst::CreateInBounds(strType, cast, {ConstantInt::get(type32, 0), ConstantInt::get(type32, 0)}, "ptr", block);
+ const auto pointer = new LoadInst(type, ptr, "pointer", block);
+ return pointer;
+ } else {
+ const auto half = CastInst::Create(Instruction::Trunc, value, type64, "half", block);
+ const auto pointer = CastInst::Create(Instruction::IntToPtr, half, type, "pointer", block);
+ return pointer;
+ }
+}
+
+ui32 MyCompareStrings(NUdf::TUnboxedValuePod lhs, NUdf::TUnboxedValuePod rhs) {
+ return NUdf::CompareStrings(lhs, rhs);
+}
+
+bool MyEquteStrings(NUdf::TUnboxedValuePod lhs, NUdf::TUnboxedValuePod rhs) {
+ return NUdf::EquateStrings(lhs, rhs);
+}
+
+NUdf::THashType MyHashString(NUdf::TUnboxedValuePod val) {
+ return NUdf::GetStringHash(val);
+}
+
+template <bool IsOptional>
+Value* GenEqualsFunction(NUdf::EDataSlot slot, Value* lv, Value* rv, TCodegenContext& ctx, BasicBlock*& block);
+
+template <>
+Value* GenEqualsFunction<false>(NUdf::EDataSlot slot, Value* lv, Value* rv, TCodegenContext& ctx, BasicBlock*& block) {
+ auto& context = ctx.Codegen.GetContext();
+
+ const auto& info = NUdf::GetDataTypeInfo(slot);
+
+ if ((info.Features & NUdf::EDataTypeFeatures::CommonType) && (info.Features & NUdf::EDataTypeFeatures::StringType || NUdf::EDataSlot::Uuid == slot || NUdf::EDataSlot::DyNumber == slot)) {
+ return CallBinaryUnboxedValueFunction(&MyEquteStrings, Type::getInt1Ty(context), lv, rv, ctx.Codegen, block);
+ }
+
+ const auto lhs = GetterFor(slot, lv, context, block);
+ const auto rhs = GetterFor(slot, rv, context, block);
+
+ if (info.Features & (NUdf::EDataTypeFeatures::IntegralType | NUdf::EDataTypeFeatures::DateType | NUdf::EDataTypeFeatures::TimeIntervalType | NUdf::EDataTypeFeatures::DecimalType) || NUdf::EDataSlot::Bool == slot) {
+ const auto equal = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, lhs, rhs, "equal", block);
+ return equal;
+ }
+
+ if (info.Features & NUdf::EDataTypeFeatures::FloatType) {
+ const auto ueq = CmpInst::Create(Instruction::FCmp, FCmpInst::FCMP_UEQ, lhs, rhs, "equals", block);
+ const auto lord = CmpInst::Create(Instruction::FCmp, FCmpInst::FCMP_ORD, ConstantFP::get(lhs->getType(), 0.0), lhs, "lord", block);
+ const auto runo = CmpInst::Create(Instruction::FCmp, FCmpInst::FCMP_UNO, ConstantFP::get(rhs->getType(), 0.0), rhs, "runo", block);
+ const auto once = BinaryOperator::CreateXor(lord, runo, "xor", block);
+ return BinaryOperator::CreateAnd(ueq, once, "and", block);
+ }
+
+ if (info.Features & NUdf::EDataTypeFeatures::TzDateType) {
+ const auto ltz = GetterForTimezone(context, lv, block);
+ const auto rtz = GetterForTimezone(context, rv, block);
+
+ const auto one = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, lhs, rhs, "one", block);
+ const auto two = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, ltz, rtz, "two", block);
+
+ return BinaryOperator::CreateAnd(one, two, "and", block);
+ }
+
+ return nullptr;
+}
+
+template <>
+Value* GenEqualsFunction<true>(NUdf::EDataSlot slot, Value* lv, Value* rv, TCodegenContext& ctx, BasicBlock*& block) {
+ auto& context = ctx.Codegen.GetContext();
+
+ const auto tiny = BasicBlock::Create(context, "tiny", ctx.Func);
+ const auto test = BasicBlock::Create(context, "test", ctx.Func);
+ const auto done = BasicBlock::Create(context, "done", ctx.Func);
+
+ const auto res = PHINode::Create(Type::getInt1Ty(context), 2U, "result", done);
+
+ const auto le = IsEmpty(lv, block);
+ const auto re = IsEmpty(rv, block);
+
+ const auto any = BinaryOperator::CreateOr(le, re, "or", block);
+
+ BranchInst::Create(tiny, test, any, block);
+
+ block = tiny;
+
+ const auto both = BinaryOperator::CreateAnd(le, re, "and", block);
+ res->addIncoming(both, block);
+ BranchInst::Create(done, block);
+
+ block = test;
+
+ const auto comp = GenEqualsFunction<false>(slot, lv, rv, ctx, block);
+ res->addIncoming(comp, block);
+ BranchInst::Create(done, block);
+
+ block = done;
+ return res;
+}
+
+Value* GenEqualsFunction(NUdf::EDataSlot slot, bool isOptional, Value* lv, Value* rv, TCodegenContext& ctx, BasicBlock*& block) {
+ return isOptional ? GenEqualsFunction<true>(slot, lv, rv, ctx, block) : GenEqualsFunction<false>(slot, lv, rv, ctx, block);
+}
+
+template <bool IsOptional>
+Value* GenCompareFunction(NUdf::EDataSlot slot, Value* lv, Value* rv, TCodegenContext& ctx, BasicBlock*& block);
+
+template <>
+Value* GenCompareFunction<false>(NUdf::EDataSlot slot, Value* lv, Value* rv, TCodegenContext& ctx, BasicBlock*& block) {
+ auto& context = ctx.Codegen.GetContext();
+
+ const auto& info = NUdf::GetDataTypeInfo(slot);
+
+ if ((info.Features & NUdf::EDataTypeFeatures::CommonType) && (info.Features & NUdf::EDataTypeFeatures::StringType || NUdf::EDataSlot::Uuid == slot || NUdf::EDataSlot::DyNumber == slot)) {
+ return CallBinaryUnboxedValueFunction(&MyCompareStrings, Type::getInt32Ty(context), lv, rv, ctx.Codegen, block);
+ }
+
+ const bool extra = info.Features & (NUdf::EDataTypeFeatures::FloatType | NUdf::EDataTypeFeatures::TzDateType);
+ const auto resultType = Type::getInt32Ty(context);
+
+ const auto exit = BasicBlock::Create(context, "exit", ctx.Func);
+ const auto test = BasicBlock::Create(context, "test", ctx.Func);
+
+ const auto res = PHINode::Create(resultType, extra ? 3U : 2U, "result", exit);
+
+ const auto lhs = GetterFor(slot, lv, context, block);
+ const auto rhs = GetterFor(slot, rv, context, block);
+
+ if (info.Features & NUdf::EDataTypeFeatures::FloatType) {
+ const auto more = BasicBlock::Create(context, "more", ctx.Func);
+ const auto next = BasicBlock::Create(context, "next", ctx.Func);
+
+ const auto uno = CmpInst::Create(Instruction::FCmp, FCmpInst::FCMP_UNO, lhs, rhs, "unorded", block);
+
+ BranchInst::Create(more, next, uno, block);
+ block = more;
+
+ const auto luno = CmpInst::Create(Instruction::FCmp, FCmpInst::FCMP_UNO, ConstantFP::get(lhs->getType(), 0.0), lhs, "luno", block);
+ const auto runo = CmpInst::Create(Instruction::FCmp, FCmpInst::FCMP_UNO, ConstantFP::get(rhs->getType(), 0.0), rhs, "runo", block);
+ const auto once = BinaryOperator::CreateXor(luno, runo, "xor", block);
+
+ const auto left = SelectInst::Create(luno, ConstantInt::get(resultType, 1), ConstantInt::get(resultType, -1), "left", block);
+ const auto both = SelectInst::Create(once, left, ConstantInt::get(resultType, 0), "both", block);
+
+ res->addIncoming(both, block);
+ BranchInst::Create(exit, block);
+
+ block = next;
+ }
+
+ const auto equals = info.Features & NUdf::EDataTypeFeatures::FloatType ?
+ CmpInst::Create(Instruction::FCmp, FCmpInst::FCMP_OEQ, lhs, rhs, "equals", block):
+ CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, lhs, rhs, "equals", block);
+
+ if (info.Features & NUdf::EDataTypeFeatures::TzDateType) {
+ const auto more = BasicBlock::Create(context, "more", ctx.Func);
+ const auto next = BasicBlock::Create(context, "next", ctx.Func);
+
+ BranchInst::Create(more, test, equals, block);
+
+ block = more;
+
+ const auto ltz = GetterForTimezone(context, lv, block);
+ const auto rtz = GetterForTimezone(context, rv, block);
+ const auto tzeq = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, ltz, rtz, "tzeq", block);
+ res->addIncoming(ConstantInt::get(resultType, 0), block);
+ BranchInst::Create(exit, next, tzeq, block);
+
+ block = next;
+ const auto tzlt = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_ULT, ltz, rtz, "tzlt", block);
+ const auto tzout = SelectInst::Create(tzlt, ConstantInt::get(resultType, -1), ConstantInt::get(resultType, 1), "tzout", block);
+ res->addIncoming(tzout, block);
+ BranchInst::Create(exit, block);
+ } else {
+ res->addIncoming(ConstantInt::get(resultType, 0), block);
+ BranchInst::Create(exit, test, equals, block);
+ }
+
+ block = test;
+
+ const auto less = info.Features & NUdf::EDataTypeFeatures::FloatType ?
+ CmpInst::Create(Instruction::FCmp, FCmpInst::FCMP_OLT, lhs, rhs, "less", block): // float
+ info.Features & (NUdf::EDataTypeFeatures::SignedIntegralType | NUdf::EDataTypeFeatures::TimeIntervalType | NUdf::EDataTypeFeatures::DecimalType) ?
+ CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_SLT, lhs, rhs, "less", block): // signed
+ info.Features & (NUdf::EDataTypeFeatures::UnsignedIntegralType | NUdf::EDataTypeFeatures::DateType | NUdf::EDataTypeFeatures::TzDateType) ?
+ CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_ULT, lhs, rhs, "less", block): // unsigned
+ rhs; // bool
+
+ const auto out = SelectInst::Create(less, ConstantInt::get(resultType, -1), ConstantInt::get(resultType, 1), "out", block);
+ res->addIncoming(out, block);
+ BranchInst::Create(exit, block);
+
+ block = exit;
+ return res;
+}
+
+template <>
+Value* GenCompareFunction<true>(NUdf::EDataSlot slot, Value* lv, Value* rv, TCodegenContext& ctx, BasicBlock*& block) {
+ auto& context = ctx.Codegen.GetContext();
+
+ const auto tiny = BasicBlock::Create(context, "tiny", ctx.Func);
+ const auto side = BasicBlock::Create(context, "side", ctx.Func);
+ const auto test = BasicBlock::Create(context, "test", ctx.Func);
+ const auto done = BasicBlock::Create(context, "done", ctx.Func);
+
+ const auto resultType = Type::getInt32Ty(context);
+ const auto res = PHINode::Create(resultType, 3U, "result", done);
+
+ const auto le = IsEmpty(lv, block);
+ const auto re = IsEmpty(rv, block);
+
+ const auto any = BinaryOperator::CreateOr(le, re, "or", block);
+
+ BranchInst::Create(tiny, test, any, block);
+
+ block = tiny;
+
+ const auto both = BinaryOperator::CreateAnd(le, re, "and", block);
+ res->addIncoming(ConstantInt::get(resultType, 0), block);
+ BranchInst::Create(done, side, both, block);
+
+ block = side;
+
+ const auto out = SelectInst::Create(le, ConstantInt::get(resultType, -1), ConstantInt::get(resultType, 1), "out", block);
+ res->addIncoming(out, block);
+ BranchInst::Create(done, block);
+
+ block = test;
+
+ const auto comp = GenCompareFunction<false>(slot, lv, rv, ctx, block);
+ res->addIncoming(comp, block);
+ BranchInst::Create(done, block);
+
+ block = done;
+ return res;
+}
+
+Value* GenCompareFunction(NUdf::EDataSlot slot, bool isOptional, Value* lv, Value* rv, TCodegenContext& ctx, BasicBlock*& block) {
+ return isOptional ? GenCompareFunction<true>(slot, lv, rv, ctx, block) : GenCompareFunction<false>(slot, lv, rv, ctx, block);
+}
+
+Value* GenCombineHashes(Value* first, Value* second, BasicBlock* block) {
+// key += ~(key << 32);
+ const auto x01 = BinaryOperator::CreateShl(first, ConstantInt::get(first->getType(), 32), "x01", block);
+ const auto x02 = BinaryOperator::CreateXor(x01, ConstantInt::get(x01->getType(), ~0), "x02", block);
+ const auto x03 = BinaryOperator::CreateAdd(x02, first, "x03", block);
+// key ^= (key >> 22);
+ const auto x04 = BinaryOperator::CreateLShr(x03, ConstantInt::get(x03->getType(), 22), "x04", block);
+ const auto x05 = BinaryOperator::CreateXor(x04, x03, "x05", block);
+// key += ~(key << 13);
+ const auto x06 = BinaryOperator::CreateShl(x05, ConstantInt::get(x05->getType(), 13), "x06", block);
+ const auto x07 = BinaryOperator::CreateXor(x06, ConstantInt::get(x06->getType(), ~0), "x07", block);
+ const auto x08 = BinaryOperator::CreateAdd(x05, x07, "x08", block);
+// key ^= (key >> 8);
+ const auto x09 = BinaryOperator::CreateLShr(x08, ConstantInt::get(x08->getType(), 8), "x09", block);
+ const auto x10 = BinaryOperator::CreateXor(x08, x09, "x10", block);
+// key += (key << 3);
+ const auto x11 = BinaryOperator::CreateShl(x10, ConstantInt::get(x10->getType(), 3), "x11", block);
+ const auto x12 = BinaryOperator::CreateAdd(x10, x11, "x12", block);
+// key ^= (key >> 15);
+ const auto x13 = BinaryOperator::CreateLShr(x12, ConstantInt::get(x12->getType(), 15), "x13", block);
+ const auto x14 = BinaryOperator::CreateXor(x13, x12, "x14", block);
+// key += ~(key << 27);
+ const auto x15 = BinaryOperator::CreateShl(x14, ConstantInt::get(x14->getType(), 27), "x15", block);
+ const auto x16 = BinaryOperator::CreateXor(x15, ConstantInt::get(x15->getType(), ~0), "x16", block);
+ const auto x17 = BinaryOperator::CreateAdd(x14, x16, "x17", block);
+// key ^= (key >> 31);
+ const auto x18 = BinaryOperator::CreateLShr(x17, ConstantInt::get(x17->getType(), 31), "x18", block);
+ const auto x19 = BinaryOperator::CreateXor(x17, x18, "x19", block);
+
+ return BinaryOperator::CreateXor(x19, second, "both", block);
+}
+
+template <bool IsOptional>
+Value* GenHashFunction(NUdf::EDataSlot slot, Value* value, TCodegenContext& ctx, BasicBlock*& block);
+
+template <>
+Value* GenHashFunction<false>(NUdf::EDataSlot slot, Value* value, TCodegenContext& ctx, BasicBlock*& block) {
+ auto& context = ctx.Codegen.GetContext();
+
+ const auto& info = NUdf::GetDataTypeInfo(slot);
+
+ if ((info.Features & NUdf::EDataTypeFeatures::CommonType) && (info.Features & NUdf::EDataTypeFeatures::StringType || NUdf::EDataSlot::Uuid == slot || NUdf::EDataSlot::DyNumber == slot)) {
+ return CallUnaryUnboxedValueFunction(&MyHashString, Type::getInt64Ty(context), value, ctx.Codegen, block);
+ }
+
+ const auto val = GetterFor(slot, value, context, block);
+ const auto hashType = Type::getInt64Ty(context);
+
+ if (info.Features & (NUdf::EDataTypeFeatures::IntegralType | NUdf::EDataTypeFeatures::DateType | NUdf::EDataTypeFeatures::TimeIntervalType) || NUdf::EDataSlot::Bool == slot) {
+ if (val->getType() == hashType) {
+ return val;
+ }
+ const auto ext = CastInst::Create(Instruction::ZExt, val, hashType, "ext", block);
+ return ext;
+ }
+
+ if (info.Features & NUdf::EDataTypeFeatures::FloatType) {
+ const auto nan = CmpInst::Create(Instruction::FCmp, FCmpInst::FCMP_UNO, val, val, "nan", block);
+ const auto zero = CmpInst::Create(Instruction::FCmp, FCmpInst::FCMP_OEQ, val, ConstantFP::get(val->getType(), 0), "zero", block);
+ if (NUdf::EDataSlot::Float == slot) {
+ const auto cast = CastInst::Create(Instruction::BitCast, val, Type::getInt32Ty(context), "cast", block);
+ const auto ext = CastInst::Create(Instruction::ZExt, cast, hashType, "ext", block);
+ const auto first = SelectInst::Create(nan, ConstantInt::get(hashType, ~0), ext, "first", block);
+ const auto second = SelectInst::Create(zero, ConstantInt::get(hashType, 0), first, "second", block);
+ return second;
+ } else {
+ const auto cast = CastInst::Create(Instruction::BitCast, val, hashType, "cast", block);
+ const auto first = SelectInst::Create(nan, ConstantInt::get(hashType, ~0), cast, "first", block);
+ const auto second = SelectInst::Create(zero, ConstantInt::get(hashType, 0), first, "second", block);
+ return second;
+ }
+ }
+
+ if (info.Features & NUdf::EDataTypeFeatures::TzDateType) {
+ const auto tz = GetterForTimezone(context, value, block);
+ const auto ext = val->getType() == hashType ? val : CastInst::Create(Instruction::ZExt, val, hashType, "ext", block);
+ const auto etz = CastInst::Create(Instruction::ZExt, tz, hashType, "etz", block);
+
+ return GenCombineHashes(ext, etz, block);
+ }
+
+ if (info.Features & NUdf::EDataTypeFeatures::DecimalType) {
+ const auto low = CastInst::Create(Instruction::Trunc, val, hashType, "low", block);
+ const auto lshr = BinaryOperator::CreateLShr(val, ConstantInt::get(val->getType(), 64), "lshr", block);
+ const auto high = CastInst::Create(Instruction::Trunc, lshr, hashType, "high", block);
+ return GenCombineHashes(low, high, block);
+ }
+
+ return nullptr;
+}
+
+template <>
+Value* GenHashFunction<true>(NUdf::EDataSlot slot, Value* value, TCodegenContext& ctx, BasicBlock*& block) {
+ auto& context = ctx.Codegen.GetContext();
+
+ const auto tiny = BasicBlock::Create(context, "tiny", ctx.Func);
+ const auto test = BasicBlock::Create(context, "test", ctx.Func);
+ const auto done = BasicBlock::Create(context, "done", ctx.Func);
+
+ const auto res = PHINode::Create(Type::getInt64Ty(context), 2U, "result", done);
+
+ BranchInst::Create(tiny, test, IsEmpty(value, block), block);
+
+ block = tiny;
+
+ res->addIncoming(ConstantInt::get(Type::getInt64Ty(context), ~0ULL), block);
+ BranchInst::Create(done, block);
+
+ block = test;
+
+ const auto comp = GenHashFunction<false>(slot, value, ctx, block);
+ res->addIncoming(comp, block);
+ BranchInst::Create(done, block);
+
+ block = done;
+ return res;
+}
+
+Value* GenHashFunction(NUdf::EDataSlot slot, bool isOptional, Value* value, TCodegenContext& ctx, BasicBlock*& block) {
+ return isOptional ? GenHashFunction<true>(slot, value, ctx, block) : GenHashFunction<false>(slot, value, ctx, block);
+}
+
+Value* LoadIfPointer(Value* value, BasicBlock* block) {
+ return value->getType()->isPointerTy() ? new LoadInst(value->getType()->getPointerElementType(), value, "load_value", block) : value;
+}
+
+}
+
+Function* GenerateEqualsFunction(NYql::NCodegen::ICodegen& codegen, const TString& name, bool isTuple, const TKeyTypes& types) {
+ auto& module = codegen.GetModule();
+ if (const auto f = module.getFunction(name.c_str()))
+ return f;
+
+ auto& context = codegen.GetContext();
+ const auto valueType = Type::getInt128Ty(context);
+ const auto ptrType = PointerType::getUnqual(valueType);
+ const auto returnType = Type::getInt1Ty(context);
+
+ const auto funcType = codegen.GetEffectiveTarget() != NYql::NCodegen::ETarget::Windows ?
+ FunctionType::get(returnType, {valueType, valueType}, false):
+ FunctionType::get(returnType, {ptrType, ptrType}, false);
+
+ TCodegenContext ctx(codegen);
+ ctx.AlwaysInline = true;
+ ctx.Func = cast<Function>(module.getOrInsertFunction(name.c_str(), funcType).getCallee());
+
+ DISubprogramAnnotator annotator(ctx, ctx.Func);
+
+
+ auto args = ctx.Func->arg_begin();
+
+ const auto main = BasicBlock::Create(context, "main", ctx.Func);
+ auto block = main;
+
+ const auto lv = LoadIfPointer(&*args, block);
+ const auto rv = LoadIfPointer(&*++args, block);
+
+ if (isTuple) {
+ if (types.empty()) {
+ ReturnInst::Create(context, ConstantInt::getTrue(context), block);
+ return ctx.Func;
+ }
+
+ const auto fast = BasicBlock::Create(context, "fast", ctx.Func);
+ const auto slow = BasicBlock::Create(context, "slow", ctx.Func);
+ const auto stop = BasicBlock::Create(context, "stop", ctx.Func);
+
+ const auto elementsType = ArrayType::get(valueType, types.size());
+ const auto elementsPtrType = PointerType::getUnqual(elementsType);
+ const auto elementsPtrOne = CallBoxedValueVirtualMethod<NUdf::TBoxedValueAccessor::EMethod::GetElements>(elementsPtrType, lv, ctx.Codegen, block);
+ const auto elementsPtrTwo = CallBoxedValueVirtualMethod<NUdf::TBoxedValueAccessor::EMethod::GetElements>(elementsPtrType, rv, ctx.Codegen, block);
+
+ const auto goodOne = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_NE, elementsPtrOne, ConstantPointerNull::get(elementsPtrType), "good_one", block);
+ const auto goodTwo = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_NE, elementsPtrTwo, ConstantPointerNull::get(elementsPtrType), "good_two", block);
+ const auto good = BinaryOperator::CreateAnd(goodOne, goodTwo, "good", block);
+
+ BranchInst::Create(fast, slow, good, block);
+
+ const auto last = types.size() - 1U;
+
+ {
+ block = fast;
+
+ const auto elementsOne = new LoadInst(elementsType, elementsPtrOne, "elements_one", block);
+ const auto elementsTwo = new LoadInst(elementsType, elementsPtrTwo, "elements_two", block);
+
+ for (ui32 i = 0U; i < last; ++i) {
+ const auto nextOne = ExtractValueInst::Create(elementsOne, i, (TString("next_one_") += ToString(i)).c_str(), block);
+ const auto nextTwo = ExtractValueInst::Create(elementsTwo, i, (TString("next_two_") += ToString(i)).c_str(), block);
+
+ const auto step = BasicBlock::Create(context, (TString("step") += ToString(i)).c_str(), ctx.Func);
+
+ const auto test = GenEqualsFunction(types[i].first, types[i].second, nextOne, nextTwo, ctx, block);
+
+ BranchInst::Create(step, stop, test, block);
+
+ block = step;
+ }
+
+ const auto backOne = ExtractValueInst::Create(elementsOne, last, "back_one", block);
+ const auto backTwo = ExtractValueInst::Create(elementsTwo, last, "back_two", block);
+
+ const auto result = GenEqualsFunction(types.back().first, types.back().second, backOne, backTwo, ctx, block);
+ ReturnInst::Create(context, result, block);
+ }
+
+ {
+ block = slow;
+
+ const auto elementOne = new AllocaInst(valueType, 0U, "element_one", block);
+ const auto elementTwo = new AllocaInst(valueType, 0U, "element_two", block);
+
+ const auto indexType = Type::getInt32Ty(context);
+
+ for (ui32 i = 0U; i < last; ++i) {
+ CallBoxedValueVirtualMethod<NUdf::TBoxedValueAccessor::EMethod::GetElement>(elementOne, lv, ctx.Codegen, block, ConstantInt::get(indexType, i));
+ CallBoxedValueVirtualMethod<NUdf::TBoxedValueAccessor::EMethod::GetElement>(elementTwo, rv, ctx.Codegen, block, ConstantInt::get(indexType, i));
+
+ const auto nextOne = new LoadInst(valueType, elementOne, (TString("next_one_") += ToString(i)).c_str(), block);
+ const auto nextTwo = new LoadInst(valueType, elementTwo, (TString("next_two_") += ToString(i)).c_str(), block);
+
+ if (NUdf::GetDataTypeInfo(types[i].first).Features & NUdf::EDataTypeFeatures::StringType) {
+ ValueRelease(EValueRepresentation::String, nextOne, ctx, block);
+ ValueRelease(EValueRepresentation::String, nextTwo, ctx, block);
+ }
+
+ const auto step = BasicBlock::Create(context, (TString("step") += ToString(i)).c_str(), ctx.Func);
+ const auto test = GenEqualsFunction(types[i].first, types[i].second, nextOne, nextTwo, ctx, block);
+
+ BranchInst::Create(step, stop, test, block);
+
+ block = step;
+ }
+
+ CallBoxedValueVirtualMethod<NUdf::TBoxedValueAccessor::EMethod::GetElement>(elementOne, lv, ctx.Codegen, block, ConstantInt::get(indexType, last));
+ CallBoxedValueVirtualMethod<NUdf::TBoxedValueAccessor::EMethod::GetElement>(elementTwo, rv, ctx.Codegen, block, ConstantInt::get(indexType, last));
+
+ const auto backOne = new LoadInst(valueType, elementOne, "back_one", block);
+ const auto backTwo = new LoadInst(valueType, elementTwo, "back_two", block);
+
+ if (NUdf::GetDataTypeInfo(types.back().first).Features & NUdf::EDataTypeFeatures::StringType) {
+ ValueRelease(EValueRepresentation::String, backOne, ctx, block);
+ ValueRelease(EValueRepresentation::String, backTwo, ctx, block);
+ }
+
+ const auto result = GenEqualsFunction(types.back().first, types.back().second, backOne, backTwo, ctx, block);
+ ReturnInst::Create(context, result, block);
+ }
+
+ block = stop;
+ ReturnInst::Create(context, ConstantInt::getFalse(context), block);
+
+ } else {
+ const auto result = GenEqualsFunction(types.front().first, types.front().second, lv, rv, ctx, block);
+ ReturnInst::Create(context, result, block);
+ }
+
+ return ctx.Func;
+}
+
+Function* GenerateHashFunction(NYql::NCodegen::ICodegen& codegen, const TString& name, bool isTuple, const TKeyTypes& types) {
+ auto& module = codegen.GetModule();
+ if (const auto f = module.getFunction(name.c_str()))
+ return f;
+
+ auto& context = codegen.GetContext();
+ const auto valueType = Type::getInt128Ty(context);
+ const auto ptrType = PointerType::getUnqual(valueType);
+ const auto returnType = Type::getInt64Ty(context);
+
+ const auto funcType = codegen.GetEffectiveTarget() != NYql::NCodegen::ETarget::Windows ?
+ FunctionType::get(returnType, {valueType}, false):
+ FunctionType::get(returnType, {ptrType}, false);
+
+ TCodegenContext ctx(codegen);
+ ctx.AlwaysInline = true;
+ ctx.Func = cast<Function>(module.getOrInsertFunction(name.c_str(), funcType).getCallee());
+
+ DISubprogramAnnotator annotator(ctx, ctx.Func);
+
+
+ const auto main = BasicBlock::Create(context, "main", ctx.Func);
+ auto block = main;
+
+ const auto arg = LoadIfPointer(&*ctx.Func->arg_begin(), block);
+
+ if (isTuple) {
+ if (types.empty()) {
+ ReturnInst::Create(context, ConstantInt::get(returnType, 0), block);
+ return ctx.Func;
+ }
+
+ const auto fast = BasicBlock::Create(context, "fast", ctx.Func);
+ const auto slow = BasicBlock::Create(context, "slow", ctx.Func);
+
+ const auto elementsType = ArrayType::get(valueType, types.size());
+ const auto elementsPtrType = PointerType::getUnqual(elementsType);
+ const auto elementsPtr = CallBoxedValueVirtualMethod<NUdf::TBoxedValueAccessor::EMethod::GetElements>(elementsPtrType, arg, ctx.Codegen, block);
+
+ const auto null = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, elementsPtr, ConstantPointerNull::get(elementsPtrType), "null", block);
+
+ BranchInst::Create(slow, fast, null, block);
+
+ {
+ block = fast;
+
+ const auto elements = new LoadInst(elementsType, elementsPtr, "elements", block);
+
+ auto result = static_cast<Value*>(ConstantInt::get(returnType, 0));
+
+ for (auto i = 0U; i < types.size(); ++i) {
+ const auto next = ExtractValueInst::Create(elements, i, (TString("next_") += ToString(i)).c_str(), block);
+
+ const auto plus = GenHashFunction(types[i].first, types[i].second, next, ctx, block);
+
+ result = GenCombineHashes(result, plus, block);
+ }
+
+ ReturnInst::Create(context, result, block);
+ }
+
+ {
+ block = slow;
+
+ const auto element = new AllocaInst(valueType, 0U, "element", block);
+
+ auto result = static_cast<Value*>(ConstantInt::get(returnType, 0));
+
+ for (auto i = 0U; i < types.size(); ++i) {
+ CallBoxedValueVirtualMethod<NUdf::TBoxedValueAccessor::EMethod::GetElement>(element, arg, ctx.Codegen, block, ConstantInt::get(Type::getInt32Ty(context), i));
+
+ const auto next = new LoadInst(valueType, element, (TString("next_") += ToString(i)).c_str(), block);
+ if (NUdf::GetDataTypeInfo(types[i].first).Features & NUdf::EDataTypeFeatures::StringType) {
+ ValueRelease(EValueRepresentation::String, next, ctx, block);
+ }
+
+ const auto plus = GenHashFunction(types[i].first, types[i].second, next, ctx, block);
+
+ result = GenCombineHashes(result, plus, block);
+ }
+
+ ReturnInst::Create(context, result, block);
+ }
+
+ } else {
+ const auto result = GenHashFunction(types.front().first, types.front().second, arg, ctx, block);
+ ReturnInst::Create(context, result, block);
+ }
+
+ return ctx.Func;
+}
+
+Function* GenerateEqualsFunction(NYql::NCodegen::ICodegen& codegen, const TString& name, const TKeyTypes& types) {
+ auto& module = codegen.GetModule();
+ if (const auto f = module.getFunction(name.c_str()))
+ return f;
+
+ auto& context = codegen.GetContext();
+ const auto valueType = Type::getInt128Ty(context);
+ const auto elementsType = ArrayType::get(valueType, types.size());
+ const auto ptrType = PointerType::getUnqual(elementsType);
+ const auto returnType = Type::getInt1Ty(context);
+
+ const auto funcType = FunctionType::get(returnType, {ptrType, ptrType}, false);
+
+ TCodegenContext ctx(codegen);
+ ctx.AlwaysInline = true;
+ ctx.Func = cast<Function>(module.getOrInsertFunction(name.c_str(), funcType).getCallee());
+
+ DISubprogramAnnotator annotator(ctx, ctx.Func);
+
+
+ auto args = ctx.Func->arg_begin();
+
+ const auto main = BasicBlock::Create(context, "main", ctx.Func);
+ auto block = main;
+
+ const auto lv = &*args;
+ const auto rv = &*++args;
+
+ if (types.empty()) {
+ ReturnInst::Create(context, ConstantInt::getTrue(context), block);
+ return ctx.Func;
+ }
+
+ const auto elementsOne = new LoadInst(elementsType, lv, "elements_one", block);
+ const auto elementsTwo = new LoadInst(elementsType, rv, "elements_two", block);
+
+ const auto stop = BasicBlock::Create(context, "stop", ctx.Func);
+ ReturnInst::Create(context, ConstantInt::getFalse(context), stop);
+
+ const auto last = types.size() - 1U;
+ for (ui32 i = 0U; i < last; ++i) {
+ const auto nextOne = ExtractValueInst::Create(elementsOne, i, (TString("next_one_") += ToString(i)).c_str(), block);
+ const auto nextTwo = ExtractValueInst::Create(elementsTwo, i, (TString("next_two_") += ToString(i)).c_str(), block);
+
+ const auto step = BasicBlock::Create(context, (TString("step_") += ToString(i)).c_str(), ctx.Func);
+
+ const auto test = GenEqualsFunction(types[i].first, types[i].second, nextOne, nextTwo, ctx, block);
+
+ BranchInst::Create(step, stop, test, block);
+
+ block = step;
+ }
+
+ const auto backOne = ExtractValueInst::Create(elementsOne, last, "back_one", block);
+ const auto backTwo = ExtractValueInst::Create(elementsTwo, last, "back_two", block);
+
+ const auto result = GenEqualsFunction(types.back().first, types.back().second, backOne, backTwo, ctx, block);
+ ReturnInst::Create(context, result, block);
+
+ return ctx.Func;
+}
+
+Function* GenerateHashFunction(NYql::NCodegen::ICodegen& codegen, const TString& name, const TKeyTypes& types) {
+ auto& module = codegen.GetModule();
+ if (const auto f = module.getFunction(name.c_str()))
+ return f;
+
+ auto& context = codegen.GetContext();
+ const auto valueType = Type::getInt128Ty(context);
+ const auto elementsType = ArrayType::get(valueType, types.size());
+ const auto ptrType = PointerType::getUnqual(elementsType);
+ const auto returnType = Type::getInt64Ty(context);
+
+ const auto funcType = FunctionType::get(returnType, {ptrType}, false);
+
+ TCodegenContext ctx(codegen);
+ ctx.AlwaysInline = true;
+ ctx.Func = cast<Function>(module.getOrInsertFunction(name.c_str(), funcType).getCallee());
+
+ DISubprogramAnnotator annotator(ctx, ctx.Func);
+
+
+ const auto main = BasicBlock::Create(context, "main", ctx.Func);
+ auto block = main;
+
+ if (types.empty()) {
+ ReturnInst::Create(context, ConstantInt::get(returnType, 0), block);
+ return ctx.Func;
+ }
+
+ const auto arg = &*ctx.Func->arg_begin();
+ const auto elements = new LoadInst(elementsType, arg, "elements", block);
+
+ if (types.size() > 1U) {
+ auto result = static_cast<Value*>(ConstantInt::get(returnType, 0));
+
+ for (auto i = 0U; i < types.size(); ++i) {
+ const auto item = ExtractValueInst::Create(elements, i, (TString("item_") += ToString(i)).c_str(), block);
+
+ const auto plus = GenHashFunction(types[i].first, types[i].second, item, ctx, block);
+
+ result = GenCombineHashes(result, plus, block);
+ }
+
+ ReturnInst::Create(context, result, block);
+ } else {
+ const auto value = ExtractValueInst::Create(elements, 0, "value", block);
+ const auto result = GenHashFunction(types.front().first, types.front().second, value, ctx, block);
+ ReturnInst::Create(context, result, block);
+ }
+
+ return ctx.Func;
+}
+
+Function* GenerateCompareFunction(NYql::NCodegen::ICodegen& codegen, const TString& name, const TKeyTypes& types) {
+ auto& module = codegen.GetModule();
+ if (const auto f = module.getFunction(name.c_str()))
+ return f;
+
+ auto& context = codegen.GetContext();
+ const auto valueType = Type::getInt128Ty(context);
+ const auto elementsType = ArrayType::get(valueType, types.size());
+ const auto ptrType = PointerType::getUnqual(elementsType);
+ const auto dirsType = ArrayType::get(Type::getInt1Ty(context), types.size());
+ const auto ptrDirsType = PointerType::getUnqual(dirsType);
+ const auto returnType = Type::getInt32Ty(context);
+
+ const auto funcType = FunctionType::get(returnType, {ptrDirsType, ptrType, ptrType}, false);
+
+ TCodegenContext ctx(codegen);
+ ctx.AlwaysInline = true;
+ ctx.Func = cast<Function>(module.getOrInsertFunction(name.c_str(), funcType).getCallee());
+
+ DISubprogramAnnotator annotator(ctx, ctx.Func);
+
+
+ auto args = ctx.Func->arg_begin();
+
+ const auto main = BasicBlock::Create(context, "main", ctx.Func);
+ auto block = main;
+
+ const auto dp = &*args;
+ const auto lv = &*++args;
+ const auto rv = &*++args;
+
+ if (types.empty()) {
+ ReturnInst::Create(context, ConstantInt::get(returnType, 0), block);
+ return ctx.Func;
+ }
+
+ const auto directions = new LoadInst(dirsType, dp, "directions", block);
+ const auto elementsOne = new LoadInst(elementsType, lv, "elements_one", block);
+ const auto elementsTwo = new LoadInst(elementsType, rv, "elements_two", block);
+ const auto zero = ConstantInt::get(returnType, 0);
+
+ for (auto i = 0U; i < types.size(); ++i) {
+ const auto nextOne = ExtractValueInst::Create(elementsOne, i, (TString("next_one_") += ToString(i)).c_str(), block);
+ const auto nextTwo = ExtractValueInst::Create(elementsTwo, i, (TString("next_two_") += ToString(i)).c_str(), block);
+
+ const auto exit = BasicBlock::Create(context, (TString("exit_") += ToString(i)).c_str(), ctx.Func);
+ const auto step = BasicBlock::Create(context, (TString("step_") += ToString(i)).c_str(), ctx.Func);
+
+ const auto test = GenCompareFunction(types[i].first, types[i].second, nextOne, nextTwo, ctx, block);
+ const auto skip = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, zero, test, (TString("skip_") += ToString(i)).c_str(), block);
+
+ BranchInst::Create(step, exit, skip, block);
+
+ block = exit;
+
+ const auto dir = ExtractValueInst::Create(directions, i, (TString("dir_") += ToString(i)).c_str(), block);
+ const auto neg = BinaryOperator::CreateNeg(test, (TString("neg_") += ToString(i)).c_str(), block);
+ const auto out = SelectInst::Create(dir, test, neg, (TString("neg_") += ToString(i)).c_str(), block);
+
+ ReturnInst::Create(context, out, block);
+
+ block = step;
+ }
+
+ ReturnInst::Create(context, zero, block);
+ return ctx.Func;
+}
+
+void GenInvalidate(const TCodegenContext& ctx, const std::vector<std::pair<ui32, EValueRepresentation>>& invalidationSet, BasicBlock*& block) {
+ auto& context = ctx.Codegen.GetContext();
+ const auto indexType = Type::getInt32Ty(context);
+ const auto valueType = Type::getInt128Ty(context);
+ const auto values = ctx.GetMutables();
+
+ for (const auto& index : invalidationSet) {
+ const auto invPtr = GetElementPtrInst::CreateInBounds(valueType, values, {ConstantInt::get(indexType, index.first)}, "inv_ptr", block);
+ ValueUnRef(index.second, invPtr, ctx, block);
+ new StoreInst(GetInvalid(context), invPtr, block);
+ }
+}
+
+TUnboxedImmutableCodegeneratorNode::TUnboxedImmutableCodegeneratorNode(TMemoryUsageInfo* memInfo, NUdf::TUnboxedValue&& value)
+ : TUnboxedImmutableComputationNode(memInfo, std::move(value))
+{}
+
+Value* TUnboxedImmutableCodegeneratorNode::CreateGetValue(const TCodegenContext& ctx, BasicBlock*&) const {
+ return ConstantInt::get(Type::getInt128Ty(ctx.Codegen.GetContext()), APInt(128, 2, reinterpret_cast<const uint64_t*>(&UnboxedValue)));
+}
+
+TExternalCodegeneratorNode::TExternalCodegeneratorNode(TComputationMutables& mutables, EValueRepresentation kind)
+ : TExternalComputationNode(mutables, kind)
+{}
+
+TExternalCodegeneratorRootNode::TExternalCodegeneratorRootNode(TComputationMutables& mutables, EValueRepresentation kind)
+ : TExternalCodegeneratorNode(mutables, kind)
+{}
+
+NUdf::TUnboxedValue TExternalCodegeneratorRootNode::GetValue(TComputationContext& compCtx) const {
+ if (compCtx.ExecuteLLVM && GetFunction)
+ return GetFunction(&compCtx);
+ return TExternalComputationNode::GetValue(compCtx);
+}
+
+void TExternalCodegeneratorRootNode::SetValue(TComputationContext& compCtx, NUdf::TUnboxedValue&& newValue) const {
+ if (compCtx.ExecuteLLVM && SetFunction)
+ return SetFunction(&compCtx, newValue.Release());
+
+ TExternalComputationNode::SetValue(compCtx, std::move(newValue));
+}
+
+TString TExternalCodegeneratorRootNode::MakeName(const TString& method) const {
+ TStringStream out;
+ out << DebugString() << "::" << method << "_(" << static_cast<const void*>(this) << ").";
+ return out.Str();
+}
+
+void TExternalCodegeneratorRootNode::FinalizeFunctions(NYql::NCodegen::ICodegen& codegen) {
+ if (GetValueFunc)
+ GetFunction = reinterpret_cast<TGetPtr>(codegen.GetPointerToFunction(GetValueFunc));
+
+ if (SetValueFunc)
+ SetFunction = reinterpret_cast<TSetPtr>(codegen.GetPointerToFunction(SetValueFunc));
+}
+
+void TExternalCodegeneratorRootNode::GenerateFunctions(NYql::NCodegen::ICodegen& codegen) {
+ GetValueFunc = GenerateGetValue(codegen);
+ SetValueFunc = GenerateSetValue(codegen);
+ codegen.ExportSymbol(GetValueFunc);
+ codegen.ExportSymbol(SetValueFunc);
+}
+
+Function* TExternalCodegeneratorRootNode::GenerateGetValue(NYql::NCodegen::ICodegen& codegen) {
+ auto& module = codegen.GetModule();
+ auto& context = codegen.GetContext();
+ const auto& name = MakeName("Get");
+ if (const auto f = module.getFunction(name.c_str()))
+ return f;
+
+ const auto valueType = Type::getInt128Ty(context);
+ const auto contextType = GetCompContextType(context);
+
+ const auto funcType = codegen.GetEffectiveTarget() != NYql::NCodegen::ETarget::Windows ?
+ FunctionType::get(valueType, {PointerType::getUnqual(contextType)}, false):
+ FunctionType::get(Type::getVoidTy(context), {PointerType::getUnqual(valueType), PointerType::getUnqual(contextType)}, false);
+
+ TCodegenContext ctx(codegen);
+ ctx.Func = cast<Function>(module.getOrInsertFunction(name.c_str(), funcType).getCallee());
+
+ DISubprogramAnnotator annotator(ctx, ctx.Func);
+
+
+ auto args = ctx.Func->arg_begin();
+ if (codegen.GetEffectiveTarget() == NYql::NCodegen::ETarget::Windows) {
+ auto& firstArg = *args++;
+ firstArg.addAttr(Attribute::StructRet);
+ firstArg.addAttr(Attribute::NoAlias);
+ }
+
+ auto main = BasicBlock::Create(context, "main", ctx.Func);
+ ctx.Ctx = &*args;
+ ctx.Ctx->addAttr(Attribute::NonNull);
+
+ const auto get = CreateGetValue(ctx, main);
+
+ if (codegen.GetEffectiveTarget() != NYql::NCodegen::ETarget::Windows) {
+ ReturnInst::Create(context, get, main);
+ } else {
+ new StoreInst(get, &*--args, main);
+ ReturnInst::Create(context, main);
+ }
+
+ return ctx.Func;
+}
+
+Function* TExternalCodegeneratorRootNode::GenerateSetValue(NYql::NCodegen::ICodegen& codegen) {
+ auto& module = codegen.GetModule();
+ auto& context = codegen.GetContext();
+ const auto& name = MakeName("Set");
+ if (const auto f = module.getFunction(name.c_str()))
+ return f;
+
+ const auto intType = Type::getInt128Ty(context);
+ const auto contextType = GetCompContextType(context);
+ const auto valueType = codegen.GetEffectiveTarget() == NYql::NCodegen::ETarget::Windows ?
+ (Type*)PointerType::getUnqual(intType) : (Type*)intType;
+
+ const auto funcType = FunctionType::get(Type::getVoidTy(context), {PointerType::getUnqual(contextType), valueType}, false);
+ TCodegenContext ctx(codegen);
+ ctx.Func = cast<Function>(module.getOrInsertFunction(name.c_str(), funcType).getCallee());
+
+ DISubprogramAnnotator annotator(ctx, ctx.Func);
+
+
+ auto args = ctx.Func->arg_begin();
+
+ auto main = BasicBlock::Create(context, "main", ctx.Func);
+ ctx.Ctx = &*args;
+ ctx.Ctx->addAttr(Attribute::NonNull);
+
+ const auto valueArg = &*++args;
+
+ if (codegen.GetEffectiveTarget() == NYql::NCodegen::ETarget::Windows) {
+ const auto value = new LoadInst(valueArg->getType()->getPointerElementType(), valueArg, "load_value", main);
+ CreateSetValue(ctx, main, value);
+ } else {
+ CreateSetValue(ctx, main, valueArg);
+ }
+ ReturnInst::Create(context, main);
+ return ctx.Func;
+}
+
+Value* TExternalCodegeneratorNode::CreateGetValue(const TCodegenContext& ctx, BasicBlock*& block) const {
+ if (ValueGetterBuilder) {
+ llvm::Function * ValueGetter = ValueGetterBuilder(ctx);
+ return CallInst::Create(ValueGetter, {ctx.Ctx}, "getter", block);
+ }
+
+ if (ValueBuilder) {
+ llvm::Value * TemporaryValue = ValueBuilder(ctx);
+ return LoadIfPointer(TemporaryValue, block);
+ }
+
+ MKQL_ENSURE(!Getter, "Wrong LLVM function generation order.");
+ auto& context = ctx.Codegen.GetContext();
+ const auto indexType = Type::getInt32Ty(context);
+ const auto valueType = Type::getInt128Ty(context);
+ const auto valuePtr = GetElementPtrInst::CreateInBounds(valueType, ctx.GetMutables(), {ConstantInt::get(indexType, ValueIndex)}, "value_ptr", block);
+ const auto value = new LoadInst(valueType, valuePtr, "value", block);
+ return value;
+}
+
+Value* TExternalCodegeneratorNode::CreateRefValue(const TCodegenContext& ctx, BasicBlock*& block) const {
+ CreateInvalidate(ctx, block);
+
+ auto& context = ctx.Codegen.GetContext();
+ const auto indexType = Type::getInt32Ty(context);
+ const auto valueType = Type::getInt128Ty(context);
+ const auto values = ctx.GetMutables();
+ const auto valuePtr = GetElementPtrInst::CreateInBounds(valueType, values, {ConstantInt::get(indexType, ValueIndex)}, "value_ptr", block);
+ return valuePtr;
+}
+
+void TExternalCodegeneratorNode::CreateSetValue(const TCodegenContext& ctx, BasicBlock*& block, Value* value) const {
+ auto& context = ctx.Codegen.GetContext();
+ const auto indexType = Type::getInt32Ty(context);
+ const auto valueType = Type::getInt128Ty(context);
+ const auto values = ctx.GetMutables();
+ const auto valuePtr = GetElementPtrInst::CreateInBounds(valueType, values, {ConstantInt::get(indexType, ValueIndex)}, "value_ptr", block);
+
+
+ if (value->getType()->isPointerTy()) {
+ ValueUnRef(RepresentationKind, valuePtr, ctx, block);
+ const auto load = new LoadInst(valueType, value, "value", block);
+ new StoreInst(load, valuePtr, block);
+ new StoreInst(ConstantInt::get(load->getType(), 0), value, block);
+ } else {
+ if (EValueRepresentation::Embedded == RepresentationKind) {
+ new StoreInst(value, valuePtr, block);
+ } else {
+ const auto load = new LoadInst(valueType, valuePtr, "value", block);
+ const auto equal = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, value, load, "equal", block);
+
+ const auto skip = BasicBlock::Create(context, "skip", ctx.Func);
+ const auto refs = BasicBlock::Create(context, "refs", ctx.Func);
+ BranchInst::Create(skip, refs, equal, block);
+
+ block = refs;
+ ValueUnRef(RepresentationKind, valuePtr, ctx, block);
+ new StoreInst(value, valuePtr, block);
+ ValueAddRef(RepresentationKind, valuePtr, ctx, block);
+
+ BranchInst::Create(skip, block);
+ block = skip;
+ }
+
+ }
+ CreateInvalidate(ctx, block);
+}
+
+Value* TExternalCodegeneratorNode::CreateSwapValue(const TCodegenContext& ctx, BasicBlock*& block, Value* value) const {
+ auto& context = ctx.Codegen.GetContext();
+ const auto indexType = Type::getInt32Ty(context);
+ const auto valueType = Type::getInt128Ty(context);
+ const auto values = ctx.GetMutables();
+ const auto valuePtr = GetElementPtrInst::CreateInBounds(valueType, values, {ConstantInt::get(indexType, ValueIndex)}, "value_ptr", block);
+ const auto output = new LoadInst(valueType, valuePtr, "output", block);
+ ValueRelease(RepresentationKind, output, ctx, block);
+
+ if (value->getType()->isPointerTy()) {
+ const auto load = new LoadInst(valueType, value, "load", block);
+ new StoreInst(load, valuePtr, block);
+ new StoreInst(ConstantInt::get(load->getType(), 0), value, block);
+ } else {
+ ValueAddRef(RepresentationKind, value, ctx, block);
+ new StoreInst(value, valuePtr, block);
+ }
+
+ CreateInvalidate(ctx, block);
+ return output;
+}
+
+void TExternalCodegeneratorNode::CreateInvalidate(const TCodegenContext& ctx, BasicBlock*& block) const {
+ GenInvalidate(ctx, InvalidationSet, block);
+}
+
+void TExternalCodegeneratorNode::SetValueBuilder(TValueBuilder valueBuilder)
+{
+ ValueBuilder = std::move(valueBuilder);
+}
+
+void TExternalCodegeneratorNode::SetValueGetterBuilder(TValueGetterBuilder valueGetterBuilder)
+{
+ ValueGetterBuilder = std::move(valueGetterBuilder);
+}
+
+void TWideFlowProxyCodegeneratorNode::CreateInvalidate(const TCodegenContext& ctx, BasicBlock*& block) const {
+ GenInvalidate(ctx, InvalidationSet, block);
+}
+
+void TWideFlowProxyCodegeneratorNode::SetGenerator(TGenerator&& generator) {
+ Generator = std::move(generator);
+}
+
+ICodegeneratorInlineWideNode::TGenerateResult
+TWideFlowProxyCodegeneratorNode::GenGetValues(const TCodegenContext& ctx, BasicBlock*& block) const {
+ return Generator(ctx, block);
+}
+
+Value* GetOptionalValue(LLVMContext& context, Value* value, BasicBlock* block) {
+ const auto type = Type::getInt128Ty(context);
+ const auto data = ConstantInt::get(type, 0xFFFFFFFFFFFFFFFFULL);
+ const auto check = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_UGT, value, data, "check", block);
+ const auto decr = BinaryOperator::CreateSub(value, ConstantInt::get(type, 1), "decr", block);
+ const auto result = SelectInst::Create(check, value, decr, "result", block);
+ return result;
+}
+
+Value* MakeOptional(LLVMContext& context, Value* value, BasicBlock* block) {
+ const auto type = Type::getInt128Ty(context);
+ const auto data = ConstantInt::get(type, 0xFFFFFFFFFFFFFFFFULL);
+ const auto check = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_UGT, value, data, "check", block);
+ const auto incr = BinaryOperator::CreateAdd(value, ConstantInt::get(type, 1), "incr", block);
+ const auto result = SelectInst::Create(check, value, incr, "result", block);
+ return result;
+}
+
+ConstantInt* GetTrue(LLVMContext &context) {
+ const uint64_t init[] = {1ULL, 0x100000000000000ULL};
+ return ConstantInt::get(context, APInt(128, 2, init));
+}
+
+ConstantInt* GetFalse(LLVMContext &context) {
+ const uint64_t init[] = {0ULL, 0x100000000000000ULL};
+ return ConstantInt::get(context, APInt(128, 2, init));
+}
+
+ConstantInt* GetDecimalPlusInf(LLVMContext &context) {
+ const auto& pair = NYql::NDecimal::MakePair(+NYql::NDecimal::Inf());
+ const uint64_t init[] = {pair.first, pair.second};
+ return ConstantInt::get(context, APInt(128, 2, init));
+}
+
+ConstantInt* GetDecimalMinusInf(LLVMContext &context) {
+ const auto& pair = NYql::NDecimal::MakePair(-NYql::NDecimal::Inf());
+ const uint64_t init[] = {pair.first, pair.second};
+ return ConstantInt::get(context, APInt(128, 2, init));
+}
+
+ConstantInt* GetDecimalNan(LLVMContext &context) {
+ const auto& pair = NYql::NDecimal::MakePair(NYql::NDecimal::Nan());
+ const uint64_t init[] = {pair.first, pair.second};
+ return ConstantInt::get(context, APInt(128, 2, init));
+}
+
+
+ConstantInt* GetDecimalMinusNan(LLVMContext &context) {
+ const auto& pair = NYql::NDecimal::MakePair(-NYql::NDecimal::Nan());
+ const uint64_t init[] = {pair.first, pair.second};
+ return ConstantInt::get(context, APInt(128, 2, init));
+}
+
+static constexpr ui64 InvalidData = std::numeric_limits<ui64>::max();
+static constexpr ui64 FinishData = InvalidData - 1ULL;
+static constexpr ui64 YieldData = InvalidData;
+
+ConstantInt* GetEmpty(LLVMContext &context) {
+ return ConstantInt::get(Type::getInt128Ty(context), 0ULL);
+}
+
+ConstantInt* GetInvalid(LLVMContext &context) {
+ return ConstantInt::get(Type::getInt128Ty(context), InvalidData);
+}
+
+ConstantInt* GetFinish(LLVMContext &context) {
+ return ConstantInt::get(Type::getInt128Ty(context), FinishData);
+}
+
+ConstantInt* GetYield(LLVMContext &context) {
+ return ConstantInt::get(Type::getInt128Ty(context), YieldData);
+}
+
+ConstantInt* GetConstant(ui64 value, LLVMContext &context) {
+ const uint64_t init[] = {value, 0x100000000000000ULL};
+ return ConstantInt::get(context, APInt(128, 2, init));
+}
+
+Value* IsExists(Value* value, BasicBlock* block) {
+ const auto v = LoadIfPointer(value, block);
+ return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_NE, v, ConstantInt::get(v->getType(), 0ULL), "exists", block);
+}
+
+Value* IsEmpty(Value* value, BasicBlock* block) {
+ const auto v = LoadIfPointer(value, block);
+ return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, v, ConstantInt::get(v->getType(), 0ULL), "empty", block);
+}
+
+Value* IsInvalid(Value* value, BasicBlock* block) {
+ const auto v = LoadIfPointer(value, block);
+ return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, v, ConstantInt::get(v->getType(), InvalidData), "invalid", block);
+}
+
+Value* IsValid(Value* value, BasicBlock* block) {
+ const auto v = LoadIfPointer(value, block);
+ return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_NE, v, ConstantInt::get(v->getType(), InvalidData), "valid", block);
+}
+
+Value* IsFinish(Value* value, BasicBlock* block) {
+ const auto v = LoadIfPointer(value, block);
+ return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, v, ConstantInt::get(v->getType(), FinishData), "finish", block);
+}
+
+Value* IsYield(Value* value, BasicBlock* block) {
+ const auto v = LoadIfPointer(value, block);
+ return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, v, ConstantInt::get(v->getType(), YieldData), "yield", block);
+}
+
+Value* IsSpecial(Value* value, BasicBlock* block) {
+ const auto v = LoadIfPointer(value, block);
+ return BinaryOperator::CreateOr(IsFinish(v, block), IsYield(v, block), "special", block);
+}
+
+Value* HasValue(Value* value, BasicBlock* block) {
+ const auto v = LoadIfPointer(value, block);
+ return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_UGT, v, ConstantInt::get(v->getType(), InvalidData), "has", block);
+}
+
+Value* MakeBoolean(Value* boolean , LLVMContext &context, BasicBlock* block) {
+ return SelectInst::Create(boolean, GetTrue(context), GetFalse(context), "result", block);
+}
+
+Value* SetterForInt128(Value* value, BasicBlock* block) {
+ const uint64_t mask[] = {0xFFFFFFFFFFFFFFFFULL, 0xFFFFFFFFFFFFFFULL};
+ const auto drop = ConstantInt::get(value->getType(), APInt(128, 2, mask));
+ const auto data = BinaryOperator::CreateAnd(value, drop, "and", block);
+ const uint64_t init[] = {0ULL, 0x100000000000000ULL}; // Embedded
+ const auto meta = ConstantInt::get(value->getType(), APInt(128, 2, init));
+ const auto full = BinaryOperator::CreateOr(data, meta, "or", block);
+ return full;
+}
+
+Value* GetterForInt128(Value* value, BasicBlock* block) {
+ const uint64_t init[] = {0ULL, 0x80000000000000ULL};
+ const auto test = ConstantInt::get(value->getType(), APInt(128, 2, init));
+ const auto sign = BinaryOperator::CreateAnd(value, test, "and", block);
+
+ const uint64_t fill[] = {0ULL, 0xFF00000000000000ULL};
+ const auto sext = ConstantInt::get(value->getType(), APInt(128, 2, fill));
+ const auto minus = BinaryOperator::CreateOr(value, sext, "or", block);
+
+ const uint64_t mask[] = {0xFFFFFFFFFFFFFFFFULL, 0xFFFFFFFFFFFFFFULL};
+ const auto trun = ConstantInt::get(value->getType(), APInt(128, 2, mask));
+ const auto plus = BinaryOperator::CreateAnd(value, trun, "and", block);
+
+ const auto check = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, sign, ConstantInt::get(sign->getType(), 0), "check", block);
+ const auto result = SelectInst::Create(check, plus, minus, "result", block);
+ return result;
+}
+
+Value* GetterForTimezone(LLVMContext& context, Value* value, BasicBlock* block) {
+ const auto lshr = BinaryOperator::CreateLShr(value, ConstantInt::get(value->getType(), 64ULL), "lshr", block);
+ const auto trunc = CastInst::Create(Instruction::Trunc, lshr, Type::getInt16Ty(context), "trunc", block);
+ return trunc;
+}
+
+template<> Type* GetTypeFor<bool>(LLVMContext &context) { return Type::getInt1Ty(context); }
+template<> Type* GetTypeFor<ui8>(LLVMContext &context) { return Type::getInt8Ty(context); }
+template<> Type* GetTypeFor<i8>(LLVMContext &context) { return Type::getInt8Ty(context); }
+
+template<> Type* GetTypeFor<i16>(LLVMContext &context) { return Type::getInt16Ty(context); }
+template<> Type* GetTypeFor<ui16>(LLVMContext &context) { return Type::getInt16Ty(context); }
+
+template<> Type* GetTypeFor<i32>(LLVMContext &context) { return Type::getInt32Ty(context); }
+template<> Type* GetTypeFor<ui32>(LLVMContext &context) { return Type::getInt32Ty(context); }
+
+template<> Type* GetTypeFor<i64>(LLVMContext &context) { return Type::getInt64Ty(context); }
+template<> Type* GetTypeFor<ui64>(LLVMContext &context) { return Type::getInt64Ty(context); }
+
+template<> Type* GetTypeFor<float>(LLVMContext &context) { return Type::getFloatTy(context); }
+template<> Type* GetTypeFor<double>(LLVMContext &context) { return Type::getDoubleTy(context); }
+
+void AddRefBoxed(Value* value, const TCodegenContext& ctx, BasicBlock*& block) {
+ auto& context = ctx.Codegen.GetContext();
+ const auto load = value->getType()->isPointerTy() ? new LoadInst(value->getType()->getPointerElementType(), value, "load", block) : value;
+ const auto half = CastInst::Create(Instruction::Trunc, load, Type::getInt64Ty(context), "half", block);
+ const auto counterType = Type::getInt32Ty(context);
+ const auto type = StructType::get(context, {PointerType::getUnqual(StructType::get(context)), counterType, Type::getInt16Ty(context)});
+ const auto boxptr = CastInst::Create(Instruction::IntToPtr, half, PointerType::getUnqual(type), "boxptr", block);
+ const auto cntptr = GetElementPtrInst::CreateInBounds(type, boxptr, {ConstantInt::get(Type::getInt32Ty(context), 0), ConstantInt::get(Type::getInt32Ty(context), 1)}, "cntptr", block);
+ const auto refs = new LoadInst(counterType, cntptr, "refs", block);
+#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 4)
+ if constexpr (EnableStaticRefcount) {
+ const auto work = BasicBlock::Create(context, "work", ctx.Func);
+ const auto skip = BasicBlock::Create(context, "skip", ctx.Func);
+ const auto magic = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_SLT, refs, ConstantInt::get(refs->getType(), 0), "magic", block);
+ BranchInst::Create(skip, work, magic, block);
+
+ block = work;
+ const auto incr = BinaryOperator::CreateAdd(refs, ConstantInt::get(refs->getType(), 1), "incr", block);
+ new StoreInst(incr, cntptr, block);
+ BranchInst::Create(skip, block);
+
+ block = skip;
+ } else {
+ const auto incr = BinaryOperator::CreateAdd(refs, ConstantInt::get(refs->getType(), 1), "incr", block);
+ new StoreInst(incr, cntptr, block);
+ }
+#else
+ const auto incr = BinaryOperator::CreateAdd(refs, ConstantInt::get(refs->getType(), 1), "incr", block);
+ new StoreInst(incr, cntptr, block);
+#endif
+}
+
+void UnRefBoxed(Value* value, const TCodegenContext& ctx, BasicBlock*& block) {
+ auto& context = ctx.Codegen.GetContext();
+ const auto load = value->getType()->isPointerTy() ? new LoadInst(value->getType()->getPointerElementType(), value, "load", block) : value;
+ const auto half = CastInst::Create(Instruction::Trunc, load, Type::getInt64Ty(context), "half", block);
+ const auto counterType = Type::getInt32Ty(context);
+ const auto type = StructType::get(context, {PointerType::getUnqual(StructType::get(context)), counterType, Type::getInt16Ty(context)});
+ const auto boxptr = CastInst::Create(Instruction::IntToPtr, half, PointerType::getUnqual(type), "boxptr", block);
+ const auto cntptr = GetElementPtrInst::CreateInBounds(type, boxptr, {ConstantInt::get(Type::getInt32Ty(context), 0), ConstantInt::get(Type::getInt32Ty(context), 1)}, "cntptr", block);
+ const auto refs = new LoadInst(counterType, cntptr, "refs", block);
+
+ const auto live = BasicBlock::Create(context, "live", ctx.Func);
+
+#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 4)
+ if constexpr (EnableStaticRefcount) {
+ const auto magic = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_SLT, refs, ConstantInt::get(refs->getType(), 0), "magic", block);
+
+ const auto work = BasicBlock::Create(context, "work", ctx.Func);
+ BranchInst::Create(live, work, magic, block);
+
+ block = work;
+ }
+#endif
+
+ const auto decr = BinaryOperator::CreateSub(refs, ConstantInt::get(refs->getType(), 1), "decr", block);
+ new StoreInst(decr, cntptr, block);
+ const auto test = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_UGT, decr, ConstantInt::get(decr->getType(), 0), "many", block);
+
+ const auto kill = BasicBlock::Create(context, "kill", ctx.Func);
+
+ BranchInst::Create(live, kill, test, block);
+
+ block = kill;
+
+ const auto fnType = FunctionType::get(Type::getVoidTy(context), {boxptr->getType()}, false);
+ const auto name = "DeleteBoxed";
+ ctx.Codegen.AddGlobalMapping(name, reinterpret_cast<const void*>(&DeleteBoxed));
+ const auto func = ctx.Codegen.GetModule().getOrInsertFunction(name, fnType).getCallee();
+ CallInst::Create(fnType, func, {boxptr}, "", block);
+
+ BranchInst::Create(live, block);
+ block = live;
+}
+
+void CleanupBoxed(Value* value, const TCodegenContext& ctx, BasicBlock*& block) {
+ auto& context = ctx.Codegen.GetContext();
+ const auto load = value->getType()->isPointerTy() ? new LoadInst(value->getType()->getPointerElementType(), value, "load", block) : value;
+ const auto half = CastInst::Create(Instruction::Trunc, load, Type::getInt64Ty(context), "half", block);
+ const auto counterType = Type::getInt32Ty(context);
+ const auto type = StructType::get(context, {PointerType::getUnqual(StructType::get(context)), counterType, Type::getInt16Ty(context)});
+ const auto boxptr = CastInst::Create(Instruction::IntToPtr, half, PointerType::getUnqual(type), "boxptr", block);
+ const auto cntptr = GetElementPtrInst::CreateInBounds(type, boxptr, {ConstantInt::get(Type::getInt32Ty(context), 0), ConstantInt::get(Type::getInt32Ty(context), 1)}, "cntptr", block);
+ const auto refs = new LoadInst(counterType, cntptr, "refs", block);
+ const auto test = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_NE, refs, ConstantInt::get(refs->getType(), 0), "many", block);
+
+ const auto live = BasicBlock::Create(context, "live", ctx.Func);
+ const auto kill = BasicBlock::Create(context, "kill", ctx.Func);
+
+ BranchInst::Create(live, kill, test, block);
+
+ block = kill;
+
+ const auto fnType = FunctionType::get(Type::getVoidTy(context), {boxptr->getType()}, false);
+ const auto name = "DeleteBoxed";
+ ctx.Codegen.AddGlobalMapping(name, reinterpret_cast<const void*>(&DeleteBoxed));
+ const auto func = ctx.Codegen.GetModule().getOrInsertFunction(name, fnType).getCallee();
+ CallInst::Create(fnType, func, {boxptr}, "", block);
+
+ BranchInst::Create(live, block);
+ block = live;
+}
+
+
+template<bool IncOrDec>
+void ChangeRefUnboxed(Value* value, const TCodegenContext& ctx, BasicBlock*& block) {
+ auto& context = ctx.Codegen.GetContext();
+
+ const auto type8 = Type::getInt8Ty(context);
+ const auto type32 = Type::getInt32Ty(context);
+
+ const auto mark = GetMarkFromUnboxed(value, ctx, block);
+
+ const auto boxb = BasicBlock::Create(context, "boxb", ctx.Func);
+ const auto strb = BasicBlock::Create(context, "strb", ctx.Func);
+ const auto doit = BasicBlock::Create(context, "doit", ctx.Func);
+ const auto done = BasicBlock::Create(context, "done", ctx.Func);
+
+ const auto refsPtrType = PointerType::getUnqual(type32);
+ const auto refsptr = PHINode::Create(refsPtrType, 2U, "refsptr", doit);
+
+ const auto choise = SwitchInst::Create(mark, done, 2U, block);
+ choise->addCase(ConstantInt::get(type8, 2), strb);
+ choise->addCase(ConstantInt::get(type8, 3), boxb);
+
+ {
+ block = strb;
+
+ const auto strptr = GetPointerFromUnboxed<false>(value, ctx, block);
+ const auto elemptr = GetElementPtrInst::CreateInBounds(strptr->getType()->getPointerElementType(), strptr, {ConstantInt::get(type32, 0), ConstantInt::get(type32, 1)}, "elemptr", block);
+ refsptr->addIncoming(elemptr, block);
+ BranchInst::Create(doit, block);
+ }
+
+ {
+ block = boxb;
+
+ const auto boxptr = GetPointerFromUnboxed<true>(value, ctx, block);;
+ const auto elemptr = GetElementPtrInst::CreateInBounds(boxptr->getType()->getPointerElementType(), boxptr, {ConstantInt::get(type32, 0), ConstantInt::get(type32, 1)}, "elemptr", block);
+ refsptr->addIncoming(elemptr, block);
+ BranchInst::Create(doit, block);
+ }
+
+ block = doit;
+
+ const auto refs = new LoadInst(type32, refsptr, "refs", block);
+#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 4)
+ if constexpr (EnableStaticRefcount) {
+ const auto magic = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_SLT, refs, ConstantInt::get(refs->getType(), 0), "magic", block);
+
+ const auto work = BasicBlock::Create(context, "work", ctx.Func);
+ BranchInst::Create(done, work, magic, block);
+
+ block = work;
+ }
+#endif
+ const auto next = IncOrDec ?
+ BinaryOperator::CreateAdd(refs, ConstantInt::get(refs->getType(), 1), "incr", block):
+ BinaryOperator::CreateSub(refs, ConstantInt::get(refs->getType(), 1), "decr", block);
+ new StoreInst(next, refsptr, block);
+ BranchInst::Create(done, block);
+
+ block = done;
+}
+
+template<bool Decrement>
+void CheckRefUnboxed(Value* value, const TCodegenContext& ctx, BasicBlock*& block) {
+ auto& context = ctx.Codegen.GetContext();
+
+ const auto type8 = Type::getInt8Ty(context);
+ const auto type32 = Type::getInt32Ty(context);
+
+ const auto mark = GetMarkFromUnboxed(value, ctx, block);
+
+ const auto boxb = BasicBlock::Create(context, "boxb", ctx.Func);
+ const auto strb = BasicBlock::Create(context, "strb", ctx.Func);
+ const auto nope = BasicBlock::Create(context, "nope", ctx.Func);
+
+ const auto choise = SwitchInst::Create(mark, nope, 2U, block);
+ choise->addCase(ConstantInt::get(type8, 2), strb);
+ choise->addCase(ConstantInt::get(type8, 3), boxb);
+
+ {
+ block = strb;
+
+ const auto strptr = GetPointerFromUnboxed<false>(value, ctx, block);
+ const auto refptr = GetElementPtrInst::CreateInBounds(strptr->getType()->getPointerElementType(), strptr, {ConstantInt::get(type32, 0), ConstantInt::get(type32, 1)}, "refptr", block);
+ const auto refs = new LoadInst(type32, refptr, "refs", block);
+
+#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 4)
+ if constexpr (EnableStaticRefcount) {
+ const auto magic = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_SLT, refs, ConstantInt::get(refs->getType(), 0), "magic", block);
+
+ const auto work = BasicBlock::Create(context, "work", ctx.Func);
+ BranchInst::Create(nope, work, magic, block);
+
+ block = work;
+ }
+#endif
+ Value* test = refs;
+
+ if constexpr (Decrement) {
+ const auto decr = BinaryOperator::CreateSub(refs, ConstantInt::get(refs->getType(), 1), "decr", block);
+ new StoreInst(decr, refptr, block);
+ test = decr;
+ }
+
+ const auto good = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_UGT, test, ConstantInt::get(test->getType(), 0), "test", block);
+
+ const auto free = BasicBlock::Create(context, "free", ctx.Func);
+
+ BranchInst::Create(nope, free, good, block);
+
+ block = free;
+
+ const auto fnType = FunctionType::get(Type::getVoidTy(context), {strptr->getType()}, false);
+ const auto name = "DeleteString";
+ ctx.Codegen.AddGlobalMapping(name, reinterpret_cast<const void*>(&DeleteString));
+ const auto func = ctx.Codegen.GetModule().getOrInsertFunction(name, fnType).getCallee();
+ CallInst::Create(fnType, func, {strptr}, "", block);
+ BranchInst::Create(nope, block);
+ }
+
+ {
+ block = boxb;
+
+ const auto boxptr = GetPointerFromUnboxed<true>(value, ctx, block);;
+ const auto refptr = GetElementPtrInst::CreateInBounds(boxptr->getType()->getPointerElementType(), boxptr, {ConstantInt::get(type32, 0), ConstantInt::get(type32, 1)}, "cntptr", block);
+ const auto refs = new LoadInst(type32, refptr, "refs", block);
+
+#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 4)
+ if constexpr (EnableStaticRefcount) {
+ const auto magic = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_SLT, refs, ConstantInt::get(refs->getType(), 0), "magic", block);
+
+ const auto work = BasicBlock::Create(context, "work", ctx.Func);
+ BranchInst::Create(nope, work, magic, block);
+
+ block = work;
+ }
+#endif
+
+ Value* test = refs;
+
+ if constexpr (Decrement) {
+ const auto decr = BinaryOperator::CreateSub(refs, ConstantInt::get(refs->getType(), 1), "decr", block);
+ new StoreInst(decr, refptr, block);
+ test = decr;
+ }
+
+ const auto good = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_UGT, test, ConstantInt::get(test->getType(), 0), "test", block);
+
+ const auto kill = BasicBlock::Create(context, "kill", ctx.Func);
+
+ BranchInst::Create(nope, kill, good, block);
+
+ block = kill;
+
+ const auto fnType = FunctionType::get(Type::getVoidTy(context), {boxptr->getType()}, false);
+ const auto name = "DeleteBoxed";
+ ctx.Codegen.AddGlobalMapping(name, reinterpret_cast<const void*>(&DeleteBoxed));
+ const auto func = ctx.Codegen.GetModule().getOrInsertFunction(name, fnType).getCallee();
+
+ CallInst::Create(fnType, func, {boxptr}, "", block);
+ BranchInst::Create(nope, block);
+ }
+
+ block = nope;
+}
+#ifdef MAKE_UNBOXED_VALUE_LLVM_REFCOUNTION_FUNCTIONS
+Function* GenRefCountFunction(const char* label, void (*func)(Value*, const TCodegenContext&, BasicBlock*&), Type* type, NYql::NCodegen::ICodegen& codegen) {
+ auto& module = codegen.GetModule();
+ auto& context = codegen.GetContext();
+ const auto name = TString(label) += (type->isPointerTy() ? "Ptr" : "Val");
+ if (const auto f = module.getFunction(name.c_str()))
+ return f;
+
+ const auto funcType = FunctionType::get(Type::getVoidTy(context), {type}, false);
+
+ TCodegenContext ctx(codegen);
+ ctx.Func = cast<Function>(module.getOrInsertFunction(name.c_str(), funcType).getCallee()).getCallee();
+
+ auto main = BasicBlock::Create(context, "main", ctx.Func);
+ auto value = &*ctx.Func->arg_begin();
+
+ func(value, ctx, main);
+ ReturnInst::Create(context, main);
+ return ctx.Func;
+}
+
+void AddRefUnboxed(Value* value, const TCodegenContext& ctx, BasicBlock*& block) {
+ CallInst::Create(GenRefCountFunction(__func__, &ChangeRefUnboxed<true>, value->getType(), ctx.Codegen), {value}, "", block);
+}
+
+void ReleaseUnboxed(Value* value, const TCodegenContext& ctx, BasicBlock*& block) {
+ CallInst::Create(GenRefCountFunction(__func__, &ChangeRefUnboxed<false>, value->getType(), ctx.Codegen), {value}, "", block);
+}
+
+void UnRefUnboxed(Value* value, const TCodegenContext& ctx, BasicBlock*& block) {
+ CallInst::Create(GenRefCountFunction(__func__, &CheckRefUnboxed<true>, value->getType(), ctx.Codegen), {value}, "", block);
+}
+
+void CleanupUnboxed(Value* value, const TCodegenContext& ctx, BasicBlock*& block) {
+ CallInst::Create(GenRefCountFunction(__func__, &CheckRefUnboxed<false>, value->getType(), ctx.Codegen), {value}, "", block);
+}
+#else
+void AddRefUnboxed(Value* value, const TCodegenContext& ctx, BasicBlock*& block) {
+ return ChangeRefUnboxed<true>(value, ctx, block);
+}
+
+void ReleaseUnboxed(Value* value, const TCodegenContext& ctx, BasicBlock*& block) {
+ return ChangeRefUnboxed<false>(value, ctx, block);
+}
+
+void UnRefUnboxed(Value* value, const TCodegenContext& ctx, BasicBlock*& block) {
+ return CheckRefUnboxed<true>(value, ctx, block);
+}
+
+void CleanupUnboxed(Value* value, const TCodegenContext& ctx, BasicBlock*& block) {
+ return CheckRefUnboxed<false>(value, ctx, block);
+}
+#endif
+
+void SafeUnRefUnboxed(Value* pointer, const TCodegenContext& ctx, BasicBlock*& block) {
+ if (const auto itemType = pointer->getType()->getPointerElementType(); itemType->isArrayTy()) {
+ const auto indexType = Type::getInt64Ty(ctx.Codegen.GetContext());
+ Value* zeros = UndefValue::get(itemType);
+ for (ui32 idx = 0U; idx < itemType->getArrayNumElements(); ++idx) {
+ const auto item = GetElementPtrInst::CreateInBounds(itemType, pointer, { ConstantInt::get(indexType, 0), ConstantInt::get(indexType, idx) }, (TString("item_") += ToString(idx)).c_str(), block);
+ UnRefUnboxed(item, ctx, block);
+ zeros = InsertValueInst::Create(zeros, ConstantInt::get(itemType->getArrayElementType(), 0), {idx}, (TString("zero_") += ToString(idx)).c_str(), block);
+ }
+ new StoreInst(zeros, pointer, block);
+ } else {
+ UnRefUnboxed(pointer, ctx, block);
+ new StoreInst(ConstantInt::get(itemType, 0), pointer, block);
+ }
+}
+
+void ValueAddRef(EValueRepresentation kind, Value* pointer, const TCodegenContext& ctx, BasicBlock*& block) {
+ switch (kind) {
+ case EValueRepresentation::Embedded: return;
+ case EValueRepresentation::Boxed: // TODO
+ case EValueRepresentation::String: // TODO
+ case EValueRepresentation::Any: return AddRefUnboxed(pointer, ctx, block);
+ }
+}
+
+void ValueUnRef(EValueRepresentation kind, Value* pointer, const TCodegenContext& ctx, BasicBlock*& block) {
+ switch (kind) {
+ case EValueRepresentation::Embedded: return;
+ case EValueRepresentation::Boxed: // TODO
+ case EValueRepresentation::String: // TODO
+ case EValueRepresentation::Any: return UnRefUnboxed(pointer, ctx, block);
+ }
+}
+
+void ValueCleanup(EValueRepresentation kind, Value* pointer, const TCodegenContext& ctx, BasicBlock*& block) {
+ switch (kind) {
+ case EValueRepresentation::Embedded: return;
+ case EValueRepresentation::Boxed: // TODO
+ case EValueRepresentation::String: // TODO
+ case EValueRepresentation::Any: return CleanupUnboxed(pointer, ctx, block);
+ }
+}
+
+void ValueRelease(EValueRepresentation kind, Value* pointer, const TCodegenContext& ctx, BasicBlock*& block) {
+ switch (kind) {
+ case EValueRepresentation::Embedded: return;
+ case EValueRepresentation::Boxed: // TODO
+ case EValueRepresentation::String: // TODO
+ case EValueRepresentation::Any: return ReleaseUnboxed(pointer, ctx, block);
+ }
+}
+
+std::pair<Value*, Value*> GetVariantParts(Value* variant, const TCodegenContext& ctx, BasicBlock*& block) {
+ auto& context = ctx.Codegen.GetContext();
+
+ const auto type = Type::getInt32Ty(context);
+ const auto lshr = BinaryOperator::CreateLShr(variant, ConstantInt::get(variant->getType(), 122), "lshr", block);
+ const auto trunc = CastInst::Create(Instruction::Trunc, lshr, type, "trunc", block);
+
+ const auto check = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_NE, trunc, ConstantInt::get(type , 0), "check", block);
+
+ const auto boxed = BasicBlock::Create(context, "boxed", ctx.Func);
+ const auto embed = BasicBlock::Create(context, "embed", ctx.Func);
+ const auto done = BasicBlock::Create(context, "done", ctx.Func);
+
+ const auto index = PHINode::Create(type, 2U, "index", done);
+ const auto item = PHINode::Create(variant->getType(), 2U, "index", done);
+
+ BranchInst::Create(embed, boxed, check, block);
+
+ {
+ block = embed;
+
+ const uint64_t init[] = {0xFFFFFFFFFFFFFFFFULL, 0x3FFFFFFFFFFFFFFULL};
+ const auto mask = ConstantInt::get(variant->getType(), APInt(128, 2, init));
+ const auto clean = BinaryOperator::CreateAnd(variant, mask, "clean", block);
+
+ const auto dec = BinaryOperator::CreateSub(trunc, ConstantInt::get(type, 1), "dec", block);
+ index->addIncoming(dec, block);
+ item->addIncoming(clean, block);
+ BranchInst::Create(done, block);
+ }
+
+ {
+ block = boxed;
+
+ const auto place = new AllocaInst(item->getType(), 0U, "place", &ctx.Func->getEntryBlock().back());
+ const auto idx = CallBoxedValueVirtualMethod<NUdf::TBoxedValueAccessor::EMethod::GetVariantIndex>(type, variant, ctx.Codegen, block);
+ CallBoxedValueVirtualMethod<NUdf::TBoxedValueAccessor::EMethod::GetVariantItem>(place, variant, ctx.Codegen, block);
+ const auto clean = new LoadInst(item->getType(), place, "clean", block);
+ ValueRelease(EValueRepresentation::Any, clean, ctx, block);
+ index->addIncoming(idx, block);
+ item->addIncoming(clean, block);
+ BranchInst::Create(done, block);
+ }
+
+ block = done;
+ return std::make_pair(index, item);
+}
+
+Value* MakeVariant(Value* item, Value* variant, const TCodegenContext& ctx, BasicBlock*& block) {
+ auto& context = ctx.Codegen.GetContext();
+
+ const auto boxed = BasicBlock::Create(context, "boxed", ctx.Func);
+ const auto embed = BasicBlock::Create(context, "embed", ctx.Func);
+ const auto done = BasicBlock::Create(context, "done", ctx.Func);
+
+ const auto result = PHINode::Create(item->getType(), 1U, "index", done);
+
+ const auto offset = ConstantInt::get(item->getType(), 122);
+ const auto lshr = BinaryOperator::CreateLShr(item, offset, "lshr", block);
+
+ const auto checkItem = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, lshr, ConstantInt::get(lshr->getType(), 0), "check_item", block);
+ const auto checkIndex = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_ULT, variant, ConstantInt::get(variant->getType(), (1U << 6U) - 1U), "check_index", block);
+ const auto check = BinaryOperator::CreateAnd(checkItem, checkIndex, "and", block);
+
+ BranchInst::Create(embed, boxed, check, block);
+
+ {
+ block = embed;
+
+ const auto index = BinaryOperator::CreateAdd(variant, ConstantInt::get(variant->getType(), 1), "index", block);
+ const auto extend = CastInst::Create(Instruction::ZExt, index, item->getType(), "extend", block);
+ const auto shift = BinaryOperator::CreateShl(extend, offset, "shift", block);
+ const auto output = BinaryOperator::CreateOr(item, shift, "output", block);
+ result->addIncoming(output, block);
+ BranchInst::Create(done, block);
+ }
+
+ {
+ block = boxed;
+
+ const auto factory = ctx.GetFactory();
+ const auto func = ConstantInt::get(Type::getInt64Ty(context), GetMethodPtr(&THolderFactory::CreateBoxedVariantHolder));
+
+ if (NYql::NCodegen::ETarget::Windows != ctx.Codegen.GetEffectiveTarget()) {
+ const auto signature = FunctionType::get(item->getType(), {factory->getType(), item->getType(), variant->getType()}, false);
+ const auto creator = CastInst::Create(Instruction::IntToPtr, func, PointerType::getUnqual(signature), "creator", block);
+ const auto output = CallInst::Create(signature, creator, {factory, item, variant}, "output", block);
+ result->addIncoming(output, block);
+ } else {
+ const auto place = new AllocaInst(item->getType(), 0U, "place", block);
+ new StoreInst(item, place, block);
+ const auto signature = FunctionType::get(Type::getVoidTy(context), {factory->getType(), place->getType(), place->getType(), variant->getType()}, false);
+ const auto creator = CastInst::Create(Instruction::IntToPtr, func, PointerType::getUnqual(signature), "creator", block);
+ CallInst::Create(signature, creator, {factory, place, place, variant}, "", block);
+ const auto output = new LoadInst(item->getType(), place, "output", block);
+ result->addIncoming(output, block);
+ }
+
+ BranchInst::Create(done, block);
+ }
+
+ block = done;
+ return result;
+}
+
+Value* GetNodeValue(IComputationNode* node, const TCodegenContext& ctx, BasicBlock*& block) {
+ if (const auto codegen = dynamic_cast<ICodegeneratorInlineNode*>(node))
+ return codegen->CreateGetValue(ctx, block);
+
+ auto& context = ctx.Codegen.GetContext();
+ const auto ptr = ConstantInt::get(Type::getInt64Ty(context), intptr_t(node));
+ const auto ptrType = PointerType::getUnqual(StructType::get(context));
+ const auto nodeThis = CastInst::Create(Instruction::IntToPtr, ptr, ptrType, "node_this", block);
+
+ const auto valueType = Type::getInt128Ty(context);
+ const auto retPtr = new AllocaInst(valueType, 0U, "return_ptr", &ctx.Func->getEntryBlock().back());
+ const auto funType = ctx.Codegen.GetEffectiveTarget() != NYql::NCodegen::ETarget::Windows ?
+ FunctionType::get(Type::getVoidTy(context), {retPtr->getType(), nodeThis->getType(), ctx.Ctx->getType()}, false):
+ FunctionType::get(Type::getVoidTy(context), {nodeThis->getType(), retPtr->getType(), ctx.Ctx->getType()}, false);
+ const auto ptrFunType = PointerType::getUnqual(funType);
+ const auto tableType = PointerType::getUnqual(ptrFunType);
+ const auto nodeVTable = CastInst::Create(Instruction::IntToPtr, ptr, PointerType::getUnqual(tableType), "node_vtable", block);
+
+ const auto table = new LoadInst(tableType, nodeVTable, "table", false, block);
+ const auto elem = GetElementPtrInst::CreateInBounds(ptrFunType, table, {ConstantInt::get(Type::getInt64Ty(context), GetMethodIndex(&IComputationNode::GetValue))}, "element", block);
+ const auto func = new LoadInst(ptrFunType, elem, "func", false, block);
+
+ if (ctx.Codegen.GetEffectiveTarget() != NYql::NCodegen::ETarget::Windows) {
+ CallInst::Create(funType, func, {retPtr, nodeThis, ctx.Ctx}, "", block);
+ } else {
+ CallInst::Create(funType, func, {nodeThis, retPtr, ctx.Ctx}, "", block);
+ }
+
+ ValueRelease(node->GetRepresentation(), retPtr, ctx, block);
+ const auto result = new LoadInst(valueType, retPtr, "return", false, block);
+ return result;
+}
+
+void GetNodeValue(Value* value, IComputationNode* node, const TCodegenContext& ctx, BasicBlock*& block) {
+ if (const auto codegen = dynamic_cast<ICodegeneratorInlineNode*>(node)) {
+ const auto v = codegen->CreateGetValue(ctx, block);
+ new StoreInst(v, value, block);
+ ValueAddRef(node->GetRepresentation(), value, ctx, block);
+ return;
+ }
+
+ auto& context = ctx.Codegen.GetContext();
+ const auto ptr = ConstantInt::get(Type::getInt64Ty(context), intptr_t(node));
+ const auto ptrType = PointerType::getUnqual(StructType::get(context));
+ const auto nodeThis = CastInst::Create(Instruction::IntToPtr, ptr, ptrType, "node_this", block);
+
+ const auto funType = ctx.Codegen.GetEffectiveTarget() != NYql::NCodegen::ETarget::Windows ?
+ FunctionType::get(Type::getVoidTy(context), {value->getType(), nodeThis->getType(), ctx.Ctx->getType()}, false):
+ FunctionType::get(Type::getVoidTy(context), {nodeThis->getType(), value->getType(), ctx.Ctx->getType()}, false);
+ const auto ptrFunType = PointerType::getUnqual(funType);
+ const auto tableType = PointerType::getUnqual(ptrFunType);
+ const auto nodeVTable = CastInst::Create(Instruction::IntToPtr, ptr, PointerType::getUnqual(tableType), "node_vtable", block);
+
+ const auto table = new LoadInst(tableType, nodeVTable, "table", false, block);
+ const auto elem = GetElementPtrInst::CreateInBounds(ptrFunType, table, {ConstantInt::get(Type::getInt64Ty(context), GetMethodIndex(&IComputationNode::GetValue))}, "element", block);
+ const auto func = new LoadInst(ptrFunType, elem, "func", false, block);
+
+ if (ctx.Codegen.GetEffectiveTarget() != NYql::NCodegen::ETarget::Windows) {
+ CallInst::Create(funType, func, {value, nodeThis, ctx.Ctx}, "", block);
+ } else {
+ CallInst::Create(funType, func, {nodeThis, value, ctx.Ctx}, "", block);
+ }
+}
+
+ICodegeneratorInlineWideNode::TGenerateResult GetNodeValues(IComputationWideFlowNode* node, const TCodegenContext& ctx, BasicBlock*& block) {
+ if (const auto codegen = dynamic_cast<ICodegeneratorInlineWideNode*>(node))
+ return codegen->GenGetValues(ctx, block);
+ throw TNoCodegen();
+}
+
+Value* GenNewArray(const TCodegenContext& ctx, Value* size, Value* items, BasicBlock* block) {
+ auto& context = ctx.Codegen.GetContext();
+ const auto fact = ctx.GetFactory();
+ const auto func = ConstantInt::get(Type::getInt64Ty(context), GetMethodPtr(&THolderFactory::CreateDirectArrayHolder));
+ const auto valueType = Type::getInt128Ty(context);
+ if (NYql::NCodegen::ETarget::Windows != ctx.Codegen.GetEffectiveTarget()) {
+ const auto funType = FunctionType::get(valueType, {fact->getType(), size->getType(), items->getType()}, false);
+ const auto funcPtr = CastInst::Create(Instruction::IntToPtr, func, PointerType::getUnqual(funType), "function", block);
+ return CallInst::Create(funType, funcPtr, {fact, size, items}, "array", block);
+ } else {
+ const auto resultPtr = new AllocaInst(valueType, 0U, "return", block);
+ const auto funType = FunctionType::get(Type::getVoidTy(context), {fact->getType(), resultPtr->getType(), size->getType(), items->getType()}, false);
+ const auto funcPtr = CastInst::Create(Instruction::IntToPtr, func, PointerType::getUnqual(funType), "function", block);
+ CallInst::Create(funType, funcPtr, {fact, resultPtr, size, items}, "", block);
+ return new LoadInst(valueType, resultPtr, "array", block);
+ }
+}
+
+Value* GetMemoryUsed(ui64 limit, const TCodegenContext& ctx, BasicBlock* block) {
+ if (!limit) {
+ return nullptr;
+ }
+
+ auto& context = ctx.Codegen.GetContext();
+ const auto fact = ctx.GetFactory();
+ const auto func = ConstantInt::get(Type::getInt64Ty(context), GetMethodPtr(&THolderFactory::GetMemoryUsed));
+ const auto funType = FunctionType::get(Type::getInt64Ty(context), {fact->getType()}, false);
+ const auto funcPtr = CastInst::Create(Instruction::IntToPtr, func, PointerType::getUnqual(funType), "get_used", block);
+ return CallInst::Create(funType, funcPtr, {fact}, "mem_used", block);
+}
+
+template <bool TrackRss>
+Value* CheckAdjustedMemLimit(ui64 limit, Value* init, const TCodegenContext& ctx, BasicBlock*& block) {
+ auto& context = ctx.Codegen.GetContext();
+
+ if (!limit || !init) {
+ return ConstantInt::getFalse(context);
+ }
+
+ const auto indexType = Type::getInt32Ty(context);
+
+ if constexpr (TrackRss) {
+ const auto rssPtr = GetElementPtrInst::CreateInBounds(GetCompContextType(context), ctx.Ctx, {ConstantInt::get(indexType, 0), ConstantInt::get(indexType, 5)}, "rss_ptr", block);
+ const auto rss = new LoadInst(Type::getInt32Ty(context), rssPtr, "rsscounter", block);
+ const auto inc = BinaryOperator::CreateAdd(rss, ConstantInt::get(rss->getType(), 1), "inc", block);
+ new StoreInst(inc, rssPtr, block);
+ const auto mod = BinaryOperator::CreateURem(rss, ConstantInt::get(rss->getType(), STEP_FOR_RSS_CHECK), "mod", block);
+ const auto now = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, mod, ConstantInt::get(mod->getType() , 0), "now", block);
+
+ const auto call = BasicBlock::Create(context, "call", ctx.Func);
+ const auto skip = BasicBlock::Create(context, "skip", ctx.Func);
+
+ BranchInst::Create(call, skip, now, block);
+
+ block = call;
+ const auto func = ConstantInt::get(Type::getInt64Ty(context), GetMethodPtr(&TComputationContext::UpdateUsageAdjustor));
+ const auto funType = FunctionType::get(Type::getVoidTy(context), {ctx.Ctx->getType(), Type::getInt64Ty(context)}, false);
+ const auto funcPtr = CastInst::Create(Instruction::IntToPtr, func, PointerType::getUnqual(funType), "update", block);
+ CallInst::Create(funType, funcPtr, {ctx.Ctx, ConstantInt::get(init->getType(), limit)}, "", block);
+
+ BranchInst::Create(skip, block);
+
+ block = skip;
+ }
+
+ const auto adjPtr = GetElementPtrInst::CreateInBounds(GetCompContextType(context), ctx.Ctx, {ConstantInt::get(indexType, 0), ConstantInt::get(indexType, 4)}, "adj_ptr", block);
+ const auto adjustor = new LoadInst(Type::getFloatTy(context), adjPtr, "adjustor", block);
+
+ const auto curr = GetMemoryUsed(limit, ctx, block);
+ const auto cast = CastInst::Create(Instruction::UIToFP, curr, adjustor->getType(), "cast", block);
+ const auto used = BinaryOperator::CreateFMul(cast, adjustor, "used", block);
+ const auto add = BinaryOperator::CreateAdd(init, ConstantInt::get(init->getType(), limit), "add", block);
+ const auto upper = CastInst::Create(Instruction::UIToFP, add, adjustor->getType(), "upper", block);
+ return CmpInst::Create(Instruction::FCmp, FCmpInst::FCMP_OGE, used, upper, "enough", block);
+}
+
+template Value* CheckAdjustedMemLimit<false>(ui64 limit, Value* init, const TCodegenContext& ctx, BasicBlock*& block);
+template Value* CheckAdjustedMemLimit<true>(ui64 limit, Value* init, const TCodegenContext& ctx, BasicBlock*& block);
+
+Value* WrapArgumentForWindows(Value* arg, const TCodegenContext& ctx, BasicBlock* block) {
+ if (ctx.Codegen.GetEffectiveTarget() != NYql::NCodegen::ETarget::Windows) {
+ return arg;
+ }
+
+ const auto newArg = new AllocaInst(arg->getType(), 0, "argument", block);
+ new StoreInst(arg, newArg, block);
+ return newArg;
+}
+
+Value* CallBoxedValueVirtualMethodImpl(uintptr_t methodPtr, Type* returnType, Value* value, NYql::NCodegen::ICodegen& codegen, BasicBlock* block) {
+ auto& context = codegen.GetContext();
+
+ const auto data = CastInst::Create(Instruction::Trunc, value, Type::getInt64Ty(context), "data", block);
+ const auto ptrStructType = PointerType::getUnqual(StructType::get(context));
+ const auto boxed = CastInst::Create(Instruction::IntToPtr, data, ptrStructType, "boxed", block);
+
+ const auto funType = FunctionType::get(returnType, {boxed->getType()}, false);
+ const auto ptrFunType = PointerType::getUnqual(funType);
+ const auto tableType = PointerType::getUnqual(ptrFunType);
+ const auto vTable = CastInst::Create(Instruction::IntToPtr, data, PointerType::getUnqual(tableType), "vtable", block);
+
+ const auto table = new LoadInst(tableType, vTable, "table", false, block);
+ const auto elem = GetElementPtrInst::CreateInBounds(ptrFunType, table, {ConstantInt::get(Type::getInt64Ty(context), GetMethodPtrIndex(methodPtr))}, "element", block);
+ const auto func = new LoadInst(ptrFunType, elem, "func", false, block);
+
+ const auto call = CallInst::Create(funType, func, {boxed}, returnType->isVoidTy() ? "" : "return", block);
+ return call;
+}
+
+void CallBoxedValueVirtualMethodImpl(uintptr_t methodPtr, Value* output, Value* value, NYql::NCodegen::ICodegen& codegen, BasicBlock* block) {
+ auto& context = codegen.GetContext();
+
+ const auto data = CastInst::Create(Instruction::Trunc, value, Type::getInt64Ty(context), "data", block);
+ const auto ptrStructType = PointerType::getUnqual(StructType::get(context));
+ const auto boxed = CastInst::Create(Instruction::IntToPtr, data, ptrStructType, "boxed", block);
+
+ const auto funType = (codegen.GetEffectiveTarget() != NYql::NCodegen::ETarget::Windows) ?
+ FunctionType::get(Type::getVoidTy(context), {output->getType(), boxed->getType()}, false):
+ FunctionType::get(Type::getVoidTy(context), {boxed->getType(), output->getType()}, false);
+ const auto ptrFunType = PointerType::getUnqual(funType);
+ const auto tableType = PointerType::getUnqual(ptrFunType);
+ const auto vTable = CastInst::Create(Instruction::IntToPtr, data, PointerType::getUnqual(tableType), "vtable", block);
+
+ const auto table = new LoadInst(tableType, vTable, "table", false, block);
+ const auto elem = GetElementPtrInst::CreateInBounds(ptrFunType, table, {ConstantInt::get(Type::getInt64Ty(context), GetMethodPtrIndex(methodPtr))}, "element", block);
+ const auto func = new LoadInst(ptrFunType, elem, "func", false, block);
+
+ if (codegen.GetEffectiveTarget() != NYql::NCodegen::ETarget::Windows) {
+ CallInst::Create(funType, func, {output, boxed}, "", block);
+ } else {
+ CallInst::Create(funType, func, {boxed, output}, "", block);
+ }
+}
+
+void CallBoxedValueVirtualMethodImpl(uintptr_t methodPtr, Value* output, Value* value, NYql::NCodegen::ICodegen& codegen, BasicBlock* block, Value* argument) {
+ auto& context = codegen.GetContext();
+
+ const auto data = CastInst::Create(Instruction::Trunc, value, Type::getInt64Ty(context), "data", block);
+ const auto ptrStructType = PointerType::getUnqual(StructType::get(context));
+ const auto boxed = CastInst::Create(Instruction::IntToPtr, data, ptrStructType, "boxed", block);
+
+ const auto funType = (codegen.GetEffectiveTarget() != NYql::NCodegen::ETarget::Windows) ?
+ FunctionType::get(Type::getVoidTy(context), {output->getType(), boxed->getType(), argument->getType()}, false):
+ FunctionType::get(Type::getVoidTy(context), {boxed->getType(), output->getType(), argument->getType()}, false);
+ const auto ptrFunType = PointerType::getUnqual(funType);
+ const auto tableType = PointerType::getUnqual(ptrFunType);
+ const auto vTable = CastInst::Create(Instruction::IntToPtr, data, PointerType::getUnqual(tableType), "vtable", block);
+
+ const auto table = new LoadInst(tableType, vTable, "table", false, block);
+ const auto elem = GetElementPtrInst::CreateInBounds(ptrFunType, table, {ConstantInt::get(Type::getInt64Ty(context), GetMethodPtrIndex(methodPtr))}, "element", block);
+ const auto func = new LoadInst(ptrFunType, elem, "func", false, block);
+
+ if (codegen.GetEffectiveTarget() != NYql::NCodegen::ETarget::Windows) {
+ CallInst::Create(funType, func, {output, boxed, argument}, "", block);
+ } else {
+ CallInst::Create(funType, func, {boxed, output, argument}, "", block);
+ }
+}
+
+Value* CallBoxedValueVirtualMethodImpl(uintptr_t methodPtr, Type* returnType, Value* value, NYql::NCodegen::ICodegen& codegen, BasicBlock* block, Value* argument) {
+ auto& context = codegen.GetContext();
+
+ const auto data = CastInst::Create(Instruction::Trunc, value, Type::getInt64Ty(context), "data", block);
+ const auto ptrStructType = PointerType::getUnqual(StructType::get(context));
+ const auto boxed = CastInst::Create(Instruction::IntToPtr, data, ptrStructType, "boxed", block);
+
+ const auto funType = FunctionType::get(returnType, {boxed->getType(), argument->getType()}, false);
+ const auto ptrFunType = PointerType::getUnqual(funType);
+ const auto tableType = PointerType::getUnqual(ptrFunType);
+ const auto vTable = CastInst::Create(Instruction::IntToPtr, data, PointerType::getUnqual(tableType), "vtable", block);
+
+ const auto table = new LoadInst(tableType, vTable, "table", false, block);
+ const auto elem = GetElementPtrInst::CreateInBounds(ptrFunType, table, {ConstantInt::get(Type::getInt64Ty(context), GetMethodPtrIndex(methodPtr))}, "element", block);
+ const auto func = new LoadInst(ptrFunType, elem, "func", false, block);
+
+ const auto call = CallInst::Create(funType, func, {boxed, argument}, returnType->isVoidTy() ? "" : "return", block);
+ return call;
+}
+
+void CallBoxedValueVirtualMethodImpl(uintptr_t methodPtr, Value* output, Value* value, NYql::NCodegen::ICodegen& codegen, BasicBlock* block, Value* arg1, Value* arg2) {
+ auto& context = codegen.GetContext();
+
+ const auto data = CastInst::Create(Instruction::Trunc, value, Type::getInt64Ty(context), "data", block);
+ const auto ptrStructType = PointerType::getUnqual(StructType::get(context));
+ const auto boxed = CastInst::Create(Instruction::IntToPtr, data, ptrStructType, "boxed", block);
+
+ const auto funType = (codegen.GetEffectiveTarget() != NYql::NCodegen::ETarget::Windows) ?
+ FunctionType::get(Type::getVoidTy(context), {output->getType(), boxed->getType(), arg1->getType(), arg2->getType()}, false):
+ FunctionType::get(Type::getVoidTy(context), {boxed->getType(), output->getType(), arg1->getType(), arg2->getType()}, false);
+ const auto ptrFunType = PointerType::getUnqual(funType);
+ const auto tableType = PointerType::getUnqual(ptrFunType);
+ const auto vTable = CastInst::Create(Instruction::IntToPtr, data, PointerType::getUnqual(tableType), "vtable", block);
+
+ const auto table = new LoadInst(tableType, vTable, "table", false, block);
+ const auto elem = GetElementPtrInst::CreateInBounds(ptrFunType, table, {ConstantInt::get(Type::getInt64Ty(context), GetMethodPtrIndex(methodPtr))}, "element", block);
+ const auto func = new LoadInst(ptrFunType, elem, "func", false, block);
+
+ if (codegen.GetEffectiveTarget() != NYql::NCodegen::ETarget::Windows) {
+ CallInst::Create(funType, func, {output, boxed, arg1, arg2}, "", block);
+ } else {
+ CallInst::Create(funType, func, {boxed, output, arg1, arg2}, "", block);
+ }
+}
+
+Value* CallBoxedValueVirtualMethodImpl(uintptr_t methodPtr, Type* returnType, Value* value, NYql::NCodegen::ICodegen& codegen, BasicBlock* block, Value* arg1, Value* arg2) {
+ auto& context = codegen.GetContext();
+
+ const auto data = CastInst::Create(Instruction::Trunc, value, Type::getInt64Ty(context), "data", block);
+ const auto ptrStructType = PointerType::getUnqual(StructType::get(context));
+ const auto boxed = CastInst::Create(Instruction::IntToPtr, data, ptrStructType, "boxed", block);
+
+ const auto funType = FunctionType::get(returnType, {boxed->getType(), arg1->getType(), arg2->getType()}, false);
+ const auto ptrFunType = PointerType::getUnqual(funType);
+ const auto tableType = PointerType::getUnqual(ptrFunType);
+ const auto vTable = CastInst::Create(Instruction::IntToPtr, data, PointerType::getUnqual(tableType), "vtable", block);
+
+ const auto table = new LoadInst(tableType, vTable, "table", false, block);
+ const auto elem = GetElementPtrInst::CreateInBounds(ptrFunType, table, {ConstantInt::get(Type::getInt64Ty(context), GetMethodPtrIndex(methodPtr))}, "element", block);
+ const auto func = new LoadInst(ptrFunType, elem, "func", false, block);
+
+ const auto call = CallInst::Create(funType, func, {boxed, arg1, arg2}, returnType->isVoidTy() ? "" : "return", block);
+ return call;
+}
+
+Value* CallUnaryUnboxedValueFunctionImpl(uintptr_t methodPtr, Type* result, Value* arg, NYql::NCodegen::ICodegen& codegen, BasicBlock* block) {
+ auto& context = codegen.GetContext();
+ const auto doFunc = ConstantInt::get(Type::getInt64Ty(context), methodPtr);
+ if (NYql::NCodegen::ETarget::Windows != codegen.GetEffectiveTarget()) {
+ const auto funType = FunctionType::get(result, {arg->getType()}, false);
+ const auto funcPtr = CastInst::Create(Instruction::IntToPtr, doFunc, PointerType::getUnqual(funType), "ptr", block);
+ const auto call = CallInst::Create(funType, funcPtr, {arg}, "call", block);
+ return call;
+ } else {
+ const auto ptrArg = new AllocaInst(arg->getType(), 0U, "arg", block);
+ new StoreInst(arg, ptrArg, block);
+
+ if (Type::getInt128Ty(context) == result) {
+ const auto ptrResult = new AllocaInst(result, 0U, "result", block);
+ const auto funType = FunctionType::get(Type::getVoidTy(context), {ptrResult->getType(), ptrArg->getType()}, false);
+ const auto funcPtr = CastInst::Create(Instruction::IntToPtr, doFunc, PointerType::getUnqual(funType), "ptr", block);
+ CallInst::Create(funType, funcPtr, {ptrResult, ptrArg}, "", block);
+ const auto res = new LoadInst(result, ptrResult, "res", block);
+ return res;
+ } else {
+ const auto funType = FunctionType::get(result, {ptrArg->getType()}, false);
+ const auto funcPtr = CastInst::Create(Instruction::IntToPtr, doFunc, PointerType::getUnqual(funType), "ptr", block);
+ const auto call = CallInst::Create(funType, funcPtr, {ptrArg}, "call", block);
+ return call;
+ }
+ }
+}
+
+Value* CallBinaryUnboxedValueFunctionImpl(uintptr_t methodPtr, Type* result, Value* left, Value* right, NYql::NCodegen::ICodegen& codegen, BasicBlock* block) {
+ auto& context = codegen.GetContext();
+ const auto doFunc = ConstantInt::get(Type::getInt64Ty(context), methodPtr);
+ if (NYql::NCodegen::ETarget::Windows != codegen.GetEffectiveTarget()) {
+ const auto funType = FunctionType::get(result, {left->getType(), right->getType()}, false);
+ const auto funcPtr = CastInst::Create(Instruction::IntToPtr, doFunc, PointerType::getUnqual(funType), "ptr", block);
+ const auto call = CallInst::Create(funType, funcPtr, {left, right}, "call", block);
+ return call;
+ } else {
+ const auto ptrLeft = new AllocaInst(left->getType(), 0U, "left", block);
+ const auto ptrRight = new AllocaInst(right->getType(), 0U, "right", block);
+ new StoreInst(left, ptrLeft, block);
+ new StoreInst(right, ptrRight, block);
+
+ if (Type::getInt128Ty(context) == result) {
+ const auto ptrResult = new AllocaInst(result, 0U, "result", block);
+ const auto funType = FunctionType::get(Type::getVoidTy(context), {ptrResult->getType(), ptrLeft->getType(), ptrRight->getType()}, false);
+ const auto funcPtr = CastInst::Create(Instruction::IntToPtr, doFunc, PointerType::getUnqual(funType), "ptr", block);
+ CallInst::Create(funType, funcPtr, {ptrResult, ptrLeft, ptrRight}, "", block);
+ const auto res = new LoadInst(result, ptrResult, "res", block);
+ return res;
+ } else {
+ const auto funType = FunctionType::get(result, {ptrLeft->getType(), ptrRight->getType()}, false);
+ const auto funcPtr = CastInst::Create(Instruction::IntToPtr, doFunc, PointerType::getUnqual(funType), "ptr", block);
+ const auto call = CallInst::Create(funType, funcPtr, {ptrLeft, ptrRight}, "call", block);
+ return call;
+ }
+ }
+}
+
+Y_NO_INLINE Value* TDecoratorCodegeneratorNodeBase::CreateGetValueImpl(IComputationNode* node,
+ const TCodegenContext& ctx, BasicBlock*& block) const {
+ const auto arg = GetNodeValue(node, ctx, block);
+ const auto value = DoGenerateGetValue(ctx, arg, block);
+ if (value->getType()->isPointerTy()) {
+ const auto load = new LoadInst(Type::getInt128Ty(ctx.Codegen.GetContext()), value, "load", block);
+ ValueRelease(node->GetRepresentation(), load, ctx, block);
+ return load;
+ } else {
+ return value;
+ }
+}
+
+Y_NO_INLINE Value* TStatelessFlowCodegeneratorNodeBase::CreateGetValueImpl(const IComputationNode* node,
+ const TCodegenContext& ctx, BasicBlock*& block) const {
+ const auto value = DoGenerateGetValue(ctx, block);
+ if (value->getType()->isPointerTy()) {
+ const auto load = new LoadInst(Type::getInt128Ty(ctx.Codegen.GetContext()), value, "load", block);
+ ValueRelease(node->GetRepresentation(), load, ctx, block);
+ return load;
+ } else {
+ return value;
+ }
+}
+
+Y_NO_INLINE ICodegeneratorInlineWideNode::TGenerateResult TStatelessWideFlowCodegeneratorNodeBase::GenGetValuesImpl(
+ const TCodegenContext& ctx, BasicBlock*& block) const {
+ return DoGenGetValues(ctx, block);
+}
+
+Y_NO_INLINE Value* TFlowSourceCodegeneratorNodeBase::CreateGetValueImpl(
+ const IComputationNode* node, const TCodegenContext& ctx, BasicBlock*& block) const {
+ auto& context = ctx.Codegen.GetContext();
+ const auto valueType = Type::getInt128Ty(context);
+ const auto statePtr = GetElementPtrInst::CreateInBounds(valueType, ctx.GetMutables(), {ConstantInt::get(Type::getInt32Ty(context), node->GetIndex())}, "state_ptr", block);
+
+ const auto value = DoGenerateGetValue(ctx, statePtr, block);
+ if (value->getType()->isPointerTy()) {
+ const auto load = new LoadInst(valueType, value, "load", block);
+ ValueRelease(node->GetRepresentation(), load, ctx, block);
+ return load;
+ } else {
+ return value;
+ }
+}
+
+Y_NO_INLINE ICodegeneratorInlineWideNode::TGenerateResult TWideFlowSourceCodegeneratorNodeBase::GenGetValuesImpl(
+ const IComputationNode* node, const TCodegenContext& ctx, BasicBlock*& block) const {
+ auto& context = ctx.Codegen.GetContext();
+ const auto valueType = Type::getInt128Ty(context);
+ const auto statePtr = GetElementPtrInst::CreateInBounds(valueType, ctx.GetMutables(), {ConstantInt::get(Type::getInt32Ty(context), node->GetIndex())}, "state_ptr", block);
+ return DoGenGetValues(ctx, statePtr, block);
+}
+
+Y_NO_INLINE Value* TStatefulFlowCodegeneratorNodeBase::CreateGetValueImpl(
+ const IComputationNode* node, const TCodegenContext& ctx, BasicBlock*& block) const {
+ auto& context = ctx.Codegen.GetContext();
+ const auto valueType = Type::getInt128Ty(context);
+ const auto statePtr = GetElementPtrInst::CreateInBounds(valueType, ctx.GetMutables(), {ConstantInt::get(Type::getInt32Ty(context), node->GetIndex())}, "state_ptr", block);
+
+ const auto value = DoGenerateGetValue(ctx, statePtr, block);
+ if (value->getType()->isPointerTy()) {
+ const auto load = new LoadInst(valueType, value, "load", block);
+ ValueRelease(node->GetRepresentation(), load, ctx, block);
+ return load;
+ } else {
+ return value;
+ }
+}
+
+Y_NO_INLINE ICodegeneratorInlineWideNode::TGenerateResult TStatefulWideFlowCodegeneratorNodeBase::GenGetValuesImpl(
+ const IComputationNode* node, const TCodegenContext& ctx, BasicBlock*& block) const {
+ auto& context = ctx.Codegen.GetContext();
+ const auto valueType = Type::getInt128Ty(context);
+ const auto statePtr = GetElementPtrInst::CreateInBounds(valueType, ctx.GetMutables(), {ConstantInt::get(Type::getInt32Ty(context), node->GetIndex())}, "state_ptr", block);
+ return DoGenGetValues(ctx, statePtr, block);
+}
+
+Y_NO_INLINE ICodegeneratorInlineWideNode::TGenerateResult TPairStateWideFlowCodegeneratorNodeBase::GenGetValuesImpl(
+ const IComputationNode* node, const TCodegenContext& ctx, BasicBlock*& block) const {
+ auto& context = ctx.Codegen.GetContext();
+ auto idx = node->GetIndex();
+ const auto valueType = Type::getInt128Ty(context);
+ const auto firstPtr = GetElementPtrInst::CreateInBounds(valueType, ctx.GetMutables(), {ConstantInt::get(Type::getInt32Ty(context), idx)}, "first_ptr", block);
+ const auto secondPtr = GetElementPtrInst::CreateInBounds(valueType, ctx.GetMutables(), {ConstantInt::get(Type::getInt32Ty(context), ++idx)}, "second_ptr", block);
+ return DoGenGetValues(ctx, firstPtr, secondPtr, block);
+}
+
+Y_NO_INLINE Value* TPairStateFlowCodegeneratorNodeBase::CreateGetValueImpl(
+ const IComputationNode* node, const TCodegenContext& ctx, BasicBlock*& block) const {
+ auto& context = ctx.Codegen.GetContext();
+ auto idx = node->GetIndex();
+ const auto valueType = Type::getInt128Ty(context);
+ const auto firstPtr = GetElementPtrInst::CreateInBounds(valueType, ctx.GetMutables(), {ConstantInt::get(Type::getInt32Ty(context), idx)}, "first_ptr", block);
+ const auto secondPtr = GetElementPtrInst::CreateInBounds(valueType, ctx.GetMutables(), {ConstantInt::get(Type::getInt32Ty(context), ++idx)}, "second_ptr", block);
+
+ const auto value = DoGenerateGetValue(ctx, firstPtr, secondPtr, block);
+ if (value->getType()->isPointerTy()) {
+ const auto load = new LoadInst(valueType, value, "load", block);
+ ValueRelease(node->GetRepresentation(), load, ctx, block);
+ return load;
+ } else {
+ return value;
+ }
+}
+
+Y_NO_INLINE Value* TBinaryCodegeneratorNodeBase::CreateGetValueImpl(const IComputationNode* node,
+ const TCodegenContext& ctx, BasicBlock*& block) const {
+ const auto value = DoGenerateGetValue(ctx, block);
+ if (value->getType()->isPointerTy()) {
+ ValueRelease(node->GetRepresentation(), value, ctx, block);
+ const auto load = new LoadInst(Type::getInt128Ty(ctx.Codegen.GetContext()), value, "load", block);
+ return load;
+ } else {
+ return value;
+ }
+}
+
+Y_NO_INLINE Value* TMutableCodegeneratorNodeBase::CreateGetValueImpl(
+ bool stateless, EValueRepresentation representation, ui32 valueIndex,
+ const TString& name, const TCodegenContext& ctx, BasicBlock*& block) const {
+ if (stateless) {
+ const auto newValue = DoGenerateGetValue(ctx, block);
+ if (newValue->getType()->isPointerTy()) {
+ ValueRelease(representation, newValue, ctx, block);
+ const auto load = new LoadInst(Type::getInt128Ty(ctx.Codegen.GetContext()), newValue, "load", block);
+ return load;
+ } else {
+ return newValue;
+ }
+ }
+
+ return ctx.AlwaysInline ? MakeGetValueBody(representation, valueIndex, ctx, block) :
+ CallInst::Create(GenerateInternalGetValue(name, representation, valueIndex, ctx.Codegen), {ctx.Ctx}, "getter", block);
+}
+
+Function* TMutableCodegeneratorNodeBase::GenerateInternalGetValue(const TString& name,
+ EValueRepresentation representation, ui32 valueIndex, NYql::NCodegen::ICodegen& codegen) const {
+ auto& module = codegen.GetModule();
+ auto& context = codegen.GetContext();
+ if (const auto f = module.getFunction(name.c_str()))
+ return f;
+
+ const auto funcType = FunctionType::get(Type::getInt128Ty(context), {PointerType::getUnqual(GetCompContextType(context))}, false);
+
+ TCodegenContext ctx(codegen);
+ ctx.Func = cast<Function>(module.getOrInsertFunction(name.c_str(), funcType).getCallee());
+
+ DISubprogramAnnotator annotator(ctx, ctx.Func);
+
+
+ auto main = BasicBlock::Create(context, "main", ctx.Func);
+ ctx.Ctx = &*ctx.Func->arg_begin();
+ ctx.Ctx->addAttr(Attribute::NonNull);
+
+ const auto get = MakeGetValueBody(representation, valueIndex, ctx, main);
+
+ ReturnInst::Create(context, get, main);
+ return ctx.Func;
+}
+
+Value* TMutableCodegeneratorNodeBase::MakeGetValueBody(EValueRepresentation representation, ui32 valueIndex, const TCodegenContext& ctx, BasicBlock*& block) const {
+ auto& context = ctx.Codegen.GetContext();
+ const auto indexType = Type::getInt32Ty(context);
+ const auto valueType = Type::getInt128Ty(context);
+ const auto valuePtr = GetElementPtrInst::CreateInBounds(valueType, ctx.GetMutables(), {ConstantInt::get(indexType, valueIndex)}, "value_ptr", block);
+ const auto value = new LoadInst(valueType, valuePtr, "value", block);
+
+ const auto invv = ConstantInt::get(value->getType(), 0xFFFFFFFFFFFFFFFFULL);
+
+ const auto check = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, value, invv, "check", block);
+
+ const auto comp = BasicBlock::Create(context, "comp", ctx.Func);
+ const auto done = BasicBlock::Create(context, "done", ctx.Func);
+
+ BranchInst::Create(comp, done, check, block);
+
+ block = comp;
+
+ const auto newValue = DoGenerateGetValue(ctx, block);
+
+ if (newValue->getType()->isPointerTy()) {
+ const auto load = new LoadInst(valueType, newValue, "value", block);
+ new StoreInst(load, valuePtr, block);
+ new StoreInst(ConstantInt::get(load->getType(), 0), newValue, block);
+ } else {
+ new StoreInst(newValue, valuePtr, block);
+ ValueAddRef(representation, valuePtr, ctx, block);
+ }
+
+ BranchInst::Create(done, block);
+ block = done;
+
+ const auto result = new LoadInst(valueType, valuePtr, "result", false, block);
+ return result;
+}
+
+Y_NO_INLINE Value* TMutableCodegeneratorPtrNodeBase::CreateGetValueImpl(
+ bool stateless, EValueRepresentation representation, ui32 valueIndex,
+ const TString& name, const TCodegenContext& ctx, BasicBlock*& block) const {
+ if (stateless) {
+ const auto type = Type::getInt128Ty(ctx.Codegen.GetContext());
+ const auto pointer = ctx.Func->getEntryBlock().empty() ?
+ new AllocaInst(type, 0U, "output", &ctx.Func->getEntryBlock()):
+ new AllocaInst(type, 0U, "output", &ctx.Func->getEntryBlock().back());
+
+ DoGenerateGetValue(ctx, pointer, block);
+ ValueRelease(representation, pointer, ctx, block);
+ const auto load = new LoadInst(type, pointer, "load", block);
+ return load;
+ }
+
+ return ctx.AlwaysInline ? MakeGetValueBody(valueIndex, ctx, block) :
+ CallInst::Create(GenerateInternalGetValue(name, valueIndex, ctx.Codegen), {ctx.Ctx}, "getter", block);
+}
+
+Value* TMutableCodegeneratorPtrNodeBase::MakeGetValueBody(ui32 valueIndex, const TCodegenContext& ctx, BasicBlock*& block) const {
+ auto& context = ctx.Codegen.GetContext();
+ const auto indexType = Type::getInt32Ty(context);
+ const auto valueType = Type::getInt128Ty(context);
+ const auto valuePtr = GetElementPtrInst::CreateInBounds(valueType, ctx.GetMutables(), {ConstantInt::get(indexType, valueIndex)}, "value_ptr", block);
+ const auto value = new LoadInst(valueType, valuePtr, "value", block);
+
+ const auto invv = ConstantInt::get(value->getType(), 0xFFFFFFFFFFFFFFFFULL);
+
+ const auto check = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, value, invv, "check", block);
+
+ const auto comp = BasicBlock::Create(context, "comp", ctx.Func);
+ const auto done = BasicBlock::Create(context, "done", ctx.Func);
+
+ BranchInst::Create(comp, done, check, block);
+
+ block = comp;
+
+ DoGenerateGetValue(ctx, valuePtr, block);
+
+ BranchInst::Create(done, block);
+ block = done;
+
+ const auto result = new LoadInst(valueType, valuePtr, "result", false, block);
+ return result;
+}
+
+Function* TMutableCodegeneratorPtrNodeBase::GenerateInternalGetValue(const TString& name, ui32 valueIndex, NYql::NCodegen::ICodegen& codegen) const {
+ auto& module = codegen.GetModule();
+ auto& context = codegen.GetContext();
+ if (const auto f = module.getFunction(name.c_str()))
+ return f;
+
+ const auto contextType = GetCompContextType(context);
+
+ const auto funcType = FunctionType::get(Type::getInt128Ty(context), {PointerType::getUnqual(contextType)}, false);
+
+ TCodegenContext ctx(codegen);
+ ctx.Func = cast<Function>(module.getOrInsertFunction(name.c_str(), funcType).getCallee());
+
+ DISubprogramAnnotator annotator(ctx, ctx.Func);
+
+
+ auto main = BasicBlock::Create(context, "main", ctx.Func);
+ ctx.Ctx = &*ctx.Func->arg_begin();
+ ctx.Ctx->addAttr(Attribute::NonNull);
+
+ const auto get = MakeGetValueBody(valueIndex, ctx, main);
+
+ ReturnInst::Create(context, get, main);
+ return ctx.Func;
+}
+
+Y_NO_INLINE Value* TMutableCodegeneratorFallbackNodeBase::DoGenerateGetValueImpl(
+ uintptr_t methodPtr, uintptr_t thisPtr, const TCodegenContext& ctx, BasicBlock*& block) const {
+ auto& context = ctx.Codegen.GetContext();
+ const auto type = Type::getInt128Ty(context);
+ const auto ptrType = PointerType::getUnqual(StructType::get(context));
+ const auto doFunc = ConstantInt::get(Type::getInt64Ty(context), methodPtr);
+ const auto self = CastInst::Create(Instruction::IntToPtr, ConstantInt::get(Type::getInt64Ty(context), thisPtr), ptrType, "self", block);
+ if (NYql::NCodegen::ETarget::Windows != ctx.Codegen.GetEffectiveTarget()) {
+ const auto funType = FunctionType::get(type, {self->getType(), ctx.Ctx->getType()}, false);
+ const auto doFuncPtr = CastInst::Create(Instruction::IntToPtr, doFunc, PointerType::getUnqual(funType), "function", block);
+ const auto value = CallInst::Create(funType, doFuncPtr, {self, ctx.Ctx}, "value", block);
+ return value;
+ } else {
+ const auto resultPtr = new AllocaInst(type, 0U, "return", block);
+ const auto funType = FunctionType::get(Type::getVoidTy(context), {self->getType(), resultPtr->getType(), ctx.Ctx->getType()}, false);
+ const auto doFuncPtr = CastInst::Create(Instruction::IntToPtr, doFunc, PointerType::getUnqual(funType), "function", block);
+ CallInst::Create(funType, doFuncPtr, {self, resultPtr, ctx.Ctx}, "", block);
+ const auto value = new LoadInst(type, resultPtr, "value", block);
+ return value;
+ }
+}
+
+Y_NO_INLINE Function* TCodegeneratorRootNodeBase::GenerateGetValueImpl(
+ const TString& name, const ICodegeneratorInlineNode* gen, NYql::NCodegen::ICodegen& codegen) {
+ auto& module = codegen.GetModule();
+ auto& context = codegen.GetContext();
+
+ if (const auto f = module.getFunction(name.c_str()))
+ return f;
+
+ const auto valueType = Type::getInt128Ty(context);
+ const auto contextType = GetCompContextType(context);
+
+ const auto funcType = codegen.GetEffectiveTarget() != NYql::NCodegen::ETarget::Windows ?
+ FunctionType::get(valueType, {PointerType::getUnqual(contextType)}, false):
+ FunctionType::get(Type::getVoidTy(context) , {PointerType::getUnqual(valueType), PointerType::getUnqual(contextType)}, false);
+
+ TCodegenContext ctx(codegen);
+ ctx.Func = cast<Function>(module.getOrInsertFunction(name.c_str(), funcType).getCallee());
+
+ DISubprogramAnnotator annotator(ctx, ctx.Func);
+
+
+ auto args = ctx.Func->arg_begin();
+ if (codegen.GetEffectiveTarget() == NYql::NCodegen::ETarget::Windows) {
+ auto& firstArg = *args++;
+ firstArg.addAttr(Attribute::StructRet);
+ firstArg.addAttr(Attribute::NoAlias);
+ }
+
+ auto main = BasicBlock::Create(context, "main", ctx.Func);
+ ctx.Ctx = &*args;
+ ctx.Ctx->addAttr(Attribute::NonNull);
+
+ const auto get = gen->CreateGetValue(ctx, main);
+
+ if (codegen.GetEffectiveTarget() != NYql::NCodegen::ETarget::Windows) {
+ ReturnInst::Create(context, get, main);
+ } else {
+ new StoreInst(get, &*--args, main);
+ ReturnInst::Create(context, main);
+ }
+
+ return ctx.Func;
+}
+
+#if __clang__ && (__clang_major__ < 16)
+TSrcLocation TSrcLocation::current() {
+ return {};
+}
+
+const char* TSrcLocation::file_name() const {
+ return __FILE__;
+}
+
+size_t TSrcLocation::line() const {
+ return __LINE__;
+}
+
+size_t TSrcLocation::column() const {
+ return 0;
+}
+#endif
+
+DISubprogramAnnotator::DISubprogramAnnotator(TCodegenContext& ctx, Function* subprogramFunc, const TSrcLocation& location)
+ : Ctx(ctx)
+ , DebugBuilder(std::make_unique<DIBuilder>(ctx.Codegen.GetModule()))
+ , Subprogram(MakeDISubprogram(subprogramFunc->getName(), location))
+ , Func(subprogramFunc)
+{
+ subprogramFunc->setSubprogram(Subprogram);
+ Ctx.Annotator = this;
+}
+
+DISubprogramAnnotator::~DISubprogramAnnotator() {
+ Ctx.Annotator = nullptr;
+ { // necessary stub annotation of "CallInst"s
+ DIScopeAnnotator stubAnnotate(this);
+ for (BasicBlock& block : *Func) {
+ for (Instruction& inst : block) {
+ if (CallInst* callInst = dyn_cast_or_null<CallInst>(&inst)) {
+ const auto& debugLoc = callInst->getDebugLoc();
+ if (!debugLoc) {
+ stubAnnotate(callInst);
+ }
+ }
+ }
+ }
+ }
+ DebugBuilder->finalizeSubprogram(Subprogram);
+}
+
+DIFile* DISubprogramAnnotator::MakeDIFile(const TSrcLocation& location) {
+ TFsPath path = TString(location.file_name());
+ return DebugBuilder->createFile(path.GetName().c_str(), path.Parent().GetPath().c_str());
+}
+
+DISubprogram* DISubprogramAnnotator::MakeDISubprogram(const StringRef& name, const TSrcLocation& location) {
+ const auto file = MakeDIFile(location);
+ const auto unit = DebugBuilder->createCompileUnit(llvm::dwarf::DW_LANG_C_plus_plus, file, "MKQL", false, "", 0);
+ const auto subroutineType = DebugBuilder->createSubroutineType(DebugBuilder->getOrCreateTypeArray({}));
+ return DebugBuilder->createFunction(
+ unit,
+ name,
+ llvm::StringRef(),
+ file, 0,
+ subroutineType, 0, llvm::DINode::FlagPrototyped, llvm::DISubprogram::SPFlagDefinition
+ );
+}
+
+DIScopeAnnotator::DIScopeAnnotator(DISubprogramAnnotator* subprogramAnnotator, const TSrcLocation& location)
+ : SubprogramAnnotator(nullptr)
+ , Scope(nullptr)
+{
+ Y_ENSURE(subprogramAnnotator != nullptr);
+ SubprogramAnnotator = subprogramAnnotator;
+ Scope = SubprogramAnnotator->DebugBuilder->createLexicalBlock(SubprogramAnnotator->Subprogram, SubprogramAnnotator->MakeDIFile(location), location.line(), location.column());
+}
+
+Instruction* DIScopeAnnotator::operator()(Instruction* inst, const TSrcLocation& location) const {
+ inst->setDebugLoc(DILocation::get(SubprogramAnnotator->Ctx.Codegen.GetContext(), location.line(), location.column(), Scope));
+ return inst;
+}
+
+}
+}
+#endif