diff options
author | vvvv <[email protected]> | 2024-11-07 04:19:26 +0300 |
---|---|---|
committer | vvvv <[email protected]> | 2024-11-07 04:29:50 +0300 |
commit | 2661be00f3bc47590fda9218bf0386d6355c8c88 (patch) | |
tree | 3d316c07519191283d31c5f537efc6aabb42a2f0 /yql/essentials/minikql/computation/mkql_computation_node_codegen.cpp | |
parent | cf2a23963ac10add28c50cc114fbf48953eca5aa (diff) |
Moved yql/minikql YQL-19206
init
[nodiff:caesar]
commit_hash:d1182ef7d430ccf7e4d37ed933c7126d7bd5d6e4
Diffstat (limited to 'yql/essentials/minikql/computation/mkql_computation_node_codegen.cpp')
-rw-r--r-- | yql/essentials/minikql/computation/mkql_computation_node_codegen.cpp | 2670 |
1 files changed, 2670 insertions, 0 deletions
diff --git a/yql/essentials/minikql/computation/mkql_computation_node_codegen.cpp b/yql/essentials/minikql/computation/mkql_computation_node_codegen.cpp new file mode 100644 index 00000000000..113e9da8dcc --- /dev/null +++ b/yql/essentials/minikql/computation/mkql_computation_node_codegen.cpp @@ -0,0 +1,2670 @@ +#include "mkql_computation_node_codegen.h" // Y_IGNORE +#include "mkql_computation_node_holders.h" + +#include <yql/essentials/minikql/codegen/codegen.h> +#include <yql/essentials/public/decimal/yql_decimal.h> + +#include <util/string/cast.h> +#include <util/folder/path.h> + +#ifndef MKQL_DISABLE_CODEGEN + +extern "C" void DeleteBoxed(NKikimr::NUdf::IBoxedValue *const boxed) { + delete boxed; +} + +extern "C" void DeleteString(void* strData) { + auto& str = *(NKikimr::NUdf::TStringValue*)(&strData); + UdfFreeWithSize(strData, 16 + str.Capacity()); +} + +namespace NKikimr { +namespace NMiniKQL { + +constexpr bool EnableStaticRefcount = true; + +using namespace llvm; + +Type* GetCompContextType(LLVMContext &context) { + const auto ptrValueType = PointerType::getUnqual(Type::getInt128Ty(context)); + const auto structPtrType = PointerType::getUnqual(StructType::get(context)); + const auto stringRefType = StructType::get(context, { + Type::getInt8PtrTy(context), + Type::getInt32Ty(context), + Type::getInt32Ty(context) + }); + const auto sourcePosType = StructType::get(context, { + Type::getInt32Ty(context), + Type::getInt32Ty(context), + stringRefType + }); + return StructType::get(context, { + structPtrType, // factory + structPtrType, // stats + ptrValueType, // mutables + structPtrType, // builder + Type::getFloatTy(context), // adjustor + Type::getInt32Ty(context), // rsscounter + PointerType::getUnqual(sourcePosType) + }); +} + +Value* TCodegenContext::GetFactory() const { + if (!Factory) { + auto& context = Codegen.GetContext(); + const auto indexType = Type::getInt32Ty(context); + const auto ptrType = PointerType::getUnqual(StructType::get(context)); + if (Func->getEntryBlock().empty()) { + const auto ptr = GetElementPtrInst::CreateInBounds(GetCompContextType(context), Ctx, {ConstantInt::get(indexType, 0), ConstantInt::get(indexType, 0)}, "factory_ptr", &Func->getEntryBlock()); + const_cast<Value*&>(Factory) = new LoadInst(ptrType, ptr, "factory", &Func->getEntryBlock()); + } else { + const auto ptr = GetElementPtrInst::CreateInBounds(GetCompContextType(context), Ctx, {ConstantInt::get(indexType, 0), ConstantInt::get(indexType, 0)}, "factory_ptr", &Func->getEntryBlock().front()); + const_cast<Value*&>(Factory) = new LoadInst(ptrType, ptr, "factory", &Func->getEntryBlock().back()); + } + } + return Factory; +} + +Value* TCodegenContext::GetStat() const { + if (!Stat) { + auto& context = Codegen.GetContext(); + const auto indexType = Type::getInt32Ty(context); + const auto ptrType = PointerType::getUnqual(StructType::get(context)); + if (Func->getEntryBlock().empty()) { + const auto ptr = GetElementPtrInst::CreateInBounds(GetCompContextType(context), Ctx, {ConstantInt::get(indexType, 0), ConstantInt::get(indexType, 1)}, "stat_ptr", &Func->getEntryBlock()); + const_cast<Value*&>(Stat) = new LoadInst(ptrType, ptr, "stat", &Func->getEntryBlock()); + } else { + const auto ptr = GetElementPtrInst::CreateInBounds(GetCompContextType(context), Ctx, {ConstantInt::get(indexType, 0), ConstantInt::get(indexType, 1)}, "stat_ptr", &Func->getEntryBlock().front()); + const_cast<Value*&>(Stat) = new LoadInst(ptrType, ptr, "stat", &Func->getEntryBlock().back()); + } + } + return Stat; +} + +Value* TCodegenContext::GetMutables() const { + if (!Mutables) { + auto& context = Codegen.GetContext(); + const auto indexType = Type::getInt32Ty(context); + const auto ptrType = PointerType::getUnqual(Type::getInt128Ty(context)); + if (Func->getEntryBlock().empty()) { + const auto ptr = GetElementPtrInst::CreateInBounds(GetCompContextType(context), Ctx, {ConstantInt::get(indexType, 0), ConstantInt::get(indexType, 2)}, "mutables_ptr", &Func->getEntryBlock()); + const_cast<Value*&>(Mutables) = new LoadInst(ptrType, ptr, "mutables", &Func->getEntryBlock()); + } else { + const auto ptr = GetElementPtrInst::CreateInBounds(GetCompContextType(context), Ctx, {ConstantInt::get(indexType, 0), ConstantInt::get(indexType, 2)}, "mutables_ptr", &Func->getEntryBlock().front()); + const_cast<Value*&>(Mutables) = new LoadInst(ptrType, ptr, "mutables", &Func->getEntryBlock().back()); + } + } + return Mutables; +} + +Value* TCodegenContext::GetBuilder() const { + if (!Builder) { + auto& context = Codegen.GetContext(); + const auto indexType = Type::getInt32Ty(context); + const auto ptrType = PointerType::getUnqual(StructType::get(context)); + if (Func->getEntryBlock().empty()) { + const auto ptr = GetElementPtrInst::CreateInBounds(GetCompContextType(context), Ctx, {ConstantInt::get(indexType, 0), ConstantInt::get(indexType, 3)}, "builder_ptr", &Func->getEntryBlock()); + const_cast<Value*&>(Builder) = new LoadInst(ptrType, ptr, "builder", &Func->getEntryBlock()); + } else { + const auto ptr = GetElementPtrInst::CreateInBounds(GetCompContextType(context), Ctx, {ConstantInt::get(indexType, 0), ConstantInt::get(indexType, 3)}, "builder_ptr", &Func->getEntryBlock().front()); + const_cast<Value*&>(Builder) = new LoadInst(ptrType, ptr, "builder", &Func->getEntryBlock().back()); + } + } + return Builder; +} + +Function* GenerateCompareFunction(NYql::NCodegen::ICodegen& codegen, const TString& name, IComputationExternalNode* left, + IComputationExternalNode* right, IComputationNode* compare) { + auto& module = codegen.GetModule(); + if (const auto f = module.getFunction(name.c_str())) + return f; + + const auto codegenLeft = dynamic_cast<ICodegeneratorExternalNode*>(left); + const auto codegenRight = dynamic_cast<ICodegeneratorExternalNode*>(right); + MKQL_ENSURE(codegenLeft, "Left must be codegenerator node."); + MKQL_ENSURE(codegenRight, "Right must be codegenerator node."); + + auto& context = codegen.GetContext(); + const auto valueType = Type::getInt128Ty(context); + const auto ptrType = PointerType::getUnqual(valueType); + const auto returnType = Type::getInt1Ty(context); + const auto contextType = GetCompContextType(context); + + const auto funcType = codegen.GetEffectiveTarget() != NYql::NCodegen::ETarget::Windows ? + FunctionType::get(returnType, {PointerType::getUnqual(contextType), valueType, valueType}, false): + FunctionType::get(returnType, {PointerType::getUnqual(contextType), ptrType, ptrType}, false); + + TCodegenContext ctx(codegen); + ctx.AlwaysInline = true; + ctx.Func = cast<Function>(module.getOrInsertFunction(name.c_str(), funcType).getCallee()); + + DISubprogramAnnotator annotator(ctx, ctx.Func); + + + auto args = ctx.Func->arg_begin(); + + const auto main = BasicBlock::Create(context, "main", ctx.Func); + auto block = main; + + ctx.Ctx = &*args; + ctx.Ctx->addAttr(Attribute::NonNull); + + const auto lv = &*++args; + const auto rv = &*++args; + + codegenLeft->SetValueBuilder([lv](const TCodegenContext&) { return lv; }); + codegenRight->SetValueBuilder([rv](const TCodegenContext&) { return rv; }); + + codegenLeft->CreateInvalidate(ctx, block); + codegenRight->CreateInvalidate(ctx, block); + + const auto res = GetNodeValue(compare, ctx, block); + const auto cast = CastInst::Create(Instruction::Trunc, res, returnType, "bool", block); + ReturnInst::Create(context, cast, block); + + codegenLeft->SetValueBuilder({}); + codegenRight->SetValueBuilder({}); + + return ctx.Func; +} + +Value* GetterFor(NUdf::EDataSlot slot, Value* value, LLVMContext &context, BasicBlock* block) { + switch (slot) { + case NUdf::EDataSlot::Bool: return GetterFor<bool>(value, context, block); + case NUdf::EDataSlot::Decimal: return GetterForInt128(value, block); + case NUdf::EDataSlot::Float: return GetterFor<float>(value, context, block); + case NUdf::EDataSlot::Double: return GetterFor<double>(value, context, block); + default: break; + } + + const auto trunc = CastInst::Create(Instruction::Trunc, value, IntegerType::get(context, NUdf::GetDataTypeInfo(slot).FixedSize << 3U), "trunc", block); + return trunc; +} + +namespace { + +Value* GetMarkFromUnboxed(Value* value, const TCodegenContext& ctx, BasicBlock* block) { + auto& context = ctx.Codegen.GetContext(); + + const auto type8 = Type::getInt8Ty(context); + if (value->getType()->isPointerTy()) { + const auto type = StructType::get(context, {PointerType::getUnqual(StructType::get(context)), ArrayType::get(type8, 8U)}); + const auto cast = CastInst::Create(Instruction::BitCast, value, PointerType::getUnqual(type), "cast", block); + const auto type32 = Type::getInt32Ty(context); + const auto metaptr = GetElementPtrInst::CreateInBounds(type, cast, {ConstantInt::get(type32, 0), ConstantInt::get(type32, 1), ConstantInt::get(type32, 7)}, "metaptr", block); + const auto meta = new LoadInst(type8, metaptr, "meta", block); + const auto mark = BinaryOperator::CreateAnd(meta, ConstantInt::get(meta->getType(), 3), "mark", block); + return mark; + } else { + const auto lshr = BinaryOperator::CreateLShr(value, ConstantInt::get(value->getType(), 120), "lshr", block); + const auto meta = CastInst::Create(Instruction::Trunc, lshr, type8, "meta", block); + const auto mark = BinaryOperator::CreateAnd(ConstantInt::get(meta->getType(), 3), meta, "mark", block); + return mark; + } + +} + +template<bool BoxedOrString> +Value* GetPointerFromUnboxed(Value* value, const TCodegenContext& ctx, BasicBlock* block) { + auto& context = ctx.Codegen.GetContext(); + + const auto type32 = Type::getInt32Ty(context); + const auto type64 = Type::getInt64Ty(context); + const auto type = PointerType::getUnqual(BoxedOrString ? + StructType::get(context, {PointerType::getUnqual(StructType::get(context)), type32, Type::getInt16Ty(context)}): + StructType::get(context, {type32, type32, type32, type32}) + ); + if (value->getType()->isPointerTy()) { + const auto strType = StructType::get(context, {type, type64}); + const auto cast = CastInst::Create(Instruction::BitCast, value, PointerType::getUnqual(strType), "cast", block); + const auto ptr = GetElementPtrInst::CreateInBounds(strType, cast, {ConstantInt::get(type32, 0), ConstantInt::get(type32, 0)}, "ptr", block); + const auto pointer = new LoadInst(type, ptr, "pointer", block); + return pointer; + } else { + const auto half = CastInst::Create(Instruction::Trunc, value, type64, "half", block); + const auto pointer = CastInst::Create(Instruction::IntToPtr, half, type, "pointer", block); + return pointer; + } +} + +ui32 MyCompareStrings(NUdf::TUnboxedValuePod lhs, NUdf::TUnboxedValuePod rhs) { + return NUdf::CompareStrings(lhs, rhs); +} + +bool MyEquteStrings(NUdf::TUnboxedValuePod lhs, NUdf::TUnboxedValuePod rhs) { + return NUdf::EquateStrings(lhs, rhs); +} + +NUdf::THashType MyHashString(NUdf::TUnboxedValuePod val) { + return NUdf::GetStringHash(val); +} + +template <bool IsOptional> +Value* GenEqualsFunction(NUdf::EDataSlot slot, Value* lv, Value* rv, TCodegenContext& ctx, BasicBlock*& block); + +template <> +Value* GenEqualsFunction<false>(NUdf::EDataSlot slot, Value* lv, Value* rv, TCodegenContext& ctx, BasicBlock*& block) { + auto& context = ctx.Codegen.GetContext(); + + const auto& info = NUdf::GetDataTypeInfo(slot); + + if ((info.Features & NUdf::EDataTypeFeatures::CommonType) && (info.Features & NUdf::EDataTypeFeatures::StringType || NUdf::EDataSlot::Uuid == slot || NUdf::EDataSlot::DyNumber == slot)) { + return CallBinaryUnboxedValueFunction(&MyEquteStrings, Type::getInt1Ty(context), lv, rv, ctx.Codegen, block); + } + + const auto lhs = GetterFor(slot, lv, context, block); + const auto rhs = GetterFor(slot, rv, context, block); + + if (info.Features & (NUdf::EDataTypeFeatures::IntegralType | NUdf::EDataTypeFeatures::DateType | NUdf::EDataTypeFeatures::TimeIntervalType | NUdf::EDataTypeFeatures::DecimalType) || NUdf::EDataSlot::Bool == slot) { + const auto equal = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, lhs, rhs, "equal", block); + return equal; + } + + if (info.Features & NUdf::EDataTypeFeatures::FloatType) { + const auto ueq = CmpInst::Create(Instruction::FCmp, FCmpInst::FCMP_UEQ, lhs, rhs, "equals", block); + const auto lord = CmpInst::Create(Instruction::FCmp, FCmpInst::FCMP_ORD, ConstantFP::get(lhs->getType(), 0.0), lhs, "lord", block); + const auto runo = CmpInst::Create(Instruction::FCmp, FCmpInst::FCMP_UNO, ConstantFP::get(rhs->getType(), 0.0), rhs, "runo", block); + const auto once = BinaryOperator::CreateXor(lord, runo, "xor", block); + return BinaryOperator::CreateAnd(ueq, once, "and", block); + } + + if (info.Features & NUdf::EDataTypeFeatures::TzDateType) { + const auto ltz = GetterForTimezone(context, lv, block); + const auto rtz = GetterForTimezone(context, rv, block); + + const auto one = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, lhs, rhs, "one", block); + const auto two = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, ltz, rtz, "two", block); + + return BinaryOperator::CreateAnd(one, two, "and", block); + } + + return nullptr; +} + +template <> +Value* GenEqualsFunction<true>(NUdf::EDataSlot slot, Value* lv, Value* rv, TCodegenContext& ctx, BasicBlock*& block) { + auto& context = ctx.Codegen.GetContext(); + + const auto tiny = BasicBlock::Create(context, "tiny", ctx.Func); + const auto test = BasicBlock::Create(context, "test", ctx.Func); + const auto done = BasicBlock::Create(context, "done", ctx.Func); + + const auto res = PHINode::Create(Type::getInt1Ty(context), 2U, "result", done); + + const auto le = IsEmpty(lv, block); + const auto re = IsEmpty(rv, block); + + const auto any = BinaryOperator::CreateOr(le, re, "or", block); + + BranchInst::Create(tiny, test, any, block); + + block = tiny; + + const auto both = BinaryOperator::CreateAnd(le, re, "and", block); + res->addIncoming(both, block); + BranchInst::Create(done, block); + + block = test; + + const auto comp = GenEqualsFunction<false>(slot, lv, rv, ctx, block); + res->addIncoming(comp, block); + BranchInst::Create(done, block); + + block = done; + return res; +} + +Value* GenEqualsFunction(NUdf::EDataSlot slot, bool isOptional, Value* lv, Value* rv, TCodegenContext& ctx, BasicBlock*& block) { + return isOptional ? GenEqualsFunction<true>(slot, lv, rv, ctx, block) : GenEqualsFunction<false>(slot, lv, rv, ctx, block); +} + +template <bool IsOptional> +Value* GenCompareFunction(NUdf::EDataSlot slot, Value* lv, Value* rv, TCodegenContext& ctx, BasicBlock*& block); + +template <> +Value* GenCompareFunction<false>(NUdf::EDataSlot slot, Value* lv, Value* rv, TCodegenContext& ctx, BasicBlock*& block) { + auto& context = ctx.Codegen.GetContext(); + + const auto& info = NUdf::GetDataTypeInfo(slot); + + if ((info.Features & NUdf::EDataTypeFeatures::CommonType) && (info.Features & NUdf::EDataTypeFeatures::StringType || NUdf::EDataSlot::Uuid == slot || NUdf::EDataSlot::DyNumber == slot)) { + return CallBinaryUnboxedValueFunction(&MyCompareStrings, Type::getInt32Ty(context), lv, rv, ctx.Codegen, block); + } + + const bool extra = info.Features & (NUdf::EDataTypeFeatures::FloatType | NUdf::EDataTypeFeatures::TzDateType); + const auto resultType = Type::getInt32Ty(context); + + const auto exit = BasicBlock::Create(context, "exit", ctx.Func); + const auto test = BasicBlock::Create(context, "test", ctx.Func); + + const auto res = PHINode::Create(resultType, extra ? 3U : 2U, "result", exit); + + const auto lhs = GetterFor(slot, lv, context, block); + const auto rhs = GetterFor(slot, rv, context, block); + + if (info.Features & NUdf::EDataTypeFeatures::FloatType) { + const auto more = BasicBlock::Create(context, "more", ctx.Func); + const auto next = BasicBlock::Create(context, "next", ctx.Func); + + const auto uno = CmpInst::Create(Instruction::FCmp, FCmpInst::FCMP_UNO, lhs, rhs, "unorded", block); + + BranchInst::Create(more, next, uno, block); + block = more; + + const auto luno = CmpInst::Create(Instruction::FCmp, FCmpInst::FCMP_UNO, ConstantFP::get(lhs->getType(), 0.0), lhs, "luno", block); + const auto runo = CmpInst::Create(Instruction::FCmp, FCmpInst::FCMP_UNO, ConstantFP::get(rhs->getType(), 0.0), rhs, "runo", block); + const auto once = BinaryOperator::CreateXor(luno, runo, "xor", block); + + const auto left = SelectInst::Create(luno, ConstantInt::get(resultType, 1), ConstantInt::get(resultType, -1), "left", block); + const auto both = SelectInst::Create(once, left, ConstantInt::get(resultType, 0), "both", block); + + res->addIncoming(both, block); + BranchInst::Create(exit, block); + + block = next; + } + + const auto equals = info.Features & NUdf::EDataTypeFeatures::FloatType ? + CmpInst::Create(Instruction::FCmp, FCmpInst::FCMP_OEQ, lhs, rhs, "equals", block): + CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, lhs, rhs, "equals", block); + + if (info.Features & NUdf::EDataTypeFeatures::TzDateType) { + const auto more = BasicBlock::Create(context, "more", ctx.Func); + const auto next = BasicBlock::Create(context, "next", ctx.Func); + + BranchInst::Create(more, test, equals, block); + + block = more; + + const auto ltz = GetterForTimezone(context, lv, block); + const auto rtz = GetterForTimezone(context, rv, block); + const auto tzeq = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, ltz, rtz, "tzeq", block); + res->addIncoming(ConstantInt::get(resultType, 0), block); + BranchInst::Create(exit, next, tzeq, block); + + block = next; + const auto tzlt = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_ULT, ltz, rtz, "tzlt", block); + const auto tzout = SelectInst::Create(tzlt, ConstantInt::get(resultType, -1), ConstantInt::get(resultType, 1), "tzout", block); + res->addIncoming(tzout, block); + BranchInst::Create(exit, block); + } else { + res->addIncoming(ConstantInt::get(resultType, 0), block); + BranchInst::Create(exit, test, equals, block); + } + + block = test; + + const auto less = info.Features & NUdf::EDataTypeFeatures::FloatType ? + CmpInst::Create(Instruction::FCmp, FCmpInst::FCMP_OLT, lhs, rhs, "less", block): // float + info.Features & (NUdf::EDataTypeFeatures::SignedIntegralType | NUdf::EDataTypeFeatures::TimeIntervalType | NUdf::EDataTypeFeatures::DecimalType) ? + CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_SLT, lhs, rhs, "less", block): // signed + info.Features & (NUdf::EDataTypeFeatures::UnsignedIntegralType | NUdf::EDataTypeFeatures::DateType | NUdf::EDataTypeFeatures::TzDateType) ? + CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_ULT, lhs, rhs, "less", block): // unsigned + rhs; // bool + + const auto out = SelectInst::Create(less, ConstantInt::get(resultType, -1), ConstantInt::get(resultType, 1), "out", block); + res->addIncoming(out, block); + BranchInst::Create(exit, block); + + block = exit; + return res; +} + +template <> +Value* GenCompareFunction<true>(NUdf::EDataSlot slot, Value* lv, Value* rv, TCodegenContext& ctx, BasicBlock*& block) { + auto& context = ctx.Codegen.GetContext(); + + const auto tiny = BasicBlock::Create(context, "tiny", ctx.Func); + const auto side = BasicBlock::Create(context, "side", ctx.Func); + const auto test = BasicBlock::Create(context, "test", ctx.Func); + const auto done = BasicBlock::Create(context, "done", ctx.Func); + + const auto resultType = Type::getInt32Ty(context); + const auto res = PHINode::Create(resultType, 3U, "result", done); + + const auto le = IsEmpty(lv, block); + const auto re = IsEmpty(rv, block); + + const auto any = BinaryOperator::CreateOr(le, re, "or", block); + + BranchInst::Create(tiny, test, any, block); + + block = tiny; + + const auto both = BinaryOperator::CreateAnd(le, re, "and", block); + res->addIncoming(ConstantInt::get(resultType, 0), block); + BranchInst::Create(done, side, both, block); + + block = side; + + const auto out = SelectInst::Create(le, ConstantInt::get(resultType, -1), ConstantInt::get(resultType, 1), "out", block); + res->addIncoming(out, block); + BranchInst::Create(done, block); + + block = test; + + const auto comp = GenCompareFunction<false>(slot, lv, rv, ctx, block); + res->addIncoming(comp, block); + BranchInst::Create(done, block); + + block = done; + return res; +} + +Value* GenCompareFunction(NUdf::EDataSlot slot, bool isOptional, Value* lv, Value* rv, TCodegenContext& ctx, BasicBlock*& block) { + return isOptional ? GenCompareFunction<true>(slot, lv, rv, ctx, block) : GenCompareFunction<false>(slot, lv, rv, ctx, block); +} + +Value* GenCombineHashes(Value* first, Value* second, BasicBlock* block) { +// key += ~(key << 32); + const auto x01 = BinaryOperator::CreateShl(first, ConstantInt::get(first->getType(), 32), "x01", block); + const auto x02 = BinaryOperator::CreateXor(x01, ConstantInt::get(x01->getType(), ~0), "x02", block); + const auto x03 = BinaryOperator::CreateAdd(x02, first, "x03", block); +// key ^= (key >> 22); + const auto x04 = BinaryOperator::CreateLShr(x03, ConstantInt::get(x03->getType(), 22), "x04", block); + const auto x05 = BinaryOperator::CreateXor(x04, x03, "x05", block); +// key += ~(key << 13); + const auto x06 = BinaryOperator::CreateShl(x05, ConstantInt::get(x05->getType(), 13), "x06", block); + const auto x07 = BinaryOperator::CreateXor(x06, ConstantInt::get(x06->getType(), ~0), "x07", block); + const auto x08 = BinaryOperator::CreateAdd(x05, x07, "x08", block); +// key ^= (key >> 8); + const auto x09 = BinaryOperator::CreateLShr(x08, ConstantInt::get(x08->getType(), 8), "x09", block); + const auto x10 = BinaryOperator::CreateXor(x08, x09, "x10", block); +// key += (key << 3); + const auto x11 = BinaryOperator::CreateShl(x10, ConstantInt::get(x10->getType(), 3), "x11", block); + const auto x12 = BinaryOperator::CreateAdd(x10, x11, "x12", block); +// key ^= (key >> 15); + const auto x13 = BinaryOperator::CreateLShr(x12, ConstantInt::get(x12->getType(), 15), "x13", block); + const auto x14 = BinaryOperator::CreateXor(x13, x12, "x14", block); +// key += ~(key << 27); + const auto x15 = BinaryOperator::CreateShl(x14, ConstantInt::get(x14->getType(), 27), "x15", block); + const auto x16 = BinaryOperator::CreateXor(x15, ConstantInt::get(x15->getType(), ~0), "x16", block); + const auto x17 = BinaryOperator::CreateAdd(x14, x16, "x17", block); +// key ^= (key >> 31); + const auto x18 = BinaryOperator::CreateLShr(x17, ConstantInt::get(x17->getType(), 31), "x18", block); + const auto x19 = BinaryOperator::CreateXor(x17, x18, "x19", block); + + return BinaryOperator::CreateXor(x19, second, "both", block); +} + +template <bool IsOptional> +Value* GenHashFunction(NUdf::EDataSlot slot, Value* value, TCodegenContext& ctx, BasicBlock*& block); + +template <> +Value* GenHashFunction<false>(NUdf::EDataSlot slot, Value* value, TCodegenContext& ctx, BasicBlock*& block) { + auto& context = ctx.Codegen.GetContext(); + + const auto& info = NUdf::GetDataTypeInfo(slot); + + if ((info.Features & NUdf::EDataTypeFeatures::CommonType) && (info.Features & NUdf::EDataTypeFeatures::StringType || NUdf::EDataSlot::Uuid == slot || NUdf::EDataSlot::DyNumber == slot)) { + return CallUnaryUnboxedValueFunction(&MyHashString, Type::getInt64Ty(context), value, ctx.Codegen, block); + } + + const auto val = GetterFor(slot, value, context, block); + const auto hashType = Type::getInt64Ty(context); + + if (info.Features & (NUdf::EDataTypeFeatures::IntegralType | NUdf::EDataTypeFeatures::DateType | NUdf::EDataTypeFeatures::TimeIntervalType) || NUdf::EDataSlot::Bool == slot) { + if (val->getType() == hashType) { + return val; + } + const auto ext = CastInst::Create(Instruction::ZExt, val, hashType, "ext", block); + return ext; + } + + if (info.Features & NUdf::EDataTypeFeatures::FloatType) { + const auto nan = CmpInst::Create(Instruction::FCmp, FCmpInst::FCMP_UNO, val, val, "nan", block); + const auto zero = CmpInst::Create(Instruction::FCmp, FCmpInst::FCMP_OEQ, val, ConstantFP::get(val->getType(), 0), "zero", block); + if (NUdf::EDataSlot::Float == slot) { + const auto cast = CastInst::Create(Instruction::BitCast, val, Type::getInt32Ty(context), "cast", block); + const auto ext = CastInst::Create(Instruction::ZExt, cast, hashType, "ext", block); + const auto first = SelectInst::Create(nan, ConstantInt::get(hashType, ~0), ext, "first", block); + const auto second = SelectInst::Create(zero, ConstantInt::get(hashType, 0), first, "second", block); + return second; + } else { + const auto cast = CastInst::Create(Instruction::BitCast, val, hashType, "cast", block); + const auto first = SelectInst::Create(nan, ConstantInt::get(hashType, ~0), cast, "first", block); + const auto second = SelectInst::Create(zero, ConstantInt::get(hashType, 0), first, "second", block); + return second; + } + } + + if (info.Features & NUdf::EDataTypeFeatures::TzDateType) { + const auto tz = GetterForTimezone(context, value, block); + const auto ext = val->getType() == hashType ? val : CastInst::Create(Instruction::ZExt, val, hashType, "ext", block); + const auto etz = CastInst::Create(Instruction::ZExt, tz, hashType, "etz", block); + + return GenCombineHashes(ext, etz, block); + } + + if (info.Features & NUdf::EDataTypeFeatures::DecimalType) { + const auto low = CastInst::Create(Instruction::Trunc, val, hashType, "low", block); + const auto lshr = BinaryOperator::CreateLShr(val, ConstantInt::get(val->getType(), 64), "lshr", block); + const auto high = CastInst::Create(Instruction::Trunc, lshr, hashType, "high", block); + return GenCombineHashes(low, high, block); + } + + return nullptr; +} + +template <> +Value* GenHashFunction<true>(NUdf::EDataSlot slot, Value* value, TCodegenContext& ctx, BasicBlock*& block) { + auto& context = ctx.Codegen.GetContext(); + + const auto tiny = BasicBlock::Create(context, "tiny", ctx.Func); + const auto test = BasicBlock::Create(context, "test", ctx.Func); + const auto done = BasicBlock::Create(context, "done", ctx.Func); + + const auto res = PHINode::Create(Type::getInt64Ty(context), 2U, "result", done); + + BranchInst::Create(tiny, test, IsEmpty(value, block), block); + + block = tiny; + + res->addIncoming(ConstantInt::get(Type::getInt64Ty(context), ~0ULL), block); + BranchInst::Create(done, block); + + block = test; + + const auto comp = GenHashFunction<false>(slot, value, ctx, block); + res->addIncoming(comp, block); + BranchInst::Create(done, block); + + block = done; + return res; +} + +Value* GenHashFunction(NUdf::EDataSlot slot, bool isOptional, Value* value, TCodegenContext& ctx, BasicBlock*& block) { + return isOptional ? GenHashFunction<true>(slot, value, ctx, block) : GenHashFunction<false>(slot, value, ctx, block); +} + +Value* LoadIfPointer(Value* value, BasicBlock* block) { + return value->getType()->isPointerTy() ? new LoadInst(value->getType()->getPointerElementType(), value, "load_value", block) : value; +} + +} + +Function* GenerateEqualsFunction(NYql::NCodegen::ICodegen& codegen, const TString& name, bool isTuple, const TKeyTypes& types) { + auto& module = codegen.GetModule(); + if (const auto f = module.getFunction(name.c_str())) + return f; + + auto& context = codegen.GetContext(); + const auto valueType = Type::getInt128Ty(context); + const auto ptrType = PointerType::getUnqual(valueType); + const auto returnType = Type::getInt1Ty(context); + + const auto funcType = codegen.GetEffectiveTarget() != NYql::NCodegen::ETarget::Windows ? + FunctionType::get(returnType, {valueType, valueType}, false): + FunctionType::get(returnType, {ptrType, ptrType}, false); + + TCodegenContext ctx(codegen); + ctx.AlwaysInline = true; + ctx.Func = cast<Function>(module.getOrInsertFunction(name.c_str(), funcType).getCallee()); + + DISubprogramAnnotator annotator(ctx, ctx.Func); + + + auto args = ctx.Func->arg_begin(); + + const auto main = BasicBlock::Create(context, "main", ctx.Func); + auto block = main; + + const auto lv = LoadIfPointer(&*args, block); + const auto rv = LoadIfPointer(&*++args, block); + + if (isTuple) { + if (types.empty()) { + ReturnInst::Create(context, ConstantInt::getTrue(context), block); + return ctx.Func; + } + + const auto fast = BasicBlock::Create(context, "fast", ctx.Func); + const auto slow = BasicBlock::Create(context, "slow", ctx.Func); + const auto stop = BasicBlock::Create(context, "stop", ctx.Func); + + const auto elementsType = ArrayType::get(valueType, types.size()); + const auto elementsPtrType = PointerType::getUnqual(elementsType); + const auto elementsPtrOne = CallBoxedValueVirtualMethod<NUdf::TBoxedValueAccessor::EMethod::GetElements>(elementsPtrType, lv, ctx.Codegen, block); + const auto elementsPtrTwo = CallBoxedValueVirtualMethod<NUdf::TBoxedValueAccessor::EMethod::GetElements>(elementsPtrType, rv, ctx.Codegen, block); + + const auto goodOne = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_NE, elementsPtrOne, ConstantPointerNull::get(elementsPtrType), "good_one", block); + const auto goodTwo = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_NE, elementsPtrTwo, ConstantPointerNull::get(elementsPtrType), "good_two", block); + const auto good = BinaryOperator::CreateAnd(goodOne, goodTwo, "good", block); + + BranchInst::Create(fast, slow, good, block); + + const auto last = types.size() - 1U; + + { + block = fast; + + const auto elementsOne = new LoadInst(elementsType, elementsPtrOne, "elements_one", block); + const auto elementsTwo = new LoadInst(elementsType, elementsPtrTwo, "elements_two", block); + + for (ui32 i = 0U; i < last; ++i) { + const auto nextOne = ExtractValueInst::Create(elementsOne, i, (TString("next_one_") += ToString(i)).c_str(), block); + const auto nextTwo = ExtractValueInst::Create(elementsTwo, i, (TString("next_two_") += ToString(i)).c_str(), block); + + const auto step = BasicBlock::Create(context, (TString("step") += ToString(i)).c_str(), ctx.Func); + + const auto test = GenEqualsFunction(types[i].first, types[i].second, nextOne, nextTwo, ctx, block); + + BranchInst::Create(step, stop, test, block); + + block = step; + } + + const auto backOne = ExtractValueInst::Create(elementsOne, last, "back_one", block); + const auto backTwo = ExtractValueInst::Create(elementsTwo, last, "back_two", block); + + const auto result = GenEqualsFunction(types.back().first, types.back().second, backOne, backTwo, ctx, block); + ReturnInst::Create(context, result, block); + } + + { + block = slow; + + const auto elementOne = new AllocaInst(valueType, 0U, "element_one", block); + const auto elementTwo = new AllocaInst(valueType, 0U, "element_two", block); + + const auto indexType = Type::getInt32Ty(context); + + for (ui32 i = 0U; i < last; ++i) { + CallBoxedValueVirtualMethod<NUdf::TBoxedValueAccessor::EMethod::GetElement>(elementOne, lv, ctx.Codegen, block, ConstantInt::get(indexType, i)); + CallBoxedValueVirtualMethod<NUdf::TBoxedValueAccessor::EMethod::GetElement>(elementTwo, rv, ctx.Codegen, block, ConstantInt::get(indexType, i)); + + const auto nextOne = new LoadInst(valueType, elementOne, (TString("next_one_") += ToString(i)).c_str(), block); + const auto nextTwo = new LoadInst(valueType, elementTwo, (TString("next_two_") += ToString(i)).c_str(), block); + + if (NUdf::GetDataTypeInfo(types[i].first).Features & NUdf::EDataTypeFeatures::StringType) { + ValueRelease(EValueRepresentation::String, nextOne, ctx, block); + ValueRelease(EValueRepresentation::String, nextTwo, ctx, block); + } + + const auto step = BasicBlock::Create(context, (TString("step") += ToString(i)).c_str(), ctx.Func); + const auto test = GenEqualsFunction(types[i].first, types[i].second, nextOne, nextTwo, ctx, block); + + BranchInst::Create(step, stop, test, block); + + block = step; + } + + CallBoxedValueVirtualMethod<NUdf::TBoxedValueAccessor::EMethod::GetElement>(elementOne, lv, ctx.Codegen, block, ConstantInt::get(indexType, last)); + CallBoxedValueVirtualMethod<NUdf::TBoxedValueAccessor::EMethod::GetElement>(elementTwo, rv, ctx.Codegen, block, ConstantInt::get(indexType, last)); + + const auto backOne = new LoadInst(valueType, elementOne, "back_one", block); + const auto backTwo = new LoadInst(valueType, elementTwo, "back_two", block); + + if (NUdf::GetDataTypeInfo(types.back().first).Features & NUdf::EDataTypeFeatures::StringType) { + ValueRelease(EValueRepresentation::String, backOne, ctx, block); + ValueRelease(EValueRepresentation::String, backTwo, ctx, block); + } + + const auto result = GenEqualsFunction(types.back().first, types.back().second, backOne, backTwo, ctx, block); + ReturnInst::Create(context, result, block); + } + + block = stop; + ReturnInst::Create(context, ConstantInt::getFalse(context), block); + + } else { + const auto result = GenEqualsFunction(types.front().first, types.front().second, lv, rv, ctx, block); + ReturnInst::Create(context, result, block); + } + + return ctx.Func; +} + +Function* GenerateHashFunction(NYql::NCodegen::ICodegen& codegen, const TString& name, bool isTuple, const TKeyTypes& types) { + auto& module = codegen.GetModule(); + if (const auto f = module.getFunction(name.c_str())) + return f; + + auto& context = codegen.GetContext(); + const auto valueType = Type::getInt128Ty(context); + const auto ptrType = PointerType::getUnqual(valueType); + const auto returnType = Type::getInt64Ty(context); + + const auto funcType = codegen.GetEffectiveTarget() != NYql::NCodegen::ETarget::Windows ? + FunctionType::get(returnType, {valueType}, false): + FunctionType::get(returnType, {ptrType}, false); + + TCodegenContext ctx(codegen); + ctx.AlwaysInline = true; + ctx.Func = cast<Function>(module.getOrInsertFunction(name.c_str(), funcType).getCallee()); + + DISubprogramAnnotator annotator(ctx, ctx.Func); + + + const auto main = BasicBlock::Create(context, "main", ctx.Func); + auto block = main; + + const auto arg = LoadIfPointer(&*ctx.Func->arg_begin(), block); + + if (isTuple) { + if (types.empty()) { + ReturnInst::Create(context, ConstantInt::get(returnType, 0), block); + return ctx.Func; + } + + const auto fast = BasicBlock::Create(context, "fast", ctx.Func); + const auto slow = BasicBlock::Create(context, "slow", ctx.Func); + + const auto elementsType = ArrayType::get(valueType, types.size()); + const auto elementsPtrType = PointerType::getUnqual(elementsType); + const auto elementsPtr = CallBoxedValueVirtualMethod<NUdf::TBoxedValueAccessor::EMethod::GetElements>(elementsPtrType, arg, ctx.Codegen, block); + + const auto null = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, elementsPtr, ConstantPointerNull::get(elementsPtrType), "null", block); + + BranchInst::Create(slow, fast, null, block); + + { + block = fast; + + const auto elements = new LoadInst(elementsType, elementsPtr, "elements", block); + + auto result = static_cast<Value*>(ConstantInt::get(returnType, 0)); + + for (auto i = 0U; i < types.size(); ++i) { + const auto next = ExtractValueInst::Create(elements, i, (TString("next_") += ToString(i)).c_str(), block); + + const auto plus = GenHashFunction(types[i].first, types[i].second, next, ctx, block); + + result = GenCombineHashes(result, plus, block); + } + + ReturnInst::Create(context, result, block); + } + + { + block = slow; + + const auto element = new AllocaInst(valueType, 0U, "element", block); + + auto result = static_cast<Value*>(ConstantInt::get(returnType, 0)); + + for (auto i = 0U; i < types.size(); ++i) { + CallBoxedValueVirtualMethod<NUdf::TBoxedValueAccessor::EMethod::GetElement>(element, arg, ctx.Codegen, block, ConstantInt::get(Type::getInt32Ty(context), i)); + + const auto next = new LoadInst(valueType, element, (TString("next_") += ToString(i)).c_str(), block); + if (NUdf::GetDataTypeInfo(types[i].first).Features & NUdf::EDataTypeFeatures::StringType) { + ValueRelease(EValueRepresentation::String, next, ctx, block); + } + + const auto plus = GenHashFunction(types[i].first, types[i].second, next, ctx, block); + + result = GenCombineHashes(result, plus, block); + } + + ReturnInst::Create(context, result, block); + } + + } else { + const auto result = GenHashFunction(types.front().first, types.front().second, arg, ctx, block); + ReturnInst::Create(context, result, block); + } + + return ctx.Func; +} + +Function* GenerateEqualsFunction(NYql::NCodegen::ICodegen& codegen, const TString& name, const TKeyTypes& types) { + auto& module = codegen.GetModule(); + if (const auto f = module.getFunction(name.c_str())) + return f; + + auto& context = codegen.GetContext(); + const auto valueType = Type::getInt128Ty(context); + const auto elementsType = ArrayType::get(valueType, types.size()); + const auto ptrType = PointerType::getUnqual(elementsType); + const auto returnType = Type::getInt1Ty(context); + + const auto funcType = FunctionType::get(returnType, {ptrType, ptrType}, false); + + TCodegenContext ctx(codegen); + ctx.AlwaysInline = true; + ctx.Func = cast<Function>(module.getOrInsertFunction(name.c_str(), funcType).getCallee()); + + DISubprogramAnnotator annotator(ctx, ctx.Func); + + + auto args = ctx.Func->arg_begin(); + + const auto main = BasicBlock::Create(context, "main", ctx.Func); + auto block = main; + + const auto lv = &*args; + const auto rv = &*++args; + + if (types.empty()) { + ReturnInst::Create(context, ConstantInt::getTrue(context), block); + return ctx.Func; + } + + const auto elementsOne = new LoadInst(elementsType, lv, "elements_one", block); + const auto elementsTwo = new LoadInst(elementsType, rv, "elements_two", block); + + const auto stop = BasicBlock::Create(context, "stop", ctx.Func); + ReturnInst::Create(context, ConstantInt::getFalse(context), stop); + + const auto last = types.size() - 1U; + for (ui32 i = 0U; i < last; ++i) { + const auto nextOne = ExtractValueInst::Create(elementsOne, i, (TString("next_one_") += ToString(i)).c_str(), block); + const auto nextTwo = ExtractValueInst::Create(elementsTwo, i, (TString("next_two_") += ToString(i)).c_str(), block); + + const auto step = BasicBlock::Create(context, (TString("step_") += ToString(i)).c_str(), ctx.Func); + + const auto test = GenEqualsFunction(types[i].first, types[i].second, nextOne, nextTwo, ctx, block); + + BranchInst::Create(step, stop, test, block); + + block = step; + } + + const auto backOne = ExtractValueInst::Create(elementsOne, last, "back_one", block); + const auto backTwo = ExtractValueInst::Create(elementsTwo, last, "back_two", block); + + const auto result = GenEqualsFunction(types.back().first, types.back().second, backOne, backTwo, ctx, block); + ReturnInst::Create(context, result, block); + + return ctx.Func; +} + +Function* GenerateHashFunction(NYql::NCodegen::ICodegen& codegen, const TString& name, const TKeyTypes& types) { + auto& module = codegen.GetModule(); + if (const auto f = module.getFunction(name.c_str())) + return f; + + auto& context = codegen.GetContext(); + const auto valueType = Type::getInt128Ty(context); + const auto elementsType = ArrayType::get(valueType, types.size()); + const auto ptrType = PointerType::getUnqual(elementsType); + const auto returnType = Type::getInt64Ty(context); + + const auto funcType = FunctionType::get(returnType, {ptrType}, false); + + TCodegenContext ctx(codegen); + ctx.AlwaysInline = true; + ctx.Func = cast<Function>(module.getOrInsertFunction(name.c_str(), funcType).getCallee()); + + DISubprogramAnnotator annotator(ctx, ctx.Func); + + + const auto main = BasicBlock::Create(context, "main", ctx.Func); + auto block = main; + + if (types.empty()) { + ReturnInst::Create(context, ConstantInt::get(returnType, 0), block); + return ctx.Func; + } + + const auto arg = &*ctx.Func->arg_begin(); + const auto elements = new LoadInst(elementsType, arg, "elements", block); + + if (types.size() > 1U) { + auto result = static_cast<Value*>(ConstantInt::get(returnType, 0)); + + for (auto i = 0U; i < types.size(); ++i) { + const auto item = ExtractValueInst::Create(elements, i, (TString("item_") += ToString(i)).c_str(), block); + + const auto plus = GenHashFunction(types[i].first, types[i].second, item, ctx, block); + + result = GenCombineHashes(result, plus, block); + } + + ReturnInst::Create(context, result, block); + } else { + const auto value = ExtractValueInst::Create(elements, 0, "value", block); + const auto result = GenHashFunction(types.front().first, types.front().second, value, ctx, block); + ReturnInst::Create(context, result, block); + } + + return ctx.Func; +} + +Function* GenerateCompareFunction(NYql::NCodegen::ICodegen& codegen, const TString& name, const TKeyTypes& types) { + auto& module = codegen.GetModule(); + if (const auto f = module.getFunction(name.c_str())) + return f; + + auto& context = codegen.GetContext(); + const auto valueType = Type::getInt128Ty(context); + const auto elementsType = ArrayType::get(valueType, types.size()); + const auto ptrType = PointerType::getUnqual(elementsType); + const auto dirsType = ArrayType::get(Type::getInt1Ty(context), types.size()); + const auto ptrDirsType = PointerType::getUnqual(dirsType); + const auto returnType = Type::getInt32Ty(context); + + const auto funcType = FunctionType::get(returnType, {ptrDirsType, ptrType, ptrType}, false); + + TCodegenContext ctx(codegen); + ctx.AlwaysInline = true; + ctx.Func = cast<Function>(module.getOrInsertFunction(name.c_str(), funcType).getCallee()); + + DISubprogramAnnotator annotator(ctx, ctx.Func); + + + auto args = ctx.Func->arg_begin(); + + const auto main = BasicBlock::Create(context, "main", ctx.Func); + auto block = main; + + const auto dp = &*args; + const auto lv = &*++args; + const auto rv = &*++args; + + if (types.empty()) { + ReturnInst::Create(context, ConstantInt::get(returnType, 0), block); + return ctx.Func; + } + + const auto directions = new LoadInst(dirsType, dp, "directions", block); + const auto elementsOne = new LoadInst(elementsType, lv, "elements_one", block); + const auto elementsTwo = new LoadInst(elementsType, rv, "elements_two", block); + const auto zero = ConstantInt::get(returnType, 0); + + for (auto i = 0U; i < types.size(); ++i) { + const auto nextOne = ExtractValueInst::Create(elementsOne, i, (TString("next_one_") += ToString(i)).c_str(), block); + const auto nextTwo = ExtractValueInst::Create(elementsTwo, i, (TString("next_two_") += ToString(i)).c_str(), block); + + const auto exit = BasicBlock::Create(context, (TString("exit_") += ToString(i)).c_str(), ctx.Func); + const auto step = BasicBlock::Create(context, (TString("step_") += ToString(i)).c_str(), ctx.Func); + + const auto test = GenCompareFunction(types[i].first, types[i].second, nextOne, nextTwo, ctx, block); + const auto skip = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, zero, test, (TString("skip_") += ToString(i)).c_str(), block); + + BranchInst::Create(step, exit, skip, block); + + block = exit; + + const auto dir = ExtractValueInst::Create(directions, i, (TString("dir_") += ToString(i)).c_str(), block); + const auto neg = BinaryOperator::CreateNeg(test, (TString("neg_") += ToString(i)).c_str(), block); + const auto out = SelectInst::Create(dir, test, neg, (TString("neg_") += ToString(i)).c_str(), block); + + ReturnInst::Create(context, out, block); + + block = step; + } + + ReturnInst::Create(context, zero, block); + return ctx.Func; +} + +void GenInvalidate(const TCodegenContext& ctx, const std::vector<std::pair<ui32, EValueRepresentation>>& invalidationSet, BasicBlock*& block) { + auto& context = ctx.Codegen.GetContext(); + const auto indexType = Type::getInt32Ty(context); + const auto valueType = Type::getInt128Ty(context); + const auto values = ctx.GetMutables(); + + for (const auto& index : invalidationSet) { + const auto invPtr = GetElementPtrInst::CreateInBounds(valueType, values, {ConstantInt::get(indexType, index.first)}, "inv_ptr", block); + ValueUnRef(index.second, invPtr, ctx, block); + new StoreInst(GetInvalid(context), invPtr, block); + } +} + +TUnboxedImmutableCodegeneratorNode::TUnboxedImmutableCodegeneratorNode(TMemoryUsageInfo* memInfo, NUdf::TUnboxedValue&& value) + : TUnboxedImmutableComputationNode(memInfo, std::move(value)) +{} + +Value* TUnboxedImmutableCodegeneratorNode::CreateGetValue(const TCodegenContext& ctx, BasicBlock*&) const { + return ConstantInt::get(Type::getInt128Ty(ctx.Codegen.GetContext()), APInt(128, 2, reinterpret_cast<const uint64_t*>(&UnboxedValue))); +} + +TExternalCodegeneratorNode::TExternalCodegeneratorNode(TComputationMutables& mutables, EValueRepresentation kind) + : TExternalComputationNode(mutables, kind) +{} + +TExternalCodegeneratorRootNode::TExternalCodegeneratorRootNode(TComputationMutables& mutables, EValueRepresentation kind) + : TExternalCodegeneratorNode(mutables, kind) +{} + +NUdf::TUnboxedValue TExternalCodegeneratorRootNode::GetValue(TComputationContext& compCtx) const { + if (compCtx.ExecuteLLVM && GetFunction) + return GetFunction(&compCtx); + return TExternalComputationNode::GetValue(compCtx); +} + +void TExternalCodegeneratorRootNode::SetValue(TComputationContext& compCtx, NUdf::TUnboxedValue&& newValue) const { + if (compCtx.ExecuteLLVM && SetFunction) + return SetFunction(&compCtx, newValue.Release()); + + TExternalComputationNode::SetValue(compCtx, std::move(newValue)); +} + +TString TExternalCodegeneratorRootNode::MakeName(const TString& method) const { + TStringStream out; + out << DebugString() << "::" << method << "_(" << static_cast<const void*>(this) << ")."; + return out.Str(); +} + +void TExternalCodegeneratorRootNode::FinalizeFunctions(NYql::NCodegen::ICodegen& codegen) { + if (GetValueFunc) + GetFunction = reinterpret_cast<TGetPtr>(codegen.GetPointerToFunction(GetValueFunc)); + + if (SetValueFunc) + SetFunction = reinterpret_cast<TSetPtr>(codegen.GetPointerToFunction(SetValueFunc)); +} + +void TExternalCodegeneratorRootNode::GenerateFunctions(NYql::NCodegen::ICodegen& codegen) { + GetValueFunc = GenerateGetValue(codegen); + SetValueFunc = GenerateSetValue(codegen); + codegen.ExportSymbol(GetValueFunc); + codegen.ExportSymbol(SetValueFunc); +} + +Function* TExternalCodegeneratorRootNode::GenerateGetValue(NYql::NCodegen::ICodegen& codegen) { + auto& module = codegen.GetModule(); + auto& context = codegen.GetContext(); + const auto& name = MakeName("Get"); + if (const auto f = module.getFunction(name.c_str())) + return f; + + const auto valueType = Type::getInt128Ty(context); + const auto contextType = GetCompContextType(context); + + const auto funcType = codegen.GetEffectiveTarget() != NYql::NCodegen::ETarget::Windows ? + FunctionType::get(valueType, {PointerType::getUnqual(contextType)}, false): + FunctionType::get(Type::getVoidTy(context), {PointerType::getUnqual(valueType), PointerType::getUnqual(contextType)}, false); + + TCodegenContext ctx(codegen); + ctx.Func = cast<Function>(module.getOrInsertFunction(name.c_str(), funcType).getCallee()); + + DISubprogramAnnotator annotator(ctx, ctx.Func); + + + auto args = ctx.Func->arg_begin(); + if (codegen.GetEffectiveTarget() == NYql::NCodegen::ETarget::Windows) { + auto& firstArg = *args++; + firstArg.addAttr(Attribute::StructRet); + firstArg.addAttr(Attribute::NoAlias); + } + + auto main = BasicBlock::Create(context, "main", ctx.Func); + ctx.Ctx = &*args; + ctx.Ctx->addAttr(Attribute::NonNull); + + const auto get = CreateGetValue(ctx, main); + + if (codegen.GetEffectiveTarget() != NYql::NCodegen::ETarget::Windows) { + ReturnInst::Create(context, get, main); + } else { + new StoreInst(get, &*--args, main); + ReturnInst::Create(context, main); + } + + return ctx.Func; +} + +Function* TExternalCodegeneratorRootNode::GenerateSetValue(NYql::NCodegen::ICodegen& codegen) { + auto& module = codegen.GetModule(); + auto& context = codegen.GetContext(); + const auto& name = MakeName("Set"); + if (const auto f = module.getFunction(name.c_str())) + return f; + + const auto intType = Type::getInt128Ty(context); + const auto contextType = GetCompContextType(context); + const auto valueType = codegen.GetEffectiveTarget() == NYql::NCodegen::ETarget::Windows ? + (Type*)PointerType::getUnqual(intType) : (Type*)intType; + + const auto funcType = FunctionType::get(Type::getVoidTy(context), {PointerType::getUnqual(contextType), valueType}, false); + TCodegenContext ctx(codegen); + ctx.Func = cast<Function>(module.getOrInsertFunction(name.c_str(), funcType).getCallee()); + + DISubprogramAnnotator annotator(ctx, ctx.Func); + + + auto args = ctx.Func->arg_begin(); + + auto main = BasicBlock::Create(context, "main", ctx.Func); + ctx.Ctx = &*args; + ctx.Ctx->addAttr(Attribute::NonNull); + + const auto valueArg = &*++args; + + if (codegen.GetEffectiveTarget() == NYql::NCodegen::ETarget::Windows) { + const auto value = new LoadInst(valueArg->getType()->getPointerElementType(), valueArg, "load_value", main); + CreateSetValue(ctx, main, value); + } else { + CreateSetValue(ctx, main, valueArg); + } + ReturnInst::Create(context, main); + return ctx.Func; +} + +Value* TExternalCodegeneratorNode::CreateGetValue(const TCodegenContext& ctx, BasicBlock*& block) const { + if (ValueGetterBuilder) { + llvm::Function * ValueGetter = ValueGetterBuilder(ctx); + return CallInst::Create(ValueGetter, {ctx.Ctx}, "getter", block); + } + + if (ValueBuilder) { + llvm::Value * TemporaryValue = ValueBuilder(ctx); + return LoadIfPointer(TemporaryValue, block); + } + + MKQL_ENSURE(!Getter, "Wrong LLVM function generation order."); + auto& context = ctx.Codegen.GetContext(); + const auto indexType = Type::getInt32Ty(context); + const auto valueType = Type::getInt128Ty(context); + const auto valuePtr = GetElementPtrInst::CreateInBounds(valueType, ctx.GetMutables(), {ConstantInt::get(indexType, ValueIndex)}, "value_ptr", block); + const auto value = new LoadInst(valueType, valuePtr, "value", block); + return value; +} + +Value* TExternalCodegeneratorNode::CreateRefValue(const TCodegenContext& ctx, BasicBlock*& block) const { + CreateInvalidate(ctx, block); + + auto& context = ctx.Codegen.GetContext(); + const auto indexType = Type::getInt32Ty(context); + const auto valueType = Type::getInt128Ty(context); + const auto values = ctx.GetMutables(); + const auto valuePtr = GetElementPtrInst::CreateInBounds(valueType, values, {ConstantInt::get(indexType, ValueIndex)}, "value_ptr", block); + return valuePtr; +} + +void TExternalCodegeneratorNode::CreateSetValue(const TCodegenContext& ctx, BasicBlock*& block, Value* value) const { + auto& context = ctx.Codegen.GetContext(); + const auto indexType = Type::getInt32Ty(context); + const auto valueType = Type::getInt128Ty(context); + const auto values = ctx.GetMutables(); + const auto valuePtr = GetElementPtrInst::CreateInBounds(valueType, values, {ConstantInt::get(indexType, ValueIndex)}, "value_ptr", block); + + + if (value->getType()->isPointerTy()) { + ValueUnRef(RepresentationKind, valuePtr, ctx, block); + const auto load = new LoadInst(valueType, value, "value", block); + new StoreInst(load, valuePtr, block); + new StoreInst(ConstantInt::get(load->getType(), 0), value, block); + } else { + if (EValueRepresentation::Embedded == RepresentationKind) { + new StoreInst(value, valuePtr, block); + } else { + const auto load = new LoadInst(valueType, valuePtr, "value", block); + const auto equal = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, value, load, "equal", block); + + const auto skip = BasicBlock::Create(context, "skip", ctx.Func); + const auto refs = BasicBlock::Create(context, "refs", ctx.Func); + BranchInst::Create(skip, refs, equal, block); + + block = refs; + ValueUnRef(RepresentationKind, valuePtr, ctx, block); + new StoreInst(value, valuePtr, block); + ValueAddRef(RepresentationKind, valuePtr, ctx, block); + + BranchInst::Create(skip, block); + block = skip; + } + + } + CreateInvalidate(ctx, block); +} + +Value* TExternalCodegeneratorNode::CreateSwapValue(const TCodegenContext& ctx, BasicBlock*& block, Value* value) const { + auto& context = ctx.Codegen.GetContext(); + const auto indexType = Type::getInt32Ty(context); + const auto valueType = Type::getInt128Ty(context); + const auto values = ctx.GetMutables(); + const auto valuePtr = GetElementPtrInst::CreateInBounds(valueType, values, {ConstantInt::get(indexType, ValueIndex)}, "value_ptr", block); + const auto output = new LoadInst(valueType, valuePtr, "output", block); + ValueRelease(RepresentationKind, output, ctx, block); + + if (value->getType()->isPointerTy()) { + const auto load = new LoadInst(valueType, value, "load", block); + new StoreInst(load, valuePtr, block); + new StoreInst(ConstantInt::get(load->getType(), 0), value, block); + } else { + ValueAddRef(RepresentationKind, value, ctx, block); + new StoreInst(value, valuePtr, block); + } + + CreateInvalidate(ctx, block); + return output; +} + +void TExternalCodegeneratorNode::CreateInvalidate(const TCodegenContext& ctx, BasicBlock*& block) const { + GenInvalidate(ctx, InvalidationSet, block); +} + +void TExternalCodegeneratorNode::SetValueBuilder(TValueBuilder valueBuilder) +{ + ValueBuilder = std::move(valueBuilder); +} + +void TExternalCodegeneratorNode::SetValueGetterBuilder(TValueGetterBuilder valueGetterBuilder) +{ + ValueGetterBuilder = std::move(valueGetterBuilder); +} + +void TWideFlowProxyCodegeneratorNode::CreateInvalidate(const TCodegenContext& ctx, BasicBlock*& block) const { + GenInvalidate(ctx, InvalidationSet, block); +} + +void TWideFlowProxyCodegeneratorNode::SetGenerator(TGenerator&& generator) { + Generator = std::move(generator); +} + +ICodegeneratorInlineWideNode::TGenerateResult +TWideFlowProxyCodegeneratorNode::GenGetValues(const TCodegenContext& ctx, BasicBlock*& block) const { + return Generator(ctx, block); +} + +Value* GetOptionalValue(LLVMContext& context, Value* value, BasicBlock* block) { + const auto type = Type::getInt128Ty(context); + const auto data = ConstantInt::get(type, 0xFFFFFFFFFFFFFFFFULL); + const auto check = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_UGT, value, data, "check", block); + const auto decr = BinaryOperator::CreateSub(value, ConstantInt::get(type, 1), "decr", block); + const auto result = SelectInst::Create(check, value, decr, "result", block); + return result; +} + +Value* MakeOptional(LLVMContext& context, Value* value, BasicBlock* block) { + const auto type = Type::getInt128Ty(context); + const auto data = ConstantInt::get(type, 0xFFFFFFFFFFFFFFFFULL); + const auto check = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_UGT, value, data, "check", block); + const auto incr = BinaryOperator::CreateAdd(value, ConstantInt::get(type, 1), "incr", block); + const auto result = SelectInst::Create(check, value, incr, "result", block); + return result; +} + +ConstantInt* GetTrue(LLVMContext &context) { + const uint64_t init[] = {1ULL, 0x100000000000000ULL}; + return ConstantInt::get(context, APInt(128, 2, init)); +} + +ConstantInt* GetFalse(LLVMContext &context) { + const uint64_t init[] = {0ULL, 0x100000000000000ULL}; + return ConstantInt::get(context, APInt(128, 2, init)); +} + +ConstantInt* GetDecimalPlusInf(LLVMContext &context) { + const auto& pair = NYql::NDecimal::MakePair(+NYql::NDecimal::Inf()); + const uint64_t init[] = {pair.first, pair.second}; + return ConstantInt::get(context, APInt(128, 2, init)); +} + +ConstantInt* GetDecimalMinusInf(LLVMContext &context) { + const auto& pair = NYql::NDecimal::MakePair(-NYql::NDecimal::Inf()); + const uint64_t init[] = {pair.first, pair.second}; + return ConstantInt::get(context, APInt(128, 2, init)); +} + +ConstantInt* GetDecimalNan(LLVMContext &context) { + const auto& pair = NYql::NDecimal::MakePair(NYql::NDecimal::Nan()); + const uint64_t init[] = {pair.first, pair.second}; + return ConstantInt::get(context, APInt(128, 2, init)); +} + + +ConstantInt* GetDecimalMinusNan(LLVMContext &context) { + const auto& pair = NYql::NDecimal::MakePair(-NYql::NDecimal::Nan()); + const uint64_t init[] = {pair.first, pair.second}; + return ConstantInt::get(context, APInt(128, 2, init)); +} + +static constexpr ui64 InvalidData = std::numeric_limits<ui64>::max(); +static constexpr ui64 FinishData = InvalidData - 1ULL; +static constexpr ui64 YieldData = InvalidData; + +ConstantInt* GetEmpty(LLVMContext &context) { + return ConstantInt::get(Type::getInt128Ty(context), 0ULL); +} + +ConstantInt* GetInvalid(LLVMContext &context) { + return ConstantInt::get(Type::getInt128Ty(context), InvalidData); +} + +ConstantInt* GetFinish(LLVMContext &context) { + return ConstantInt::get(Type::getInt128Ty(context), FinishData); +} + +ConstantInt* GetYield(LLVMContext &context) { + return ConstantInt::get(Type::getInt128Ty(context), YieldData); +} + +ConstantInt* GetConstant(ui64 value, LLVMContext &context) { + const uint64_t init[] = {value, 0x100000000000000ULL}; + return ConstantInt::get(context, APInt(128, 2, init)); +} + +Value* IsExists(Value* value, BasicBlock* block) { + const auto v = LoadIfPointer(value, block); + return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_NE, v, ConstantInt::get(v->getType(), 0ULL), "exists", block); +} + +Value* IsEmpty(Value* value, BasicBlock* block) { + const auto v = LoadIfPointer(value, block); + return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, v, ConstantInt::get(v->getType(), 0ULL), "empty", block); +} + +Value* IsInvalid(Value* value, BasicBlock* block) { + const auto v = LoadIfPointer(value, block); + return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, v, ConstantInt::get(v->getType(), InvalidData), "invalid", block); +} + +Value* IsValid(Value* value, BasicBlock* block) { + const auto v = LoadIfPointer(value, block); + return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_NE, v, ConstantInt::get(v->getType(), InvalidData), "valid", block); +} + +Value* IsFinish(Value* value, BasicBlock* block) { + const auto v = LoadIfPointer(value, block); + return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, v, ConstantInt::get(v->getType(), FinishData), "finish", block); +} + +Value* IsYield(Value* value, BasicBlock* block) { + const auto v = LoadIfPointer(value, block); + return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, v, ConstantInt::get(v->getType(), YieldData), "yield", block); +} + +Value* IsSpecial(Value* value, BasicBlock* block) { + const auto v = LoadIfPointer(value, block); + return BinaryOperator::CreateOr(IsFinish(v, block), IsYield(v, block), "special", block); +} + +Value* HasValue(Value* value, BasicBlock* block) { + const auto v = LoadIfPointer(value, block); + return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_UGT, v, ConstantInt::get(v->getType(), InvalidData), "has", block); +} + +Value* MakeBoolean(Value* boolean , LLVMContext &context, BasicBlock* block) { + return SelectInst::Create(boolean, GetTrue(context), GetFalse(context), "result", block); +} + +Value* SetterForInt128(Value* value, BasicBlock* block) { + const uint64_t mask[] = {0xFFFFFFFFFFFFFFFFULL, 0xFFFFFFFFFFFFFFULL}; + const auto drop = ConstantInt::get(value->getType(), APInt(128, 2, mask)); + const auto data = BinaryOperator::CreateAnd(value, drop, "and", block); + const uint64_t init[] = {0ULL, 0x100000000000000ULL}; // Embedded + const auto meta = ConstantInt::get(value->getType(), APInt(128, 2, init)); + const auto full = BinaryOperator::CreateOr(data, meta, "or", block); + return full; +} + +Value* GetterForInt128(Value* value, BasicBlock* block) { + const uint64_t init[] = {0ULL, 0x80000000000000ULL}; + const auto test = ConstantInt::get(value->getType(), APInt(128, 2, init)); + const auto sign = BinaryOperator::CreateAnd(value, test, "and", block); + + const uint64_t fill[] = {0ULL, 0xFF00000000000000ULL}; + const auto sext = ConstantInt::get(value->getType(), APInt(128, 2, fill)); + const auto minus = BinaryOperator::CreateOr(value, sext, "or", block); + + const uint64_t mask[] = {0xFFFFFFFFFFFFFFFFULL, 0xFFFFFFFFFFFFFFULL}; + const auto trun = ConstantInt::get(value->getType(), APInt(128, 2, mask)); + const auto plus = BinaryOperator::CreateAnd(value, trun, "and", block); + + const auto check = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, sign, ConstantInt::get(sign->getType(), 0), "check", block); + const auto result = SelectInst::Create(check, plus, minus, "result", block); + return result; +} + +Value* GetterForTimezone(LLVMContext& context, Value* value, BasicBlock* block) { + const auto lshr = BinaryOperator::CreateLShr(value, ConstantInt::get(value->getType(), 64ULL), "lshr", block); + const auto trunc = CastInst::Create(Instruction::Trunc, lshr, Type::getInt16Ty(context), "trunc", block); + return trunc; +} + +template<> Type* GetTypeFor<bool>(LLVMContext &context) { return Type::getInt1Ty(context); } +template<> Type* GetTypeFor<ui8>(LLVMContext &context) { return Type::getInt8Ty(context); } +template<> Type* GetTypeFor<i8>(LLVMContext &context) { return Type::getInt8Ty(context); } + +template<> Type* GetTypeFor<i16>(LLVMContext &context) { return Type::getInt16Ty(context); } +template<> Type* GetTypeFor<ui16>(LLVMContext &context) { return Type::getInt16Ty(context); } + +template<> Type* GetTypeFor<i32>(LLVMContext &context) { return Type::getInt32Ty(context); } +template<> Type* GetTypeFor<ui32>(LLVMContext &context) { return Type::getInt32Ty(context); } + +template<> Type* GetTypeFor<i64>(LLVMContext &context) { return Type::getInt64Ty(context); } +template<> Type* GetTypeFor<ui64>(LLVMContext &context) { return Type::getInt64Ty(context); } + +template<> Type* GetTypeFor<float>(LLVMContext &context) { return Type::getFloatTy(context); } +template<> Type* GetTypeFor<double>(LLVMContext &context) { return Type::getDoubleTy(context); } + +void AddRefBoxed(Value* value, const TCodegenContext& ctx, BasicBlock*& block) { + auto& context = ctx.Codegen.GetContext(); + const auto load = value->getType()->isPointerTy() ? new LoadInst(value->getType()->getPointerElementType(), value, "load", block) : value; + const auto half = CastInst::Create(Instruction::Trunc, load, Type::getInt64Ty(context), "half", block); + const auto counterType = Type::getInt32Ty(context); + const auto type = StructType::get(context, {PointerType::getUnqual(StructType::get(context)), counterType, Type::getInt16Ty(context)}); + const auto boxptr = CastInst::Create(Instruction::IntToPtr, half, PointerType::getUnqual(type), "boxptr", block); + const auto cntptr = GetElementPtrInst::CreateInBounds(type, boxptr, {ConstantInt::get(Type::getInt32Ty(context), 0), ConstantInt::get(Type::getInt32Ty(context), 1)}, "cntptr", block); + const auto refs = new LoadInst(counterType, cntptr, "refs", block); +#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 4) + if constexpr (EnableStaticRefcount) { + const auto work = BasicBlock::Create(context, "work", ctx.Func); + const auto skip = BasicBlock::Create(context, "skip", ctx.Func); + const auto magic = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_SLT, refs, ConstantInt::get(refs->getType(), 0), "magic", block); + BranchInst::Create(skip, work, magic, block); + + block = work; + const auto incr = BinaryOperator::CreateAdd(refs, ConstantInt::get(refs->getType(), 1), "incr", block); + new StoreInst(incr, cntptr, block); + BranchInst::Create(skip, block); + + block = skip; + } else { + const auto incr = BinaryOperator::CreateAdd(refs, ConstantInt::get(refs->getType(), 1), "incr", block); + new StoreInst(incr, cntptr, block); + } +#else + const auto incr = BinaryOperator::CreateAdd(refs, ConstantInt::get(refs->getType(), 1), "incr", block); + new StoreInst(incr, cntptr, block); +#endif +} + +void UnRefBoxed(Value* value, const TCodegenContext& ctx, BasicBlock*& block) { + auto& context = ctx.Codegen.GetContext(); + const auto load = value->getType()->isPointerTy() ? new LoadInst(value->getType()->getPointerElementType(), value, "load", block) : value; + const auto half = CastInst::Create(Instruction::Trunc, load, Type::getInt64Ty(context), "half", block); + const auto counterType = Type::getInt32Ty(context); + const auto type = StructType::get(context, {PointerType::getUnqual(StructType::get(context)), counterType, Type::getInt16Ty(context)}); + const auto boxptr = CastInst::Create(Instruction::IntToPtr, half, PointerType::getUnqual(type), "boxptr", block); + const auto cntptr = GetElementPtrInst::CreateInBounds(type, boxptr, {ConstantInt::get(Type::getInt32Ty(context), 0), ConstantInt::get(Type::getInt32Ty(context), 1)}, "cntptr", block); + const auto refs = new LoadInst(counterType, cntptr, "refs", block); + + const auto live = BasicBlock::Create(context, "live", ctx.Func); + +#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 4) + if constexpr (EnableStaticRefcount) { + const auto magic = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_SLT, refs, ConstantInt::get(refs->getType(), 0), "magic", block); + + const auto work = BasicBlock::Create(context, "work", ctx.Func); + BranchInst::Create(live, work, magic, block); + + block = work; + } +#endif + + const auto decr = BinaryOperator::CreateSub(refs, ConstantInt::get(refs->getType(), 1), "decr", block); + new StoreInst(decr, cntptr, block); + const auto test = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_UGT, decr, ConstantInt::get(decr->getType(), 0), "many", block); + + const auto kill = BasicBlock::Create(context, "kill", ctx.Func); + + BranchInst::Create(live, kill, test, block); + + block = kill; + + const auto fnType = FunctionType::get(Type::getVoidTy(context), {boxptr->getType()}, false); + const auto name = "DeleteBoxed"; + ctx.Codegen.AddGlobalMapping(name, reinterpret_cast<const void*>(&DeleteBoxed)); + const auto func = ctx.Codegen.GetModule().getOrInsertFunction(name, fnType).getCallee(); + CallInst::Create(fnType, func, {boxptr}, "", block); + + BranchInst::Create(live, block); + block = live; +} + +void CleanupBoxed(Value* value, const TCodegenContext& ctx, BasicBlock*& block) { + auto& context = ctx.Codegen.GetContext(); + const auto load = value->getType()->isPointerTy() ? new LoadInst(value->getType()->getPointerElementType(), value, "load", block) : value; + const auto half = CastInst::Create(Instruction::Trunc, load, Type::getInt64Ty(context), "half", block); + const auto counterType = Type::getInt32Ty(context); + const auto type = StructType::get(context, {PointerType::getUnqual(StructType::get(context)), counterType, Type::getInt16Ty(context)}); + const auto boxptr = CastInst::Create(Instruction::IntToPtr, half, PointerType::getUnqual(type), "boxptr", block); + const auto cntptr = GetElementPtrInst::CreateInBounds(type, boxptr, {ConstantInt::get(Type::getInt32Ty(context), 0), ConstantInt::get(Type::getInt32Ty(context), 1)}, "cntptr", block); + const auto refs = new LoadInst(counterType, cntptr, "refs", block); + const auto test = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_NE, refs, ConstantInt::get(refs->getType(), 0), "many", block); + + const auto live = BasicBlock::Create(context, "live", ctx.Func); + const auto kill = BasicBlock::Create(context, "kill", ctx.Func); + + BranchInst::Create(live, kill, test, block); + + block = kill; + + const auto fnType = FunctionType::get(Type::getVoidTy(context), {boxptr->getType()}, false); + const auto name = "DeleteBoxed"; + ctx.Codegen.AddGlobalMapping(name, reinterpret_cast<const void*>(&DeleteBoxed)); + const auto func = ctx.Codegen.GetModule().getOrInsertFunction(name, fnType).getCallee(); + CallInst::Create(fnType, func, {boxptr}, "", block); + + BranchInst::Create(live, block); + block = live; +} + + +template<bool IncOrDec> +void ChangeRefUnboxed(Value* value, const TCodegenContext& ctx, BasicBlock*& block) { + auto& context = ctx.Codegen.GetContext(); + + const auto type8 = Type::getInt8Ty(context); + const auto type32 = Type::getInt32Ty(context); + + const auto mark = GetMarkFromUnboxed(value, ctx, block); + + const auto boxb = BasicBlock::Create(context, "boxb", ctx.Func); + const auto strb = BasicBlock::Create(context, "strb", ctx.Func); + const auto doit = BasicBlock::Create(context, "doit", ctx.Func); + const auto done = BasicBlock::Create(context, "done", ctx.Func); + + const auto refsPtrType = PointerType::getUnqual(type32); + const auto refsptr = PHINode::Create(refsPtrType, 2U, "refsptr", doit); + + const auto choise = SwitchInst::Create(mark, done, 2U, block); + choise->addCase(ConstantInt::get(type8, 2), strb); + choise->addCase(ConstantInt::get(type8, 3), boxb); + + { + block = strb; + + const auto strptr = GetPointerFromUnboxed<false>(value, ctx, block); + const auto elemptr = GetElementPtrInst::CreateInBounds(strptr->getType()->getPointerElementType(), strptr, {ConstantInt::get(type32, 0), ConstantInt::get(type32, 1)}, "elemptr", block); + refsptr->addIncoming(elemptr, block); + BranchInst::Create(doit, block); + } + + { + block = boxb; + + const auto boxptr = GetPointerFromUnboxed<true>(value, ctx, block);; + const auto elemptr = GetElementPtrInst::CreateInBounds(boxptr->getType()->getPointerElementType(), boxptr, {ConstantInt::get(type32, 0), ConstantInt::get(type32, 1)}, "elemptr", block); + refsptr->addIncoming(elemptr, block); + BranchInst::Create(doit, block); + } + + block = doit; + + const auto refs = new LoadInst(type32, refsptr, "refs", block); +#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 4) + if constexpr (EnableStaticRefcount) { + const auto magic = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_SLT, refs, ConstantInt::get(refs->getType(), 0), "magic", block); + + const auto work = BasicBlock::Create(context, "work", ctx.Func); + BranchInst::Create(done, work, magic, block); + + block = work; + } +#endif + const auto next = IncOrDec ? + BinaryOperator::CreateAdd(refs, ConstantInt::get(refs->getType(), 1), "incr", block): + BinaryOperator::CreateSub(refs, ConstantInt::get(refs->getType(), 1), "decr", block); + new StoreInst(next, refsptr, block); + BranchInst::Create(done, block); + + block = done; +} + +template<bool Decrement> +void CheckRefUnboxed(Value* value, const TCodegenContext& ctx, BasicBlock*& block) { + auto& context = ctx.Codegen.GetContext(); + + const auto type8 = Type::getInt8Ty(context); + const auto type32 = Type::getInt32Ty(context); + + const auto mark = GetMarkFromUnboxed(value, ctx, block); + + const auto boxb = BasicBlock::Create(context, "boxb", ctx.Func); + const auto strb = BasicBlock::Create(context, "strb", ctx.Func); + const auto nope = BasicBlock::Create(context, "nope", ctx.Func); + + const auto choise = SwitchInst::Create(mark, nope, 2U, block); + choise->addCase(ConstantInt::get(type8, 2), strb); + choise->addCase(ConstantInt::get(type8, 3), boxb); + + { + block = strb; + + const auto strptr = GetPointerFromUnboxed<false>(value, ctx, block); + const auto refptr = GetElementPtrInst::CreateInBounds(strptr->getType()->getPointerElementType(), strptr, {ConstantInt::get(type32, 0), ConstantInt::get(type32, 1)}, "refptr", block); + const auto refs = new LoadInst(type32, refptr, "refs", block); + +#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 4) + if constexpr (EnableStaticRefcount) { + const auto magic = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_SLT, refs, ConstantInt::get(refs->getType(), 0), "magic", block); + + const auto work = BasicBlock::Create(context, "work", ctx.Func); + BranchInst::Create(nope, work, magic, block); + + block = work; + } +#endif + Value* test = refs; + + if constexpr (Decrement) { + const auto decr = BinaryOperator::CreateSub(refs, ConstantInt::get(refs->getType(), 1), "decr", block); + new StoreInst(decr, refptr, block); + test = decr; + } + + const auto good = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_UGT, test, ConstantInt::get(test->getType(), 0), "test", block); + + const auto free = BasicBlock::Create(context, "free", ctx.Func); + + BranchInst::Create(nope, free, good, block); + + block = free; + + const auto fnType = FunctionType::get(Type::getVoidTy(context), {strptr->getType()}, false); + const auto name = "DeleteString"; + ctx.Codegen.AddGlobalMapping(name, reinterpret_cast<const void*>(&DeleteString)); + const auto func = ctx.Codegen.GetModule().getOrInsertFunction(name, fnType).getCallee(); + CallInst::Create(fnType, func, {strptr}, "", block); + BranchInst::Create(nope, block); + } + + { + block = boxb; + + const auto boxptr = GetPointerFromUnboxed<true>(value, ctx, block);; + const auto refptr = GetElementPtrInst::CreateInBounds(boxptr->getType()->getPointerElementType(), boxptr, {ConstantInt::get(type32, 0), ConstantInt::get(type32, 1)}, "cntptr", block); + const auto refs = new LoadInst(type32, refptr, "refs", block); + +#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 4) + if constexpr (EnableStaticRefcount) { + const auto magic = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_SLT, refs, ConstantInt::get(refs->getType(), 0), "magic", block); + + const auto work = BasicBlock::Create(context, "work", ctx.Func); + BranchInst::Create(nope, work, magic, block); + + block = work; + } +#endif + + Value* test = refs; + + if constexpr (Decrement) { + const auto decr = BinaryOperator::CreateSub(refs, ConstantInt::get(refs->getType(), 1), "decr", block); + new StoreInst(decr, refptr, block); + test = decr; + } + + const auto good = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_UGT, test, ConstantInt::get(test->getType(), 0), "test", block); + + const auto kill = BasicBlock::Create(context, "kill", ctx.Func); + + BranchInst::Create(nope, kill, good, block); + + block = kill; + + const auto fnType = FunctionType::get(Type::getVoidTy(context), {boxptr->getType()}, false); + const auto name = "DeleteBoxed"; + ctx.Codegen.AddGlobalMapping(name, reinterpret_cast<const void*>(&DeleteBoxed)); + const auto func = ctx.Codegen.GetModule().getOrInsertFunction(name, fnType).getCallee(); + + CallInst::Create(fnType, func, {boxptr}, "", block); + BranchInst::Create(nope, block); + } + + block = nope; +} +#ifdef MAKE_UNBOXED_VALUE_LLVM_REFCOUNTION_FUNCTIONS +Function* GenRefCountFunction(const char* label, void (*func)(Value*, const TCodegenContext&, BasicBlock*&), Type* type, NYql::NCodegen::ICodegen& codegen) { + auto& module = codegen.GetModule(); + auto& context = codegen.GetContext(); + const auto name = TString(label) += (type->isPointerTy() ? "Ptr" : "Val"); + if (const auto f = module.getFunction(name.c_str())) + return f; + + const auto funcType = FunctionType::get(Type::getVoidTy(context), {type}, false); + + TCodegenContext ctx(codegen); + ctx.Func = cast<Function>(module.getOrInsertFunction(name.c_str(), funcType).getCallee()).getCallee(); + + auto main = BasicBlock::Create(context, "main", ctx.Func); + auto value = &*ctx.Func->arg_begin(); + + func(value, ctx, main); + ReturnInst::Create(context, main); + return ctx.Func; +} + +void AddRefUnboxed(Value* value, const TCodegenContext& ctx, BasicBlock*& block) { + CallInst::Create(GenRefCountFunction(__func__, &ChangeRefUnboxed<true>, value->getType(), ctx.Codegen), {value}, "", block); +} + +void ReleaseUnboxed(Value* value, const TCodegenContext& ctx, BasicBlock*& block) { + CallInst::Create(GenRefCountFunction(__func__, &ChangeRefUnboxed<false>, value->getType(), ctx.Codegen), {value}, "", block); +} + +void UnRefUnboxed(Value* value, const TCodegenContext& ctx, BasicBlock*& block) { + CallInst::Create(GenRefCountFunction(__func__, &CheckRefUnboxed<true>, value->getType(), ctx.Codegen), {value}, "", block); +} + +void CleanupUnboxed(Value* value, const TCodegenContext& ctx, BasicBlock*& block) { + CallInst::Create(GenRefCountFunction(__func__, &CheckRefUnboxed<false>, value->getType(), ctx.Codegen), {value}, "", block); +} +#else +void AddRefUnboxed(Value* value, const TCodegenContext& ctx, BasicBlock*& block) { + return ChangeRefUnboxed<true>(value, ctx, block); +} + +void ReleaseUnboxed(Value* value, const TCodegenContext& ctx, BasicBlock*& block) { + return ChangeRefUnboxed<false>(value, ctx, block); +} + +void UnRefUnboxed(Value* value, const TCodegenContext& ctx, BasicBlock*& block) { + return CheckRefUnboxed<true>(value, ctx, block); +} + +void CleanupUnboxed(Value* value, const TCodegenContext& ctx, BasicBlock*& block) { + return CheckRefUnboxed<false>(value, ctx, block); +} +#endif + +void SafeUnRefUnboxed(Value* pointer, const TCodegenContext& ctx, BasicBlock*& block) { + if (const auto itemType = pointer->getType()->getPointerElementType(); itemType->isArrayTy()) { + const auto indexType = Type::getInt64Ty(ctx.Codegen.GetContext()); + Value* zeros = UndefValue::get(itemType); + for (ui32 idx = 0U; idx < itemType->getArrayNumElements(); ++idx) { + const auto item = GetElementPtrInst::CreateInBounds(itemType, pointer, { ConstantInt::get(indexType, 0), ConstantInt::get(indexType, idx) }, (TString("item_") += ToString(idx)).c_str(), block); + UnRefUnboxed(item, ctx, block); + zeros = InsertValueInst::Create(zeros, ConstantInt::get(itemType->getArrayElementType(), 0), {idx}, (TString("zero_") += ToString(idx)).c_str(), block); + } + new StoreInst(zeros, pointer, block); + } else { + UnRefUnboxed(pointer, ctx, block); + new StoreInst(ConstantInt::get(itemType, 0), pointer, block); + } +} + +void ValueAddRef(EValueRepresentation kind, Value* pointer, const TCodegenContext& ctx, BasicBlock*& block) { + switch (kind) { + case EValueRepresentation::Embedded: return; + case EValueRepresentation::Boxed: // TODO + case EValueRepresentation::String: // TODO + case EValueRepresentation::Any: return AddRefUnboxed(pointer, ctx, block); + } +} + +void ValueUnRef(EValueRepresentation kind, Value* pointer, const TCodegenContext& ctx, BasicBlock*& block) { + switch (kind) { + case EValueRepresentation::Embedded: return; + case EValueRepresentation::Boxed: // TODO + case EValueRepresentation::String: // TODO + case EValueRepresentation::Any: return UnRefUnboxed(pointer, ctx, block); + } +} + +void ValueCleanup(EValueRepresentation kind, Value* pointer, const TCodegenContext& ctx, BasicBlock*& block) { + switch (kind) { + case EValueRepresentation::Embedded: return; + case EValueRepresentation::Boxed: // TODO + case EValueRepresentation::String: // TODO + case EValueRepresentation::Any: return CleanupUnboxed(pointer, ctx, block); + } +} + +void ValueRelease(EValueRepresentation kind, Value* pointer, const TCodegenContext& ctx, BasicBlock*& block) { + switch (kind) { + case EValueRepresentation::Embedded: return; + case EValueRepresentation::Boxed: // TODO + case EValueRepresentation::String: // TODO + case EValueRepresentation::Any: return ReleaseUnboxed(pointer, ctx, block); + } +} + +std::pair<Value*, Value*> GetVariantParts(Value* variant, const TCodegenContext& ctx, BasicBlock*& block) { + auto& context = ctx.Codegen.GetContext(); + + const auto type = Type::getInt32Ty(context); + const auto lshr = BinaryOperator::CreateLShr(variant, ConstantInt::get(variant->getType(), 122), "lshr", block); + const auto trunc = CastInst::Create(Instruction::Trunc, lshr, type, "trunc", block); + + const auto check = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_NE, trunc, ConstantInt::get(type , 0), "check", block); + + const auto boxed = BasicBlock::Create(context, "boxed", ctx.Func); + const auto embed = BasicBlock::Create(context, "embed", ctx.Func); + const auto done = BasicBlock::Create(context, "done", ctx.Func); + + const auto index = PHINode::Create(type, 2U, "index", done); + const auto item = PHINode::Create(variant->getType(), 2U, "index", done); + + BranchInst::Create(embed, boxed, check, block); + + { + block = embed; + + const uint64_t init[] = {0xFFFFFFFFFFFFFFFFULL, 0x3FFFFFFFFFFFFFFULL}; + const auto mask = ConstantInt::get(variant->getType(), APInt(128, 2, init)); + const auto clean = BinaryOperator::CreateAnd(variant, mask, "clean", block); + + const auto dec = BinaryOperator::CreateSub(trunc, ConstantInt::get(type, 1), "dec", block); + index->addIncoming(dec, block); + item->addIncoming(clean, block); + BranchInst::Create(done, block); + } + + { + block = boxed; + + const auto place = new AllocaInst(item->getType(), 0U, "place", &ctx.Func->getEntryBlock().back()); + const auto idx = CallBoxedValueVirtualMethod<NUdf::TBoxedValueAccessor::EMethod::GetVariantIndex>(type, variant, ctx.Codegen, block); + CallBoxedValueVirtualMethod<NUdf::TBoxedValueAccessor::EMethod::GetVariantItem>(place, variant, ctx.Codegen, block); + const auto clean = new LoadInst(item->getType(), place, "clean", block); + ValueRelease(EValueRepresentation::Any, clean, ctx, block); + index->addIncoming(idx, block); + item->addIncoming(clean, block); + BranchInst::Create(done, block); + } + + block = done; + return std::make_pair(index, item); +} + +Value* MakeVariant(Value* item, Value* variant, const TCodegenContext& ctx, BasicBlock*& block) { + auto& context = ctx.Codegen.GetContext(); + + const auto boxed = BasicBlock::Create(context, "boxed", ctx.Func); + const auto embed = BasicBlock::Create(context, "embed", ctx.Func); + const auto done = BasicBlock::Create(context, "done", ctx.Func); + + const auto result = PHINode::Create(item->getType(), 1U, "index", done); + + const auto offset = ConstantInt::get(item->getType(), 122); + const auto lshr = BinaryOperator::CreateLShr(item, offset, "lshr", block); + + const auto checkItem = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, lshr, ConstantInt::get(lshr->getType(), 0), "check_item", block); + const auto checkIndex = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_ULT, variant, ConstantInt::get(variant->getType(), (1U << 6U) - 1U), "check_index", block); + const auto check = BinaryOperator::CreateAnd(checkItem, checkIndex, "and", block); + + BranchInst::Create(embed, boxed, check, block); + + { + block = embed; + + const auto index = BinaryOperator::CreateAdd(variant, ConstantInt::get(variant->getType(), 1), "index", block); + const auto extend = CastInst::Create(Instruction::ZExt, index, item->getType(), "extend", block); + const auto shift = BinaryOperator::CreateShl(extend, offset, "shift", block); + const auto output = BinaryOperator::CreateOr(item, shift, "output", block); + result->addIncoming(output, block); + BranchInst::Create(done, block); + } + + { + block = boxed; + + const auto factory = ctx.GetFactory(); + const auto func = ConstantInt::get(Type::getInt64Ty(context), GetMethodPtr(&THolderFactory::CreateBoxedVariantHolder)); + + if (NYql::NCodegen::ETarget::Windows != ctx.Codegen.GetEffectiveTarget()) { + const auto signature = FunctionType::get(item->getType(), {factory->getType(), item->getType(), variant->getType()}, false); + const auto creator = CastInst::Create(Instruction::IntToPtr, func, PointerType::getUnqual(signature), "creator", block); + const auto output = CallInst::Create(signature, creator, {factory, item, variant}, "output", block); + result->addIncoming(output, block); + } else { + const auto place = new AllocaInst(item->getType(), 0U, "place", block); + new StoreInst(item, place, block); + const auto signature = FunctionType::get(Type::getVoidTy(context), {factory->getType(), place->getType(), place->getType(), variant->getType()}, false); + const auto creator = CastInst::Create(Instruction::IntToPtr, func, PointerType::getUnqual(signature), "creator", block); + CallInst::Create(signature, creator, {factory, place, place, variant}, "", block); + const auto output = new LoadInst(item->getType(), place, "output", block); + result->addIncoming(output, block); + } + + BranchInst::Create(done, block); + } + + block = done; + return result; +} + +Value* GetNodeValue(IComputationNode* node, const TCodegenContext& ctx, BasicBlock*& block) { + if (const auto codegen = dynamic_cast<ICodegeneratorInlineNode*>(node)) + return codegen->CreateGetValue(ctx, block); + + auto& context = ctx.Codegen.GetContext(); + const auto ptr = ConstantInt::get(Type::getInt64Ty(context), intptr_t(node)); + const auto ptrType = PointerType::getUnqual(StructType::get(context)); + const auto nodeThis = CastInst::Create(Instruction::IntToPtr, ptr, ptrType, "node_this", block); + + const auto valueType = Type::getInt128Ty(context); + const auto retPtr = new AllocaInst(valueType, 0U, "return_ptr", &ctx.Func->getEntryBlock().back()); + const auto funType = ctx.Codegen.GetEffectiveTarget() != NYql::NCodegen::ETarget::Windows ? + FunctionType::get(Type::getVoidTy(context), {retPtr->getType(), nodeThis->getType(), ctx.Ctx->getType()}, false): + FunctionType::get(Type::getVoidTy(context), {nodeThis->getType(), retPtr->getType(), ctx.Ctx->getType()}, false); + const auto ptrFunType = PointerType::getUnqual(funType); + const auto tableType = PointerType::getUnqual(ptrFunType); + const auto nodeVTable = CastInst::Create(Instruction::IntToPtr, ptr, PointerType::getUnqual(tableType), "node_vtable", block); + + const auto table = new LoadInst(tableType, nodeVTable, "table", false, block); + const auto elem = GetElementPtrInst::CreateInBounds(ptrFunType, table, {ConstantInt::get(Type::getInt64Ty(context), GetMethodIndex(&IComputationNode::GetValue))}, "element", block); + const auto func = new LoadInst(ptrFunType, elem, "func", false, block); + + if (ctx.Codegen.GetEffectiveTarget() != NYql::NCodegen::ETarget::Windows) { + CallInst::Create(funType, func, {retPtr, nodeThis, ctx.Ctx}, "", block); + } else { + CallInst::Create(funType, func, {nodeThis, retPtr, ctx.Ctx}, "", block); + } + + ValueRelease(node->GetRepresentation(), retPtr, ctx, block); + const auto result = new LoadInst(valueType, retPtr, "return", false, block); + return result; +} + +void GetNodeValue(Value* value, IComputationNode* node, const TCodegenContext& ctx, BasicBlock*& block) { + if (const auto codegen = dynamic_cast<ICodegeneratorInlineNode*>(node)) { + const auto v = codegen->CreateGetValue(ctx, block); + new StoreInst(v, value, block); + ValueAddRef(node->GetRepresentation(), value, ctx, block); + return; + } + + auto& context = ctx.Codegen.GetContext(); + const auto ptr = ConstantInt::get(Type::getInt64Ty(context), intptr_t(node)); + const auto ptrType = PointerType::getUnqual(StructType::get(context)); + const auto nodeThis = CastInst::Create(Instruction::IntToPtr, ptr, ptrType, "node_this", block); + + const auto funType = ctx.Codegen.GetEffectiveTarget() != NYql::NCodegen::ETarget::Windows ? + FunctionType::get(Type::getVoidTy(context), {value->getType(), nodeThis->getType(), ctx.Ctx->getType()}, false): + FunctionType::get(Type::getVoidTy(context), {nodeThis->getType(), value->getType(), ctx.Ctx->getType()}, false); + const auto ptrFunType = PointerType::getUnqual(funType); + const auto tableType = PointerType::getUnqual(ptrFunType); + const auto nodeVTable = CastInst::Create(Instruction::IntToPtr, ptr, PointerType::getUnqual(tableType), "node_vtable", block); + + const auto table = new LoadInst(tableType, nodeVTable, "table", false, block); + const auto elem = GetElementPtrInst::CreateInBounds(ptrFunType, table, {ConstantInt::get(Type::getInt64Ty(context), GetMethodIndex(&IComputationNode::GetValue))}, "element", block); + const auto func = new LoadInst(ptrFunType, elem, "func", false, block); + + if (ctx.Codegen.GetEffectiveTarget() != NYql::NCodegen::ETarget::Windows) { + CallInst::Create(funType, func, {value, nodeThis, ctx.Ctx}, "", block); + } else { + CallInst::Create(funType, func, {nodeThis, value, ctx.Ctx}, "", block); + } +} + +ICodegeneratorInlineWideNode::TGenerateResult GetNodeValues(IComputationWideFlowNode* node, const TCodegenContext& ctx, BasicBlock*& block) { + if (const auto codegen = dynamic_cast<ICodegeneratorInlineWideNode*>(node)) + return codegen->GenGetValues(ctx, block); + throw TNoCodegen(); +} + +Value* GenNewArray(const TCodegenContext& ctx, Value* size, Value* items, BasicBlock* block) { + auto& context = ctx.Codegen.GetContext(); + const auto fact = ctx.GetFactory(); + const auto func = ConstantInt::get(Type::getInt64Ty(context), GetMethodPtr(&THolderFactory::CreateDirectArrayHolder)); + const auto valueType = Type::getInt128Ty(context); + if (NYql::NCodegen::ETarget::Windows != ctx.Codegen.GetEffectiveTarget()) { + const auto funType = FunctionType::get(valueType, {fact->getType(), size->getType(), items->getType()}, false); + const auto funcPtr = CastInst::Create(Instruction::IntToPtr, func, PointerType::getUnqual(funType), "function", block); + return CallInst::Create(funType, funcPtr, {fact, size, items}, "array", block); + } else { + const auto resultPtr = new AllocaInst(valueType, 0U, "return", block); + const auto funType = FunctionType::get(Type::getVoidTy(context), {fact->getType(), resultPtr->getType(), size->getType(), items->getType()}, false); + const auto funcPtr = CastInst::Create(Instruction::IntToPtr, func, PointerType::getUnqual(funType), "function", block); + CallInst::Create(funType, funcPtr, {fact, resultPtr, size, items}, "", block); + return new LoadInst(valueType, resultPtr, "array", block); + } +} + +Value* GetMemoryUsed(ui64 limit, const TCodegenContext& ctx, BasicBlock* block) { + if (!limit) { + return nullptr; + } + + auto& context = ctx.Codegen.GetContext(); + const auto fact = ctx.GetFactory(); + const auto func = ConstantInt::get(Type::getInt64Ty(context), GetMethodPtr(&THolderFactory::GetMemoryUsed)); + const auto funType = FunctionType::get(Type::getInt64Ty(context), {fact->getType()}, false); + const auto funcPtr = CastInst::Create(Instruction::IntToPtr, func, PointerType::getUnqual(funType), "get_used", block); + return CallInst::Create(funType, funcPtr, {fact}, "mem_used", block); +} + +template <bool TrackRss> +Value* CheckAdjustedMemLimit(ui64 limit, Value* init, const TCodegenContext& ctx, BasicBlock*& block) { + auto& context = ctx.Codegen.GetContext(); + + if (!limit || !init) { + return ConstantInt::getFalse(context); + } + + const auto indexType = Type::getInt32Ty(context); + + if constexpr (TrackRss) { + const auto rssPtr = GetElementPtrInst::CreateInBounds(GetCompContextType(context), ctx.Ctx, {ConstantInt::get(indexType, 0), ConstantInt::get(indexType, 5)}, "rss_ptr", block); + const auto rss = new LoadInst(Type::getInt32Ty(context), rssPtr, "rsscounter", block); + const auto inc = BinaryOperator::CreateAdd(rss, ConstantInt::get(rss->getType(), 1), "inc", block); + new StoreInst(inc, rssPtr, block); + const auto mod = BinaryOperator::CreateURem(rss, ConstantInt::get(rss->getType(), STEP_FOR_RSS_CHECK), "mod", block); + const auto now = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, mod, ConstantInt::get(mod->getType() , 0), "now", block); + + const auto call = BasicBlock::Create(context, "call", ctx.Func); + const auto skip = BasicBlock::Create(context, "skip", ctx.Func); + + BranchInst::Create(call, skip, now, block); + + block = call; + const auto func = ConstantInt::get(Type::getInt64Ty(context), GetMethodPtr(&TComputationContext::UpdateUsageAdjustor)); + const auto funType = FunctionType::get(Type::getVoidTy(context), {ctx.Ctx->getType(), Type::getInt64Ty(context)}, false); + const auto funcPtr = CastInst::Create(Instruction::IntToPtr, func, PointerType::getUnqual(funType), "update", block); + CallInst::Create(funType, funcPtr, {ctx.Ctx, ConstantInt::get(init->getType(), limit)}, "", block); + + BranchInst::Create(skip, block); + + block = skip; + } + + const auto adjPtr = GetElementPtrInst::CreateInBounds(GetCompContextType(context), ctx.Ctx, {ConstantInt::get(indexType, 0), ConstantInt::get(indexType, 4)}, "adj_ptr", block); + const auto adjustor = new LoadInst(Type::getFloatTy(context), adjPtr, "adjustor", block); + + const auto curr = GetMemoryUsed(limit, ctx, block); + const auto cast = CastInst::Create(Instruction::UIToFP, curr, adjustor->getType(), "cast", block); + const auto used = BinaryOperator::CreateFMul(cast, adjustor, "used", block); + const auto add = BinaryOperator::CreateAdd(init, ConstantInt::get(init->getType(), limit), "add", block); + const auto upper = CastInst::Create(Instruction::UIToFP, add, adjustor->getType(), "upper", block); + return CmpInst::Create(Instruction::FCmp, FCmpInst::FCMP_OGE, used, upper, "enough", block); +} + +template Value* CheckAdjustedMemLimit<false>(ui64 limit, Value* init, const TCodegenContext& ctx, BasicBlock*& block); +template Value* CheckAdjustedMemLimit<true>(ui64 limit, Value* init, const TCodegenContext& ctx, BasicBlock*& block); + +Value* WrapArgumentForWindows(Value* arg, const TCodegenContext& ctx, BasicBlock* block) { + if (ctx.Codegen.GetEffectiveTarget() != NYql::NCodegen::ETarget::Windows) { + return arg; + } + + const auto newArg = new AllocaInst(arg->getType(), 0, "argument", block); + new StoreInst(arg, newArg, block); + return newArg; +} + +Value* CallBoxedValueVirtualMethodImpl(uintptr_t methodPtr, Type* returnType, Value* value, NYql::NCodegen::ICodegen& codegen, BasicBlock* block) { + auto& context = codegen.GetContext(); + + const auto data = CastInst::Create(Instruction::Trunc, value, Type::getInt64Ty(context), "data", block); + const auto ptrStructType = PointerType::getUnqual(StructType::get(context)); + const auto boxed = CastInst::Create(Instruction::IntToPtr, data, ptrStructType, "boxed", block); + + const auto funType = FunctionType::get(returnType, {boxed->getType()}, false); + const auto ptrFunType = PointerType::getUnqual(funType); + const auto tableType = PointerType::getUnqual(ptrFunType); + const auto vTable = CastInst::Create(Instruction::IntToPtr, data, PointerType::getUnqual(tableType), "vtable", block); + + const auto table = new LoadInst(tableType, vTable, "table", false, block); + const auto elem = GetElementPtrInst::CreateInBounds(ptrFunType, table, {ConstantInt::get(Type::getInt64Ty(context), GetMethodPtrIndex(methodPtr))}, "element", block); + const auto func = new LoadInst(ptrFunType, elem, "func", false, block); + + const auto call = CallInst::Create(funType, func, {boxed}, returnType->isVoidTy() ? "" : "return", block); + return call; +} + +void CallBoxedValueVirtualMethodImpl(uintptr_t methodPtr, Value* output, Value* value, NYql::NCodegen::ICodegen& codegen, BasicBlock* block) { + auto& context = codegen.GetContext(); + + const auto data = CastInst::Create(Instruction::Trunc, value, Type::getInt64Ty(context), "data", block); + const auto ptrStructType = PointerType::getUnqual(StructType::get(context)); + const auto boxed = CastInst::Create(Instruction::IntToPtr, data, ptrStructType, "boxed", block); + + const auto funType = (codegen.GetEffectiveTarget() != NYql::NCodegen::ETarget::Windows) ? + FunctionType::get(Type::getVoidTy(context), {output->getType(), boxed->getType()}, false): + FunctionType::get(Type::getVoidTy(context), {boxed->getType(), output->getType()}, false); + const auto ptrFunType = PointerType::getUnqual(funType); + const auto tableType = PointerType::getUnqual(ptrFunType); + const auto vTable = CastInst::Create(Instruction::IntToPtr, data, PointerType::getUnqual(tableType), "vtable", block); + + const auto table = new LoadInst(tableType, vTable, "table", false, block); + const auto elem = GetElementPtrInst::CreateInBounds(ptrFunType, table, {ConstantInt::get(Type::getInt64Ty(context), GetMethodPtrIndex(methodPtr))}, "element", block); + const auto func = new LoadInst(ptrFunType, elem, "func", false, block); + + if (codegen.GetEffectiveTarget() != NYql::NCodegen::ETarget::Windows) { + CallInst::Create(funType, func, {output, boxed}, "", block); + } else { + CallInst::Create(funType, func, {boxed, output}, "", block); + } +} + +void CallBoxedValueVirtualMethodImpl(uintptr_t methodPtr, Value* output, Value* value, NYql::NCodegen::ICodegen& codegen, BasicBlock* block, Value* argument) { + auto& context = codegen.GetContext(); + + const auto data = CastInst::Create(Instruction::Trunc, value, Type::getInt64Ty(context), "data", block); + const auto ptrStructType = PointerType::getUnqual(StructType::get(context)); + const auto boxed = CastInst::Create(Instruction::IntToPtr, data, ptrStructType, "boxed", block); + + const auto funType = (codegen.GetEffectiveTarget() != NYql::NCodegen::ETarget::Windows) ? + FunctionType::get(Type::getVoidTy(context), {output->getType(), boxed->getType(), argument->getType()}, false): + FunctionType::get(Type::getVoidTy(context), {boxed->getType(), output->getType(), argument->getType()}, false); + const auto ptrFunType = PointerType::getUnqual(funType); + const auto tableType = PointerType::getUnqual(ptrFunType); + const auto vTable = CastInst::Create(Instruction::IntToPtr, data, PointerType::getUnqual(tableType), "vtable", block); + + const auto table = new LoadInst(tableType, vTable, "table", false, block); + const auto elem = GetElementPtrInst::CreateInBounds(ptrFunType, table, {ConstantInt::get(Type::getInt64Ty(context), GetMethodPtrIndex(methodPtr))}, "element", block); + const auto func = new LoadInst(ptrFunType, elem, "func", false, block); + + if (codegen.GetEffectiveTarget() != NYql::NCodegen::ETarget::Windows) { + CallInst::Create(funType, func, {output, boxed, argument}, "", block); + } else { + CallInst::Create(funType, func, {boxed, output, argument}, "", block); + } +} + +Value* CallBoxedValueVirtualMethodImpl(uintptr_t methodPtr, Type* returnType, Value* value, NYql::NCodegen::ICodegen& codegen, BasicBlock* block, Value* argument) { + auto& context = codegen.GetContext(); + + const auto data = CastInst::Create(Instruction::Trunc, value, Type::getInt64Ty(context), "data", block); + const auto ptrStructType = PointerType::getUnqual(StructType::get(context)); + const auto boxed = CastInst::Create(Instruction::IntToPtr, data, ptrStructType, "boxed", block); + + const auto funType = FunctionType::get(returnType, {boxed->getType(), argument->getType()}, false); + const auto ptrFunType = PointerType::getUnqual(funType); + const auto tableType = PointerType::getUnqual(ptrFunType); + const auto vTable = CastInst::Create(Instruction::IntToPtr, data, PointerType::getUnqual(tableType), "vtable", block); + + const auto table = new LoadInst(tableType, vTable, "table", false, block); + const auto elem = GetElementPtrInst::CreateInBounds(ptrFunType, table, {ConstantInt::get(Type::getInt64Ty(context), GetMethodPtrIndex(methodPtr))}, "element", block); + const auto func = new LoadInst(ptrFunType, elem, "func", false, block); + + const auto call = CallInst::Create(funType, func, {boxed, argument}, returnType->isVoidTy() ? "" : "return", block); + return call; +} + +void CallBoxedValueVirtualMethodImpl(uintptr_t methodPtr, Value* output, Value* value, NYql::NCodegen::ICodegen& codegen, BasicBlock* block, Value* arg1, Value* arg2) { + auto& context = codegen.GetContext(); + + const auto data = CastInst::Create(Instruction::Trunc, value, Type::getInt64Ty(context), "data", block); + const auto ptrStructType = PointerType::getUnqual(StructType::get(context)); + const auto boxed = CastInst::Create(Instruction::IntToPtr, data, ptrStructType, "boxed", block); + + const auto funType = (codegen.GetEffectiveTarget() != NYql::NCodegen::ETarget::Windows) ? + FunctionType::get(Type::getVoidTy(context), {output->getType(), boxed->getType(), arg1->getType(), arg2->getType()}, false): + FunctionType::get(Type::getVoidTy(context), {boxed->getType(), output->getType(), arg1->getType(), arg2->getType()}, false); + const auto ptrFunType = PointerType::getUnqual(funType); + const auto tableType = PointerType::getUnqual(ptrFunType); + const auto vTable = CastInst::Create(Instruction::IntToPtr, data, PointerType::getUnqual(tableType), "vtable", block); + + const auto table = new LoadInst(tableType, vTable, "table", false, block); + const auto elem = GetElementPtrInst::CreateInBounds(ptrFunType, table, {ConstantInt::get(Type::getInt64Ty(context), GetMethodPtrIndex(methodPtr))}, "element", block); + const auto func = new LoadInst(ptrFunType, elem, "func", false, block); + + if (codegen.GetEffectiveTarget() != NYql::NCodegen::ETarget::Windows) { + CallInst::Create(funType, func, {output, boxed, arg1, arg2}, "", block); + } else { + CallInst::Create(funType, func, {boxed, output, arg1, arg2}, "", block); + } +} + +Value* CallBoxedValueVirtualMethodImpl(uintptr_t methodPtr, Type* returnType, Value* value, NYql::NCodegen::ICodegen& codegen, BasicBlock* block, Value* arg1, Value* arg2) { + auto& context = codegen.GetContext(); + + const auto data = CastInst::Create(Instruction::Trunc, value, Type::getInt64Ty(context), "data", block); + const auto ptrStructType = PointerType::getUnqual(StructType::get(context)); + const auto boxed = CastInst::Create(Instruction::IntToPtr, data, ptrStructType, "boxed", block); + + const auto funType = FunctionType::get(returnType, {boxed->getType(), arg1->getType(), arg2->getType()}, false); + const auto ptrFunType = PointerType::getUnqual(funType); + const auto tableType = PointerType::getUnqual(ptrFunType); + const auto vTable = CastInst::Create(Instruction::IntToPtr, data, PointerType::getUnqual(tableType), "vtable", block); + + const auto table = new LoadInst(tableType, vTable, "table", false, block); + const auto elem = GetElementPtrInst::CreateInBounds(ptrFunType, table, {ConstantInt::get(Type::getInt64Ty(context), GetMethodPtrIndex(methodPtr))}, "element", block); + const auto func = new LoadInst(ptrFunType, elem, "func", false, block); + + const auto call = CallInst::Create(funType, func, {boxed, arg1, arg2}, returnType->isVoidTy() ? "" : "return", block); + return call; +} + +Value* CallUnaryUnboxedValueFunctionImpl(uintptr_t methodPtr, Type* result, Value* arg, NYql::NCodegen::ICodegen& codegen, BasicBlock* block) { + auto& context = codegen.GetContext(); + const auto doFunc = ConstantInt::get(Type::getInt64Ty(context), methodPtr); + if (NYql::NCodegen::ETarget::Windows != codegen.GetEffectiveTarget()) { + const auto funType = FunctionType::get(result, {arg->getType()}, false); + const auto funcPtr = CastInst::Create(Instruction::IntToPtr, doFunc, PointerType::getUnqual(funType), "ptr", block); + const auto call = CallInst::Create(funType, funcPtr, {arg}, "call", block); + return call; + } else { + const auto ptrArg = new AllocaInst(arg->getType(), 0U, "arg", block); + new StoreInst(arg, ptrArg, block); + + if (Type::getInt128Ty(context) == result) { + const auto ptrResult = new AllocaInst(result, 0U, "result", block); + const auto funType = FunctionType::get(Type::getVoidTy(context), {ptrResult->getType(), ptrArg->getType()}, false); + const auto funcPtr = CastInst::Create(Instruction::IntToPtr, doFunc, PointerType::getUnqual(funType), "ptr", block); + CallInst::Create(funType, funcPtr, {ptrResult, ptrArg}, "", block); + const auto res = new LoadInst(result, ptrResult, "res", block); + return res; + } else { + const auto funType = FunctionType::get(result, {ptrArg->getType()}, false); + const auto funcPtr = CastInst::Create(Instruction::IntToPtr, doFunc, PointerType::getUnqual(funType), "ptr", block); + const auto call = CallInst::Create(funType, funcPtr, {ptrArg}, "call", block); + return call; + } + } +} + +Value* CallBinaryUnboxedValueFunctionImpl(uintptr_t methodPtr, Type* result, Value* left, Value* right, NYql::NCodegen::ICodegen& codegen, BasicBlock* block) { + auto& context = codegen.GetContext(); + const auto doFunc = ConstantInt::get(Type::getInt64Ty(context), methodPtr); + if (NYql::NCodegen::ETarget::Windows != codegen.GetEffectiveTarget()) { + const auto funType = FunctionType::get(result, {left->getType(), right->getType()}, false); + const auto funcPtr = CastInst::Create(Instruction::IntToPtr, doFunc, PointerType::getUnqual(funType), "ptr", block); + const auto call = CallInst::Create(funType, funcPtr, {left, right}, "call", block); + return call; + } else { + const auto ptrLeft = new AllocaInst(left->getType(), 0U, "left", block); + const auto ptrRight = new AllocaInst(right->getType(), 0U, "right", block); + new StoreInst(left, ptrLeft, block); + new StoreInst(right, ptrRight, block); + + if (Type::getInt128Ty(context) == result) { + const auto ptrResult = new AllocaInst(result, 0U, "result", block); + const auto funType = FunctionType::get(Type::getVoidTy(context), {ptrResult->getType(), ptrLeft->getType(), ptrRight->getType()}, false); + const auto funcPtr = CastInst::Create(Instruction::IntToPtr, doFunc, PointerType::getUnqual(funType), "ptr", block); + CallInst::Create(funType, funcPtr, {ptrResult, ptrLeft, ptrRight}, "", block); + const auto res = new LoadInst(result, ptrResult, "res", block); + return res; + } else { + const auto funType = FunctionType::get(result, {ptrLeft->getType(), ptrRight->getType()}, false); + const auto funcPtr = CastInst::Create(Instruction::IntToPtr, doFunc, PointerType::getUnqual(funType), "ptr", block); + const auto call = CallInst::Create(funType, funcPtr, {ptrLeft, ptrRight}, "call", block); + return call; + } + } +} + +Y_NO_INLINE Value* TDecoratorCodegeneratorNodeBase::CreateGetValueImpl(IComputationNode* node, + const TCodegenContext& ctx, BasicBlock*& block) const { + const auto arg = GetNodeValue(node, ctx, block); + const auto value = DoGenerateGetValue(ctx, arg, block); + if (value->getType()->isPointerTy()) { + const auto load = new LoadInst(Type::getInt128Ty(ctx.Codegen.GetContext()), value, "load", block); + ValueRelease(node->GetRepresentation(), load, ctx, block); + return load; + } else { + return value; + } +} + +Y_NO_INLINE Value* TStatelessFlowCodegeneratorNodeBase::CreateGetValueImpl(const IComputationNode* node, + const TCodegenContext& ctx, BasicBlock*& block) const { + const auto value = DoGenerateGetValue(ctx, block); + if (value->getType()->isPointerTy()) { + const auto load = new LoadInst(Type::getInt128Ty(ctx.Codegen.GetContext()), value, "load", block); + ValueRelease(node->GetRepresentation(), load, ctx, block); + return load; + } else { + return value; + } +} + +Y_NO_INLINE ICodegeneratorInlineWideNode::TGenerateResult TStatelessWideFlowCodegeneratorNodeBase::GenGetValuesImpl( + const TCodegenContext& ctx, BasicBlock*& block) const { + return DoGenGetValues(ctx, block); +} + +Y_NO_INLINE Value* TFlowSourceCodegeneratorNodeBase::CreateGetValueImpl( + const IComputationNode* node, const TCodegenContext& ctx, BasicBlock*& block) const { + auto& context = ctx.Codegen.GetContext(); + const auto valueType = Type::getInt128Ty(context); + const auto statePtr = GetElementPtrInst::CreateInBounds(valueType, ctx.GetMutables(), {ConstantInt::get(Type::getInt32Ty(context), node->GetIndex())}, "state_ptr", block); + + const auto value = DoGenerateGetValue(ctx, statePtr, block); + if (value->getType()->isPointerTy()) { + const auto load = new LoadInst(valueType, value, "load", block); + ValueRelease(node->GetRepresentation(), load, ctx, block); + return load; + } else { + return value; + } +} + +Y_NO_INLINE ICodegeneratorInlineWideNode::TGenerateResult TWideFlowSourceCodegeneratorNodeBase::GenGetValuesImpl( + const IComputationNode* node, const TCodegenContext& ctx, BasicBlock*& block) const { + auto& context = ctx.Codegen.GetContext(); + const auto valueType = Type::getInt128Ty(context); + const auto statePtr = GetElementPtrInst::CreateInBounds(valueType, ctx.GetMutables(), {ConstantInt::get(Type::getInt32Ty(context), node->GetIndex())}, "state_ptr", block); + return DoGenGetValues(ctx, statePtr, block); +} + +Y_NO_INLINE Value* TStatefulFlowCodegeneratorNodeBase::CreateGetValueImpl( + const IComputationNode* node, const TCodegenContext& ctx, BasicBlock*& block) const { + auto& context = ctx.Codegen.GetContext(); + const auto valueType = Type::getInt128Ty(context); + const auto statePtr = GetElementPtrInst::CreateInBounds(valueType, ctx.GetMutables(), {ConstantInt::get(Type::getInt32Ty(context), node->GetIndex())}, "state_ptr", block); + + const auto value = DoGenerateGetValue(ctx, statePtr, block); + if (value->getType()->isPointerTy()) { + const auto load = new LoadInst(valueType, value, "load", block); + ValueRelease(node->GetRepresentation(), load, ctx, block); + return load; + } else { + return value; + } +} + +Y_NO_INLINE ICodegeneratorInlineWideNode::TGenerateResult TStatefulWideFlowCodegeneratorNodeBase::GenGetValuesImpl( + const IComputationNode* node, const TCodegenContext& ctx, BasicBlock*& block) const { + auto& context = ctx.Codegen.GetContext(); + const auto valueType = Type::getInt128Ty(context); + const auto statePtr = GetElementPtrInst::CreateInBounds(valueType, ctx.GetMutables(), {ConstantInt::get(Type::getInt32Ty(context), node->GetIndex())}, "state_ptr", block); + return DoGenGetValues(ctx, statePtr, block); +} + +Y_NO_INLINE ICodegeneratorInlineWideNode::TGenerateResult TPairStateWideFlowCodegeneratorNodeBase::GenGetValuesImpl( + const IComputationNode* node, const TCodegenContext& ctx, BasicBlock*& block) const { + auto& context = ctx.Codegen.GetContext(); + auto idx = node->GetIndex(); + const auto valueType = Type::getInt128Ty(context); + const auto firstPtr = GetElementPtrInst::CreateInBounds(valueType, ctx.GetMutables(), {ConstantInt::get(Type::getInt32Ty(context), idx)}, "first_ptr", block); + const auto secondPtr = GetElementPtrInst::CreateInBounds(valueType, ctx.GetMutables(), {ConstantInt::get(Type::getInt32Ty(context), ++idx)}, "second_ptr", block); + return DoGenGetValues(ctx, firstPtr, secondPtr, block); +} + +Y_NO_INLINE Value* TPairStateFlowCodegeneratorNodeBase::CreateGetValueImpl( + const IComputationNode* node, const TCodegenContext& ctx, BasicBlock*& block) const { + auto& context = ctx.Codegen.GetContext(); + auto idx = node->GetIndex(); + const auto valueType = Type::getInt128Ty(context); + const auto firstPtr = GetElementPtrInst::CreateInBounds(valueType, ctx.GetMutables(), {ConstantInt::get(Type::getInt32Ty(context), idx)}, "first_ptr", block); + const auto secondPtr = GetElementPtrInst::CreateInBounds(valueType, ctx.GetMutables(), {ConstantInt::get(Type::getInt32Ty(context), ++idx)}, "second_ptr", block); + + const auto value = DoGenerateGetValue(ctx, firstPtr, secondPtr, block); + if (value->getType()->isPointerTy()) { + const auto load = new LoadInst(valueType, value, "load", block); + ValueRelease(node->GetRepresentation(), load, ctx, block); + return load; + } else { + return value; + } +} + +Y_NO_INLINE Value* TBinaryCodegeneratorNodeBase::CreateGetValueImpl(const IComputationNode* node, + const TCodegenContext& ctx, BasicBlock*& block) const { + const auto value = DoGenerateGetValue(ctx, block); + if (value->getType()->isPointerTy()) { + ValueRelease(node->GetRepresentation(), value, ctx, block); + const auto load = new LoadInst(Type::getInt128Ty(ctx.Codegen.GetContext()), value, "load", block); + return load; + } else { + return value; + } +} + +Y_NO_INLINE Value* TMutableCodegeneratorNodeBase::CreateGetValueImpl( + bool stateless, EValueRepresentation representation, ui32 valueIndex, + const TString& name, const TCodegenContext& ctx, BasicBlock*& block) const { + if (stateless) { + const auto newValue = DoGenerateGetValue(ctx, block); + if (newValue->getType()->isPointerTy()) { + ValueRelease(representation, newValue, ctx, block); + const auto load = new LoadInst(Type::getInt128Ty(ctx.Codegen.GetContext()), newValue, "load", block); + return load; + } else { + return newValue; + } + } + + return ctx.AlwaysInline ? MakeGetValueBody(representation, valueIndex, ctx, block) : + CallInst::Create(GenerateInternalGetValue(name, representation, valueIndex, ctx.Codegen), {ctx.Ctx}, "getter", block); +} + +Function* TMutableCodegeneratorNodeBase::GenerateInternalGetValue(const TString& name, + EValueRepresentation representation, ui32 valueIndex, NYql::NCodegen::ICodegen& codegen) const { + auto& module = codegen.GetModule(); + auto& context = codegen.GetContext(); + if (const auto f = module.getFunction(name.c_str())) + return f; + + const auto funcType = FunctionType::get(Type::getInt128Ty(context), {PointerType::getUnqual(GetCompContextType(context))}, false); + + TCodegenContext ctx(codegen); + ctx.Func = cast<Function>(module.getOrInsertFunction(name.c_str(), funcType).getCallee()); + + DISubprogramAnnotator annotator(ctx, ctx.Func); + + + auto main = BasicBlock::Create(context, "main", ctx.Func); + ctx.Ctx = &*ctx.Func->arg_begin(); + ctx.Ctx->addAttr(Attribute::NonNull); + + const auto get = MakeGetValueBody(representation, valueIndex, ctx, main); + + ReturnInst::Create(context, get, main); + return ctx.Func; +} + +Value* TMutableCodegeneratorNodeBase::MakeGetValueBody(EValueRepresentation representation, ui32 valueIndex, const TCodegenContext& ctx, BasicBlock*& block) const { + auto& context = ctx.Codegen.GetContext(); + const auto indexType = Type::getInt32Ty(context); + const auto valueType = Type::getInt128Ty(context); + const auto valuePtr = GetElementPtrInst::CreateInBounds(valueType, ctx.GetMutables(), {ConstantInt::get(indexType, valueIndex)}, "value_ptr", block); + const auto value = new LoadInst(valueType, valuePtr, "value", block); + + const auto invv = ConstantInt::get(value->getType(), 0xFFFFFFFFFFFFFFFFULL); + + const auto check = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, value, invv, "check", block); + + const auto comp = BasicBlock::Create(context, "comp", ctx.Func); + const auto done = BasicBlock::Create(context, "done", ctx.Func); + + BranchInst::Create(comp, done, check, block); + + block = comp; + + const auto newValue = DoGenerateGetValue(ctx, block); + + if (newValue->getType()->isPointerTy()) { + const auto load = new LoadInst(valueType, newValue, "value", block); + new StoreInst(load, valuePtr, block); + new StoreInst(ConstantInt::get(load->getType(), 0), newValue, block); + } else { + new StoreInst(newValue, valuePtr, block); + ValueAddRef(representation, valuePtr, ctx, block); + } + + BranchInst::Create(done, block); + block = done; + + const auto result = new LoadInst(valueType, valuePtr, "result", false, block); + return result; +} + +Y_NO_INLINE Value* TMutableCodegeneratorPtrNodeBase::CreateGetValueImpl( + bool stateless, EValueRepresentation representation, ui32 valueIndex, + const TString& name, const TCodegenContext& ctx, BasicBlock*& block) const { + if (stateless) { + const auto type = Type::getInt128Ty(ctx.Codegen.GetContext()); + const auto pointer = ctx.Func->getEntryBlock().empty() ? + new AllocaInst(type, 0U, "output", &ctx.Func->getEntryBlock()): + new AllocaInst(type, 0U, "output", &ctx.Func->getEntryBlock().back()); + + DoGenerateGetValue(ctx, pointer, block); + ValueRelease(representation, pointer, ctx, block); + const auto load = new LoadInst(type, pointer, "load", block); + return load; + } + + return ctx.AlwaysInline ? MakeGetValueBody(valueIndex, ctx, block) : + CallInst::Create(GenerateInternalGetValue(name, valueIndex, ctx.Codegen), {ctx.Ctx}, "getter", block); +} + +Value* TMutableCodegeneratorPtrNodeBase::MakeGetValueBody(ui32 valueIndex, const TCodegenContext& ctx, BasicBlock*& block) const { + auto& context = ctx.Codegen.GetContext(); + const auto indexType = Type::getInt32Ty(context); + const auto valueType = Type::getInt128Ty(context); + const auto valuePtr = GetElementPtrInst::CreateInBounds(valueType, ctx.GetMutables(), {ConstantInt::get(indexType, valueIndex)}, "value_ptr", block); + const auto value = new LoadInst(valueType, valuePtr, "value", block); + + const auto invv = ConstantInt::get(value->getType(), 0xFFFFFFFFFFFFFFFFULL); + + const auto check = CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, value, invv, "check", block); + + const auto comp = BasicBlock::Create(context, "comp", ctx.Func); + const auto done = BasicBlock::Create(context, "done", ctx.Func); + + BranchInst::Create(comp, done, check, block); + + block = comp; + + DoGenerateGetValue(ctx, valuePtr, block); + + BranchInst::Create(done, block); + block = done; + + const auto result = new LoadInst(valueType, valuePtr, "result", false, block); + return result; +} + +Function* TMutableCodegeneratorPtrNodeBase::GenerateInternalGetValue(const TString& name, ui32 valueIndex, NYql::NCodegen::ICodegen& codegen) const { + auto& module = codegen.GetModule(); + auto& context = codegen.GetContext(); + if (const auto f = module.getFunction(name.c_str())) + return f; + + const auto contextType = GetCompContextType(context); + + const auto funcType = FunctionType::get(Type::getInt128Ty(context), {PointerType::getUnqual(contextType)}, false); + + TCodegenContext ctx(codegen); + ctx.Func = cast<Function>(module.getOrInsertFunction(name.c_str(), funcType).getCallee()); + + DISubprogramAnnotator annotator(ctx, ctx.Func); + + + auto main = BasicBlock::Create(context, "main", ctx.Func); + ctx.Ctx = &*ctx.Func->arg_begin(); + ctx.Ctx->addAttr(Attribute::NonNull); + + const auto get = MakeGetValueBody(valueIndex, ctx, main); + + ReturnInst::Create(context, get, main); + return ctx.Func; +} + +Y_NO_INLINE Value* TMutableCodegeneratorFallbackNodeBase::DoGenerateGetValueImpl( + uintptr_t methodPtr, uintptr_t thisPtr, const TCodegenContext& ctx, BasicBlock*& block) const { + auto& context = ctx.Codegen.GetContext(); + const auto type = Type::getInt128Ty(context); + const auto ptrType = PointerType::getUnqual(StructType::get(context)); + const auto doFunc = ConstantInt::get(Type::getInt64Ty(context), methodPtr); + const auto self = CastInst::Create(Instruction::IntToPtr, ConstantInt::get(Type::getInt64Ty(context), thisPtr), ptrType, "self", block); + if (NYql::NCodegen::ETarget::Windows != ctx.Codegen.GetEffectiveTarget()) { + const auto funType = FunctionType::get(type, {self->getType(), ctx.Ctx->getType()}, false); + const auto doFuncPtr = CastInst::Create(Instruction::IntToPtr, doFunc, PointerType::getUnqual(funType), "function", block); + const auto value = CallInst::Create(funType, doFuncPtr, {self, ctx.Ctx}, "value", block); + return value; + } else { + const auto resultPtr = new AllocaInst(type, 0U, "return", block); + const auto funType = FunctionType::get(Type::getVoidTy(context), {self->getType(), resultPtr->getType(), ctx.Ctx->getType()}, false); + const auto doFuncPtr = CastInst::Create(Instruction::IntToPtr, doFunc, PointerType::getUnqual(funType), "function", block); + CallInst::Create(funType, doFuncPtr, {self, resultPtr, ctx.Ctx}, "", block); + const auto value = new LoadInst(type, resultPtr, "value", block); + return value; + } +} + +Y_NO_INLINE Function* TCodegeneratorRootNodeBase::GenerateGetValueImpl( + const TString& name, const ICodegeneratorInlineNode* gen, NYql::NCodegen::ICodegen& codegen) { + auto& module = codegen.GetModule(); + auto& context = codegen.GetContext(); + + if (const auto f = module.getFunction(name.c_str())) + return f; + + const auto valueType = Type::getInt128Ty(context); + const auto contextType = GetCompContextType(context); + + const auto funcType = codegen.GetEffectiveTarget() != NYql::NCodegen::ETarget::Windows ? + FunctionType::get(valueType, {PointerType::getUnqual(contextType)}, false): + FunctionType::get(Type::getVoidTy(context) , {PointerType::getUnqual(valueType), PointerType::getUnqual(contextType)}, false); + + TCodegenContext ctx(codegen); + ctx.Func = cast<Function>(module.getOrInsertFunction(name.c_str(), funcType).getCallee()); + + DISubprogramAnnotator annotator(ctx, ctx.Func); + + + auto args = ctx.Func->arg_begin(); + if (codegen.GetEffectiveTarget() == NYql::NCodegen::ETarget::Windows) { + auto& firstArg = *args++; + firstArg.addAttr(Attribute::StructRet); + firstArg.addAttr(Attribute::NoAlias); + } + + auto main = BasicBlock::Create(context, "main", ctx.Func); + ctx.Ctx = &*args; + ctx.Ctx->addAttr(Attribute::NonNull); + + const auto get = gen->CreateGetValue(ctx, main); + + if (codegen.GetEffectiveTarget() != NYql::NCodegen::ETarget::Windows) { + ReturnInst::Create(context, get, main); + } else { + new StoreInst(get, &*--args, main); + ReturnInst::Create(context, main); + } + + return ctx.Func; +} + +#if __clang__ && (__clang_major__ < 16) +TSrcLocation TSrcLocation::current() { + return {}; +} + +const char* TSrcLocation::file_name() const { + return __FILE__; +} + +size_t TSrcLocation::line() const { + return __LINE__; +} + +size_t TSrcLocation::column() const { + return 0; +} +#endif + +DISubprogramAnnotator::DISubprogramAnnotator(TCodegenContext& ctx, Function* subprogramFunc, const TSrcLocation& location) + : Ctx(ctx) + , DebugBuilder(std::make_unique<DIBuilder>(ctx.Codegen.GetModule())) + , Subprogram(MakeDISubprogram(subprogramFunc->getName(), location)) + , Func(subprogramFunc) +{ + subprogramFunc->setSubprogram(Subprogram); + Ctx.Annotator = this; +} + +DISubprogramAnnotator::~DISubprogramAnnotator() { + Ctx.Annotator = nullptr; + { // necessary stub annotation of "CallInst"s + DIScopeAnnotator stubAnnotate(this); + for (BasicBlock& block : *Func) { + for (Instruction& inst : block) { + if (CallInst* callInst = dyn_cast_or_null<CallInst>(&inst)) { + const auto& debugLoc = callInst->getDebugLoc(); + if (!debugLoc) { + stubAnnotate(callInst); + } + } + } + } + } + DebugBuilder->finalizeSubprogram(Subprogram); +} + +DIFile* DISubprogramAnnotator::MakeDIFile(const TSrcLocation& location) { + TFsPath path = TString(location.file_name()); + return DebugBuilder->createFile(path.GetName().c_str(), path.Parent().GetPath().c_str()); +} + +DISubprogram* DISubprogramAnnotator::MakeDISubprogram(const StringRef& name, const TSrcLocation& location) { + const auto file = MakeDIFile(location); + const auto unit = DebugBuilder->createCompileUnit(llvm::dwarf::DW_LANG_C_plus_plus, file, "MKQL", false, "", 0); + const auto subroutineType = DebugBuilder->createSubroutineType(DebugBuilder->getOrCreateTypeArray({})); + return DebugBuilder->createFunction( + unit, + name, + llvm::StringRef(), + file, 0, + subroutineType, 0, llvm::DINode::FlagPrototyped, llvm::DISubprogram::SPFlagDefinition + ); +} + +DIScopeAnnotator::DIScopeAnnotator(DISubprogramAnnotator* subprogramAnnotator, const TSrcLocation& location) + : SubprogramAnnotator(nullptr) + , Scope(nullptr) +{ + Y_ENSURE(subprogramAnnotator != nullptr); + SubprogramAnnotator = subprogramAnnotator; + Scope = SubprogramAnnotator->DebugBuilder->createLexicalBlock(SubprogramAnnotator->Subprogram, SubprogramAnnotator->MakeDIFile(location), location.line(), location.column()); +} + +Instruction* DIScopeAnnotator::operator()(Instruction* inst, const TSrcLocation& location) const { + inst->setDebugLoc(DILocation::get(SubprogramAnnotator->Ctx.Codegen.GetContext(), location.line(), location.column(), Scope)); + return inst; +} + +} +} +#endif |