#include "compiler.h"
#include "defaults.h"
#include "event.h"
#include "thread.h"
#include <cstdlib> //for abort()
#if defined(_win_)
#include "winint.h"
#endif
#if defined(_unix_)
#include <cxxabi.h>
#if !defined(Y_CXA_EH_GLOBALS_COMPLETE)
namespace __cxxabiv1 {
struct __cxa_eh_globals {
void* caughtExceptions;
unsigned int uncaughtExceptions;
};
extern "C" __cxa_eh_globals* __cxa_get_globals();
}
#endif
#endif
#include <util/stream/output.h>
#include <util/generic/yexception.h>
#define FROM_CONTEXT_IMPL
#include "context.h"
void ITrampoLine::DoRun() {
}
void ITrampoLine::DoRunNaked() {
try {
DoRun();
} catch (...) {
Cerr << "Uncaught exception in coroutine: " << CurrentExceptionMessage() << "\n";
}
abort();
}
static inline void Run(void* arg) {
((ITrampoLine*)arg)->DoRunNaked();
}
#if defined(USE_JUMP_CONT)
extern "C" void __mylongjmp(__myjmp_buf env, int val) __attribute__((__noreturn__));
extern "C" int __mysetjmp(__myjmp_buf env) __attribute__((__returns_twice__));
namespace {
class TStackType {
public:
inline TStackType(TArrayRef<char> range) noexcept
#if defined(STACK_GROW_DOWN)
: Data_(range.data() + range.size())
#else
: Data_(range.data() + STACK_ALIGN)
#endif
{
ReAlign();
}
inline ~TStackType() = default;
inline void ReAlign() noexcept {
Data_ = AlignStackPtr(Data_);
}
template <class T>
inline void Push(T t) noexcept {
#if defined(STACK_GROW_DOWN)
Data_ -= sizeof(T);
*((T*)Data_) = t;
#else
*((T*)Data_) = t;
Data_ += sizeof(T);
#endif
}
inline char* StackPtr() noexcept {
return Data_;
}
private:
static inline char* AlignStackPtr(char* ptr) noexcept {
#if defined(STACK_GROW_DOWN)
return AlignDown(ptr, STACK_ALIGN);
#else
return AlignUp(ptr, STACK_ALIGN);
#endif
}
private:
char* Data_;
};
static inline void*& JmpBufReg(__myjmp_buf& buf, size_t n) noexcept {
return (((void**)(void*)(buf))[n]);
}
static inline void*& JmpBufStackReg(__myjmp_buf& buf) noexcept {
return JmpBufReg(buf, STACK_CNT);
}
static inline void*& JmpBufProgrReg(__myjmp_buf& buf) noexcept {
return JmpBufReg(buf, PROGR_CNT);
}
static inline void*& JmpBufFrameReg(__myjmp_buf& buf) noexcept {
return JmpBufReg(buf, FRAME_CNT);
}
#if defined(_x86_64_)
// not sure if Y_NO_SANITIZE is needed
Y_NO_SANITIZE("address")
Y_NO_SANITIZE("memory") extern "C" void ContextTrampoLine(void*, void*, void*, void*, void*, void*, // register arguments, no defined value
/* first argument passed through the stack */ void* t1,
/* second argument passed through the stack */ void* t2) {
Y_ASSERT(t1 == t2);
Run(t1);
}
#else
Y_NO_SANITIZE("address")
Y_NO_SANITIZE("memory") static void ContextTrampoLine() {
void** argPtr = (void**)((char*)AlignUp(&argPtr + EXTRA_PUSH_ARGS, STACK_ALIGN) + STACK_ALIGN);
Y_ASSERT(*(argPtr - 1) == *(argPtr - 2));
Run(*(argPtr - 1));
}
#endif
}
TContMachineContext::TSan::TSan() noexcept
: TL(nullptr)
{
}
TContMachineContext::TSan::TSan(const TContClosure& c) noexcept
: NSan::TFiberContext(c.Stack.data(), c.Stack.size(), c.ContName)
, TL(c.TrampoLine)
{
}
void TContMachineContext::TSan::DoRunNaked() {
AfterSwitch();
TL->DoRunNaked();
BeforeFinish();
}
TContMachineContext::TContMachineContext(const TContClosure& c)
#if defined(_asan_enabled_) || defined(_tsan_enabled_)
: San_(c)
#endif
{
TStackType stack(c.Stack);
/*
* arg, and align data
*/
#if defined(_asan_enabled_)
auto trampoline = &San_;
#else
auto trampoline = c.TrampoLine;
#endif
#if defined(_x86_64_)
stack.ReAlign();
// push twice to preserve alignment by 16
stack.Push(trampoline); // second stack argument
stack.Push(trampoline); // first stack argument
stack.Push(nullptr); // fake return address
#else
stack.Push(trampoline);
stack.Push(trampoline);
stack.ReAlign();
/*
* fake return address
*/
for (size_t i = 0; i < EXTRA_PUSH_ARGS; ++i) {
stack.Push(nullptr);
}
#endif
__mysetjmp(Buf_);
JmpBufProgrReg(Buf_) = reinterpret_cast<void*>(ContextTrampoLine);
JmpBufStackReg(Buf_) = stack.StackPtr();
JmpBufFrameReg(Buf_) = nullptr;
}
void TContMachineContext::SwitchTo(TContMachineContext* next) noexcept {
if (Y_LIKELY(__mysetjmp(Buf_) == 0)) {
#if defined(_asan_enabled_) || defined(_tsan_enabled_)
next->San_.BeforeSwitch(&San_);
#endif
__mylongjmp(next->Buf_, 1);
} else {
#if defined(_asan_enabled_)
San_.AfterSwitch();
#endif
}
}
#elif defined(_win_) && defined(_32_)
void __stdcall ContextTrampoLine(void* arg) {
Run(arg);
}
#else
void ContextTrampoLine(void* arg) {
Run(arg);
}
#endif
#if defined(USE_FIBER_CONT)
TContMachineContext::TContMachineContext()
: Fiber_(ConvertThreadToFiber(this))
, MainFiber_(true)
{
Y_ENSURE(Fiber_, TStringBuf("fiber error"));
}
TContMachineContext::TContMachineContext(const TContClosure& c)
: Fiber_(CreateFiber(c.Stack.size(), (LPFIBER_START_ROUTINE)ContextTrampoLine, (LPVOID)c.TrampoLine))
, MainFiber_(false)
{
Y_ENSURE(Fiber_, TStringBuf("fiber error"));
}
TContMachineContext::~TContMachineContext() {
if (MainFiber_) {
ConvertFiberToThread();
} else {
DeleteFiber(Fiber_);
}
}
void TContMachineContext::SwitchTo(TContMachineContext* next) noexcept {
SwitchToFiber(next->Fiber_);
}
#endif
#if defined(USE_GENERIC_CONT)
#include <pthread.h>
struct TContMachineContext::TImpl {
inline TImpl()
: TL(nullptr)
, Finish(false)
{
}
inline TImpl(const TContClosure& c)
: TL(c.TrampoLine)
, Finish(false)
{
Thread.Reset(new TThread(TThread::TParams(Run, this).SetStackSize(c.Stack.size()).SetStackPointer((void*)c.Stack.data())));
Thread->Start();
}
inline ~TImpl() {
if (Thread) {
Finish = true;
Signal();
Thread->Join();
}
}
inline void SwitchTo(TImpl* next) noexcept {
next->Signal();
Wait();
}
static void* Run(void* self) {
((TImpl*)self)->DoRun();
return nullptr;
}
inline void DoRun() {
Wait();
TL->DoRun();
}
inline void Signal() noexcept {
Event.Signal();
}
inline void Wait() noexcept {
Event.Wait();
if (Finish) {
// TODO - need proper TThread::Exit(), have some troubles in win32 now
pthread_exit(0);
}
}
TAutoEvent Event;
THolder<TThread> Thread;
ITrampoLine* TL;
bool Finish;
};
TContMachineContext::TContMachineContext()
: Impl_(new TImpl())
{
}
TContMachineContext::TContMachineContext(const TContClosure& c)
: Impl_(new TImpl(c))
{
}
TContMachineContext::~TContMachineContext() {
}
void TContMachineContext::SwitchTo(TContMachineContext* next) noexcept {
Impl_->SwitchTo(next->Impl_.Get());
}
#endif
void TExceptionSafeContext::SwitchTo(TExceptionSafeContext* to) noexcept {
#if defined(_unix_)
static_assert(sizeof(__cxxabiv1::__cxa_eh_globals) == sizeof(Buf_), "size mismatch of __cxa_eh_globals structure");
auto* eh = __cxxabiv1::__cxa_get_globals();
::memcpy(Buf_, eh, sizeof(Buf_));
::memcpy(eh, to->Buf_, sizeof(Buf_));
#endif
TContMachineContext::SwitchTo(to);
}