diff options
Diffstat (limited to 'contrib/tools/python3/Python/ceval_gil.c')
| -rw-r--r-- | contrib/tools/python3/Python/ceval_gil.c | 1009 |
1 files changed, 626 insertions, 383 deletions
diff --git a/contrib/tools/python3/Python/ceval_gil.c b/contrib/tools/python3/Python/ceval_gil.c index c1ab5883568..a1433e5b25f 100644 --- a/contrib/tools/python3/Python/ceval_gil.c +++ b/contrib/tools/python3/Python/ceval_gil.c @@ -1,12 +1,13 @@ #include "Python.h" -#include "pycore_atomic.h" // _Py_atomic_int #include "pycore_ceval.h" // _PyEval_SignalReceived() -#include "pycore_pyerrors.h" // _PyErr_GetRaisedException() -#include "pycore_pylifecycle.h" // _PyErr_Print() #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_interp.h" // _Py_RunGC() +#include "pycore_pyerrors.h" // _PyErr_GetRaisedException() +#include "pycore_pylifecycle.h" // _PyErr_Print() #include "pycore_pymem.h" // _PyMem_IsPtrFreed() +#include "pycore_pystats.h" // _Py_PrintSpecializationStats() +#include "pycore_pythread.h" // PyThread_hang_thread() /* Notes about the implementation: @@ -56,113 +57,54 @@ #define _Py_atomic_load_relaxed_int32(ATOMIC_VAL) _Py_atomic_load_relaxed(ATOMIC_VAL) #endif -/* This can set eval_breaker to 0 even though gil_drop_request became - 1. We believe this is all right because the eval loop will release - the GIL eventually anyway. */ -static inline void -COMPUTE_EVAL_BREAKER(PyInterpreterState *interp, - struct _ceval_runtime_state *ceval, - struct _ceval_state *ceval2) -{ - _Py_atomic_store_relaxed(&ceval2->eval_breaker, - _Py_atomic_load_relaxed_int32(&ceval2->gil_drop_request) - | (_Py_atomic_load_relaxed_int32(&ceval->signals_pending) - && _Py_ThreadCanHandleSignals(interp)) - | (_Py_atomic_load_relaxed_int32(&ceval2->pending.calls_to_do)) - | (_Py_IsMainThread() && _Py_IsMainInterpreter(interp) - &&_Py_atomic_load_relaxed_int32(&ceval->pending_mainthread.calls_to_do)) - | ceval2->pending.async_exc - | _Py_atomic_load_relaxed_int32(&ceval2->gc_scheduled)); -} - - +// Atomically copy the bits indicated by mask between two values. static inline void -SET_GIL_DROP_REQUEST(PyInterpreterState *interp) +copy_eval_breaker_bits(uintptr_t *from, uintptr_t *to, uintptr_t mask) { - struct _ceval_state *ceval2 = &interp->ceval; - _Py_atomic_store_relaxed(&ceval2->gil_drop_request, 1); - _Py_atomic_store_relaxed(&ceval2->eval_breaker, 1); -} - + uintptr_t from_bits = _Py_atomic_load_uintptr_relaxed(from) & mask; + uintptr_t old_value = _Py_atomic_load_uintptr_relaxed(to); + uintptr_t to_bits = old_value & mask; + if (from_bits == to_bits) { + return; + } -static inline void -RESET_GIL_DROP_REQUEST(PyInterpreterState *interp) -{ - struct _ceval_runtime_state *ceval = &interp->runtime->ceval; - struct _ceval_state *ceval2 = &interp->ceval; - _Py_atomic_store_relaxed(&ceval2->gil_drop_request, 0); - COMPUTE_EVAL_BREAKER(interp, ceval, ceval2); + uintptr_t new_value; + do { + new_value = (old_value & ~mask) | from_bits; + } while (!_Py_atomic_compare_exchange_uintptr(to, &old_value, new_value)); } - +// When attaching a thread, set the global instrumentation version and +// _PY_CALLS_TO_DO_BIT from the current state of the interpreter. static inline void -SIGNAL_PENDING_CALLS(struct _pending_calls *pending, PyInterpreterState *interp) +update_eval_breaker_for_thread(PyInterpreterState *interp, PyThreadState *tstate) { - struct _ceval_runtime_state *ceval = &interp->runtime->ceval; - struct _ceval_state *ceval2 = &interp->ceval; - _Py_atomic_store_relaxed(&pending->calls_to_do, 1); - COMPUTE_EVAL_BREAKER(interp, ceval, ceval2); -} - - -static inline void -UNSIGNAL_PENDING_CALLS(PyInterpreterState *interp) -{ - struct _ceval_runtime_state *ceval = &interp->runtime->ceval; - struct _ceval_state *ceval2 = &interp->ceval; - if (_Py_IsMainThread() && _Py_IsMainInterpreter(interp)) { - _Py_atomic_store_relaxed(&ceval->pending_mainthread.calls_to_do, 0); - } - _Py_atomic_store_relaxed(&ceval2->pending.calls_to_do, 0); - COMPUTE_EVAL_BREAKER(interp, ceval, ceval2); -} - +#ifdef Py_GIL_DISABLED + // Free-threaded builds eagerly update the eval_breaker on *all* threads as + // needed, so this function doesn't apply. + return; +#endif -static inline void -SIGNAL_PENDING_SIGNALS(PyInterpreterState *interp, int force) -{ - struct _ceval_runtime_state *ceval = &interp->runtime->ceval; - struct _ceval_state *ceval2 = &interp->ceval; - _Py_atomic_store_relaxed(&ceval->signals_pending, 1); - if (force) { - _Py_atomic_store_relaxed(&ceval2->eval_breaker, 1); + int32_t npending = _Py_atomic_load_int32_relaxed( + &interp->ceval.pending.npending); + if (npending) { + _Py_set_eval_breaker_bit(tstate, _PY_CALLS_TO_DO_BIT); } - else { - /* eval_breaker is not set to 1 if thread_can_handle_signals() is false */ - COMPUTE_EVAL_BREAKER(interp, ceval, ceval2); + else if (_Py_IsMainThread()) { + npending = _Py_atomic_load_int32_relaxed( + &_PyRuntime.ceval.pending_mainthread.npending); + if (npending) { + _Py_set_eval_breaker_bit(tstate, _PY_CALLS_TO_DO_BIT); + } } -} - -static inline void -UNSIGNAL_PENDING_SIGNALS(PyInterpreterState *interp) -{ - struct _ceval_runtime_state *ceval = &interp->runtime->ceval; - struct _ceval_state *ceval2 = &interp->ceval; - _Py_atomic_store_relaxed(&ceval->signals_pending, 0); - COMPUTE_EVAL_BREAKER(interp, ceval, ceval2); + // _PY_CALLS_TO_DO_BIT was derived from other state above, so the only bits + // we copy from our interpreter's state are the instrumentation version. + copy_eval_breaker_bits(&interp->ceval.instrumentation_version, + &tstate->eval_breaker, + ~_PY_EVAL_EVENTS_MASK); } - -static inline void -SIGNAL_ASYNC_EXC(PyInterpreterState *interp) -{ - struct _ceval_state *ceval2 = &interp->ceval; - ceval2->pending.async_exc = 1; - _Py_atomic_store_relaxed(&ceval2->eval_breaker, 1); -} - - -static inline void -UNSIGNAL_ASYNC_EXC(PyInterpreterState *interp) -{ - struct _ceval_runtime_state *ceval = &interp->runtime->ceval; - struct _ceval_state *ceval2 = &interp->ceval; - ceval2->pending.async_exc = 0; - COMPUTE_EVAL_BREAKER(interp, ceval, ceval2); -} - - /* * Implementation of the Global Interpreter Lock (GIL). */ @@ -170,9 +112,6 @@ UNSIGNAL_ASYNC_EXC(PyInterpreterState *interp) #include <stdlib.h> #include <errno.h> -#include "pycore_atomic.h" - - #include "condvar.h" #define MUTEX_INIT(mut) \ @@ -216,8 +155,7 @@ UNSIGNAL_ASYNC_EXC(PyInterpreterState *interp) static void _gil_initialize(struct _gil_runtime_state *gil) { - _Py_atomic_int uninitialized = {-1}; - gil->locked = uninitialized; + gil->locked = -1; gil->interval = DEFAULT_INTERVAL; } @@ -226,7 +164,7 @@ static int gil_created(struct _gil_runtime_state *gil) if (gil == NULL) { return 0; } - return (_Py_atomic_load_explicit(&gil->locked, _Py_memory_order_acquire) >= 0); + return (_Py_atomic_load_int_acquire(&gil->locked) >= 0); } static void create_gil(struct _gil_runtime_state *gil) @@ -239,9 +177,9 @@ static void create_gil(struct _gil_runtime_state *gil) #ifdef FORCE_SWITCHING COND_INIT(gil->switch_cond); #endif - _Py_atomic_store_relaxed(&gil->last_holder, 0); + _Py_atomic_store_ptr_relaxed(&gil->last_holder, 0); _Py_ANNOTATE_RWLOCK_CREATE(&gil->locked); - _Py_atomic_store_explicit(&gil->locked, 0, _Py_memory_order_release); + _Py_atomic_store_int_release(&gil->locked, 0); } static void destroy_gil(struct _gil_runtime_state *gil) @@ -255,8 +193,7 @@ static void destroy_gil(struct _gil_runtime_state *gil) COND_FINI(gil->switch_cond); MUTEX_FINI(gil->switch_mutex); #endif - _Py_atomic_store_explicit(&gil->locked, -1, - _Py_memory_order_release); + _Py_atomic_store_int_release(&gil->locked, -1); _Py_ANNOTATE_RWLOCK_DESTROY(&gil->locked); } @@ -269,53 +206,70 @@ static void recreate_gil(struct _gil_runtime_state *gil) } #endif +static inline void +drop_gil_impl(PyThreadState *tstate, struct _gil_runtime_state *gil) +{ + MUTEX_LOCK(gil->mutex); + _Py_ANNOTATE_RWLOCK_RELEASED(&gil->locked, /*is_write=*/1); + _Py_atomic_store_int_relaxed(&gil->locked, 0); + if (tstate != NULL) { + tstate->_status.holds_gil = 0; + } + COND_SIGNAL(gil->cond); + MUTEX_UNLOCK(gil->mutex); +} + static void -drop_gil(struct _ceval_state *ceval, PyThreadState *tstate) +drop_gil(PyInterpreterState *interp, PyThreadState *tstate, int final_release) { - /* If tstate is NULL, the caller is indicating that we're releasing + struct _ceval_state *ceval = &interp->ceval; + /* If final_release is true, the caller is indicating that we're releasing the GIL for the last time in this thread. This is particularly relevant when the current thread state is finalizing or its interpreter is finalizing (either may be in an inconsistent state). In that case the current thread will definitely never try to acquire the GIL again. */ // XXX It may be more correct to check tstate->_status.finalizing. - // XXX assert(tstate == NULL || !tstate->_status.cleared); + // XXX assert(final_release || !tstate->_status.cleared); + assert(final_release || tstate != NULL); struct _gil_runtime_state *gil = ceval->gil; - if (!_Py_atomic_load_relaxed(&gil->locked)) { +#ifdef Py_GIL_DISABLED + // Check if we have the GIL before dropping it. tstate will be NULL if + // take_gil() detected that this thread has been destroyed, in which case + // we know we have the GIL. + if (tstate != NULL && !tstate->_status.holds_gil) { + return; + } +#endif + if (!_Py_atomic_load_int_relaxed(&gil->locked)) { Py_FatalError("drop_gil: GIL is not locked"); } - /* tstate is allowed to be NULL (early interpreter init) */ - if (tstate != NULL) { + if (!final_release) { /* Sub-interpreter support: threads might have been switched under our feet using PyThreadState_Swap(). Fix the GIL last holder variable so that our heuristics work. */ - _Py_atomic_store_relaxed(&gil->last_holder, (uintptr_t)tstate); + _Py_atomic_store_ptr_relaxed(&gil->last_holder, tstate); } - MUTEX_LOCK(gil->mutex); - _Py_ANNOTATE_RWLOCK_RELEASED(&gil->locked, /*is_write=*/1); - _Py_atomic_store_relaxed(&gil->locked, 0); - COND_SIGNAL(gil->cond); - MUTEX_UNLOCK(gil->mutex); + drop_gil_impl(tstate, gil); #ifdef FORCE_SWITCHING - /* We check tstate first in case we might be releasing the GIL for - the last time in this thread. In that case there's a possible - race with tstate->interp getting deleted after gil->mutex is - unlocked and before the following code runs, leading to a crash. - We can use (tstate == NULL) to indicate the thread is done with - the GIL, and that's the only time we might delete the - interpreter, so checking tstate first prevents the crash. - See https://github.com/python/cpython/issues/104341. */ - if (tstate != NULL && _Py_atomic_load_relaxed(&ceval->gil_drop_request)) { + /* We might be releasing the GIL for the last time in this thread. In that + case there's a possible race with tstate->interp getting deleted after + gil->mutex is unlocked and before the following code runs, leading to a + crash. We can use final_release to indicate the thread is done with the + GIL, and that's the only time we might delete the interpreter. See + https://github.com/python/cpython/issues/104341. */ + if (!final_release && + _Py_eval_breaker_bit_is_set(tstate, _PY_GIL_DROP_REQUEST_BIT)) { MUTEX_LOCK(gil->switch_mutex); /* Not switched yet => wait */ - if (((PyThreadState*)_Py_atomic_load_relaxed(&gil->last_holder)) == tstate) + if (((PyThreadState*)_Py_atomic_load_ptr_relaxed(&gil->last_holder)) == tstate) { assert(_PyThreadState_CheckConsistency(tstate)); - RESET_GIL_DROP_REQUEST(tstate->interp); + _Py_unset_eval_breaker_bit(tstate, _PY_GIL_DROP_REQUEST_BIT); /* NOTE: if COND_WAIT does not atomically start waiting when releasing the mutex, another thread can run through, take the GIL and drop it again, and reset the condition @@ -331,6 +285,7 @@ drop_gil(struct _ceval_state *ceval, PyThreadState *tstate) /* Take the GIL. The function saves errno at entry and restores its value at exit. + It may hang rather than return if the interpreter has been finalized. tstate must be non-NULL. */ static void @@ -345,30 +300,36 @@ take_gil(PyThreadState *tstate) if (_PyThreadState_MustExit(tstate)) { /* bpo-39877: If Py_Finalize() has been called and tstate is not the - thread which called Py_Finalize(), exit immediately the thread. + thread which called Py_Finalize(), this thread cannot continue. This code path can be reached by a daemon thread after Py_Finalize() completes. In this case, tstate is a dangling pointer: points to - PyThreadState freed memory. */ - PyThread_exit_thread(); + PyThreadState freed memory. + + This used to call a *thread_exit API, but that was not safe as it + lacks stack unwinding and local variable destruction important to + C++. gh-87135: The best that can be done is to hang the thread as + the public APIs calling this have no error reporting mechanism (!). + */ + PyThread_hang_thread(); } assert(_PyThreadState_CheckConsistency(tstate)); PyInterpreterState *interp = tstate->interp; - struct _ceval_state *ceval = &interp->ceval; - struct _gil_runtime_state *gil = ceval->gil; + struct _gil_runtime_state *gil = interp->ceval.gil; +#ifdef Py_GIL_DISABLED + if (!_Py_atomic_load_int_relaxed(&gil->enabled)) { + return; + } +#endif /* Check that _PyEval_InitThreads() was called to create the lock */ assert(gil_created(gil)); MUTEX_LOCK(gil->mutex); - if (!_Py_atomic_load_relaxed(&gil->locked)) { - goto _ready; - } - int drop_requested = 0; - while (_Py_atomic_load_relaxed(&gil->locked)) { + while (_Py_atomic_load_int_relaxed(&gil->locked)) { unsigned long saved_switchnum = gil->switch_number; unsigned long interval = (gil->interval >= 1 ? gil->interval : 1); @@ -378,9 +339,11 @@ take_gil(PyThreadState *tstate) /* If we timed out and no switch occurred in the meantime, it is time to ask the GIL-holding thread to drop it. */ if (timed_out && - _Py_atomic_load_relaxed(&gil->locked) && + _Py_atomic_load_int_relaxed(&gil->locked) && gil->switch_number == saved_switchnum) { + PyThreadState *holder_tstate = + (PyThreadState*)_Py_atomic_load_ptr_relaxed(&gil->last_holder); if (_PyThreadState_MustExit(tstate)) { MUTEX_UNLOCK(gil->mutex); // gh-96387: If the loop requested a drop request in a previous @@ -390,29 +353,41 @@ take_gil(PyThreadState *tstate) // may have to request again a drop request (iterate one more // time). if (drop_requested) { - RESET_GIL_DROP_REQUEST(interp); + _Py_unset_eval_breaker_bit(holder_tstate, _PY_GIL_DROP_REQUEST_BIT); } - PyThread_exit_thread(); + // gh-87135: hang the thread as *thread_exit() is not a safe + // API. It lacks stack unwind and local variable destruction. + PyThread_hang_thread(); } assert(_PyThreadState_CheckConsistency(tstate)); - SET_GIL_DROP_REQUEST(interp); + _Py_set_eval_breaker_bit(holder_tstate, _PY_GIL_DROP_REQUEST_BIT); drop_requested = 1; } } -_ready: +#ifdef Py_GIL_DISABLED + if (!_Py_atomic_load_int_relaxed(&gil->enabled)) { + // Another thread disabled the GIL between our check above and + // now. Don't take the GIL, signal any other waiting threads, and + // return. + COND_SIGNAL(gil->cond); + MUTEX_UNLOCK(gil->mutex); + return; + } +#endif + #ifdef FORCE_SWITCHING /* This mutex must be taken before modifying gil->last_holder: see drop_gil(). */ MUTEX_LOCK(gil->switch_mutex); #endif /* We now hold the GIL */ - _Py_atomic_store_relaxed(&gil->locked, 1); + _Py_atomic_store_int_relaxed(&gil->locked, 1); _Py_ANNOTATE_RWLOCK_ACQUIRED(&gil->locked, /*is_write=*/1); - if (tstate != (PyThreadState*)_Py_atomic_load_relaxed(&gil->last_holder)) { - _Py_atomic_store_relaxed(&gil->last_holder, (uintptr_t)tstate); + if (tstate != (PyThreadState*)_Py_atomic_load_ptr_relaxed(&gil->last_holder)) { + _Py_atomic_store_ptr_relaxed(&gil->last_holder, tstate); ++gil->switch_number; } @@ -423,43 +398,33 @@ _ready: if (_PyThreadState_MustExit(tstate)) { /* bpo-36475: If Py_Finalize() has been called and tstate is not - the thread which called Py_Finalize(), exit immediately the + the thread which called Py_Finalize(), gh-87135: hang the thread. This code path can be reached by a daemon thread which was waiting in take_gil() while the main thread called wait_for_thread_shutdown() from Py_Finalize(). */ MUTEX_UNLOCK(gil->mutex); - drop_gil(ceval, tstate); - PyThread_exit_thread(); + /* tstate could be a dangling pointer, so don't pass it to + drop_gil(). */ + drop_gil(interp, NULL, 1); + PyThread_hang_thread(); } assert(_PyThreadState_CheckConsistency(tstate)); - if (_Py_atomic_load_relaxed(&ceval->gil_drop_request)) { - RESET_GIL_DROP_REQUEST(interp); - } - else { - /* bpo-40010: eval_breaker should be recomputed to be set to 1 if there - is a pending signal: signal received by another thread which cannot - handle signals. - - Note: RESET_GIL_DROP_REQUEST() calls COMPUTE_EVAL_BREAKER(). */ - COMPUTE_EVAL_BREAKER(interp, &_PyRuntime.ceval, ceval); - } - - /* Don't access tstate if the thread must exit */ - if (tstate->async_exc != NULL) { - _PyEval_SignalAsyncExc(tstate->interp); - } + tstate->_status.holds_gil = 1; + _Py_unset_eval_breaker_bit(tstate, _PY_GIL_DROP_REQUEST_BIT); + update_eval_breaker_for_thread(interp, tstate); MUTEX_UNLOCK(gil->mutex); errno = err; + return; } void _PyEval_SetSwitchInterval(unsigned long microseconds) { - PyInterpreterState *interp = _PyInterpreterState_Get(); + PyInterpreterState *interp = _PyInterpreterState_GET(); struct _gil_runtime_state *gil = interp->ceval.gil; assert(gil != NULL); gil->interval = microseconds; @@ -467,7 +432,7 @@ void _PyEval_SetSwitchInterval(unsigned long microseconds) unsigned long _PyEval_GetSwitchInterval(void) { - PyInterpreterState *interp = _PyInterpreterState_Get(); + PyInterpreterState *interp = _PyInterpreterState_GET(); struct _gil_runtime_state *gil = interp->ceval.gil; assert(gil != NULL); return gil->interval; @@ -488,20 +453,30 @@ _PyEval_ThreadsInitialized(void) return gil_created(gil); } -int +// Function removed in the Python 3.13 API but kept in the stable ABI. +PyAPI_FUNC(int) PyEval_ThreadsInitialized(void) { return _PyEval_ThreadsInitialized(); } +#ifndef NDEBUG static inline int current_thread_holds_gil(struct _gil_runtime_state *gil, PyThreadState *tstate) { - if (((PyThreadState*)_Py_atomic_load_relaxed(&gil->last_holder)) != tstate) { - return 0; - } - return _Py_atomic_load_relaxed(&gil->locked); + int holds_gil = tstate->_status.holds_gil; + + // holds_gil is the source of truth; check that last_holder and gil->locked + // are consistent with it. + int locked = _Py_atomic_load_int_relaxed(&gil->locked); + int is_last_holder = + ((PyThreadState*)_Py_atomic_load_ptr_relaxed(&gil->last_holder)) == tstate; + assert(!holds_gil || locked); + assert(!holds_gil || is_last_holder); + + return holds_gil; } +#endif static void init_shared_gil(PyInterpreterState *interp, struct _gil_runtime_state *gil) @@ -515,35 +490,35 @@ static void init_own_gil(PyInterpreterState *interp, struct _gil_runtime_state *gil) { assert(!gil_created(gil)); +#ifdef Py_GIL_DISABLED + const PyConfig *config = _PyInterpreterState_GetConfig(interp); + gil->enabled = config->enable_gil == _PyConfig_GIL_ENABLE ? INT_MAX : 0; +#endif create_gil(gil); assert(gil_created(gil)); interp->ceval.gil = gil; interp->ceval.own_gil = 1; } -PyStatus +void _PyEval_InitGIL(PyThreadState *tstate, int own_gil) { assert(tstate->interp->ceval.gil == NULL); - int locked; if (!own_gil) { /* The interpreter will share the main interpreter's instead. */ PyInterpreterState *main_interp = _PyInterpreterState_Main(); assert(tstate->interp != main_interp); struct _gil_runtime_state *gil = main_interp->ceval.gil; init_shared_gil(tstate->interp, gil); - locked = current_thread_holds_gil(gil, tstate); + assert(!current_thread_holds_gil(gil, tstate)); } else { PyThread_init_thread(); init_own_gil(tstate->interp, &tstate->interp->_gil); - locked = 0; - } - if (!locked) { - take_gil(tstate); } - return _PyStatus_OK(); + // Lock the GIL and mark the current thread as attached. + _PyThreadState_Attach(tstate); } void @@ -589,7 +564,9 @@ _PyEval_Fini(void) _Py_PrintSpecializationStats(1); #endif } -void + +// Function removed in the Python 3.13 API but kept in the stable ABI. +PyAPI_FUNC(void) PyEval_AcquireLock(void) { PyThreadState *tstate = _PyThreadState_GET(); @@ -598,15 +575,15 @@ PyEval_AcquireLock(void) take_gil(tstate); } -void +// Function removed in the Python 3.13 API but kept in the stable ABI. +PyAPI_FUNC(void) PyEval_ReleaseLock(void) { PyThreadState *tstate = _PyThreadState_GET(); /* This function must succeed when the current thread state is NULL. We therefore avoid PyThreadState_Get() which dumps a fatal error in debug mode. */ - struct _ceval_state *ceval = &tstate->interp->ceval; - drop_gil(ceval, tstate); + drop_gil(tstate->interp, tstate, 0); } void @@ -617,44 +594,32 @@ _PyEval_AcquireLock(PyThreadState *tstate) } void -_PyEval_ReleaseLock(PyInterpreterState *interp, PyThreadState *tstate) +_PyEval_ReleaseLock(PyInterpreterState *interp, + PyThreadState *tstate, + int final_release) { - /* If tstate is NULL then we do not expect the current thread - to acquire the GIL ever again. */ - assert(tstate == NULL || tstate->interp == interp); - struct _ceval_state *ceval = &interp->ceval; - drop_gil(ceval, tstate); + assert(tstate != NULL); + assert(tstate->interp == interp); + drop_gil(interp, tstate, final_release); } void PyEval_AcquireThread(PyThreadState *tstate) { _Py_EnsureTstateNotNULL(tstate); - - take_gil(tstate); - - if (_PyThreadState_SwapNoGIL(tstate) != NULL) { - Py_FatalError("non-NULL old thread state"); - } + _PyThreadState_Attach(tstate); } void PyEval_ReleaseThread(PyThreadState *tstate) { assert(_PyThreadState_CheckConsistency(tstate)); - - PyThreadState *new_tstate = _PyThreadState_SwapNoGIL(NULL); - if (new_tstate != tstate) { - Py_FatalError("wrong thread state"); - } - struct _ceval_state *ceval = &tstate->interp->ceval; - drop_gil(ceval, tstate); + _PyThreadState_Detach(tstate); } #ifdef HAVE_FORK -/* This function is called from PyOS_AfterFork_Child to destroy all threads - which are not running in the child process, and clear internal locks - which might be held by those threads. */ +/* This function is called from PyOS_AfterFork_Child to re-initialize the + GIL and pending calls lock. */ PyStatus _PyEval_ReInitThreads(PyThreadState *tstate) { @@ -669,46 +634,62 @@ _PyEval_ReInitThreads(PyThreadState *tstate) take_gil(tstate); struct _pending_calls *pending = &tstate->interp->ceval.pending; - if (_PyThread_at_fork_reinit(&pending->lock) < 0) { - return _PyStatus_ERR("Can't reinitialize pending calls lock"); - } + _PyMutex_at_fork_reinit(&pending->mutex); - /* Destroy all threads except the current one */ - _PyThreadState_DeleteExcept(tstate); return _PyStatus_OK(); } #endif -/* This function is used to signal that async exceptions are waiting to be - raised. */ - -void -_PyEval_SignalAsyncExc(PyInterpreterState *interp) -{ - SIGNAL_ASYNC_EXC(interp); -} - PyThreadState * PyEval_SaveThread(void) { - PyThreadState *tstate = _PyThreadState_SwapNoGIL(NULL); - _Py_EnsureTstateNotNULL(tstate); - - struct _ceval_state *ceval = &tstate->interp->ceval; - assert(gil_created(ceval->gil)); - drop_gil(ceval, tstate); + PyThreadState *tstate = _PyThreadState_GET(); + _PyThreadState_Detach(tstate); return tstate; } void PyEval_RestoreThread(PyThreadState *tstate) { +#ifdef MS_WINDOWS + int err = GetLastError(); +#endif + _Py_EnsureTstateNotNULL(tstate); + _PyThreadState_Attach(tstate); + +#ifdef MS_WINDOWS + SetLastError(err); +#endif +} + + +void +_PyEval_SignalReceived(void) +{ + _Py_set_eval_breaker_bit(_PyRuntime.main_tstate, _PY_SIGNALS_PENDING_BIT); +} - take_gil(tstate); - _PyThreadState_SwapNoGIL(tstate); +#ifndef Py_GIL_DISABLED +static void +signal_active_thread(PyInterpreterState *interp, uintptr_t bit) +{ + struct _gil_runtime_state *gil = interp->ceval.gil; + + // If a thread from the targeted interpreter is holding the GIL, signal + // that thread. Otherwise, the next thread to run from the targeted + // interpreter will have its bit set as part of taking the GIL. + MUTEX_LOCK(gil->mutex); + if (_Py_atomic_load_int_relaxed(&gil->locked)) { + PyThreadState *holder = (PyThreadState*)_Py_atomic_load_ptr_relaxed(&gil->last_holder); + if (holder->interp == interp) { + _Py_set_eval_breaker_bit(holder, bit); + } + } + MUTEX_UNLOCK(gil->mutex); } +#endif /* Mechanism whereby asynchronously executing callbacks (e.g. UNIX @@ -733,67 +714,61 @@ PyEval_RestoreThread(PyThreadState *tstate) threadstate. */ -void -_PyEval_SignalReceived(PyInterpreterState *interp) -{ -#ifdef MS_WINDOWS - // bpo-42296: On Windows, _PyEval_SignalReceived() is called from a signal - // handler which can run in a thread different than the Python thread, in - // which case _Py_ThreadCanHandleSignals() is wrong. Ignore - // _Py_ThreadCanHandleSignals() and always set eval_breaker to 1. - // - // The next eval_frame_handle_pending() call will call - // _Py_ThreadCanHandleSignals() to recompute eval_breaker. - int force = 1; -#else - int force = 0; -#endif - /* bpo-30703: Function called when the C signal handler of Python gets a - signal. We cannot queue a callback using _PyEval_AddPendingCall() since - that function is not async-signal-safe. */ - SIGNAL_PENDING_SIGNALS(interp, force); -} - /* Push one item onto the queue while holding the lock. */ static int _push_pending_call(struct _pending_calls *pending, - int (*func)(void *), void *arg) + _Py_pending_call_func func, void *arg, int flags) { - int i = pending->last; - int j = (i + 1) % NPENDINGCALLS; - if (j == pending->first) { - return -1; /* Queue full */ + if (pending->npending == pending->max) { + return _Py_ADD_PENDING_FULL; } + assert(pending->npending < pending->max); + + int i = pending->next; + assert(pending->calls[i].func == NULL); + pending->calls[i].func = func; pending->calls[i].arg = arg; - pending->last = j; - return 0; + pending->calls[i].flags = flags; + + assert(pending->npending < PENDINGCALLSARRAYSIZE); + _Py_atomic_add_int32(&pending->npending, 1); + + pending->next = (i + 1) % PENDINGCALLSARRAYSIZE; + assert(pending->next != pending->first + || pending->npending == pending->max); + + return _Py_ADD_PENDING_SUCCESS; } static int _next_pending_call(struct _pending_calls *pending, - int (**func)(void *), void **arg) + int (**func)(void *), void **arg, int *flags) { int i = pending->first; - if (i == pending->last) { + if (pending->npending == 0) { /* Queue empty */ + assert(i == pending->next); assert(pending->calls[i].func == NULL); return -1; } *func = pending->calls[i].func; *arg = pending->calls[i].arg; + *flags = pending->calls[i].flags; return i; } /* Pop one item off the queue while holding the lock. */ static void _pop_pending_call(struct _pending_calls *pending, - int (**func)(void *), void **arg) + int (**func)(void *), void **arg, int *flags) { - int i = _next_pending_call(pending, func, arg); + int i = _next_pending_call(pending, func, arg, flags); if (i >= 0) { pending->calls[i] = (struct _pending_call){0}; - pending->first = (i + 1) % NPENDINGCALLS; + pending->first = (i + 1) % PENDINGCALLSARRAYSIZE; + assert(pending->npending > 0); + _Py_atomic_add_int32(&pending->npending, -1); } } @@ -802,157 +777,268 @@ _pop_pending_call(struct _pending_calls *pending, callback. */ -int +_Py_add_pending_call_result _PyEval_AddPendingCall(PyInterpreterState *interp, - int (*func)(void *), void *arg, - int mainthreadonly) + _Py_pending_call_func func, void *arg, int flags) { - assert(!mainthreadonly || _Py_IsMainInterpreter(interp)); struct _pending_calls *pending = &interp->ceval.pending; - if (mainthreadonly) { + int main_only = (flags & _Py_PENDING_MAINTHREADONLY) != 0; + if (main_only) { /* The main thread only exists in the main interpreter. */ assert(_Py_IsMainInterpreter(interp)); pending = &_PyRuntime.ceval.pending_mainthread; } - /* Ensure that _PyEval_InitState() was called - and that _PyEval_FiniState() is not called yet. */ - assert(pending->lock != NULL); - PyThread_acquire_lock(pending->lock, WAIT_LOCK); - int result = _push_pending_call(pending, func, arg); - PyThread_release_lock(pending->lock); + PyMutex_Lock(&pending->mutex); + _Py_add_pending_call_result result = + _push_pending_call(pending, func, arg, flags); + PyMutex_Unlock(&pending->mutex); + + if (main_only) { + _Py_set_eval_breaker_bit(_PyRuntime.main_tstate, _PY_CALLS_TO_DO_BIT); + } + else { +#ifdef Py_GIL_DISABLED + _Py_set_eval_breaker_bit_all(interp, _PY_CALLS_TO_DO_BIT); +#else + signal_active_thread(interp, _PY_CALLS_TO_DO_BIT); +#endif + } - /* signal main loop */ - SIGNAL_PENDING_CALLS(pending, interp); return result; } int -Py_AddPendingCall(int (*func)(void *), void *arg) +Py_AddPendingCall(_Py_pending_call_func func, void *arg) { /* Legacy users of this API will continue to target the main thread (of the main interpreter). */ PyInterpreterState *interp = _PyInterpreterState_Main(); - return _PyEval_AddPendingCall(interp, func, arg, 1); + _Py_add_pending_call_result r = + _PyEval_AddPendingCall(interp, func, arg, _Py_PENDING_MAINTHREADONLY); + if (r == _Py_ADD_PENDING_FULL) { + return -1; + } + else { + assert(r == _Py_ADD_PENDING_SUCCESS); + return 0; + } } static int handle_signals(PyThreadState *tstate) { assert(_PyThreadState_CheckConsistency(tstate)); + _Py_unset_eval_breaker_bit(tstate, _PY_SIGNALS_PENDING_BIT); if (!_Py_ThreadCanHandleSignals(tstate->interp)) { return 0; } - - UNSIGNAL_PENDING_SIGNALS(tstate->interp); if (_PyErr_CheckSignalsTstate(tstate) < 0) { /* On failure, re-schedule a call to handle_signals(). */ - SIGNAL_PENDING_SIGNALS(tstate->interp, 0); + _Py_set_eval_breaker_bit(tstate, _PY_SIGNALS_PENDING_BIT); return -1; } return 0; } -static inline int -maybe_has_pending_calls(PyInterpreterState *interp) +static int +_make_pending_calls(struct _pending_calls *pending, int32_t *p_npending) { - struct _pending_calls *pending = &interp->ceval.pending; - if (_Py_atomic_load_relaxed_int32(&pending->calls_to_do)) { - return 1; - } - if (!_Py_IsMainThread() || !_Py_IsMainInterpreter(interp)) { - return 0; + int res = 0; + int32_t npending = -1; + + assert(sizeof(pending->max) <= sizeof(size_t) + && ((size_t)pending->max) <= Py_ARRAY_LENGTH(pending->calls)); + int32_t maxloop = pending->maxloop; + if (maxloop == 0) { + maxloop = pending->max; } - pending = &_PyRuntime.ceval.pending_mainthread; - return _Py_atomic_load_relaxed_int32(&pending->calls_to_do); -} + assert(maxloop > 0 && maxloop <= pending->max); -static int -_make_pending_calls(struct _pending_calls *pending) -{ /* perform a bounded number of calls, in case of recursion */ - for (int i=0; i<NPENDINGCALLS; i++) { - int (*func)(void *) = NULL; + for (int i=0; i<maxloop; i++) { + _Py_pending_call_func func = NULL; void *arg = NULL; + int flags = 0; /* pop one item off the queue while holding the lock */ - PyThread_acquire_lock(pending->lock, WAIT_LOCK); - _pop_pending_call(pending, &func, &arg); - PyThread_release_lock(pending->lock); + PyMutex_Lock(&pending->mutex); + _pop_pending_call(pending, &func, &arg, &flags); + npending = pending->npending; + PyMutex_Unlock(&pending->mutex); - /* having released the lock, perform the callback */ + /* Check if there are any more pending calls. */ if (func == NULL) { + assert(npending == 0); break; } - if (func(arg) != 0) { - return -1; + + /* having released the lock, perform the callback */ + res = func(arg); + if ((flags & _Py_PENDING_RAWFREE) && arg != NULL) { + PyMem_RawFree(arg); + } + if (res != 0) { + res = -1; + goto finally; } } - return 0; + +finally: + *p_npending = npending; + return res; +} + +static void +signal_pending_calls(PyThreadState *tstate, PyInterpreterState *interp) +{ +#ifdef Py_GIL_DISABLED + _Py_set_eval_breaker_bit_all(interp, _PY_CALLS_TO_DO_BIT); +#else + _Py_set_eval_breaker_bit(tstate, _PY_CALLS_TO_DO_BIT); +#endif +} + +static void +unsignal_pending_calls(PyThreadState *tstate, PyInterpreterState *interp) +{ +#ifdef Py_GIL_DISABLED + _Py_unset_eval_breaker_bit_all(interp, _PY_CALLS_TO_DO_BIT); +#else + _Py_unset_eval_breaker_bit(tstate, _PY_CALLS_TO_DO_BIT); +#endif +} + +static void +clear_pending_handling_thread(struct _pending_calls *pending) +{ +#ifdef Py_GIL_DISABLED + PyMutex_Lock(&pending->mutex); + pending->handling_thread = NULL; + PyMutex_Unlock(&pending->mutex); +#else + pending->handling_thread = NULL; +#endif } static int -make_pending_calls(PyInterpreterState *interp) +make_pending_calls(PyThreadState *tstate) { + PyInterpreterState *interp = tstate->interp; struct _pending_calls *pending = &interp->ceval.pending; struct _pending_calls *pending_main = &_PyRuntime.ceval.pending_mainthread; /* Only one thread (per interpreter) may run the pending calls at once. In the same way, we don't do recursive pending calls. */ - PyThread_acquire_lock(pending->lock, WAIT_LOCK); - if (pending->busy) { + PyMutex_Lock(&pending->mutex); + if (pending->handling_thread != NULL) { /* A pending call was added after another thread was already handling the pending calls (and had already "unsignaled"). Once that thread is done, it may have taken care of all the pending calls, or there might be some still waiting. - Regardless, this interpreter's pending calls will stay - "signaled" until that first thread has finished. At that - point the next thread to trip the eval breaker will take - care of any remaining pending calls. Until then, though, - all the interpreter's threads will be tripping the eval - breaker every time it's checked. */ - PyThread_release_lock(pending->lock); + To avoid all threads constantly stopping on the eval breaker, + we clear the bit for this thread and make sure it is set + for the thread currently handling the pending call. */ + _Py_set_eval_breaker_bit(pending->handling_thread, _PY_CALLS_TO_DO_BIT); + _Py_unset_eval_breaker_bit(tstate, _PY_CALLS_TO_DO_BIT); + PyMutex_Unlock(&pending->mutex); return 0; } - pending->busy = 1; - PyThread_release_lock(pending->lock); + pending->handling_thread = tstate; + PyMutex_Unlock(&pending->mutex); /* unsignal before starting to call callbacks, so that any callback added in-between re-signals */ - UNSIGNAL_PENDING_CALLS(interp); + unsignal_pending_calls(tstate, interp); - if (_make_pending_calls(pending) != 0) { - pending->busy = 0; + int32_t npending; + if (_make_pending_calls(pending, &npending) != 0) { + clear_pending_handling_thread(pending); /* There might not be more calls to make, but we play it safe. */ - SIGNAL_PENDING_CALLS(pending, interp); + signal_pending_calls(tstate, interp); return -1; } + if (npending > 0) { + /* We hit pending->maxloop. */ + signal_pending_calls(tstate, interp); + } if (_Py_IsMainThread() && _Py_IsMainInterpreter(interp)) { - if (_make_pending_calls(pending_main) != 0) { - pending->busy = 0; + if (_make_pending_calls(pending_main, &npending) != 0) { + clear_pending_handling_thread(pending); /* There might not be more calls to make, but we play it safe. */ - SIGNAL_PENDING_CALLS(pending_main, interp); + signal_pending_calls(tstate, interp); return -1; } + if (npending > 0) { + /* We hit pending_main->maxloop. */ + signal_pending_calls(tstate, interp); + } } - pending->busy = 0; + clear_pending_handling_thread(pending); return 0; } + +void +_Py_set_eval_breaker_bit_all(PyInterpreterState *interp, uintptr_t bit) +{ + _PyRuntimeState *runtime = &_PyRuntime; + + HEAD_LOCK(runtime); + for (PyThreadState *tstate = interp->threads.head; tstate != NULL; tstate = tstate->next) { + _Py_set_eval_breaker_bit(tstate, bit); + } + HEAD_UNLOCK(runtime); +} + +void +_Py_unset_eval_breaker_bit_all(PyInterpreterState *interp, uintptr_t bit) +{ + _PyRuntimeState *runtime = &_PyRuntime; + + HEAD_LOCK(runtime); + for (PyThreadState *tstate = interp->threads.head; tstate != NULL; tstate = tstate->next) { + _Py_unset_eval_breaker_bit(tstate, bit); + } + HEAD_UNLOCK(runtime); +} + void _Py_FinishPendingCalls(PyThreadState *tstate) { assert(PyGILState_Check()); assert(_PyThreadState_CheckConsistency(tstate)); - if (make_pending_calls(tstate->interp) < 0) { - PyObject *exc = _PyErr_GetRaisedException(tstate); - PyErr_BadInternalCall(); - _PyErr_ChainExceptions1(exc); - _PyErr_Print(tstate); - } + struct _pending_calls *pending = &tstate->interp->ceval.pending; + struct _pending_calls *pending_main = + _Py_IsMainThread() && _Py_IsMainInterpreter(tstate->interp) + ? &_PyRuntime.ceval.pending_mainthread + : NULL; + /* make_pending_calls() may return early without making all pending + calls, so we keep trying until we're actually done. */ + int32_t npending; +#ifndef NDEBUG + int32_t npending_prev = INT32_MAX; +#endif + do { + if (make_pending_calls(tstate) < 0) { + PyObject *exc = _PyErr_GetRaisedException(tstate); + PyErr_BadInternalCall(); + _PyErr_ChainExceptions1(exc); + _PyErr_Print(tstate); + } + + npending = _Py_atomic_load_int32_relaxed(&pending->npending); + if (pending_main != NULL) { + npending += _Py_atomic_load_int32_relaxed(&pending_main->npending); + } +#ifndef NDEBUG + assert(npending_prev > npending); + npending_prev = npending; +#endif + } while (npending > 0); } int @@ -970,7 +1056,7 @@ _PyEval_MakePendingCalls(PyThreadState *tstate) } } - res = make_pending_calls(tstate->interp); + res = make_pending_calls(tstate); if (res != 0) { return res; } @@ -996,94 +1082,251 @@ Py_MakePendingCalls(void) } void -_PyEval_InitState(PyInterpreterState *interp, PyThread_type_lock pending_lock) +_PyEval_InitState(PyInterpreterState *interp) { _gil_initialize(&interp->_gil); +} - struct _pending_calls *pending = &interp->ceval.pending; - assert(pending->lock == NULL); - pending->lock = pending_lock; +#ifdef Py_GIL_DISABLED +int +_PyEval_EnableGILTransient(PyThreadState *tstate) +{ + const PyConfig *config = _PyInterpreterState_GetConfig(tstate->interp); + if (config->enable_gil != _PyConfig_GIL_DEFAULT) { + return 0; + } + struct _gil_runtime_state *gil = tstate->interp->ceval.gil; + + int enabled = _Py_atomic_load_int_relaxed(&gil->enabled); + if (enabled == INT_MAX) { + // The GIL is already enabled permanently. + return 0; + } + if (enabled == INT_MAX - 1) { + Py_FatalError("Too many transient requests to enable the GIL"); + } + if (enabled > 0) { + // If enabled is nonzero, we know we hold the GIL. This means that no + // other threads are attached, and nobody else can be concurrently + // mutating it. + _Py_atomic_store_int_relaxed(&gil->enabled, enabled + 1); + return 0; + } + + // Enabling the GIL changes what it means to be an "attached" thread. To + // safely make this transition, we: + // 1. Detach the current thread. + // 2. Stop the world to detach (and suspend) all other threads. + // 3. Enable the GIL, if nobody else did between our check above and when + // our stop-the-world begins. + // 4. Start the world. + // 5. Attach the current thread. Other threads may attach and hold the GIL + // before this thread, which is harmless. + _PyThreadState_Detach(tstate); + + // This could be an interpreter-local stop-the-world in situations where we + // know that this interpreter's GIL is not shared, and that it won't become + // shared before the stop-the-world begins. For now, we always stop all + // interpreters for simplicity. + _PyEval_StopTheWorldAll(&_PyRuntime); + + enabled = _Py_atomic_load_int_relaxed(&gil->enabled); + int this_thread_enabled = enabled == 0; + _Py_atomic_store_int_relaxed(&gil->enabled, enabled + 1); + + _PyEval_StartTheWorldAll(&_PyRuntime); + _PyThreadState_Attach(tstate); + + return this_thread_enabled; } -void -_PyEval_FiniState(struct _ceval_state *ceval) +int +_PyEval_EnableGILPermanent(PyThreadState *tstate) +{ + const PyConfig *config = _PyInterpreterState_GetConfig(tstate->interp); + if (config->enable_gil != _PyConfig_GIL_DEFAULT) { + return 0; + } + + struct _gil_runtime_state *gil = tstate->interp->ceval.gil; + assert(current_thread_holds_gil(gil, tstate)); + + int enabled = _Py_atomic_load_int_relaxed(&gil->enabled); + if (enabled == INT_MAX) { + return 0; + } + + _Py_atomic_store_int_relaxed(&gil->enabled, INT_MAX); + return 1; +} + +int +_PyEval_DisableGIL(PyThreadState *tstate) { - struct _pending_calls *pending = &ceval->pending; - if (pending->lock != NULL) { - PyThread_free_lock(pending->lock); - pending->lock = NULL; + const PyConfig *config = _PyInterpreterState_GetConfig(tstate->interp); + if (config->enable_gil != _PyConfig_GIL_DEFAULT) { + return 0; + } + + struct _gil_runtime_state *gil = tstate->interp->ceval.gil; + assert(current_thread_holds_gil(gil, tstate)); + + int enabled = _Py_atomic_load_int_relaxed(&gil->enabled); + if (enabled == INT_MAX) { + return 0; } + + assert(enabled >= 1); + enabled--; + + // Disabling the GIL is much simpler than enabling it, since we know we are + // the only attached thread. Other threads may start free-threading as soon + // as this store is complete, if it sets gil->enabled to 0. + _Py_atomic_store_int_relaxed(&gil->enabled, enabled); + + if (enabled == 0) { + // We're attached, so we know the GIL will remain disabled until at + // least the next time we detach, which must be after this function + // returns. + // + // Drop the GIL, which will wake up any threads waiting in take_gil() + // and let them resume execution without the GIL. + drop_gil_impl(tstate, gil); + + // If another thread asked us to drop the GIL, they should be + // free-threading by now. Remove any such request so we have a clean + // slate if/when the GIL is enabled again. + _Py_unset_eval_breaker_bit(tstate, _PY_GIL_DROP_REQUEST_BIT); + return 1; + } + return 0; } +#endif + -/* Handle signals, pending calls, GIL drop request - and asynchronous exception */ +/* Do periodic things, like check for signals and async I/0. +* We need to do reasonably frequently, but not too frequently. +* All loops should include a check of the eval breaker. +* We also check on return from any builtin function. +* +* ## More Details ### +* +* The eval loop (this function) normally executes the instructions +* of a code object sequentially. However, the runtime supports a +* number of out-of-band execution scenarios that may pause that +* sequential execution long enough to do that out-of-band work +* in the current thread using the current PyThreadState. +* +* The scenarios include: +* +* - cyclic garbage collection +* - GIL drop requests +* - "async" exceptions +* - "pending calls" (some only in the main thread) +* - signal handling (only in the main thread) +* +* When the need for one of the above is detected, the eval loop +* pauses long enough to handle the detected case. Then, if doing +* so didn't trigger an exception, the eval loop resumes executing +* the sequential instructions. +* +* To make this work, the eval loop periodically checks if any +* of the above needs to happen. The individual checks can be +* expensive if computed each time, so a while back we switched +* to using pre-computed, per-interpreter variables for the checks, +* and later consolidated that to a single "eval breaker" variable +* (now a PyInterpreterState field). +* +* For the longest time, the eval breaker check would happen +* frequently, every 5 or so times through the loop, regardless +* of what instruction ran last or what would run next. Then, in +* early 2021 (gh-18334, commit 4958f5d), we switched to checking +* the eval breaker less frequently, by hard-coding the check to +* specific places in the eval loop (e.g. certain instructions). +* The intent then was to check after returning from calls +* and on the back edges of loops. +* +* In addition to being more efficient, that approach keeps +* the eval loop from running arbitrary code between instructions +* that don't handle that well. (See gh-74174.) +* +* Currently, the eval breaker check happens on back edges in +* the control flow graph, which pretty much applies to all loops, +* and most calls. +* (See bytecodes.c for exact information.) +* +* One consequence of this approach is that it might not be obvious +* how to force any specific thread to pick up the eval breaker, +* or for any specific thread to not pick it up. Mostly this +* involves judicious uses of locks and careful ordering of code, +* while avoiding code that might trigger the eval breaker +* until so desired. +*/ int _Py_HandlePending(PyThreadState *tstate) { - _PyRuntimeState * const runtime = &_PyRuntime; - struct _ceval_runtime_state *ceval = &runtime->ceval; - struct _ceval_state *interp_ceval_state = &tstate->interp->ceval; + uintptr_t breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); + + /* Stop-the-world */ + if ((breaker & _PY_EVAL_PLEASE_STOP_BIT) != 0) { + _Py_unset_eval_breaker_bit(tstate, _PY_EVAL_PLEASE_STOP_BIT); + _PyThreadState_Suspend(tstate); + + /* The attach blocks until the stop-the-world event is complete. */ + _PyThreadState_Attach(tstate); + } /* Pending signals */ - if (_Py_atomic_load_relaxed_int32(&ceval->signals_pending)) { + if ((breaker & _PY_SIGNALS_PENDING_BIT) != 0) { if (handle_signals(tstate) != 0) { return -1; } } /* Pending calls */ - if (maybe_has_pending_calls(tstate->interp)) { - if (make_pending_calls(tstate->interp) != 0) { + if ((breaker & _PY_CALLS_TO_DO_BIT) != 0) { + if (make_pending_calls(tstate) != 0) { return -1; } } +#ifdef Py_GIL_DISABLED + /* Objects with refcounts to merge */ + if ((breaker & _PY_EVAL_EXPLICIT_MERGE_BIT) != 0) { + _Py_unset_eval_breaker_bit(tstate, _PY_EVAL_EXPLICIT_MERGE_BIT); + _Py_brc_merge_refcounts(tstate); + } + /* Process deferred memory frees held by QSBR */ + if (_Py_qsbr_should_process(((_PyThreadStateImpl *)tstate)->qsbr)) { + _PyMem_ProcessDelayed(tstate); + } +#endif + /* GC scheduled to run */ - if (_Py_atomic_load_relaxed_int32(&interp_ceval_state->gc_scheduled)) { - _Py_atomic_store_relaxed(&interp_ceval_state->gc_scheduled, 0); - COMPUTE_EVAL_BREAKER(tstate->interp, ceval, interp_ceval_state); + if ((breaker & _PY_GC_SCHEDULED_BIT) != 0) { + _Py_unset_eval_breaker_bit(tstate, _PY_GC_SCHEDULED_BIT); _Py_RunGC(tstate); } /* GIL drop request */ - if (_Py_atomic_load_relaxed_int32(&interp_ceval_state->gil_drop_request)) { + if ((breaker & _PY_GIL_DROP_REQUEST_BIT) != 0) { /* Give another thread a chance */ - if (_PyThreadState_SwapNoGIL(NULL) != tstate) { - Py_FatalError("tstate mix-up"); - } - drop_gil(interp_ceval_state, tstate); + _PyThreadState_Detach(tstate); /* Other threads may run now */ - take_gil(tstate); - - if (_PyThreadState_SwapNoGIL(tstate) != NULL) { - Py_FatalError("orphan tstate"); - } + _PyThreadState_Attach(tstate); } /* Check for asynchronous exception. */ - if (tstate->async_exc != NULL) { - PyObject *exc = tstate->async_exc; - tstate->async_exc = NULL; - UNSIGNAL_ASYNC_EXC(tstate->interp); - _PyErr_SetNone(tstate, exc); - Py_DECREF(exc); - return -1; + if ((breaker & _PY_ASYNC_EXCEPTION_BIT) != 0) { + _Py_unset_eval_breaker_bit(tstate, _PY_ASYNC_EXCEPTION_BIT); + PyObject *exc = _Py_atomic_exchange_ptr(&tstate->async_exc, NULL); + if (exc != NULL) { + _PyErr_SetNone(tstate, exc); + Py_DECREF(exc); + return -1; + } } - - - // It is possible that some of the conditions that trigger the eval breaker - // are called in a different thread than the Python thread. An example of - // this is bpo-42296: On Windows, _PyEval_SignalReceived() can be called in - // a different thread than the Python thread, in which case - // _Py_ThreadCanHandleSignals() is wrong. Recompute eval_breaker in the - // current Python thread with the correct _Py_ThreadCanHandleSignals() - // value. It prevents to interrupt the eval loop at every instruction if - // the current Python thread cannot handle signals (if - // _Py_ThreadCanHandleSignals() is false). - COMPUTE_EVAL_BREAKER(tstate->interp, ceval, interp_ceval_state); - return 0; } - |