diff options
author | orivej <orivej@yandex-team.ru> | 2022-02-10 16:44:49 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:44:49 +0300 |
commit | 718c552901d703c502ccbefdfc3c9028d608b947 (patch) | |
tree | 46534a98bbefcd7b1f3faa5b52c138ab27db75b7 /contrib/tools/python3/src/Python/ceval_gil.h | |
parent | e9656aae26e0358d5378e5b63dcac5c8dbe0e4d0 (diff) | |
download | ydb-718c552901d703c502ccbefdfc3c9028d608b947.tar.gz |
Restoring authorship annotation for <orivej@yandex-team.ru>. Commit 1 of 2.
Diffstat (limited to 'contrib/tools/python3/src/Python/ceval_gil.h')
-rw-r--r-- | contrib/tools/python3/src/Python/ceval_gil.h | 346 |
1 files changed, 173 insertions, 173 deletions
diff --git a/contrib/tools/python3/src/Python/ceval_gil.h b/contrib/tools/python3/src/Python/ceval_gil.h index 3510675a69..7c77767e81 100644 --- a/contrib/tools/python3/src/Python/ceval_gil.h +++ b/contrib/tools/python3/src/Python/ceval_gil.h @@ -1,187 +1,187 @@ -/* - * Implementation of the Global Interpreter Lock (GIL). - */ - -#include <stdlib.h> -#include <errno.h> - +/* + * Implementation of the Global Interpreter Lock (GIL). + */ + +#include <stdlib.h> +#include <errno.h> + #include "pycore_atomic.h" - - -/* - Notes about the implementation: - - - The GIL is just a boolean variable (locked) whose access is protected - by a mutex (gil_mutex), and whose changes are signalled by a condition - variable (gil_cond). gil_mutex is taken for short periods of time, - and therefore mostly uncontended. - - - In the GIL-holding thread, the main loop (PyEval_EvalFrameEx) must be - able to release the GIL on demand by another thread. A volatile boolean - variable (gil_drop_request) is used for that purpose, which is checked - at every turn of the eval loop. That variable is set after a wait of - `interval` microseconds on `gil_cond` has timed out. - - [Actually, another volatile boolean variable (eval_breaker) is used - which ORs several conditions into one. Volatile booleans are - sufficient as inter-thread signalling means since Python is run - on cache-coherent architectures only.] - - - A thread wanting to take the GIL will first let pass a given amount of - time (`interval` microseconds) before setting gil_drop_request. This - encourages a defined switching period, but doesn't enforce it since - opcodes can take an arbitrary time to execute. - - The `interval` value is available for the user to read and modify - using the Python API `sys.{get,set}switchinterval()`. - - - When a thread releases the GIL and gil_drop_request is set, that thread - ensures that another GIL-awaiting thread gets scheduled. - It does so by waiting on a condition variable (switch_cond) until - the value of last_holder is changed to something else than its - own thread state pointer, indicating that another thread was able to - take the GIL. - - This is meant to prohibit the latency-adverse behaviour on multi-core - machines where one thread would speculatively release the GIL, but still - run and end up being the first to re-acquire it, making the "timeslices" - much longer than expected. - (Note: this mechanism is enabled with FORCE_SWITCHING above) -*/ - -#include "condvar.h" - -#define MUTEX_INIT(mut) \ - if (PyMUTEX_INIT(&(mut))) { \ - Py_FatalError("PyMUTEX_INIT(" #mut ") failed"); }; -#define MUTEX_FINI(mut) \ - if (PyMUTEX_FINI(&(mut))) { \ - Py_FatalError("PyMUTEX_FINI(" #mut ") failed"); }; -#define MUTEX_LOCK(mut) \ - if (PyMUTEX_LOCK(&(mut))) { \ - Py_FatalError("PyMUTEX_LOCK(" #mut ") failed"); }; -#define MUTEX_UNLOCK(mut) \ - if (PyMUTEX_UNLOCK(&(mut))) { \ - Py_FatalError("PyMUTEX_UNLOCK(" #mut ") failed"); }; - -#define COND_INIT(cond) \ - if (PyCOND_INIT(&(cond))) { \ - Py_FatalError("PyCOND_INIT(" #cond ") failed"); }; -#define COND_FINI(cond) \ - if (PyCOND_FINI(&(cond))) { \ - Py_FatalError("PyCOND_FINI(" #cond ") failed"); }; -#define COND_SIGNAL(cond) \ - if (PyCOND_SIGNAL(&(cond))) { \ - Py_FatalError("PyCOND_SIGNAL(" #cond ") failed"); }; -#define COND_WAIT(cond, mut) \ - if (PyCOND_WAIT(&(cond), &(mut))) { \ - Py_FatalError("PyCOND_WAIT(" #cond ") failed"); }; -#define COND_TIMED_WAIT(cond, mut, microseconds, timeout_result) \ - { \ - int r = PyCOND_TIMEDWAIT(&(cond), &(mut), (microseconds)); \ - if (r < 0) \ - Py_FatalError("PyCOND_WAIT(" #cond ") failed"); \ - if (r) /* 1 == timeout, 2 == impl. can't say, so assume timeout */ \ - timeout_result = 1; \ - else \ - timeout_result = 0; \ - } \ - - -#define DEFAULT_INTERVAL 5000 - + + +/* + Notes about the implementation: + + - The GIL is just a boolean variable (locked) whose access is protected + by a mutex (gil_mutex), and whose changes are signalled by a condition + variable (gil_cond). gil_mutex is taken for short periods of time, + and therefore mostly uncontended. + + - In the GIL-holding thread, the main loop (PyEval_EvalFrameEx) must be + able to release the GIL on demand by another thread. A volatile boolean + variable (gil_drop_request) is used for that purpose, which is checked + at every turn of the eval loop. That variable is set after a wait of + `interval` microseconds on `gil_cond` has timed out. + + [Actually, another volatile boolean variable (eval_breaker) is used + which ORs several conditions into one. Volatile booleans are + sufficient as inter-thread signalling means since Python is run + on cache-coherent architectures only.] + + - A thread wanting to take the GIL will first let pass a given amount of + time (`interval` microseconds) before setting gil_drop_request. This + encourages a defined switching period, but doesn't enforce it since + opcodes can take an arbitrary time to execute. + + The `interval` value is available for the user to read and modify + using the Python API `sys.{get,set}switchinterval()`. + + - When a thread releases the GIL and gil_drop_request is set, that thread + ensures that another GIL-awaiting thread gets scheduled. + It does so by waiting on a condition variable (switch_cond) until + the value of last_holder is changed to something else than its + own thread state pointer, indicating that another thread was able to + take the GIL. + + This is meant to prohibit the latency-adverse behaviour on multi-core + machines where one thread would speculatively release the GIL, but still + run and end up being the first to re-acquire it, making the "timeslices" + much longer than expected. + (Note: this mechanism is enabled with FORCE_SWITCHING above) +*/ + +#include "condvar.h" + +#define MUTEX_INIT(mut) \ + if (PyMUTEX_INIT(&(mut))) { \ + Py_FatalError("PyMUTEX_INIT(" #mut ") failed"); }; +#define MUTEX_FINI(mut) \ + if (PyMUTEX_FINI(&(mut))) { \ + Py_FatalError("PyMUTEX_FINI(" #mut ") failed"); }; +#define MUTEX_LOCK(mut) \ + if (PyMUTEX_LOCK(&(mut))) { \ + Py_FatalError("PyMUTEX_LOCK(" #mut ") failed"); }; +#define MUTEX_UNLOCK(mut) \ + if (PyMUTEX_UNLOCK(&(mut))) { \ + Py_FatalError("PyMUTEX_UNLOCK(" #mut ") failed"); }; + +#define COND_INIT(cond) \ + if (PyCOND_INIT(&(cond))) { \ + Py_FatalError("PyCOND_INIT(" #cond ") failed"); }; +#define COND_FINI(cond) \ + if (PyCOND_FINI(&(cond))) { \ + Py_FatalError("PyCOND_FINI(" #cond ") failed"); }; +#define COND_SIGNAL(cond) \ + if (PyCOND_SIGNAL(&(cond))) { \ + Py_FatalError("PyCOND_SIGNAL(" #cond ") failed"); }; +#define COND_WAIT(cond, mut) \ + if (PyCOND_WAIT(&(cond), &(mut))) { \ + Py_FatalError("PyCOND_WAIT(" #cond ") failed"); }; +#define COND_TIMED_WAIT(cond, mut, microseconds, timeout_result) \ + { \ + int r = PyCOND_TIMEDWAIT(&(cond), &(mut), (microseconds)); \ + if (r < 0) \ + Py_FatalError("PyCOND_WAIT(" #cond ") failed"); \ + if (r) /* 1 == timeout, 2 == impl. can't say, so assume timeout */ \ + timeout_result = 1; \ + else \ + timeout_result = 0; \ + } \ + + +#define DEFAULT_INTERVAL 5000 + static void _gil_initialize(struct _gil_runtime_state *gil) -{ - _Py_atomic_int uninitialized = {-1}; +{ + _Py_atomic_int uninitialized = {-1}; gil->locked = uninitialized; gil->interval = DEFAULT_INTERVAL; -} - +} + static int gil_created(struct _gil_runtime_state *gil) -{ +{ return (_Py_atomic_load_explicit(&gil->locked, _Py_memory_order_acquire) >= 0); -} - +} + static void create_gil(struct _gil_runtime_state *gil) -{ +{ MUTEX_INIT(gil->mutex); -#ifdef FORCE_SWITCHING +#ifdef FORCE_SWITCHING MUTEX_INIT(gil->switch_mutex); -#endif +#endif COND_INIT(gil->cond); -#ifdef FORCE_SWITCHING +#ifdef FORCE_SWITCHING COND_INIT(gil->switch_cond); -#endif +#endif _Py_atomic_store_relaxed(&gil->last_holder, 0); _Py_ANNOTATE_RWLOCK_CREATE(&gil->locked); _Py_atomic_store_explicit(&gil->locked, 0, _Py_memory_order_release); -} - +} + static void destroy_gil(struct _gil_runtime_state *gil) -{ - /* some pthread-like implementations tie the mutex to the cond - * and must have the cond destroyed first. - */ +{ + /* some pthread-like implementations tie the mutex to the cond + * and must have the cond destroyed first. + */ COND_FINI(gil->cond); MUTEX_FINI(gil->mutex); -#ifdef FORCE_SWITCHING +#ifdef FORCE_SWITCHING COND_FINI(gil->switch_cond); MUTEX_FINI(gil->switch_mutex); -#endif +#endif _Py_atomic_store_explicit(&gil->locked, -1, - _Py_memory_order_release); + _Py_memory_order_release); _Py_ANNOTATE_RWLOCK_DESTROY(&gil->locked); -} - +} + static void recreate_gil(struct _gil_runtime_state *gil) -{ +{ _Py_ANNOTATE_RWLOCK_DESTROY(&gil->locked); - /* XXX should we destroy the old OS resources here? */ + /* XXX should we destroy the old OS resources here? */ create_gil(gil); -} - +} + static void drop_gil(struct _ceval_runtime_state *ceval, struct _ceval_state *ceval2, PyThreadState *tstate) -{ +{ struct _gil_runtime_state *gil = &ceval->gil; if (!_Py_atomic_load_relaxed(&gil->locked)) { - Py_FatalError("drop_gil: GIL is not locked"); + Py_FatalError("drop_gil: GIL is not locked"); } - /* tstate is allowed to be NULL (early interpreter init) */ - if (tstate != NULL) { - /* Sub-interpreter support: threads might have been switched - under our feet using PyThreadState_Swap(). Fix the GIL last - holder variable so that our heuristics work. */ + /* tstate is allowed to be NULL (early interpreter init) */ + if (tstate != NULL) { + /* Sub-interpreter support: threads might have been switched + under our feet using PyThreadState_Swap(). Fix the GIL last + holder variable so that our heuristics work. */ _Py_atomic_store_relaxed(&gil->last_holder, (uintptr_t)tstate); - } - + } + MUTEX_LOCK(gil->mutex); _Py_ANNOTATE_RWLOCK_RELEASED(&gil->locked, /*is_write=*/1); _Py_atomic_store_relaxed(&gil->locked, 0); COND_SIGNAL(gil->cond); MUTEX_UNLOCK(gil->mutex); - -#ifdef FORCE_SWITCHING + +#ifdef FORCE_SWITCHING if (_Py_atomic_load_relaxed(&ceval2->gil_drop_request) && tstate != NULL) { MUTEX_LOCK(gil->switch_mutex); - /* Not switched yet => wait */ + /* Not switched yet => wait */ if (((PyThreadState*)_Py_atomic_load_relaxed(&gil->last_holder)) == tstate) - { + { assert(is_tstate_valid(tstate)); RESET_GIL_DROP_REQUEST(tstate->interp); - /* NOTE: if COND_WAIT does not atomically start waiting when - releasing the mutex, another thread can run through, take - the GIL and drop it again, and reset the condition - before we even had a chance to wait for it. */ + /* NOTE: if COND_WAIT does not atomically start waiting when + releasing the mutex, another thread can run through, take + the GIL and drop it again, and reset the condition + before we even had a chance to wait for it. */ COND_WAIT(gil->switch_cond, gil->switch_mutex); } MUTEX_UNLOCK(gil->switch_mutex); - } -#endif -} - + } +#endif +} + /* Check if a Python thread must exit immediately, rather than taking the GIL if Py_Finalize() has been called. @@ -209,7 +209,7 @@ tstate_must_exit(PyThreadState *tstate) tstate must be non-NULL. */ static void take_gil(PyThreadState *tstate) -{ +{ int err = errno; assert(tstate != NULL); @@ -223,7 +223,7 @@ take_gil(PyThreadState *tstate) PyThreadState freed memory. */ PyThread_exit_thread(); } - + assert(is_tstate_valid(tstate)); PyInterpreterState *interp = tstate->interp; struct _ceval_runtime_state *ceval = &interp->runtime->ceval; @@ -234,21 +234,21 @@ take_gil(PyThreadState *tstate) assert(gil_created(gil)); MUTEX_LOCK(gil->mutex); - + if (!_Py_atomic_load_relaxed(&gil->locked)) { - goto _ready; + goto _ready; } - + while (_Py_atomic_load_relaxed(&gil->locked)) { unsigned long saved_switchnum = gil->switch_number; - + unsigned long interval = (gil->interval >= 1 ? gil->interval : 1); int timed_out = 0; COND_TIMED_WAIT(gil->cond, gil->mutex, interval, timed_out); - /* If we timed out and no switch occurred in the meantime, it is time - to ask the GIL-holding thread to drop it. */ - if (timed_out && + /* If we timed out and no switch occurred in the meantime, it is time + to ask the GIL-holding thread to drop it. */ + if (timed_out && _Py_atomic_load_relaxed(&gil->locked) && gil->switch_number == saved_switchnum) { @@ -259,28 +259,28 @@ take_gil(PyThreadState *tstate) assert(is_tstate_valid(tstate)); SET_GIL_DROP_REQUEST(interp); - } - } + } + } -_ready: -#ifdef FORCE_SWITCHING +_ready: +#ifdef FORCE_SWITCHING /* This mutex must be taken before modifying gil->last_holder: see drop_gil(). */ MUTEX_LOCK(gil->switch_mutex); -#endif - /* We now hold the GIL */ +#endif + /* We now hold the GIL */ _Py_atomic_store_relaxed(&gil->locked, 1); _Py_ANNOTATE_RWLOCK_ACQUIRED(&gil->locked, /*is_write=*/1); - + if (tstate != (PyThreadState*)_Py_atomic_load_relaxed(&gil->last_holder)) { _Py_atomic_store_relaxed(&gil->last_holder, (uintptr_t)tstate); ++gil->switch_number; - } - -#ifdef FORCE_SWITCHING + } + +#ifdef FORCE_SWITCHING COND_SIGNAL(gil->switch_cond); MUTEX_UNLOCK(gil->switch_mutex); -#endif +#endif if (tstate_must_exit(tstate)) { /* bpo-36475: If Py_Finalize() has been called and tstate is not @@ -293,7 +293,7 @@ _ready: MUTEX_UNLOCK(gil->mutex); drop_gil(ceval, ceval2, tstate); PyThread_exit_thread(); - } + } assert(is_tstate_valid(tstate)); if (_Py_atomic_load_relaxed(&ceval2->gil_drop_request)) { @@ -309,23 +309,23 @@ _ready: } /* Don't access tstate if the thread must exit */ - if (tstate->async_exc != NULL) { + if (tstate->async_exc != NULL) { _PyEval_SignalAsyncExc(tstate); - } - + } + MUTEX_UNLOCK(gil->mutex); - errno = err; -} - -void _PyEval_SetSwitchInterval(unsigned long microseconds) -{ + errno = err; +} + +void _PyEval_SetSwitchInterval(unsigned long microseconds) +{ struct _gil_runtime_state *gil = &_PyRuntime.ceval.gil; gil->interval = microseconds; -} - -unsigned long _PyEval_GetSwitchInterval() -{ +} + +unsigned long _PyEval_GetSwitchInterval() +{ struct _gil_runtime_state *gil = &_PyRuntime.ceval.gil; return gil->interval; -} +} |