diff options
| author | shadchin <shadchin@yandex-team.com> | 2024-02-12 07:53:52 +0300 |
|---|---|---|
| committer | shadchin <shadchin@yandex-team.com> | 2024-02-12 08:07:36 +0300 |
| commit | ce1b7ca3171f9158180640c6a02a74b4afffedea (patch) | |
| tree | e47c1e8391b1b0128262c1e9b1e6ed4c8fff2348 /contrib/tools/python3/src/Python/pystate.c | |
| parent | 57350d96f030db90f220ce50ee591d5c5d403df7 (diff) | |
| download | ydb-ce1b7ca3171f9158180640c6a02a74b4afffedea.tar.gz | |
Update Python from 3.11.8 to 3.12.2
Diffstat (limited to 'contrib/tools/python3/src/Python/pystate.c')
| -rw-r--r-- | contrib/tools/python3/src/Python/pystate.c | 2148 |
1 files changed, 1465 insertions, 683 deletions
diff --git a/contrib/tools/python3/src/Python/pystate.c b/contrib/tools/python3/src/Python/pystate.c index db2ce878af..0ebbdfbfb4 100644 --- a/contrib/tools/python3/src/Python/pystate.c +++ b/contrib/tools/python3/src/Python/pystate.c @@ -3,14 +3,15 @@ #include "Python.h" #include "pycore_ceval.h" -#include "pycore_code.h" // stats +#include "pycore_code.h" // stats +#include "pycore_dtoa.h" // _dtoa_state_INIT() #include "pycore_frame.h" #include "pycore_initconfig.h" #include "pycore_object.h" // _PyType_InitCache() #include "pycore_pyerrors.h" #include "pycore_pylifecycle.h" #include "pycore_pymem.h" // _PyMem_SetDefaultAllocator() -#include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_pystate.h" #include "pycore_runtime_init.h" // _PyRuntimeState_INIT #include "pycore_sysmodule.h" @@ -37,56 +38,388 @@ to avoid the expense of doing their own locking). extern "C" { #endif -#define _PyRuntimeGILState_GetThreadState(gilstate) \ - ((PyThreadState*)_Py_atomic_load_relaxed(&(gilstate)->tstate_current)) -#define _PyRuntimeGILState_SetThreadState(gilstate, value) \ - _Py_atomic_store_relaxed(&(gilstate)->tstate_current, \ - (uintptr_t)(value)) -/* Forward declarations */ -static PyThreadState *_PyGILState_GetThisThreadState(struct _gilstate_runtime_state *gilstate); -static void _PyThreadState_Delete(PyThreadState *tstate, int check_current); +/****************************************/ +/* helpers for the current thread state */ +/****************************************/ + +// API for the current thread state is further down. + +/* "current" means one of: + - bound to the current OS thread + - holds the GIL + */ + +//------------------------------------------------- +// a highly efficient lookup for the current thread +//------------------------------------------------- + +/* + The stored thread state is set by PyThreadState_Swap(). + + For each of these functions, the GIL must be held by the current thread. + */ + + +#ifdef HAVE_THREAD_LOCAL +_Py_thread_local PyThreadState *_Py_tss_tstate = NULL; +#endif + +static inline PyThreadState * +current_fast_get(_PyRuntimeState *Py_UNUSED(runtime)) +{ +#ifdef HAVE_THREAD_LOCAL + return _Py_tss_tstate; +#else + // XXX Fall back to the PyThread_tss_*() API. +# error "no supported thread-local variable storage classifier" +#endif +} + +static inline void +current_fast_set(_PyRuntimeState *Py_UNUSED(runtime), PyThreadState *tstate) +{ + assert(tstate != NULL); +#ifdef HAVE_THREAD_LOCAL + _Py_tss_tstate = tstate; +#else + // XXX Fall back to the PyThread_tss_*() API. +# error "no supported thread-local variable storage classifier" +#endif +} + +static inline void +current_fast_clear(_PyRuntimeState *Py_UNUSED(runtime)) +{ +#ifdef HAVE_THREAD_LOCAL + _Py_tss_tstate = NULL; +#else + // XXX Fall back to the PyThread_tss_*() API. +# error "no supported thread-local variable storage classifier" +#endif +} + +#define tstate_verify_not_active(tstate) \ + if (tstate == current_fast_get((tstate)->interp->runtime)) { \ + _Py_FatalErrorFormat(__func__, "tstate %p is still current", tstate); \ + } + +PyThreadState * +_PyThreadState_GetCurrent(void) +{ + return current_fast_get(&_PyRuntime); +} + + +//------------------------------------------------ +// the thread state bound to the current OS thread +//------------------------------------------------ + +static inline int +tstate_tss_initialized(Py_tss_t *key) +{ + return PyThread_tss_is_created(key); +} + +static inline int +tstate_tss_init(Py_tss_t *key) +{ + assert(!tstate_tss_initialized(key)); + return PyThread_tss_create(key); +} + +static inline void +tstate_tss_fini(Py_tss_t *key) +{ + assert(tstate_tss_initialized(key)); + PyThread_tss_delete(key); +} + +static inline PyThreadState * +tstate_tss_get(Py_tss_t *key) +{ + assert(tstate_tss_initialized(key)); + return (PyThreadState *)PyThread_tss_get(key); +} + +static inline int +tstate_tss_set(Py_tss_t *key, PyThreadState *tstate) +{ + assert(tstate != NULL); + assert(tstate_tss_initialized(key)); + return PyThread_tss_set(key, (void *)tstate); +} + +static inline int +tstate_tss_clear(Py_tss_t *key) +{ + assert(tstate_tss_initialized(key)); + return PyThread_tss_set(key, (void *)NULL); +} + +#ifdef HAVE_FORK +/* Reset the TSS key - called by PyOS_AfterFork_Child(). + * This should not be necessary, but some - buggy - pthread implementations + * don't reset TSS upon fork(), see issue #10517. + */ +static PyStatus +tstate_tss_reinit(Py_tss_t *key) +{ + if (!tstate_tss_initialized(key)) { + return _PyStatus_OK(); + } + PyThreadState *tstate = tstate_tss_get(key); + + tstate_tss_fini(key); + if (tstate_tss_init(key) != 0) { + return _PyStatus_NO_MEMORY(); + } + + /* If the thread had an associated auto thread state, reassociate it with + * the new key. */ + if (tstate && tstate_tss_set(key, tstate) != 0) { + return _PyStatus_ERR("failed to re-set autoTSSkey"); + } + return _PyStatus_OK(); +} +#endif + + +/* + The stored thread state is set by bind_tstate() (AKA PyThreadState_Bind(). + + The GIL does no need to be held for these. + */ + +#define gilstate_tss_initialized(runtime) \ + tstate_tss_initialized(&(runtime)->autoTSSkey) +#define gilstate_tss_init(runtime) \ + tstate_tss_init(&(runtime)->autoTSSkey) +#define gilstate_tss_fini(runtime) \ + tstate_tss_fini(&(runtime)->autoTSSkey) +#define gilstate_tss_get(runtime) \ + tstate_tss_get(&(runtime)->autoTSSkey) +#define _gilstate_tss_set(runtime, tstate) \ + tstate_tss_set(&(runtime)->autoTSSkey, tstate) +#define _gilstate_tss_clear(runtime) \ + tstate_tss_clear(&(runtime)->autoTSSkey) +#define gilstate_tss_reinit(runtime) \ + tstate_tss_reinit(&(runtime)->autoTSSkey) + +static inline void +gilstate_tss_set(_PyRuntimeState *runtime, PyThreadState *tstate) +{ + assert(tstate != NULL && tstate->interp->runtime == runtime); + if (_gilstate_tss_set(runtime, tstate) != 0) { + Py_FatalError("failed to set current tstate (TSS)"); + } +} + +static inline void +gilstate_tss_clear(_PyRuntimeState *runtime) +{ + if (_gilstate_tss_clear(runtime) != 0) { + Py_FatalError("failed to clear current tstate (TSS)"); + } +} + + +#ifndef NDEBUG +static inline int tstate_is_alive(PyThreadState *tstate); + +static inline int +tstate_is_bound(PyThreadState *tstate) +{ + return tstate->_status.bound && !tstate->_status.unbound; +} +#endif // !NDEBUG + +static void bind_gilstate_tstate(PyThreadState *); +static void unbind_gilstate_tstate(PyThreadState *); + +static void +bind_tstate(PyThreadState *tstate) +{ + assert(tstate != NULL); + assert(tstate_is_alive(tstate) && !tstate->_status.bound); + assert(!tstate->_status.unbound); // just in case + assert(!tstate->_status.bound_gilstate); + assert(tstate != gilstate_tss_get(tstate->interp->runtime)); + assert(!tstate->_status.active); + assert(tstate->thread_id == 0); + assert(tstate->native_thread_id == 0); + + // Currently we don't necessarily store the thread state + // in thread-local storage (e.g. per-interpreter). + + tstate->thread_id = PyThread_get_thread_ident(); +#ifdef PY_HAVE_THREAD_NATIVE_ID + tstate->native_thread_id = PyThread_get_thread_native_id(); +#endif + + tstate->_status.bound = 1; +} + +static void +unbind_tstate(PyThreadState *tstate) +{ + assert(tstate != NULL); + assert(tstate_is_bound(tstate)); +#ifndef HAVE_PTHREAD_STUBS + assert(tstate->thread_id > 0); +#endif +#ifdef PY_HAVE_THREAD_NATIVE_ID + assert(tstate->native_thread_id > 0); +#endif + + // We leave thread_id and native_thread_id alone + // since they can be useful for debugging. + // Check the `_status` field to know if these values + // are still valid. + + // We leave tstate->_status.bound set to 1 + // to indicate it was previously bound. + tstate->_status.unbound = 1; +} + + +/* Stick the thread state for this thread in thread specific storage. + + When a thread state is created for a thread by some mechanism + other than PyGILState_Ensure(), it's important that the GILState + machinery knows about it so it doesn't try to create another + thread state for the thread. + (This is a better fix for SF bug #1010677 than the first one attempted.) + + The only situation where you can legitimately have more than one + thread state for an OS level thread is when there are multiple + interpreters. + + Before 3.12, the PyGILState_*() APIs didn't work with multiple + interpreters (see bpo-10915 and bpo-15751), so this function used + to set TSS only once. Thus, the first thread state created for that + given OS level thread would "win", which seemed reasonable behaviour. +*/ + +static void +bind_gilstate_tstate(PyThreadState *tstate) +{ + assert(tstate != NULL); + assert(tstate_is_alive(tstate)); + assert(tstate_is_bound(tstate)); + // XXX assert(!tstate->_status.active); + assert(!tstate->_status.bound_gilstate); + + _PyRuntimeState *runtime = tstate->interp->runtime; + PyThreadState *tcur = gilstate_tss_get(runtime); + assert(tstate != tcur); + + if (tcur != NULL) { + tcur->_status.bound_gilstate = 0; + } + gilstate_tss_set(runtime, tstate); + tstate->_status.bound_gilstate = 1; +} + +static void +unbind_gilstate_tstate(PyThreadState *tstate) +{ + assert(tstate != NULL); + // XXX assert(tstate_is_alive(tstate)); + assert(tstate_is_bound(tstate)); + // XXX assert(!tstate->_status.active); + assert(tstate->_status.bound_gilstate); + assert(tstate == gilstate_tss_get(tstate->interp->runtime)); + + gilstate_tss_clear(tstate->interp->runtime); + tstate->_status.bound_gilstate = 0; +} + + +//---------------------------------------------- +// the thread state that currently holds the GIL +//---------------------------------------------- + +/* This is not exported, as it is not reliable! It can only + ever be compared to the state for the *current* thread. + * If not equal, then it doesn't matter that the actual + value may change immediately after comparison, as it can't + possibly change to the current thread's state. + * If equal, then the current thread holds the lock, so the value can't + change until we yield the lock. +*/ +static int +holds_gil(PyThreadState *tstate) +{ + // XXX Fall back to tstate->interp->runtime->ceval.gil.last_holder + // (and tstate->interp->runtime->ceval.gil.locked). + assert(tstate != NULL); +#ifndef NDEBUG + if (!tstate_is_alive(tstate)) { + return 0; + } +#endif + _PyRuntimeState *runtime = tstate->interp->runtime; + /* Must be the tstate for this thread */ + assert(tstate == gilstate_tss_get(runtime)); + return tstate == current_fast_get(runtime); +} + + +/****************************/ +/* the global runtime state */ +/****************************/ + +//---------- +// lifecycle +//---------- /* Suppress deprecation warning for PyBytesObject.ob_shash */ _Py_COMP_DIAG_PUSH _Py_COMP_DIAG_IGNORE_DEPR_DECLS /* We use "initial" if the runtime gets re-used - (e.g. Py_Finalize() followed by Py_Initialize(). */ -static const _PyRuntimeState initial = _PyRuntimeState_INIT; + (e.g. Py_Finalize() followed by Py_Initialize(). + Note that we initialize "initial" relative to _PyRuntime, + to ensure pre-initialized pointers point to the active + runtime state (and not "initial"). */ +static const _PyRuntimeState initial = _PyRuntimeState_INIT(_PyRuntime); _Py_COMP_DIAG_POP +#define NUMLOCKS 9 +#define LOCKS_INIT(runtime) \ + { \ + &(runtime)->interpreters.mutex, \ + &(runtime)->xidregistry.mutex, \ + &(runtime)->getargs.mutex, \ + &(runtime)->unicode_state.ids.lock, \ + &(runtime)->imports.extensions.mutex, \ + &(runtime)->ceval.pending_mainthread.lock, \ + &(runtime)->atexit.mutex, \ + &(runtime)->audit_hooks.mutex, \ + &(runtime)->allocators.mutex, \ + } + static int -alloc_for_runtime(PyThread_type_lock *plock1, PyThread_type_lock *plock2, - PyThread_type_lock *plock3) +alloc_for_runtime(PyThread_type_lock locks[NUMLOCKS]) { /* Force default allocator, since _PyRuntimeState_Fini() must use the same allocator than this function. */ PyMemAllocatorEx old_alloc; _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - PyThread_type_lock lock1 = PyThread_allocate_lock(); - if (lock1 == NULL) { - return -1; - } - - PyThread_type_lock lock2 = PyThread_allocate_lock(); - if (lock2 == NULL) { - PyThread_free_lock(lock1); - return -1; - } - - PyThread_type_lock lock3 = PyThread_allocate_lock(); - if (lock3 == NULL) { - PyThread_free_lock(lock1); - PyThread_free_lock(lock2); - return -1; + for (int i = 0; i < NUMLOCKS; i++) { + PyThread_type_lock lock = PyThread_allocate_lock(); + if (lock == NULL) { + for (int j = 0; j < i; j++) { + PyThread_free_lock(locks[j]); + locks[j] = NULL; + } + break; + } + locks[i] = lock; } PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - - *plock1 = lock1; - *plock2 = lock2; - *plock3 = lock3; return 0; } @@ -95,9 +428,7 @@ init_runtime(_PyRuntimeState *runtime, void *open_code_hook, void *open_code_userdata, _Py_AuditHookEntry *audit_hook_head, Py_ssize_t unicode_next_index, - PyThread_type_lock unicode_ids_mutex, - PyThread_type_lock interpreters_mutex, - PyThread_type_lock xidregistry_mutex) + PyThread_type_lock locks[NUMLOCKS]) { if (runtime->_initialized) { Py_FatalError("runtime already initialized"); @@ -109,21 +440,20 @@ init_runtime(_PyRuntimeState *runtime, runtime->open_code_hook = open_code_hook; runtime->open_code_userdata = open_code_userdata; - runtime->audit_hook_head = audit_hook_head; - - _PyEval_InitRuntimeState(&runtime->ceval); + runtime->audit_hooks.head = audit_hook_head; PyPreConfig_InitPythonConfig(&runtime->preconfig); - runtime->interpreters.mutex = interpreters_mutex; - - runtime->xidregistry.mutex = xidregistry_mutex; + PyThread_type_lock *lockptrs[NUMLOCKS] = LOCKS_INIT(runtime); + for (int i = 0; i < NUMLOCKS; i++) { + assert(locks[i] != NULL); + *lockptrs[i] = locks[i]; + } // Set it to the ID of the main thread of the main interpreter. runtime->main_thread = PyThread_get_thread_ident(); - runtime->unicode_ids.next_index = unicode_next_index; - runtime->unicode_ids.lock = unicode_ids_mutex; + runtime->unicode_state.ids.next_index = unicode_next_index; runtime->_initialized = 1; } @@ -136,13 +466,13 @@ _PyRuntimeState_Init(_PyRuntimeState *runtime) initialization and interpreter initialization. */ void *open_code_hook = runtime->open_code_hook; void *open_code_userdata = runtime->open_code_userdata; - _Py_AuditHookEntry *audit_hook_head = runtime->audit_hook_head; + _Py_AuditHookEntry *audit_hook_head = runtime->audit_hooks.head; // bpo-42882: Preserve next_index value if Py_Initialize()/Py_Finalize() // is called multiple times. - Py_ssize_t unicode_next_index = runtime->unicode_ids.next_index; + Py_ssize_t unicode_next_index = runtime->unicode_state.ids.next_index; - PyThread_type_lock lock1, lock2, lock3; - if (alloc_for_runtime(&lock1, &lock2, &lock3) != 0) { + PyThread_type_lock locks[NUMLOCKS]; + if (alloc_for_runtime(locks) != 0) { return _PyStatus_NO_MEMORY(); } @@ -151,15 +481,43 @@ _PyRuntimeState_Init(_PyRuntimeState *runtime) // Reset to _PyRuntimeState_INIT. memcpy(runtime, &initial, sizeof(*runtime)); } + + if (gilstate_tss_init(runtime) != 0) { + _PyRuntimeState_Fini(runtime); + return _PyStatus_NO_MEMORY(); + } + + if (PyThread_tss_create(&runtime->trashTSSkey) != 0) { + _PyRuntimeState_Fini(runtime); + return _PyStatus_NO_MEMORY(); + } + init_runtime(runtime, open_code_hook, open_code_userdata, audit_hook_head, - unicode_next_index, lock1, lock2, lock3); + unicode_next_index, locks); return _PyStatus_OK(); } +static void _xidregistry_clear(struct _xidregistry *); + void _PyRuntimeState_Fini(_PyRuntimeState *runtime) { +#ifdef Py_REF_DEBUG + /* The count is cleared by _Py_FinalizeRefTotal(). */ + assert(runtime->object_state.interpreter_leaks == 0); +#endif + + _xidregistry_clear(&runtime->xidregistry); + + if (gilstate_tss_initialized(runtime)) { + gilstate_tss_fini(runtime); + } + + if (PyThread_tss_is_created(&runtime->trashTSSkey)) { + PyThread_tss_delete(&runtime->trashTSSkey); + } + /* Force the allocator used by _PyRuntimeState_Init(). */ PyMemAllocatorEx old_alloc; _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); @@ -169,11 +527,16 @@ _PyRuntimeState_Fini(_PyRuntimeState *runtime) LOCK = NULL; \ } - FREE_LOCK(runtime->interpreters.mutex); - FREE_LOCK(runtime->xidregistry.mutex); - FREE_LOCK(runtime->unicode_ids.lock); + PyThread_type_lock *lockptrs[NUMLOCKS] = LOCKS_INIT(runtime); + for (int i = 0; i < NUMLOCKS; i++) { + FREE_LOCK(*lockptrs[i]); + } #undef FREE_LOCK + if (runtime->sys_path_0 != NULL) { + PyMem_RawFree(runtime->sys_path_0); + runtime->sys_path_0 = NULL; + } PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); } @@ -191,36 +554,53 @@ _PyRuntimeState_ReInitThreads(_PyRuntimeState *runtime) PyMemAllocatorEx old_alloc; _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - int reinit_interp = _PyThread_at_fork_reinit(&runtime->interpreters.mutex); - int reinit_xidregistry = _PyThread_at_fork_reinit(&runtime->xidregistry.mutex); - int reinit_unicode_ids = _PyThread_at_fork_reinit(&runtime->unicode_ids.lock); + PyThread_type_lock *lockptrs[NUMLOCKS] = LOCKS_INIT(runtime); + int reinit_err = 0; + for (int i = 0; i < NUMLOCKS; i++) { + reinit_err += _PyThread_at_fork_reinit(lockptrs[i]); + } + /* PyOS_AfterFork_Child(), which calls this function, later calls + _PyInterpreterState_DeleteExceptMain(), so we only need to update + the main interpreter here. */ + assert(runtime->interpreters.main != NULL); + runtime->interpreters.main->xidregistry.mutex = runtime->xidregistry.mutex; PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); /* bpo-42540: id_mutex is freed by _PyInterpreterState_Delete, which does * not force the default allocator. */ - int reinit_main_id = _PyThread_at_fork_reinit(&runtime->interpreters.main->id_mutex); + reinit_err += _PyThread_at_fork_reinit(&runtime->interpreters.main->id_mutex); - if (reinit_interp < 0 - || reinit_main_id < 0 - || reinit_xidregistry < 0 - || reinit_unicode_ids < 0) - { + if (reinit_err < 0) { return _PyStatus_ERR("Failed to reinitialize runtime locks"); + } + PyStatus status = gilstate_tss_reinit(runtime); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + + if (PyThread_tss_is_created(&runtime->trashTSSkey)) { + PyThread_tss_delete(&runtime->trashTSSkey); + } + if (PyThread_tss_create(&runtime->trashTSSkey) != 0) { + return _PyStatus_NO_MEMORY(); } + return _PyStatus_OK(); } #endif -#define HEAD_LOCK(runtime) \ - PyThread_acquire_lock((runtime)->interpreters.mutex, WAIT_LOCK) -#define HEAD_UNLOCK(runtime) \ - PyThread_release_lock((runtime)->interpreters.mutex) -/* Forward declaration */ -static void _PyGILState_NoteThreadState( - struct _gilstate_runtime_state *gilstate, PyThreadState* tstate); +/*************************************/ +/* the per-interpreter runtime state */ +/*************************************/ + +//---------- +// lifecycle +//---------- + +/* Calling this indicates that the runtime is ready to create interpreters. */ PyStatus _PyInterpreterState_Enable(_PyRuntimeState *runtime) @@ -248,6 +628,7 @@ _PyInterpreterState_Enable(_PyRuntimeState *runtime) return _PyStatus_OK(); } + static PyInterpreterState * alloc_interpreter(void) { @@ -257,7 +638,9 @@ alloc_interpreter(void) static void free_interpreter(PyInterpreterState *interp) { - if (!interp->_static) { + // The main interpreter is statically allocated so + // should not be freed. + if (interp != &_PyRuntime._main_interpreter) { PyMem_RawFree(interp); } } @@ -268,8 +651,19 @@ free_interpreter(PyInterpreterState *interp) e.g. by PyMem_RawCalloc() or memset(), or otherwise pre-initialized. The runtime state is not manipulated. Instead it is assumed that the interpreter is getting added to the runtime. - */ + Note that the main interpreter was statically initialized as part + of the runtime and most state is already set properly. That leaves + a small number of fields to initialize dynamically, as well as some + that are initialized lazily. + + For subinterpreters we memcpy() the main interpreter in + PyInterpreterState_New(), leaving it in the same mostly-initialized + state. The only difference is that the interpreter has some + self-referential state that is statically initializexd to the + main interpreter. We fix those fields here, in addition + to the other dynamically initialized fields. + */ static void init_interpreter(PyInterpreterState *interp, _PyRuntimeState *runtime, int64_t id, @@ -290,10 +684,38 @@ init_interpreter(PyInterpreterState *interp, assert(next != NULL || (interp == runtime->interpreters.main)); interp->next = next; - _PyEval_InitState(&interp->ceval, pending_lock); + /* Initialize obmalloc, but only for subinterpreters, + since the main interpreter is initialized statically. */ + if (interp != &runtime->_main_interpreter) { + poolp temp[OBMALLOC_USED_POOLS_SIZE] = \ + _obmalloc_pools_INIT(interp->obmalloc.pools); + memcpy(&interp->obmalloc.pools.used, temp, sizeof(temp)); + } + _PyObject_InitState(interp); + + _PyEval_InitState(interp, pending_lock); _PyGC_InitState(&interp->gc); PyConfig_InitPythonConfig(&interp->config); _PyType_InitCache(interp); + for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) { + interp->monitors.tools[i] = 0; + } + for (int t = 0; t < PY_MONITORING_TOOL_IDS; t++) { + for (int e = 0; e < _PY_MONITORING_EVENTS; e++) { + interp->monitoring_callables[t][e] = NULL; + + } + } + interp->sys_profile_initialized = false; + interp->sys_trace_initialized = false; + if (interp != &runtime->_main_interpreter) { + /* Fix the self-referential, statically initialized fields. */ + interp->dtoa = (struct _dtoa_state)_dtoa_state_INIT(interp); + } + interp->f_opcode_trace_set = false; + + assert(runtime->xidregistry.mutex != NULL); + interp->xidregistry.mutex = runtime->xidregistry.mutex; interp->_initialized = 1; } @@ -302,7 +724,8 @@ PyInterpreterState * PyInterpreterState_New(void) { PyInterpreterState *interp; - PyThreadState *tstate = _PyThreadState_GET(); + _PyRuntimeState *runtime = &_PyRuntime; + PyThreadState *tstate = current_fast_get(runtime); /* tstate is NULL when Py_InitializeFromConfig() calls PyInterpreterState_New() to create the main interpreter. */ @@ -319,7 +742,6 @@ PyInterpreterState_New(void) } /* Don't get runtime from tstate since tstate can be NULL. */ - _PyRuntimeState *runtime = &_PyRuntime; struct pyinterpreters *interpreters = &runtime->interpreters; /* We completely serialize creation of multiple interpreters, since @@ -341,7 +763,6 @@ PyInterpreterState_New(void) interp = &runtime->_main_interpreter; assert(interp->id == 0); assert(interp->next == NULL); - assert(interp->_static); interpreters->main = interp; } @@ -356,9 +777,6 @@ PyInterpreterState_New(void) // Set to _PyInterpreterState_INIT. memcpy(interp, &initial._main_interpreter, sizeof(*interp)); - // We need to adjust any fields that are different from the initial - // interpreter (as defined in _PyInterpreterState_INIT): - interp->_static = false; if (id < 0) { /* overflow or Py_Initialize() not called yet! */ @@ -390,8 +808,20 @@ error: static void interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) { + assert(interp != NULL); + assert(tstate != NULL); _PyRuntimeState *runtime = interp->runtime; + /* XXX Conditions we need to enforce: + + * the GIL must be held by the current thread + * tstate must be the "current" thread state (current_fast_get()) + * tstate->interp must be interp + * for the main interpreter, tstate must be the main thread + */ + // XXX Ideally, we would not rely on any thread state in this function + // (and we would drop the "tstate" argument). + if (_PySys_Audit(tstate, "cpython.PyInterpreterState_Clear", NULL) < 0) { _PyErr_Clear(tstate); } @@ -409,18 +839,48 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) p = p->next; HEAD_UNLOCK(runtime); } + if (tstate->interp == interp) { + /* We fix tstate->_status below when we for sure aren't using it + (e.g. no longer need the GIL). */ + // XXX Eliminate the need to do this. + tstate->_status.cleared = 0; + } + + /* It is possible that any of the objects below have a finalizer + that runs Python code or otherwise relies on a thread state + or even the interpreter state. For now we trust that isn't + a problem. + */ + // XXX Make sure we properly deal with problematic finalizers. Py_CLEAR(interp->audit_hooks); + for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) { + interp->monitors.tools[i] = 0; + } + for (int t = 0; t < PY_MONITORING_TOOL_IDS; t++) { + for (int e = 0; e < _PY_MONITORING_EVENTS; e++) { + Py_CLEAR(interp->monitoring_callables[t][e]); + } + } + interp->sys_profile_initialized = false; + interp->sys_trace_initialized = false; + for (int t = 0; t < PY_MONITORING_TOOL_IDS; t++) { + Py_CLEAR(interp->monitoring_tool_names[t]); + } + PyConfig_Clear(&interp->config); Py_CLEAR(interp->codec_search_path); Py_CLEAR(interp->codec_search_cache); Py_CLEAR(interp->codec_error_registry); - Py_CLEAR(interp->modules); - Py_CLEAR(interp->modules_by_index); + + assert(interp->imports.modules == NULL); + assert(interp->imports.modules_by_index == NULL); + assert(interp->imports.importlib == NULL); + assert(interp->imports.import_func == NULL); + + Py_CLEAR(interp->sysdict_copy); Py_CLEAR(interp->builtins_copy); - Py_CLEAR(interp->importlib); - Py_CLEAR(interp->import_func); Py_CLEAR(interp->dict); #ifdef HAVE_FORK Py_CLEAR(interp->before_forkers); @@ -447,7 +907,36 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) PyDict_Clear(interp->builtins); Py_CLEAR(interp->sysdict); Py_CLEAR(interp->builtins); + Py_CLEAR(interp->interpreter_trampoline); + + _xidregistry_clear(&interp->xidregistry); + /* The lock is owned by the runtime, so we don't free it here. */ + interp->xidregistry.mutex = NULL; + + if (tstate->interp == interp) { + /* We are now safe to fix tstate->_status.cleared. */ + // XXX Do this (much) earlier? + tstate->_status.cleared = 1; + } + + for (int i=0; i < DICT_MAX_WATCHERS; i++) { + interp->dict_state.watchers[i] = NULL; + } + for (int i=0; i < TYPE_MAX_WATCHERS; i++) { + interp->type_watchers[i] = NULL; + } + + for (int i=0; i < FUNC_MAX_WATCHERS; i++) { + interp->func_watchers[i] = NULL; + } + interp->active_func_watchers = 0; + + for (int i=0; i < CODE_MAX_WATCHERS; i++) { + interp->code_watchers[i] = NULL; + } + interp->active_code_watchers = 0; + interp->f_opcode_trace_set = false; // XXX Once we have one allocator per interpreter (i.e. // per-interpreter GC) we must ensure that all of the interpreter's // objects have been cleaned up at the point. @@ -460,8 +949,8 @@ PyInterpreterState_Clear(PyInterpreterState *interp) // Use the current Python thread state to call audit hooks and to collect // garbage. It can be different than the current Python thread state // of 'interp'. - PyThreadState *current_tstate = _PyThreadState_GET(); - + PyThreadState *current_tstate = current_fast_get(interp->runtime); + _PyImport_ClearCore(interp); interpreter_clear(interp, current_tstate); } @@ -469,33 +958,40 @@ PyInterpreterState_Clear(PyInterpreterState *interp) void _PyInterpreterState_Clear(PyThreadState *tstate) { + _PyImport_ClearCore(tstate->interp); interpreter_clear(tstate->interp, tstate); } -static void -zapthreads(PyInterpreterState *interp, int check_current) -{ - PyThreadState *tstate; - /* No need to lock the mutex here because this should only happen - when the threads are all really dead (XXX famous last words). */ - while ((tstate = interp->threads.head) != NULL) { - _PyThreadState_Delete(tstate, check_current); - } -} - +static inline void tstate_deactivate(PyThreadState *tstate); +static void zapthreads(PyInterpreterState *interp); void PyInterpreterState_Delete(PyInterpreterState *interp) { _PyRuntimeState *runtime = interp->runtime; struct pyinterpreters *interpreters = &runtime->interpreters; - zapthreads(interp, 0); + + // XXX Clearing the "current" thread state should happen before + // we start finalizing the interpreter (or the current thread state). + PyThreadState *tcur = current_fast_get(runtime); + if (tcur != NULL && interp == tcur->interp) { + /* Unset current thread. After this, many C API calls become crashy. */ + current_fast_clear(runtime); + tstate_deactivate(tcur); + _PyEval_ReleaseLock(interp, NULL); + } + + zapthreads(interp); _PyEval_FiniState(&interp->ceval); - /* Delete current thread. After this, many C API calls become crashy. */ - _PyThreadState_Swap(&runtime->gilstate, NULL); + // XXX These two calls should be done at the end of clear_interpreter(), + // but currently some objects get decref'ed after that. +#ifdef Py_REF_DEBUG + _PyInterpreterState_FinalizeRefTotal(interp); +#endif + _PyInterpreterState_FinalizeAllocatedBlocks(interp); HEAD_LOCK(runtime); PyInterpreterState **p; @@ -535,10 +1031,9 @@ PyInterpreterState_Delete(PyInterpreterState *interp) PyStatus _PyInterpreterState_DeleteExceptMain(_PyRuntimeState *runtime) { - struct _gilstate_runtime_state *gilstate = &runtime->gilstate; struct pyinterpreters *interpreters = &runtime->interpreters; - PyThreadState *tstate = _PyThreadState_Swap(gilstate, NULL); + PyThreadState *tstate = _PyThreadState_Swap(runtime, NULL); if (tstate != NULL && tstate->interp != interpreters->main) { return _PyStatus_ERR("not main interpreter"); } @@ -554,8 +1049,10 @@ _PyInterpreterState_DeleteExceptMain(_PyRuntimeState *runtime) continue; } + // XXX Won't this fail since PyInterpreterState_Clear() requires + // the "current" tstate to be set? PyInterpreterState_Clear(interp); // XXX must activate? - zapthreads(interp, 1); + zapthreads(interp); if (interp->id_mutex != NULL) { PyThread_free_lock(interp->id_mutex); } @@ -568,24 +1065,48 @@ _PyInterpreterState_DeleteExceptMain(_PyRuntimeState *runtime) if (interpreters->head == NULL) { return _PyStatus_ERR("missing main interpreter"); } - _PyThreadState_Swap(gilstate, tstate); + _PyThreadState_Swap(runtime, tstate); return _PyStatus_OK(); } #endif -PyInterpreterState * -PyInterpreterState_Get(void) +int +_PyInterpreterState_SetRunningMain(PyInterpreterState *interp) { - PyThreadState *tstate = _PyThreadState_GET(); + if (interp->threads_main != NULL) { + PyErr_SetString(PyExc_RuntimeError, + "interpreter already running"); + return -1; + } + PyThreadState *tstate = current_fast_get(&_PyRuntime); _Py_EnsureTstateNotNULL(tstate); - PyInterpreterState *interp = tstate->interp; - if (interp == NULL) { - Py_FatalError("no current interpreter"); + if (tstate->interp != interp) { + PyErr_SetString(PyExc_RuntimeError, + "current tstate has wrong interpreter"); + return -1; } - return interp; + interp->threads_main = tstate; + return 0; +} + +void +_PyInterpreterState_SetNotRunningMain(PyInterpreterState *interp) +{ + assert(interp->threads_main == current_fast_get(&_PyRuntime)); + interp->threads_main = NULL; } +int +_PyInterpreterState_IsRunningMain(PyInterpreterState *interp) +{ + return (interp->threads_main != NULL); +} + + +//---------- +// accessors +//---------- int64_t PyInterpreterState_GetID(PyInterpreterState *interp) @@ -598,41 +1119,6 @@ PyInterpreterState_GetID(PyInterpreterState *interp) } -static PyInterpreterState * -interp_look_up_id(_PyRuntimeState *runtime, int64_t requested_id) -{ - PyInterpreterState *interp = runtime->interpreters.head; - while (interp != NULL) { - int64_t id = PyInterpreterState_GetID(interp); - if (id < 0) { - return NULL; - } - if (requested_id == id) { - return interp; - } - interp = PyInterpreterState_Next(interp); - } - return NULL; -} - -PyInterpreterState * -_PyInterpreterState_LookUpID(int64_t requested_id) -{ - PyInterpreterState *interp = NULL; - if (requested_id >= 0) { - _PyRuntimeState *runtime = &_PyRuntime; - HEAD_LOCK(runtime); - interp = interp_look_up_id(runtime, requested_id); - HEAD_UNLOCK(runtime); - } - if (interp == NULL && !PyErr_Occurred()) { - PyErr_Format(PyExc_RuntimeError, - "unrecognized interpreter ID %lld", requested_id); - } - return interp; -} - - int _PyInterpreterState_IDInitref(PyInterpreterState *interp) { @@ -668,8 +1154,8 @@ void _PyInterpreterState_IDDecref(PyInterpreterState *interp) { assert(interp->id_mutex != NULL); + _PyRuntimeState *runtime = interp->runtime; - struct _gilstate_runtime_state *gilstate = &_PyRuntime.gilstate; PyThread_acquire_lock(interp->id_mutex, WAIT_LOCK); assert(interp->id_refcount != 0); interp->id_refcount -= 1; @@ -680,9 +1166,9 @@ _PyInterpreterState_IDDecref(PyInterpreterState *interp) // XXX Using the "head" thread isn't strictly correct. PyThreadState *tstate = PyInterpreterState_ThreadHead(interp); // XXX Possible GILState issues? - PyThreadState *save_tstate = _PyThreadState_Swap(gilstate, tstate); + PyThreadState *save_tstate = _PyThreadState_Swap(runtime, tstate); Py_EndInterpreter(tstate); - _PyThreadState_Swap(gilstate, save_tstate); + _PyThreadState_Swap(runtime, save_tstate); } } @@ -701,11 +1187,12 @@ _PyInterpreterState_RequireIDRef(PyInterpreterState *interp, int required) PyObject * _PyInterpreterState_GetMainModule(PyInterpreterState *interp) { - if (interp->modules == NULL) { + PyObject *modules = _PyImport_GetModules(interp); + if (modules == NULL) { PyErr_SetString(PyExc_RuntimeError, "interpreter not initialized"); return NULL; } - return PyMapping_GetItemString(interp->modules, "__main__"); + return PyMapping_GetItemString(modules, "__main__"); } PyObject * @@ -721,6 +1208,88 @@ PyInterpreterState_GetDict(PyInterpreterState *interp) return interp->dict; } + +//----------------------------- +// look up an interpreter state +//----------------------------- + +/* Return the interpreter associated with the current OS thread. + + The GIL must be held. + */ + +PyInterpreterState * +PyInterpreterState_Get(void) +{ + PyThreadState *tstate = current_fast_get(&_PyRuntime); + _Py_EnsureTstateNotNULL(tstate); + PyInterpreterState *interp = tstate->interp; + if (interp == NULL) { + Py_FatalError("no current interpreter"); + } + return interp; +} + + +static PyInterpreterState * +interp_look_up_id(_PyRuntimeState *runtime, int64_t requested_id) +{ + PyInterpreterState *interp = runtime->interpreters.head; + while (interp != NULL) { + int64_t id = PyInterpreterState_GetID(interp); + if (id < 0) { + return NULL; + } + if (requested_id == id) { + return interp; + } + interp = PyInterpreterState_Next(interp); + } + return NULL; +} + +/* Return the interpreter state with the given ID. + + Fail with RuntimeError if the interpreter is not found. */ + +PyInterpreterState * +_PyInterpreterState_LookUpID(int64_t requested_id) +{ + PyInterpreterState *interp = NULL; + if (requested_id >= 0) { + _PyRuntimeState *runtime = &_PyRuntime; + HEAD_LOCK(runtime); + interp = interp_look_up_id(runtime, requested_id); + HEAD_UNLOCK(runtime); + } + if (interp == NULL && !PyErr_Occurred()) { + PyErr_Format(PyExc_RuntimeError, + "unrecognized interpreter ID %lld", requested_id); + } + return interp; +} + + +/********************************/ +/* the per-thread runtime state */ +/********************************/ + +#ifndef NDEBUG +static inline int +tstate_is_alive(PyThreadState *tstate) +{ + return (tstate->_status.initialized && + !tstate->_status.finalized && + !tstate->_status.cleared && + !tstate->_status.finalizing); +} +#endif + + +//---------- +// lifecycle +//---------- + /* Minimum size of data stack chunk */ #define DATA_STACK_CHUNK_SIZE (16*1024) @@ -747,7 +1316,9 @@ alloc_threadstate(void) static void free_threadstate(PyThreadState *tstate) { - if (!tstate->_static) { + // The initial thread state of the interpreter is allocated + // as part of the interpreter state so should not be freed. + if (tstate != &tstate->interp->_initial_thread) { PyMem_RawFree(tstate); } } @@ -762,44 +1333,57 @@ free_threadstate(PyThreadState *tstate) static void init_threadstate(PyThreadState *tstate, - PyInterpreterState *interp, uint64_t id, - PyThreadState *next) + PyInterpreterState *interp, uint64_t id) { - if (tstate->_initialized) { + if (tstate->_status.initialized) { Py_FatalError("thread state already initialized"); } assert(interp != NULL); tstate->interp = interp; + // next/prev are set in add_threadstate(). + assert(tstate->next == NULL); + assert(tstate->prev == NULL); + assert(id > 0); tstate->id = id; - assert(interp->threads.head == tstate); - assert((next != NULL && id != 1) || (next == NULL && id == 1)); - if (next != NULL) { - assert(next->prev == NULL || next->prev == tstate); - next->prev = tstate; - } - tstate->next = next; - assert(tstate->prev == NULL); - - tstate->thread_id = PyThread_get_thread_ident(); -#ifdef PY_HAVE_THREAD_NATIVE_ID - tstate->native_thread_id = PyThread_get_thread_native_id(); -#endif + // thread_id and native_thread_id are set in bind_tstate(). - tstate->recursion_limit = interp->ceval.recursion_limit, - tstate->recursion_remaining = interp->ceval.recursion_limit, + tstate->py_recursion_limit = interp->ceval.recursion_limit, + tstate->py_recursion_remaining = interp->ceval.recursion_limit, + tstate->c_recursion_remaining = C_RECURSION_LIMIT; tstate->exc_info = &tstate->exc_state; + // PyGILState_Release must not try to delete this thread state. + // This is cleared when PyGILState_Ensure() creates the thread state. + tstate->gilstate_counter = 1; + tstate->cframe = &tstate->root_cframe; tstate->datastack_chunk = NULL; tstate->datastack_top = NULL; tstate->datastack_limit = NULL; + tstate->what_event = -1; - tstate->_initialized = 1; + tstate->_status.initialized = 1; +} + +static void +add_threadstate(PyInterpreterState *interp, PyThreadState *tstate, + PyThreadState *next) +{ + assert(interp->threads.head != tstate); + assert((next != NULL && tstate->id != 1) || + (next == NULL && tstate->id == 1)); + if (next != NULL) { + assert(next->prev == NULL || next->prev == tstate); + next->prev = tstate; + } + tstate->next = next; + assert(tstate->prev == NULL); + interp->threads.head = tstate; } static PyThreadState * @@ -829,7 +1413,6 @@ new_threadstate(PyInterpreterState *interp) assert(id == 1); used_newtstate = 0; tstate = &interp->_initial_thread; - assert(tstate->_static); } else { // Every valid interpreter must have at least one thread. @@ -841,13 +1424,10 @@ new_threadstate(PyInterpreterState *interp) memcpy(tstate, &initial._main_interpreter._initial_thread, sizeof(*tstate)); - // We need to adjust any fields that are different from the initial - // thread (as defined in _PyThreadState_INIT): - tstate->_static = false; } - interp->threads.head = tstate; - init_threadstate(tstate, interp, id, old_head); + init_threadstate(tstate, interp, id); + add_threadstate(interp, tstate, old_head); HEAD_UNLOCK(runtime); if (!used_newtstate) { @@ -861,189 +1441,99 @@ PyThreadState * PyThreadState_New(PyInterpreterState *interp) { PyThreadState *tstate = new_threadstate(interp); - _PyThreadState_SetCurrent(tstate); + if (tstate) { + bind_tstate(tstate); + // This makes sure there's a gilstate tstate bound + // as soon as possible. + if (gilstate_tss_get(tstate->interp->runtime) == NULL) { + bind_gilstate_tstate(tstate); + } + } return tstate; } +// This must be followed by a call to _PyThreadState_Bind(); PyThreadState * -_PyThreadState_Prealloc(PyInterpreterState *interp) +_PyThreadState_New(PyInterpreterState *interp) { return new_threadstate(interp); } -// We keep this around for (accidental) stable ABI compatibility. -// Realisically, no extensions are using it. -void -_PyThreadState_Init(PyThreadState *tstate) +// We keep this for stable ABI compabibility. +PyThreadState * +_PyThreadState_Prealloc(PyInterpreterState *interp) { - Py_FatalError("_PyThreadState_Init() is for internal use only"); + return _PyThreadState_New(interp); } +// We keep this around for (accidental) stable ABI compatibility. +// Realistically, no extensions are using it. void -_PyThreadState_SetCurrent(PyThreadState *tstate) -{ - // gh-104690: If Python is being finalized and PyInterpreterState_Delete() - // was called, tstate becomes a dangling pointer. - assert(_PyThreadState_CheckConsistency(tstate)); - - _PyGILState_NoteThreadState(&tstate->interp->runtime->gilstate, tstate); -} - -PyObject* -PyState_FindModule(PyModuleDef* module) -{ - Py_ssize_t index = module->m_base.m_index; - PyInterpreterState *state = _PyInterpreterState_GET(); - PyObject *res; - if (module->m_slots) { - return NULL; - } - if (index == 0) - return NULL; - if (state->modules_by_index == NULL) - return NULL; - if (index >= PyList_GET_SIZE(state->modules_by_index)) - return NULL; - res = PyList_GET_ITEM(state->modules_by_index, index); - return res==Py_None ? NULL : res; -} - -int -_PyState_AddModule(PyThreadState *tstate, PyObject* module, PyModuleDef* def) +_PyThreadState_Init(PyThreadState *tstate) { - if (!def) { - assert(_PyErr_Occurred(tstate)); - return -1; - } - if (def->m_slots) { - _PyErr_SetString(tstate, - PyExc_SystemError, - "PyState_AddModule called on module with slots"); - return -1; - } - - PyInterpreterState *interp = tstate->interp; - if (!interp->modules_by_index) { - interp->modules_by_index = PyList_New(0); - if (!interp->modules_by_index) { - return -1; - } - } - - while (PyList_GET_SIZE(interp->modules_by_index) <= def->m_base.m_index) { - if (PyList_Append(interp->modules_by_index, Py_None) < 0) { - return -1; - } - } - - Py_INCREF(module); - return PyList_SetItem(interp->modules_by_index, - def->m_base.m_index, module); + Py_FatalError("_PyThreadState_Init() is for internal use only"); } -int -PyState_AddModule(PyObject* module, PyModuleDef* def) -{ - if (!def) { - Py_FatalError("module definition is NULL"); - return -1; - } - - PyThreadState *tstate = _PyThreadState_GET(); - PyInterpreterState *interp = tstate->interp; - Py_ssize_t index = def->m_base.m_index; - if (interp->modules_by_index && - index < PyList_GET_SIZE(interp->modules_by_index) && - module == PyList_GET_ITEM(interp->modules_by_index, index)) - { - _Py_FatalErrorFormat(__func__, "module %p already added", module); - return -1; - } - return _PyState_AddModule(tstate, module, def); -} -int -PyState_RemoveModule(PyModuleDef* def) +static void +clear_datastack(PyThreadState *tstate) { - PyThreadState *tstate = _PyThreadState_GET(); - PyInterpreterState *interp = tstate->interp; - - if (def->m_slots) { - _PyErr_SetString(tstate, - PyExc_SystemError, - "PyState_RemoveModule called on module with slots"); - return -1; - } - - Py_ssize_t index = def->m_base.m_index; - if (index == 0) { - Py_FatalError("invalid module index"); - } - if (interp->modules_by_index == NULL) { - Py_FatalError("Interpreters module-list not accessible."); - } - if (index > PyList_GET_SIZE(interp->modules_by_index)) { - Py_FatalError("Module index out of bounds."); + _PyStackChunk *chunk = tstate->datastack_chunk; + tstate->datastack_chunk = NULL; + while (chunk != NULL) { + _PyStackChunk *prev = chunk->previous; + _PyObject_VirtualFree(chunk, chunk->size); + chunk = prev; } - - Py_INCREF(Py_None); - return PyList_SetItem(interp->modules_by_index, index, Py_None); } -// Used by finalize_modules() void -_PyInterpreterState_ClearModules(PyInterpreterState *interp) +PyThreadState_Clear(PyThreadState *tstate) { - if (!interp->modules_by_index) { - return; - } + assert(tstate->_status.initialized && !tstate->_status.cleared); + // XXX assert(!tstate->_status.bound || tstate->_status.unbound); + tstate->_status.finalizing = 1; // just in case - Py_ssize_t i; - for (i = 0; i < PyList_GET_SIZE(interp->modules_by_index); i++) { - PyObject *m = PyList_GET_ITEM(interp->modules_by_index, i); - if (PyModule_Check(m)) { - /* cleanup the saved copy of module dicts */ - PyModuleDef *md = PyModule_GetDef(m); - if (md) { - Py_CLEAR(md->m_base.m_copy); - } - } - } + /* XXX Conditions we need to enforce: - /* Setting modules_by_index to NULL could be dangerous, so we - clear the list instead. */ - if (PyList_SetSlice(interp->modules_by_index, - 0, PyList_GET_SIZE(interp->modules_by_index), - NULL)) { - PyErr_WriteUnraisable(interp->modules_by_index); - } -} + * the GIL must be held by the current thread + * current_fast_get()->interp must match tstate->interp + * for the main interpreter, current_fast_get() must be the main thread + */ -void -PyThreadState_Clear(PyThreadState *tstate) -{ int verbose = _PyInterpreterState_GetConfig(tstate->interp)->verbose; if (verbose && tstate->cframe->current_frame != NULL) { /* bpo-20526: After the main thread calls - _PyRuntimeState_SetFinalizing() in Py_FinalizeEx(), threads must - exit when trying to take the GIL. If a thread exit in the middle of - _PyEval_EvalFrameDefault(), tstate->frame is not reset to its - previous value. It is more likely with daemon threads, but it can - happen with regular threads if threading._shutdown() fails + _PyInterpreterState_SetFinalizing() in Py_FinalizeEx() + (or in Py_EndInterpreter() for subinterpreters), + threads must exit when trying to take the GIL. + If a thread exit in the middle of _PyEval_EvalFrameDefault(), + tstate->frame is not reset to its previous value. + It is more likely with daemon threads, but it can happen + with regular threads if threading._shutdown() fails (ex: interrupted by CTRL+C). */ fprintf(stderr, "PyThreadState_Clear: warning: thread still has a frame\n"); } + /* At this point tstate shouldn't be used any more, + neither to run Python code nor for other uses. + + This is tricky when current_fast_get() == tstate, in the same way + as noted in interpreter_clear() above. The below finalizers + can possibly run Python code or otherwise use the partially + cleared thread state. For now we trust that isn't a problem + in practice. + */ + // XXX Deal with the possibility of problematic finalizers. + /* Don't clear tstate->pyframe: it is a borrowed reference */ Py_CLEAR(tstate->dict); Py_CLEAR(tstate->async_exc); - Py_CLEAR(tstate->curexc_type); - Py_CLEAR(tstate->curexc_value); - Py_CLEAR(tstate->curexc_traceback); + Py_CLEAR(tstate->current_exception); Py_CLEAR(tstate->exc_state.exc_value); @@ -1053,8 +1543,14 @@ PyThreadState_Clear(PyThreadState *tstate) "PyThreadState_Clear: warning: thread still has a generator\n"); } - tstate->c_profilefunc = NULL; - tstate->c_tracefunc = NULL; + if (tstate->c_profilefunc != NULL) { + tstate->interp->sys_profiling_threads--; + tstate->c_profilefunc = NULL; + } + if (tstate->c_tracefunc != NULL) { + tstate->interp->sys_tracing_threads--; + tstate->c_tracefunc = NULL; + } Py_CLEAR(tstate->c_profileobj); Py_CLEAR(tstate->c_traceobj); @@ -1066,15 +1562,19 @@ PyThreadState_Clear(PyThreadState *tstate) if (tstate->on_delete != NULL) { tstate->on_delete(tstate->on_delete_data); } -} + tstate->_status.cleared = 1; + + // XXX Call _PyThreadStateSwap(runtime, NULL) here if "current". + // XXX Do it as early in the function as possible. +} /* Common code for PyThreadState_Delete() and PyThreadState_DeleteCurrent() */ static void -tstate_delete_common(PyThreadState *tstate, - struct _gilstate_runtime_state *gilstate) +tstate_delete_common(PyThreadState *tstate) { - _Py_EnsureTstateNotNULL(tstate); + assert(tstate->_status.cleared && !tstate->_status.finalized); + PyInterpreterState *interp = tstate->interp; if (interp == NULL) { Py_FatalError("NULL interpreter"); @@ -1093,38 +1593,40 @@ tstate_delete_common(PyThreadState *tstate, } HEAD_UNLOCK(runtime); - if (gilstate->autoInterpreterState && - PyThread_tss_get(&gilstate->autoTSSkey) == tstate) - { - PyThread_tss_set(&gilstate->autoTSSkey, NULL); - } - _PyStackChunk *chunk = tstate->datastack_chunk; - tstate->datastack_chunk = NULL; - while (chunk != NULL) { - _PyStackChunk *prev = chunk->previous; - _PyObject_VirtualFree(chunk, chunk->size); - chunk = prev; + // XXX Unbind in PyThreadState_Clear(), or earlier + // (and assert not-equal here)? + if (tstate->_status.bound_gilstate) { + unbind_gilstate_tstate(tstate); } + unbind_tstate(tstate); + + // XXX Move to PyThreadState_Clear()? + clear_datastack(tstate); + + tstate->_status.finalized = 1; } static void -_PyThreadState_Delete(PyThreadState *tstate, int check_current) +zapthreads(PyInterpreterState *interp) { - struct _gilstate_runtime_state *gilstate = &tstate->interp->runtime->gilstate; - if (check_current) { - if (tstate == _PyRuntimeGILState_GetThreadState(gilstate)) { - _Py_FatalErrorFormat(__func__, "tstate %p is still current", tstate); - } + PyThreadState *tstate; + /* No need to lock the mutex here because this should only happen + when the threads are all really dead (XXX famous last words). */ + while ((tstate = interp->threads.head) != NULL) { + tstate_verify_not_active(tstate); + tstate_delete_common(tstate); + free_threadstate(tstate); } - tstate_delete_common(tstate, gilstate); - free_threadstate(tstate); } void PyThreadState_Delete(PyThreadState *tstate) { - _PyThreadState_Delete(tstate, 1); + _Py_EnsureTstateNotNULL(tstate); + tstate_verify_not_active(tstate); + tstate_delete_common(tstate); + free_threadstate(tstate); } @@ -1132,18 +1634,16 @@ void _PyThreadState_DeleteCurrent(PyThreadState *tstate) { _Py_EnsureTstateNotNULL(tstate); - struct _gilstate_runtime_state *gilstate = &tstate->interp->runtime->gilstate; - tstate_delete_common(tstate, gilstate); - _PyRuntimeGILState_SetThreadState(gilstate, NULL); - _PyEval_ReleaseLock(tstate); + tstate_delete_common(tstate); + current_fast_clear(tstate->interp->runtime); + _PyEval_ReleaseLock(tstate->interp, NULL); free_threadstate(tstate); } void PyThreadState_DeleteCurrent(void) { - struct _gilstate_runtime_state *gilstate = &_PyRuntime.gilstate; - PyThreadState *tstate = _PyRuntimeGILState_GetThreadState(gilstate); + PyThreadState *tstate = current_fast_get(&_PyRuntime); _PyThreadState_DeleteCurrent(tstate); } @@ -1156,9 +1656,11 @@ PyThreadState_DeleteCurrent(void) * be kept in those other interpreters. */ void -_PyThreadState_DeleteExcept(_PyRuntimeState *runtime, PyThreadState *tstate) +_PyThreadState_DeleteExcept(PyThreadState *tstate) { + assert(tstate != NULL); PyInterpreterState *interp = tstate->interp; + _PyRuntimeState *runtime = interp->runtime; HEAD_LOCK(runtime); /* Remove all thread states, except tstate, from the linked list of @@ -1190,52 +1692,9 @@ _PyThreadState_DeleteExcept(_PyRuntimeState *runtime, PyThreadState *tstate) } -PyThreadState * -_PyThreadState_UncheckedGet(void) -{ - return _PyThreadState_GET(); -} - - -PyThreadState * -PyThreadState_Get(void) -{ - PyThreadState *tstate = _PyThreadState_GET(); - _Py_EnsureTstateNotNULL(tstate); - return tstate; -} - - -PyThreadState * -_PyThreadState_Swap(struct _gilstate_runtime_state *gilstate, PyThreadState *newts) -{ - PyThreadState *oldts = _PyRuntimeGILState_GetThreadState(gilstate); - - _PyRuntimeGILState_SetThreadState(gilstate, newts); - /* It should not be possible for more than one thread state - to be used for a thread. Check this the best we can in debug - builds. - */ -#if defined(Py_DEBUG) - if (newts) { - /* This can be called from PyEval_RestoreThread(). Similar - to it, we need to ensure errno doesn't change. - */ - int err = errno; - PyThreadState *check = _PyGILState_GetThisThreadState(gilstate); - if (check && check->interp == newts->interp && check != newts) - Py_FatalError("Invalid thread state for this thread"); - errno = err; - } -#endif - return oldts; -} - -PyThreadState * -PyThreadState_Swap(PyThreadState *newts) -{ - return _PyThreadState_Swap(&_PyRuntime.gilstate, newts); -} +//---------- +// accessors +//---------- /* An extension mechanism to store arbitrary additional per-thread state. PyThreadState_GetDict() returns a dictionary that can be used to hold such @@ -1260,7 +1719,7 @@ _PyThreadState_GetDict(PyThreadState *tstate) PyObject * PyThreadState_GetDict(void) { - PyThreadState *tstate = _PyThreadState_GET(); + PyThreadState *tstate = current_fast_get(&_PyRuntime); if (tstate == NULL) { return NULL; } @@ -1280,10 +1739,7 @@ PyFrameObject* PyThreadState_GetFrame(PyThreadState *tstate) { assert(tstate != NULL); - _PyInterpreterFrame *f = tstate->cframe->current_frame; - while (f && _PyFrame_IsIncomplete(f)) { - f = f->previous; - } + _PyInterpreterFrame *f = _PyThreadState_GetFrame(tstate); if (f == NULL) { return NULL; } @@ -1291,8 +1747,7 @@ PyThreadState_GetFrame(PyThreadState *tstate) if (frame == NULL) { PyErr_Clear(); } - Py_XINCREF(frame); - return frame; + return (PyFrameObject*)Py_XNewRef(frame); } @@ -1304,6 +1759,42 @@ PyThreadState_GetID(PyThreadState *tstate) } +static inline void +tstate_activate(PyThreadState *tstate) +{ + assert(tstate != NULL); + // XXX assert(tstate_is_alive(tstate)); + assert(tstate_is_bound(tstate)); + assert(!tstate->_status.active); + + assert(!tstate->_status.bound_gilstate || + tstate == gilstate_tss_get((tstate->interp->runtime))); + if (!tstate->_status.bound_gilstate) { + bind_gilstate_tstate(tstate); + } + + tstate->_status.active = 1; +} + +static inline void +tstate_deactivate(PyThreadState *tstate) +{ + assert(tstate != NULL); + // XXX assert(tstate_is_alive(tstate)); + assert(tstate_is_bound(tstate)); + assert(tstate->_status.active); + + tstate->_status.active = 0; + + // We do not unbind the gilstate tstate here. + // It will still be used in PyGILState_Ensure(). +} + + +//---------- +// other API +//---------- + /* Asynchronously raise an exception in a thread. Requested by Just van Rossum and Alex Martelli. To prevent naive misuse, you must write your own extension @@ -1312,11 +1803,13 @@ PyThreadState_GetID(PyThreadState *tstate) match any known thread id). Can be called with exc=NULL to clear an existing async exception. This raises no exceptions. */ +// XXX Move this to Python/ceval_gil.c? +// XXX Deprecate this. int PyThreadState_SetAsyncExc(unsigned long id, PyObject *exc) { _PyRuntimeState *runtime = &_PyRuntime; - PyInterpreterState *interp = _PyRuntimeState_GetThreadState(runtime)->interp; + PyInterpreterState *interp = _PyInterpreterState_GET(); /* Although the GIL is held, a few C API functions can be called * without the GIL held, and in particular some that create and @@ -1338,8 +1831,7 @@ PyThreadState_SetAsyncExc(unsigned long id, PyObject *exc) * the decref. */ PyObject *old_exc = tstate->async_exc; - Py_XINCREF(exc); - tstate->async_exc = exc; + tstate->async_exc = Py_XNewRef(exc); HEAD_UNLOCK(runtime); Py_XDECREF(old_exc); @@ -1350,8 +1842,109 @@ PyThreadState_SetAsyncExc(unsigned long id, PyObject *exc) return 0; } -/* Routines for advanced debuggers, requested by David Beazley. - Don't use unless you know what you are doing! */ + +//--------------------------------- +// API for the current thread state +//--------------------------------- + +PyThreadState * +_PyThreadState_UncheckedGet(void) +{ + return current_fast_get(&_PyRuntime); +} + + +PyThreadState * +PyThreadState_Get(void) +{ + PyThreadState *tstate = current_fast_get(&_PyRuntime); + _Py_EnsureTstateNotNULL(tstate); + return tstate; +} + + +static void +_swap_thread_states(_PyRuntimeState *runtime, + PyThreadState *oldts, PyThreadState *newts) +{ + // XXX Do this only if oldts != NULL? + current_fast_clear(runtime); + + if (oldts != NULL) { + // XXX assert(tstate_is_alive(oldts) && tstate_is_bound(oldts)); + tstate_deactivate(oldts); + } + + if (newts != NULL) { + // XXX assert(tstate_is_alive(newts)); + assert(tstate_is_bound(newts)); + current_fast_set(runtime, newts); + tstate_activate(newts); + } +} + +PyThreadState * +_PyThreadState_SwapNoGIL(PyThreadState *newts) +{ +#if defined(Py_DEBUG) + /* This can be called from PyEval_RestoreThread(). Similar + to it, we need to ensure errno doesn't change. + */ + int err = errno; +#endif + + PyThreadState *oldts = current_fast_get(&_PyRuntime); + _swap_thread_states(&_PyRuntime, oldts, newts); + +#if defined(Py_DEBUG) + errno = err; +#endif + return oldts; +} + +PyThreadState * +_PyThreadState_Swap(_PyRuntimeState *runtime, PyThreadState *newts) +{ + PyThreadState *oldts = current_fast_get(runtime); + if (oldts != NULL) { + _PyEval_ReleaseLock(oldts->interp, oldts); + } + _swap_thread_states(runtime, oldts, newts); + if (newts != NULL) { + _PyEval_AcquireLock(newts); + } + return oldts; +} + +PyThreadState * +PyThreadState_Swap(PyThreadState *newts) +{ + return _PyThreadState_Swap(&_PyRuntime, newts); +} + + +void +_PyThreadState_Bind(PyThreadState *tstate) +{ + // gh-104690: If Python is being finalized and PyInterpreterState_Delete() + // was called, tstate becomes a dangling pointer. + assert(_PyThreadState_CheckConsistency(tstate)); + + bind_tstate(tstate); + // This makes sure there's a gilstate tstate bound + // as soon as possible. + if (gilstate_tss_get(tstate->interp->runtime) == NULL) { + bind_gilstate_tstate(tstate); + } +} + + +/***********************************/ +/* routines for advanced debuggers */ +/***********************************/ + +// (requested by David Beazley) +// Don't use unless you know what you are doing! PyInterpreterState * PyInterpreterState_Head(void) @@ -1380,6 +1973,11 @@ PyThreadState_Next(PyThreadState *tstate) { return tstate->next; } + +/********************************************/ +/* reporting execution state of all threads */ +/********************************************/ + /* The implementation of sys._current_frames(). This is intended to be called with the GIL held, as it will be when called via sys._current_frames(). It's possible it would work fine even without @@ -1388,7 +1986,8 @@ PyThreadState_Next(PyThreadState *tstate) { PyObject * _PyThread_CurrentFrames(void) { - PyThreadState *tstate = _PyThreadState_GET(); + _PyRuntimeState *runtime = &_PyRuntime; + PyThreadState *tstate = current_fast_get(runtime); if (_PySys_Audit(tstate, "sys._current_frames", NULL) < 0) { return NULL; } @@ -1404,16 +2003,13 @@ _PyThread_CurrentFrames(void) * Because these lists can mutate even when the GIL is held, we * need to grab head_mutex for the duration. */ - _PyRuntimeState *runtime = tstate->interp->runtime; HEAD_LOCK(runtime); PyInterpreterState *i; for (i = runtime->interpreters.head; i != NULL; i = i->next) { PyThreadState *t; for (t = i->threads.head; t != NULL; t = t->next) { _PyInterpreterFrame *frame = t->cframe->current_frame; - while (frame && _PyFrame_IsIncomplete(frame)) { - frame = frame->previous; - } + frame = _PyFrame_GetFirstComplete(frame); if (frame == NULL) { continue; } @@ -1443,10 +2039,16 @@ done: return result; } +/* The implementation of sys._current_exceptions(). This is intended to be + called with the GIL held, as it will be when called via + sys._current_exceptions(). It's possible it would work fine even without + the GIL held, but haven't thought enough about that. +*/ PyObject * _PyThread_CurrentExceptions(void) { - PyThreadState *tstate = _PyThreadState_GET(); + _PyRuntimeState *runtime = &_PyRuntime; + PyThreadState *tstate = current_fast_get(runtime); _Py_EnsureTstateNotNULL(tstate); @@ -1465,7 +2067,6 @@ _PyThread_CurrentExceptions(void) * Because these lists can mutate even when the GIL is held, we * need to grab head_mutex for the duration. */ - _PyRuntimeState *runtime = tstate->interp->runtime; HEAD_LOCK(runtime); PyInterpreterState *i; for (i = runtime->interpreters.head; i != NULL; i = i->next) { @@ -1479,14 +2080,13 @@ _PyThread_CurrentExceptions(void) if (id == NULL) { goto fail; } - PyObject *exc_info = _PyErr_StackItemToExcInfoTuple(err_info); - if (exc_info == NULL) { - Py_DECREF(id); - goto fail; - } - int stat = PyDict_SetItem(result, id, exc_info); + PyObject *exc = err_info->exc_value; + assert(exc == NULL || + exc == Py_None || + PyExceptionInstance_Check(exc)); + + int stat = PyDict_SetItem(result, id, exc == NULL ? Py_None : exc); Py_DECREF(id); - Py_DECREF(exc_info); if (stat < 0) { goto fail; } @@ -1502,62 +2102,63 @@ done: return result; } -/* Python "auto thread state" API. */ -/* Keep this as a static, as it is not reliable! It can only - ever be compared to the state for the *current* thread. - * If not equal, then it doesn't matter that the actual - value may change immediately after comparison, as it can't - possibly change to the current thread's state. - * If equal, then the current thread holds the lock, so the value can't - change until we yield the lock. -*/ -static int -PyThreadState_IsCurrent(PyThreadState *tstate) -{ - /* Must be the tstate for this thread */ - struct _gilstate_runtime_state *gilstate = &_PyRuntime.gilstate; - assert(_PyGILState_GetThisThreadState(gilstate) == tstate); - return tstate == _PyRuntimeGILState_GetThreadState(gilstate); -} +/***********************************/ +/* Python "auto thread state" API. */ +/***********************************/ /* Internal initialization/finalization functions called by Py_Initialize/Py_FinalizeEx */ PyStatus -_PyGILState_Init(_PyRuntimeState *runtime) +_PyGILState_Init(PyInterpreterState *interp) { - struct _gilstate_runtime_state *gilstate = &runtime->gilstate; - if (PyThread_tss_create(&gilstate->autoTSSkey) != 0) { - return _PyStatus_NO_MEMORY(); + if (!_Py_IsMainInterpreter(interp)) { + /* Currently, PyGILState is shared by all interpreters. The main + * interpreter is responsible to initialize it. */ + return _PyStatus_OK(); } - // PyThreadState_New() calls _PyGILState_NoteThreadState() which does - // nothing before autoInterpreterState is set. - assert(gilstate->autoInterpreterState == NULL); + _PyRuntimeState *runtime = interp->runtime; + assert(gilstate_tss_get(runtime) == NULL); + assert(runtime->gilstate.autoInterpreterState == NULL); + runtime->gilstate.autoInterpreterState = interp; return _PyStatus_OK(); } +void +_PyGILState_Fini(PyInterpreterState *interp) +{ + if (!_Py_IsMainInterpreter(interp)) { + /* Currently, PyGILState is shared by all interpreters. The main + * interpreter is responsible to initialize it. */ + return; + } + interp->runtime->gilstate.autoInterpreterState = NULL; +} + +// XXX Drop this. PyStatus _PyGILState_SetTstate(PyThreadState *tstate) { + /* must init with valid states */ + assert(tstate != NULL); + assert(tstate->interp != NULL); + if (!_Py_IsMainInterpreter(tstate->interp)) { /* Currently, PyGILState is shared by all interpreters. The main * interpreter is responsible to initialize it. */ return _PyStatus_OK(); } - /* must init with valid states */ - assert(tstate != NULL); - assert(tstate->interp != NULL); - - struct _gilstate_runtime_state *gilstate = &tstate->interp->runtime->gilstate; +#ifndef NDEBUG + _PyRuntimeState *runtime = tstate->interp->runtime; - gilstate->autoInterpreterState = tstate->interp; - assert(PyThread_tss_get(&gilstate->autoTSSkey) == NULL); - assert(tstate->gilstate_counter == 0); + assert(runtime->gilstate.autoInterpreterState == tstate->interp); + assert(gilstate_tss_get(runtime) == tstate); + assert(tstate->gilstate_counter == 1); +#endif - _PyGILState_NoteThreadState(gilstate, tstate); return _PyStatus_OK(); } @@ -1567,118 +2168,42 @@ _PyGILState_GetInterpreterStateUnsafe(void) return _PyRuntime.gilstate.autoInterpreterState; } -void -_PyGILState_Fini(PyInterpreterState *interp) -{ - struct _gilstate_runtime_state *gilstate = &interp->runtime->gilstate; - PyThread_tss_delete(&gilstate->autoTSSkey); - gilstate->autoInterpreterState = NULL; -} - -#ifdef HAVE_FORK -/* Reset the TSS key - called by PyOS_AfterFork_Child(). - * This should not be necessary, but some - buggy - pthread implementations - * don't reset TSS upon fork(), see issue #10517. - */ -PyStatus -_PyGILState_Reinit(_PyRuntimeState *runtime) -{ - struct _gilstate_runtime_state *gilstate = &runtime->gilstate; - PyThreadState *tstate = _PyGILState_GetThisThreadState(gilstate); - - PyThread_tss_delete(&gilstate->autoTSSkey); - if (PyThread_tss_create(&gilstate->autoTSSkey) != 0) { - return _PyStatus_NO_MEMORY(); - } - - /* If the thread had an associated auto thread state, reassociate it with - * the new key. */ - if (tstate && - PyThread_tss_set(&gilstate->autoTSSkey, (void *)tstate) != 0) - { - return _PyStatus_ERR("failed to set autoTSSkey"); - } - return _PyStatus_OK(); -} -#endif - -/* When a thread state is created for a thread by some mechanism other than - PyGILState_Ensure, it's important that the GILState machinery knows about - it so it doesn't try to create another thread state for the thread (this is - a better fix for SF bug #1010677 than the first one attempted). -*/ -static void -_PyGILState_NoteThreadState(struct _gilstate_runtime_state *gilstate, PyThreadState* tstate) -{ - /* If autoTSSkey isn't initialized, this must be the very first - threadstate created in Py_Initialize(). Don't do anything for now - (we'll be back here when _PyGILState_Init is called). */ - if (!gilstate->autoInterpreterState) { - return; - } - - /* Stick the thread state for this thread in thread specific storage. - - The only situation where you can legitimately have more than one - thread state for an OS level thread is when there are multiple - interpreters. - - You shouldn't really be using the PyGILState_ APIs anyway (see issues - #10915 and #15751). - - The first thread state created for that given OS level thread will - "win", which seems reasonable behaviour. - */ - if (PyThread_tss_get(&gilstate->autoTSSkey) == NULL) { - if ((PyThread_tss_set(&gilstate->autoTSSkey, (void *)tstate)) != 0) { - Py_FatalError("Couldn't create autoTSSkey mapping"); - } - } - - /* PyGILState_Release must not try to delete this thread state. */ - tstate->gilstate_counter = 1; -} - /* The public functions */ -static PyThreadState * -_PyGILState_GetThisThreadState(struct _gilstate_runtime_state *gilstate) -{ - if (gilstate->autoInterpreterState == NULL) - return NULL; - return (PyThreadState *)PyThread_tss_get(&gilstate->autoTSSkey); -} PyThreadState * PyGILState_GetThisThreadState(void) { - return _PyGILState_GetThisThreadState(&_PyRuntime.gilstate); + _PyRuntimeState *runtime = &_PyRuntime; + if (!gilstate_tss_initialized(runtime)) { + return NULL; + } + return gilstate_tss_get(runtime); } int PyGILState_Check(void) { - struct _gilstate_runtime_state *gilstate = &_PyRuntime.gilstate; - if (!gilstate->check_enabled) { + _PyRuntimeState *runtime = &_PyRuntime; + if (!runtime->gilstate.check_enabled) { return 1; } - if (!PyThread_tss_is_created(&gilstate->autoTSSkey)) { + if (!gilstate_tss_initialized(runtime)) { return 1; } - PyThreadState *tstate = _PyRuntimeGILState_GetThreadState(gilstate); + PyThreadState *tstate = current_fast_get(runtime); if (tstate == NULL) { return 0; } - return (tstate == _PyGILState_GetThisThreadState(gilstate)); + return (tstate == gilstate_tss_get(runtime)); } PyGILState_STATE PyGILState_Ensure(void) { _PyRuntimeState *runtime = &_PyRuntime; - struct _gilstate_runtime_state *gilstate = &runtime->gilstate; /* Note that we do not auto-init Python here - apart from potential races with 2 threads auto-initializing, pep-311 @@ -1687,28 +2212,32 @@ PyGILState_Ensure(void) /* Ensure that _PyEval_InitThreads() and _PyGILState_Init() have been called by Py_Initialize() */ - assert(_PyEval_ThreadsInitialized(runtime)); - assert(gilstate->autoInterpreterState); + assert(_PyEval_ThreadsInitialized()); + assert(gilstate_tss_initialized(runtime)); + assert(runtime->gilstate.autoInterpreterState != NULL); - PyThreadState *tcur = (PyThreadState *)PyThread_tss_get(&gilstate->autoTSSkey); - int current; + PyThreadState *tcur = gilstate_tss_get(runtime); + int has_gil; if (tcur == NULL) { /* Create a new Python thread state for this thread */ - tcur = PyThreadState_New(gilstate->autoInterpreterState); + tcur = new_threadstate(runtime->gilstate.autoInterpreterState); if (tcur == NULL) { Py_FatalError("Couldn't create thread-state for new thread"); } + bind_tstate(tcur); + bind_gilstate_tstate(tcur); /* This is our thread state! We'll need to delete it in the matching call to PyGILState_Release(). */ + assert(tcur->gilstate_counter == 1); tcur->gilstate_counter = 0; - current = 0; /* new thread state is never current */ + has_gil = 0; /* new thread state is never current */ } else { - current = PyThreadState_IsCurrent(tcur); + has_gil = holds_gil(tcur); } - if (current == 0) { + if (!has_gil) { PyEval_RestoreThread(tcur); } @@ -1719,14 +2248,14 @@ PyGILState_Ensure(void) */ ++tcur->gilstate_counter; - return current ? PyGILState_LOCKED : PyGILState_UNLOCKED; + return has_gil ? PyGILState_LOCKED : PyGILState_UNLOCKED; } void PyGILState_Release(PyGILState_STATE oldstate) { _PyRuntimeState *runtime = &_PyRuntime; - PyThreadState *tstate = PyThread_tss_get(&runtime->gilstate.autoTSSkey); + PyThreadState *tstate = gilstate_tss_get(runtime); if (tstate == NULL) { Py_FatalError("auto-releasing thread-state, " "but no thread-state for this thread"); @@ -1737,12 +2266,12 @@ PyGILState_Release(PyGILState_STATE oldstate) but while this is very new (April 2003), the extra check by release-only users can't hurt. */ - if (!PyThreadState_IsCurrent(tstate)) { + if (!holds_gil(tstate)) { _Py_FatalErrorFormat(__func__, "thread state %p must be current when releasing", tstate); } - assert(PyThreadState_IsCurrent(tstate)); + assert(holds_gil(tstate)); --tstate->gilstate_counter; assert(tstate->gilstate_counter >= 0); /* illegal counter value */ @@ -1752,18 +2281,20 @@ PyGILState_Release(PyGILState_STATE oldstate) if (tstate->gilstate_counter == 0) { /* can't have been locked when we created it */ assert(oldstate == PyGILState_UNLOCKED); + // XXX Unbind tstate here. PyThreadState_Clear(tstate); /* Delete the thread-state. Note this releases the GIL too! * It's vital that the GIL be held here, to avoid shutdown * races; see bugs 225673 and 1061968 (that nasty bug has a * habit of coming back). */ - assert(_PyRuntimeGILState_GetThreadState(&runtime->gilstate) == tstate); + assert(current_fast_get(runtime) == tstate); _PyThreadState_DeleteCurrent(tstate); } /* Release the lock if necessary */ - else if (oldstate == PyGILState_UNLOCKED) + else if (oldstate == PyGILState_UNLOCKED) { PyEval_SaveThread(); + } } @@ -1773,30 +2304,84 @@ PyGILState_Release(PyGILState_STATE oldstate) /* cross-interpreter data */ -crossinterpdatafunc _PyCrossInterpreterData_Lookup(PyObject *); +static inline void +_xidata_init(_PyCrossInterpreterData *data) +{ + // If the value is being reused + // then _xidata_clear() should have been called already. + assert(data->data == NULL); + assert(data->obj == NULL); + *data = (_PyCrossInterpreterData){0}; + data->interp = -1; +} -/* This is a separate func from _PyCrossInterpreterData_Lookup in order - to keep the registry code separate. */ -static crossinterpdatafunc -_lookup_getdata(PyObject *obj) +static inline void +_xidata_clear(_PyCrossInterpreterData *data) { - crossinterpdatafunc getdata = _PyCrossInterpreterData_Lookup(obj); - if (getdata == NULL && PyErr_Occurred() == 0) - PyErr_Format(PyExc_ValueError, - "%S does not support cross-interpreter data", obj); - return getdata; + // _PyCrossInterpreterData only has two members that need to be + // cleaned up, if set: "data" must be freed and "obj" must be decref'ed. + // In both cases the original (owning) interpreter must be used, + // which is the caller's responsibility to ensure. + if (data->data != NULL) { + if (data->free != NULL) { + data->free(data->data); + } + data->data = NULL; + } + Py_CLEAR(data->obj); +} + +void +_PyCrossInterpreterData_Init(_PyCrossInterpreterData *data, + PyInterpreterState *interp, + void *shared, PyObject *obj, + xid_newobjectfunc new_object) +{ + assert(data != NULL); + assert(new_object != NULL); + _xidata_init(data); + data->data = shared; + if (obj != NULL) { + assert(interp != NULL); + // released in _PyCrossInterpreterData_Clear() + data->obj = Py_NewRef(obj); + } + // Ideally every object would know its owning interpreter. + // Until then, we have to rely on the caller to identify it + // (but we don't need it in all cases). + data->interp = (interp != NULL) ? interp->id : -1; + data->new_object = new_object; } int -_PyObject_CheckCrossInterpreterData(PyObject *obj) -{ - crossinterpdatafunc getdata = _lookup_getdata(obj); - if (getdata == NULL) { +_PyCrossInterpreterData_InitWithSize(_PyCrossInterpreterData *data, + PyInterpreterState *interp, + const size_t size, PyObject *obj, + xid_newobjectfunc new_object) +{ + assert(size > 0); + // For now we always free the shared data in the same interpreter + // where it was allocated, so the interpreter is required. + assert(interp != NULL); + _PyCrossInterpreterData_Init(data, interp, NULL, obj, new_object); + data->data = PyMem_RawMalloc(size); + if (data->data == NULL) { return -1; } + data->free = PyMem_RawFree; return 0; } +void +_PyCrossInterpreterData_Clear(PyInterpreterState *interp, + _PyCrossInterpreterData *data) +{ + assert(data != NULL); + // This must be called in the owning interpreter. + assert(interp == NULL || data->interp == interp->id); + _xidata_clear(data); +} + static int _check_xidata(PyThreadState *tstate, _PyCrossInterpreterData *data) { @@ -1819,10 +2404,35 @@ _check_xidata(PyThreadState *tstate, _PyCrossInterpreterData *data) return 0; } +crossinterpdatafunc _PyCrossInterpreterData_Lookup(PyObject *); + +/* This is a separate func from _PyCrossInterpreterData_Lookup in order + to keep the registry code separate. */ +static crossinterpdatafunc +_lookup_getdata(PyObject *obj) +{ + crossinterpdatafunc getdata = _PyCrossInterpreterData_Lookup(obj); + if (getdata == NULL && PyErr_Occurred() == 0) + PyErr_Format(PyExc_ValueError, + "%S does not support cross-interpreter data", obj); + return getdata; +} + +int +_PyObject_CheckCrossInterpreterData(PyObject *obj) +{ + crossinterpdatafunc getdata = _lookup_getdata(obj); + if (getdata == NULL) { + return -1; + } + return 0; +} + int _PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data) { - PyThreadState *tstate = _PyThreadState_GET(); + _PyRuntimeState *runtime = &_PyRuntime; + PyThreadState *tstate = current_fast_get(runtime); #ifdef Py_DEBUG // The caller must hold the GIL _Py_EnsureTstateNotNULL(tstate); @@ -1831,7 +2441,7 @@ _PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data) // Reset data before re-populating. *data = (_PyCrossInterpreterData){0}; - data->free = PyMem_RawFree; // Set a default that may be overridden. + data->interp = -1; // Call the "getdata" func for the object. Py_INCREF(obj); @@ -1840,7 +2450,7 @@ _PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data) Py_DECREF(obj); return -1; } - int res = getdata(obj, data); + int res = getdata(tstate, obj, data); Py_DECREF(obj); if (res != 0) { return -1; @@ -1849,75 +2459,89 @@ _PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data) // Fill in the blanks and validate the result. data->interp = interp->id; if (_check_xidata(tstate, data) != 0) { - _PyCrossInterpreterData_Release(data); + (void)_PyCrossInterpreterData_Release(data); return -1; } return 0; } -static void -_release_xidata(void *arg) +PyObject * +_PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *data) { - _PyCrossInterpreterData *data = (_PyCrossInterpreterData *)arg; - if (data->free != NULL) { - data->free(data->data); - } - Py_XDECREF(data->obj); + return data->new_object(data); } -static void -_call_in_interpreter(struct _gilstate_runtime_state *gilstate, - PyInterpreterState *interp, - void (*func)(void *), void *arg) +static int +_release_xidata_pending(void *data) { - /* We would use Py_AddPendingCall() if it weren't specific to the - * main interpreter (see bpo-33608). In the meantime we take a - * naive approach. - */ - PyThreadState *save_tstate = NULL; - if (interp != _PyRuntimeGILState_GetThreadState(gilstate)->interp) { - // XXX Using the "head" thread isn't strictly correct. - PyThreadState *tstate = PyInterpreterState_ThreadHead(interp); - // XXX Possible GILState issues? - save_tstate = _PyThreadState_Swap(gilstate, tstate); - } - - func(arg); + _xidata_clear((_PyCrossInterpreterData *)data); + return 0; +} - // Switch back. - if (save_tstate != NULL) { - _PyThreadState_Swap(gilstate, save_tstate); - } +static int +_xidata_release_and_rawfree_pending(void *data) +{ + _xidata_clear((_PyCrossInterpreterData *)data); + PyMem_RawFree(data); + return 0; } -void -_PyCrossInterpreterData_Release(_PyCrossInterpreterData *data) +static int +_xidata_release(_PyCrossInterpreterData *data, int rawfree) { - if (data->data == NULL && data->obj == NULL) { + if ((data->data == NULL || data->free == NULL) && data->obj == NULL) { // Nothing to release! - return; + if (rawfree) { + PyMem_RawFree(data); + } + else { + data->data = NULL; + } + return 0; } // Switch to the original interpreter. PyInterpreterState *interp = _PyInterpreterState_LookUpID(data->interp); if (interp == NULL) { // The interpreter was already destroyed. - if (data->free != NULL) { - // XXX Someone leaked some memory... + // This function shouldn't have been called. + // XXX Someone leaked some memory... + assert(PyErr_Occurred()); + if (rawfree) { + PyMem_RawFree(data); } - return; + return -1; } // "Release" the data and/or the object. - struct _gilstate_runtime_state *gilstate = &_PyRuntime.gilstate; - _call_in_interpreter(gilstate, interp, _release_xidata, data); + if (interp == current_fast_get(interp->runtime)->interp) { + _xidata_clear(data); + if (rawfree) { + PyMem_RawFree(data); + } + } + else { + int (*func)(void *) = _release_xidata_pending; + if (rawfree) { + func = _xidata_release_and_rawfree_pending; + } + // XXX Emit a warning if this fails? + _PyEval_AddPendingCall(interp, func, data, 0); + } + return 0; } -PyObject * -_PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *data) +int +_PyCrossInterpreterData_Release(_PyCrossInterpreterData *data) { - return data->new_object(data); + return _xidata_release(data, 0); +} + +int +_PyCrossInterpreterData_ReleaseAndRawFree(_PyCrossInterpreterData *data) +{ + return _xidata_release(data, 1); } /* registry of {type -> crossinterpdatafunc} */ @@ -1927,26 +2551,123 @@ _PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *data) crossinterpdatafunc. It would be simpler and more efficient. */ static int -_register_xidata(struct _xidregistry *xidregistry, PyTypeObject *cls, - crossinterpdatafunc getdata) +_xidregistry_add_type(struct _xidregistry *xidregistry, + PyTypeObject *cls, crossinterpdatafunc getdata) { - // Note that we effectively replace already registered classes - // rather than failing. struct _xidregitem *newhead = PyMem_RawMalloc(sizeof(struct _xidregitem)); - if (newhead == NULL) + if (newhead == NULL) { return -1; - newhead->cls = cls; - newhead->getdata = getdata; + } + *newhead = (struct _xidregitem){ + // We do not keep a reference, to avoid keeping the class alive. + .cls = cls, + .refcount = 1, + .getdata = getdata, + }; + if (cls->tp_flags & Py_TPFLAGS_HEAPTYPE) { + // XXX Assign a callback to clear the entry from the registry? + newhead->weakref = PyWeakref_NewRef((PyObject *)cls, NULL); + if (newhead->weakref == NULL) { + PyMem_RawFree(newhead); + return -1; + } + } newhead->next = xidregistry->head; + if (newhead->next != NULL) { + newhead->next->prev = newhead; + } xidregistry->head = newhead; return 0; } +static struct _xidregitem * +_xidregistry_remove_entry(struct _xidregistry *xidregistry, + struct _xidregitem *entry) +{ + struct _xidregitem *next = entry->next; + if (entry->prev != NULL) { + assert(entry->prev->next == entry); + entry->prev->next = next; + } + else { + assert(xidregistry->head == entry); + xidregistry->head = next; + } + if (next != NULL) { + next->prev = entry->prev; + } + Py_XDECREF(entry->weakref); + PyMem_RawFree(entry); + return next; +} + +static void +_xidregistry_clear(struct _xidregistry *xidregistry) +{ + struct _xidregitem *cur = xidregistry->head; + xidregistry->head = NULL; + while (cur != NULL) { + struct _xidregitem *next = cur->next; + Py_XDECREF(cur->weakref); + PyMem_RawFree(cur); + cur = next; + } +} + +static struct _xidregitem * +_xidregistry_find_type(struct _xidregistry *xidregistry, PyTypeObject *cls) +{ + struct _xidregitem *cur = xidregistry->head; + while (cur != NULL) { + if (cur->weakref != NULL) { + // cur is/was a heap type. + PyObject *registered = PyWeakref_GetObject(cur->weakref); + assert(registered != NULL); + if (registered == Py_None) { + // The weakly ref'ed object was freed. + cur = _xidregistry_remove_entry(xidregistry, cur); + continue; + } + assert(PyType_Check(registered)); + assert(cur->cls == (PyTypeObject *)registered); + assert(cur->cls->tp_flags & Py_TPFLAGS_HEAPTYPE); + //Py_DECREF(registered); + } + if (cur->cls == cls) { + return cur; + } + cur = cur->next; + } + return NULL; +} + +static inline struct _xidregistry * +_get_xidregistry(PyInterpreterState *interp, PyTypeObject *cls) +{ + struct _xidregistry *xidregistry = &interp->runtime->xidregistry; + if (cls->tp_flags & Py_TPFLAGS_HEAPTYPE) { + assert(interp->xidregistry.mutex == xidregistry->mutex); + xidregistry = &interp->xidregistry; + } + return xidregistry; +} + static void _register_builtins_for_crossinterpreter_data(struct _xidregistry *xidregistry); +static inline void +_ensure_builtins_xid(PyInterpreterState *interp, struct _xidregistry *xidregistry) +{ + if (xidregistry != &interp->xidregistry) { + assert(xidregistry == &interp->runtime->xidregistry); + if (xidregistry->head == NULL) { + _register_builtins_for_crossinterpreter_data(xidregistry); + } + } +} + int _PyCrossInterpreterData_RegisterClass(PyTypeObject *cls, - crossinterpdatafunc getdata) + crossinterpdatafunc getdata) { if (!PyType_Check(cls)) { PyErr_Format(PyExc_ValueError, "only classes may be registered"); @@ -1957,19 +2678,50 @@ _PyCrossInterpreterData_RegisterClass(PyTypeObject *cls, return -1; } - // Make sure the class isn't ever deallocated. - Py_INCREF((PyObject *)cls); + int res = 0; + PyInterpreterState *interp = _PyInterpreterState_GET(); + struct _xidregistry *xidregistry = _get_xidregistry(interp, cls); + PyThread_acquire_lock(xidregistry->mutex, WAIT_LOCK); + + _ensure_builtins_xid(interp, xidregistry); + + struct _xidregitem *matched = _xidregistry_find_type(xidregistry, cls); + if (matched != NULL) { + assert(matched->getdata == getdata); + matched->refcount += 1; + goto finally; + } + + res = _xidregistry_add_type(xidregistry, cls, getdata); + +finally: + PyThread_release_lock(xidregistry->mutex); + return res; +} - struct _xidregistry *xidregistry = &_PyRuntime.xidregistry ; +int +_PyCrossInterpreterData_UnregisterClass(PyTypeObject *cls) +{ + int res = 0; + PyInterpreterState *interp = _PyInterpreterState_GET(); + struct _xidregistry *xidregistry = _get_xidregistry(interp, cls); PyThread_acquire_lock(xidregistry->mutex, WAIT_LOCK); - if (xidregistry->head == NULL) { - _register_builtins_for_crossinterpreter_data(xidregistry); + + struct _xidregitem *matched = _xidregistry_find_type(xidregistry, cls); + if (matched != NULL) { + assert(matched->refcount > 0); + matched->refcount -= 1; + if (matched->refcount == 0) { + (void)_xidregistry_remove_entry(xidregistry, matched); + } + res = 1; } - int res = _register_xidata(xidregistry, cls, getdata); + PyThread_release_lock(xidregistry->mutex); return res; } + /* Cross-interpreter objects are looked up by exact match on the class. We can reassess this policy when we move from a global registry to a tp_* slot. */ @@ -1977,24 +2729,19 @@ _PyCrossInterpreterData_RegisterClass(PyTypeObject *cls, crossinterpdatafunc _PyCrossInterpreterData_Lookup(PyObject *obj) { - struct _xidregistry *xidregistry = &_PyRuntime.xidregistry ; - PyObject *cls = PyObject_Type(obj); - crossinterpdatafunc getdata = NULL; + PyTypeObject *cls = Py_TYPE(obj); + + PyInterpreterState *interp = _PyInterpreterState_GET(); + struct _xidregistry *xidregistry = _get_xidregistry(interp, cls); PyThread_acquire_lock(xidregistry->mutex, WAIT_LOCK); - struct _xidregitem *cur = xidregistry->head; - if (cur == NULL) { - _register_builtins_for_crossinterpreter_data(xidregistry); - cur = xidregistry->head; - } - for(; cur != NULL; cur = cur->next) { - if (cur->cls == (PyTypeObject *)cls) { - getdata = cur->getdata; - break; - } - } - Py_DECREF(cls); + + _ensure_builtins_xid(interp, xidregistry); + + struct _xidregitem *matched = _xidregistry_find_type(xidregistry, cls); + crossinterpdatafunc func = matched != NULL ? matched->getdata : NULL; + PyThread_release_lock(xidregistry->mutex); - return getdata; + return func; } /* cross-interpreter data for builtin types */ @@ -2012,17 +2759,21 @@ _new_bytes_object(_PyCrossInterpreterData *data) } static int -_bytes_shared(PyObject *obj, _PyCrossInterpreterData *data) +_bytes_shared(PyThreadState *tstate, PyObject *obj, + _PyCrossInterpreterData *data) { - struct _shared_bytes_data *shared = PyMem_NEW(struct _shared_bytes_data, 1); + if (_PyCrossInterpreterData_InitWithSize( + data, tstate->interp, sizeof(struct _shared_bytes_data), obj, + _new_bytes_object + ) < 0) + { + return -1; + } + struct _shared_bytes_data *shared = (struct _shared_bytes_data *)data->data; if (PyBytes_AsStringAndSize(obj, &shared->bytes, &shared->len) < 0) { + _PyCrossInterpreterData_Clear(tstate->interp, data); return -1; } - data->data = (void *)shared; - Py_INCREF(obj); - data->obj = obj; // Will be "released" (decref'ed) when data released. - data->new_object = _new_bytes_object; - data->free = PyMem_Free; return 0; } @@ -2040,17 +2791,20 @@ _new_str_object(_PyCrossInterpreterData *data) } static int -_str_shared(PyObject *obj, _PyCrossInterpreterData *data) +_str_shared(PyThreadState *tstate, PyObject *obj, + _PyCrossInterpreterData *data) { - struct _shared_str_data *shared = PyMem_NEW(struct _shared_str_data, 1); + if (_PyCrossInterpreterData_InitWithSize( + data, tstate->interp, sizeof(struct _shared_str_data), obj, + _new_str_object + ) < 0) + { + return -1; + } + struct _shared_str_data *shared = (struct _shared_str_data *)data->data; shared->kind = PyUnicode_KIND(obj); shared->buffer = PyUnicode_DATA(obj); shared->len = PyUnicode_GET_LENGTH(obj); - data->data = (void *)shared; - Py_INCREF(obj); - data->obj = obj; // Will be "released" (decref'ed) when data released. - data->new_object = _new_str_object; - data->free = PyMem_Free; return 0; } @@ -2061,7 +2815,8 @@ _new_long_object(_PyCrossInterpreterData *data) } static int -_long_shared(PyObject *obj, _PyCrossInterpreterData *data) +_long_shared(PyThreadState *tstate, PyObject *obj, + _PyCrossInterpreterData *data) { /* Note that this means the size of shareable ints is bounded by * sys.maxsize. Hence on 32-bit architectures that is half the @@ -2074,10 +2829,9 @@ _long_shared(PyObject *obj, _PyCrossInterpreterData *data) } return -1; } - data->data = (void *)value; - data->obj = NULL; - data->new_object = _new_long_object; - data->free = NULL; + _PyCrossInterpreterData_Init(data, tstate->interp, (void *)value, NULL, + _new_long_object); + // data->obj and data->free remain NULL return 0; } @@ -2085,17 +2839,16 @@ static PyObject * _new_none_object(_PyCrossInterpreterData *data) { // XXX Singleton refcounts are problematic across interpreters... - Py_INCREF(Py_None); - return Py_None; + return Py_NewRef(Py_None); } static int -_none_shared(PyObject *obj, _PyCrossInterpreterData *data) +_none_shared(PyThreadState *tstate, PyObject *obj, + _PyCrossInterpreterData *data) { - data->data = NULL; - // data->obj remains NULL - data->new_object = _new_none_object; - data->free = NULL; // There is nothing to free. + _PyCrossInterpreterData_Init(data, tstate->interp, NULL, NULL, + _new_none_object); + // data->data, data->obj and data->free remain NULL return 0; } @@ -2103,27 +2856,31 @@ static void _register_builtins_for_crossinterpreter_data(struct _xidregistry *xidregistry) { // None - if (_register_xidata(xidregistry, (PyTypeObject *)PyObject_Type(Py_None), _none_shared) != 0) { + if (_xidregistry_add_type(xidregistry, (PyTypeObject *)PyObject_Type(Py_None), _none_shared) != 0) { Py_FatalError("could not register None for cross-interpreter sharing"); } // int - if (_register_xidata(xidregistry, &PyLong_Type, _long_shared) != 0) { + if (_xidregistry_add_type(xidregistry, &PyLong_Type, _long_shared) != 0) { Py_FatalError("could not register int for cross-interpreter sharing"); } // bytes - if (_register_xidata(xidregistry, &PyBytes_Type, _bytes_shared) != 0) { + if (_xidregistry_add_type(xidregistry, &PyBytes_Type, _bytes_shared) != 0) { Py_FatalError("could not register bytes for cross-interpreter sharing"); } // str - if (_register_xidata(xidregistry, &PyUnicode_Type, _str_shared) != 0) { + if (_xidregistry_add_type(xidregistry, &PyUnicode_Type, _str_shared) != 0) { Py_FatalError("could not register str for cross-interpreter sharing"); } } +/*************/ +/* Other API */ +/*************/ + _PyFrameEvalFunction _PyInterpreterState_GetEvalFrameFunc(PyInterpreterState *interp) { @@ -2171,11 +2928,21 @@ _PyInterpreterState_GetConfigCopy(PyConfig *config) const PyConfig* _Py_GetConfig(void) { + _PyRuntimeState *runtime = &_PyRuntime; assert(PyGILState_Check()); - PyThreadState *tstate = _PyThreadState_GET(); + PyThreadState *tstate = current_fast_get(runtime); + _Py_EnsureTstateNotNULL(tstate); return _PyInterpreterState_GetConfig(tstate->interp); } + +int +_PyInterpreterState_HasFeature(PyInterpreterState *interp, unsigned long feature) +{ + return ((interp->feature_flags & feature) != 0); +} + + #define MINIMUM_OVERHEAD 1000 static PyObject ** @@ -2204,17 +2971,14 @@ push_chunk(PyThreadState *tstate, int size) } _PyInterpreterFrame * -_PyThreadState_BumpFramePointerSlow(PyThreadState *tstate, size_t size) +_PyThreadState_PushFrame(PyThreadState *tstate, size_t size) { - if (_PyThreadState_HasStackSpace(tstate, size)) { + assert(size < INT_MAX/sizeof(PyObject *)); + if (_PyThreadState_HasStackSpace(tstate, (int)size)) { _PyInterpreterFrame *res = (_PyInterpreterFrame *)tstate->datastack_top; tstate->datastack_top += size; return res; } - if (size > INT_MAX/2) { - PyErr_NoMemory(); - return NULL; - } return (_PyInterpreterFrame *)push_chunk(tstate, (int)size); } @@ -2276,8 +3040,26 @@ _PyThreadState_MustExit(PyThreadState *tstate) tstate->interp->runtime to support calls from Python daemon threads. After Py_Finalize() has been called, tstate can be a dangling pointer: point to PyThreadState freed memory. */ + unsigned long finalizing_id = _PyRuntimeState_GetFinalizingID(&_PyRuntime); PyThreadState *finalizing = _PyRuntimeState_GetFinalizing(&_PyRuntime); - return (finalizing != NULL && finalizing != tstate); + if (finalizing == NULL) { + // XXX This isn't completely safe from daemon thraeds, + // since tstate might be a dangling pointer. + finalizing = _PyInterpreterState_GetFinalizing(tstate->interp); + finalizing_id = _PyInterpreterState_GetFinalizingID(tstate->interp); + } + // XXX else check &_PyRuntime._main_interpreter._initial_thread + if (finalizing == NULL) { + return 0; + } + else if (finalizing == tstate) { + return 0; + } + else if (finalizing_id == PyThread_get_thread_ident()) { + /* gh-109793: we must have switched interpreters. */ + return 0; + } + return 1; } |