diff options
Diffstat (limited to 'contrib/tools/python3/Objects/object.c')
| -rw-r--r-- | contrib/tools/python3/Objects/object.c | 1171 |
1 files changed, 770 insertions, 401 deletions
diff --git a/contrib/tools/python3/Objects/object.c b/contrib/tools/python3/Objects/object.c index 706719116da..c6d5e163528 100644 --- a/contrib/tools/python3/Objects/object.c +++ b/contrib/tools/python3/Objects/object.c @@ -2,32 +2,36 @@ /* Generic object operations; and implementation of None */ #include "Python.h" +#include "pycore_brc.h" // _Py_brc_queue_object() #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _Py_EnterRecursiveCallTstate() #include "pycore_context.h" // _PyContextTokenMissing_Type +#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION, Py_END_CRITICAL_SECTION +#include "pycore_descrobject.h" // _PyMethodWrapper_Type #include "pycore_dict.h" // _PyObject_MakeDictFromInstanceAttributes() #include "pycore_floatobject.h" // _PyFloat_DebugMallocStats() #include "pycore_initconfig.h" // _PyStatus_EXCEPTION() +#include "pycore_instruction_sequence.h" // _PyInstructionSequence_Type +#include "pycore_hashtable.h" // _Py_hashtable_new() +#include "pycore_memoryobject.h" // _PyManagedBuffer_Type #include "pycore_namespace.h" // _PyNamespace_Type -#include "pycore_object.h" // _PyType_CheckConsistency(), _Py_FatalRefcountError() +#include "pycore_object.h" // PyAPI_DATA() _Py_SwappedOp definition +#include "pycore_long.h" // _PyLong_GetZero() +#include "pycore_optimizer.h" // _PyUOpExecutor_Type, _PyUOpOptimizer_Type, ... #include "pycore_pyerrors.h" // _PyErr_Occurred() #include "pycore_pymem.h" // _PyMem_IsPtrFreed() #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_symtable.h" // PySTEntry_Type -#include "pycore_typevarobject.h" // _PyTypeAlias_Type, _Py_initialize_generic #include "pycore_typeobject.h" // _PyBufferWrapper_Type +#include "pycore_typevarobject.h" // _PyTypeAlias_Type, _Py_initialize_generic #include "pycore_unionobject.h" // _PyUnion_Type -#include "interpreteridobject.h" // _PyInterpreterID_Type + #ifdef Py_LIMITED_API // Prevent recursive call _Py_IncRef() <=> Py_INCREF() # error "Py_LIMITED_API macro must not be defined" #endif -#ifdef __cplusplus -extern "C" { -#endif - /* Defined in tracemalloc.c */ extern void _PyMem_DumpTraceback(int fd, const void *ptr); @@ -72,21 +76,16 @@ get_legacy_reftotal(void) interp->object_state.reftotal static inline void -reftotal_increment(PyInterpreterState *interp) -{ - REFTOTAL(interp)++; -} - -static inline void -reftotal_decrement(PyInterpreterState *interp) +reftotal_add(PyThreadState *tstate, Py_ssize_t n) { - REFTOTAL(interp)--; -} - -static inline void -reftotal_add(PyInterpreterState *interp, Py_ssize_t n) -{ - REFTOTAL(interp) += n; +#ifdef Py_GIL_DISABLED + _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; + // relaxed store to avoid data race with read in get_reftotal() + Py_ssize_t reftotal = tstate_impl->reftotal + n; + _Py_atomic_store_ssize_relaxed(&tstate_impl->reftotal, reftotal); +#else + REFTOTAL(tstate->interp) += n; +#endif } static inline Py_ssize_t get_global_reftotal(_PyRuntimeState *); @@ -116,7 +115,15 @@ get_reftotal(PyInterpreterState *interp) { /* For a single interpreter, we ignore the legacy _Py_RefTotal, since we can't determine which interpreter updated it. */ - return REFTOTAL(interp); + Py_ssize_t total = REFTOTAL(interp); +#ifdef Py_GIL_DISABLED + for (PyThreadState *p = interp->threads.head; p != NULL; p = p->next) { + /* This may race with other threads modifications to their reftotal */ + _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)p; + total += _Py_atomic_load_ssize_relaxed(&tstate_impl->reftotal); + } +#endif + return total; } static inline Py_ssize_t @@ -128,7 +135,7 @@ get_global_reftotal(_PyRuntimeState *runtime) HEAD_LOCK(&_PyRuntime); PyInterpreterState *interp = PyInterpreterState_Head(); for (; interp != NULL; interp = PyInterpreterState_Next(interp)) { - total += REFTOTAL(interp); + total += get_reftotal(interp); } HEAD_UNLOCK(&_PyRuntime); @@ -160,6 +167,7 @@ _PyDebug_PrintTotalRefs(void) { #ifdef Py_TRACE_REFS #define REFCHAIN(interp) interp->object_state.refchain +#define REFCHAIN_VALUE ((void*)(uintptr_t)1) static inline int has_own_refchain(PyInterpreterState *interp) @@ -171,48 +179,80 @@ has_own_refchain(PyInterpreterState *interp) return 1; } -static inline void -init_refchain(PyInterpreterState *interp) +static int +refchain_init(PyInterpreterState *interp) { if (!has_own_refchain(interp)) { // Legacy subinterpreters share a refchain with the main interpreter. REFCHAIN(interp) = REFCHAIN(_PyInterpreterState_Main()); - return; + return 0; } - REFCHAIN(interp) = &interp->object_state._refchain_obj; - PyObject *refchain = REFCHAIN(interp); - refchain->_ob_prev = refchain; - refchain->_ob_next = refchain; + _Py_hashtable_allocator_t alloc = { + // Don't use default PyMem_Malloc() and PyMem_Free() which + // require the caller to hold the GIL. + .malloc = PyMem_RawMalloc, + .free = PyMem_RawFree, + }; + REFCHAIN(interp) = _Py_hashtable_new_full( + _Py_hashtable_hash_ptr, _Py_hashtable_compare_direct, + NULL, NULL, &alloc); + if (REFCHAIN(interp) == NULL) { + return -1; + } + return 0; } -/* Insert op at the front of the list of all objects. If force is true, - * op is added even if _ob_prev and _ob_next are non-NULL already. If - * force is false amd _ob_prev or _ob_next are non-NULL, do nothing. - * force should be true if and only if op points to freshly allocated, - * uninitialized memory, or you've unlinked op from the list and are - * relinking it into the front. - * Note that objects are normally added to the list via _Py_NewReference, - * which is called by PyObject_Init. Not all objects are initialized that - * way, though; exceptions include statically allocated type objects, and - * statically allocated singletons (like Py_True and Py_None). - */ -void -_Py_AddToAllObjects(PyObject *op, int force) +static void +refchain_fini(PyInterpreterState *interp) { -#ifdef Py_DEBUG - if (!force) { - /* If it's initialized memory, op must be in or out of - * the list unambiguously. - */ - _PyObject_ASSERT(op, (op->_ob_prev == NULL) == (op->_ob_next == NULL)); + if (has_own_refchain(interp) && REFCHAIN(interp) != NULL) { + _Py_hashtable_destroy(REFCHAIN(interp)); } + REFCHAIN(interp) = NULL; +} + +bool +_PyRefchain_IsTraced(PyInterpreterState *interp, PyObject *obj) +{ + return (_Py_hashtable_get(REFCHAIN(interp), obj) == REFCHAIN_VALUE); +} + + +static void +_PyRefchain_Trace(PyInterpreterState *interp, PyObject *obj) +{ + if (_Py_hashtable_set(REFCHAIN(interp), obj, REFCHAIN_VALUE) < 0) { + // Use a fatal error because _Py_NewReference() cannot report + // the error to the caller. + Py_FatalError("_Py_hashtable_set() memory allocation failed"); + } +} + + +static void +_PyRefchain_Remove(PyInterpreterState *interp, PyObject *obj) +{ + void *value = _Py_hashtable_steal(REFCHAIN(interp), obj); +#ifndef NDEBUG + assert(value == REFCHAIN_VALUE); +#else + (void)value; #endif - if (force || op->_ob_prev == NULL) { - PyObject *refchain = REFCHAIN(_PyInterpreterState_GET()); - op->_ob_next = refchain->_ob_next; - op->_ob_prev = refchain; - refchain->_ob_next->_ob_prev = op; - refchain->_ob_next = op; +} + + +/* Add an object to the refchain hash table. + * + * Note that objects are normally added to the list by PyObject_Init() + * indirectly. Not all objects are initialized that way, though; exceptions + * include statically allocated type objects, and statically allocated + * singletons (like Py_True and Py_None). */ +void +_Py_AddToAllObjects(PyObject *op) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (!_PyRefchain_IsTraced(interp, op)) { + _PyRefchain_Trace(interp, op); } } #endif /* Py_TRACE_REFS */ @@ -230,32 +270,32 @@ _Py_NegativeRefcount(const char *filename, int lineno, PyObject *op) void _Py_INCREF_IncRefTotal(void) { - reftotal_increment(_PyInterpreterState_GET()); + reftotal_add(_PyThreadState_GET(), 1); } /* This is used strictly by Py_DECREF(). */ void _Py_DECREF_DecRefTotal(void) { - reftotal_decrement(_PyInterpreterState_GET()); + reftotal_add(_PyThreadState_GET(), -1); } void -_Py_IncRefTotal(PyInterpreterState *interp) +_Py_IncRefTotal(PyThreadState *tstate) { - reftotal_increment(interp); + reftotal_add(tstate, 1); } void -_Py_DecRefTotal(PyInterpreterState *interp) +_Py_DecRefTotal(PyThreadState *tstate) { - reftotal_decrement(interp); + reftotal_add(tstate, -1); } void -_Py_AddRefTotal(PyInterpreterState *interp, Py_ssize_t n) +_Py_AddRefTotal(PyThreadState *tstate, Py_ssize_t n) { - reftotal_add(interp, n); + reftotal_add(tstate, n); } /* This includes the legacy total @@ -275,7 +315,10 @@ _Py_GetLegacyRefTotal(void) Py_ssize_t _PyInterpreterState_GetRefTotal(PyInterpreterState *interp) { - return get_reftotal(interp); + HEAD_LOCK(&_PyRuntime); + Py_ssize_t total = get_reftotal(interp); + HEAD_UNLOCK(&_PyRuntime); + return total; } #endif /* Py_REF_DEBUG */ @@ -304,6 +347,162 @@ _Py_DecRef(PyObject *o) Py_DECREF(o); } +#ifdef Py_GIL_DISABLED +# ifdef Py_REF_DEBUG +static int +is_dead(PyObject *o) +{ +# if SIZEOF_SIZE_T == 8 + return (uintptr_t)o->ob_type == 0xDDDDDDDDDDDDDDDD; +# else + return (uintptr_t)o->ob_type == 0xDDDDDDDD; +# endif +} +# endif + +// Decrement the shared reference count of an object. Return 1 if the object +// is dead and should be deallocated, 0 otherwise. +static int +_Py_DecRefSharedIsDead(PyObject *o, const char *filename, int lineno) +{ + // Should we queue the object for the owning thread to merge? + int should_queue; + + Py_ssize_t new_shared; + Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&o->ob_ref_shared); + do { + should_queue = (shared == 0 || shared == _Py_REF_MAYBE_WEAKREF); + + if (should_queue) { + // If the object had refcount zero, not queued, and not merged, + // then we enqueue the object to be merged by the owning thread. + // In this case, we don't subtract one from the reference count + // because the queue holds a reference. + new_shared = _Py_REF_QUEUED; + } + else { + // Otherwise, subtract one from the reference count. This might + // be negative! + new_shared = shared - (1 << _Py_REF_SHARED_SHIFT); + } + +#ifdef Py_REF_DEBUG + if ((new_shared < 0 && _Py_REF_IS_MERGED(new_shared)) || + (should_queue && is_dead(o))) + { + _Py_NegativeRefcount(filename, lineno, o); + } +#endif + } while (!_Py_atomic_compare_exchange_ssize(&o->ob_ref_shared, + &shared, new_shared)); + + if (should_queue) { +#ifdef Py_REF_DEBUG + _Py_IncRefTotal(_PyThreadState_GET()); +#endif + _Py_brc_queue_object(o); + } + else if (new_shared == _Py_REF_MERGED) { + // refcount is zero AND merged + return 1; + } + return 0; +} + +void +_Py_DecRefSharedDebug(PyObject *o, const char *filename, int lineno) +{ + if (_Py_DecRefSharedIsDead(o, filename, lineno)) { + _Py_Dealloc(o); + } +} + +void +_Py_DecRefShared(PyObject *o) +{ + _Py_DecRefSharedDebug(o, NULL, 0); +} + +void +_Py_MergeZeroLocalRefcount(PyObject *op) +{ + assert(op->ob_ref_local == 0); + + Py_ssize_t shared = _Py_atomic_load_ssize_acquire(&op->ob_ref_shared); + if (shared == 0) { + // Fast-path: shared refcount is zero (including flags) + _Py_Dealloc(op); + return; + } + + // gh-121794: This must be before the store to `ob_ref_shared` (gh-119999), + // but should outside the fast-path to maintain the invariant that + // a zero `ob_tid` implies a merged refcount. + _Py_atomic_store_uintptr_relaxed(&op->ob_tid, 0); + + // Slow-path: atomically set the flags (low two bits) to _Py_REF_MERGED. + Py_ssize_t new_shared; + do { + new_shared = (shared & ~_Py_REF_SHARED_FLAG_MASK) | _Py_REF_MERGED; + } while (!_Py_atomic_compare_exchange_ssize(&op->ob_ref_shared, + &shared, new_shared)); + + if (new_shared == _Py_REF_MERGED) { + // i.e., the shared refcount is zero (only the flags are set) so we + // deallocate the object. + _Py_Dealloc(op); + } +} + +Py_ssize_t +_Py_ExplicitMergeRefcount(PyObject *op, Py_ssize_t extra) +{ + assert(!_Py_IsImmortal(op)); + +#ifdef Py_REF_DEBUG + _Py_AddRefTotal(_PyThreadState_GET(), extra); +#endif + + // gh-119999: Write to ob_ref_local and ob_tid before merging the refcount. + Py_ssize_t local = (Py_ssize_t)op->ob_ref_local; + _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, 0); + _Py_atomic_store_uintptr_relaxed(&op->ob_tid, 0); + + Py_ssize_t refcnt; + Py_ssize_t new_shared; + Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared); + do { + refcnt = Py_ARITHMETIC_RIGHT_SHIFT(Py_ssize_t, shared, _Py_REF_SHARED_SHIFT); + refcnt += local; + refcnt += extra; + + new_shared = _Py_REF_SHARED(refcnt, _Py_REF_MERGED); + } while (!_Py_atomic_compare_exchange_ssize(&op->ob_ref_shared, + &shared, new_shared)); + return refcnt; +} + +// The more complicated "slow" path for undoing the resurrection of an object. +int +_PyObject_ResurrectEndSlow(PyObject *op) +{ + if (_Py_IsImmortal(op)) { + return 1; + } + if (_Py_IsOwnedByCurrentThread(op)) { + // If the object is owned by the current thread, give up ownership and + // merge the refcount. This isn't necessary in all cases, but it + // simplifies the implementation. + Py_ssize_t refcount = _Py_ExplicitMergeRefcount(op, -1); + return refcount != 0; + } + int is_dead = _Py_DecRefSharedIsDead(op, NULL, 0); + return !is_dead; +} + + +#endif /* Py_GIL_DISABLED */ + /**************************************/ @@ -380,7 +579,7 @@ PyObject_CallFinalizerFromDealloc(PyObject *self) } /* Temporarily resurrect the object. */ - Py_SET_REFCNT(self, 1); + _PyObject_ResurrectStart(self); PyObject_CallFinalizer(self); @@ -390,16 +589,13 @@ PyObject_CallFinalizerFromDealloc(PyObject *self) /* Undo the temporary resurrection; can't use DECREF here, it would * cause a recursive call. */ - Py_SET_REFCNT(self, Py_REFCNT(self) - 1); - if (Py_REFCNT(self) == 0) { + if (!_PyObject_ResurrectEnd(self)) { return 0; /* this is the normal path out */ } /* tp_finalize resurrected it! Make it look like the original Py_DECREF * never happened. */ - Py_ssize_t refcnt = Py_REFCNT(self); - _Py_NewReferenceNoTotal(self); - Py_SET_REFCNT(self, refcnt); + _Py_ResurrectReference(self); _PyObject_ASSERT(self, (!_PyType_IS_GC(Py_TYPE(self)) @@ -411,6 +607,7 @@ int PyObject_Print(PyObject *op, FILE *fp, int flags) { int ret = 0; + int write_error = 0; if (PyErr_CheckSignals()) return -1; #ifdef USE_STACKCHECK @@ -449,14 +646,20 @@ PyObject_Print(PyObject *op, FILE *fp, int flags) ret = -1; } else { - fwrite(t, 1, len, fp); + /* Versions of Android and OpenBSD from before 2023 fail to + set the `ferror` indicator when writing to a read-only + stream, so we need to check the return value. + (https://github.com/openbsd/src/commit/fc99cf9338942ecd9adc94ea08bf6188f0428c15) */ + if (fwrite(t, 1, len, fp) != (size_t)len) { + write_error = 1; + } } Py_DECREF(s); } } } if (ret == 0) { - if (ferror(fp)) { + if (write_error || ferror(fp)) { PyErr_SetFromErrno(PyExc_OSError); clearerr(fp); ret = -1; @@ -484,21 +687,11 @@ _PyObject_IsFreed(PyObject *op) if (_PyMem_IsPtrFreed(op) || _PyMem_IsPtrFreed(Py_TYPE(op))) { return 1; } - /* ignore op->ob_ref: its value can have be modified - by Py_INCREF() and Py_DECREF(). */ -#ifdef Py_TRACE_REFS - if (op->_ob_next != NULL && _PyMem_IsPtrFreed(op->_ob_next)) { - return 1; - } - if (op->_ob_prev != NULL && _PyMem_IsPtrFreed(op->_ob_prev)) { - return 1; - } -#endif return 0; } -/* For debugging convenience. See Misc/gdbinit for some useful gdb hooks */ +/* For debugging convenience. */ void _PyObject_Dump(PyObject* op) { @@ -582,11 +775,6 @@ PyObject_Repr(PyObject *v) Py_DECREF(res); return NULL; } -#ifndef Py_DEBUG - if (PyUnicode_READY(res) < 0) { - return NULL; - } -#endif return res; } @@ -605,10 +793,6 @@ PyObject_Str(PyObject *v) if (v == NULL) return PyUnicode_FromString("<NULL>"); if (PyUnicode_CheckExact(v)) { -#ifndef Py_DEBUG - if (PyUnicode_READY(v) < 0) - return NULL; -#endif return Py_NewRef(v); } if (Py_TYPE(v)->tp_str == NULL) @@ -640,11 +824,6 @@ PyObject_Str(PyObject *v) Py_DECREF(res); return NULL; } -#ifndef Py_DEBUG - if (PyUnicode_READY(res) < 0) { - return NULL; - } -#endif assert(_PyUnicode_CheckConsistency(res, 1)); return res; } @@ -708,6 +887,21 @@ PyObject_Bytes(PyObject *v) return PyBytes_FromObject(v); } +void +_PyObject_ClearFreeLists(struct _Py_object_freelists *freelists, int is_finalization) +{ + // In the free-threaded build, freelists are per-PyThreadState and cleared in PyThreadState_Clear() + // In the default build, freelists are per-interpreter and cleared in finalize_interp_types() + _PyFloat_ClearFreeList(freelists, is_finalization); + _PyTuple_ClearFreeList(freelists, is_finalization); + _PyList_ClearFreeList(freelists, is_finalization); + _PyDict_ClearFreeList(freelists, is_finalization); + _PyContext_ClearFreeList(freelists, is_finalization); + _PyAsyncGen_ClearFreeLists(freelists, is_finalization); + // Only be cleared if is_finalization is true. + _PyObjectStackChunk_ClearFreeList(freelists, is_finalization); + _PySlice_ClearFreeList(freelists, is_finalization); +} /* def _PyObject_FunctionStr(x): @@ -728,7 +922,7 @@ _PyObject_FunctionStr(PyObject *x) { assert(!PyErr_Occurred()); PyObject *qualname; - int ret = _PyObject_LookupAttr(x, &_Py_ID(__qualname__), &qualname); + int ret = PyObject_GetOptionalAttr(x, &_Py_ID(__qualname__), &qualname); if (qualname == NULL) { if (ret < 0) { return NULL; @@ -737,7 +931,7 @@ _PyObject_FunctionStr(PyObject *x) } PyObject *module; PyObject *result = NULL; - ret = _PyObject_LookupAttr(x, &_Py_ID(__module__), &module); + ret = PyObject_GetOptionalAttr(x, &_Py_ID(__module__), &module); if (module != NULL && module != Py_None) { ret = PyObject_RichCompareBool(module, &_Py_ID(builtins), Py_NE); if (ret < 0) { @@ -940,26 +1134,27 @@ PyObject_GetAttrString(PyObject *v, const char *name) } int -PyObject_HasAttrString(PyObject *v, const char *name) +PyObject_HasAttrStringWithError(PyObject *obj, const char *name) { - if (Py_TYPE(v)->tp_getattr != NULL) { - PyObject *res = (*Py_TYPE(v)->tp_getattr)(v, (char*)name); - if (res != NULL) { - Py_DECREF(res); - return 1; - } - PyErr_Clear(); - return 0; - } + PyObject *res; + int rc = PyObject_GetOptionalAttrString(obj, name, &res); + Py_XDECREF(res); + return rc; +} - PyObject *attr_name = PyUnicode_FromString(name); - if (attr_name == NULL) { - PyErr_Clear(); + +int +PyObject_HasAttrString(PyObject *obj, const char *name) +{ + int rc = PyObject_HasAttrStringWithError(obj, name); + if (rc < 0) { + PyErr_FormatUnraisable( + "Exception ignored in PyObject_HasAttrString(); consider using " + "PyObject_HasAttrStringWithError(), " + "PyObject_GetOptionalAttrString() or PyObject_GetAttrString()"); return 0; } - int ok = PyObject_HasAttr(v, attr_name); - Py_DECREF(attr_name); - return ok; + return rc; } int @@ -979,6 +1174,12 @@ PyObject_SetAttrString(PyObject *v, const char *name, PyObject *w) } int +PyObject_DelAttrString(PyObject *v, const char *name) +{ + return PyObject_SetAttrString(v, name, NULL); +} + +int _PyObject_IsAbstract(PyObject *obj) { int res; @@ -987,7 +1188,7 @@ _PyObject_IsAbstract(PyObject *obj) if (obj == NULL) return 0; - res = _PyObject_LookupAttr(obj, &_Py_ID(__isabstractmethod__), &isabstract); + res = PyObject_GetOptionalAttr(obj, &_Py_ID(__isabstractmethod__), &isabstract); if (res > 0) { res = PyObject_IsTrue(isabstract); Py_DECREF(isabstract); @@ -1017,8 +1218,8 @@ _PyObject_SetAttrId(PyObject *v, _Py_Identifier *name, PyObject *w) return result; } -static inline int -set_attribute_error_context(PyObject* v, PyObject* name) +int +_PyObject_SetAttributeErrorContext(PyObject* v, PyObject* name) { assert(PyErr_Occurred()); if (!PyErr_ExceptionMatches(PyExc_AttributeError)){ @@ -1073,13 +1274,13 @@ PyObject_GetAttr(PyObject *v, PyObject *name) } if (result == NULL) { - set_attribute_error_context(v, name); + _PyObject_SetAttributeErrorContext(v, name); } return result; } int -_PyObject_LookupAttr(PyObject *v, PyObject *name, PyObject **result) +PyObject_GetOptionalAttr(PyObject *v, PyObject *name, PyObject **result) { PyTypeObject *tp = Py_TYPE(v); @@ -1101,7 +1302,7 @@ _PyObject_LookupAttr(PyObject *v, PyObject *name, PyObject **result) } return 0; } - if (tp->tp_getattro == (getattrofunc)_Py_type_getattro) { + if (tp->tp_getattro == _Py_type_getattro) { int supress_missing_attribute_exception = 0; *result = _Py_type_getattro_impl((PyTypeObject*)v, name, &supress_missing_attribute_exception); if (supress_missing_attribute_exception) { @@ -1147,29 +1348,51 @@ _PyObject_LookupAttr(PyObject *v, PyObject *name, PyObject **result) } int -_PyObject_LookupAttrId(PyObject *v, _Py_Identifier *name, PyObject **result) +PyObject_GetOptionalAttrString(PyObject *obj, const char *name, PyObject **result) { - PyObject *oname = _PyUnicode_FromId(name); /* borrowed */ - if (!oname) { - *result = NULL; + if (Py_TYPE(obj)->tp_getattr == NULL) { + PyObject *oname = PyUnicode_FromString(name); + if (oname == NULL) { + *result = NULL; + return -1; + } + int rc = PyObject_GetOptionalAttr(obj, oname, result); + Py_DECREF(oname); + return rc; + } + + *result = (*Py_TYPE(obj)->tp_getattr)(obj, (char*)name); + if (*result != NULL) { + return 1; + } + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { return -1; } - return _PyObject_LookupAttr(v, oname, result); + PyErr_Clear(); + return 0; } int -PyObject_HasAttr(PyObject *v, PyObject *name) +PyObject_HasAttrWithError(PyObject *obj, PyObject *name) { PyObject *res; - if (_PyObject_LookupAttr(v, name, &res) < 0) { - PyErr_Clear(); - return 0; - } - if (res == NULL) { + int rc = PyObject_GetOptionalAttr(obj, name, &res); + Py_XDECREF(res); + return rc; +} + +int +PyObject_HasAttr(PyObject *obj, PyObject *name) +{ + int rc = PyObject_HasAttrWithError(obj, name); + if (rc < 0) { + PyErr_FormatUnraisable( + "Exception ignored in PyObject_HasAttr(); consider using " + "PyObject_HasAttrWithError(), " + "PyObject_GetOptionalAttr() or PyObject_GetAttr()"); return 0; } - Py_DECREF(res); - return 1; + return rc; } int @@ -1222,6 +1445,12 @@ PyObject_SetAttr(PyObject *v, PyObject *name, PyObject *value) return -1; } +int +PyObject_DelAttr(PyObject *v, PyObject *name) +{ + return PyObject_SetAttr(v, name, NULL); +} + PyObject ** _PyObject_ComputedDictPointer(PyObject *obj) { @@ -1263,16 +1492,15 @@ _PyObject_GetDictPtr(PyObject *obj) if ((Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) { return _PyObject_ComputedDictPointer(obj); } - PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(*dorv_ptr)) { - PyObject *dict = _PyObject_MakeDictFromInstanceAttributes(obj, _PyDictOrValues_GetValues(*dorv_ptr)); + PyDictObject *dict = _PyObject_GetManagedDict(obj); + if (dict == NULL && Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES) { + dict = _PyObject_MaterializeManagedDict(obj); if (dict == NULL) { PyErr_Clear(); return NULL; } - dorv_ptr->dict = dict; } - return &dorv_ptr->dict; + return (PyObject **)&_PyObject_ManagedDictPointer(obj)->dict; } PyObject * @@ -1325,13 +1553,13 @@ _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method) return 0; } - PyObject *descr = _PyType_Lookup(tp, name); + PyObject *descr = _PyType_LookupRef(tp, name); descrgetfunc f = NULL; if (descr != NULL) { - Py_INCREF(descr); if (_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)) { meth_found = 1; - } else { + } + else { f = Py_TYPE(descr)->tp_descr_get; if (f != NULL && PyDescr_IsData(descr)) { *method = f(descr, obj, (PyObject *)Py_TYPE(obj)); @@ -1340,27 +1568,23 @@ _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method) } } } - PyObject *dict; - if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) { - PyDictOrValues* dorv_ptr = _PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(*dorv_ptr)) { - PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr); - PyObject *attr = _PyObject_GetInstanceAttribute(obj, values, name); - if (attr != NULL) { - *method = attr; - Py_XDECREF(descr); - return 0; - } - dict = NULL; - } - else { - dict = dorv_ptr->dict; + PyObject *dict, *attr; + if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) && + _PyObject_TryGetInstanceAttribute(obj, name, &attr)) { + if (attr != NULL) { + *method = attr; + Py_XDECREF(descr); + return 0; } + dict = NULL; + } + else if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) { + dict = (PyObject *)_PyObject_GetManagedDict(obj); } else { PyObject **dictptr = _PyObject_ComputedDictPointer(obj); if (dictptr != NULL) { - dict = *dictptr; + dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*dictptr); } else { dict = NULL; @@ -1368,19 +1592,14 @@ _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method) } if (dict != NULL) { Py_INCREF(dict); - PyObject *attr = PyDict_GetItemWithError(dict, name); - if (attr != NULL) { - *method = Py_NewRef(attr); + if (PyDict_GetItemRef(dict, name, method) != 0) { + // found or error Py_DECREF(dict); Py_XDECREF(descr); return 0; } + // not found Py_DECREF(dict); - - if (PyErr_Occurred()) { - Py_XDECREF(descr); - return 0; - } } if (meth_found) { @@ -1403,7 +1622,7 @@ _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method) "'%.100s' object has no attribute '%U'", tp->tp_name, name); - set_attribute_error_context(obj, name); + _PyObject_SetAttributeErrorContext(obj, name); return 0; } @@ -1437,11 +1656,10 @@ _PyObject_GenericGetAttrWithDict(PyObject *obj, PyObject *name, goto done; } - descr = _PyType_Lookup(tp, name); + descr = _PyType_LookupRef(tp, name); f = NULL; if (descr != NULL) { - Py_INCREF(descr); f = Py_TYPE(descr)->tp_descr_get; if (f != NULL && PyDescr_IsData(descr)) { res = f(descr, obj, (PyObject *)Py_TYPE(obj)); @@ -1453,53 +1671,48 @@ _PyObject_GenericGetAttrWithDict(PyObject *obj, PyObject *name, } } if (dict == NULL) { - if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) { - PyDictOrValues* dorv_ptr = _PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(*dorv_ptr)) { - PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr); - if (PyUnicode_CheckExact(name)) { - res = _PyObject_GetInstanceAttribute(obj, values, name); - if (res != NULL) { - goto done; - } - } - else { - dict = _PyObject_MakeDictFromInstanceAttributes(obj, values); - if (dict == NULL) { - res = NULL; - goto done; - } - dorv_ptr->dict = dict; + if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES)) { + if (PyUnicode_CheckExact(name) && + _PyObject_TryGetInstanceAttribute(obj, name, &res)) { + if (res != NULL) { + goto done; } } else { - dict = _PyDictOrValues_GetDict(*dorv_ptr); + dict = (PyObject *)_PyObject_MaterializeManagedDict(obj); + if (dict == NULL) { + res = NULL; + goto done; + } } } + else if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) { + dict = (PyObject *)_PyObject_GetManagedDict(obj); + } else { PyObject **dictptr = _PyObject_ComputedDictPointer(obj); if (dictptr) { +#ifdef Py_GIL_DISABLED + dict = _Py_atomic_load_ptr_acquire(dictptr); +#else dict = *dictptr; +#endif } } } if (dict != NULL) { Py_INCREF(dict); - res = PyDict_GetItemWithError(dict, name); + int rc = PyDict_GetItemRef(dict, name, &res); + Py_DECREF(dict); if (res != NULL) { - Py_INCREF(res); - Py_DECREF(dict); goto done; } - else { - Py_DECREF(dict); - if (PyErr_Occurred()) { - if (suppress && PyErr_ExceptionMatches(PyExc_AttributeError)) { - PyErr_Clear(); - } - else { - goto done; - } + else if (rc < 0) { + if (suppress && PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + } + else { + goto done; } } } @@ -1524,7 +1737,7 @@ _PyObject_GenericGetAttrWithDict(PyObject *obj, PyObject *name, "'%.100s' object has no attribute '%U'", tp->tp_name, name); - set_attribute_error_context(obj, name); + _PyObject_SetAttributeErrorContext(obj, name); } done: Py_XDECREF(descr); @@ -1547,6 +1760,7 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, descrsetfunc f; int res = -1; + assert(!PyType_IsSubtype(tp, &PyType_Type)); if (!PyUnicode_Check(name)){ PyErr_Format(PyExc_TypeError, "attribute name must be string, not '%.200s'", @@ -1560,10 +1774,9 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, Py_INCREF(name); Py_INCREF(tp); - descr = _PyType_Lookup(tp, name); + descr = _PyType_LookupRef(tp, name); if (descr != NULL) { - Py_INCREF(descr); f = Py_TYPE(descr)->tp_descr_set; if (f != NULL) { res = f(descr, obj, value); @@ -1573,23 +1786,33 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, if (dict == NULL) { PyObject **dictptr; + + if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES)) { + res = _PyObject_StoreInstanceAttribute(obj, name, value); + goto error_check; + } + if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) { - PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj); - if (_PyDictOrValues_IsValues(*dorv_ptr)) { - res = _PyObject_StoreInstanceAttribute( - obj, _PyDictOrValues_GetValues(*dorv_ptr), name, value); - goto error_check; - } - dictptr = &dorv_ptr->dict; + PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(obj); + dictptr = (PyObject **)&managed_dict->dict; } else { dictptr = _PyObject_ComputedDictPointer(obj); } if (dictptr == NULL) { if (descr == NULL) { - PyErr_Format(PyExc_AttributeError, - "'%.100s' object has no attribute '%U'", - tp->tp_name, name); + if (tp->tp_setattro == PyObject_GenericSetAttr) { + PyErr_Format(PyExc_AttributeError, + "'%.100s' object has no attribute '%U' and no " + "__dict__ for setting new attributes", + tp->tp_name, name); + } + else { + PyErr_Format(PyExc_AttributeError, + "'%.100s' object has no attribute '%U'", + tp->tp_name, name); + } + _PyObject_SetAttributeErrorContext(obj, name); } else { PyErr_Format(PyExc_AttributeError, @@ -1599,7 +1822,7 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, goto done; } else { - res = _PyObjectDict_SetItem(tp, dictptr, name, value); + res = _PyObjectDict_SetItem(tp, obj, dictptr, name, value); } } else { @@ -1612,16 +1835,10 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, } error_check: if (res < 0 && PyErr_ExceptionMatches(PyExc_KeyError)) { - if (PyType_IsSubtype(tp, &PyType_Type)) { - PyErr_Format(PyExc_AttributeError, - "type object '%.50s' has no attribute '%U'", - ((PyTypeObject*)obj)->tp_name, name); - } - else { - PyErr_Format(PyExc_AttributeError, - "'%.100s' object has no attribute '%U'", - tp->tp_name, name); - } + PyErr_Format(PyExc_AttributeError, + "'%.100s' object has no attribute '%U'", + tp->tp_name, name); + _PyObject_SetAttributeErrorContext(obj, name); } done: Py_XDECREF(descr); @@ -1641,9 +1858,9 @@ PyObject_GenericSetDict(PyObject *obj, PyObject *value, void *context) { PyObject **dictptr = _PyObject_GetDictPtr(obj); if (dictptr == NULL) { - if (_PyType_HasFeature(Py_TYPE(obj), Py_TPFLAGS_MANAGED_DICT) && - _PyDictOrValues_IsValues(*_PyObject_DictOrValuesPointer(obj))) - { + if (_PyType_HasFeature(Py_TYPE(obj), Py_TPFLAGS_INLINE_VALUES) && + _PyObject_GetManagedDict(obj) == NULL + ) { /* Was unable to convert to dict */ PyErr_NoMemory(); } @@ -1663,7 +1880,9 @@ PyObject_GenericSetDict(PyObject *obj, PyObject *value, void *context) "not a '%.200s'", Py_TYPE(value)->tp_name); return -1; } + Py_BEGIN_CRITICAL_SECTION(obj); Py_XSETREF(*dictptr, Py_NewRef(value)); + Py_END_CRITICAL_SECTION(); return 0; } @@ -1871,6 +2090,11 @@ static PyNumberMethods none_as_number = { 0, /* nb_index */ }; +PyDoc_STRVAR(none_doc, +"NoneType()\n" +"--\n\n" +"The type of the None singleton."); + PyTypeObject _PyNone_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "NoneType", @@ -1892,7 +2116,7 @@ PyTypeObject _PyNone_Type = { 0, /*tp_setattro */ 0, /*tp_as_buffer */ Py_TPFLAGS_DEFAULT, /*tp_flags */ - 0, /*tp_doc */ + none_doc, /*tp_doc */ 0, /*tp_traverse */ 0, /*tp_clear */ _Py_BaseObject_RichCompare, /*tp_richcompare */ @@ -1912,11 +2136,7 @@ PyTypeObject _PyNone_Type = { none_new, /*tp_new */ }; -PyObject _Py_NoneStruct = { - _PyObject_EXTRA_INIT - { _Py_IMMORTAL_REFCNT }, - &_PyNone_Type -}; +PyObject _Py_NoneStruct = _PyObject_HEAD_INIT(&_PyNone_Type); /* NotImplemented is an object that can be used to signal that an operation is not implemented for the given type combination. */ @@ -1974,6 +2194,11 @@ static PyNumberMethods notimplemented_as_number = { .nb_bool = notimplemented_bool, }; +PyDoc_STRVAR(notimplemented_doc, +"NotImplementedType()\n" +"--\n\n" +"The type of the NotImplemented singleton."); + PyTypeObject _PyNotImplemented_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "NotImplementedType", @@ -1995,7 +2220,7 @@ PyTypeObject _PyNotImplemented_Type = { 0, /*tp_setattro */ 0, /*tp_as_buffer */ Py_TPFLAGS_DEFAULT, /*tp_flags */ - 0, /*tp_doc */ + notimplemented_doc, /*tp_doc */ 0, /*tp_traverse */ 0, /*tp_clear */ 0, /*tp_richcompare */ @@ -2015,27 +2240,35 @@ PyTypeObject _PyNotImplemented_Type = { notimplemented_new, /*tp_new */ }; -PyObject _Py_NotImplementedStruct = { - _PyObject_EXTRA_INIT - { _Py_IMMORTAL_REFCNT }, - &_PyNotImplemented_Type -}; +PyObject _Py_NotImplementedStruct = _PyObject_HEAD_INIT(&_PyNotImplemented_Type); -void +PyStatus _PyObject_InitState(PyInterpreterState *interp) { #ifdef Py_TRACE_REFS - init_refchain(interp); + if (refchain_init(interp) < 0) { + return _PyStatus_NO_MEMORY(); + } #endif + return _PyStatus_OK(); } +void +_PyObject_FiniState(PyInterpreterState *interp) +{ +#ifdef Py_TRACE_REFS + refchain_fini(interp); +#endif +} -extern PyTypeObject _Py_GenericAliasIterType; -extern PyTypeObject _PyMemoryIter_Type; + +extern PyTypeObject _PyAnextAwaitable_Type; +extern PyTypeObject _PyLegacyEventHandler_Type; extern PyTypeObject _PyLineIterator; +extern PyTypeObject _PyMemoryIter_Type; extern PyTypeObject _PyPositionsIterator; -extern PyTypeObject _PyLegacyEventHandler_Type; +extern PyTypeObject _Py_GenericAliasIterType; static PyTypeObject* static_types[] = { // The two most important base types: must be initialized first and @@ -2077,6 +2310,7 @@ static PyTypeObject* static_types[] = { &PyFilter_Type, &PyFloat_Type, &PyFrame_Type, + &PyFrameLocalsProxy_Type, &PyFrozenSet_Type, &PyFunction_Type, &PyGen_Type, @@ -2123,6 +2357,11 @@ static PyTypeObject* static_types[] = { &_PyBufferWrapper_Type, &_PyContextTokenMissing_Type, &_PyCoroWrapper_Type, +#ifdef _Py_TIER2 + &_PyCounterExecutor_Type, + &_PyCounterOptimizer_Type, + &_PyDefaultOptimizer_Type, +#endif &_Py_GenericAliasIterType, &_PyHamtItems_Type, &_PyHamtKeys_Type, @@ -2131,8 +2370,8 @@ static PyTypeObject* static_types[] = { &_PyHamt_BitmapNode_Type, &_PyHamt_CollisionNode_Type, &_PyHamt_Type, + &_PyInstructionSequence_Type, &_PyLegacyEventHandler_Type, - &_PyInterpreterID_Type, &_PyLineIterator, &_PyManagedBuffer_Type, &_PyMemoryIter_Type, @@ -2143,10 +2382,15 @@ static PyTypeObject* static_types[] = { &_PyPositionsIterator, &_PyUnicodeASCIIIter_Type, &_PyUnion_Type, +#ifdef _Py_TIER2 + &_PyUOpExecutor_Type, + &_PyUOpOptimizer_Type, +#endif &_PyWeakref_CallableProxyType, &_PyWeakref_ProxyType, &_PyWeakref_RefType, &_PyTypeAlias_Type, + &_PyNoDefault_Type, // subclasses: _PyTypes_FiniTypes() deallocates them before their base // class @@ -2175,6 +2419,14 @@ _PyTypes_InitTypes(PyInterpreterState *interp) } } + // Cache __reduce__ from PyBaseObject_Type object + PyObject *baseobj_dict = _PyType_GetDict(&PyBaseObject_Type); + PyObject *baseobj_reduce = PyDict_GetItemWithError(baseobj_dict, &_Py_ID(__reduce__)); + if (baseobj_reduce == NULL && PyErr_Occurred()) { + return _PyStatus_ERR("Can't get __reduce__ from base object"); + } + _Py_INTERP_CACHED_OBJECT(interp, objreduce) = baseobj_reduce; + // Must be after static types are initialized if (_Py_initialize_generic(interp) < 0) { return _PyStatus_ERR("Can't initialize generic types"); @@ -2197,7 +2449,7 @@ _PyTypes_FiniTypes(PyInterpreterState *interp) // their base classes. for (Py_ssize_t i=Py_ARRAY_LENGTH(static_types)-1; i>=0; i--) { PyTypeObject *type = static_types[i]; - _PyStaticType_Dealloc(interp, type); + _PyStaticType_FiniBuiltin(interp, type); } } @@ -2205,21 +2457,28 @@ _PyTypes_FiniTypes(PyInterpreterState *interp) static inline void new_reference(PyObject *op) { - if (_PyRuntime.tracemalloc.config.tracing) { - _PyTraceMalloc_NewReference(op); - } // Skip the immortal object check in Py_SET_REFCNT; always set refcnt to 1 +#if !defined(Py_GIL_DISABLED) op->ob_refcnt = 1; +#else + op->ob_tid = _Py_ThreadId(); + op->_padding = 0; + op->ob_mutex = (PyMutex){ 0 }; + op->ob_gc_bits = 0; + op->ob_ref_local = 1; + op->ob_ref_shared = 0; +#endif #ifdef Py_TRACE_REFS - _Py_AddToAllObjects(op, 1); + _Py_AddToAllObjects(op); #endif + _PyReftracerTrack(op, PyRefTracer_CREATE); } void _Py_NewReference(PyObject *op) { #ifdef Py_REF_DEBUG - reftotal_increment(_PyInterpreterState_GET()); + _Py_IncRefTotal(_PyThreadState_GET()); #endif new_reference(op); } @@ -2230,6 +2489,62 @@ _Py_NewReferenceNoTotal(PyObject *op) new_reference(op); } +void +_Py_SetImmortalUntracked(PyObject *op) +{ +#ifdef Py_DEBUG + // For strings, use _PyUnicode_InternImmortal instead. + if (PyUnicode_CheckExact(op)) { + assert(PyUnicode_CHECK_INTERNED(op) == SSTATE_INTERNED_IMMORTAL + || PyUnicode_CHECK_INTERNED(op) == SSTATE_INTERNED_IMMORTAL_STATIC); + } +#endif +#ifdef Py_GIL_DISABLED + op->ob_tid = _Py_UNOWNED_TID; + op->ob_ref_local = _Py_IMMORTAL_REFCNT_LOCAL; + op->ob_ref_shared = 0; +#else + op->ob_refcnt = _Py_IMMORTAL_REFCNT; +#endif +} + +void +_Py_SetImmortal(PyObject *op) +{ + if (PyObject_IS_GC(op) && _PyObject_GC_IS_TRACKED(op)) { + _PyObject_GC_UNTRACK(op); + } + _Py_SetImmortalUntracked(op); +} + +void +_PyObject_SetDeferredRefcount(PyObject *op) +{ +#ifdef Py_GIL_DISABLED + assert(PyType_IS_GC(Py_TYPE(op))); + assert(_Py_IsOwnedByCurrentThread(op)); + assert(op->ob_ref_shared == 0); + _PyObject_SET_GC_BITS(op, _PyGC_BITS_DEFERRED); + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (_Py_atomic_load_int_relaxed(&interp->gc.immortalize) == 1) { + // gh-117696: immortalize objects instead of using deferred reference + // counting for now. + _Py_SetImmortal(op); + return; + } + op->ob_ref_local += 1; + op->ob_ref_shared = _Py_REF_QUEUED; +#endif +} + +void +_Py_ResurrectReference(PyObject *op) +{ +#ifdef Py_TRACE_REFS + _Py_AddToAllObjects(op); +#endif +} + #ifdef Py_TRACE_REFS void @@ -2239,53 +2554,62 @@ _Py_ForgetReference(PyObject *op) _PyObject_ASSERT_FAILED_MSG(op, "negative refcnt"); } - PyObject *refchain = REFCHAIN(_PyInterpreterState_GET()); - if (op == refchain || - op->_ob_prev->_ob_next != op || op->_ob_next->_ob_prev != op) - { - _PyObject_ASSERT_FAILED_MSG(op, "invalid object chain"); - } + PyInterpreterState *interp = _PyInterpreterState_GET(); #ifdef SLOW_UNREF_CHECK - PyObject *p; - for (p = refchain->_ob_next; p != refchain; p = p->_ob_next) { - if (p == op) { - break; - } - } - if (p == refchain) { + if (!_PyRefchain_Get(interp, op)) { /* Not found */ _PyObject_ASSERT_FAILED_MSG(op, "object not found in the objects list"); } #endif - op->_ob_next->_ob_prev = op->_ob_prev; - op->_ob_prev->_ob_next = op->_ob_next; - op->_ob_next = op->_ob_prev = NULL; + _PyRefchain_Remove(interp, op); } +static int +_Py_PrintReference(_Py_hashtable_t *ht, + const void *key, const void *value, + void *user_data) +{ + PyObject *op = (PyObject*)key; + FILE *fp = (FILE *)user_data; + fprintf(fp, "%p [%zd] ", (void *)op, Py_REFCNT(op)); + if (PyObject_Print(op, fp, 0) != 0) { + PyErr_Clear(); + } + putc('\n', fp); + return 0; +} + + /* Print all live objects. Because PyObject_Print is called, the * interpreter must be in a healthy state. */ void _Py_PrintReferences(PyInterpreterState *interp, FILE *fp) { - PyObject *op; if (interp == NULL) { interp = _PyInterpreterState_Main(); } fprintf(fp, "Remaining objects:\n"); - PyObject *refchain = REFCHAIN(interp); - for (op = refchain->_ob_next; op != refchain; op = op->_ob_next) { - fprintf(fp, "%p [%zd] ", (void *)op, Py_REFCNT(op)); - if (PyObject_Print(op, fp, 0) != 0) { - PyErr_Clear(); - } - putc('\n', fp); - } + _Py_hashtable_foreach(REFCHAIN(interp), _Py_PrintReference, fp); +} + + +static int +_Py_PrintReferenceAddress(_Py_hashtable_t *ht, + const void *key, const void *value, + void *user_data) +{ + PyObject *op = (PyObject*)key; + FILE *fp = (FILE *)user_data; + fprintf(fp, "%p [%zd] %s\n", + (void *)op, Py_REFCNT(op), Py_TYPE(op)->tp_name); + return 0; } + /* Print the addresses of all live objects. Unlike _Py_PrintReferences, this * doesn't make any calls to the Python C API, so is always safe to call. */ @@ -2296,47 +2620,96 @@ _Py_PrintReferences(PyInterpreterState *interp, FILE *fp) void _Py_PrintReferenceAddresses(PyInterpreterState *interp, FILE *fp) { - PyObject *op; - PyObject *refchain = REFCHAIN(interp); fprintf(fp, "Remaining object addresses:\n"); - for (op = refchain->_ob_next; op != refchain; op = op->_ob_next) - fprintf(fp, "%p [%zd] %s\n", (void *)op, - Py_REFCNT(op), Py_TYPE(op)->tp_name); + _Py_hashtable_foreach(REFCHAIN(interp), _Py_PrintReferenceAddress, fp); +} + + +typedef struct { + PyObject *self; + PyObject *args; + PyObject *list; + PyObject *type; + Py_ssize_t limit; +} _Py_GetObjectsData; + +enum { + _PY_GETOBJECTS_IGNORE = 0, + _PY_GETOBJECTS_ERROR = 1, + _PY_GETOBJECTS_STOP = 2, +}; + +static int +_Py_GetObject(_Py_hashtable_t *ht, + const void *key, const void *value, + void *user_data) +{ + PyObject *op = (PyObject *)key; + _Py_GetObjectsData *data = user_data; + if (data->limit > 0) { + if (PyList_GET_SIZE(data->list) >= data->limit) { + return _PY_GETOBJECTS_STOP; + } + } + + if (op == data->self) { + return _PY_GETOBJECTS_IGNORE; + } + if (op == data->args) { + return _PY_GETOBJECTS_IGNORE; + } + if (op == data->list) { + return _PY_GETOBJECTS_IGNORE; + } + if (data->type != NULL) { + if (op == data->type) { + return _PY_GETOBJECTS_IGNORE; + } + if (!Py_IS_TYPE(op, (PyTypeObject *)data->type)) { + return _PY_GETOBJECTS_IGNORE; + } + } + + if (PyList_Append(data->list, op) < 0) { + return _PY_GETOBJECTS_ERROR; + } + return 0; } + /* The implementation of sys.getobjects(). */ PyObject * _Py_GetObjects(PyObject *self, PyObject *args) { - int i, n; - PyObject *t = NULL; - PyObject *res, *op; - PyInterpreterState *interp = _PyInterpreterState_GET(); + Py_ssize_t limit; + PyObject *type = NULL; + if (!PyArg_ParseTuple(args, "n|O", &limit, &type)) { + return NULL; + } - if (!PyArg_ParseTuple(args, "i|O", &n, &t)) + PyObject *list = PyList_New(0); + if (list == NULL) { return NULL; - PyObject *refchain = REFCHAIN(interp); - op = refchain->_ob_next; - res = PyList_New(0); - if (res == NULL) + } + + _Py_GetObjectsData data = { + .self = self, + .args = args, + .list = list, + .type = type, + .limit = limit, + }; + PyInterpreterState *interp = _PyInterpreterState_GET(); + int res = _Py_hashtable_foreach(REFCHAIN(interp), _Py_GetObject, &data); + if (res == _PY_GETOBJECTS_ERROR) { + Py_DECREF(list); return NULL; - for (i = 0; (n == 0 || i < n) && op != refchain; i++) { - while (op == self || op == args || op == res || op == t || - (t != NULL && !Py_IS_TYPE(op, (PyTypeObject *) t))) { - op = op->_ob_next; - if (op == refchain) - return res; - } - if (PyList_Append(res, op) < 0) { - Py_DECREF(res); - return NULL; - } - op = op->_ob_next; } - return res; + return list; } #undef REFCHAIN +#undef REFCHAIN_VALUE #endif /* Py_TRACE_REFS */ @@ -2433,28 +2806,30 @@ finally: /* Trashcan support. */ -#define _PyTrash_UNWIND_LEVEL 50 - /* Add op to the gcstate->trash_delete_later list. Called when the current * call-stack depth gets large. op must be a currently untracked gc'ed * object, with refcount 0. Py_DECREF must already have been called on it. */ -static void -_PyTrash_thread_deposit_object(struct _py_trashcan *trash, PyObject *op) +void +_PyTrash_thread_deposit_object(PyThreadState *tstate, PyObject *op) { _PyObject_ASSERT(op, _PyObject_IS_GC(op)); _PyObject_ASSERT(op, !_PyObject_GC_IS_TRACKED(op)); _PyObject_ASSERT(op, Py_REFCNT(op) == 0); - _PyGCHead_SET_PREV(_Py_AS_GC(op), (PyGC_Head*)trash->delete_later); - trash->delete_later = op; +#ifdef Py_GIL_DISABLED + op->ob_tid = (uintptr_t)tstate->delete_later; +#else + _PyGCHead_SET_PREV(_Py_AS_GC(op), (PyGC_Head*)tstate->delete_later); +#endif + tstate->delete_later = op; } /* Deallocate all the objects in the gcstate->trash_delete_later list. * Called when the call-stack unwinds again. */ -static void -_PyTrash_thread_destroy_chain(struct _py_trashcan *trash) +void +_PyTrash_thread_destroy_chain(PyThreadState *tstate) { - /* We need to increase trash_delete_nesting here, otherwise, + /* We need to increase c_recursion_remaining here, otherwise, _PyTrash_thread_destroy_chain will be called recursively and then possibly crash. An example that may crash without increase: @@ -2465,14 +2840,19 @@ _PyTrash_thread_destroy_chain(struct _py_trashcan *trash) tups = [(tup,) for tup in tups] del tups */ - assert(trash->delete_nesting == 0); - ++trash->delete_nesting; - while (trash->delete_later) { - PyObject *op = trash->delete_later; + assert(tstate->c_recursion_remaining > Py_TRASHCAN_HEADROOM); + tstate->c_recursion_remaining--; + while (tstate->delete_later) { + PyObject *op = tstate->delete_later; destructor dealloc = Py_TYPE(op)->tp_dealloc; - trash->delete_later = - (PyObject*) _PyGCHead_PREV(_Py_AS_GC(op)); +#ifdef Py_GIL_DISABLED + tstate->delete_later = (PyObject*) op->ob_tid; + op->ob_tid = 0; + _Py_atomic_store_ssize_relaxed(&op->ob_ref_shared, _Py_REF_MERGED); +#else + tstate->delete_later = (PyObject*) _PyGCHead_PREV(_Py_AS_GC(op)); +#endif /* Call the deallocator directly. This used to try to * fool Py_DECREF into calling it indirectly, but @@ -2482,92 +2862,10 @@ _PyTrash_thread_destroy_chain(struct _py_trashcan *trash) */ _PyObject_ASSERT(op, Py_REFCNT(op) == 0); (*dealloc)(op); - assert(trash->delete_nesting == 1); - } - --trash->delete_nesting; -} - - -static struct _py_trashcan * -_PyTrash_get_state(PyThreadState *tstate) -{ - if (tstate != NULL) { - return &tstate->trash; - } - // The current thread must be finalizing. - // Fall back to using thread-local state. - // XXX Use thread-local variable syntax? - assert(PyThread_tss_is_created(&_PyRuntime.trashTSSkey)); - struct _py_trashcan *trash = - (struct _py_trashcan *)PyThread_tss_get(&_PyRuntime.trashTSSkey); - if (trash == NULL) { - trash = PyMem_RawMalloc(sizeof(struct _py_trashcan)); - if (trash == NULL) { - Py_FatalError("Out of memory"); - } - PyThread_tss_set(&_PyRuntime.trashTSSkey, (void *)trash); - } - return trash; -} - -static void -_PyTrash_clear_state(PyThreadState *tstate) -{ - if (tstate != NULL) { - assert(tstate->trash.delete_later == NULL); - return; - } - if (PyThread_tss_is_created(&_PyRuntime.trashTSSkey)) { - struct _py_trashcan *trash = - (struct _py_trashcan *)PyThread_tss_get(&_PyRuntime.trashTSSkey); - if (trash != NULL) { - PyThread_tss_set(&_PyRuntime.trashTSSkey, (void *)NULL); - PyMem_RawFree(trash); - } } + tstate->c_recursion_remaining++; } - -int -_PyTrash_begin(PyThreadState *tstate, PyObject *op) -{ - // XXX Make sure the GIL is held. - struct _py_trashcan *trash = _PyTrash_get_state(tstate); - if (trash->delete_nesting >= _PyTrash_UNWIND_LEVEL) { - /* Store the object (to be deallocated later) and jump past - * Py_TRASHCAN_END, skipping the body of the deallocator */ - _PyTrash_thread_deposit_object(trash, op); - return 1; - } - ++trash->delete_nesting; - return 0; -} - - -void -_PyTrash_end(PyThreadState *tstate) -{ - // XXX Make sure the GIL is held. - struct _py_trashcan *trash = _PyTrash_get_state(tstate); - --trash->delete_nesting; - if (trash->delete_nesting <= 0) { - if (trash->delete_later != NULL) { - _PyTrash_thread_destroy_chain(trash); - } - _PyTrash_clear_state(tstate); - } -} - - -/* bpo-40170: It's only be used in Py_TRASHCAN_BEGIN macro to hide - implementation details. */ -int -_PyTrash_cond(PyObject *op, destructor dealloc) -{ - return Py_TYPE(op)->tp_dealloc == dealloc; -} - - void _Py_NO_RETURN _PyObject_AssertFailed(PyObject *obj, const char *expr, const char *msg, const char *file, int line, const char *function) @@ -2637,6 +2935,7 @@ _Py_Dealloc(PyObject *op) #ifdef Py_TRACE_REFS _Py_ForgetReference(op); #endif + _PyReftracerTrack(op, PyRefTracer_DESTROY); (*dealloc)(op); #ifdef Py_DEBUG @@ -2714,6 +3013,76 @@ int Py_IsFalse(PyObject *x) return Py_Is(x, Py_False); } -#ifdef __cplusplus + +// Py_SET_REFCNT() implementation for stable ABI +void +_Py_SetRefcnt(PyObject *ob, Py_ssize_t refcnt) +{ + Py_SET_REFCNT(ob, refcnt); +} + +int PyRefTracer_SetTracer(PyRefTracer tracer, void *data) { + assert(PyGILState_Check()); + _PyRuntime.ref_tracer.tracer_func = tracer; + _PyRuntime.ref_tracer.tracer_data = data; + return 0; +} + +PyRefTracer PyRefTracer_GetTracer(void** data) { + assert(PyGILState_Check()); + if (data != NULL) { + *data = _PyRuntime.ref_tracer.tracer_data; + } + return _PyRuntime.ref_tracer.tracer_func; } + + + +static PyObject* constants[] = { + &_Py_NoneStruct, // Py_CONSTANT_NONE + (PyObject*)(&_Py_FalseStruct), // Py_CONSTANT_FALSE + (PyObject*)(&_Py_TrueStruct), // Py_CONSTANT_TRUE + &_Py_EllipsisObject, // Py_CONSTANT_ELLIPSIS + &_Py_NotImplementedStruct, // Py_CONSTANT_NOT_IMPLEMENTED + NULL, // Py_CONSTANT_ZERO + NULL, // Py_CONSTANT_ONE + NULL, // Py_CONSTANT_EMPTY_STR + NULL, // Py_CONSTANT_EMPTY_BYTES + NULL, // Py_CONSTANT_EMPTY_TUPLE +}; + +void +_Py_GetConstant_Init(void) +{ + constants[Py_CONSTANT_ZERO] = _PyLong_GetZero(); + constants[Py_CONSTANT_ONE] = _PyLong_GetOne(); + constants[Py_CONSTANT_EMPTY_STR] = PyUnicode_New(0, 0); + constants[Py_CONSTANT_EMPTY_BYTES] = PyBytes_FromStringAndSize(NULL, 0); + constants[Py_CONSTANT_EMPTY_TUPLE] = PyTuple_New(0); +#ifndef NDEBUG + for (size_t i=0; i < Py_ARRAY_LENGTH(constants); i++) { + assert(constants[i] != NULL); + assert(_Py_IsImmortal(constants[i])); + } #endif +} + +PyObject* +Py_GetConstant(unsigned int constant_id) +{ + if (constant_id < Py_ARRAY_LENGTH(constants)) { + return constants[constant_id]; + } + else { + PyErr_BadInternalCall(); + return NULL; + } +} + + +PyObject* +Py_GetConstantBorrowed(unsigned int constant_id) +{ + // All constants are immortal + return Py_GetConstant(constant_id); +} |