diff options
Diffstat (limited to 'contrib/tools/python3/Objects/codeobject.c')
| -rw-r--r-- | contrib/tools/python3/Objects/codeobject.c | 738 |
1 files changed, 530 insertions, 208 deletions
diff --git a/contrib/tools/python3/Objects/codeobject.c b/contrib/tools/python3/Objects/codeobject.c index 1681d97613e..8a0b1027b94 100644 --- a/contrib/tools/python3/Objects/codeobject.c +++ b/contrib/tools/python3/Objects/codeobject.c @@ -1,17 +1,21 @@ -#include <stdbool.h> - #include "Python.h" #include "opcode.h" -#include "structmember.h" // PyMemberDef + #include "pycore_code.h" // _PyCodeConstructor #include "pycore_frame.h" // FRAME_SPECIALS_SIZE +#include "pycore_hashtable.h" // _Py_hashtable_t +#include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_interp.h" // PyInterpreterState.co_extra_freefuncs -#include "pycore_opcode.h" // _PyOpcode_Deopt +#include "pycore_object.h" // _PyObject_SetDeferredRefcount +#include "pycore_opcode_metadata.h" // _PyOpcode_Deopt, _PyOpcode_Caches +#include "pycore_opcode_utils.h" // RESUME_AT_FUNC_START #include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_setobject.h" // _PySet_NextEntry() #include "pycore_tuple.h" // _PyTuple_ITEMS() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include "clinic/codeobject.c.h" -static PyObject* code_repr(PyCodeObject *co); +#include <stdbool.h> static const char * code_event_name(PyCodeEvent event) { @@ -40,21 +44,9 @@ notify_code_watchers(PyCodeEvent event, PyCodeObject *co) // callback must be non-null if the watcher bit is set assert(cb != NULL); if (cb(event, co) < 0) { - // Don't risk resurrecting the object if an unraisablehook keeps - // a reference; pass a string as context. - PyObject *context = NULL; - PyObject *repr = code_repr(co); - if (repr) { - context = PyUnicode_FromFormat( - "%s watcher callback for %U", - code_event_name(event), repr); - Py_DECREF(repr); - } - if (context == NULL) { - context = Py_NewRef(Py_None); - } - PyErr_WriteUnraisable(context); - Py_DECREF(context); + PyErr_FormatUnraisable( + "Exception ignored in %s watcher callback for %R", + code_event_name(event), co); } } i++; @@ -111,10 +103,20 @@ PyCode_ClearWatcher(int watcher_id) * generic helpers ******************/ -/* all_name_chars(s): true iff s matches [a-zA-Z0-9_]* */ static int -all_name_chars(PyObject *o) +should_intern_string(PyObject *o) { +#ifdef Py_GIL_DISABLED + // The free-threaded build interns (and immortalizes) all string constants + // unless we've disabled immortalizing objects that use deferred reference + // counting. + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (_Py_atomic_load_int(&interp->gc.immortalize) < 0) { + return 1; + } +#endif + + // compute if s matches [a-zA-Z0-9_] const unsigned char *s, *e; if (!PyUnicode_IS_ASCII(o)) @@ -129,6 +131,42 @@ all_name_chars(PyObject *o) return 1; } +#ifdef Py_GIL_DISABLED +static PyObject *intern_one_constant(PyObject *op); + +// gh-130851: In the free threading build, we intern and immortalize most +// constants, except code objects. However, users can generate code objects +// with arbitrary co_consts. We don't want to immortalize or intern unexpected +// constants or tuples/sets containing unexpected constants. +static int +should_immortalize_constant(PyObject *v) +{ + // Only immortalize containers if we've already immortalized all their + // elements. + if (PyTuple_CheckExact(v)) { + for (Py_ssize_t i = PyTuple_GET_SIZE(v); --i >= 0; ) { + if (!_Py_IsImmortal(PyTuple_GET_ITEM(v, i))) { + return 0; + } + } + return 1; + } + else if (PyFrozenSet_CheckExact(v)) { + PyObject *item; + Py_hash_t hash; + Py_ssize_t pos = 0; + while (_PySet_NextEntry(v, &pos, &item, &hash)) { + if (!_Py_IsImmortal(item)) { + return 0; + } + } + return 1; + } + return (PyLong_CheckExact(v) || PyFloat_CheckExact(v) || + PyComplex_Check(v) || PyBytes_CheckExact(v)); +} +#endif + static int intern_strings(PyObject *tuple) { @@ -147,19 +185,17 @@ intern_strings(PyObject *tuple) return 0; } -/* Intern selected string constants */ +/* Intern constants. In the default build, this interns selected string + constants. In the free-threaded build, this also interns non-string + constants. */ static int -intern_string_constants(PyObject *tuple, int *modified) +intern_constants(PyObject *tuple, int *modified) { PyInterpreterState *interp = _PyInterpreterState_GET(); for (Py_ssize_t i = PyTuple_GET_SIZE(tuple); --i >= 0; ) { PyObject *v = PyTuple_GET_ITEM(tuple, i); if (PyUnicode_CheckExact(v)) { - if (PyUnicode_READY(v) == -1) { - return -1; - } - - if (all_name_chars(v)) { + if (should_intern_string(v)) { PyObject *w = v; _PyUnicode_InternMortal(interp, &v); if (w != v) { @@ -171,7 +207,7 @@ intern_string_constants(PyObject *tuple, int *modified) } } else if (PyTuple_CheckExact(v)) { - if (intern_string_constants(v, NULL) < 0) { + if (intern_constants(v, NULL) < 0) { return -1; } } @@ -182,7 +218,7 @@ intern_string_constants(PyObject *tuple, int *modified) return -1; } int tmp_modified = 0; - if (intern_string_constants(tmp, &tmp_modified) < 0) { + if (intern_constants(tmp, &tmp_modified) < 0) { Py_DECREF(tmp); return -1; } @@ -201,6 +237,59 @@ intern_string_constants(PyObject *tuple, int *modified) } Py_DECREF(tmp); } +#ifdef Py_GIL_DISABLED + else if (PySlice_Check(v)) { + PySliceObject *slice = (PySliceObject *)v; + PyObject *tmp = PyTuple_New(3); + if (tmp == NULL) { + return -1; + } + PyTuple_SET_ITEM(tmp, 0, Py_NewRef(slice->start)); + PyTuple_SET_ITEM(tmp, 1, Py_NewRef(slice->stop)); + PyTuple_SET_ITEM(tmp, 2, Py_NewRef(slice->step)); + int tmp_modified = 0; + if (intern_constants(tmp, &tmp_modified) < 0) { + Py_DECREF(tmp); + return -1; + } + if (tmp_modified) { + v = PySlice_New(PyTuple_GET_ITEM(tmp, 0), + PyTuple_GET_ITEM(tmp, 1), + PyTuple_GET_ITEM(tmp, 2)); + if (v == NULL) { + Py_DECREF(tmp); + return -1; + } + PyTuple_SET_ITEM(tuple, i, v); + Py_DECREF(slice); + if (modified) { + *modified = 1; + } + } + Py_DECREF(tmp); + } + + // Intern non-string consants in the free-threaded build, but only if + // we are also immortalizing objects that use deferred reference + // counting. + PyThreadState *tstate = PyThreadState_GET(); + if (!_Py_IsImmortal(v) && !PyUnicode_CheckExact(v) && + should_immortalize_constant(v) && + _Py_atomic_load_int(&tstate->interp->gc.immortalize) >= 0) + { + PyObject *interned = intern_one_constant(v); + if (interned == NULL) { + return -1; + } + else if (interned != v) { + PyTuple_SET_ITEM(tuple, i, interned); + Py_SETREF(v, interned); + if (modified) { + *modified = 1; + } + } + } +#endif } return 0; } @@ -248,21 +337,32 @@ validate_and_copy_tuple(PyObject *tup) } static int -init_co_cached(PyCodeObject *self) { - if (self->_co_cached == NULL) { - self->_co_cached = PyMem_New(_PyCoCached, 1); - if (self->_co_cached == NULL) { +init_co_cached(PyCodeObject *self) +{ + _PyCoCached *cached = FT_ATOMIC_LOAD_PTR(self->_co_cached); + if (cached != NULL) { + return 0; + } + + Py_BEGIN_CRITICAL_SECTION(self); + cached = self->_co_cached; + if (cached == NULL) { + cached = PyMem_New(_PyCoCached, 1); + if (cached == NULL) { PyErr_NoMemory(); - return -1; } - self->_co_cached->_co_code = NULL; - self->_co_cached->_co_cellvars = NULL; - self->_co_cached->_co_freevars = NULL; - self->_co_cached->_co_varnames = NULL; + else { + cached->_co_code = NULL; + cached->_co_cellvars = NULL; + cached->_co_freevars = NULL; + cached->_co_varnames = NULL; + FT_ATOMIC_STORE_PTR(self->_co_cached, cached); + } } - return 0; - + Py_END_CRITICAL_SECTION(); + return cached != NULL ? 0 : -1; } + /****************** * _PyCode_New() ******************/ @@ -400,10 +500,17 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) int nlocals, ncellvars, nfreevars; get_localsplus_counts(con->localsplusnames, con->localspluskinds, &nlocals, &ncellvars, &nfreevars); + if (con->stacksize == 0) { + con->stacksize = 1; + } + PyInterpreterState *interp = _PyInterpreterState_GET(); co->co_filename = Py_NewRef(con->filename); co->co_name = Py_NewRef(con->name); co->co_qualname = Py_NewRef(con->qualname); + _PyUnicode_InternMortal(interp, &co->co_filename); + _PyUnicode_InternMortal(interp, &co->co_name); + _PyUnicode_InternMortal(interp, &co->co_qualname); co->co_flags = con->flags; co->co_firstlineno = con->firstlineno; @@ -429,16 +536,23 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) co->co_framesize = nlocalsplus + con->stacksize + FRAME_SPECIALS_SIZE; co->co_ncellvars = ncellvars; co->co_nfreevars = nfreevars; - co->co_version = _Py_next_func_version; - if (_Py_next_func_version != 0) { - _Py_next_func_version++; +#ifdef Py_GIL_DISABLED + PyMutex_Lock(&interp->func_state.mutex); +#endif + co->co_version = interp->func_state.next_version; + if (interp->func_state.next_version != 0) { + interp->func_state.next_version++; } +#ifdef Py_GIL_DISABLED + PyMutex_Unlock(&interp->func_state.mutex); +#endif co->_co_monitoring = NULL; co->_co_instrumentation_version = 0; /* not set */ co->co_weakreflist = NULL; co->co_extra = NULL; co->_co_cached = NULL; + co->co_executors = NULL; memcpy(_PyCode_CODE(co), PyBytes_AS_STRING(con->code), PyBytes_GET_SIZE(con->code)); @@ -543,29 +657,41 @@ remove_column_info(PyObject *locations) return res; } -/* The caller is responsible for ensuring that the given data is valid. */ - -PyCodeObject * -_PyCode_New(struct _PyCodeConstructor *con) +static int +intern_code_constants(struct _PyCodeConstructor *con) { - /* Ensure that strings are ready Unicode string */ - if (PyUnicode_READY(con->name) < 0) { - return NULL; - } - if (PyUnicode_READY(con->qualname) < 0) { - return NULL; - } - if (PyUnicode_READY(con->filename) < 0) { - return NULL; - } - +#ifdef Py_GIL_DISABLED + PyInterpreterState *interp = _PyInterpreterState_GET(); + struct _py_code_state *state = &interp->code_state; + PyMutex_Lock(&state->mutex); +#endif if (intern_strings(con->names) < 0) { - return NULL; + goto error; } - if (intern_string_constants(con->consts, NULL) < 0) { - return NULL; + if (intern_constants(con->consts, NULL) < 0) { + goto error; } if (intern_strings(con->localsplusnames) < 0) { + goto error; + } +#ifdef Py_GIL_DISABLED + PyMutex_Unlock(&state->mutex); +#endif + return 0; + +error: +#ifdef Py_GIL_DISABLED + PyMutex_Unlock(&state->mutex); +#endif + return -1; +} + +/* The caller is responsible for ensuring that the given data is valid. */ + +PyCodeObject * +_PyCode_New(struct _PyCodeConstructor *con) +{ + if (intern_code_constants(con) < 0) { return NULL; } @@ -581,13 +707,22 @@ _PyCode_New(struct _PyCodeConstructor *con) } Py_ssize_t size = PyBytes_GET_SIZE(con->code) / sizeof(_Py_CODEUNIT); - PyCodeObject *co = PyObject_NewVar(PyCodeObject, &PyCode_Type, size); + PyCodeObject *co; +#ifdef Py_GIL_DISABLED + co = PyObject_GC_NewVar(PyCodeObject, &PyCode_Type, size); +#else + co = PyObject_NewVar(PyCodeObject, &PyCode_Type, size); +#endif if (co == NULL) { Py_XDECREF(replacement_locations); PyErr_NoMemory(); return NULL; } init_code(co, con); +#ifdef Py_GIL_DISABLED + _PyObject_SetDeferredRefcount((PyObject *)co); + _PyObject_GC_TRACK(co); +#endif Py_XDECREF(replacement_locations); return co; } @@ -773,7 +908,7 @@ PyUnstable_Code_New(int argcount, int kwonlyargcount, // test.test_code.CodeLocationTest.test_code_new_empty to keep it in sync! static const uint8_t assert0[6] = { - RESUME, 0, + RESUME, RESUME_AT_FUNC_START, LOAD_ASSERTION_ERROR, 0, RAISE_VARARGS, 1 }; @@ -853,7 +988,27 @@ PyCode_Addr2Line(PyCodeObject *co, int addrq) if (addrq < 0) { return co->co_firstlineno; } - assert(addrq >= 0 && addrq <= _PyCode_NBYTES(co)); + if (co->_co_monitoring && co->_co_monitoring->lines) { + return _Py_Instrumentation_GetLine(co, addrq/sizeof(_Py_CODEUNIT)); + } + // assert(addrq >= 0 && addrq < _PyCode_NBYTES(co)); + PyCodeAddressRange bounds; + _PyCode_InitAddressRange(co, &bounds); + return _PyCode_CheckLineNumber(addrq, &bounds); +} + +int +_PyCode_SafeAddr2Line(PyCodeObject *co, int addrq) +{ + if (addrq < 0) { + return co->co_firstlineno; + } + if (co->_co_monitoring && co->_co_monitoring->lines) { + return _Py_Instrumentation_GetLine(co, addrq/sizeof(_Py_CODEUNIT)); + } + if (!(addrq >= 0 && addrq < _PyCode_NBYTES(co))) { + return -1; + } PyCodeAddressRange bounds; _PyCode_InitAddressRange(co, &bounds); return _PyCode_CheckLineNumber(addrq, &bounds); @@ -1453,16 +1608,21 @@ get_cached_locals(PyCodeObject *co, PyObject **cached_field, { assert(cached_field != NULL); assert(co->_co_cached != NULL); - if (*cached_field != NULL) { - return Py_NewRef(*cached_field); + PyObject *varnames = FT_ATOMIC_LOAD_PTR(*cached_field); + if (varnames != NULL) { + return Py_NewRef(varnames); } - assert(*cached_field == NULL); - PyObject *varnames = get_localsplus_names(co, kind, num); + + Py_BEGIN_CRITICAL_SECTION(co); + varnames = *cached_field; if (varnames == NULL) { - return NULL; + varnames = get_localsplus_names(co, kind, num); + if (varnames != NULL) { + FT_ATOMIC_STORE_PTR(*cached_field, varnames); + } } - *cached_field = Py_NewRef(varnames); - return varnames; + Py_END_CRITICAL_SECTION(); + return Py_XNewRef(varnames); } PyObject * @@ -1510,12 +1670,42 @@ PyCode_GetFreevars(PyCodeObject *code) return _PyCode_GetFreevars(code); } +#ifdef _Py_TIER2 + +static void +clear_executors(PyCodeObject *co) +{ + assert(co->co_executors); + for (int i = 0; i < co->co_executors->size; i++) { + if (co->co_executors->executors[i]) { + _Py_ExecutorDetach(co->co_executors->executors[i]); + assert(co->co_executors->executors[i] == NULL); + } + } + PyMem_Free(co->co_executors); + co->co_executors = NULL; +} + +void +_PyCode_Clear_Executors(PyCodeObject *code) +{ + clear_executors(code); +} + +#endif + static void deopt_code(PyCodeObject *code, _Py_CODEUNIT *instructions) { Py_ssize_t len = Py_SIZE(code); for (int i = 0; i < len; i++) { int opcode = _Py_GetBaseOpcode(code, i); + if (opcode == ENTER_EXECUTOR) { + _PyExecutorObject *exec = code->co_executors->executors[instructions[i].op.arg]; + opcode = _PyOpcode_Deopt[exec->vm_data.opcode]; + instructions[i].op.arg = exec->vm_data.oparg; + } + assert(opcode != ENTER_EXECUTOR); int caches = _PyOpcode_Caches[opcode]; instructions[i].op.code = opcode; for (int j = 1; j <= caches; j++) { @@ -1531,18 +1721,26 @@ _PyCode_GetCode(PyCodeObject *co) if (init_co_cached(co)) { return NULL; } - if (co->_co_cached->_co_code != NULL) { - return Py_NewRef(co->_co_cached->_co_code); + + _PyCoCached *cached = co->_co_cached; + PyObject *code = FT_ATOMIC_LOAD_PTR(cached->_co_code); + if (code != NULL) { + return Py_NewRef(code); } - PyObject *code = PyBytes_FromStringAndSize((const char *)_PyCode_CODE(co), - _PyCode_NBYTES(co)); + + Py_BEGIN_CRITICAL_SECTION(co); + code = cached->_co_code; if (code == NULL) { - return NULL; + code = PyBytes_FromStringAndSize((const char *)_PyCode_CODE(co), + _PyCode_NBYTES(co)); + if (code != NULL) { + deopt_code(co, (_Py_CODEUNIT *)PyBytes_AS_STRING(code)); + assert(cached->_co_code == NULL); + FT_ATOMIC_STORE_PTR(cached->_co_code, code); + } } - deopt_code(co, (_Py_CODEUNIT *)PyBytes_AS_STRING(code)); - assert(co->_co_cached->_co_code == NULL); - co->_co_cached->_co_code = Py_NewRef(code); - return code; + Py_END_CRITICAL_SECTION(); + return Py_XNewRef(code); } PyObject * @@ -1700,15 +1898,17 @@ free_monitoring_data(_PyCoMonitoringData *data) static void code_dealloc(PyCodeObject *co) { - assert(Py_REFCNT(co) == 0); - Py_SET_REFCNT(co, 1); + _PyObject_ResurrectStart((PyObject *)co); notify_code_watchers(PY_CODE_EVENT_DESTROY, co); - if (Py_REFCNT(co) > 1) { - Py_SET_REFCNT(co, Py_REFCNT(co) - 1); + if (_PyObject_ResurrectEnd((PyObject *)co)) { return; } - Py_SET_REFCNT(co, 0); +#ifdef Py_GIL_DISABLED + PyObject_GC_UnTrack(co); +#endif + + _PyFunction_ClearCodeByVersion(co->co_version); if (co->co_extra != NULL) { PyInterpreterState *interp = _PyInterpreterState_GET(); _PyCodeObjectExtra *co_extra = co->co_extra; @@ -1723,6 +1923,11 @@ code_dealloc(PyCodeObject *co) PyMem_Free(co_extra); } +#ifdef _Py_TIER2 + if (co->co_executors != NULL) { + clear_executors(co); + } +#endif Py_XDECREF(co->co_consts); Py_XDECREF(co->co_names); @@ -1740,13 +1945,20 @@ code_dealloc(PyCodeObject *co) Py_XDECREF(co->_co_cached->_co_varnames); PyMem_Free(co->_co_cached); } - if (co->co_weakreflist != NULL) { - PyObject_ClearWeakRefs((PyObject*)co); - } + FT_CLEAR_WEAKREFS((PyObject*)co, co->co_weakreflist); free_monitoring_data(co->_co_monitoring); PyObject_Free(co); } +#ifdef Py_GIL_DISABLED +static int +code_traverse(PyCodeObject *co, visitproc visit, void *arg) +{ + Py_VISIT(co->co_consts); + return 0; +} +#endif + static PyObject * code_repr(PyCodeObject *co) { @@ -1802,13 +2014,31 @@ code_richcompare(PyObject *self, PyObject *other, int op) for (int i = 0; i < Py_SIZE(co); i++) { _Py_CODEUNIT co_instr = _PyCode_CODE(co)[i]; _Py_CODEUNIT cp_instr = _PyCode_CODE(cp)[i]; - co_instr.op.code = _Py_GetBaseOpcode(co, i); - cp_instr.op.code = _Py_GetBaseOpcode(cp, i); - eq = co_instr.cache == cp_instr.cache; - if (!eq) { + uint8_t co_code = _Py_GetBaseOpcode(co, i); + uint8_t co_arg = co_instr.op.arg; + uint8_t cp_code = _Py_GetBaseOpcode(cp, i); + uint8_t cp_arg = cp_instr.op.arg; + + if (co_code == ENTER_EXECUTOR) { + const int exec_index = co_arg; + _PyExecutorObject *exec = co->co_executors->executors[exec_index]; + co_code = _PyOpcode_Deopt[exec->vm_data.opcode]; + co_arg = exec->vm_data.oparg; + } + assert(co_code != ENTER_EXECUTOR); + + if (cp_code == ENTER_EXECUTOR) { + const int exec_index = cp_arg; + _PyExecutorObject *exec = cp->co_executors->executors[exec_index]; + cp_code = _PyOpcode_Deopt[exec->vm_data.opcode]; + cp_arg = exec->vm_data.oparg; + } + assert(cp_code != ENTER_EXECUTOR); + + if (co_code != cp_code || co_arg != cp_arg) { goto unequal; } - i += _PyOpcode_Caches[co_instr.op.code]; + i += _PyOpcode_Caches[co_code]; } /* compare constants */ @@ -1864,7 +2094,7 @@ code_hash(PyCodeObject *co) Py_uhash_t uhash = 20221211; #define SCRAMBLE_IN(H) do { \ uhash ^= (Py_uhash_t)(H); \ - uhash *= _PyHASH_MULTIPLIER; \ + uhash *= PyHASH_MULTIPLIER; \ } while (0) #define SCRAMBLE_IN_HASH(EXPR) do { \ Py_hash_t h = PyObject_Hash(EXPR); \ @@ -1887,10 +2117,22 @@ code_hash(PyCodeObject *co) SCRAMBLE_IN(co->co_firstlineno); SCRAMBLE_IN(Py_SIZE(co)); for (int i = 0; i < Py_SIZE(co); i++) { - int deop = _Py_GetBaseOpcode(co, i); - SCRAMBLE_IN(deop); - SCRAMBLE_IN(_PyCode_CODE(co)[i].op.arg); - i += _PyOpcode_Caches[deop]; + _Py_CODEUNIT co_instr = _PyCode_CODE(co)[i]; + uint8_t co_code = co_instr.op.code; + uint8_t co_arg = co_instr.op.arg; + if (co_code == ENTER_EXECUTOR) { + _PyExecutorObject *exec = co->co_executors->executors[co_arg]; + assert(exec != NULL); + assert(exec->vm_data.opcode != ENTER_EXECUTOR); + co_code = _PyOpcode_Deopt[exec->vm_data.opcode]; + co_arg = exec->vm_data.oparg; + } + else { + co_code = _Py_GetBaseOpcode(co, i); + } + SCRAMBLE_IN(co_code); + SCRAMBLE_IN(co_arg); + i += _PyOpcode_Caches[co_code]; } if ((Py_hash_t)uhash == -1) { return -2; @@ -1902,20 +2144,20 @@ code_hash(PyCodeObject *co) #define OFF(x) offsetof(PyCodeObject, x) static PyMemberDef code_memberlist[] = { - {"co_argcount", T_INT, OFF(co_argcount), READONLY}, - {"co_posonlyargcount", T_INT, OFF(co_posonlyargcount), READONLY}, - {"co_kwonlyargcount", T_INT, OFF(co_kwonlyargcount), READONLY}, - {"co_stacksize", T_INT, OFF(co_stacksize), READONLY}, - {"co_flags", T_INT, OFF(co_flags), READONLY}, - {"co_nlocals", T_INT, OFF(co_nlocals), READONLY}, - {"co_consts", T_OBJECT, OFF(co_consts), READONLY}, - {"co_names", T_OBJECT, OFF(co_names), READONLY}, - {"co_filename", T_OBJECT, OFF(co_filename), READONLY}, - {"co_name", T_OBJECT, OFF(co_name), READONLY}, - {"co_qualname", T_OBJECT, OFF(co_qualname), READONLY}, - {"co_firstlineno", T_INT, OFF(co_firstlineno), READONLY}, - {"co_linetable", T_OBJECT, OFF(co_linetable), READONLY}, - {"co_exceptiontable", T_OBJECT, OFF(co_exceptiontable), READONLY}, + {"co_argcount", Py_T_INT, OFF(co_argcount), Py_READONLY}, + {"co_posonlyargcount", Py_T_INT, OFF(co_posonlyargcount), Py_READONLY}, + {"co_kwonlyargcount", Py_T_INT, OFF(co_kwonlyargcount), Py_READONLY}, + {"co_stacksize", Py_T_INT, OFF(co_stacksize), Py_READONLY}, + {"co_flags", Py_T_INT, OFF(co_flags), Py_READONLY}, + {"co_nlocals", Py_T_INT, OFF(co_nlocals), Py_READONLY}, + {"co_consts", _Py_T_OBJECT, OFF(co_consts), Py_READONLY}, + {"co_names", _Py_T_OBJECT, OFF(co_names), Py_READONLY}, + {"co_filename", _Py_T_OBJECT, OFF(co_filename), Py_READONLY}, + {"co_name", _Py_T_OBJECT, OFF(co_name), Py_READONLY}, + {"co_qualname", _Py_T_OBJECT, OFF(co_qualname), Py_READONLY}, + {"co_firstlineno", Py_T_INT, OFF(co_firstlineno), Py_READONLY}, + {"co_linetable", _Py_T_OBJECT, OFF(co_linetable), Py_READONLY}, + {"co_exceptiontable", _Py_T_OBJECT, OFF(co_exceptiontable), Py_READONLY}, {NULL} /* Sentinel */ }; @@ -2135,6 +2377,8 @@ static struct PyMethodDef code_methods[] = { {"co_positions", (PyCFunction)code_positionsiterator, METH_NOARGS}, CODE_REPLACE_METHODDEF CODE__VARNAME_FROM_OPARG_METHODDEF + {"__replace__", _PyCFunction_CAST(code_replace), METH_FASTCALL|METH_KEYWORDS, + PyDoc_STR("__replace__($self, /, **changes)\n--\n\nThe same as replace().")}, {NULL, NULL} /* sentinel */ }; @@ -2159,9 +2403,17 @@ PyTypeObject PyCode_Type = { PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ 0, /* tp_as_buffer */ +#ifdef Py_GIL_DISABLED + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ +#else Py_TPFLAGS_DEFAULT, /* tp_flags */ +#endif code_new__doc__, /* tp_doc */ +#ifdef Py_GIL_DISABLED + (traverseproc)code_traverse, /* tp_traverse */ +#else 0, /* tp_traverse */ +#endif 0, /* tp_clear */ code_richcompare, /* tp_richcompare */ offsetof(PyCodeObject, co_weakreflist), /* tp_weaklistoffset */ @@ -2314,118 +2566,188 @@ _PyCode_ConstantKey(PyObject *op) return key; } -void -_PyStaticCode_Fini(PyCodeObject *co) +#ifdef Py_GIL_DISABLED +static PyObject * +intern_one_constant(PyObject *op) { - deopt_code(co, _PyCode_CODE(co)); - PyMem_Free(co->co_extra); - if (co->_co_cached != NULL) { - Py_CLEAR(co->_co_cached->_co_code); - Py_CLEAR(co->_co_cached->_co_cellvars); - Py_CLEAR(co->_co_cached->_co_freevars); - Py_CLEAR(co->_co_cached->_co_varnames); - PyMem_Free(co->_co_cached); - co->_co_cached = NULL; - } - co->co_extra = NULL; - if (co->co_weakreflist != NULL) { - PyObject_ClearWeakRefs((PyObject *)co); - co->co_weakreflist = NULL; + PyInterpreterState *interp = _PyInterpreterState_GET(); + _Py_hashtable_t *consts = interp->code_state.constants; + + assert(!PyUnicode_CheckExact(op)); // strings are interned separately + + _Py_hashtable_entry_t *entry = _Py_hashtable_get_entry(consts, op); + if (entry == NULL) { + if (_Py_hashtable_set(consts, op, op) != 0) { + PyErr_NoMemory(); + return NULL; + } + +#ifdef Py_REF_DEBUG + Py_ssize_t refcnt = Py_REFCNT(op); + if (refcnt != 1) { + // Adjust the reftotal to account for the fact that we only + // restore a single reference in _PyCode_Fini. + _Py_AddRefTotal(_PyThreadState_GET(), -(refcnt - 1)); + } +#endif + + _Py_SetImmortal(op); + return op; } - free_monitoring_data(co->_co_monitoring); - co->_co_monitoring = NULL; + + assert(_Py_IsImmortal(entry->value)); + return (PyObject *)entry->value; } -int -_PyStaticCode_Init(PyCodeObject *co) +static int +compare_constants(const void *key1, const void *key2) { - int res = intern_strings(co->co_names); - if (res < 0) { - return -1; + PyObject *op1 = (PyObject *)key1; + PyObject *op2 = (PyObject *)key2; + if (op1 == op2) { + return 1; } - res = intern_string_constants(co->co_consts, NULL); - if (res < 0) { - return -1; + if (Py_TYPE(op1) != Py_TYPE(op2)) { + return 0; } - res = intern_strings(co->co_localsplusnames); - if (res < 0) { - return -1; + // We compare container contents by identity because we have already + // internalized the items. + if (PyTuple_CheckExact(op1)) { + Py_ssize_t size = PyTuple_GET_SIZE(op1); + if (size != PyTuple_GET_SIZE(op2)) { + return 0; + } + for (Py_ssize_t i = 0; i < size; i++) { + if (PyTuple_GET_ITEM(op1, i) != PyTuple_GET_ITEM(op2, i)) { + return 0; + } + } + return 1; } - _PyCode_Quicken(co); + else if (PyFrozenSet_CheckExact(op1)) { + if (PySet_GET_SIZE(op1) != PySet_GET_SIZE(op2)) { + return 0; + } + Py_ssize_t pos1 = 0, pos2 = 0; + PyObject *obj1, *obj2; + Py_hash_t hash1, hash2; + while ((_PySet_NextEntry(op1, &pos1, &obj1, &hash1)) && + (_PySet_NextEntry(op2, &pos2, &obj2, &hash2))) + { + if (obj1 != obj2) { + return 0; + } + } + return 1; + } + else if (PySlice_Check(op1)) { + PySliceObject *s1 = (PySliceObject *)op1; + PySliceObject *s2 = (PySliceObject *)op2; + return (s1->start == s2->start && + s1->stop == s2->stop && + s1->step == s2->step); + } + else if (PyBytes_CheckExact(op1) || PyLong_CheckExact(op1)) { + return PyObject_RichCompareBool(op1, op2, Py_EQ); + } + else if (PyFloat_CheckExact(op1)) { + // Ensure that, for example, +0.0 and -0.0 are distinct + double f1 = PyFloat_AS_DOUBLE(op1); + double f2 = PyFloat_AS_DOUBLE(op2); + return memcmp(&f1, &f2, sizeof(double)) == 0; + } + else if (PyComplex_CheckExact(op1)) { + Py_complex c1 = ((PyComplexObject *)op1)->cval; + Py_complex c2 = ((PyComplexObject *)op2)->cval; + return memcmp(&c1, &c2, sizeof(Py_complex)) == 0; + } + // gh-130851: Treat instances of unexpected types as distinct if they are + // not the same object. return 0; } -#define MAX_CODE_UNITS_PER_LOC_ENTRY 8 - -PyCodeObject * -_Py_MakeShimCode(const _PyShimCodeDef *codedef) +static Py_uhash_t +hash_const(const void *key) { - PyObject *name = NULL; - PyObject *co_code = NULL; - PyObject *lines = NULL; - PyCodeObject *codeobj = NULL; - uint8_t *loc_table = NULL; - - name = _PyUnicode_FromASCII(codedef->cname, strlen(codedef->cname)); - if (name == NULL) { - goto cleanup; + PyObject *op = (PyObject *)key; + if (PySlice_Check(op)) { + PySliceObject *s = (PySliceObject *)op; + PyObject *data[3] = { s->start, s->stop, s->step }; + return _Py_HashBytes(&data, sizeof(data)); } - co_code = PyBytes_FromStringAndSize( - (const char *)codedef->code, codedef->codelen); - if (co_code == NULL) { - goto cleanup; + else if (PyTuple_CheckExact(op)) { + Py_ssize_t size = PyTuple_GET_SIZE(op); + PyObject **data = _PyTuple_ITEMS(op); + return _Py_HashBytes(data, sizeof(PyObject *) * size); } - int code_units = codedef->codelen / sizeof(_Py_CODEUNIT); - int loc_entries = (code_units + MAX_CODE_UNITS_PER_LOC_ENTRY - 1) / - MAX_CODE_UNITS_PER_LOC_ENTRY; - loc_table = PyMem_Malloc(loc_entries); - if (loc_table == NULL) { - PyErr_NoMemory(); - goto cleanup; + Py_hash_t h = PyObject_Hash(op); + if (h == -1) { + // gh-130851: Other than slice objects, every constant that the + // bytecode compiler generates is hashable. However, users can + // provide their own constants, when constructing code objects via + // types.CodeType(). If the user-provided constant is unhashable, we + // use the memory address of the object as a fallback hash value. + PyErr_Clear(); + return (Py_uhash_t)(uintptr_t)key; } - for (int i = 0; i < loc_entries-1; i++) { - loc_table[i] = 0x80 | (PY_CODE_LOCATION_INFO_NONE << 3) | 7; - code_units -= MAX_CODE_UNITS_PER_LOC_ENTRY; + return (Py_uhash_t)h; +} + +static int +clear_containers(_Py_hashtable_t *ht, const void *key, const void *value, + void *user_data) +{ + // First clear containers to avoid recursive deallocation later on in + // destroy_key. + PyObject *op = (PyObject *)key; + if (PyTuple_CheckExact(op)) { + for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(op); i++) { + Py_CLEAR(_PyTuple_ITEMS(op)[i]); + } } - assert(loc_entries > 0); - assert(code_units > 0 && code_units <= MAX_CODE_UNITS_PER_LOC_ENTRY); - loc_table[loc_entries-1] = 0x80 | - (PY_CODE_LOCATION_INFO_NONE << 3) | (code_units-1); - lines = PyBytes_FromStringAndSize((const char *)loc_table, loc_entries); - PyMem_Free(loc_table); - if (lines == NULL) { - goto cleanup; + else if (PySlice_Check(op)) { + PySliceObject *slice = (PySliceObject *)op; + Py_SETREF(slice->start, Py_None); + Py_SETREF(slice->stop, Py_None); + Py_SETREF(slice->step, Py_None); } - _Py_DECLARE_STR(shim_name, "<shim>"); - struct _PyCodeConstructor con = { - .filename = &_Py_STR(shim_name), - .name = name, - .qualname = name, - .flags = CO_NEWLOCALS | CO_OPTIMIZED, - - .code = co_code, - .firstlineno = 1, - .linetable = lines, - - .consts = (PyObject *)&_Py_SINGLETON(tuple_empty), - .names = (PyObject *)&_Py_SINGLETON(tuple_empty), - - .localsplusnames = (PyObject *)&_Py_SINGLETON(tuple_empty), - .localspluskinds = (PyObject *)&_Py_SINGLETON(bytes_empty), - - .argcount = 0, - .posonlyargcount = 0, - .kwonlyargcount = 0, + else if (PyFrozenSet_CheckExact(op)) { + _PySet_ClearInternal((PySetObject *)op); + } + return 0; +} - .stacksize = codedef->stacksize, +static void +destroy_key(void *key) +{ + _Py_ClearImmortal(key); +} +#endif - .exceptiontable = (PyObject *)&_Py_SINGLETON(bytes_empty), - }; +PyStatus +_PyCode_Init(PyInterpreterState *interp) +{ +#ifdef Py_GIL_DISABLED + struct _py_code_state *state = &interp->code_state; + state->constants = _Py_hashtable_new_full(&hash_const, &compare_constants, + &destroy_key, NULL, NULL); + if (state->constants == NULL) { + return _PyStatus_NO_MEMORY(); + } +#endif + return _PyStatus_OK(); +} - codeobj = _PyCode_New(&con); -cleanup: - Py_XDECREF(name); - Py_XDECREF(co_code); - Py_XDECREF(lines); - return codeobj; +void +_PyCode_Fini(PyInterpreterState *interp) +{ +#ifdef Py_GIL_DISABLED + // Free interned constants + struct _py_code_state *state = &interp->code_state; + if (state->constants) { + _Py_hashtable_foreach(state->constants, &clear_containers, NULL); + _Py_hashtable_destroy(state->constants); + state->constants = NULL; + } +#endif } |