diff options
Diffstat (limited to 'contrib/tools/python3/Include/cpython')
48 files changed, 4351 insertions, 1676 deletions
diff --git a/contrib/tools/python3/Include/cpython/abstract.h b/contrib/tools/python3/Include/cpython/abstract.h index 3b27aab2fc4..4e7b7a46703 100644 --- a/contrib/tools/python3/Include/cpython/abstract.h +++ b/contrib/tools/python3/Include/cpython/abstract.h @@ -4,9 +4,12 @@ /* === Object Protocol ================================================== */ -#ifdef PY_SSIZE_T_CLEAN -# define _PyObject_CallMethodId _PyObject_CallMethodId_SizeT -#endif +/* Like PyObject_CallMethod(), but expect a _Py_Identifier* + as the method name. */ +PyAPI_FUNC(PyObject*) _PyObject_CallMethodId( + PyObject *obj, + _Py_Identifier *name, + const char *format, ...); /* Convert keyword arguments from the FASTCALL (stack: C array, kwnames: tuple) format to a Python dictionary ("kwargs" dict). @@ -18,51 +21,25 @@ Duplicate keys are merged using the last value. If duplicate keys must raise an exception, the caller is responsible to implement an explicit keys on kwnames. */ -PyAPI_FUNC(PyObject *) _PyStack_AsDict( - PyObject *const *values, - PyObject *kwnames); - -/* Suggested size (number of positional arguments) for arrays of PyObject* - allocated on a C stack to avoid allocating memory on the heap memory. Such - array is used to pass positional arguments to call functions of the - PyObject_Vectorcall() family. +PyAPI_FUNC(PyObject*) _PyStack_AsDict(PyObject *const *values, PyObject *kwnames); - The size is chosen to not abuse the C stack and so limit the risk of stack - overflow. The size is also chosen to allow using the small stack for most - function calls of the Python standard library. On 64-bit CPU, it allocates - 40 bytes on the stack. */ -#define _PY_FASTCALL_SMALL_STACK 5 - -PyAPI_FUNC(PyObject *) _Py_CheckFunctionResult( - PyThreadState *tstate, - PyObject *callable, - PyObject *result, - const char *where); /* === Vectorcall protocol (PEP 590) ============================= */ -/* Call callable using tp_call. Arguments are like PyObject_Vectorcall() - or PyObject_FastCallDict() (both forms are supported), - except that nargs is plainly the number of arguments without flags. */ -PyAPI_FUNC(PyObject *) _PyObject_MakeTpCall( - PyThreadState *tstate, - PyObject *callable, - PyObject *const *args, Py_ssize_t nargs, - PyObject *keywords); - // PyVectorcall_NARGS() is exported as a function for the stable ABI. // Here (when we are not using the stable ABI), the name is overridden to // call a static inline function for best performance. -#define PyVectorcall_NARGS(n) _PyVectorcall_NARGS(n) static inline Py_ssize_t _PyVectorcall_NARGS(size_t n) { return n & ~PY_VECTORCALL_ARGUMENTS_OFFSET; } +#define PyVectorcall_NARGS(n) _PyVectorcall_NARGS(n) PyAPI_FUNC(vectorcallfunc) PyVectorcall_Function(PyObject *callable); -// Backwards compatibility aliases for API that was provisional in Python 3.8 +// Backwards compatibility aliases (PEP 590) for API that was provisional +// in Python 3.8 #define _PyObject_Vectorcall PyObject_Vectorcall #define _PyObject_VectorcallMethod PyObject_VectorcallMethod #define _PyObject_FastCallDict PyObject_VectorcallDict @@ -79,12 +56,6 @@ PyAPI_FUNC(PyObject *) PyObject_VectorcallDict( size_t nargsf, PyObject *kwargs); -// Same as PyObject_Vectorcall(), except without keyword arguments -PyAPI_FUNC(PyObject *) _PyObject_FastCall( - PyObject *func, - PyObject *const *args, - Py_ssize_t nargs); - PyAPI_FUNC(PyObject *) PyObject_CallOneArg(PyObject *func, PyObject *arg); static inline PyObject * @@ -103,56 +74,6 @@ PyObject_CallMethodOneArg(PyObject *self, PyObject *name, PyObject *arg) return PyObject_VectorcallMethod(name, args, nargsf, _Py_NULL); } -PyAPI_FUNC(PyObject *) _PyObject_CallMethod(PyObject *obj, - PyObject *name, - const char *format, ...); - -/* Like PyObject_CallMethod(), but expect a _Py_Identifier* - as the method name. */ -PyAPI_FUNC(PyObject *) _PyObject_CallMethodId(PyObject *obj, - _Py_Identifier *name, - const char *format, ...); - -PyAPI_FUNC(PyObject *) _PyObject_CallMethodId_SizeT(PyObject *obj, - _Py_Identifier *name, - const char *format, - ...); - -PyAPI_FUNC(PyObject *) _PyObject_CallMethodIdObjArgs( - PyObject *obj, - _Py_Identifier *name, - ...); - -static inline PyObject * -_PyObject_VectorcallMethodId( - _Py_Identifier *name, PyObject *const *args, - size_t nargsf, PyObject *kwnames) -{ - PyObject *oname = _PyUnicode_FromId(name); /* borrowed */ - if (!oname) { - return _Py_NULL; - } - return PyObject_VectorcallMethod(oname, args, nargsf, kwnames); -} - -static inline PyObject * -_PyObject_CallMethodIdNoArgs(PyObject *self, _Py_Identifier *name) -{ - size_t nargsf = 1 | PY_VECTORCALL_ARGUMENTS_OFFSET; - return _PyObject_VectorcallMethodId(name, &self, nargsf, _Py_NULL); -} - -static inline PyObject * -_PyObject_CallMethodIdOneArg(PyObject *self, _Py_Identifier *name, PyObject *arg) -{ - PyObject *args[2] = {self, arg}; - size_t nargsf = 2 | PY_VECTORCALL_ARGUMENTS_OFFSET; - assert(arg != NULL); - return _PyObject_VectorcallMethodId(name, args, nargsf, _Py_NULL); -} - -PyAPI_FUNC(int) _PyObject_HasLen(PyObject *o); - /* Guess the size of object 'o' using len(o) or o.__length_hint__(). If neither of those return a non-negative value, then return the default value. If one of the calls fails, this function returns -1. */ @@ -164,43 +85,3 @@ PyAPI_FUNC(Py_ssize_t) PyObject_LengthHint(PyObject *o, Py_ssize_t); need to be corrected for a negative index. */ #define PySequence_ITEM(o, i)\ ( Py_TYPE(o)->tp_as_sequence->sq_item((o), (i)) ) - -#define PY_ITERSEARCH_COUNT 1 -#define PY_ITERSEARCH_INDEX 2 -#define PY_ITERSEARCH_CONTAINS 3 - -/* Iterate over seq. - - Result depends on the operation: - - PY_ITERSEARCH_COUNT: return # of times obj appears in seq; -1 if - error. - PY_ITERSEARCH_INDEX: return 0-based index of first occurrence of - obj in seq; set ValueError and return -1 if none found; - also return -1 on error. - PY_ITERSEARCH_CONTAINS: return 1 if obj in seq, else 0; -1 on - error. */ -PyAPI_FUNC(Py_ssize_t) _PySequence_IterSearch(PyObject *seq, - PyObject *obj, int operation); - -/* === Mapping protocol ================================================= */ - -PyAPI_FUNC(int) _PyObject_RealIsInstance(PyObject *inst, PyObject *cls); - -PyAPI_FUNC(int) _PyObject_RealIsSubclass(PyObject *derived, PyObject *cls); - -PyAPI_FUNC(char *const *) _PySequence_BytesToCharpArray(PyObject* self); - -PyAPI_FUNC(void) _Py_FreeCharPArray(char *const array[]); - -/* For internal use by buffer API functions */ -PyAPI_FUNC(void) _Py_add_one_to_index_F(int nd, Py_ssize_t *index, - const Py_ssize_t *shape); -PyAPI_FUNC(void) _Py_add_one_to_index_C(int nd, Py_ssize_t *index, - const Py_ssize_t *shape); - -/* Convert Python int to Py_ssize_t. Do nothing if the argument is None. */ -PyAPI_FUNC(int) _Py_convert_optional_to_ssize_t(PyObject *, void *); - -/* Same as PyNumber_Index but can return an instance of a subclass of int. */ -PyAPI_FUNC(PyObject *) _PyNumber_Index(PyObject *o); diff --git a/contrib/tools/python3/Include/cpython/bytesobject.h b/contrib/tools/python3/Include/cpython/bytesobject.h index e982031c107..41537210b74 100644 --- a/contrib/tools/python3/Include/cpython/bytesobject.h +++ b/contrib/tools/python3/Include/cpython/bytesobject.h @@ -15,18 +15,6 @@ typedef struct { } PyBytesObject; PyAPI_FUNC(int) _PyBytes_Resize(PyObject **, Py_ssize_t); -PyAPI_FUNC(PyObject*) _PyBytes_FormatEx( - const char *format, - Py_ssize_t format_len, - PyObject *args, - int use_bytearray); -PyAPI_FUNC(PyObject*) _PyBytes_FromHex( - PyObject *string, - int use_bytearray); - -/* Helper for PyBytes_DecodeEscape that detects invalid escape chars. */ -PyAPI_FUNC(PyObject *) _PyBytes_DecodeEscape(const char *, Py_ssize_t, - const char *, const char **); /* Macros and static inline functions, trading safety for speed */ #define _PyBytes_CAST(op) \ @@ -46,84 +34,4 @@ static inline Py_ssize_t PyBytes_GET_SIZE(PyObject *op) { /* _PyBytes_Join(sep, x) is like sep.join(x). sep must be PyBytesObject*, x must be an iterable object. */ -PyAPI_FUNC(PyObject *) _PyBytes_Join(PyObject *sep, PyObject *x); - - -/* The _PyBytesWriter structure is big: it contains an embedded "stack buffer". - A _PyBytesWriter variable must be declared at the end of variables in a - function to optimize the memory allocation on the stack. */ -typedef struct { - /* bytes, bytearray or NULL (when the small buffer is used) */ - PyObject *buffer; - - /* Number of allocated size. */ - Py_ssize_t allocated; - - /* Minimum number of allocated bytes, - incremented by _PyBytesWriter_Prepare() */ - Py_ssize_t min_size; - - /* If non-zero, use a bytearray instead of a bytes object for buffer. */ - int use_bytearray; - - /* If non-zero, overallocate the buffer (default: 0). - This flag must be zero if use_bytearray is non-zero. */ - int overallocate; - - /* Stack buffer */ - int use_small_buffer; - char small_buffer[512]; -} _PyBytesWriter; - -/* Initialize a bytes writer - - By default, the overallocation is disabled. Set the overallocate attribute - to control the allocation of the buffer. */ -PyAPI_FUNC(void) _PyBytesWriter_Init(_PyBytesWriter *writer); - -/* Get the buffer content and reset the writer. - Return a bytes object, or a bytearray object if use_bytearray is non-zero. - Raise an exception and return NULL on error. */ -PyAPI_FUNC(PyObject *) _PyBytesWriter_Finish(_PyBytesWriter *writer, - void *str); - -/* Deallocate memory of a writer (clear its internal buffer). */ -PyAPI_FUNC(void) _PyBytesWriter_Dealloc(_PyBytesWriter *writer); - -/* Allocate the buffer to write size bytes. - Return the pointer to the beginning of buffer data. - Raise an exception and return NULL on error. */ -PyAPI_FUNC(void*) _PyBytesWriter_Alloc(_PyBytesWriter *writer, - Py_ssize_t size); - -/* Ensure that the buffer is large enough to write *size* bytes. - Add size to the writer minimum size (min_size attribute). - - str is the current pointer inside the buffer. - Return the updated current pointer inside the buffer. - Raise an exception and return NULL on error. */ -PyAPI_FUNC(void*) _PyBytesWriter_Prepare(_PyBytesWriter *writer, - void *str, - Py_ssize_t size); - -/* Resize the buffer to make it larger. - The new buffer may be larger than size bytes because of overallocation. - Return the updated current pointer inside the buffer. - Raise an exception and return NULL on error. - - Note: size must be greater than the number of allocated bytes in the writer. - - This function doesn't use the writer minimum size (min_size attribute). - - See also _PyBytesWriter_Prepare(). - */ -PyAPI_FUNC(void*) _PyBytesWriter_Resize(_PyBytesWriter *writer, - void *str, - Py_ssize_t size); - -/* Write bytes. - Raise an exception and return NULL on error. */ -PyAPI_FUNC(void*) _PyBytesWriter_WriteBytes(_PyBytesWriter *writer, - void *str, - const void *bytes, - Py_ssize_t size); +PyAPI_FUNC(PyObject*) _PyBytes_Join(PyObject *sep, PyObject *x); diff --git a/contrib/tools/python3/Include/cpython/ceval.h b/contrib/tools/python3/Include/cpython/ceval.h index a9616bd6a4f..78f74056616 100644 --- a/contrib/tools/python3/Include/cpython/ceval.h +++ b/contrib/tools/python3/Include/cpython/ceval.h @@ -4,14 +4,9 @@ PyAPI_FUNC(void) PyEval_SetProfile(Py_tracefunc, PyObject *); PyAPI_FUNC(void) PyEval_SetProfileAllThreads(Py_tracefunc, PyObject *); -PyAPI_DATA(int) _PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg); PyAPI_FUNC(void) PyEval_SetTrace(Py_tracefunc, PyObject *); PyAPI_FUNC(void) PyEval_SetTraceAllThreads(Py_tracefunc, PyObject *); -PyAPI_FUNC(int) _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg); -/* Helper to look up a builtin object */ -PyAPI_FUNC(PyObject *) _PyEval_GetBuiltin(PyObject *); -PyAPI_FUNC(PyObject *) _PyEval_GetBuiltinId(_Py_Identifier *); /* Look at the current frame's (if any) code's co_flags, and turn on the corresponding compiler flags in cf->cf_flags. Return 1 if any flag was set, else return 0. */ @@ -19,11 +14,6 @@ PyAPI_FUNC(int) PyEval_MergeCompilerFlags(PyCompilerFlags *cf); PyAPI_FUNC(PyObject *) _PyEval_EvalFrameDefault(PyThreadState *tstate, struct _PyInterpreterFrame *f, int exc); -PyAPI_FUNC(void) _PyEval_SetSwitchInterval(unsigned long microseconds); -PyAPI_FUNC(unsigned long) _PyEval_GetSwitchInterval(void); - -PyAPI_FUNC(int) _PyEval_MakePendingCalls(PyThreadState *); - PyAPI_FUNC(Py_ssize_t) PyUnstable_Eval_RequestCodeExtraIndex(freefunc); // Old name -- remove when this API changes: _Py_DEPRECATED_EXTERNALLY(3.12) static inline Py_ssize_t diff --git a/contrib/tools/python3/Include/cpython/code.h b/contrib/tools/python3/Include/cpython/code.h index 81741c547b7..bd8afab8f93 100644 --- a/contrib/tools/python3/Include/cpython/code.h +++ b/contrib/tools/python3/Include/cpython/code.h @@ -16,57 +16,14 @@ extern "C" { #define _PY_MONITORING_EVENTS 17 /* Tables of which tools are active for each monitored event. */ -/* For 3.12 ABI compatibility this is over sized */ typedef struct _Py_LocalMonitors { - /* Only _PY_MONITORING_LOCAL_EVENTS of these are used */ - uint8_t tools[_PY_MONITORING_UNGROUPED_EVENTS]; + uint8_t tools[_PY_MONITORING_LOCAL_EVENTS]; } _Py_LocalMonitors; typedef struct _Py_GlobalMonitors { uint8_t tools[_PY_MONITORING_UNGROUPED_EVENTS]; } _Py_GlobalMonitors; -/* Each instruction in a code object is a fixed-width value, - * currently 2 bytes: 1-byte opcode + 1-byte oparg. The EXTENDED_ARG - * opcode allows for larger values but the current limit is 3 uses - * of EXTENDED_ARG (see Python/compile.c), for a maximum - * 32-bit value. This aligns with the note in Python/compile.c - * (compiler_addop_i_line) indicating that the max oparg value is - * 2**32 - 1, rather than INT_MAX. - */ - -typedef union { - uint16_t cache; - struct { - uint8_t code; - uint8_t arg; - } op; -} _Py_CODEUNIT; - - -/* These macros only remain defined for compatibility. */ -#define _Py_OPCODE(word) ((word).op.code) -#define _Py_OPARG(word) ((word).op.arg) - -static inline _Py_CODEUNIT -_py_make_codeunit(uint8_t opcode, uint8_t oparg) -{ - // No designated initialisers because of C++ compat - _Py_CODEUNIT word; - word.op.code = opcode; - word.op.arg = oparg; - return word; -} - -static inline void -_py_set_opcode(_Py_CODEUNIT *word, uint8_t opcode) -{ - word->op.code = opcode; -} - -#define _Py_MAKE_CODEUNIT(opcode, oparg) _py_make_codeunit((opcode), (oparg)) -#define _Py_SET_OPCODE(word, opcode) _py_set_opcode(&(word), (opcode)) - typedef struct { PyObject *_co_code; @@ -76,13 +33,21 @@ typedef struct { } _PyCoCached; /* Ancillary data structure used for instrumentation. - Line instrumentation creates an array of - these. One entry per code unit.*/ + Line instrumentation creates this with sufficient + space for one entry per code unit. The total size + of the data will be `bytes_per_entry * Py_SIZE(code)` */ typedef struct { - uint8_t original_opcode; - int8_t line_delta; + uint8_t bytes_per_entry; + uint8_t data[1]; } _PyCoLineInstrumentationData; + +typedef struct { + int size; + int capacity; + struct _PyExecutorObject *executors[1]; +} _PyExecutorArray; + /* Main data structure used for instrumentation. * This is allocated when needed for instrumentation */ @@ -160,8 +125,9 @@ typedef struct { PyObject *co_qualname; /* unicode (qualname, for reference) */ \ PyObject *co_linetable; /* bytes object that holds location info */ \ PyObject *co_weakreflist; /* to support weakrefs to code objects */ \ + _PyExecutorArray *co_executors; /* executors from optimizer */ \ _PyCoCached *_co_cached; /* cached co_* attributes */ \ - uint64_t _co_instrumentation_version; /* current instrumentation version */ \ + uintptr_t _co_instrumentation_version; /* current instrumentation version */ \ _PyCoMonitoringData *_co_monitoring; /* Monitoring data */ \ int _co_firsttraceable; /* index of first traceable instruction */ \ /* Scratch space for extra data relating to the code object. \ @@ -202,6 +168,8 @@ struct PyCodeObject _PyCode_DEF(1); #define CO_FUTURE_GENERATOR_STOP 0x800000 #define CO_FUTURE_ANNOTATIONS 0x1000000 +#define CO_NO_MONITORING_EVENTS 0x2000000 + /* This should be defined if a future statement modifies the syntax. For example, when a keyword is added. */ @@ -218,13 +186,14 @@ static inline Py_ssize_t PyCode_GetNumFree(PyCodeObject *op) { return op->co_nfreevars; } -static inline int PyCode_GetFirstFree(PyCodeObject *op) { +static inline int PyUnstable_Code_GetFirstFree(PyCodeObject *op) { assert(PyCode_Check(op)); return op->co_nlocalsplus - op->co_nfreevars; } -#define _PyCode_CODE(CO) _Py_RVALUE((_Py_CODEUNIT *)(CO)->co_code_adaptive) -#define _PyCode_NBYTES(CO) (Py_SIZE(CO) * (Py_ssize_t)sizeof(_Py_CODEUNIT)) +Py_DEPRECATED(3.13) static inline int PyCode_GetFirstFree(PyCodeObject *op) { + return PyUnstable_Code_GetFirstFree(op); +} /* Unstable public interface */ PyAPI_FUNC(PyCodeObject *) PyUnstable_Code_New( diff --git a/contrib/tools/python3/Include/cpython/compile.h b/contrib/tools/python3/Include/cpython/compile.h index f5a62a8ec6d..cfdb7080d45 100644 --- a/contrib/tools/python3/Include/cpython/compile.h +++ b/contrib/tools/python3/Include/cpython/compile.h @@ -19,9 +19,10 @@ #define PyCF_TYPE_COMMENTS 0x1000 #define PyCF_ALLOW_TOP_LEVEL_AWAIT 0x2000 #define PyCF_ALLOW_INCOMPLETE_INPUT 0x4000 +#define PyCF_OPTIMIZED_AST (0x8000 | PyCF_ONLY_AST) #define PyCF_COMPILE_MASK (PyCF_ONLY_AST | PyCF_ALLOW_TOP_LEVEL_AWAIT | \ PyCF_TYPE_COMMENTS | PyCF_DONT_IMPLY_DEDENT | \ - PyCF_ALLOW_INCOMPLETE_INPUT) + PyCF_ALLOW_INCOMPLETE_INPUT | PyCF_OPTIMIZED_AST) typedef struct { int cf_flags; /* bitmask of CO_xxx flags relevant to future */ @@ -31,28 +32,8 @@ typedef struct { #define _PyCompilerFlags_INIT \ (PyCompilerFlags){.cf_flags = 0, .cf_feature_version = PY_MINOR_VERSION} -/* source location information */ -typedef struct { - int lineno; - int end_lineno; - int col_offset; - int end_col_offset; -} _PyCompilerSrcLocation; - -#define SRC_LOCATION_FROM_AST(n) \ - (_PyCompilerSrcLocation){ \ - .lineno = (n)->lineno, \ - .end_lineno = (n)->end_lineno, \ - .col_offset = (n)->col_offset, \ - .end_col_offset = (n)->end_col_offset } - /* Future feature support */ -typedef struct { - int ff_features; /* flags set by future statements */ - _PyCompilerSrcLocation ff_location; /* location of last future statement */ -} PyFutureFeatures; - #define FUTURE_NESTED_SCOPES "nested_scopes" #define FUTURE_GENERATORS "generators" #define FUTURE_DIVISION "division" diff --git a/contrib/tools/python3/Include/cpython/complexobject.h b/contrib/tools/python3/Include/cpython/complexobject.h index b7d7283ae88..fbdc6a91fe8 100644 --- a/contrib/tools/python3/Include/cpython/complexobject.h +++ b/contrib/tools/python3/Include/cpython/complexobject.h @@ -7,8 +7,7 @@ typedef struct { double imag; } Py_complex; -/* Operations on complex numbers from complexmodule.c */ - +// Operations on complex numbers. PyAPI_FUNC(Py_complex) _Py_c_sum(Py_complex, Py_complex); PyAPI_FUNC(Py_complex) _Py_c_diff(Py_complex, Py_complex); PyAPI_FUNC(Py_complex) _Py_c_neg(Py_complex); @@ -17,6 +16,7 @@ PyAPI_FUNC(Py_complex) _Py_c_quot(Py_complex, Py_complex); PyAPI_FUNC(Py_complex) _Py_c_pow(Py_complex, Py_complex); PyAPI_FUNC(double) _Py_c_abs(Py_complex); + /* Complex object interface */ /* @@ -31,14 +31,3 @@ typedef struct { PyAPI_FUNC(PyObject *) PyComplex_FromCComplex(Py_complex); PyAPI_FUNC(Py_complex) PyComplex_AsCComplex(PyObject *op); - -#ifdef Py_BUILD_CORE -/* Format the object based on the format_spec, as defined in PEP 3101 - (Advanced String Formatting). */ -extern int _PyComplex_FormatAdvancedWriter( - _PyUnicodeWriter *writer, - PyObject *obj, - PyObject *format_spec, - Py_ssize_t start, - Py_ssize_t end); -#endif // Py_BUILD_CORE diff --git a/contrib/tools/python3/Include/cpython/context.h b/contrib/tools/python3/Include/cpython/context.h index 9879fc7192e..a3249fc29b0 100644 --- a/contrib/tools/python3/Include/cpython/context.h +++ b/contrib/tools/python3/Include/cpython/context.h @@ -67,10 +67,6 @@ PyAPI_FUNC(PyObject *) PyContextVar_Set(PyObject *var, PyObject *value); PyAPI_FUNC(int) PyContextVar_Reset(PyObject *var, PyObject *token); -/* This method is exposed only for CPython tests. Don not use it. */ -PyAPI_FUNC(PyObject *) _PyContext_NewHamtForTests(void); - - #ifdef __cplusplus } #endif diff --git a/contrib/tools/python3/Include/cpython/critical_section.h b/contrib/tools/python3/Include/cpython/critical_section.h new file mode 100644 index 00000000000..35db3fb6a59 --- /dev/null +++ b/contrib/tools/python3/Include/cpython/critical_section.h @@ -0,0 +1,134 @@ +#ifndef Py_CPYTHON_CRITICAL_SECTION_H +# error "this header file must not be included directly" +#endif + +// Python critical sections +// +// Conceptually, critical sections are a deadlock avoidance layer on top of +// per-object locks. These helpers, in combination with those locks, replace +// our usage of the global interpreter lock to provide thread-safety for +// otherwise thread-unsafe objects, such as dict. +// +// NOTE: These APIs are no-ops in non-free-threaded builds. +// +// Straightforward per-object locking could introduce deadlocks that were not +// present when running with the GIL. Threads may hold locks for multiple +// objects simultaneously because Python operations can nest. If threads were +// to acquire the same locks in different orders, they would deadlock. +// +// One way to avoid deadlocks is to allow threads to hold only the lock (or +// locks) for a single operation at a time (typically a single lock, but some +// operations involve two locks). When a thread begins a nested operation it +// could suspend the locks for any outer operation: before beginning the nested +// operation, the locks for the outer operation are released and when the +// nested operation completes, the locks for the outer operation are +// reacquired. +// +// To improve performance, this API uses a variation of the above scheme. +// Instead of immediately suspending locks any time a nested operation begins, +// locks are only suspended if the thread would block. This reduces the number +// of lock acquisitions and releases for nested operations, while still +// avoiding deadlocks. +// +// Additionally, the locks for any active operation are suspended around +// other potentially blocking operations, such as I/O. This is because the +// interaction between locks and blocking operations can lead to deadlocks in +// the same way as the interaction between multiple locks. +// +// Each thread's critical sections and their corresponding locks are tracked in +// a stack in `PyThreadState.critical_section`. When a thread calls +// `_PyThreadState_Detach()`, such as before a blocking I/O operation or when +// waiting to acquire a lock, the thread suspends all of its active critical +// sections, temporarily releasing the associated locks. When the thread calls +// `_PyThreadState_Attach()`, it resumes the top-most (i.e., most recent) +// critical section by reacquiring the associated lock or locks. See +// `_PyCriticalSection_Resume()`. +// +// NOTE: Only the top-most critical section is guaranteed to be active. +// Operations that need to lock two objects at once must use +// `Py_BEGIN_CRITICAL_SECTION2()`. You *CANNOT* use nested critical sections +// to lock more than one object at once, because the inner critical section +// may suspend the outer critical sections. This API does not provide a way +// to lock more than two objects at once (though it could be added later +// if actually needed). +// +// NOTE: Critical sections implicitly behave like reentrant locks because +// attempting to acquire the same lock will suspend any outer (earlier) +// critical sections. However, they are less efficient for this use case than +// purposefully designed reentrant locks. +// +// Example usage: +// Py_BEGIN_CRITICAL_SECTION(op); +// ... +// Py_END_CRITICAL_SECTION(); +// +// To lock two objects at once: +// Py_BEGIN_CRITICAL_SECTION2(op1, op2); +// ... +// Py_END_CRITICAL_SECTION2(); + +typedef struct PyCriticalSection PyCriticalSection; +typedef struct PyCriticalSection2 PyCriticalSection2; + +PyAPI_FUNC(void) +PyCriticalSection_Begin(PyCriticalSection *c, PyObject *op); + +PyAPI_FUNC(void) +PyCriticalSection_End(PyCriticalSection *c); + +PyAPI_FUNC(void) +PyCriticalSection2_Begin(PyCriticalSection2 *c, PyObject *a, PyObject *b); + +PyAPI_FUNC(void) +PyCriticalSection2_End(PyCriticalSection2 *c); + +#ifndef Py_GIL_DISABLED +# define Py_BEGIN_CRITICAL_SECTION(op) \ + { +# define Py_END_CRITICAL_SECTION() \ + } +# define Py_BEGIN_CRITICAL_SECTION2(a, b) \ + { +# define Py_END_CRITICAL_SECTION2() \ + } +#else /* !Py_GIL_DISABLED */ + +// NOTE: the contents of this struct are private and may change betweeen +// Python releases without a deprecation period. +struct PyCriticalSection { + // Tagged pointer to an outer active critical section (or 0). + uintptr_t _cs_prev; + + // Mutex used to protect critical section + PyMutex *_cs_mutex; +}; + +// A critical section protected by two mutexes. Use +// Py_BEGIN_CRITICAL_SECTION2 and Py_END_CRITICAL_SECTION2. +// NOTE: the contents of this struct are private and may change betweeen +// Python releases without a deprecation period. +struct PyCriticalSection2 { + PyCriticalSection _cs_base; + + PyMutex *_cs_mutex2; +}; + +# define Py_BEGIN_CRITICAL_SECTION(op) \ + { \ + PyCriticalSection _py_cs; \ + PyCriticalSection_Begin(&_py_cs, _PyObject_CAST(op)) + +# define Py_END_CRITICAL_SECTION() \ + PyCriticalSection_End(&_py_cs); \ + } + +# define Py_BEGIN_CRITICAL_SECTION2(a, b) \ + { \ + PyCriticalSection2 _py_cs2; \ + PyCriticalSection2_Begin(&_py_cs2, _PyObject_CAST(a), _PyObject_CAST(b)) + +# define Py_END_CRITICAL_SECTION2() \ + PyCriticalSection2_End(&_py_cs2); \ + } + +#endif diff --git a/contrib/tools/python3/Include/cpython/descrobject.h b/contrib/tools/python3/Include/cpython/descrobject.h index e2ea1b9a2d3..bbad8b59c22 100644 --- a/contrib/tools/python3/Include/cpython/descrobject.h +++ b/contrib/tools/python3/Include/cpython/descrobject.h @@ -57,8 +57,6 @@ typedef struct { void *d_wrapped; /* This can be any function pointer */ } PyWrapperDescrObject; -PyAPI_DATA(PyTypeObject) _PyMethodWrapper_Type; - PyAPI_FUNC(PyObject *) PyDescr_NewWrapper(PyTypeObject *, struct wrapperbase *, void *); PyAPI_FUNC(int) PyDescr_IsData(PyObject *); diff --git a/contrib/tools/python3/Include/cpython/dictobject.h b/contrib/tools/python3/Include/cpython/dictobject.h index ddada922020..3fd23b9313c 100644 --- a/contrib/tools/python3/Include/cpython/dictobject.h +++ b/contrib/tools/python3/Include/cpython/dictobject.h @@ -17,6 +17,9 @@ typedef struct { /* Dictionary version: globally unique, value change each time the dictionary is modified */ #ifdef Py_BUILD_CORE + /* Bits 0-7 are for dict watchers. + * Bits 8-11 are for the watched mutation counter (used by tier2 optimization) + * The remaining bits (12-63) are the actual version tag. */ uint64_t ma_version_tag; #else Py_DEPRECATED(3.12) uint64_t ma_version_tag; @@ -33,60 +36,41 @@ typedef struct { } PyDictObject; PyAPI_FUNC(PyObject *) _PyDict_GetItem_KnownHash(PyObject *mp, PyObject *key, - Py_hash_t hash); -PyAPI_FUNC(PyObject *) _PyDict_GetItemWithError(PyObject *dp, PyObject *key); -PyAPI_FUNC(PyObject *) _PyDict_GetItemIdWithError(PyObject *dp, - _Py_Identifier *key); + Py_hash_t hash); PyAPI_FUNC(PyObject *) _PyDict_GetItemStringWithError(PyObject *, const char *); PyAPI_FUNC(PyObject *) PyDict_SetDefault( PyObject *mp, PyObject *key, PyObject *defaultobj); -PyAPI_FUNC(int) _PyDict_SetItem_KnownHash(PyObject *mp, PyObject *key, - PyObject *item, Py_hash_t hash); -PyAPI_FUNC(int) _PyDict_DelItem_KnownHash(PyObject *mp, PyObject *key, - Py_hash_t hash); -PyAPI_FUNC(int) _PyDict_DelItemIf(PyObject *mp, PyObject *key, - int (*predicate)(PyObject *value)); -PyAPI_FUNC(int) _PyDict_Next( - PyObject *mp, Py_ssize_t *pos, PyObject **key, PyObject **value, Py_hash_t *hash); + +// Inserts `key` with a value `default_value`, if `key` is not already present +// in the dictionary. If `result` is not NULL, then the value associated +// with `key` is returned in `*result` (either the existing value, or the now +// inserted `default_value`). +// Returns: +// -1 on error +// 0 if `key` was not present and `default_value` was inserted +// 1 if `key` was present and `default_value` was not inserted +PyAPI_FUNC(int) PyDict_SetDefaultRef(PyObject *mp, PyObject *key, PyObject *default_value, PyObject **result); /* Get the number of items of a dictionary. */ static inline Py_ssize_t PyDict_GET_SIZE(PyObject *op) { PyDictObject *mp; assert(PyDict_Check(op)); mp = _Py_CAST(PyDictObject*, op); +#ifdef Py_GIL_DISABLED + return _Py_atomic_load_ssize_relaxed(&mp->ma_used); +#else return mp->ma_used; +#endif } #define PyDict_GET_SIZE(op) PyDict_GET_SIZE(_PyObject_CAST(op)) -PyAPI_FUNC(int) _PyDict_Contains_KnownHash(PyObject *, PyObject *, Py_hash_t); -PyAPI_FUNC(int) _PyDict_ContainsId(PyObject *, _Py_Identifier *); -PyAPI_FUNC(PyObject *) _PyDict_NewPresized(Py_ssize_t minused); -PyAPI_FUNC(void) _PyDict_MaybeUntrack(PyObject *mp); -PyAPI_FUNC(int) _PyDict_HasOnlyStringKeys(PyObject *mp); -PyAPI_FUNC(Py_ssize_t) _PyDict_SizeOf(PyDictObject *); -PyAPI_FUNC(PyObject *) _PyDict_Pop(PyObject *, PyObject *, PyObject *); -#define _PyDict_HasSplitTable(d) ((d)->ma_values != NULL) - -/* Like PyDict_Merge, but override can be 0, 1 or 2. If override is 0, - the first occurrence of a key wins, if override is 1, the last occurrence - of a key wins, if override is 2, a KeyError with conflicting key as - argument is raised. -*/ -PyAPI_FUNC(int) _PyDict_MergeEx(PyObject *mp, PyObject *other, int override); -PyAPI_FUNC(int) _PyDict_SetItemId(PyObject *dp, _Py_Identifier *key, PyObject *item); +PyAPI_FUNC(int) PyDict_ContainsString(PyObject *mp, const char *key); -PyAPI_FUNC(int) _PyDict_DelItemId(PyObject *mp, _Py_Identifier *key); -PyAPI_FUNC(void) _PyDict_DebugMallocStats(FILE *out); - -/* _PyDictView */ - -typedef struct { - PyObject_HEAD - PyDictObject *dv_dict; -} _PyDictViewObject; +PyAPI_FUNC(PyObject *) _PyDict_NewPresized(Py_ssize_t minused); -PyAPI_FUNC(PyObject *) _PyDictView_New(PyObject *, PyTypeObject *); -PyAPI_FUNC(PyObject *) _PyDictView_Intersect(PyObject* self, PyObject *other); +PyAPI_FUNC(int) PyDict_Pop(PyObject *dict, PyObject *key, PyObject **result); +PyAPI_FUNC(int) PyDict_PopString(PyObject *dict, const char *key, PyObject **result); +PyAPI_FUNC(PyObject *) _PyDict_Pop(PyObject *dict, PyObject *key, PyObject *default_value); /* Dictionary watchers */ diff --git a/contrib/tools/python3/Include/cpython/fileobject.h b/contrib/tools/python3/Include/cpython/fileobject.h index b70ec318986..e2d89c522bd 100644 --- a/contrib/tools/python3/Include/cpython/fileobject.h +++ b/contrib/tools/python3/Include/cpython/fileobject.h @@ -3,7 +3,6 @@ #endif PyAPI_FUNC(char *) Py_UniversalNewlineFgets(char *, int, FILE*, PyObject *); -PyAPI_FUNC(char *) _Py_UniversalNewlineFgetsWithSize(char *, int, FILE*, PyObject *, size_t*); /* The std printer acts as a preliminary sys.stderr until the new io infrastructure is in place. */ @@ -15,5 +14,3 @@ typedef PyObject * (*Py_OpenCodeHookFunction)(PyObject *, void *); PyAPI_FUNC(PyObject *) PyFile_OpenCode(const char *utf8path); PyAPI_FUNC(PyObject *) PyFile_OpenCodeObject(PyObject *path); PyAPI_FUNC(int) PyFile_SetOpenCodeHook(Py_OpenCodeHookFunction hook, void *userData); - -PyAPI_FUNC(int) _PyLong_FileDescriptor_Converter(PyObject *, void *); diff --git a/contrib/tools/python3/Include/cpython/frameobject.h b/contrib/tools/python3/Include/cpython/frameobject.h index 4e19535c656..dbbfbb5105b 100644 --- a/contrib/tools/python3/Include/cpython/frameobject.h +++ b/contrib/tools/python3/Include/cpython/frameobject.h @@ -27,3 +27,9 @@ PyAPI_FUNC(int) _PyFrame_IsEntryFrame(PyFrameObject *frame); PyAPI_FUNC(int) PyFrame_FastToLocalsWithError(PyFrameObject *f); PyAPI_FUNC(void) PyFrame_FastToLocals(PyFrameObject *); + + +typedef struct { + PyObject_HEAD + PyFrameObject* frame; +} PyFrameLocalsProxyObject; diff --git a/contrib/tools/python3/Include/cpython/funcobject.h b/contrib/tools/python3/Include/cpython/funcobject.h index 23c8009c38d..5433ba48eef 100644 --- a/contrib/tools/python3/Include/cpython/funcobject.h +++ b/contrib/tools/python3/Include/cpython/funcobject.h @@ -81,12 +81,6 @@ PyAPI_FUNC(int) PyFunction_SetClosure(PyObject *, PyObject *); PyAPI_FUNC(PyObject *) PyFunction_GetAnnotations(PyObject *); PyAPI_FUNC(int) PyFunction_SetAnnotations(PyObject *, PyObject *); -PyAPI_FUNC(PyObject *) _PyFunction_Vectorcall( - PyObject *func, - PyObject *const *stack, - size_t nargsf, - PyObject *kwnames); - #define _PyFunction_CAST(func) \ (assert(PyFunction_Check(func)), _Py_CAST(PyFunctionObject*, func)) diff --git a/contrib/tools/python3/Include/cpython/genobject.h b/contrib/tools/python3/Include/cpython/genobject.h index 7856481b5db..49e46c277d7 100644 --- a/contrib/tools/python3/Include/cpython/genobject.h +++ b/contrib/tools/python3/Include/cpython/genobject.h @@ -41,9 +41,6 @@ PyAPI_DATA(PyTypeObject) PyGen_Type; PyAPI_FUNC(PyObject *) PyGen_New(PyFrameObject *); PyAPI_FUNC(PyObject *) PyGen_NewWithQualName(PyFrameObject *, PyObject *name, PyObject *qualname); -PyAPI_FUNC(int) _PyGen_SetStopIterationValue(PyObject *); -PyAPI_FUNC(int) _PyGen_FetchStopIterationValue(PyObject **); -PyAPI_FUNC(void) _PyGen_Finalize(PyObject *self); PyAPI_FUNC(PyCodeObject *) PyGen_GetCode(PyGenObject *gen); @@ -54,7 +51,6 @@ typedef struct { } PyCoroObject; PyAPI_DATA(PyTypeObject) PyCoro_Type; -PyAPI_DATA(PyTypeObject) _PyCoroWrapper_Type; #define PyCoro_CheckExact(op) Py_IS_TYPE((op), &PyCoro_Type) PyAPI_FUNC(PyObject *) PyCoro_New(PyFrameObject *, @@ -69,8 +65,6 @@ typedef struct { PyAPI_DATA(PyTypeObject) PyAsyncGen_Type; PyAPI_DATA(PyTypeObject) _PyAsyncGenASend_Type; -PyAPI_DATA(PyTypeObject) _PyAsyncGenWrappedValue_Type; -PyAPI_DATA(PyTypeObject) _PyAsyncGenAThrow_Type; PyAPI_FUNC(PyObject *) PyAsyncGen_New(PyFrameObject *, PyObject *name, PyObject *qualname); diff --git a/contrib/tools/python3/Include/cpython/import.h b/contrib/tools/python3/Include/cpython/import.h index 2bca4ade4c4..7daf0b84fcf 100644 --- a/contrib/tools/python3/Include/cpython/import.h +++ b/contrib/tools/python3/Include/cpython/import.h @@ -4,23 +4,6 @@ PyMODINIT_FUNC PyInit__imp(void); -PyAPI_FUNC(int) _PyImport_IsInitialized(PyInterpreterState *); - -PyAPI_FUNC(PyObject *) _PyImport_GetModuleId(_Py_Identifier *name); -PyAPI_FUNC(int) _PyImport_SetModule(PyObject *name, PyObject *module); -PyAPI_FUNC(int) _PyImport_SetModuleString(const char *name, PyObject* module); - -PyAPI_FUNC(void) _PyImport_AcquireLock(PyInterpreterState *interp); -PyAPI_FUNC(int) _PyImport_ReleaseLock(PyInterpreterState *interp); - -PyAPI_FUNC(int) _PyImport_FixupBuiltin( - PyObject *mod, - const char *name, /* UTF-8 encoded string */ - PyObject *modules - ); -PyAPI_FUNC(int) _PyImport_FixupExtensionObject(PyObject*, PyObject *, - PyObject *, PyObject *); - struct _inittab { const char *name; /* ASCII encoded string */ PyObject* (*initfunc)(void); @@ -34,13 +17,9 @@ struct _frozen { const unsigned char *code; int size; int is_package; - PyObject *(*get_code)(void); }; /* Embedding apps may change this pointer to point to their favorite collection of frozen modules: */ PyAPI_DATA(const struct _frozen *) PyImport_FrozenModules; - -PyAPI_DATA(PyObject *) _PyImport_GetModuleAttr(PyObject *, PyObject *); -PyAPI_DATA(PyObject *) _PyImport_GetModuleAttrString(const char *, const char *); diff --git a/contrib/tools/python3/Include/cpython/initconfig.h b/contrib/tools/python3/Include/cpython/initconfig.h index cbae97f12f5..5da5ef9e543 100644 --- a/contrib/tools/python3/Include/cpython/initconfig.h +++ b/contrib/tools/python3/Include/cpython/initconfig.h @@ -25,7 +25,6 @@ PyAPI_FUNC(PyStatus) PyStatus_Exit(int exitcode); PyAPI_FUNC(int) PyStatus_IsError(PyStatus err); PyAPI_FUNC(int) PyStatus_IsExit(PyStatus err); PyAPI_FUNC(int) PyStatus_Exception(PyStatus err); -PyAPI_FUNC(PyObject *) _PyErr_SetFromPyStatus(PyStatus status); /* --- PyWideStringList ------------------------------------------------ */ @@ -181,6 +180,11 @@ typedef struct PyConfig { int safe_path; int int_max_str_digits; + int cpu_count; +#ifdef Py_GIL_DISABLED + int enable_gil; +#endif + /* --- Path configuration inputs ------------ */ int pathconfig_warnings; wchar_t *program_name; @@ -205,6 +209,9 @@ typedef struct PyConfig { wchar_t *run_module; wchar_t *run_filename; + /* --- Set by Py_Main() -------------------------- */ + wchar_t *sys_path_0; + /* --- Private fields ---------------------------- */ // Install importlib? If equals to 0, importlib is not initialized at all. @@ -216,6 +223,17 @@ typedef struct PyConfig { // If non-zero, we believe we're running from a source tree. int _is_python_build; + +#ifdef Py_STATS + // If non-zero, turns on statistics gathering. + int _pystats; +#endif + +#ifdef Py_DEBUG + // If not empty, import a non-__main__ module before site.py is executed. + // PYTHON_PRESITE=package.module or -X presite=package.module + wchar_t *run_presite; +#endif } PyConfig; PyAPI_FUNC(void) PyConfig_InitPythonConfig(PyConfig *config); diff --git a/contrib/tools/python3/Include/cpython/interpreteridobject.h b/contrib/tools/python3/Include/cpython/interpreteridobject.h deleted file mode 100644 index 5076584209b..00000000000 --- a/contrib/tools/python3/Include/cpython/interpreteridobject.h +++ /dev/null @@ -1,11 +0,0 @@ -#ifndef Py_CPYTHON_INTERPRETERIDOBJECT_H -# error "this header file must not be included directly" -#endif - -/* Interpreter ID Object */ - -PyAPI_DATA(PyTypeObject) _PyInterpreterID_Type; - -PyAPI_FUNC(PyObject *) _PyInterpreterID_New(int64_t); -PyAPI_FUNC(PyObject *) _PyInterpreterState_GetIDObject(PyInterpreterState *); -PyAPI_FUNC(PyInterpreterState *) _PyInterpreterID_LookUp(PyObject *); diff --git a/contrib/tools/python3/Include/cpython/listobject.h b/contrib/tools/python3/Include/cpython/listobject.h index 8fa82122d8d..49f5e8d6d1a 100644 --- a/contrib/tools/python3/Include/cpython/listobject.h +++ b/contrib/tools/python3/Include/cpython/listobject.h @@ -21,9 +21,6 @@ typedef struct { Py_ssize_t allocated; } PyListObject; -PyAPI_FUNC(PyObject *) _PyList_Extend(PyListObject *, PyObject *); -PyAPI_FUNC(void) _PyList_DebugMallocStats(FILE *out); - /* Cast argument to PyListObject* type. */ #define _PyList_CAST(op) \ (assert(PyList_Check(op)), _Py_CAST(PyListObject*, (op))) @@ -32,7 +29,11 @@ PyAPI_FUNC(void) _PyList_DebugMallocStats(FILE *out); static inline Py_ssize_t PyList_GET_SIZE(PyObject *op) { PyListObject *list = _PyList_CAST(op); +#ifdef Py_GIL_DISABLED + return _Py_atomic_load_ssize_relaxed(&(_PyVarObject_CAST(list)->ob_size)); +#else return Py_SIZE(list); +#endif } #define PyList_GET_SIZE(op) PyList_GET_SIZE(_PyObject_CAST(op)) @@ -41,7 +42,12 @@ static inline Py_ssize_t PyList_GET_SIZE(PyObject *op) { static inline void PyList_SET_ITEM(PyObject *op, Py_ssize_t index, PyObject *value) { PyListObject *list = _PyList_CAST(op); + assert(0 <= index); + assert(index < list->allocated); list->ob_item[index] = value; } #define PyList_SET_ITEM(op, index, value) \ PyList_SET_ITEM(_PyObject_CAST(op), (index), _PyObject_CAST(value)) + +PyAPI_FUNC(int) PyList_Extend(PyObject *self, PyObject *iterable); +PyAPI_FUNC(int) PyList_Clear(PyObject *self); diff --git a/contrib/tools/python3/Include/cpython/lock.h b/contrib/tools/python3/Include/cpython/lock.h new file mode 100644 index 00000000000..8ee03e82f74 --- /dev/null +++ b/contrib/tools/python3/Include/cpython/lock.h @@ -0,0 +1,63 @@ +#ifndef Py_CPYTHON_LOCK_H +# error "this header file must not be included directly" +#endif + +#define _Py_UNLOCKED 0 +#define _Py_LOCKED 1 + +// A mutex that occupies one byte. The lock can be zero initialized to +// represent the unlocked state. +// +// Typical initialization: +// PyMutex m = (PyMutex){0}; +// +// Or initialize as global variables: +// static PyMutex m; +// +// Typical usage: +// PyMutex_Lock(&m); +// ... +// PyMutex_Unlock(&m); +// +// The contents of the PyMutex are not part of the public API, but are +// described to aid in understanding the implementation and debugging. Only +// the two least significant bits are used. The remaining bits are always zero: +// 0b00: unlocked +// 0b01: locked +// 0b10: unlocked and has parked threads +// 0b11: locked and has parked threads +typedef struct PyMutex { + uint8_t _bits; // (private) +} PyMutex; + +// exported function for locking the mutex +PyAPI_FUNC(void) PyMutex_Lock(PyMutex *m); + +// exported function for unlocking the mutex +PyAPI_FUNC(void) PyMutex_Unlock(PyMutex *m); + +// Locks the mutex. +// +// If the mutex is currently locked, the calling thread will be parked until +// the mutex is unlocked. If the current thread holds the GIL, then the GIL +// will be released while the thread is parked. +static inline void +_PyMutex_Lock(PyMutex *m) +{ + uint8_t expected = _Py_UNLOCKED; + if (!_Py_atomic_compare_exchange_uint8(&m->_bits, &expected, _Py_LOCKED)) { + PyMutex_Lock(m); + } +} +#define PyMutex_Lock _PyMutex_Lock + +// Unlocks the mutex. +static inline void +_PyMutex_Unlock(PyMutex *m) +{ + uint8_t expected = _Py_LOCKED; + if (!_Py_atomic_compare_exchange_uint8(&m->_bits, &expected, _Py_UNLOCKED)) { + PyMutex_Unlock(m); + } +} +#define PyMutex_Unlock _PyMutex_Unlock diff --git a/contrib/tools/python3/Include/cpython/longintrepr.h b/contrib/tools/python3/Include/cpython/longintrepr.h index 78ac79a7cb8..3246908ba98 100644 --- a/contrib/tools/python3/Include/cpython/longintrepr.h +++ b/contrib/tools/python3/Include/cpython/longintrepr.h @@ -62,21 +62,32 @@ typedef long stwodigits; /* signed variant of twodigits */ #define PyLong_MASK ((digit)(PyLong_BASE - 1)) /* Long integer representation. + + Long integers are made up of a number of 30- or 15-bit digits, depending on + the platform. The number of digits (ndigits) is stored in the high bits of + the lv_tag field (lvtag >> _PyLong_NON_SIZE_BITS). + The absolute value of a number is equal to - SUM(for i=0 through abs(ob_size)-1) ob_digit[i] * 2**(SHIFT*i) - Negative numbers are represented with ob_size < 0; - zero is represented by ob_size == 0. - In a normalized number, ob_digit[abs(ob_size)-1] (the most significant + SUM(for i=0 through ndigits-1) ob_digit[i] * 2**(PyLong_SHIFT*i) + + The sign of the value is stored in the lower 2 bits of lv_tag. + + - 0: Positive + - 1: Zero + - 2: Negative + + The third lowest bit of lv_tag is reserved for an immortality flag, but is + not currently used. + + In a normalized number, ob_digit[ndigits-1] (the most significant digit) is never zero. Also, in all cases, for all valid i, - 0 <= ob_digit[i] <= MASK. + 0 <= ob_digit[i] <= PyLong_MASK. + The allocation function takes care of allocating extra memory - so that ob_digit[0] ... ob_digit[abs(ob_size)-1] are actually available. + so that ob_digit[0] ... ob_digit[ndigits-1] are actually available. We always allocate memory for at least one digit, so accessing ob_digit[0] - is always safe. However, in the case ob_size == 0, the contents of + is always safe. However, in the case ndigits == 0, the contents of ob_digit[0] may be undefined. - - CAUTION: Generic code manipulating subtypes of PyVarObject has to - aware that ints abuse ob_size's sign bit. */ typedef struct _PyLongValue { @@ -89,13 +100,15 @@ struct _longobject { _PyLongValue long_value; }; -PyAPI_FUNC(PyLongObject *) _PyLong_New(Py_ssize_t); +PyAPI_FUNC(PyLongObject*) _PyLong_New(Py_ssize_t); -/* Return a copy of src. */ -PyAPI_FUNC(PyObject *) _PyLong_Copy(PyLongObject *src); +// Return a copy of src. +PyAPI_FUNC(PyObject*) _PyLong_Copy(PyLongObject *src); -PyAPI_FUNC(PyLongObject *) -_PyLong_FromDigits(int negative, Py_ssize_t digit_count, digit *digits); +PyAPI_FUNC(PyLongObject*) _PyLong_FromDigits( + int negative, + Py_ssize_t digit_count, + digit *digits); /* Inline some internals for speed. These should be in pycore_long.h diff --git a/contrib/tools/python3/Include/cpython/longobject.h b/contrib/tools/python3/Include/cpython/longobject.h index 90cc0f267ae..0d49242ff68 100644 --- a/contrib/tools/python3/Include/cpython/longobject.h +++ b/contrib/tools/python3/Include/cpython/longobject.h @@ -2,29 +2,65 @@ # error "this header file must not be included directly" #endif -PyAPI_FUNC(int) _PyLong_AsInt(PyObject *); +PyAPI_FUNC(PyObject*) PyLong_FromUnicodeObject(PyObject *u, int base); -PyAPI_FUNC(int) _PyLong_UnsignedShort_Converter(PyObject *, void *); -PyAPI_FUNC(int) _PyLong_UnsignedInt_Converter(PyObject *, void *); -PyAPI_FUNC(int) _PyLong_UnsignedLong_Converter(PyObject *, void *); -PyAPI_FUNC(int) _PyLong_UnsignedLongLong_Converter(PyObject *, void *); -PyAPI_FUNC(int) _PyLong_Size_t_Converter(PyObject *, void *); +#define Py_ASNATIVEBYTES_DEFAULTS -1 +#define Py_ASNATIVEBYTES_BIG_ENDIAN 0 +#define Py_ASNATIVEBYTES_LITTLE_ENDIAN 1 +#define Py_ASNATIVEBYTES_NATIVE_ENDIAN 3 +#define Py_ASNATIVEBYTES_UNSIGNED_BUFFER 4 +#define Py_ASNATIVEBYTES_REJECT_NEGATIVE 8 +#define Py_ASNATIVEBYTES_ALLOW_INDEX 16 -/* _PyLong_Frexp returns a double x and an exponent e such that the - true value is approximately equal to x * 2**e. e is >= 0. x is - 0.0 if and only if the input is 0 (in which case, e and x are both - zeroes); otherwise, 0.5 <= abs(x) < 1.0. On overflow, which is - possible if the number of bits doesn't fit into a Py_ssize_t, sets - OverflowError and returns -1.0 for x, 0 for e. */ -PyAPI_FUNC(double) _PyLong_Frexp(PyLongObject *a, Py_ssize_t *e); +/* PyLong_AsNativeBytes: Copy the integer value to a native variable. + buffer points to the first byte of the variable. + n_bytes is the number of bytes available in the buffer. Pass 0 to request + the required size for the value. + flags is a bitfield of the following flags: + * 1 - little endian + * 2 - native endian + * 4 - unsigned destination (e.g. don't reject copying 255 into one byte) + * 8 - raise an exception for negative inputs + * 16 - call __index__ on non-int types + If flags is -1 (all bits set), native endian is used, value truncation + behaves most like C (allows negative inputs and allow MSB set), and non-int + objects will raise a TypeError. + Big endian mode will write the most significant byte into the address + directly referenced by buffer; little endian will write the least significant + byte into that address. -PyAPI_FUNC(PyObject *) PyLong_FromUnicodeObject(PyObject *u, int base); -PyAPI_FUNC(PyObject *) _PyLong_FromBytes(const char *, Py_ssize_t, int); + If an exception is raised, returns a negative value. + Otherwise, returns the number of bytes that are required to store the value. + To check that the full value is represented, ensure that the return value is + equal or less than n_bytes. + All n_bytes are guaranteed to be written (unless an exception occurs), and + so ignoring a positive return value is the equivalent of a downcast in C. + In cases where the full value could not be represented, the returned value + may be larger than necessary - this function is not an accurate way to + calculate the bit length of an integer object. + */ +PyAPI_FUNC(Py_ssize_t) PyLong_AsNativeBytes(PyObject* v, void* buffer, + Py_ssize_t n_bytes, int flags); -/* _PyLong_Sign. Return 0 if v is 0, -1 if v < 0, +1 if v > 0. - v must not be NULL, and must be a normalized long. - There are no error cases. -*/ +/* PyLong_FromNativeBytes: Create an int value from a native integer + n_bytes is the number of bytes to read from the buffer. Passing 0 will + always produce the zero int. + PyLong_FromUnsignedNativeBytes always produces a non-negative int. + flags is the same as for PyLong_AsNativeBytes, but only supports selecting + the endianness or forcing an unsigned buffer. + + Returns the int object, or NULL with an exception set. */ +PyAPI_FUNC(PyObject*) PyLong_FromNativeBytes(const void* buffer, size_t n_bytes, + int flags); +PyAPI_FUNC(PyObject*) PyLong_FromUnsignedNativeBytes(const void* buffer, + size_t n_bytes, int flags); + +PyAPI_FUNC(int) PyUnstable_Long_IsCompact(const PyLongObject* op); +PyAPI_FUNC(Py_ssize_t) PyUnstable_Long_CompactValue(const PyLongObject* op); + +// _PyLong_Sign. Return 0 if v is 0, -1 if v < 0, +1 if v > 0. +// v must not be NULL, and must be a normalized long. +// There are no error cases. PyAPI_FUNC(int) _PyLong_Sign(PyObject *v); /* _PyLong_NumBits. Return the number of bits needed to represent the @@ -36,14 +72,6 @@ PyAPI_FUNC(int) _PyLong_Sign(PyObject *v); */ PyAPI_FUNC(size_t) _PyLong_NumBits(PyObject *v); -/* _PyLong_DivmodNear. Given integers a and b, compute the nearest - integer q to the exact quotient a / b, rounding to the nearest even integer - in the case of a tie. Return (q, r), where r = a - q*b. The remainder r - will satisfy abs(r) <= abs(b)/2, with equality possible only if q is - even. -*/ -PyAPI_FUNC(PyObject *) _PyLong_DivmodNear(PyObject *, PyObject *); - /* _PyLong_FromByteArray: View the n unsigned bytes as a binary integer in base 256, and return a Python int with the same numeric value. If n is 0, the integer is 0. Else: @@ -82,19 +110,7 @@ PyAPI_FUNC(PyObject *) _PyLong_FromByteArray( */ PyAPI_FUNC(int) _PyLong_AsByteArray(PyLongObject* v, unsigned char* bytes, size_t n, - int little_endian, int is_signed); - -/* _PyLong_Format: Convert the long to a string object with given base, - appending a base prefix of 0[box] if base is 2, 8 or 16. */ -PyAPI_FUNC(PyObject *) _PyLong_Format(PyObject *obj, int base); + int little_endian, int is_signed, int with_exceptions); /* For use by the gcd function in mathmodule.c */ PyAPI_FUNC(PyObject *) _PyLong_GCD(PyObject *, PyObject *); - -PyAPI_FUNC(PyObject *) _PyLong_Rshift(PyObject *, size_t); -PyAPI_FUNC(PyObject *) _PyLong_Lshift(PyObject *, size_t); - - -PyAPI_FUNC(int) PyUnstable_Long_IsCompact(const PyLongObject* op); -PyAPI_FUNC(Py_ssize_t) PyUnstable_Long_CompactValue(const PyLongObject* op); - diff --git a/contrib/tools/python3/Include/cpython/memoryobject.h b/contrib/tools/python3/Include/cpython/memoryobject.h index 3837fa8c6ab..961161b70f2 100644 --- a/contrib/tools/python3/Include/cpython/memoryobject.h +++ b/contrib/tools/python3/Include/cpython/memoryobject.h @@ -2,8 +2,6 @@ # error "this header file must not be included directly" #endif -PyAPI_DATA(PyTypeObject) _PyManagedBuffer_Type; - /* The structs are declared here so that macros can work, but they shouldn't be considered public. Don't access their fields directly, use the macros and functions instead! */ diff --git a/contrib/tools/python3/Include/cpython/modsupport.h b/contrib/tools/python3/Include/cpython/modsupport.h index 2259291aff6..d3b88f58c82 100644 --- a/contrib/tools/python3/Include/cpython/modsupport.h +++ b/contrib/tools/python3/Include/cpython/modsupport.h @@ -2,108 +2,25 @@ # error "this header file must not be included directly" #endif -/* If PY_SSIZE_T_CLEAN is defined, each functions treats #-specifier - to mean Py_ssize_t */ -#ifdef PY_SSIZE_T_CLEAN -#define _Py_VaBuildStack _Py_VaBuildStack_SizeT -#else -PyAPI_FUNC(PyObject *) _Py_VaBuildValue_SizeT(const char *, va_list); -PyAPI_FUNC(PyObject **) _Py_VaBuildStack_SizeT( - PyObject **small_stack, - Py_ssize_t small_stack_len, - const char *format, - va_list va, - Py_ssize_t *p_nargs); -#endif - -PyAPI_FUNC(int) _PyArg_UnpackStack( - PyObject *const *args, - Py_ssize_t nargs, - const char *name, - Py_ssize_t min, - Py_ssize_t max, - ...); - -PyAPI_FUNC(int) _PyArg_NoKeywords(const char *funcname, PyObject *kwargs); -PyAPI_FUNC(int) _PyArg_NoKwnames(const char *funcname, PyObject *kwnames); -PyAPI_FUNC(int) _PyArg_NoPositional(const char *funcname, PyObject *args); -#define _PyArg_NoKeywords(funcname, kwargs) \ - ((kwargs) == NULL || _PyArg_NoKeywords((funcname), (kwargs))) -#define _PyArg_NoKwnames(funcname, kwnames) \ - ((kwnames) == NULL || _PyArg_NoKwnames((funcname), (kwnames))) -#define _PyArg_NoPositional(funcname, args) \ - ((args) == NULL || _PyArg_NoPositional((funcname), (args))) - -#define _Py_ANY_VARARGS(n) ((n) == PY_SSIZE_T_MAX) - -PyAPI_FUNC(void) _PyArg_BadArgument(const char *, const char *, const char *, PyObject *); -PyAPI_FUNC(int) _PyArg_CheckPositional(const char *, Py_ssize_t, - Py_ssize_t, Py_ssize_t); -#define _PyArg_CheckPositional(funcname, nargs, min, max) \ - ((!_Py_ANY_VARARGS(max) && (min) <= (nargs) && (nargs) <= (max)) \ - || _PyArg_CheckPositional((funcname), (nargs), (min), (max))) - -PyAPI_FUNC(PyObject **) _Py_VaBuildStack( - PyObject **small_stack, - Py_ssize_t small_stack_len, - const char *format, - va_list va, - Py_ssize_t *p_nargs); +// A data structure that can be used to run initialization code once in a +// thread-safe manner. The C++11 equivalent is std::call_once. +typedef struct { + uint8_t v; +} _PyOnceFlag; typedef struct _PyArg_Parser { - int initialized; const char *format; const char * const *keywords; const char *fname; const char *custom_msg; - int pos; /* number of positional-only arguments */ - int min; /* minimal number of arguments */ - int max; /* maximal number of positional arguments */ - PyObject *kwtuple; /* tuple of keyword parameter names */ + _PyOnceFlag once; /* atomic one-time initialization flag */ + int is_kwtuple_owned; /* does this parser own the kwtuple object? */ + int pos; /* number of positional-only arguments */ + int min; /* minimal number of arguments */ + int max; /* maximal number of positional arguments */ + PyObject *kwtuple; /* tuple of keyword parameter names */ struct _PyArg_Parser *next; } _PyArg_Parser; -#ifdef PY_SSIZE_T_CLEAN -#define _PyArg_ParseTupleAndKeywordsFast _PyArg_ParseTupleAndKeywordsFast_SizeT -#define _PyArg_ParseStack _PyArg_ParseStack_SizeT -#define _PyArg_ParseStackAndKeywords _PyArg_ParseStackAndKeywords_SizeT -#define _PyArg_VaParseTupleAndKeywordsFast _PyArg_VaParseTupleAndKeywordsFast_SizeT -#endif - PyAPI_FUNC(int) _PyArg_ParseTupleAndKeywordsFast(PyObject *, PyObject *, struct _PyArg_Parser *, ...); -PyAPI_FUNC(int) _PyArg_ParseStack( - PyObject *const *args, - Py_ssize_t nargs, - const char *format, - ...); -PyAPI_FUNC(int) _PyArg_ParseStackAndKeywords( - PyObject *const *args, - Py_ssize_t nargs, - PyObject *kwnames, - struct _PyArg_Parser *, - ...); -PyAPI_FUNC(int) _PyArg_VaParseTupleAndKeywordsFast(PyObject *, PyObject *, - struct _PyArg_Parser *, va_list); -PyAPI_FUNC(PyObject * const *) _PyArg_UnpackKeywords( - PyObject *const *args, Py_ssize_t nargs, - PyObject *kwargs, PyObject *kwnames, - struct _PyArg_Parser *parser, - int minpos, int maxpos, int minkw, - PyObject **buf); - -PyAPI_FUNC(PyObject * const *) _PyArg_UnpackKeywordsWithVararg( - PyObject *const *args, Py_ssize_t nargs, - PyObject *kwargs, PyObject *kwnames, - struct _PyArg_Parser *parser, - int minpos, int maxpos, int minkw, - int vararg, PyObject **buf); - -#define _PyArg_UnpackKeywords(args, nargs, kwargs, kwnames, parser, minpos, maxpos, minkw, buf) \ - (((minkw) == 0 && (kwargs) == NULL && (kwnames) == NULL && \ - (minpos) <= (nargs) && (nargs) <= (maxpos) && (args) != NULL) ? (args) : \ - _PyArg_UnpackKeywords((args), (nargs), (kwargs), (kwnames), (parser), \ - (minpos), (maxpos), (minkw), (buf))) - -PyAPI_FUNC(PyObject *) _PyModule_CreateInitialized(PyModuleDef*, int apiver); -PyAPI_FUNC(int) _PyModule_Add(PyObject *, const char *, PyObject *); diff --git a/contrib/tools/python3/Include/cpython/monitoring.h b/contrib/tools/python3/Include/cpython/monitoring.h new file mode 100644 index 00000000000..797ba51246b --- /dev/null +++ b/contrib/tools/python3/Include/cpython/monitoring.h @@ -0,0 +1,250 @@ +#ifndef Py_CPYTHON_MONITORING_H +# error "this header file must not be included directly" +#endif + +/* Local events. + * These require bytecode instrumentation */ + +#define PY_MONITORING_EVENT_PY_START 0 +#define PY_MONITORING_EVENT_PY_RESUME 1 +#define PY_MONITORING_EVENT_PY_RETURN 2 +#define PY_MONITORING_EVENT_PY_YIELD 3 +#define PY_MONITORING_EVENT_CALL 4 +#define PY_MONITORING_EVENT_LINE 5 +#define PY_MONITORING_EVENT_INSTRUCTION 6 +#define PY_MONITORING_EVENT_JUMP 7 +#define PY_MONITORING_EVENT_BRANCH 8 +#define PY_MONITORING_EVENT_STOP_ITERATION 9 + +#define PY_MONITORING_IS_INSTRUMENTED_EVENT(ev) \ + ((ev) < _PY_MONITORING_LOCAL_EVENTS) + +/* Other events, mainly exceptions */ + +#define PY_MONITORING_EVENT_RAISE 10 +#define PY_MONITORING_EVENT_EXCEPTION_HANDLED 11 +#define PY_MONITORING_EVENT_PY_UNWIND 12 +#define PY_MONITORING_EVENT_PY_THROW 13 +#define PY_MONITORING_EVENT_RERAISE 14 + + +/* Ancillary events */ + +#define PY_MONITORING_EVENT_C_RETURN 15 +#define PY_MONITORING_EVENT_C_RAISE 16 + + +typedef struct _PyMonitoringState { + uint8_t active; + uint8_t opaque; +} PyMonitoringState; + + +PyAPI_FUNC(int) +PyMonitoring_EnterScope(PyMonitoringState *state_array, uint64_t *version, + const uint8_t *event_types, Py_ssize_t length); + +PyAPI_FUNC(int) +PyMonitoring_ExitScope(void); + + +PyAPI_FUNC(int) +_PyMonitoring_FirePyStartEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset); + +PyAPI_FUNC(int) +_PyMonitoring_FirePyResumeEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset); + +PyAPI_FUNC(int) +_PyMonitoring_FirePyReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *retval); + +PyAPI_FUNC(int) +_PyMonitoring_FirePyYieldEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *retval); + +PyAPI_FUNC(int) +_PyMonitoring_FireCallEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject* callable, PyObject *arg0); + +PyAPI_FUNC(int) +_PyMonitoring_FireLineEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + int lineno); + +PyAPI_FUNC(int) +_PyMonitoring_FireJumpEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *target_offset); + +PyAPI_FUNC(int) +_PyMonitoring_FireBranchEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *target_offset); + +PyAPI_FUNC(int) +_PyMonitoring_FireCReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *retval); + +PyAPI_FUNC(int) +_PyMonitoring_FirePyThrowEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset); + +PyAPI_FUNC(int) +_PyMonitoring_FireRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset); + +PyAPI_FUNC(int) +_PyMonitoring_FireReraiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset); + +PyAPI_FUNC(int) +_PyMonitoring_FireExceptionHandledEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset); + +PyAPI_FUNC(int) +_PyMonitoring_FireCRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset); + +PyAPI_FUNC(int) +_PyMonitoring_FirePyUnwindEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset); + +PyAPI_FUNC(int) +_PyMonitoring_FireStopIterationEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *value); + + +#define _PYMONITORING_IF_ACTIVE(STATE, X) \ + if ((STATE)->active) { \ + return (X); \ + } \ + else { \ + return 0; \ + } + +static inline int +PyMonitoring_FirePyStartEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FirePyStartEvent(state, codelike, offset)); +} + +static inline int +PyMonitoring_FirePyResumeEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FirePyResumeEvent(state, codelike, offset)); +} + +static inline int +PyMonitoring_FirePyReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *retval) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FirePyReturnEvent(state, codelike, offset, retval)); +} + +static inline int +PyMonitoring_FirePyYieldEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *retval) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FirePyYieldEvent(state, codelike, offset, retval)); +} + +static inline int +PyMonitoring_FireCallEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject* callable, PyObject *arg0) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireCallEvent(state, codelike, offset, callable, arg0)); +} + +static inline int +PyMonitoring_FireLineEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + int lineno) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireLineEvent(state, codelike, offset, lineno)); +} + +static inline int +PyMonitoring_FireJumpEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *target_offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireJumpEvent(state, codelike, offset, target_offset)); +} + +static inline int +PyMonitoring_FireBranchEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *target_offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireBranchEvent(state, codelike, offset, target_offset)); +} + +static inline int +PyMonitoring_FireCReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *retval) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireCReturnEvent(state, codelike, offset, retval)); +} + +static inline int +PyMonitoring_FirePyThrowEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FirePyThrowEvent(state, codelike, offset)); +} + +static inline int +PyMonitoring_FireRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireRaiseEvent(state, codelike, offset)); +} + +static inline int +PyMonitoring_FireReraiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireReraiseEvent(state, codelike, offset)); +} + +static inline int +PyMonitoring_FireExceptionHandledEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireExceptionHandledEvent(state, codelike, offset)); +} + +static inline int +PyMonitoring_FireCRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireCRaiseEvent(state, codelike, offset)); +} + +static inline int +PyMonitoring_FirePyUnwindEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FirePyUnwindEvent(state, codelike, offset)); +} + +static inline int +PyMonitoring_FireStopIterationEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *value) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireStopIterationEvent(state, codelike, offset, value)); +} + +#undef _PYMONITORING_IF_ACTIVE diff --git a/contrib/tools/python3/Include/cpython/object.h b/contrib/tools/python3/Include/cpython/object.h index ae7f780a931..5b3b890dcf3 100644 --- a/contrib/tools/python3/Include/cpython/object.h +++ b/contrib/tools/python3/Include/cpython/object.h @@ -4,11 +4,7 @@ PyAPI_FUNC(void) _Py_NewReference(PyObject *op); PyAPI_FUNC(void) _Py_NewReferenceNoTotal(PyObject *op); - -#ifdef Py_TRACE_REFS -/* Py_TRACE_REFS is such major surgery that we call external routines. */ -PyAPI_FUNC(void) _Py_ForgetReference(PyObject *); -#endif +PyAPI_FUNC(void) _Py_ResurrectReference(PyObject *op); #ifdef Py_REF_DEBUG /* These are useful as debugging aids when chasing down refleaks. */ @@ -44,6 +40,10 @@ typedef struct _Py_Identifier { // Index in PyInterpreterState.unicode.ids.array. It is process-wide // unique and must be initialized to -1. Py_ssize_t index; + // Hidden PyMutex struct for non free-threaded build. + struct { + uint8_t v; + } mutex; } _Py_Identifier; #ifndef Py_BUILD_CORE @@ -56,6 +56,7 @@ typedef struct _Py_Identifier { #endif /* !Py_BUILD_CORE */ + typedef struct { /* Number implementations must check *both* arguments for proper type and implement the necessary conversions @@ -220,7 +221,9 @@ struct _typeobject { PyObject *tp_weaklist; /* not used for static builtin types */ destructor tp_del; - /* Type attribute cache version tag. Added in version 2.6 */ + /* Type attribute cache version tag. Added in version 2.6. + * If zero, the cache is invalid and must be initialized. + */ unsigned int tp_version_tag; destructor tp_finalize; @@ -228,10 +231,19 @@ struct _typeobject { /* bitset of which type-watchers care about this type */ unsigned char tp_watched; + + /* Number of tp_version_tag values used. + * Set to _Py_ATTR_CACHE_UNUSED if the attribute cache is + * disabled for this type (e.g. due to custom MRO entries). + * Otherwise, limited to MAX_VERSIONS_PER_CLASS (defined elsewhere). + */ + uint16_t tp_versions_used; }; +#define _Py_ATTR_CACHE_UNUSED (30000) // (see tp_versions_used) + /* This struct is used by the specializer - * It should should be treated as an opaque blob + * It should be treated as an opaque blob * by code other than the specializer and interpreter. */ struct _specialization_cache { // In order to avoid bloating the bytecode with lots of inline caches, the @@ -246,6 +258,7 @@ struct _specialization_cache { // *args nor **kwargs (as required by BINARY_SUBSCR_GETITEM): PyObject *getitem; uint32_t getitem_version; + PyObject *init; }; /* The *real* layout of a type object when allocated on the heap */ @@ -272,46 +285,21 @@ typedef struct _heaptypeobject { PyAPI_FUNC(const char *) _PyType_Name(PyTypeObject *); PyAPI_FUNC(PyObject *) _PyType_Lookup(PyTypeObject *, PyObject *); -PyAPI_FUNC(PyObject *) _PyType_LookupId(PyTypeObject *, _Py_Identifier *); -PyAPI_FUNC(PyObject *) _PyObject_LookupSpecialId(PyObject *, _Py_Identifier *); -#ifndef Py_BUILD_CORE -// Backward compatibility for 3rd-party extensions -// that may be using the old name. -#define _PyObject_LookupSpecial _PyObject_LookupSpecialId -#endif -PyAPI_FUNC(PyTypeObject *) _PyType_CalculateMetaclass(PyTypeObject *, PyObject *); -PyAPI_FUNC(PyObject *) _PyType_GetDocFromInternalDoc(const char *, const char *); -PyAPI_FUNC(PyObject *) _PyType_GetTextSignatureFromInternalDoc(const char *, const char *); -PyAPI_FUNC(PyObject *) PyType_GetModuleByDef(PyTypeObject *, PyModuleDef *); +PyAPI_FUNC(PyObject *) _PyType_LookupRef(PyTypeObject *, PyObject *); PyAPI_FUNC(PyObject *) PyType_GetDict(PyTypeObject *); PyAPI_FUNC(int) PyObject_Print(PyObject *, FILE *, int); PyAPI_FUNC(void) _Py_BreakPoint(void); PyAPI_FUNC(void) _PyObject_Dump(PyObject *); -PyAPI_FUNC(int) _PyObject_IsFreed(PyObject *); - -PyAPI_FUNC(int) _PyObject_IsAbstract(PyObject *); -PyAPI_FUNC(PyObject *) _PyObject_GetAttrId(PyObject *, _Py_Identifier *); -PyAPI_FUNC(int) _PyObject_SetAttrId(PyObject *, _Py_Identifier *, PyObject *); -/* Replacements of PyObject_GetAttr() and _PyObject_GetAttrId() which - don't raise AttributeError. - Return 1 and set *result != NULL if an attribute is found. - Return 0 and set *result == NULL if an attribute is not found; - an AttributeError is silenced. - Return -1 and set *result == NULL if an error other than AttributeError - is raised. -*/ -PyAPI_FUNC(int) _PyObject_LookupAttr(PyObject *, PyObject *, PyObject **); -PyAPI_FUNC(int) _PyObject_LookupAttrId(PyObject *, _Py_Identifier *, PyObject **); - -PyAPI_FUNC(int) _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method); +PyAPI_FUNC(PyObject*) _PyObject_GetAttrId(PyObject *, _Py_Identifier *); PyAPI_FUNC(PyObject **) _PyObject_GetDictPtr(PyObject *); -PyAPI_FUNC(PyObject *) _PyObject_NextNotImplemented(PyObject *); PyAPI_FUNC(void) PyObject_CallFinalizer(PyObject *); PyAPI_FUNC(int) PyObject_CallFinalizerFromDealloc(PyObject *); +PyAPI_FUNC(void) PyUnstable_Object_ClearWeakRefsNoCallbacks(PyObject *); + /* Same as PyObject_Generic{Get,Set}Attr, but passing the attributes dict as the last parameter. */ PyAPI_FUNC(PyObject *) @@ -387,20 +375,6 @@ PyAPI_FUNC(PyObject *) _PyObject_FunctionStr(PyObject *); #endif -PyAPI_DATA(PyTypeObject) _PyNone_Type; -PyAPI_DATA(PyTypeObject) _PyNotImplemented_Type; - -/* Maps Py_LT to Py_GT, ..., Py_GE to Py_LE. - * Defined in object.c. - */ -PyAPI_DATA(int) _Py_SwappedOp[]; - -PyAPI_FUNC(void) -_PyDebugAllocatorStats(FILE *out, const char *block_name, int num_blocks, - size_t sizeof_block); -PyAPI_FUNC(void) -_PyObject_DebugTypeStats(FILE *out); - /* Define a pair of assertion macros: _PyObject_ASSERT_FROM(), _PyObject_ASSERT_WITH_MSG() and _PyObject_ASSERT(). @@ -449,21 +423,6 @@ PyAPI_FUNC(void) _Py_NO_RETURN _PyObject_AssertFailed( int line, const char *function); -/* Check if an object is consistent. For example, ensure that the reference - counter is greater than or equal to 1, and ensure that ob_type is not NULL. - - Call _PyObject_AssertFailed() if the object is inconsistent. - - If check_content is zero, only check header fields: reduce the overhead. - - The function always return 1. The return value is just here to be able to - write: - - assert(_PyObject_CheckConsistency(obj, 1)); */ -PyAPI_FUNC(int) _PyObject_CheckConsistency( - PyObject *op, - int check_content); - /* Trashcan mechanism, thanks to Christian Tismer. @@ -502,8 +461,8 @@ without deallocating anything (and so unbounded call-stack depth is avoided). When the call stack finishes unwinding again, code generated by the END macro notices this, and calls another routine to deallocate all the objects that may have been added to the list of deferred deallocations. In effect, a -chain of N deallocations is broken into (N-1)/(_PyTrash_UNWIND_LEVEL-1) pieces, -with the call stack never exceeding a depth of _PyTrash_UNWIND_LEVEL. +chain of N deallocations is broken into (N-1)/(Py_TRASHCAN_HEADROOM-1) pieces, +with the call stack never exceeding a depth of Py_TRASHCAN_HEADROOM. Since the tp_dealloc of a subclass typically calls the tp_dealloc of the base class, we need to ensure that the trashcan is only triggered on the tp_dealloc @@ -515,49 +474,40 @@ passed as second argument to Py_TRASHCAN_BEGIN(). /* Python 3.9 private API, invoked by the macros below. */ PyAPI_FUNC(int) _PyTrash_begin(PyThreadState *tstate, PyObject *op); PyAPI_FUNC(void) _PyTrash_end(PyThreadState *tstate); + +PyAPI_FUNC(void) _PyTrash_thread_deposit_object(PyThreadState *tstate, PyObject *op); +PyAPI_FUNC(void) _PyTrash_thread_destroy_chain(PyThreadState *tstate); + + /* Python 3.10 private API, invoked by the Py_TRASHCAN_BEGIN(). */ -PyAPI_FUNC(int) _PyTrash_cond(PyObject *op, destructor dealloc); -#define Py_TRASHCAN_BEGIN_CONDITION(op, cond) \ - do { \ - PyThreadState *_tstate = NULL; \ - /* If "cond" is false, then _tstate remains NULL and the deallocator \ - * is run normally without involving the trashcan */ \ - if (cond) { \ - _tstate = _PyThreadState_UncheckedGet(); \ - if (_PyTrash_begin(_tstate, _PyObject_CAST(op))) { \ - break; \ - } \ - } - /* The body of the deallocator is here. */ -#define Py_TRASHCAN_END \ - if (_tstate) { \ - _PyTrash_end(_tstate); \ - } \ - } while (0); +/* To avoid raising recursion errors during dealloc trigger trashcan before we reach + * recursion limit. To avoid trashing, we don't attempt to empty the trashcan until + * we have headroom above the trigger limit */ +#define Py_TRASHCAN_HEADROOM 50 #define Py_TRASHCAN_BEGIN(op, dealloc) \ - Py_TRASHCAN_BEGIN_CONDITION((op), \ - _PyTrash_cond(_PyObject_CAST(op), (destructor)(dealloc))) +do { \ + PyThreadState *tstate = PyThreadState_Get(); \ + if (tstate->c_recursion_remaining <= Py_TRASHCAN_HEADROOM && Py_TYPE(op)->tp_dealloc == (destructor)dealloc) { \ + _PyTrash_thread_deposit_object(tstate, (PyObject *)op); \ + break; \ + } \ + tstate->c_recursion_remaining--; + /* The body of the deallocator is here. */ +#define Py_TRASHCAN_END \ + tstate->c_recursion_remaining++; \ + if (tstate->delete_later && tstate->c_recursion_remaining > (Py_TRASHCAN_HEADROOM*2)) { \ + _PyTrash_thread_destroy_chain(tstate); \ + } \ +} while (0); -/* The following two macros, Py_TRASHCAN_SAFE_BEGIN and - * Py_TRASHCAN_SAFE_END, are deprecated since version 3.11 and - * will be removed in the future. - * Use Py_TRASHCAN_BEGIN and Py_TRASHCAN_END instead. - */ -Py_DEPRECATED(3.11) typedef int UsingDeprecatedTrashcanMacro; -#define Py_TRASHCAN_SAFE_BEGIN(op) \ - do { \ - UsingDeprecatedTrashcanMacro cond=1; \ - Py_TRASHCAN_BEGIN_CONDITION((op), cond); -#define Py_TRASHCAN_SAFE_END(op) \ - Py_TRASHCAN_END; \ - } while(0); PyAPI_FUNC(void *) PyObject_GetItemData(PyObject *obj); -PyAPI_FUNC(int) _PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg); -PyAPI_FUNC(void) _PyObject_ClearManagedDict(PyObject *obj); +PyAPI_FUNC(int) PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg); +PyAPI_FUNC(int) _PyObject_SetManagedDict(PyObject *obj, PyObject *new_dict); +PyAPI_FUNC(void) PyObject_ClearManagedDict(PyObject *obj); #define TYPE_MAX_WATCHERS 8 @@ -573,3 +523,13 @@ PyAPI_FUNC(int) PyType_Unwatch(int watcher_id, PyObject *type); * assigned, or 0 if a new tag could not be assigned. */ PyAPI_FUNC(int) PyUnstable_Type_AssignVersionTag(PyTypeObject *type); + + +typedef enum { + PyRefTracer_CREATE = 0, + PyRefTracer_DESTROY = 1, +} PyRefTracerEvent; + +typedef int (*PyRefTracer)(PyObject *, PyRefTracerEvent event, void *); +PyAPI_FUNC(int) PyRefTracer_SetTracer(PyRefTracer tracer, void *data); +PyAPI_FUNC(PyRefTracer) PyRefTracer_GetTracer(void**); diff --git a/contrib/tools/python3/Include/cpython/objimpl.h b/contrib/tools/python3/Include/cpython/objimpl.h index 5a8cdd57c78..e0c2ce286f1 100644 --- a/contrib/tools/python3/Include/cpython/objimpl.h +++ b/contrib/tools/python3/Include/cpython/objimpl.h @@ -78,14 +78,6 @@ PyAPI_FUNC(void) PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator); PyAPI_FUNC(int) PyObject_IS_GC(PyObject *obj); -/* Code built with Py_BUILD_CORE must include pycore_gc.h instead which - defines a different _PyGC_FINALIZED() macro. */ -#ifndef Py_BUILD_CORE - // Kept for backward compatibility with Python 3.8 -# define _PyGC_FINALIZED(o) PyObject_GC_IsFinalized(o) -#endif - - // Test if a type supports weak references PyAPI_FUNC(int) PyType_SUPPORTS_WEAKREFS(PyTypeObject *type); @@ -93,3 +85,20 @@ PyAPI_FUNC(PyObject **) PyObject_GET_WEAKREFS_LISTPTR(PyObject *op); PyAPI_FUNC(PyObject *) PyUnstable_Object_GC_NewWithExtraData(PyTypeObject *, size_t); + + +/* Visit all live GC-capable objects, similar to gc.get_objects(None). The + * supplied callback is called on every such object with the void* arg set + * to the supplied arg. Returning 0 from the callback ends iteration, returning + * 1 allows iteration to continue. Returning any other value may result in + * undefined behaviour. + * + * If new objects are (de)allocated by the callback it is undefined if they + * will be visited. + + * Garbage collection is disabled during operation. Explicitly running a + * collection in the callback may lead to undefined behaviour e.g. visiting the + * same objects multiple times or not at all. + */ +typedef int (*gcvisitobjects_t)(PyObject*, void*); +PyAPI_FUNC(void) PyUnstable_GC_VisitObjects(gcvisitobjects_t callback, void* arg); diff --git a/contrib/tools/python3/Include/cpython/pthread_stubs.h b/contrib/tools/python3/Include/cpython/pthread_stubs.h index 83f0b08693e..e6114a192fc 100644 --- a/contrib/tools/python3/Include/cpython/pthread_stubs.h +++ b/contrib/tools/python3/Include/cpython/pthread_stubs.h @@ -21,13 +21,29 @@ #ifdef __wasi__ // WASI's bits/alltypes.h provides type definitions when __NEED_ is set. // The header file can be included multiple times. -# define __NEED_pthread_cond_t 1 -# define __NEED_pthread_condattr_t 1 -# define __NEED_pthread_mutex_t 1 -# define __NEED_pthread_mutexattr_t 1 -# define __NEED_pthread_key_t 1 -# define __NEED_pthread_t 1 -# define __NEED_pthread_attr_t 1 +// +// <sys/types.h> may also define these macros. +# ifndef __NEED_pthread_cond_t +# define __NEED_pthread_cond_t 1 +# endif +# ifndef __NEED_pthread_condattr_t +# define __NEED_pthread_condattr_t 1 +# endif +# ifndef __NEED_pthread_mutex_t +# define __NEED_pthread_mutex_t 1 +# endif +# ifndef __NEED_pthread_mutexattr_t +# define __NEED_pthread_mutexattr_t 1 +# endif +# ifndef __NEED_pthread_key_t +# define __NEED_pthread_key_t 1 +# endif +# ifndef __NEED_pthread_t +# define __NEED_pthread_t 1 +# endif +# ifndef __NEED_pthread_attr_t +# define __NEED_pthread_attr_t 1 +# endif # error #include <bits/alltypes.h> #else typedef struct { void *__x; } pthread_cond_t; @@ -67,6 +83,7 @@ PyAPI_FUNC(int) pthread_create(pthread_t *restrict thread, void *(*start_routine)(void *), void *restrict arg); PyAPI_FUNC(int) pthread_detach(pthread_t thread); +PyAPI_FUNC(int) pthread_join(pthread_t thread, void** value_ptr); PyAPI_FUNC(pthread_t) pthread_self(void); PyAPI_FUNC(int) pthread_exit(void *retval) __attribute__ ((__noreturn__)); PyAPI_FUNC(int) pthread_attr_init(pthread_attr_t *attr); diff --git a/contrib/tools/python3/Include/cpython/pyatomic.h b/contrib/tools/python3/Include/cpython/pyatomic.h new file mode 100644 index 00000000000..71e91c8964b --- /dev/null +++ b/contrib/tools/python3/Include/cpython/pyatomic.h @@ -0,0 +1,569 @@ +// This header provides cross-platform low-level atomic operations +// similar to C11 atomics. +// +// Operations are sequentially consistent unless they have a suffix indicating +// otherwise. If in doubt, prefer the sequentially consistent operations. +// +// The "_relaxed" suffix for load and store operations indicates the "relaxed" +// memory order. They don't provide synchronization, but (roughly speaking) +// guarantee somewhat sane behavior for races instead of undefined behavior. +// In practice, they correspond to "normal" hardware load and store +// instructions, so they are almost as inexpensive as plain loads and stores +// in C. +// +// Note that atomic read-modify-write operations like _Py_atomic_add_* return +// the previous value of the atomic variable, not the new value. +// +// See https://en.cppreference.com/w/c/atomic for more information on C11 +// atomics. +// See https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2020/p2055r0.pdf +// "A Relaxed Guide to memory_order_relaxed" for discussion of and common usage +// or relaxed atomics. +// +// Functions with pseudo Python code: +// +// def _Py_atomic_load(obj): +// return obj # sequential consistency +// +// def _Py_atomic_load_relaxed(obj): +// return obj # relaxed consistency +// +// def _Py_atomic_store(obj, value): +// obj = value # sequential consistency +// +// def _Py_atomic_store_relaxed(obj, value): +// obj = value # relaxed consistency +// +// def _Py_atomic_exchange(obj, value): +// # sequential consistency +// old_obj = obj +// obj = value +// return old_obj +// +// def _Py_atomic_compare_exchange(obj, expected, desired): +// # sequential consistency +// if obj == expected: +// obj = desired +// return True +// else: +// expected = obj +// return False +// +// def _Py_atomic_add(obj, value): +// # sequential consistency +// old_obj = obj +// obj += value +// return old_obj +// +// def _Py_atomic_and(obj, value): +// # sequential consistency +// old_obj = obj +// obj &= value +// return old_obj +// +// def _Py_atomic_or(obj, value): +// # sequential consistency +// old_obj = obj +// obj |= value +// return old_obj +// +// Other functions: +// +// def _Py_atomic_load_ptr_acquire(obj): +// return obj # acquire +// +// def _Py_atomic_store_ptr_release(obj, value): +// obj = value # release +// +// def _Py_atomic_fence_seq_cst(): +// # sequential consistency +// ... +// +// def _Py_atomic_fence_release(): +// # release +// ... + +#ifndef Py_CPYTHON_ATOMIC_H +# error "this header file must not be included directly" +#endif + +// --- _Py_atomic_add -------------------------------------------------------- +// Atomically adds `value` to `obj` and returns the previous value + +static inline int +_Py_atomic_add_int(int *obj, int value); + +static inline int8_t +_Py_atomic_add_int8(int8_t *obj, int8_t value); + +static inline int16_t +_Py_atomic_add_int16(int16_t *obj, int16_t value); + +static inline int32_t +_Py_atomic_add_int32(int32_t *obj, int32_t value); + +static inline int64_t +_Py_atomic_add_int64(int64_t *obj, int64_t value); + +static inline intptr_t +_Py_atomic_add_intptr(intptr_t *obj, intptr_t value); + +static inline unsigned int +_Py_atomic_add_uint(unsigned int *obj, unsigned int value); + +static inline uint8_t +_Py_atomic_add_uint8(uint8_t *obj, uint8_t value); + +static inline uint16_t +_Py_atomic_add_uint16(uint16_t *obj, uint16_t value); + +static inline uint32_t +_Py_atomic_add_uint32(uint32_t *obj, uint32_t value); + +static inline uint64_t +_Py_atomic_add_uint64(uint64_t *obj, uint64_t value); + +static inline uintptr_t +_Py_atomic_add_uintptr(uintptr_t *obj, uintptr_t value); + +static inline Py_ssize_t +_Py_atomic_add_ssize(Py_ssize_t *obj, Py_ssize_t value); + + +// --- _Py_atomic_compare_exchange ------------------------------------------- +// Performs an atomic compare-and-exchange. +// +// - If `*obj` and `*expected` are equal, store `desired` into `*obj` +// and return 1 (success). +// - Otherwise, store the `*obj` current value into `*expected` +// and return 0 (failure). +// +// These correspond to the C11 atomic_compare_exchange_strong() function. + +static inline int +_Py_atomic_compare_exchange_int(int *obj, int *expected, int desired); + +static inline int +_Py_atomic_compare_exchange_int8(int8_t *obj, int8_t *expected, int8_t desired); + +static inline int +_Py_atomic_compare_exchange_int16(int16_t *obj, int16_t *expected, int16_t desired); + +static inline int +_Py_atomic_compare_exchange_int32(int32_t *obj, int32_t *expected, int32_t desired); + +static inline int +_Py_atomic_compare_exchange_int64(int64_t *obj, int64_t *expected, int64_t desired); + +static inline int +_Py_atomic_compare_exchange_intptr(intptr_t *obj, intptr_t *expected, intptr_t desired); + +static inline int +_Py_atomic_compare_exchange_uint(unsigned int *obj, unsigned int *expected, unsigned int desired); + +static inline int +_Py_atomic_compare_exchange_uint8(uint8_t *obj, uint8_t *expected, uint8_t desired); + +static inline int +_Py_atomic_compare_exchange_uint16(uint16_t *obj, uint16_t *expected, uint16_t desired); + +static inline int +_Py_atomic_compare_exchange_uint32(uint32_t *obj, uint32_t *expected, uint32_t desired); + +static inline int +_Py_atomic_compare_exchange_uint64(uint64_t *obj, uint64_t *expected, uint64_t desired); + +static inline int +_Py_atomic_compare_exchange_uintptr(uintptr_t *obj, uintptr_t *expected, uintptr_t desired); + +static inline int +_Py_atomic_compare_exchange_ssize(Py_ssize_t *obj, Py_ssize_t *expected, Py_ssize_t desired); + +// NOTE: `obj` and `expected` are logically `void**` types, but we use `void*` +// so that we can pass types like `PyObject**` without a cast. +static inline int +_Py_atomic_compare_exchange_ptr(void *obj, void *expected, void *value); + + +// --- _Py_atomic_exchange --------------------------------------------------- +// Atomically replaces `*obj` with `value` and returns the previous value of `*obj`. + +static inline int +_Py_atomic_exchange_int(int *obj, int value); + +static inline int8_t +_Py_atomic_exchange_int8(int8_t *obj, int8_t value); + +static inline int16_t +_Py_atomic_exchange_int16(int16_t *obj, int16_t value); + +static inline int32_t +_Py_atomic_exchange_int32(int32_t *obj, int32_t value); + +static inline int64_t +_Py_atomic_exchange_int64(int64_t *obj, int64_t value); + +static inline intptr_t +_Py_atomic_exchange_intptr(intptr_t *obj, intptr_t value); + +static inline unsigned int +_Py_atomic_exchange_uint(unsigned int *obj, unsigned int value); + +static inline uint8_t +_Py_atomic_exchange_uint8(uint8_t *obj, uint8_t value); + +static inline uint16_t +_Py_atomic_exchange_uint16(uint16_t *obj, uint16_t value); + +static inline uint32_t +_Py_atomic_exchange_uint32(uint32_t *obj, uint32_t value); + +static inline uint64_t +_Py_atomic_exchange_uint64(uint64_t *obj, uint64_t value); + +static inline uintptr_t +_Py_atomic_exchange_uintptr(uintptr_t *obj, uintptr_t value); + +static inline Py_ssize_t +_Py_atomic_exchange_ssize(Py_ssize_t *obj, Py_ssize_t value); + +static inline void * +_Py_atomic_exchange_ptr(void *obj, void *value); + + +// --- _Py_atomic_and -------------------------------------------------------- +// Performs `*obj &= value` atomically and returns the previous value of `*obj`. + +static inline uint8_t +_Py_atomic_and_uint8(uint8_t *obj, uint8_t value); + +static inline uint16_t +_Py_atomic_and_uint16(uint16_t *obj, uint16_t value); + +static inline uint32_t +_Py_atomic_and_uint32(uint32_t *obj, uint32_t value); + +static inline uint64_t +_Py_atomic_and_uint64(uint64_t *obj, uint64_t value); + +static inline uintptr_t +_Py_atomic_and_uintptr(uintptr_t *obj, uintptr_t value); + + +// --- _Py_atomic_or --------------------------------------------------------- +// Performs `*obj |= value` atomically and returns the previous value of `*obj`. + +static inline uint8_t +_Py_atomic_or_uint8(uint8_t *obj, uint8_t value); + +static inline uint16_t +_Py_atomic_or_uint16(uint16_t *obj, uint16_t value); + +static inline uint32_t +_Py_atomic_or_uint32(uint32_t *obj, uint32_t value); + +static inline uint64_t +_Py_atomic_or_uint64(uint64_t *obj, uint64_t value); + +static inline uintptr_t +_Py_atomic_or_uintptr(uintptr_t *obj, uintptr_t value); + + +// --- _Py_atomic_load ------------------------------------------------------- +// Atomically loads `*obj` (sequential consistency) + +static inline int +_Py_atomic_load_int(const int *obj); + +static inline int8_t +_Py_atomic_load_int8(const int8_t *obj); + +static inline int16_t +_Py_atomic_load_int16(const int16_t *obj); + +static inline int32_t +_Py_atomic_load_int32(const int32_t *obj); + +static inline int64_t +_Py_atomic_load_int64(const int64_t *obj); + +static inline intptr_t +_Py_atomic_load_intptr(const intptr_t *obj); + +static inline uint8_t +_Py_atomic_load_uint8(const uint8_t *obj); + +static inline uint16_t +_Py_atomic_load_uint16(const uint16_t *obj); + +static inline uint32_t +_Py_atomic_load_uint32(const uint32_t *obj); + +static inline uint64_t +_Py_atomic_load_uint64(const uint64_t *obj); + +static inline uintptr_t +_Py_atomic_load_uintptr(const uintptr_t *obj); + +static inline unsigned int +_Py_atomic_load_uint(const unsigned int *obj); + +static inline Py_ssize_t +_Py_atomic_load_ssize(const Py_ssize_t *obj); + +static inline void * +_Py_atomic_load_ptr(const void *obj); + + +// --- _Py_atomic_load_relaxed ----------------------------------------------- +// Loads `*obj` (relaxed consistency, i.e., no ordering) + +static inline int +_Py_atomic_load_int_relaxed(const int *obj); + +static inline int8_t +_Py_atomic_load_int8_relaxed(const int8_t *obj); + +static inline int16_t +_Py_atomic_load_int16_relaxed(const int16_t *obj); + +static inline int32_t +_Py_atomic_load_int32_relaxed(const int32_t *obj); + +static inline int64_t +_Py_atomic_load_int64_relaxed(const int64_t *obj); + +static inline intptr_t +_Py_atomic_load_intptr_relaxed(const intptr_t *obj); + +static inline uint8_t +_Py_atomic_load_uint8_relaxed(const uint8_t *obj); + +static inline uint16_t +_Py_atomic_load_uint16_relaxed(const uint16_t *obj); + +static inline uint32_t +_Py_atomic_load_uint32_relaxed(const uint32_t *obj); + +static inline uint64_t +_Py_atomic_load_uint64_relaxed(const uint64_t *obj); + +static inline uintptr_t +_Py_atomic_load_uintptr_relaxed(const uintptr_t *obj); + +static inline unsigned int +_Py_atomic_load_uint_relaxed(const unsigned int *obj); + +static inline Py_ssize_t +_Py_atomic_load_ssize_relaxed(const Py_ssize_t *obj); + +static inline void * +_Py_atomic_load_ptr_relaxed(const void *obj); + +static inline unsigned long long +_Py_atomic_load_ullong_relaxed(const unsigned long long *obj); + +// --- _Py_atomic_store ------------------------------------------------------ +// Atomically performs `*obj = value` (sequential consistency) + +static inline void +_Py_atomic_store_int(int *obj, int value); + +static inline void +_Py_atomic_store_int8(int8_t *obj, int8_t value); + +static inline void +_Py_atomic_store_int16(int16_t *obj, int16_t value); + +static inline void +_Py_atomic_store_int32(int32_t *obj, int32_t value); + +static inline void +_Py_atomic_store_int64(int64_t *obj, int64_t value); + +static inline void +_Py_atomic_store_intptr(intptr_t *obj, intptr_t value); + +static inline void +_Py_atomic_store_uint8(uint8_t *obj, uint8_t value); + +static inline void +_Py_atomic_store_uint16(uint16_t *obj, uint16_t value); + +static inline void +_Py_atomic_store_uint32(uint32_t *obj, uint32_t value); + +static inline void +_Py_atomic_store_uint64(uint64_t *obj, uint64_t value); + +static inline void +_Py_atomic_store_uintptr(uintptr_t *obj, uintptr_t value); + +static inline void +_Py_atomic_store_uint(unsigned int *obj, unsigned int value); + +static inline void +_Py_atomic_store_ptr(void *obj, void *value); + +static inline void +_Py_atomic_store_ssize(Py_ssize_t* obj, Py_ssize_t value); + + +// --- _Py_atomic_store_relaxed ---------------------------------------------- +// Stores `*obj = value` (relaxed consistency, i.e., no ordering) + +static inline void +_Py_atomic_store_int_relaxed(int *obj, int value); + +static inline void +_Py_atomic_store_int8_relaxed(int8_t *obj, int8_t value); + +static inline void +_Py_atomic_store_int16_relaxed(int16_t *obj, int16_t value); + +static inline void +_Py_atomic_store_int32_relaxed(int32_t *obj, int32_t value); + +static inline void +_Py_atomic_store_int64_relaxed(int64_t *obj, int64_t value); + +static inline void +_Py_atomic_store_intptr_relaxed(intptr_t *obj, intptr_t value); + +static inline void +_Py_atomic_store_uint8_relaxed(uint8_t* obj, uint8_t value); + +static inline void +_Py_atomic_store_uint16_relaxed(uint16_t *obj, uint16_t value); + +static inline void +_Py_atomic_store_uint32_relaxed(uint32_t *obj, uint32_t value); + +static inline void +_Py_atomic_store_uint64_relaxed(uint64_t *obj, uint64_t value); + +static inline void +_Py_atomic_store_uintptr_relaxed(uintptr_t *obj, uintptr_t value); + +static inline void +_Py_atomic_store_uint_relaxed(unsigned int *obj, unsigned int value); + +static inline void +_Py_atomic_store_ptr_relaxed(void *obj, void *value); + +static inline void +_Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value); + +static inline void +_Py_atomic_store_ullong_relaxed(unsigned long long *obj, + unsigned long long value); + + +// --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ + +// Loads `*obj` (acquire operation) +static inline void * +_Py_atomic_load_ptr_acquire(const void *obj); + +static inline uintptr_t +_Py_atomic_load_uintptr_acquire(const uintptr_t *obj); + +// Stores `*obj = value` (release operation) +static inline void +_Py_atomic_store_ptr_release(void *obj, void *value); + +static inline void +_Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value); + +static inline void +_Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value); + +static inline void +_Py_atomic_store_int_release(int *obj, int value); + +static inline int +_Py_atomic_load_int_acquire(const int *obj); + +static inline void +_Py_atomic_store_uint32_release(uint32_t *obj, uint32_t value); + +static inline void +_Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value); + +static inline uint64_t +_Py_atomic_load_uint64_acquire(const uint64_t *obj); + +static inline uint32_t +_Py_atomic_load_uint32_acquire(const uint32_t *obj); + +static inline Py_ssize_t +_Py_atomic_load_ssize_acquire(const Py_ssize_t *obj); + + + + +// --- _Py_atomic_fence ------------------------------------------------------ + +// Sequential consistency fence. C11 fences have complex semantics. When +// possible, use the atomic operations on variables defined above, which +// generally do not require explicit use of a fence. +// See https://en.cppreference.com/w/cpp/atomic/atomic_thread_fence +static inline void _Py_atomic_fence_seq_cst(void); + +// Acquire fence +static inline void _Py_atomic_fence_acquire(void); + +// Release fence +static inline void _Py_atomic_fence_release(void); + + +#ifndef _Py_USE_GCC_BUILTIN_ATOMICS +# if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)) +# define _Py_USE_GCC_BUILTIN_ATOMICS 1 +# elif defined(__clang__) +# if __has_builtin(__atomic_load) +# define _Py_USE_GCC_BUILTIN_ATOMICS 1 +# endif +# endif +#endif + +#if _Py_USE_GCC_BUILTIN_ATOMICS +# define Py_ATOMIC_GCC_H +# include "pyatomic_gcc.h" +# undef Py_ATOMIC_GCC_H +#elif __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_ATOMICS__) +# define Py_ATOMIC_STD_H +# include "pyatomic_std.h" +# undef Py_ATOMIC_STD_H +#elif defined(_MSC_VER) +# define Py_ATOMIC_MSC_H +# include "pyatomic_msc.h" +# undef Py_ATOMIC_MSC_H +#else +# error "no available pyatomic implementation for this platform/compiler" +#endif + + +// --- aliases --------------------------------------------------------------- + +#if SIZEOF_LONG == 8 +# define _Py_atomic_load_ulong(p) \ + _Py_atomic_load_uint64((uint64_t *)p) +# define _Py_atomic_load_ulong_relaxed(p) \ + _Py_atomic_load_uint64_relaxed((uint64_t *)p) +# define _Py_atomic_store_ulong(p, v) \ + _Py_atomic_store_uint64((uint64_t *)p, v) +# define _Py_atomic_store_ulong_relaxed(p, v) \ + _Py_atomic_store_uint64_relaxed((uint64_t *)p, v) +#elif SIZEOF_LONG == 4 +# define _Py_atomic_load_ulong(p) \ + _Py_atomic_load_uint32((uint32_t *)p) +# define _Py_atomic_load_ulong_relaxed(p) \ + _Py_atomic_load_uint32_relaxed((uint32_t *)p) +# define _Py_atomic_store_ulong(p, v) \ + _Py_atomic_store_uint32((uint32_t *)p, v) +# define _Py_atomic_store_ulong_relaxed(p, v) \ + _Py_atomic_store_uint32_relaxed((uint32_t *)p, v) +#else +# error "long must be 4 or 8 bytes in size" +#endif // SIZEOF_LONG diff --git a/contrib/tools/python3/Include/cpython/pyatomic_gcc.h b/contrib/tools/python3/Include/cpython/pyatomic_gcc.h new file mode 100644 index 00000000000..ef09954d53a --- /dev/null +++ b/contrib/tools/python3/Include/cpython/pyatomic_gcc.h @@ -0,0 +1,551 @@ +// This is the implementation of Python atomic operations using GCC's built-in +// functions that match the C+11 memory model. This implementation is preferred +// for GCC compatible compilers, such as Clang. These functions are available +// in GCC 4.8+ without needing to compile with --std=c11 or --std=gnu11. + +#ifndef Py_ATOMIC_GCC_H +# error "this header file must not be included directly" +#endif + + +// --- _Py_atomic_add -------------------------------------------------------- + +static inline int +_Py_atomic_add_int(int *obj, int value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + +static inline int8_t +_Py_atomic_add_int8(int8_t *obj, int8_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + +static inline int16_t +_Py_atomic_add_int16(int16_t *obj, int16_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + +static inline int32_t +_Py_atomic_add_int32(int32_t *obj, int32_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + +static inline int64_t +_Py_atomic_add_int64(int64_t *obj, int64_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + +static inline intptr_t +_Py_atomic_add_intptr(intptr_t *obj, intptr_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + +static inline unsigned int +_Py_atomic_add_uint(unsigned int *obj, unsigned int value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + +static inline uint8_t +_Py_atomic_add_uint8(uint8_t *obj, uint8_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + +static inline uint16_t +_Py_atomic_add_uint16(uint16_t *obj, uint16_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + +static inline uint32_t +_Py_atomic_add_uint32(uint32_t *obj, uint32_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + +static inline uint64_t +_Py_atomic_add_uint64(uint64_t *obj, uint64_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + +static inline uintptr_t +_Py_atomic_add_uintptr(uintptr_t *obj, uintptr_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + +static inline Py_ssize_t +_Py_atomic_add_ssize(Py_ssize_t *obj, Py_ssize_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + + +// --- _Py_atomic_compare_exchange ------------------------------------------- + +static inline int +_Py_atomic_compare_exchange_int(int *obj, int *expected, int desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + +static inline int +_Py_atomic_compare_exchange_int8(int8_t *obj, int8_t *expected, int8_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + +static inline int +_Py_atomic_compare_exchange_int16(int16_t *obj, int16_t *expected, int16_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + +static inline int +_Py_atomic_compare_exchange_int32(int32_t *obj, int32_t *expected, int32_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + +static inline int +_Py_atomic_compare_exchange_int64(int64_t *obj, int64_t *expected, int64_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + +static inline int +_Py_atomic_compare_exchange_intptr(intptr_t *obj, intptr_t *expected, intptr_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + +static inline int +_Py_atomic_compare_exchange_uint(unsigned int *obj, unsigned int *expected, unsigned int desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + +static inline int +_Py_atomic_compare_exchange_uint8(uint8_t *obj, uint8_t *expected, uint8_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + +static inline int +_Py_atomic_compare_exchange_uint16(uint16_t *obj, uint16_t *expected, uint16_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + +static inline int +_Py_atomic_compare_exchange_uint32(uint32_t *obj, uint32_t *expected, uint32_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + +static inline int +_Py_atomic_compare_exchange_uint64(uint64_t *obj, uint64_t *expected, uint64_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + +static inline int +_Py_atomic_compare_exchange_uintptr(uintptr_t *obj, uintptr_t *expected, uintptr_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + +static inline int +_Py_atomic_compare_exchange_ssize(Py_ssize_t *obj, Py_ssize_t *expected, Py_ssize_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + +static inline int +_Py_atomic_compare_exchange_ptr(void *obj, void *expected, void *desired) +{ return __atomic_compare_exchange_n((void **)obj, (void **)expected, desired, 0, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + + +// --- _Py_atomic_exchange --------------------------------------------------- + +static inline int +_Py_atomic_exchange_int(int *obj, int value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline int8_t +_Py_atomic_exchange_int8(int8_t *obj, int8_t value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline int16_t +_Py_atomic_exchange_int16(int16_t *obj, int16_t value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline int32_t +_Py_atomic_exchange_int32(int32_t *obj, int32_t value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline int64_t +_Py_atomic_exchange_int64(int64_t *obj, int64_t value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline intptr_t +_Py_atomic_exchange_intptr(intptr_t *obj, intptr_t value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline unsigned int +_Py_atomic_exchange_uint(unsigned int *obj, unsigned int value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline uint8_t +_Py_atomic_exchange_uint8(uint8_t *obj, uint8_t value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline uint16_t +_Py_atomic_exchange_uint16(uint16_t *obj, uint16_t value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline uint32_t +_Py_atomic_exchange_uint32(uint32_t *obj, uint32_t value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline uint64_t +_Py_atomic_exchange_uint64(uint64_t *obj, uint64_t value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline uintptr_t +_Py_atomic_exchange_uintptr(uintptr_t *obj, uintptr_t value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline Py_ssize_t +_Py_atomic_exchange_ssize(Py_ssize_t *obj, Py_ssize_t value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline void * +_Py_atomic_exchange_ptr(void *obj, void *value) +{ return __atomic_exchange_n((void **)obj, value, __ATOMIC_SEQ_CST); } + + +// --- _Py_atomic_and -------------------------------------------------------- + +static inline uint8_t +_Py_atomic_and_uint8(uint8_t *obj, uint8_t value) +{ return __atomic_fetch_and(obj, value, __ATOMIC_SEQ_CST); } + +static inline uint16_t +_Py_atomic_and_uint16(uint16_t *obj, uint16_t value) +{ return __atomic_fetch_and(obj, value, __ATOMIC_SEQ_CST); } + +static inline uint32_t +_Py_atomic_and_uint32(uint32_t *obj, uint32_t value) +{ return __atomic_fetch_and(obj, value, __ATOMIC_SEQ_CST); } + +static inline uint64_t +_Py_atomic_and_uint64(uint64_t *obj, uint64_t value) +{ return __atomic_fetch_and(obj, value, __ATOMIC_SEQ_CST); } + +static inline uintptr_t +_Py_atomic_and_uintptr(uintptr_t *obj, uintptr_t value) +{ return __atomic_fetch_and(obj, value, __ATOMIC_SEQ_CST); } + + +// --- _Py_atomic_or --------------------------------------------------------- + +static inline uint8_t +_Py_atomic_or_uint8(uint8_t *obj, uint8_t value) +{ return __atomic_fetch_or(obj, value, __ATOMIC_SEQ_CST); } + +static inline uint16_t +_Py_atomic_or_uint16(uint16_t *obj, uint16_t value) +{ return __atomic_fetch_or(obj, value, __ATOMIC_SEQ_CST); } + +static inline uint32_t +_Py_atomic_or_uint32(uint32_t *obj, uint32_t value) +{ return __atomic_fetch_or(obj, value, __ATOMIC_SEQ_CST); } + +static inline uint64_t +_Py_atomic_or_uint64(uint64_t *obj, uint64_t value) +{ return __atomic_fetch_or(obj, value, __ATOMIC_SEQ_CST); } + +static inline uintptr_t +_Py_atomic_or_uintptr(uintptr_t *obj, uintptr_t value) +{ return __atomic_fetch_or(obj, value, __ATOMIC_SEQ_CST); } + + +// --- _Py_atomic_load ------------------------------------------------------- + +static inline int +_Py_atomic_load_int(const int *obj) +{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); } + +static inline int8_t +_Py_atomic_load_int8(const int8_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); } + +static inline int16_t +_Py_atomic_load_int16(const int16_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); } + +static inline int32_t +_Py_atomic_load_int32(const int32_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); } + +static inline int64_t +_Py_atomic_load_int64(const int64_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); } + +static inline intptr_t +_Py_atomic_load_intptr(const intptr_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); } + +static inline uint8_t +_Py_atomic_load_uint8(const uint8_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); } + +static inline uint16_t +_Py_atomic_load_uint16(const uint16_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); } + +static inline uint32_t +_Py_atomic_load_uint32(const uint32_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); } + +static inline uint64_t +_Py_atomic_load_uint64(const uint64_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); } + +static inline uintptr_t +_Py_atomic_load_uintptr(const uintptr_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); } + +static inline unsigned int +_Py_atomic_load_uint(const unsigned int *obj) +{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); } + +static inline Py_ssize_t +_Py_atomic_load_ssize(const Py_ssize_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); } + +static inline void * +_Py_atomic_load_ptr(const void *obj) +{ return (void *)__atomic_load_n((void * const *)obj, __ATOMIC_SEQ_CST); } + + +// --- _Py_atomic_load_relaxed ----------------------------------------------- + +static inline int +_Py_atomic_load_int_relaxed(const int *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline int8_t +_Py_atomic_load_int8_relaxed(const int8_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline int16_t +_Py_atomic_load_int16_relaxed(const int16_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline int32_t +_Py_atomic_load_int32_relaxed(const int32_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline int64_t +_Py_atomic_load_int64_relaxed(const int64_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline intptr_t +_Py_atomic_load_intptr_relaxed(const intptr_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline uint8_t +_Py_atomic_load_uint8_relaxed(const uint8_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline uint16_t +_Py_atomic_load_uint16_relaxed(const uint16_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline uint32_t +_Py_atomic_load_uint32_relaxed(const uint32_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline uint64_t +_Py_atomic_load_uint64_relaxed(const uint64_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline uintptr_t +_Py_atomic_load_uintptr_relaxed(const uintptr_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline unsigned int +_Py_atomic_load_uint_relaxed(const unsigned int *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline Py_ssize_t +_Py_atomic_load_ssize_relaxed(const Py_ssize_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline void * +_Py_atomic_load_ptr_relaxed(const void *obj) +{ return (void *)__atomic_load_n((void * const *)obj, __ATOMIC_RELAXED); } + +static inline unsigned long long +_Py_atomic_load_ullong_relaxed(const unsigned long long *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + + +// --- _Py_atomic_store ------------------------------------------------------ + +static inline void +_Py_atomic_store_int(int *obj, int value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline void +_Py_atomic_store_int8(int8_t *obj, int8_t value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline void +_Py_atomic_store_int16(int16_t *obj, int16_t value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline void +_Py_atomic_store_int32(int32_t *obj, int32_t value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline void +_Py_atomic_store_int64(int64_t *obj, int64_t value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline void +_Py_atomic_store_intptr(intptr_t *obj, intptr_t value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline void +_Py_atomic_store_uint8(uint8_t *obj, uint8_t value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline void +_Py_atomic_store_uint16(uint16_t *obj, uint16_t value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline void +_Py_atomic_store_uint32(uint32_t *obj, uint32_t value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline void +_Py_atomic_store_uint64(uint64_t *obj, uint64_t value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline void +_Py_atomic_store_uintptr(uintptr_t *obj, uintptr_t value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline void +_Py_atomic_store_uint(unsigned int *obj, unsigned int value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + +static inline void +_Py_atomic_store_ptr(void *obj, void *value) +{ __atomic_store_n((void **)obj, value, __ATOMIC_SEQ_CST); } + +static inline void +_Py_atomic_store_ssize(Py_ssize_t *obj, Py_ssize_t value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + + +// --- _Py_atomic_store_relaxed ---------------------------------------------- + +static inline void +_Py_atomic_store_int_relaxed(int *obj, int value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_int8_relaxed(int8_t *obj, int8_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_int16_relaxed(int16_t *obj, int16_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_int32_relaxed(int32_t *obj, int32_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_int64_relaxed(int64_t *obj, int64_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_intptr_relaxed(intptr_t *obj, intptr_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_uint8_relaxed(uint8_t *obj, uint8_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_uint16_relaxed(uint16_t *obj, uint16_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_uint32_relaxed(uint32_t *obj, uint32_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_uint64_relaxed(uint64_t *obj, uint64_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_uintptr_relaxed(uintptr_t *obj, uintptr_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_uint_relaxed(unsigned int *obj, unsigned int value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_ptr_relaxed(void *obj, void *value) +{ __atomic_store_n((void **)obj, value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_ullong_relaxed(unsigned long long *obj, + unsigned long long value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + + +// --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ + +static inline void * +_Py_atomic_load_ptr_acquire(const void *obj) +{ return (void *)__atomic_load_n((void * const *)obj, __ATOMIC_ACQUIRE); } + +static inline uintptr_t +_Py_atomic_load_uintptr_acquire(const uintptr_t *obj) +{ return (uintptr_t)__atomic_load_n(obj, __ATOMIC_ACQUIRE); } + +static inline void +_Py_atomic_store_ptr_release(void *obj, void *value) +{ __atomic_store_n((void **)obj, value, __ATOMIC_RELEASE); } + +static inline void +_Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + +static inline void +_Py_atomic_store_int_release(int *obj, int value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + +static inline void +_Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + +static inline int +_Py_atomic_load_int_acquire(const int *obj) +{ return __atomic_load_n(obj, __ATOMIC_ACQUIRE); } + +static inline void +_Py_atomic_store_uint32_release(uint32_t *obj, uint32_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + +static inline void +_Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + +static inline uint64_t +_Py_atomic_load_uint64_acquire(const uint64_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_ACQUIRE); } + +static inline uint32_t +_Py_atomic_load_uint32_acquire(const uint32_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_ACQUIRE); } + +static inline Py_ssize_t +_Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_ACQUIRE); } + +// --- _Py_atomic_fence ------------------------------------------------------ + +static inline void +_Py_atomic_fence_seq_cst(void) +{ __atomic_thread_fence(__ATOMIC_SEQ_CST); } + + static inline void +_Py_atomic_fence_acquire(void) +{ __atomic_thread_fence(__ATOMIC_ACQUIRE); } + + static inline void +_Py_atomic_fence_release(void) +{ __atomic_thread_fence(__ATOMIC_RELEASE); } diff --git a/contrib/tools/python3/Include/cpython/pyatomic_msc.h b/contrib/tools/python3/Include/cpython/pyatomic_msc.h new file mode 100644 index 00000000000..84da21bdcbf --- /dev/null +++ b/contrib/tools/python3/Include/cpython/pyatomic_msc.h @@ -0,0 +1,1095 @@ +// This is the implementation of Python atomic operations for MSVC if the +// compiler does not support C11 or C++11 atomics. +// +// MSVC intrinsics are defined on char, short, long, __int64, and pointer +// types. Note that long and int are both 32-bits even on 64-bit Windows, +// so operations on int are cast to long. +// +// The volatile keyword has additional memory ordering semantics on MSVC. On +// x86 and x86-64, volatile accesses have acquire-release semantics. On ARM64, +// volatile accesses behave like C11's memory_order_relaxed. + +#ifndef Py_ATOMIC_MSC_H +# error "this header file must not be included directly" +#endif + +#include <intrin.h> + +#define _Py_atomic_ASSERT_ARG_TYPE(TYPE) \ + Py_BUILD_ASSERT(sizeof(*obj) == sizeof(TYPE)) + + +// --- _Py_atomic_add -------------------------------------------------------- + +static inline int8_t +_Py_atomic_add_int8(int8_t *obj, int8_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(char); + return (int8_t)_InterlockedExchangeAdd8((volatile char *)obj, (char)value); +} + +static inline int16_t +_Py_atomic_add_int16(int16_t *obj, int16_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(short); + return (int16_t)_InterlockedExchangeAdd16((volatile short *)obj, (short)value); +} + +static inline int32_t +_Py_atomic_add_int32(int32_t *obj, int32_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(long); + return (int32_t)_InterlockedExchangeAdd((volatile long *)obj, (long)value); +} + +static inline int64_t +_Py_atomic_add_int64(int64_t *obj, int64_t value) +{ +#if defined(_M_X64) || defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(__int64); + return (int64_t)_InterlockedExchangeAdd64((volatile __int64 *)obj, (__int64)value); +#else + int64_t old_value = _Py_atomic_load_int64_relaxed(obj); + for (;;) { + int64_t new_value = old_value + value; + if (_Py_atomic_compare_exchange_int64(obj, &old_value, new_value)) { + return old_value; + } + } +#endif +} + + +static inline uint8_t +_Py_atomic_add_uint8(uint8_t *obj, uint8_t value) +{ + return (uint8_t)_Py_atomic_add_int8((int8_t *)obj, (int8_t)value); +} + +static inline uint16_t +_Py_atomic_add_uint16(uint16_t *obj, uint16_t value) +{ + return (uint16_t)_Py_atomic_add_int16((int16_t *)obj, (int16_t)value); +} + +static inline uint32_t +_Py_atomic_add_uint32(uint32_t *obj, uint32_t value) +{ + return (uint32_t)_Py_atomic_add_int32((int32_t *)obj, (int32_t)value); +} + +static inline int +_Py_atomic_add_int(int *obj, int value) +{ + _Py_atomic_ASSERT_ARG_TYPE(int32_t); + return (int)_Py_atomic_add_int32((int32_t *)obj, (int32_t)value); +} + +static inline unsigned int +_Py_atomic_add_uint(unsigned int *obj, unsigned int value) +{ + _Py_atomic_ASSERT_ARG_TYPE(int32_t); + return (unsigned int)_Py_atomic_add_int32((int32_t *)obj, (int32_t)value); +} + +static inline uint64_t +_Py_atomic_add_uint64(uint64_t *obj, uint64_t value) +{ + return (uint64_t)_Py_atomic_add_int64((int64_t *)obj, (int64_t)value); +} + +static inline intptr_t +_Py_atomic_add_intptr(intptr_t *obj, intptr_t value) +{ +#if SIZEOF_VOID_P == 8 + _Py_atomic_ASSERT_ARG_TYPE(int64_t); + return (intptr_t)_Py_atomic_add_int64((int64_t *)obj, (int64_t)value); +#else + _Py_atomic_ASSERT_ARG_TYPE(int32_t); + return (intptr_t)_Py_atomic_add_int32((int32_t *)obj, (int32_t)value); +#endif +} + +static inline uintptr_t +_Py_atomic_add_uintptr(uintptr_t *obj, uintptr_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(intptr_t); + return (uintptr_t)_Py_atomic_add_intptr((intptr_t *)obj, (intptr_t)value); +} + +static inline Py_ssize_t +_Py_atomic_add_ssize(Py_ssize_t *obj, Py_ssize_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(intptr_t); + return (Py_ssize_t)_Py_atomic_add_intptr((intptr_t *)obj, (intptr_t)value); +} + + +// --- _Py_atomic_compare_exchange ------------------------------------------- + +static inline int +_Py_atomic_compare_exchange_int8(int8_t *obj, int8_t *expected, int8_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(char); + int8_t initial = (int8_t)_InterlockedCompareExchange8( + (volatile char *)obj, + (char)value, + (char)*expected); + if (initial == *expected) { + return 1; + } + *expected = initial; + return 0; +} + +static inline int +_Py_atomic_compare_exchange_int16(int16_t *obj, int16_t *expected, int16_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(short); + int16_t initial = (int16_t)_InterlockedCompareExchange16( + (volatile short *)obj, + (short)value, + (short)*expected); + if (initial == *expected) { + return 1; + } + *expected = initial; + return 0; +} + +static inline int +_Py_atomic_compare_exchange_int32(int32_t *obj, int32_t *expected, int32_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(long); + int32_t initial = (int32_t)_InterlockedCompareExchange( + (volatile long *)obj, + (long)value, + (long)*expected); + if (initial == *expected) { + return 1; + } + *expected = initial; + return 0; +} + +static inline int +_Py_atomic_compare_exchange_int64(int64_t *obj, int64_t *expected, int64_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(__int64); + int64_t initial = (int64_t)_InterlockedCompareExchange64( + (volatile __int64 *)obj, + (__int64)value, + (__int64)*expected); + if (initial == *expected) { + return 1; + } + *expected = initial; + return 0; +} + +static inline int +_Py_atomic_compare_exchange_ptr(void *obj, void *expected, void *value) +{ + void *initial = _InterlockedCompareExchangePointer( + (void**)obj, + value, + *(void**)expected); + if (initial == *(void**)expected) { + return 1; + } + *(void**)expected = initial; + return 0; +} + + +static inline int +_Py_atomic_compare_exchange_uint8(uint8_t *obj, uint8_t *expected, uint8_t value) +{ + return _Py_atomic_compare_exchange_int8((int8_t *)obj, + (int8_t *)expected, + (int8_t)value); +} + +static inline int +_Py_atomic_compare_exchange_uint16(uint16_t *obj, uint16_t *expected, uint16_t value) +{ + return _Py_atomic_compare_exchange_int16((int16_t *)obj, + (int16_t *)expected, + (int16_t)value); +} + +static inline int +_Py_atomic_compare_exchange_uint32(uint32_t *obj, uint32_t *expected, uint32_t value) +{ + return _Py_atomic_compare_exchange_int32((int32_t *)obj, + (int32_t *)expected, + (int32_t)value); +} + +static inline int +_Py_atomic_compare_exchange_int(int *obj, int *expected, int value) +{ + _Py_atomic_ASSERT_ARG_TYPE(int32_t); + return _Py_atomic_compare_exchange_int32((int32_t *)obj, + (int32_t *)expected, + (int32_t)value); +} + +static inline int +_Py_atomic_compare_exchange_uint(unsigned int *obj, unsigned int *expected, unsigned int value) +{ + _Py_atomic_ASSERT_ARG_TYPE(int32_t); + return _Py_atomic_compare_exchange_int32((int32_t *)obj, + (int32_t *)expected, + (int32_t)value); +} + +static inline int +_Py_atomic_compare_exchange_uint64(uint64_t *obj, uint64_t *expected, uint64_t value) +{ + return _Py_atomic_compare_exchange_int64((int64_t *)obj, + (int64_t *)expected, + (int64_t)value); +} + +static inline int +_Py_atomic_compare_exchange_intptr(intptr_t *obj, intptr_t *expected, intptr_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(void*); + return _Py_atomic_compare_exchange_ptr((void**)obj, + (void**)expected, + (void*)value); +} + +static inline int +_Py_atomic_compare_exchange_uintptr(uintptr_t *obj, uintptr_t *expected, uintptr_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(void*); + return _Py_atomic_compare_exchange_ptr((void**)obj, + (void**)expected, + (void*)value); +} + +static inline int +_Py_atomic_compare_exchange_ssize(Py_ssize_t *obj, Py_ssize_t *expected, Py_ssize_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(void*); + return _Py_atomic_compare_exchange_ptr((void**)obj, + (void**)expected, + (void*)value); +} + + +// --- _Py_atomic_exchange --------------------------------------------------- + +static inline int8_t +_Py_atomic_exchange_int8(int8_t *obj, int8_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(char); + return (int8_t)_InterlockedExchange8((volatile char *)obj, (char)value); +} + +static inline int16_t +_Py_atomic_exchange_int16(int16_t *obj, int16_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(short); + return (int16_t)_InterlockedExchange16((volatile short *)obj, (short)value); +} + +static inline int32_t +_Py_atomic_exchange_int32(int32_t *obj, int32_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(long); + return (int32_t)_InterlockedExchange((volatile long *)obj, (long)value); +} + +static inline int64_t +_Py_atomic_exchange_int64(int64_t *obj, int64_t value) +{ +#if defined(_M_X64) || defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(__int64); + return (int64_t)_InterlockedExchange64((volatile __int64 *)obj, (__int64)value); +#else + int64_t old_value = _Py_atomic_load_int64_relaxed(obj); + for (;;) { + if (_Py_atomic_compare_exchange_int64(obj, &old_value, value)) { + return old_value; + } + } +#endif +} + +static inline void* +_Py_atomic_exchange_ptr(void *obj, void *value) +{ + return (void*)_InterlockedExchangePointer((void * volatile *)obj, (void *)value); +} + + +static inline uint8_t +_Py_atomic_exchange_uint8(uint8_t *obj, uint8_t value) +{ + return (uint8_t)_Py_atomic_exchange_int8((int8_t *)obj, + (int8_t)value); +} + +static inline uint16_t +_Py_atomic_exchange_uint16(uint16_t *obj, uint16_t value) +{ + return (uint16_t)_Py_atomic_exchange_int16((int16_t *)obj, + (int16_t)value); +} + +static inline uint32_t +_Py_atomic_exchange_uint32(uint32_t *obj, uint32_t value) +{ + return (uint32_t)_Py_atomic_exchange_int32((int32_t *)obj, + (int32_t)value); +} + +static inline int +_Py_atomic_exchange_int(int *obj, int value) +{ + _Py_atomic_ASSERT_ARG_TYPE(int32_t); + return (int)_Py_atomic_exchange_int32((int32_t *)obj, + (int32_t)value); +} + +static inline unsigned int +_Py_atomic_exchange_uint(unsigned int *obj, unsigned int value) +{ + _Py_atomic_ASSERT_ARG_TYPE(int32_t); + return (unsigned int)_Py_atomic_exchange_int32((int32_t *)obj, + (int32_t)value); +} + +static inline uint64_t +_Py_atomic_exchange_uint64(uint64_t *obj, uint64_t value) +{ + return (uint64_t)_Py_atomic_exchange_int64((int64_t *)obj, + (int64_t)value); +} + +static inline intptr_t +_Py_atomic_exchange_intptr(intptr_t *obj, intptr_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(void*); + return (intptr_t)_Py_atomic_exchange_ptr((void**)obj, + (void*)value); +} + +static inline uintptr_t +_Py_atomic_exchange_uintptr(uintptr_t *obj, uintptr_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(void*); + return (uintptr_t)_Py_atomic_exchange_ptr((void**)obj, + (void*)value); +} + +static inline Py_ssize_t +_Py_atomic_exchange_ssize(Py_ssize_t *obj, Py_ssize_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(void*); + return (Py_ssize_t)_Py_atomic_exchange_ptr((void**)obj, + (void*)value); +} + + +// --- _Py_atomic_and -------------------------------------------------------- + +static inline uint8_t +_Py_atomic_and_uint8(uint8_t *obj, uint8_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(char); + return (uint8_t)_InterlockedAnd8((volatile char *)obj, (char)value); +} + +static inline uint16_t +_Py_atomic_and_uint16(uint16_t *obj, uint16_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(short); + return (uint16_t)_InterlockedAnd16((volatile short *)obj, (short)value); +} + +static inline uint32_t +_Py_atomic_and_uint32(uint32_t *obj, uint32_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(long); + return (uint32_t)_InterlockedAnd((volatile long *)obj, (long)value); +} + +static inline uint64_t +_Py_atomic_and_uint64(uint64_t *obj, uint64_t value) +{ +#if defined(_M_X64) || defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(__int64); + return (uint64_t)_InterlockedAnd64((volatile __int64 *)obj, (__int64)value); +#else + uint64_t old_value = _Py_atomic_load_uint64_relaxed(obj); + for (;;) { + uint64_t new_value = old_value & value; + if (_Py_atomic_compare_exchange_uint64(obj, &old_value, new_value)) { + return old_value; + } + } +#endif +} + +static inline uintptr_t +_Py_atomic_and_uintptr(uintptr_t *obj, uintptr_t value) +{ +#if SIZEOF_VOID_P == 8 + _Py_atomic_ASSERT_ARG_TYPE(uint64_t); + return (uintptr_t)_Py_atomic_and_uint64((uint64_t *)obj, + (uint64_t)value); +#else + _Py_atomic_ASSERT_ARG_TYPE(uint32_t); + return (uintptr_t)_Py_atomic_and_uint32((uint32_t *)obj, + (uint32_t)value); +#endif +} + + +// --- _Py_atomic_or --------------------------------------------------------- + +static inline uint8_t +_Py_atomic_or_uint8(uint8_t *obj, uint8_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(char); + return (uint8_t)_InterlockedOr8((volatile char *)obj, (char)value); +} + +static inline uint16_t +_Py_atomic_or_uint16(uint16_t *obj, uint16_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(short); + return (uint16_t)_InterlockedOr16((volatile short *)obj, (short)value); +} + +static inline uint32_t +_Py_atomic_or_uint32(uint32_t *obj, uint32_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(long); + return (uint32_t)_InterlockedOr((volatile long *)obj, (long)value); +} + +static inline uint64_t +_Py_atomic_or_uint64(uint64_t *obj, uint64_t value) +{ +#if defined(_M_X64) || defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(__int64); + return (uint64_t)_InterlockedOr64((volatile __int64 *)obj, (__int64)value); +#else + uint64_t old_value = _Py_atomic_load_uint64_relaxed(obj); + for (;;) { + uint64_t new_value = old_value | value; + if (_Py_atomic_compare_exchange_uint64(obj, &old_value, new_value)) { + return old_value; + } + } +#endif +} + + +static inline uintptr_t +_Py_atomic_or_uintptr(uintptr_t *obj, uintptr_t value) +{ +#if SIZEOF_VOID_P == 8 + _Py_atomic_ASSERT_ARG_TYPE(uint64_t); + return (uintptr_t)_Py_atomic_or_uint64((uint64_t *)obj, + (uint64_t)value); +#else + _Py_atomic_ASSERT_ARG_TYPE(uint32_t); + return (uintptr_t)_Py_atomic_or_uint32((uint32_t *)obj, + (uint32_t)value); +#endif +} + + +// --- _Py_atomic_load ------------------------------------------------------- + +static inline uint8_t +_Py_atomic_load_uint8(const uint8_t *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(volatile uint8_t *)obj; +#elif defined(_M_ARM64) + return (uint8_t)__ldar8((unsigned __int8 volatile *)obj); +#else +# error "no implementation of _Py_atomic_load_uint8" +#endif +} + +static inline uint16_t +_Py_atomic_load_uint16(const uint16_t *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(volatile uint16_t *)obj; +#elif defined(_M_ARM64) + return (uint16_t)__ldar16((unsigned __int16 volatile *)obj); +#else +# error "no implementation of _Py_atomic_load_uint16" +#endif +} + +static inline uint32_t +_Py_atomic_load_uint32(const uint32_t *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(volatile uint32_t *)obj; +#elif defined(_M_ARM64) + return (uint32_t)__ldar32((unsigned __int32 volatile *)obj); +#else +# error "no implementation of _Py_atomic_load_uint32" +#endif +} + +static inline uint64_t +_Py_atomic_load_uint64(const uint64_t *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(volatile uint64_t *)obj; +#elif defined(_M_ARM64) + return (uint64_t)__ldar64((unsigned __int64 volatile *)obj); +#else +# error "no implementation of _Py_atomic_load_uint64" +#endif +} + +static inline int8_t +_Py_atomic_load_int8(const int8_t *obj) +{ + return (int8_t)_Py_atomic_load_uint8((const uint8_t *)obj); +} + +static inline int16_t +_Py_atomic_load_int16(const int16_t *obj) +{ + return (int16_t)_Py_atomic_load_uint16((const uint16_t *)obj); +} + +static inline int32_t +_Py_atomic_load_int32(const int32_t *obj) +{ + return (int32_t)_Py_atomic_load_uint32((const uint32_t *)obj); +} + +static inline int +_Py_atomic_load_int(const int *obj) +{ + _Py_atomic_ASSERT_ARG_TYPE(uint32_t); + return (int)_Py_atomic_load_uint32((uint32_t *)obj); +} + +static inline unsigned int +_Py_atomic_load_uint(const unsigned int *obj) +{ + _Py_atomic_ASSERT_ARG_TYPE(uint32_t); + return (unsigned int)_Py_atomic_load_uint32((uint32_t *)obj); +} + +static inline int64_t +_Py_atomic_load_int64(const int64_t *obj) +{ + return (int64_t)_Py_atomic_load_uint64((const uint64_t *)obj); +} + +static inline void* +_Py_atomic_load_ptr(const void *obj) +{ +#if SIZEOF_VOID_P == 8 + return (void*)_Py_atomic_load_uint64((const uint64_t *)obj); +#else + return (void*)_Py_atomic_load_uint32((const uint32_t *)obj); +#endif +} + +static inline intptr_t +_Py_atomic_load_intptr(const intptr_t *obj) +{ + _Py_atomic_ASSERT_ARG_TYPE(void*); + return (intptr_t)_Py_atomic_load_ptr((void*)obj); +} + +static inline uintptr_t +_Py_atomic_load_uintptr(const uintptr_t *obj) +{ + _Py_atomic_ASSERT_ARG_TYPE(void*); + return (uintptr_t)_Py_atomic_load_ptr((void*)obj); +} + +static inline Py_ssize_t +_Py_atomic_load_ssize(const Py_ssize_t *obj) +{ + _Py_atomic_ASSERT_ARG_TYPE(void*); + return (Py_ssize_t)_Py_atomic_load_ptr((void*)obj); +} + + +// --- _Py_atomic_load_relaxed ----------------------------------------------- + +static inline int +_Py_atomic_load_int_relaxed(const int *obj) +{ + return *(volatile int *)obj; +} + +static inline int8_t +_Py_atomic_load_int8_relaxed(const int8_t *obj) +{ + return *(volatile int8_t *)obj; +} + +static inline int16_t +_Py_atomic_load_int16_relaxed(const int16_t *obj) +{ + return *(volatile int16_t *)obj; +} + +static inline int32_t +_Py_atomic_load_int32_relaxed(const int32_t *obj) +{ + return *(volatile int32_t *)obj; +} + +static inline int64_t +_Py_atomic_load_int64_relaxed(const int64_t *obj) +{ + return *(volatile int64_t *)obj; +} + +static inline intptr_t +_Py_atomic_load_intptr_relaxed(const intptr_t *obj) +{ + return *(volatile intptr_t *)obj; +} + +static inline uint8_t +_Py_atomic_load_uint8_relaxed(const uint8_t *obj) +{ + return *(volatile uint8_t *)obj; +} + +static inline uint16_t +_Py_atomic_load_uint16_relaxed(const uint16_t *obj) +{ + return *(volatile uint16_t *)obj; +} + +static inline uint32_t +_Py_atomic_load_uint32_relaxed(const uint32_t *obj) +{ + return *(volatile uint32_t *)obj; +} + +static inline uint64_t +_Py_atomic_load_uint64_relaxed(const uint64_t *obj) +{ + return *(volatile uint64_t *)obj; +} + +static inline uintptr_t +_Py_atomic_load_uintptr_relaxed(const uintptr_t *obj) +{ + return *(volatile uintptr_t *)obj; +} + +static inline unsigned int +_Py_atomic_load_uint_relaxed(const unsigned int *obj) +{ + return *(volatile unsigned int *)obj; +} + +static inline Py_ssize_t +_Py_atomic_load_ssize_relaxed(const Py_ssize_t *obj) +{ + return *(volatile Py_ssize_t *)obj; +} + +static inline void* +_Py_atomic_load_ptr_relaxed(const void *obj) +{ + return *(void * volatile *)obj; +} + +static inline unsigned long long +_Py_atomic_load_ullong_relaxed(const unsigned long long *obj) +{ + return *(volatile unsigned long long *)obj; +} + + +// --- _Py_atomic_store ------------------------------------------------------ + +static inline void +_Py_atomic_store_int(int *obj, int value) +{ + (void)_Py_atomic_exchange_int(obj, value); +} + +static inline void +_Py_atomic_store_int8(int8_t *obj, int8_t value) +{ + (void)_Py_atomic_exchange_int8(obj, value); +} + +static inline void +_Py_atomic_store_int16(int16_t *obj, int16_t value) +{ + (void)_Py_atomic_exchange_int16(obj, value); +} + +static inline void +_Py_atomic_store_int32(int32_t *obj, int32_t value) +{ + (void)_Py_atomic_exchange_int32(obj, value); +} + +static inline void +_Py_atomic_store_int64(int64_t *obj, int64_t value) +{ + (void)_Py_atomic_exchange_int64(obj, value); +} + +static inline void +_Py_atomic_store_intptr(intptr_t *obj, intptr_t value) +{ + (void)_Py_atomic_exchange_intptr(obj, value); +} + +static inline void +_Py_atomic_store_uint8(uint8_t *obj, uint8_t value) +{ + (void)_Py_atomic_exchange_uint8(obj, value); +} + +static inline void +_Py_atomic_store_uint16(uint16_t *obj, uint16_t value) +{ + (void)_Py_atomic_exchange_uint16(obj, value); +} + +static inline void +_Py_atomic_store_uint32(uint32_t *obj, uint32_t value) +{ + (void)_Py_atomic_exchange_uint32(obj, value); +} + +static inline void +_Py_atomic_store_uint64(uint64_t *obj, uint64_t value) +{ + (void)_Py_atomic_exchange_uint64(obj, value); +} + +static inline void +_Py_atomic_store_uintptr(uintptr_t *obj, uintptr_t value) +{ + (void)_Py_atomic_exchange_uintptr(obj, value); +} + +static inline void +_Py_atomic_store_uint(unsigned int *obj, unsigned int value) +{ + (void)_Py_atomic_exchange_uint(obj, value); +} + +static inline void +_Py_atomic_store_ptr(void *obj, void *value) +{ + (void)_Py_atomic_exchange_ptr(obj, value); +} + +static inline void +_Py_atomic_store_ssize(Py_ssize_t *obj, Py_ssize_t value) +{ + (void)_Py_atomic_exchange_ssize(obj, value); +} + + +// --- _Py_atomic_store_relaxed ---------------------------------------------- + +static inline void +_Py_atomic_store_int_relaxed(int *obj, int value) +{ + *(volatile int *)obj = value; +} + +static inline void +_Py_atomic_store_int8_relaxed(int8_t *obj, int8_t value) +{ + *(volatile int8_t *)obj = value; +} + +static inline void +_Py_atomic_store_int16_relaxed(int16_t *obj, int16_t value) +{ + *(volatile int16_t *)obj = value; +} + +static inline void +_Py_atomic_store_int32_relaxed(int32_t *obj, int32_t value) +{ + *(volatile int32_t *)obj = value; +} + +static inline void +_Py_atomic_store_int64_relaxed(int64_t *obj, int64_t value) +{ + *(volatile int64_t *)obj = value; +} + +static inline void +_Py_atomic_store_intptr_relaxed(intptr_t *obj, intptr_t value) +{ + *(volatile intptr_t *)obj = value; +} + +static inline void +_Py_atomic_store_uint8_relaxed(uint8_t *obj, uint8_t value) +{ + *(volatile uint8_t *)obj = value; +} + +static inline void +_Py_atomic_store_uint16_relaxed(uint16_t *obj, uint16_t value) +{ + *(volatile uint16_t *)obj = value; +} + +static inline void +_Py_atomic_store_uint32_relaxed(uint32_t *obj, uint32_t value) +{ + *(volatile uint32_t *)obj = value; +} + +static inline void +_Py_atomic_store_uint64_relaxed(uint64_t *obj, uint64_t value) +{ + *(volatile uint64_t *)obj = value; +} + +static inline void +_Py_atomic_store_uintptr_relaxed(uintptr_t *obj, uintptr_t value) +{ + *(volatile uintptr_t *)obj = value; +} + +static inline void +_Py_atomic_store_uint_relaxed(unsigned int *obj, unsigned int value) +{ + *(volatile unsigned int *)obj = value; +} + +static inline void +_Py_atomic_store_ptr_relaxed(void *obj, void* value) +{ + *(void * volatile *)obj = value; +} + +static inline void +_Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value) +{ + *(volatile Py_ssize_t *)obj = value; +} + +static inline void +_Py_atomic_store_ullong_relaxed(unsigned long long *obj, + unsigned long long value) +{ + *(volatile unsigned long long *)obj = value; +} + + +// --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ + +static inline void * +_Py_atomic_load_ptr_acquire(const void *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(void * volatile *)obj; +#elif defined(_M_ARM64) + return (void *)__ldar64((unsigned __int64 volatile *)obj); +#else +# error "no implementation of _Py_atomic_load_ptr_acquire" +#endif +} + +static inline uintptr_t +_Py_atomic_load_uintptr_acquire(const uintptr_t *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(uintptr_t volatile *)obj; +#elif defined(_M_ARM64) + return (uintptr_t)__ldar64((unsigned __int64 volatile *)obj); +#else +# error "no implementation of _Py_atomic_load_uintptr_acquire" +#endif +} + +static inline void +_Py_atomic_store_ptr_release(void *obj, void *value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(void * volatile *)obj = value; +#elif defined(_M_ARM64) + __stlr64((unsigned __int64 volatile *)obj, (uintptr_t)value); +#else +# error "no implementation of _Py_atomic_store_ptr_release" +#endif +} + +static inline void +_Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(uintptr_t volatile *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int64); + __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value); +#else +# error "no implementation of _Py_atomic_store_uintptr_release" +#endif +} + +static inline void +_Py_atomic_store_int_release(int *obj, int value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(int volatile *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32); + __stlr32((unsigned __int32 volatile *)obj, (unsigned __int32)value); +#else +# error "no implementation of _Py_atomic_store_int_release" +#endif +} + +static inline void +_Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(Py_ssize_t volatile *)obj = value; +#elif defined(_M_ARM64) + __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value); +#else +# error "no implementation of _Py_atomic_store_ssize_release" +#endif +} + +static inline int +_Py_atomic_load_int_acquire(const int *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(int volatile *)obj; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32); + return (int)__ldar32((unsigned __int32 volatile *)obj); +#else +# error "no implementation of _Py_atomic_load_int_acquire" +#endif +} + +static inline void +_Py_atomic_store_uint32_release(uint32_t *obj, uint32_t value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(uint32_t volatile *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32); + __stlr32((unsigned __int32 volatile *)obj, (unsigned __int32)value); +#else +# error "no implementation of _Py_atomic_store_uint32_release" +#endif +} + +static inline void +_Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(uint64_t volatile *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int64); + __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value); +#else +# error "no implementation of _Py_atomic_store_uint64_release" +#endif +} + +static inline uint64_t +_Py_atomic_load_uint64_acquire(const uint64_t *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(uint64_t volatile *)obj; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(__int64); + return (uint64_t)__ldar64((unsigned __int64 volatile *)obj); +#else +# error "no implementation of _Py_atomic_load_uint64_acquire" +#endif +} + +static inline uint32_t +_Py_atomic_load_uint32_acquire(const uint32_t *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(uint32_t volatile *)obj; +#elif defined(_M_ARM64) + return (uint32_t)__ldar32((uint32_t volatile *)obj); +#else +# error "no implementation of _Py_atomic_load_uint32_acquire" +#endif +} + +static inline Py_ssize_t +_Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(Py_ssize_t volatile *)obj; +#elif defined(_M_ARM64) + return (Py_ssize_t)__ldar64((unsigned __int64 volatile *)obj); +#else +# error "no implementation of _Py_atomic_load_ssize_acquire" +#endif +} + +// --- _Py_atomic_fence ------------------------------------------------------ + + static inline void +_Py_atomic_fence_seq_cst(void) +{ +#if defined(_M_ARM64) + __dmb(_ARM64_BARRIER_ISH); +#elif defined(_M_X64) + __faststorefence(); +#elif defined(_M_IX86) + _mm_mfence(); +#else +# error "no implementation of _Py_atomic_fence_seq_cst" +#endif +} + + static inline void +_Py_atomic_fence_acquire(void) +{ +#if defined(_M_ARM64) + __dmb(_ARM64_BARRIER_ISHLD); +#elif defined(_M_X64) || defined(_M_IX86) + _ReadBarrier(); +#else +# error "no implementation of _Py_atomic_fence_acquire" +#endif +} + + static inline void +_Py_atomic_fence_release(void) +{ +#if defined(_M_ARM64) + __dmb(_ARM64_BARRIER_ISH); +#elif defined(_M_X64) || defined(_M_IX86) + _ReadWriteBarrier(); +#else +# error "no implementation of _Py_atomic_fence_release" +#endif +} + +#undef _Py_atomic_ASSERT_ARG_TYPE diff --git a/contrib/tools/python3/Include/cpython/pyatomic_std.h b/contrib/tools/python3/Include/cpython/pyatomic_std.h new file mode 100644 index 00000000000..ab3a4e1c74c --- /dev/null +++ b/contrib/tools/python3/Include/cpython/pyatomic_std.h @@ -0,0 +1,976 @@ +// This is the implementation of Python atomic operations using C++11 or C11 +// atomics. Note that the pyatomic_gcc.h implementation is preferred for GCC +// compatible compilers, even if they support C++11 atomics. + +#ifndef Py_ATOMIC_STD_H +# error "this header file must not be included directly" +#endif + +#ifdef __cplusplus +extern "C++" { +# include <atomic> +} +# define _Py_USING_STD using namespace std +# define _Atomic(tp) atomic<tp> +#else +# define _Py_USING_STD +# include <stdatomic.h> +#endif + + +// --- _Py_atomic_add -------------------------------------------------------- + +static inline int +_Py_atomic_add_int(int *obj, int value) +{ + _Py_USING_STD; + return atomic_fetch_add((_Atomic(int)*)obj, value); +} + +static inline int8_t +_Py_atomic_add_int8(int8_t *obj, int8_t value) +{ + _Py_USING_STD; + return atomic_fetch_add((_Atomic(int8_t)*)obj, value); +} + +static inline int16_t +_Py_atomic_add_int16(int16_t *obj, int16_t value) +{ + _Py_USING_STD; + return atomic_fetch_add((_Atomic(int16_t)*)obj, value); +} + +static inline int32_t +_Py_atomic_add_int32(int32_t *obj, int32_t value) +{ + _Py_USING_STD; + return atomic_fetch_add((_Atomic(int32_t)*)obj, value); +} + +static inline int64_t +_Py_atomic_add_int64(int64_t *obj, int64_t value) +{ + _Py_USING_STD; + return atomic_fetch_add((_Atomic(int64_t)*)obj, value); +} + +static inline intptr_t +_Py_atomic_add_intptr(intptr_t *obj, intptr_t value) +{ + _Py_USING_STD; + return atomic_fetch_add((_Atomic(intptr_t)*)obj, value); +} + +static inline unsigned int +_Py_atomic_add_uint(unsigned int *obj, unsigned int value) +{ + _Py_USING_STD; + return atomic_fetch_add((_Atomic(unsigned int)*)obj, value); +} + +static inline uint8_t +_Py_atomic_add_uint8(uint8_t *obj, uint8_t value) +{ + _Py_USING_STD; + return atomic_fetch_add((_Atomic(uint8_t)*)obj, value); +} + +static inline uint16_t +_Py_atomic_add_uint16(uint16_t *obj, uint16_t value) +{ + _Py_USING_STD; + return atomic_fetch_add((_Atomic(uint16_t)*)obj, value); +} + +static inline uint32_t +_Py_atomic_add_uint32(uint32_t *obj, uint32_t value) +{ + _Py_USING_STD; + return atomic_fetch_add((_Atomic(uint32_t)*)obj, value); +} + +static inline uint64_t +_Py_atomic_add_uint64(uint64_t *obj, uint64_t value) +{ + _Py_USING_STD; + return atomic_fetch_add((_Atomic(uint64_t)*)obj, value); +} + +static inline uintptr_t +_Py_atomic_add_uintptr(uintptr_t *obj, uintptr_t value) +{ + _Py_USING_STD; + return atomic_fetch_add((_Atomic(uintptr_t)*)obj, value); +} + +static inline Py_ssize_t +_Py_atomic_add_ssize(Py_ssize_t *obj, Py_ssize_t value) +{ + _Py_USING_STD; + return atomic_fetch_add((_Atomic(Py_ssize_t)*)obj, value); +} + + +// --- _Py_atomic_compare_exchange ------------------------------------------- + +static inline int +_Py_atomic_compare_exchange_int(int *obj, int *expected, int desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong((_Atomic(int)*)obj, + expected, desired); +} + +static inline int +_Py_atomic_compare_exchange_int8(int8_t *obj, int8_t *expected, int8_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong((_Atomic(int8_t)*)obj, + expected, desired); +} + +static inline int +_Py_atomic_compare_exchange_int16(int16_t *obj, int16_t *expected, int16_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong((_Atomic(int16_t)*)obj, + expected, desired); +} + +static inline int +_Py_atomic_compare_exchange_int32(int32_t *obj, int32_t *expected, int32_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong((_Atomic(int32_t)*)obj, + expected, desired); +} + +static inline int +_Py_atomic_compare_exchange_int64(int64_t *obj, int64_t *expected, int64_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong((_Atomic(int64_t)*)obj, + expected, desired); +} + +static inline int +_Py_atomic_compare_exchange_intptr(intptr_t *obj, intptr_t *expected, intptr_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong((_Atomic(intptr_t)*)obj, + expected, desired); +} + +static inline int +_Py_atomic_compare_exchange_uint(unsigned int *obj, unsigned int *expected, unsigned int desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong((_Atomic(unsigned int)*)obj, + expected, desired); +} + +static inline int +_Py_atomic_compare_exchange_uint8(uint8_t *obj, uint8_t *expected, uint8_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong((_Atomic(uint8_t)*)obj, + expected, desired); +} + +static inline int +_Py_atomic_compare_exchange_uint16(uint16_t *obj, uint16_t *expected, uint16_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong((_Atomic(uint16_t)*)obj, + expected, desired); +} + +static inline int +_Py_atomic_compare_exchange_uint32(uint32_t *obj, uint32_t *expected, uint32_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong((_Atomic(uint32_t)*)obj, + expected, desired); +} + +static inline int +_Py_atomic_compare_exchange_uint64(uint64_t *obj, uint64_t *expected, uint64_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong((_Atomic(uint64_t)*)obj, + expected, desired); +} + +static inline int +_Py_atomic_compare_exchange_uintptr(uintptr_t *obj, uintptr_t *expected, uintptr_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong((_Atomic(uintptr_t)*)obj, + expected, desired); +} + +static inline int +_Py_atomic_compare_exchange_ssize(Py_ssize_t *obj, Py_ssize_t *expected, Py_ssize_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong((_Atomic(Py_ssize_t)*)obj, + expected, desired); +} + +static inline int +_Py_atomic_compare_exchange_ptr(void *obj, void *expected, void *desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong((_Atomic(void *)*)obj, + (void **)expected, desired); +} + + +// --- _Py_atomic_exchange --------------------------------------------------- + +static inline int +_Py_atomic_exchange_int(int *obj, int value) +{ + _Py_USING_STD; + return atomic_exchange((_Atomic(int)*)obj, value); +} + +static inline int8_t +_Py_atomic_exchange_int8(int8_t *obj, int8_t value) +{ + _Py_USING_STD; + return atomic_exchange((_Atomic(int8_t)*)obj, value); +} + +static inline int16_t +_Py_atomic_exchange_int16(int16_t *obj, int16_t value) +{ + _Py_USING_STD; + return atomic_exchange((_Atomic(int16_t)*)obj, value); +} + +static inline int32_t +_Py_atomic_exchange_int32(int32_t *obj, int32_t value) +{ + _Py_USING_STD; + return atomic_exchange((_Atomic(int32_t)*)obj, value); +} + +static inline int64_t +_Py_atomic_exchange_int64(int64_t *obj, int64_t value) +{ + _Py_USING_STD; + return atomic_exchange((_Atomic(int64_t)*)obj, value); +} + +static inline intptr_t +_Py_atomic_exchange_intptr(intptr_t *obj, intptr_t value) +{ + _Py_USING_STD; + return atomic_exchange((_Atomic(intptr_t)*)obj, value); +} + +static inline unsigned int +_Py_atomic_exchange_uint(unsigned int *obj, unsigned int value) +{ + _Py_USING_STD; + return atomic_exchange((_Atomic(unsigned int)*)obj, value); +} + +static inline uint8_t +_Py_atomic_exchange_uint8(uint8_t *obj, uint8_t value) +{ + _Py_USING_STD; + return atomic_exchange((_Atomic(uint8_t)*)obj, value); +} + +static inline uint16_t +_Py_atomic_exchange_uint16(uint16_t *obj, uint16_t value) +{ + _Py_USING_STD; + return atomic_exchange((_Atomic(uint16_t)*)obj, value); +} + +static inline uint32_t +_Py_atomic_exchange_uint32(uint32_t *obj, uint32_t value) +{ + _Py_USING_STD; + return atomic_exchange((_Atomic(uint32_t)*)obj, value); +} + +static inline uint64_t +_Py_atomic_exchange_uint64(uint64_t *obj, uint64_t value) +{ + _Py_USING_STD; + return atomic_exchange((_Atomic(uint64_t)*)obj, value); +} + +static inline uintptr_t +_Py_atomic_exchange_uintptr(uintptr_t *obj, uintptr_t value) +{ + _Py_USING_STD; + return atomic_exchange((_Atomic(uintptr_t)*)obj, value); +} + +static inline Py_ssize_t +_Py_atomic_exchange_ssize(Py_ssize_t *obj, Py_ssize_t value) +{ + _Py_USING_STD; + return atomic_exchange((_Atomic(Py_ssize_t)*)obj, value); +} + +static inline void* +_Py_atomic_exchange_ptr(void *obj, void *value) +{ + _Py_USING_STD; + return atomic_exchange((_Atomic(void *)*)obj, value); +} + + +// --- _Py_atomic_and -------------------------------------------------------- + +static inline uint8_t +_Py_atomic_and_uint8(uint8_t *obj, uint8_t value) +{ + _Py_USING_STD; + return atomic_fetch_and((_Atomic(uint8_t)*)obj, value); +} + +static inline uint16_t +_Py_atomic_and_uint16(uint16_t *obj, uint16_t value) +{ + _Py_USING_STD; + return atomic_fetch_and((_Atomic(uint16_t)*)obj, value); +} + +static inline uint32_t +_Py_atomic_and_uint32(uint32_t *obj, uint32_t value) +{ + _Py_USING_STD; + return atomic_fetch_and((_Atomic(uint32_t)*)obj, value); +} + +static inline uint64_t +_Py_atomic_and_uint64(uint64_t *obj, uint64_t value) +{ + _Py_USING_STD; + return atomic_fetch_and((_Atomic(uint64_t)*)obj, value); +} + +static inline uintptr_t +_Py_atomic_and_uintptr(uintptr_t *obj, uintptr_t value) +{ + _Py_USING_STD; + return atomic_fetch_and((_Atomic(uintptr_t)*)obj, value); +} + + +// --- _Py_atomic_or --------------------------------------------------------- + +static inline uint8_t +_Py_atomic_or_uint8(uint8_t *obj, uint8_t value) +{ + _Py_USING_STD; + return atomic_fetch_or((_Atomic(uint8_t)*)obj, value); +} + +static inline uint16_t +_Py_atomic_or_uint16(uint16_t *obj, uint16_t value) +{ + _Py_USING_STD; + return atomic_fetch_or((_Atomic(uint16_t)*)obj, value); +} + +static inline uint32_t +_Py_atomic_or_uint32(uint32_t *obj, uint32_t value) +{ + _Py_USING_STD; + return atomic_fetch_or((_Atomic(uint32_t)*)obj, value); +} + +static inline uint64_t +_Py_atomic_or_uint64(uint64_t *obj, uint64_t value) +{ + _Py_USING_STD; + return atomic_fetch_or((_Atomic(uint64_t)*)obj, value); +} + +static inline uintptr_t +_Py_atomic_or_uintptr(uintptr_t *obj, uintptr_t value) +{ + _Py_USING_STD; + return atomic_fetch_or((_Atomic(uintptr_t)*)obj, value); +} + + +// --- _Py_atomic_load ------------------------------------------------------- + +static inline int +_Py_atomic_load_int(const int *obj) +{ + _Py_USING_STD; + return atomic_load((const _Atomic(int)*)obj); +} + +static inline int8_t +_Py_atomic_load_int8(const int8_t *obj) +{ + _Py_USING_STD; + return atomic_load((const _Atomic(int8_t)*)obj); +} + +static inline int16_t +_Py_atomic_load_int16(const int16_t *obj) +{ + _Py_USING_STD; + return atomic_load((const _Atomic(int16_t)*)obj); +} + +static inline int32_t +_Py_atomic_load_int32(const int32_t *obj) +{ + _Py_USING_STD; + return atomic_load((const _Atomic(int32_t)*)obj); +} + +static inline int64_t +_Py_atomic_load_int64(const int64_t *obj) +{ + _Py_USING_STD; + return atomic_load((const _Atomic(int64_t)*)obj); +} + +static inline intptr_t +_Py_atomic_load_intptr(const intptr_t *obj) +{ + _Py_USING_STD; + return atomic_load((const _Atomic(intptr_t)*)obj); +} + +static inline uint8_t +_Py_atomic_load_uint8(const uint8_t *obj) +{ + _Py_USING_STD; + return atomic_load((const _Atomic(uint8_t)*)obj); +} + +static inline uint16_t +_Py_atomic_load_uint16(const uint16_t *obj) +{ + _Py_USING_STD; + return atomic_load((const _Atomic(uint16_t)*)obj); +} + +static inline uint32_t +_Py_atomic_load_uint32(const uint32_t *obj) +{ + _Py_USING_STD; + return atomic_load((const _Atomic(uint32_t)*)obj); +} + +static inline uint64_t +_Py_atomic_load_uint64(const uint64_t *obj) +{ + _Py_USING_STD; + return atomic_load((const _Atomic(uint64_t)*)obj); +} + +static inline uintptr_t +_Py_atomic_load_uintptr(const uintptr_t *obj) +{ + _Py_USING_STD; + return atomic_load((const _Atomic(uintptr_t)*)obj); +} + +static inline unsigned int +_Py_atomic_load_uint(const unsigned int *obj) +{ + _Py_USING_STD; + return atomic_load((const _Atomic(unsigned int)*)obj); +} + +static inline Py_ssize_t +_Py_atomic_load_ssize(const Py_ssize_t *obj) +{ + _Py_USING_STD; + return atomic_load((const _Atomic(Py_ssize_t)*)obj); +} + +static inline void* +_Py_atomic_load_ptr(const void *obj) +{ + _Py_USING_STD; + return atomic_load((const _Atomic(void*)*)obj); +} + + +// --- _Py_atomic_load_relaxed ----------------------------------------------- + +static inline int +_Py_atomic_load_int_relaxed(const int *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(int)*)obj, + memory_order_relaxed); +} + +static inline int8_t +_Py_atomic_load_int8_relaxed(const int8_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(int8_t)*)obj, + memory_order_relaxed); +} + +static inline int16_t +_Py_atomic_load_int16_relaxed(const int16_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(int16_t)*)obj, + memory_order_relaxed); +} + +static inline int32_t +_Py_atomic_load_int32_relaxed(const int32_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(int32_t)*)obj, + memory_order_relaxed); +} + +static inline int64_t +_Py_atomic_load_int64_relaxed(const int64_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(int64_t)*)obj, + memory_order_relaxed); +} + +static inline intptr_t +_Py_atomic_load_intptr_relaxed(const intptr_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(intptr_t)*)obj, + memory_order_relaxed); +} + +static inline uint8_t +_Py_atomic_load_uint8_relaxed(const uint8_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(uint8_t)*)obj, + memory_order_relaxed); +} + +static inline uint16_t +_Py_atomic_load_uint16_relaxed(const uint16_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(uint16_t)*)obj, + memory_order_relaxed); +} + +static inline uint32_t +_Py_atomic_load_uint32_relaxed(const uint32_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(uint32_t)*)obj, + memory_order_relaxed); +} + +static inline uint64_t +_Py_atomic_load_uint64_relaxed(const uint64_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(uint64_t)*)obj, + memory_order_relaxed); +} + +static inline uintptr_t +_Py_atomic_load_uintptr_relaxed(const uintptr_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(uintptr_t)*)obj, + memory_order_relaxed); +} + +static inline unsigned int +_Py_atomic_load_uint_relaxed(const unsigned int *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(unsigned int)*)obj, + memory_order_relaxed); +} + +static inline Py_ssize_t +_Py_atomic_load_ssize_relaxed(const Py_ssize_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(Py_ssize_t)*)obj, + memory_order_relaxed); +} + +static inline void* +_Py_atomic_load_ptr_relaxed(const void *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(void*)*)obj, + memory_order_relaxed); +} + +static inline unsigned long long +_Py_atomic_load_ullong_relaxed(const unsigned long long *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(unsigned long long)*)obj, + memory_order_relaxed); +} + + +// --- _Py_atomic_store ------------------------------------------------------ + +static inline void +_Py_atomic_store_int(int *obj, int value) +{ + _Py_USING_STD; + atomic_store((_Atomic(int)*)obj, value); +} + +static inline void +_Py_atomic_store_int8(int8_t *obj, int8_t value) +{ + _Py_USING_STD; + atomic_store((_Atomic(int8_t)*)obj, value); +} + +static inline void +_Py_atomic_store_int16(int16_t *obj, int16_t value) +{ + _Py_USING_STD; + atomic_store((_Atomic(int16_t)*)obj, value); +} + +static inline void +_Py_atomic_store_int32(int32_t *obj, int32_t value) +{ + _Py_USING_STD; + atomic_store((_Atomic(int32_t)*)obj, value); +} + +static inline void +_Py_atomic_store_int64(int64_t *obj, int64_t value) +{ + _Py_USING_STD; + atomic_store((_Atomic(int64_t)*)obj, value); +} + +static inline void +_Py_atomic_store_intptr(intptr_t *obj, intptr_t value) +{ + _Py_USING_STD; + atomic_store((_Atomic(intptr_t)*)obj, value); +} + +static inline void +_Py_atomic_store_uint8(uint8_t *obj, uint8_t value) +{ + _Py_USING_STD; + atomic_store((_Atomic(uint8_t)*)obj, value); +} + +static inline void +_Py_atomic_store_uint16(uint16_t *obj, uint16_t value) +{ + _Py_USING_STD; + atomic_store((_Atomic(uint16_t)*)obj, value); +} + +static inline void +_Py_atomic_store_uint32(uint32_t *obj, uint32_t value) +{ + _Py_USING_STD; + atomic_store((_Atomic(uint32_t)*)obj, value); +} + +static inline void +_Py_atomic_store_uint64(uint64_t *obj, uint64_t value) +{ + _Py_USING_STD; + atomic_store((_Atomic(uint64_t)*)obj, value); +} + +static inline void +_Py_atomic_store_uintptr(uintptr_t *obj, uintptr_t value) +{ + _Py_USING_STD; + atomic_store((_Atomic(uintptr_t)*)obj, value); +} + +static inline void +_Py_atomic_store_uint(unsigned int *obj, unsigned int value) +{ + _Py_USING_STD; + atomic_store((_Atomic(unsigned int)*)obj, value); +} + +static inline void +_Py_atomic_store_ptr(void *obj, void *value) +{ + _Py_USING_STD; + atomic_store((_Atomic(void*)*)obj, value); +} + +static inline void +_Py_atomic_store_ssize(Py_ssize_t *obj, Py_ssize_t value) +{ + _Py_USING_STD; + atomic_store((_Atomic(Py_ssize_t)*)obj, value); +} + + +// --- _Py_atomic_store_relaxed ---------------------------------------------- + +static inline void +_Py_atomic_store_int_relaxed(int *obj, int value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(int)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_int8_relaxed(int8_t *obj, int8_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(int8_t)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_int16_relaxed(int16_t *obj, int16_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(int16_t)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_int32_relaxed(int32_t *obj, int32_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(int32_t)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_int64_relaxed(int64_t *obj, int64_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(int64_t)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_intptr_relaxed(intptr_t *obj, intptr_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(intptr_t)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_uint8_relaxed(uint8_t *obj, uint8_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(uint8_t)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_uint16_relaxed(uint16_t *obj, uint16_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(uint16_t)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_uint32_relaxed(uint32_t *obj, uint32_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(uint32_t)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_uint64_relaxed(uint64_t *obj, uint64_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(uint64_t)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_uintptr_relaxed(uintptr_t *obj, uintptr_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(uintptr_t)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_uint_relaxed(unsigned int *obj, unsigned int value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(unsigned int)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_ptr_relaxed(void *obj, void *value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(void*)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(Py_ssize_t)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_ullong_relaxed(unsigned long long *obj, + unsigned long long value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(unsigned long long)*)obj, value, + memory_order_relaxed); +} + + +// --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ + +static inline void * +_Py_atomic_load_ptr_acquire(const void *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(void*)*)obj, + memory_order_acquire); +} + +static inline uintptr_t +_Py_atomic_load_uintptr_acquire(const uintptr_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(uintptr_t)*)obj, + memory_order_acquire); +} + +static inline void +_Py_atomic_store_ptr_release(void *obj, void *value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(void*)*)obj, value, + memory_order_release); +} + +static inline void +_Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(uintptr_t)*)obj, value, + memory_order_release); +} + +static inline void +_Py_atomic_store_int_release(int *obj, int value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(int)*)obj, value, + memory_order_release); +} + +static inline void +_Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(Py_ssize_t)*)obj, value, + memory_order_release); +} + +static inline int +_Py_atomic_load_int_acquire(const int *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(int)*)obj, + memory_order_acquire); +} + +static inline void +_Py_atomic_store_uint32_release(uint32_t *obj, uint32_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(uint32_t)*)obj, value, + memory_order_release); +} + +static inline void +_Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(uint64_t)*)obj, value, + memory_order_release); +} + +static inline uint64_t +_Py_atomic_load_uint64_acquire(const uint64_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(uint64_t)*)obj, + memory_order_acquire); +} + +static inline uint32_t +_Py_atomic_load_uint32_acquire(const uint32_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(uint32_t)*)obj, + memory_order_acquire); +} + +static inline Py_ssize_t +_Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(Py_ssize_t)*)obj, + memory_order_acquire); +} + + +// --- _Py_atomic_fence ------------------------------------------------------ + + static inline void +_Py_atomic_fence_seq_cst(void) +{ + _Py_USING_STD; + atomic_thread_fence(memory_order_seq_cst); +} + + static inline void +_Py_atomic_fence_acquire(void) +{ + _Py_USING_STD; + atomic_thread_fence(memory_order_acquire); +} + + static inline void +_Py_atomic_fence_release(void) +{ + _Py_USING_STD; + atomic_thread_fence(memory_order_release); +} diff --git a/contrib/tools/python3/Include/cpython/pyerrors.h b/contrib/tools/python3/Include/cpython/pyerrors.h index 156665cbdb1..422391c3222 100644 --- a/contrib/tools/python3/Include/cpython/pyerrors.h +++ b/contrib/tools/python3/Include/cpython/pyerrors.h @@ -18,6 +18,7 @@ typedef struct { PyException_HEAD PyObject *msg; PyObject *excs; + PyObject *excs_str; } PyBaseExceptionGroupObject; typedef struct { @@ -88,42 +89,19 @@ typedef PyOSErrorObject PyEnvironmentErrorObject; typedef PyOSErrorObject PyWindowsErrorObject; #endif -/* Error handling definitions */ - -PyAPI_FUNC(void) _PyErr_SetKeyError(PyObject *); -PyAPI_FUNC(_PyErr_StackItem*) _PyErr_GetTopmostException(PyThreadState *tstate); -PyAPI_FUNC(PyObject*) _PyErr_GetHandledException(PyThreadState *); -PyAPI_FUNC(void) _PyErr_SetHandledException(PyThreadState *, PyObject *); -PyAPI_FUNC(void) _PyErr_GetExcInfo(PyThreadState *, PyObject **, PyObject **, PyObject **); - /* Context manipulation (PEP 3134) */ -Py_DEPRECATED(3.12) PyAPI_FUNC(void) _PyErr_ChainExceptions(PyObject *, PyObject *, PyObject *); PyAPI_FUNC(void) _PyErr_ChainExceptions1(PyObject *); -/* Like PyErr_Format(), but saves current exception as __context__ and - __cause__. - */ -PyAPI_FUNC(PyObject *) _PyErr_FormatFromCause( - PyObject *exception, - const char *format, /* ASCII-encoded string */ - ... - ); - /* In exceptions.c */ -PyAPI_FUNC(int) _PyException_AddNote( - PyObject *exc, - PyObject *note); - PyAPI_FUNC(PyObject*) PyUnstable_Exc_PrepReraiseStar( PyObject *orig, PyObject *excs); /* In signalmodule.c */ -int PySignal_SetWakeupFd(int fd); -PyAPI_FUNC(int) _PyErr_CheckSignals(void); +PyAPI_FUNC(int) PySignal_SetWakeupFd(int fd); /* Support for adding program text to SyntaxErrors */ @@ -143,36 +121,12 @@ PyAPI_FUNC(PyObject *) PyErr_ProgramTextObject( PyObject *filename, int lineno); -PyAPI_FUNC(PyObject *) _PyErr_ProgramDecodedTextObject( - PyObject *filename, - int lineno, - const char* encoding); - -PyAPI_FUNC(PyObject *) _PyUnicodeTranslateError_Create( - PyObject *object, - Py_ssize_t start, - Py_ssize_t end, - const char *reason /* UTF-8 encoded string */ - ); - -PyAPI_FUNC(void) _PyErr_WriteUnraisableMsg( - const char *err_msg, - PyObject *obj); - PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalErrorFunc( const char *func, const char *message); -PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalErrorFormat( - const char *func, - const char *format, - ...); - -extern PyObject *_PyErr_SetImportErrorWithNameFrom( - PyObject *, - PyObject *, - PyObject *, - PyObject *); +PyAPI_FUNC(void) PyErr_FormatUnraisable(const char *, ...); +PyAPI_DATA(PyObject *) PyExc_PythonFinalizationError; #define Py_FatalError(message) _Py_FatalErrorFunc(__func__, (message)) diff --git a/contrib/tools/python3/Include/cpython/pyframe.h b/contrib/tools/python3/Include/cpython/pyframe.h index 0e2afff925e..eeafbb17a56 100644 --- a/contrib/tools/python3/Include/cpython/pyframe.h +++ b/contrib/tools/python3/Include/cpython/pyframe.h @@ -3,8 +3,10 @@ #endif PyAPI_DATA(PyTypeObject) PyFrame_Type; +PyAPI_DATA(PyTypeObject) PyFrameLocalsProxy_Type; #define PyFrame_Check(op) Py_IS_TYPE((op), &PyFrame_Type) +#define PyFrameLocalsProxy_Check(op) Py_IS_TYPE((op), &PyFrameLocalsProxy_Type) PyAPI_FUNC(PyFrameObject *) PyFrame_GetBack(PyFrameObject *frame); PyAPI_FUNC(PyObject *) PyFrame_GetLocals(PyFrameObject *frame); @@ -33,3 +35,11 @@ PyAPI_FUNC(int) PyUnstable_InterpreterFrame_GetLasti(struct _PyInterpreterFrame /* Returns the currently executing line number, or -1 if there is no line number. * Does not raise an exception. */ PyAPI_FUNC(int) PyUnstable_InterpreterFrame_GetLine(struct _PyInterpreterFrame *frame); + +#define PyUnstable_EXECUTABLE_KIND_SKIP 0 +#define PyUnstable_EXECUTABLE_KIND_PY_FUNCTION 1 +#define PyUnstable_EXECUTABLE_KIND_BUILTIN_FUNCTION 3 +#define PyUnstable_EXECUTABLE_KIND_METHOD_DESCRIPTOR 4 +#define PyUnstable_EXECUTABLE_KINDS 5 + +PyAPI_DATA(const PyTypeObject *) const PyUnstable_ExecutableKinds[PyUnstable_EXECUTABLE_KINDS+1]; diff --git a/contrib/tools/python3/Include/cpython/pyhash.h b/contrib/tools/python3/Include/cpython/pyhash.h new file mode 100644 index 00000000000..825c034a8d8 --- /dev/null +++ b/contrib/tools/python3/Include/cpython/pyhash.h @@ -0,0 +1,47 @@ +#ifndef Py_CPYTHON_HASH_H +# error "this header file must not be included directly" +#endif + +/* Prime multiplier used in string and various other hashes. */ +#define PyHASH_MULTIPLIER 1000003UL /* 0xf4243 */ + +/* Parameters used for the numeric hash implementation. See notes for + _Py_HashDouble in Python/pyhash.c. Numeric hashes are based on + reduction modulo the prime 2**_PyHASH_BITS - 1. */ + +#if SIZEOF_VOID_P >= 8 +# define PyHASH_BITS 61 +#else +# define PyHASH_BITS 31 +#endif + +#define PyHASH_MODULUS (((size_t)1 << _PyHASH_BITS) - 1) +#define PyHASH_INF 314159 +#define PyHASH_IMAG PyHASH_MULTIPLIER + +/* Aliases kept for backward compatibility with Python 3.12 */ +#define _PyHASH_MULTIPLIER PyHASH_MULTIPLIER +#define _PyHASH_BITS PyHASH_BITS +#define _PyHASH_MODULUS PyHASH_MODULUS +#define _PyHASH_INF PyHASH_INF +#define _PyHASH_IMAG PyHASH_IMAG + +/* Helpers for hash functions */ +PyAPI_FUNC(Py_hash_t) _Py_HashDouble(PyObject *, double); + +// Kept for backward compatibility +#define _Py_HashPointer Py_HashPointer + + +/* hash function definition */ +typedef struct { + Py_hash_t (*const hash)(const void *, Py_ssize_t); + const char *name; + const int hash_bits; + const int seed_bits; +} PyHash_FuncDef; + +PyAPI_FUNC(PyHash_FuncDef*) PyHash_GetFuncDef(void); + +PyAPI_FUNC(Py_hash_t) Py_HashPointer(const void *ptr); +PyAPI_FUNC(Py_hash_t) PyObject_GenericHash(PyObject *); diff --git a/contrib/tools/python3/Include/cpython/pylifecycle.h b/contrib/tools/python3/Include/cpython/pylifecycle.h index 4daea33bf80..e46dfe59ec4 100644 --- a/contrib/tools/python3/Include/cpython/pylifecycle.h +++ b/contrib/tools/python3/Include/cpython/pylifecycle.h @@ -6,13 +6,6 @@ in all builds of Python */ PyAPI_FUNC(int) Py_FrozenMain(int argc, char **argv); -/* Only used by applications that embed the interpreter and need to - * override the standard encoding determination mechanism - */ -Py_DEPRECATED(3.11) PyAPI_FUNC(int) Py_SetStandardStreamEncoding( - const char *encoding, - const char *errors); - /* PEP 432 Multi-phase initialization API (Private while provisional!) */ PyAPI_FUNC(PyStatus) Py_PreInitialize( @@ -26,13 +19,13 @@ PyAPI_FUNC(PyStatus) Py_PreInitializeFromArgs( Py_ssize_t argc, wchar_t **argv); -PyAPI_FUNC(int) _Py_IsCoreInitialized(void); - /* Initialization and finalization */ PyAPI_FUNC(PyStatus) Py_InitializeFromConfig( const PyConfig *config); + +// Python 3.8 provisional API (PEP 587) PyAPI_FUNC(PyStatus) _Py_InitializeMain(void); PyAPI_FUNC(int) Py_RunMain(void); @@ -40,28 +33,7 @@ PyAPI_FUNC(int) Py_RunMain(void); PyAPI_FUNC(void) _Py_NO_RETURN Py_ExitStatusException(PyStatus err); -/* Restore signals that the interpreter has called SIG_IGN on to SIG_DFL. */ -PyAPI_FUNC(void) _Py_RestoreSignals(void); - PyAPI_FUNC(int) Py_FdIsInteractive(FILE *, const char *); -PyAPI_FUNC(int) _Py_FdIsInteractive(FILE *fp, PyObject *filename); - -Py_DEPRECATED(3.11) PyAPI_FUNC(void) _Py_SetProgramFullPath(const wchar_t *); - -PyAPI_FUNC(const char *) _Py_gitidentifier(void); -PyAPI_FUNC(const char *) _Py_gitversion(void); - -PyAPI_FUNC(int) _Py_IsFinalizing(void); -PyAPI_FUNC(int) _Py_IsInterpreterFinalizing(PyInterpreterState *interp); - -/* Random */ -PyAPI_FUNC(int) _PyOS_URandom(void *buffer, Py_ssize_t size); -PyAPI_FUNC(int) _PyOS_URandomNonblock(void *buffer, Py_ssize_t size); - -/* Legacy locale support */ -PyAPI_FUNC(int) _Py_CoerceLegacyLocale(int warn); -PyAPI_FUNC(int) _Py_LegacyLocaleDetected(int warn); -PyAPI_FUNC(char *) _Py_SetLocaleFromEnv(int category); /* --- PyInterpreterConfig ------------------------------------ */ @@ -91,6 +63,15 @@ typedef struct { .gil = PyInterpreterConfig_OWN_GIL, \ } +// gh-117649: The free-threaded build does not currently support single-phase +// init extensions in subinterpreters. For now, we ensure that +// `check_multi_interp_extensions` is always `1`, even in the legacy config. +#ifdef Py_GIL_DISABLED +# define _PyInterpreterConfig_LEGACY_CHECK_MULTI_INTERP_EXTENSIONS 1 +#else +# define _PyInterpreterConfig_LEGACY_CHECK_MULTI_INTERP_EXTENSIONS 0 +#endif + #define _PyInterpreterConfig_LEGACY_INIT \ { \ .use_main_obmalloc = 1, \ @@ -98,7 +79,7 @@ typedef struct { .allow_exec = 1, \ .allow_threads = 1, \ .allow_daemon_threads = 1, \ - .check_multi_interp_extensions = 0, \ + .check_multi_interp_extensions = _PyInterpreterConfig_LEGACY_CHECK_MULTI_INTERP_EXTENSIONS, \ .gil = PyInterpreterConfig_SHARED_GIL, \ } @@ -107,5 +88,5 @@ PyAPI_FUNC(PyStatus) Py_NewInterpreterFromConfig( const PyInterpreterConfig *config); typedef void (*atexit_datacallbackfunc)(void *); -PyAPI_FUNC(int) _Py_AtExit( +PyAPI_FUNC(int) PyUnstable_AtExit( PyInterpreterState *, atexit_datacallbackfunc, void *); diff --git a/contrib/tools/python3/Include/cpython/pymem.h b/contrib/tools/python3/Include/cpython/pymem.h index d1054d76520..76b3221f7b9 100644 --- a/contrib/tools/python3/Include/cpython/pymem.h +++ b/contrib/tools/python3/Include/cpython/pymem.h @@ -2,24 +2,6 @@ # error "this header file must not be included directly" #endif -PyAPI_FUNC(void *) PyMem_RawMalloc(size_t size); -PyAPI_FUNC(void *) PyMem_RawCalloc(size_t nelem, size_t elsize); -PyAPI_FUNC(void *) PyMem_RawRealloc(void *ptr, size_t new_size); -PyAPI_FUNC(void) PyMem_RawFree(void *ptr); - -/* Try to get the allocators name set by _PyMem_SetupAllocators(). */ -PyAPI_FUNC(const char*) _PyMem_GetCurrentAllocatorName(void); - -/* strdup() using PyMem_RawMalloc() */ -PyAPI_FUNC(char *) _PyMem_RawStrdup(const char *str); - -/* strdup() using PyMem_Malloc() */ -PyAPI_FUNC(char *) _PyMem_Strdup(const char *str); - -/* wcsdup() using PyMem_RawMalloc() */ -PyAPI_FUNC(wchar_t*) _PyMem_RawWcsdup(const wchar_t *str); - - typedef enum { /* PyMem_RawMalloc(), PyMem_RawRealloc() and PyMem_RawFree() */ PYMEM_DOMAIN_RAW, @@ -41,6 +23,10 @@ typedef enum { PYMEM_ALLOCATOR_PYMALLOC = 5, PYMEM_ALLOCATOR_PYMALLOC_DEBUG = 6, #endif +#ifdef WITH_MIMALLOC + PYMEM_ALLOCATOR_MIMALLOC = 7, + PYMEM_ALLOCATOR_MIMALLOC_DEBUG = 8, +#endif } PyMemAllocatorName; diff --git a/contrib/tools/python3/Include/cpython/pystate.h b/contrib/tools/python3/Include/cpython/pystate.h index 32789e17bda..04f002772c6 100644 --- a/contrib/tools/python3/Include/cpython/pystate.h +++ b/contrib/tools/python3/Include/cpython/pystate.h @@ -3,45 +3,12 @@ #endif -/* -Runtime Feature Flags - -Each flag indicate whether or not a specific runtime feature -is available in a given context. For example, forking the process -might not be allowed in the current interpreter (i.e. os.fork() would fail). -*/ - -/* Set if the interpreter share obmalloc runtime state - with the main interpreter. */ -#define Py_RTFLAGS_USE_MAIN_OBMALLOC (1UL << 5) - -/* Set if import should check a module for subinterpreter support. */ -#define Py_RTFLAGS_MULTI_INTERP_EXTENSIONS (1UL << 8) - -/* Set if threads are allowed. */ -#define Py_RTFLAGS_THREADS (1UL << 10) - -/* Set if daemon threads are allowed. */ -#define Py_RTFLAGS_DAEMON_THREADS (1UL << 11) - -/* Set if os.fork() is allowed. */ -#define Py_RTFLAGS_FORK (1UL << 15) - -/* Set if os.exec*() is allowed. */ -#define Py_RTFLAGS_EXEC (1UL << 16) - - -PyAPI_FUNC(int) _PyInterpreterState_HasFeature(PyInterpreterState *interp, - unsigned long feature); - - /* private interpreter helpers */ PyAPI_FUNC(int) _PyInterpreterState_RequiresIDRef(PyInterpreterState *); PyAPI_FUNC(void) _PyInterpreterState_RequireIDRef(PyInterpreterState *, int); -PyAPI_FUNC(PyObject *) _PyInterpreterState_GetMainModule(PyInterpreterState *); - +PyAPI_FUNC(PyObject *) PyUnstable_InterpreterState_GetMainModule(PyInterpreterState *); /* State unique per thread */ @@ -62,24 +29,6 @@ typedef int (*Py_tracefunc)(PyObject *, PyFrameObject *, int, PyObject *); #define PyTrace_C_RETURN 6 #define PyTrace_OPCODE 7 -// Internal structure: you should not use it directly, but use public functions -// like PyThreadState_EnterTracing() and PyThreadState_LeaveTracing(). -typedef struct _PyCFrame { - /* This struct will be threaded through the C stack - * allowing fast access to per-thread state that needs - * to be accessed quickly by the interpreter, but can - * be modified outside of the interpreter. - * - * WARNING: This makes data on the C stack accessible from - * heap objects. Care must be taken to maintain stack - * discipline and make sure that instances of this struct cannot - * accessed outside of their lifetime. - */ - /* Pointer to the currently executing frame (it can be NULL) */ - struct _PyInterpreterFrame *current_frame; - struct _PyCFrame *previous; -} _PyCFrame; - typedef struct _err_stackitem { /* This struct represents a single execution context where we might * be currently handling an exception. It is a per-coroutine state @@ -107,11 +56,6 @@ typedef struct _stack_chunk { PyObject * data[1]; /* Variable sized */ } _PyStackChunk; -struct _py_trashcan { - int delete_nesting; - PyObject *delete_later; -}; - struct _ts { /* See Python/ceval.c for comments explaining most fields */ @@ -119,6 +63,11 @@ struct _ts { PyThreadState *next; PyInterpreterState *interp; + /* The global instrumentation version in high bits, plus flags indicating + when to break out of the interpreter loop in lower bits. See details in + pycore_ceval.h. */ + uintptr_t eval_breaker; + struct { /* Has been initialized to a safe state. @@ -134,6 +83,8 @@ struct _ts { unsigned int bound_gilstate:1; /* Currently in use (maybe holds the GIL). */ unsigned int active:1; + /* Currently holds the GIL. */ + unsigned int holds_gil:1; /* various stages of finalization */ unsigned int finalizing:1; @@ -141,8 +92,22 @@ struct _ts { unsigned int finalized:1; /* padding to align to 4 bytes */ - unsigned int :24; + unsigned int :23; } _status; +#ifdef Py_BUILD_CORE +# define _PyThreadState_WHENCE_NOTSET -1 +# define _PyThreadState_WHENCE_UNKNOWN 0 +# define _PyThreadState_WHENCE_INIT 1 +# define _PyThreadState_WHENCE_FINI 2 +# define _PyThreadState_WHENCE_THREADING 3 +# define _PyThreadState_WHENCE_GILSTATE 4 +# define _PyThreadState_WHENCE_EXEC 5 +#endif + int _whence; + + /* Thread state (_Py_THREAD_ATTACHED, _Py_THREAD_DETACHED, _Py_THREAD_SUSPENDED). + See Include/internal/pycore_pystate.h for more details. */ + int state; int py_recursion_remaining; int py_recursion_limit; @@ -156,9 +121,8 @@ struct _ts { int tracing; int what_event; /* The event currently being monitored, if any. */ - /* Pointer to current _PyCFrame in the C stack frame of the currently, - * or most recently, executing _PyEval_EvalFrameDefault. */ - _PyCFrame *cframe; + /* Pointer to currently executing frame. */ + struct _PyInterpreterFrame *current_frame; Py_tracefunc c_profilefunc; Py_tracefunc c_tracefunc; @@ -186,33 +150,14 @@ struct _ts { */ unsigned long native_thread_id; - struct _py_trashcan trash; + PyObject *delete_later; - /* Called when a thread state is deleted normally, but not when it - * is destroyed after fork(). - * Pain: to prevent rare but fatal shutdown errors (issue 18808), - * Thread.join() must wait for the join'ed thread's tstate to be unlinked - * from the tstate chain. That happens at the end of a thread's life, - * in pystate.c. - * The obvious way doesn't quite work: create a lock which the tstate - * unlinking code releases, and have Thread.join() wait to acquire that - * lock. The problem is that we _are_ at the end of the thread's life: - * if the thread holds the last reference to the lock, decref'ing the - * lock will delete the lock, and that may trigger arbitrary Python code - * if there's a weakref, with a callback, to the lock. But by this time - * _PyRuntime.gilstate.tstate_current is already NULL, so only the simplest - * of C code can be allowed to run (in particular it must not be possible to - * release the GIL). - * So instead of holding the lock directly, the tstate holds a weakref to - * the lock: that's the value of on_delete_data below. Decref'ing a - * weakref is harmless. - * on_delete points to _threadmodule.c's static release_sentinel() function. - * After the tstate is unlinked, release_sentinel is called with the - * weakref-to-lock (on_delete_data) argument, and release_sentinel releases - * the indirectly held lock. + /* Tagged pointer to top-most critical section, or zero if there is no + * active critical section. Critical sections are only used in + * `--disable-gil` builds (i.e., when Py_GIL_DISABLED is defined to 1). In the + * default build, this field is always zero. */ - void (*on_delete)(void *); - void *on_delete_data; + uintptr_t critical_section; int coroutine_origin_tracking_depth; @@ -244,48 +189,55 @@ struct _ts { /* The thread's exception stack entry. (Always the last entry.) */ _PyErr_StackItem exc_state; - /* The bottom-most frame on the stack. */ - _PyCFrame root_cframe; + PyObject *previous_executor; + + uint64_t dict_global_version; + + /* Used to store/retrieve `threading.local` keys/values for this thread */ + PyObject *threading_local_key; + + /* Used by `threading.local`s to be remove keys/values for dying threads. + The PyThreadObject must hold the only reference to this value. + */ + PyObject *threading_local_sentinel; + + _PyStackChunk *datastack_cached_chunk; }; -/* WASI has limited call stack. Python's recursion limit depends on code - layout, optimization, and WASI runtime. Wasmtime can handle about 700 - recursions, sometimes less. 500 is a more conservative limit. */ #ifdef Py_DEBUG -# if defined(__wasi__) -# define C_RECURSION_LIMIT 150 -# else -# define C_RECURSION_LIMIT 1500 -# endif + // A debug build is likely built with low optimization level which implies + // higher stack memory usage than a release build: use a lower limit. +# define Py_C_RECURSION_LIMIT 1500 +#elif defined(__s390x__) +# define Py_C_RECURSION_LIMIT 800 +#elif defined(_WIN32) && defined(_M_ARM64) +# define Py_C_RECURSION_LIMIT 1000 +#elif defined(_WIN32) +# define Py_C_RECURSION_LIMIT 3000 +#elif defined(__ANDROID__) + // On an ARM64 emulator, API level 34 was OK with 10000, but API level 21 + // crashed in test_compiler_recursion_limit. +# define Py_C_RECURSION_LIMIT 3000 +#elif defined(_Py_ADDRESS_SANITIZER) +# define Py_C_RECURSION_LIMIT 4000 +#elif defined(__wasi__) + // Based on wasmtime 16. +# define Py_C_RECURSION_LIMIT 5000 #else -# if defined(__wasi__) -# define C_RECURSION_LIMIT 500 -# elif defined(__s390x__) -# define C_RECURSION_LIMIT 800 -# elif defined(_WIN32) -# define C_RECURSION_LIMIT 3000 -# elif defined(_Py_ADDRESS_SANITIZER) -# define C_RECURSION_LIMIT 4000 -# else - // This value is duplicated in Lib/test/support/__init__.py -# define C_RECURSION_LIMIT 10000 -# endif + // This value is duplicated in Lib/test/support/__init__.py +# define Py_C_RECURSION_LIMIT 10000 #endif -/* other API */ - -// Alias for backward compatibility with Python 3.8 -#define _PyInterpreterState_Get PyInterpreterState_Get -/* An alias for the internal _PyThreadState_New(), - kept for stable ABI compatibility. */ -PyAPI_FUNC(PyThreadState *) _PyThreadState_Prealloc(PyInterpreterState *); +/* other API */ /* Similar to PyThreadState_Get(), but don't issue a fatal error * if it is NULL. */ -PyAPI_FUNC(PyThreadState *) _PyThreadState_UncheckedGet(void); +PyAPI_FUNC(PyThreadState *) PyThreadState_GetUnchecked(void); + +// Alias kept for backward compatibility +#define _PyThreadState_UncheckedGet PyThreadState_GetUnchecked -PyAPI_FUNC(PyObject *) _PyThreadState_GetDict(PyThreadState *tstate); // Disable tracing and profiling. PyAPI_FUNC(void) PyThreadState_EnterTracing(PyThreadState *tstate); @@ -302,24 +254,10 @@ PyAPI_FUNC(void) PyThreadState_LeaveTracing(PyThreadState *tstate); The function returns 1 if _PyGILState_check_enabled is non-zero. */ PyAPI_FUNC(int) PyGILState_Check(void); -/* Get the single PyInterpreterState used by this process' GILState - implementation. - - This function doesn't check for error. Return NULL before _PyGILState_Init() - is called and after _PyGILState_Fini() is called. - - See also _PyInterpreterState_Get() and _PyInterpreterState_GET(). */ -PyAPI_FUNC(PyInterpreterState *) _PyGILState_GetInterpreterStateUnsafe(void); - /* The implementation of sys._current_frames() Returns a dict mapping thread id to that thread's current frame. */ -PyAPI_FUNC(PyObject *) _PyThread_CurrentFrames(void); - -/* The implementation of sys._current_exceptions() Returns a dict mapping - thread id to that thread's current exception. -*/ -PyAPI_FUNC(PyObject *) _PyThread_CurrentExceptions(void); +PyAPI_FUNC(PyObject*) _PyThread_CurrentFrames(void); /* Routines for advanced debuggers, requested by David Beazley. Don't use unless you know what you are doing! */ @@ -339,118 +277,3 @@ PyAPI_FUNC(_PyFrameEvalFunction) _PyInterpreterState_GetEvalFrameFunc( PyAPI_FUNC(void) _PyInterpreterState_SetEvalFrameFunc( PyInterpreterState *interp, _PyFrameEvalFunction eval_frame); - -PyAPI_FUNC(const PyConfig*) _PyInterpreterState_GetConfig(PyInterpreterState *interp); - -/* Get a copy of the current interpreter configuration. - - Return 0 on success. Raise an exception and return -1 on error. - - The caller must initialize 'config', using PyConfig_InitPythonConfig() - for example. - - Python must be preinitialized to call this method. - The caller must hold the GIL. - - Once done with the configuration, PyConfig_Clear() must be called to clear - it. */ -PyAPI_FUNC(int) _PyInterpreterState_GetConfigCopy( - struct PyConfig *config); - -/* Set the configuration of the current interpreter. - - This function should be called during or just after the Python - initialization. - - Update the sys module with the new configuration. If the sys module was - modified directly after the Python initialization, these changes are lost. - - Some configuration like faulthandler or warnoptions can be updated in the - configuration, but don't reconfigure Python (don't enable/disable - faulthandler and don't reconfigure warnings filters). - - Return 0 on success. Raise an exception and return -1 on error. - - The configuration should come from _PyInterpreterState_GetConfigCopy(). */ -PyAPI_FUNC(int) _PyInterpreterState_SetConfig( - const struct PyConfig *config); - -// Get the configuration of the current interpreter. -// The caller must hold the GIL. -PyAPI_FUNC(const PyConfig*) _Py_GetConfig(void); - - -/* cross-interpreter data */ - -// _PyCrossInterpreterData is similar to Py_buffer as an effectively -// opaque struct that holds data outside the object machinery. This -// is necessary to pass safely between interpreters in the same process. -typedef struct _xid _PyCrossInterpreterData; - -typedef PyObject *(*xid_newobjectfunc)(_PyCrossInterpreterData *); -typedef void (*xid_freefunc)(void *); - -struct _xid { - // data is the cross-interpreter-safe derivation of a Python object - // (see _PyObject_GetCrossInterpreterData). It will be NULL if the - // new_object func (below) encodes the data. - void *data; - // obj is the Python object from which the data was derived. This - // is non-NULL only if the data remains bound to the object in some - // way, such that the object must be "released" (via a decref) when - // the data is released. In that case the code that sets the field, - // likely a registered "crossinterpdatafunc", is responsible for - // ensuring it owns the reference (i.e. incref). - PyObject *obj; - // interp is the ID of the owning interpreter of the original - // object. It corresponds to the active interpreter when - // _PyObject_GetCrossInterpreterData() was called. This should only - // be set by the cross-interpreter machinery. - // - // We use the ID rather than the PyInterpreterState to avoid issues - // with deleted interpreters. Note that IDs are never re-used, so - // each one will always correspond to a specific interpreter - // (whether still alive or not). - int64_t interp; - // new_object is a function that returns a new object in the current - // interpreter given the data. The resulting object (a new - // reference) will be equivalent to the original object. This field - // is required. - xid_newobjectfunc new_object; - // free is called when the data is released. If it is NULL then - // nothing will be done to free the data. For some types this is - // okay (e.g. bytes) and for those types this field should be set - // to NULL. However, for most the data was allocated just for - // cross-interpreter use, so it must be freed when - // _PyCrossInterpreterData_Release is called or the memory will - // leak. In that case, at the very least this field should be set - // to PyMem_RawFree (the default if not explicitly set to NULL). - // The call will happen with the original interpreter activated. - xid_freefunc free; -}; - -PyAPI_FUNC(void) _PyCrossInterpreterData_Init( - _PyCrossInterpreterData *data, - PyInterpreterState *interp, void *shared, PyObject *obj, - xid_newobjectfunc new_object); -PyAPI_FUNC(int) _PyCrossInterpreterData_InitWithSize( - _PyCrossInterpreterData *, - PyInterpreterState *interp, const size_t, PyObject *, - xid_newobjectfunc); -PyAPI_FUNC(void) _PyCrossInterpreterData_Clear( - PyInterpreterState *, _PyCrossInterpreterData *); - -PyAPI_FUNC(int) _PyObject_GetCrossInterpreterData(PyObject *, _PyCrossInterpreterData *); -PyAPI_FUNC(PyObject *) _PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *); -PyAPI_FUNC(int) _PyCrossInterpreterData_Release(_PyCrossInterpreterData *); - -PyAPI_FUNC(int) _PyObject_CheckCrossInterpreterData(PyObject *); - -/* cross-interpreter data registry */ - -typedef int (*crossinterpdatafunc)(PyThreadState *tstate, PyObject *, - _PyCrossInterpreterData *); - -PyAPI_FUNC(int) _PyCrossInterpreterData_RegisterClass(PyTypeObject *, crossinterpdatafunc); -PyAPI_FUNC(int) _PyCrossInterpreterData_UnregisterClass(PyTypeObject *); -PyAPI_FUNC(crossinterpdatafunc) _PyCrossInterpreterData_Lookup(PyObject *); diff --git a/contrib/tools/python3/Include/cpython/pystats.h b/contrib/tools/python3/Include/cpython/pystats.h new file mode 100644 index 00000000000..378c2760ec3 --- /dev/null +++ b/contrib/tools/python3/Include/cpython/pystats.h @@ -0,0 +1,175 @@ +// Statistics on Python performance. +// +// API: +// +// - _Py_INCREF_STAT_INC() and _Py_DECREF_STAT_INC() used by Py_INCREF() +// and Py_DECREF(). +// - _Py_stats variable +// +// Functions of the sys module: +// +// - sys._stats_on() +// - sys._stats_off() +// - sys._stats_clear() +// - sys._stats_dump() +// +// Python must be built with ./configure --enable-pystats to define the +// Py_STATS macro. +// +// Define _PY_INTERPRETER macro to increment interpreter_increfs and +// interpreter_decrefs. Otherwise, increment increfs and decrefs. +// +// The number of incref operations counted by `incref` and +// `interpreter_incref` is the number of increment operations, which is +// not equal to the total of all reference counts. A single increment +// operation may increase the reference count of an object by more than +// one. For example, see `_Py_RefcntAdd`. + +#ifndef Py_CPYTHON_PYSTATS_H +# error "this header file must not be included directly" +#endif + +#define PYSTATS_MAX_UOP_ID 512 + +#define SPECIALIZATION_FAILURE_KINDS 36 + +/* Stats for determining who is calling PyEval_EvalFrame */ +#define EVAL_CALL_TOTAL 0 +#define EVAL_CALL_VECTOR 1 +#define EVAL_CALL_GENERATOR 2 +#define EVAL_CALL_LEGACY 3 +#define EVAL_CALL_FUNCTION_VECTORCALL 4 +#define EVAL_CALL_BUILD_CLASS 5 +#define EVAL_CALL_SLOT 6 +#define EVAL_CALL_FUNCTION_EX 7 +#define EVAL_CALL_API 8 +#define EVAL_CALL_METHOD 9 + +#define EVAL_CALL_KINDS 10 + +typedef struct _specialization_stats { + uint64_t success; + uint64_t failure; + uint64_t hit; + uint64_t deferred; + uint64_t miss; + uint64_t deopt; + uint64_t failure_kinds[SPECIALIZATION_FAILURE_KINDS]; +} SpecializationStats; + +typedef struct _opcode_stats { + SpecializationStats specialization; + uint64_t execution_count; + uint64_t pair_count[256]; +} OpcodeStats; + +typedef struct _call_stats { + uint64_t inlined_py_calls; + uint64_t pyeval_calls; + uint64_t frames_pushed; + uint64_t frame_objects_created; + uint64_t eval_calls[EVAL_CALL_KINDS]; +} CallStats; + +typedef struct _object_stats { + uint64_t increfs; + uint64_t decrefs; + uint64_t interpreter_increfs; + uint64_t interpreter_decrefs; + uint64_t allocations; + uint64_t allocations512; + uint64_t allocations4k; + uint64_t allocations_big; + uint64_t frees; + uint64_t to_freelist; + uint64_t from_freelist; + uint64_t inline_values; + uint64_t dict_materialized_on_request; + uint64_t dict_materialized_new_key; + uint64_t dict_materialized_too_big; + uint64_t dict_materialized_str_subclass; + uint64_t type_cache_hits; + uint64_t type_cache_misses; + uint64_t type_cache_dunder_hits; + uint64_t type_cache_dunder_misses; + uint64_t type_cache_collisions; + /* Temporary value used during GC */ + uint64_t object_visits; +} ObjectStats; + +typedef struct _gc_stats { + uint64_t collections; + uint64_t object_visits; + uint64_t objects_collected; +} GCStats; + +typedef struct _uop_stats { + uint64_t execution_count; + uint64_t miss; + uint64_t pair_count[PYSTATS_MAX_UOP_ID + 1]; +} UOpStats; + +#define _Py_UOP_HIST_SIZE 32 + +typedef struct _optimization_stats { + uint64_t attempts; + uint64_t traces_created; + uint64_t traces_executed; + uint64_t uops_executed; + uint64_t trace_stack_overflow; + uint64_t trace_stack_underflow; + uint64_t trace_too_long; + uint64_t trace_too_short; + uint64_t inner_loop; + uint64_t recursive_call; + uint64_t low_confidence; + uint64_t executors_invalidated; + UOpStats opcode[PYSTATS_MAX_UOP_ID + 1]; + uint64_t unsupported_opcode[256]; + uint64_t trace_length_hist[_Py_UOP_HIST_SIZE]; + uint64_t trace_run_length_hist[_Py_UOP_HIST_SIZE]; + uint64_t optimized_trace_length_hist[_Py_UOP_HIST_SIZE]; + uint64_t optimizer_attempts; + uint64_t optimizer_successes; + uint64_t optimizer_failure_reason_no_memory; + uint64_t remove_globals_builtins_changed; + uint64_t remove_globals_incorrect_keys; + uint64_t error_in_opcode[PYSTATS_MAX_UOP_ID + 1]; +} OptimizationStats; + +typedef struct _rare_event_stats { + /* Setting an object's class, obj.__class__ = ... */ + uint64_t set_class; + /* Setting the bases of a class, cls.__bases__ = ... */ + uint64_t set_bases; + /* Setting the PEP 523 frame eval function, _PyInterpreterState_SetFrameEvalFunc() */ + uint64_t set_eval_frame_func; + /* Modifying the builtins, __builtins__.__dict__[var] = ... */ + uint64_t builtin_dict; + /* Modifying a function, e.g. func.__defaults__ = ..., etc. */ + uint64_t func_modification; + /* Modifying a dict that is being watched */ + uint64_t watched_dict_modification; + uint64_t watched_globals_modification; +} RareEventStats; + +typedef struct _stats { + OpcodeStats opcode_stats[256]; + CallStats call_stats; + ObjectStats object_stats; + OptimizationStats optimization_stats; + RareEventStats rare_event_stats; + GCStats *gc_stats; +} PyStats; + + +// Export for shared extensions like 'math' +PyAPI_DATA(PyStats*) _Py_stats; + +#ifdef _PY_INTERPRETER +# define _Py_INCREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_increfs++; } while (0) +# define _Py_DECREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_decrefs++; } while (0) +#else +# define _Py_INCREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.increfs++; } while (0) +# define _Py_DECREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.decrefs++; } while (0) +#endif diff --git a/contrib/tools/python3/Include/cpython/pythonrun.h b/contrib/tools/python3/Include/cpython/pythonrun.h index fb617655374..edc40952254 100644 --- a/contrib/tools/python3/Include/cpython/pythonrun.h +++ b/contrib/tools/python3/Include/cpython/pythonrun.h @@ -3,21 +3,11 @@ #endif PyAPI_FUNC(int) PyRun_SimpleStringFlags(const char *, PyCompilerFlags *); -PyAPI_FUNC(int) _PyRun_SimpleFileObject( - FILE *fp, - PyObject *filename, - int closeit, - PyCompilerFlags *flags); PyAPI_FUNC(int) PyRun_AnyFileExFlags( FILE *fp, const char *filename, /* decoded from the filesystem encoding */ int closeit, PyCompilerFlags *flags); -PyAPI_FUNC(int) _PyRun_AnyFileObject( - FILE *fp, - PyObject *filename, - int closeit, - PyCompilerFlags *flags); PyAPI_FUNC(int) PyRun_SimpleFileExFlags( FILE *fp, const char *filename, /* decoded from the filesystem encoding */ @@ -35,10 +25,6 @@ PyAPI_FUNC(int) PyRun_InteractiveLoopFlags( FILE *fp, const char *filename, /* decoded from the filesystem encoding */ PyCompilerFlags *flags); -PyAPI_FUNC(int) _PyRun_InteractiveLoopObject( - FILE *fp, - PyObject *filename, - PyCompilerFlags *flags); PyAPI_FUNC(PyObject *) PyRun_StringFlags(const char *, int, PyObject *, @@ -69,15 +55,6 @@ PyAPI_FUNC(PyObject *) Py_CompileStringObject( #define Py_CompileString(str, p, s) Py_CompileStringExFlags((str), (p), (s), NULL, -1) #define Py_CompileStringFlags(str, p, s, f) Py_CompileStringExFlags((str), (p), (s), (f), -1) - -PyAPI_FUNC(const char *) _Py_SourceAsString( - PyObject *cmd, - const char *funcname, - const char *what, - PyCompilerFlags *cf, - PyObject **cmd_copy); - - /* A function flavor is also exported by libpython. It is required when libpython is accessed directly rather than using header files which defines macros below. On Windows, for example, PyAPI_FUNC() uses dllexport to @@ -114,8 +91,6 @@ PyAPI_FUNC(PyObject *) PyRun_FileFlags(FILE *fp, const char *p, int s, PyObject #define PyRun_FileFlags(fp, p, s, g, l, flags) \ PyRun_FileExFlags((fp), (p), (s), (g), (l), 0, (flags)) - /* Stuff with no proper home (yet) */ PyAPI_FUNC(char *) PyOS_Readline(FILE *, FILE *, const char *); -PyAPI_DATA(PyThreadState*) _PyOS_ReadlineTState; PyAPI_DATA(char) *(*PyOS_ReadlineFunctionPointer)(FILE *, FILE *, const char *); diff --git a/contrib/tools/python3/Include/cpython/pythread.h b/contrib/tools/python3/Include/cpython/pythread.h index 449b7cc6ddd..e658b35bd90 100644 --- a/contrib/tools/python3/Include/cpython/pythread.h +++ b/contrib/tools/python3/Include/cpython/pythread.h @@ -2,14 +2,15 @@ # error "this header file must not be included directly" #endif -#define PYTHREAD_INVALID_THREAD_ID ((unsigned long)-1) +// PY_TIMEOUT_MAX is the highest usable value (in microseconds) of PY_TIMEOUT_T +// type, and depends on the system threading API. +// +// NOTE: this isn't the same value as `_thread.TIMEOUT_MAX`. The _thread module +// exposes a higher-level API, with timeouts expressed in seconds and +// floating-point numbers allowed. +PyAPI_DATA(const long long) PY_TIMEOUT_MAX; -#ifdef HAVE_FORK -/* Private function to reinitialize a lock at fork in the child process. - Reset the lock to the unlocked state. - Return 0 on success, return -1 on error. */ -PyAPI_FUNC(int) _PyThread_at_fork_reinit(PyThread_type_lock *lock); -#endif /* HAVE_FORK */ +#define PYTHREAD_INVALID_THREAD_ID ((unsigned long)-1) #ifdef HAVE_PTHREAD_H /* Darwin needs pthread.h to know type name the pthread_key_t. */ @@ -21,7 +22,7 @@ PyAPI_FUNC(int) _PyThread_at_fork_reinit(PyThread_type_lock *lock); */ # define NATIVE_TSS_KEY_T unsigned long #elif defined(HAVE_PTHREAD_STUBS) -#error # include "cpython/pthread_stubs.h" +# include "pthread_stubs.h" # define NATIVE_TSS_KEY_T pthread_key_t #else # error "Require native threads. See https://bugs.python.org/issue31370" diff --git a/contrib/tools/python3/Include/cpython/pytime.h b/contrib/tools/python3/Include/cpython/pytime.h index 46cc97bd7cf..5c68110aeed 100644 --- a/contrib/tools/python3/Include/cpython/pytime.h +++ b/contrib/tools/python3/Include/cpython/pytime.h @@ -1,331 +1,27 @@ -// The _PyTime_t API is written to use timestamp and timeout values stored in -// various formats and to read clocks. -// -// The _PyTime_t type is an integer to support directly common arithmetic -// operations like t1 + t2. -// -// The _PyTime_t API supports a resolution of 1 nanosecond. The _PyTime_t type -// is signed to support negative timestamps. The supported range is around -// [-292.3 years; +292.3 years]. Using the Unix epoch (January 1st, 1970), the -// supported date range is around [1677-09-21; 2262-04-11]. -// -// Formats: -// -// * seconds -// * seconds as a floating pointer number (C double) -// * milliseconds (10^-3 seconds) -// * microseconds (10^-6 seconds) -// * 100 nanoseconds (10^-7 seconds) -// * nanoseconds (10^-9 seconds) -// * timeval structure, 1 microsecond resolution (10^-6 seconds) -// * timespec structure, 1 nanosecond resolution (10^-9 seconds) -// -// Integer overflows are detected and raise OverflowError. Conversion to a -// resolution worse than 1 nanosecond is rounded correctly with the requested -// rounding mode. There are 4 rounding modes: floor (towards -inf), ceiling -// (towards +inf), half even and up (away from zero). -// -// Some functions clamp the result in the range [_PyTime_MIN; _PyTime_MAX], so -// the caller doesn't have to handle errors and doesn't need to hold the GIL. -// For example, _PyTime_Add(t1, t2) computes t1+t2 and clamp the result on -// overflow. -// -// Clocks: -// -// * System clock -// * Monotonic clock -// * Performance counter -// -// Operations like (t * k / q) with integers are implemented in a way to reduce -// the risk of integer overflow. Such operation is used to convert a clock -// value expressed in ticks with a frequency to _PyTime_t, like -// QueryPerformanceCounter() with QueryPerformanceFrequency(). +// PyTime_t C API: see Doc/c-api/time.rst for the documentation. #ifndef Py_LIMITED_API #ifndef Py_PYTIME_H #define Py_PYTIME_H - -/************************************************************************** -Symbols and macros to supply platform-independent interfaces to time related -functions and constants -**************************************************************************/ #ifdef __cplusplus extern "C" { #endif -#if defined(__clang__) || defined(_MSC_VER) -struct timeval; -#endif - -/* _PyTime_t: Python timestamp with subsecond precision. It can be used to - store a duration, and so indirectly a date (related to another date, like - UNIX epoch). */ -typedef int64_t _PyTime_t; -// _PyTime_MIN nanoseconds is around -292.3 years -#define _PyTime_MIN INT64_MIN -// _PyTime_MAX nanoseconds is around +292.3 years -#define _PyTime_MAX INT64_MAX -#define _SIZEOF_PYTIME_T 8 - -typedef enum { - /* Round towards minus infinity (-inf). - For example, used to read a clock. */ - _PyTime_ROUND_FLOOR=0, - /* Round towards infinity (+inf). - For example, used for timeout to wait "at least" N seconds. */ - _PyTime_ROUND_CEILING=1, - /* Round to nearest with ties going to nearest even integer. - For example, used to round from a Python float. */ - _PyTime_ROUND_HALF_EVEN=2, - /* Round away from zero - For example, used for timeout. _PyTime_ROUND_CEILING rounds - -1e-9 to 0 milliseconds which causes bpo-31786 issue. - _PyTime_ROUND_UP rounds -1e-9 to -1 millisecond which keeps - the timeout sign as expected. select.poll(timeout) must block - for negative values." */ - _PyTime_ROUND_UP=3, - /* _PyTime_ROUND_TIMEOUT (an alias for _PyTime_ROUND_UP) should be - used for timeouts. */ - _PyTime_ROUND_TIMEOUT = _PyTime_ROUND_UP -} _PyTime_round_t; - - -/* Convert a time_t to a PyLong. */ -PyAPI_FUNC(PyObject *) _PyLong_FromTime_t( - time_t sec); - -/* Convert a PyLong to a time_t. */ -PyAPI_FUNC(time_t) _PyLong_AsTime_t( - PyObject *obj); - -/* Convert a number of seconds, int or float, to time_t. */ -PyAPI_FUNC(int) _PyTime_ObjectToTime_t( - PyObject *obj, - time_t *sec, - _PyTime_round_t); - -/* Convert a number of seconds, int or float, to a timeval structure. - usec is in the range [0; 999999] and rounded towards zero. - For example, -1.2 is converted to (-2, 800000). */ -PyAPI_FUNC(int) _PyTime_ObjectToTimeval( - PyObject *obj, - time_t *sec, - long *usec, - _PyTime_round_t); - -/* Convert a number of seconds, int or float, to a timespec structure. - nsec is in the range [0; 999999999] and rounded towards zero. - For example, -1.2 is converted to (-2, 800000000). */ -PyAPI_FUNC(int) _PyTime_ObjectToTimespec( - PyObject *obj, - time_t *sec, - long *nsec, - _PyTime_round_t); - - -/* Create a timestamp from a number of seconds. */ -PyAPI_FUNC(_PyTime_t) _PyTime_FromSeconds(int seconds); - -/* Macro to create a timestamp from a number of seconds, no integer overflow. - Only use the macro for small values, prefer _PyTime_FromSeconds(). */ -#define _PYTIME_FROMSECONDS(seconds) \ - ((_PyTime_t)(seconds) * (1000 * 1000 * 1000)) - -/* Create a timestamp from a number of nanoseconds. */ -PyAPI_FUNC(_PyTime_t) _PyTime_FromNanoseconds(_PyTime_t ns); - -/* Create a timestamp from a number of microseconds. - * Clamp to [_PyTime_MIN; _PyTime_MAX] on overflow. */ -PyAPI_FUNC(_PyTime_t) _PyTime_FromMicrosecondsClamp(_PyTime_t us); - -/* Create a timestamp from nanoseconds (Python int). */ -PyAPI_FUNC(int) _PyTime_FromNanosecondsObject(_PyTime_t *t, - PyObject *obj); - -/* Convert a number of seconds (Python float or int) to a timestamp. - Raise an exception and return -1 on error, return 0 on success. */ -PyAPI_FUNC(int) _PyTime_FromSecondsObject(_PyTime_t *t, - PyObject *obj, - _PyTime_round_t round); - -/* Convert a number of milliseconds (Python float or int, 10^-3) to a timestamp. - Raise an exception and return -1 on error, return 0 on success. */ -PyAPI_FUNC(int) _PyTime_FromMillisecondsObject(_PyTime_t *t, - PyObject *obj, - _PyTime_round_t round); - -/* Convert a timestamp to a number of seconds as a C double. */ -PyAPI_FUNC(double) _PyTime_AsSecondsDouble(_PyTime_t t); - -/* Convert timestamp to a number of milliseconds (10^-3 seconds). */ -PyAPI_FUNC(_PyTime_t) _PyTime_AsMilliseconds(_PyTime_t t, - _PyTime_round_t round); - -/* Convert timestamp to a number of microseconds (10^-6 seconds). */ -PyAPI_FUNC(_PyTime_t) _PyTime_AsMicroseconds(_PyTime_t t, - _PyTime_round_t round); - -/* Convert timestamp to a number of nanoseconds (10^-9 seconds). */ -PyAPI_FUNC(_PyTime_t) _PyTime_AsNanoseconds(_PyTime_t t); - -#ifdef MS_WINDOWS -// Convert timestamp to a number of 100 nanoseconds (10^-7 seconds). -PyAPI_FUNC(_PyTime_t) _PyTime_As100Nanoseconds(_PyTime_t t, - _PyTime_round_t round); -#endif - -/* Convert timestamp to a number of nanoseconds (10^-9 seconds) as a Python int - object. */ -PyAPI_FUNC(PyObject *) _PyTime_AsNanosecondsObject(_PyTime_t t); - -#ifndef MS_WINDOWS -/* Create a timestamp from a timeval structure. - Raise an exception and return -1 on overflow, return 0 on success. */ -PyAPI_FUNC(int) _PyTime_FromTimeval(_PyTime_t *tp, struct timeval *tv); -#endif - -/* Convert a timestamp to a timeval structure (microsecond resolution). - tv_usec is always positive. - Raise an exception and return -1 if the conversion overflowed, - return 0 on success. */ -PyAPI_FUNC(int) _PyTime_AsTimeval(_PyTime_t t, - struct timeval *tv, - _PyTime_round_t round); - -/* Similar to _PyTime_AsTimeval() but don't raise an exception on overflow. - On overflow, clamp tv_sec to _PyTime_t min/max. */ -PyAPI_FUNC(void) _PyTime_AsTimeval_clamp(_PyTime_t t, - struct timeval *tv, - _PyTime_round_t round); - -/* Convert a timestamp to a number of seconds (secs) and microseconds (us). - us is always positive. This function is similar to _PyTime_AsTimeval() - except that secs is always a time_t type, whereas the timeval structure - uses a C long for tv_sec on Windows. - Raise an exception and return -1 if the conversion overflowed, - return 0 on success. */ -PyAPI_FUNC(int) _PyTime_AsTimevalTime_t( - _PyTime_t t, - time_t *secs, - int *us, - _PyTime_round_t round); +typedef int64_t PyTime_t; +#define PyTime_MIN INT64_MIN +#define PyTime_MAX INT64_MAX -#if defined(HAVE_CLOCK_GETTIME) || defined(HAVE_KQUEUE) -/* Create a timestamp from a timespec structure. - Raise an exception and return -1 on overflow, return 0 on success. */ -PyAPI_FUNC(int) _PyTime_FromTimespec(_PyTime_t *tp, struct timespec *ts); +PyAPI_FUNC(double) PyTime_AsSecondsDouble(PyTime_t t); +PyAPI_FUNC(int) PyTime_Monotonic(PyTime_t *result); +PyAPI_FUNC(int) PyTime_PerfCounter(PyTime_t *result); +PyAPI_FUNC(int) PyTime_Time(PyTime_t *result); -/* Convert a timestamp to a timespec structure (nanosecond resolution). - tv_nsec is always positive. - Raise an exception and return -1 on error, return 0 on success. */ -PyAPI_FUNC(int) _PyTime_AsTimespec(_PyTime_t t, struct timespec *ts); - -/* Similar to _PyTime_AsTimespec() but don't raise an exception on overflow. - On overflow, clamp tv_sec to _PyTime_t min/max. */ -PyAPI_FUNC(void) _PyTime_AsTimespec_clamp(_PyTime_t t, struct timespec *ts); -#endif - - -// Compute t1 + t2. Clamp to [_PyTime_MIN; _PyTime_MAX] on overflow. -PyAPI_FUNC(_PyTime_t) _PyTime_Add(_PyTime_t t1, _PyTime_t t2); - -/* Compute ticks * mul / div. - Clamp to [_PyTime_MIN; _PyTime_MAX] on overflow. - The caller must ensure that ((div - 1) * mul) cannot overflow. */ -PyAPI_FUNC(_PyTime_t) _PyTime_MulDiv(_PyTime_t ticks, - _PyTime_t mul, - _PyTime_t div); - -/* Structure used by time.get_clock_info() */ -typedef struct { - const char *implementation; - int monotonic; - int adjustable; - double resolution; -} _Py_clock_info_t; - -/* Get the current time from the system clock. - - If the internal clock fails, silently ignore the error and return 0. - On integer overflow, silently ignore the overflow and clamp the clock to - [_PyTime_MIN; _PyTime_MAX]. - - Use _PyTime_GetSystemClockWithInfo() to check for failure. */ -PyAPI_FUNC(_PyTime_t) _PyTime_GetSystemClock(void); - -/* Get the current time from the system clock. - * On success, set *t and *info (if not NULL), and return 0. - * On error, raise an exception and return -1. - */ -PyAPI_FUNC(int) _PyTime_GetSystemClockWithInfo( - _PyTime_t *t, - _Py_clock_info_t *info); - -/* Get the time of a monotonic clock, i.e. a clock that cannot go backwards. - The clock is not affected by system clock updates. The reference point of - the returned value is undefined, so that only the difference between the - results of consecutive calls is valid. - - If the internal clock fails, silently ignore the error and return 0. - On integer overflow, silently ignore the overflow and clamp the clock to - [_PyTime_MIN; _PyTime_MAX]. - - Use _PyTime_GetMonotonicClockWithInfo() to check for failure. */ -PyAPI_FUNC(_PyTime_t) _PyTime_GetMonotonicClock(void); - -/* Get the time of a monotonic clock, i.e. a clock that cannot go backwards. - The clock is not affected by system clock updates. The reference point of - the returned value is undefined, so that only the difference between the - results of consecutive calls is valid. - - Fill info (if set) with information of the function used to get the time. - - Return 0 on success, raise an exception and return -1 on error. */ -PyAPI_FUNC(int) _PyTime_GetMonotonicClockWithInfo( - _PyTime_t *t, - _Py_clock_info_t *info); - - -/* Converts a timestamp to the Gregorian time, using the local time zone. - Return 0 on success, raise an exception and return -1 on error. */ -PyAPI_FUNC(int) _PyTime_localtime(time_t t, struct tm *tm); - -/* Converts a timestamp to the Gregorian time, assuming UTC. - Return 0 on success, raise an exception and return -1 on error. */ -PyAPI_FUNC(int) _PyTime_gmtime(time_t t, struct tm *tm); - -/* Get the performance counter: clock with the highest available resolution to - measure a short duration. - - If the internal clock fails, silently ignore the error and return 0. - On integer overflow, silently ignore the overflow and clamp the clock to - [_PyTime_MIN; _PyTime_MAX]. - - Use _PyTime_GetPerfCounterWithInfo() to check for failure. */ -PyAPI_FUNC(_PyTime_t) _PyTime_GetPerfCounter(void); - -/* Get the performance counter: clock with the highest available resolution to - measure a short duration. - - Fill info (if set) with information of the function used to get the time. - - Return 0 on success, raise an exception and return -1 on error. */ -PyAPI_FUNC(int) _PyTime_GetPerfCounterWithInfo( - _PyTime_t *t, - _Py_clock_info_t *info); - - -// Create a deadline. -// Pseudo code: _PyTime_GetMonotonicClock() + timeout. -PyAPI_FUNC(_PyTime_t) _PyDeadline_Init(_PyTime_t timeout); - -// Get remaining time from a deadline. -// Pseudo code: deadline - _PyTime_GetMonotonicClock(). -PyAPI_FUNC(_PyTime_t) _PyDeadline_Get(_PyTime_t deadline); +PyAPI_FUNC(int) PyTime_MonotonicRaw(PyTime_t *result); +PyAPI_FUNC(int) PyTime_PerfCounterRaw(PyTime_t *result); +PyAPI_FUNC(int) PyTime_TimeRaw(PyTime_t *result); #ifdef __cplusplus } #endif - #endif /* Py_PYTIME_H */ #endif /* Py_LIMITED_API */ diff --git a/contrib/tools/python3/Include/cpython/setobject.h b/contrib/tools/python3/Include/cpython/setobject.h index 20fd63eaae5..89565cb2921 100644 --- a/contrib/tools/python3/Include/cpython/setobject.h +++ b/contrib/tools/python3/Include/cpython/setobject.h @@ -62,11 +62,10 @@ typedef struct { (assert(PyAnySet_Check(so)), _Py_CAST(PySetObject*, so)) static inline Py_ssize_t PySet_GET_SIZE(PyObject *so) { +#ifdef Py_GIL_DISABLED + return _Py_atomic_load_ssize_relaxed(&(_PySet_CAST(so)->used)); +#else return _PySet_CAST(so)->used; +#endif } #define PySet_GET_SIZE(so) PySet_GET_SIZE(_PyObject_CAST(so)) - -PyAPI_DATA(PyObject *) _PySet_Dummy; - -PyAPI_FUNC(int) _PySet_NextEntry(PyObject *set, Py_ssize_t *pos, PyObject **key, Py_hash_t *hash); -PyAPI_FUNC(int) _PySet_Update(PyObject *set, PyObject *iterable); diff --git a/contrib/tools/python3/Include/cpython/sysmodule.h b/contrib/tools/python3/Include/cpython/sysmodule.h index 19d9dddc344..a3ac07f538a 100644 --- a/contrib/tools/python3/Include/cpython/sysmodule.h +++ b/contrib/tools/python3/Include/cpython/sysmodule.h @@ -2,15 +2,21 @@ # error "this header file must not be included directly" #endif -PyAPI_FUNC(PyObject *) _PySys_GetAttr(PyThreadState *tstate, - PyObject *name); - -PyAPI_FUNC(size_t) _PySys_GetSizeOf(PyObject *); - typedef int(*Py_AuditHookFunction)(const char *, PyObject *, void *); -PyAPI_FUNC(int) PySys_Audit( - const char *event, - const char *argFormat, - ...); PyAPI_FUNC(int) PySys_AddAuditHook(Py_AuditHookFunction, void*); + +typedef struct { + FILE* perf_map; + PyThread_type_lock map_lock; +} PerfMapState; + +PyAPI_FUNC(int) PyUnstable_PerfMapState_Init(void); +PyAPI_FUNC(int) PyUnstable_WritePerfMapEntry( + const void *code_addr, + unsigned int code_size, + const char *entry_name); +PyAPI_FUNC(void) PyUnstable_PerfMapState_Fini(void); +PyAPI_FUNC(int) PyUnstable_CopyPerfMapFile(const char* parent_filename); +PyAPI_FUNC(int) PyUnstable_PerfTrampoline_CompileCode(PyCodeObject *); +PyAPI_FUNC(int) PyUnstable_PerfTrampoline_SetPersistAfterFork(int enable); diff --git a/contrib/tools/python3/Include/cpython/traceback.h b/contrib/tools/python3/Include/cpython/traceback.h index a4e087b2b4e..81c51944f13 100644 --- a/contrib/tools/python3/Include/cpython/traceback.h +++ b/contrib/tools/python3/Include/cpython/traceback.h @@ -11,6 +11,3 @@ struct _traceback { int tb_lasti; int tb_lineno; }; - -PyAPI_FUNC(int) _Py_DisplaySourceLine(PyObject *, PyObject *, int, int, int *, PyObject **); -PyAPI_FUNC(void) _PyTraceback_Add(const char *, const char *, int); diff --git a/contrib/tools/python3/Include/cpython/tracemalloc.h b/contrib/tools/python3/Include/cpython/tracemalloc.h new file mode 100644 index 00000000000..6d094291ae2 --- /dev/null +++ b/contrib/tools/python3/Include/cpython/tracemalloc.h @@ -0,0 +1,32 @@ +#ifndef Py_LIMITED_API +#ifndef Py_TRACEMALLOC_H +#define Py_TRACEMALLOC_H +#ifdef __cplusplus +extern "C" { +#endif + +/* Track an allocated memory block in the tracemalloc module. + Return 0 on success, return -1 on error (failed to allocate memory to store + the trace). + + Return -2 if tracemalloc is disabled. + + If memory block is already tracked, update the existing trace. */ +PyAPI_FUNC(int) PyTraceMalloc_Track( + unsigned int domain, + uintptr_t ptr, + size_t size); + +/* Untrack an allocated memory block in the tracemalloc module. + Do nothing if the block was not tracked. + + Return -2 if tracemalloc is disabled, otherwise return 0. */ +PyAPI_FUNC(int) PyTraceMalloc_Untrack( + unsigned int domain, + uintptr_t ptr); + +#ifdef __cplusplus +} +#endif +#endif // !Py_TRACEMALLOC_H +#endif // !Py_LIMITED_API diff --git a/contrib/tools/python3/Include/cpython/tupleobject.h b/contrib/tools/python3/Include/cpython/tupleobject.h index f6a1f076e03..e530c8beda4 100644 --- a/contrib/tools/python3/Include/cpython/tupleobject.h +++ b/contrib/tools/python3/Include/cpython/tupleobject.h @@ -11,7 +11,6 @@ typedef struct { } PyTupleObject; PyAPI_FUNC(int) _PyTuple_Resize(PyObject **, Py_ssize_t); -PyAPI_FUNC(void) _PyTuple_MaybeUntrack(PyObject *); /* Cast argument to PyTupleObject* type. */ #define _PyTuple_CAST(op) \ @@ -31,9 +30,9 @@ static inline Py_ssize_t PyTuple_GET_SIZE(PyObject *op) { static inline void PyTuple_SET_ITEM(PyObject *op, Py_ssize_t index, PyObject *value) { PyTupleObject *tuple = _PyTuple_CAST(op); + assert(0 <= index); + assert(index < Py_SIZE(tuple)); tuple->ob_item[index] = value; } #define PyTuple_SET_ITEM(op, index, value) \ PyTuple_SET_ITEM(_PyObject_CAST(op), (index), _PyObject_CAST(value)) - -PyAPI_FUNC(void) _PyTuple_DebugMallocStats(FILE *out); diff --git a/contrib/tools/python3/Include/cpython/unicodeobject.h b/contrib/tools/python3/Include/cpython/unicodeobject.h index f177cd9e2af..d9b54bce832 100644 --- a/contrib/tools/python3/Include/cpython/unicodeobject.h +++ b/contrib/tools/python3/Include/cpython/unicodeobject.h @@ -6,8 +6,9 @@ Python and represents a single Unicode element in the Unicode type. With PEP 393, Py_UNICODE is deprecated and replaced with a typedef to wchar_t. */ -#define PY_UNICODE_TYPE wchar_t -/* Py_DEPRECATED(3.3) */ typedef wchar_t Py_UNICODE; +Py_DEPRECATED(3.13) typedef wchar_t PY_UNICODE_TYPE; +Py_DEPRECATED(3.13) typedef wchar_t Py_UNICODE; + /* --- Internal Unicode Operations ---------------------------------------- */ @@ -43,6 +44,7 @@ static inline Py_UCS4 Py_UNICODE_LOW_SURROGATE(Py_UCS4 ch) { return (0xDC00 + (ch & 0x3FF)); } + /* --- Unicode Type ------------------------------------------------------- */ /* ASCII-only strings created through PyUnicode_New use the PyASCIIObject @@ -169,10 +171,6 @@ typedef struct { } data; /* Canonical, smallest-form Unicode buffer */ } PyUnicodeObject; -PyAPI_FUNC(int) _PyUnicode_CheckConsistency( - PyObject *op, - int check_content); - #define _PyASCIIObject_CAST(op) \ (assert(PyUnicode_Check(op)), \ @@ -379,9 +377,8 @@ static inline Py_UCS4 PyUnicode_MAX_CHAR_VALUE(PyObject *op) #define PyUnicode_MAX_CHAR_VALUE(op) \ PyUnicode_MAX_CHAR_VALUE(_PyObject_CAST(op)) -/* === Public API ========================================================= */ -/* --- Plain Py_UNICODE --------------------------------------------------- */ +/* === Public API ========================================================= */ /* With PEP 393, this is the recommended way to allocate a new unicode object. This function will allocate the object and its buffer in a single memory @@ -398,11 +395,6 @@ static inline int PyUnicode_READY(PyObject* Py_UNUSED(op)) } #define PyUnicode_READY(op) PyUnicode_READY(_PyObject_CAST(op)) -/* Get a copy of a Unicode string. */ -PyAPI_FUNC(PyObject*) _PyUnicode_Copy( - PyObject *unicode - ); - /* Copy character from one unicode object into another, this function performs character conversion when necessary and falls back to memcpy() if possible. @@ -429,17 +421,6 @@ PyAPI_FUNC(Py_ssize_t) PyUnicode_CopyCharacters( Py_ssize_t how_many ); -/* Unsafe version of PyUnicode_CopyCharacters(): don't check arguments and so - may crash if parameters are invalid (e.g. if the output string - is too short). */ -PyAPI_FUNC(void) _PyUnicode_FastCopyCharacters( - PyObject *to, - Py_ssize_t to_start, - PyObject *from, - Py_ssize_t from_start, - Py_ssize_t how_many - ); - /* Fill a string with a character: write fill_char into unicode[start:start+length]. @@ -455,15 +436,6 @@ PyAPI_FUNC(Py_ssize_t) PyUnicode_Fill( Py_UCS4 fill_char ); -/* Unsafe version of PyUnicode_Fill(): don't check arguments and so may crash - if parameters are invalid (e.g. if length is longer than the string). */ -PyAPI_FUNC(void) _PyUnicode_FastFill( - PyObject *unicode, - Py_ssize_t start, - Py_ssize_t length, - Py_UCS4 fill_char - ); - /* Create a new string from a buffer of Py_UCS1, Py_UCS2 or Py_UCS4 characters. Scan the string to find the maximum character. */ PyAPI_FUNC(PyObject*) PyUnicode_FromKindAndData( @@ -471,18 +443,6 @@ PyAPI_FUNC(PyObject*) PyUnicode_FromKindAndData( const void *buffer, Py_ssize_t size); -/* Create a new string from a buffer of ASCII characters. - WARNING: Don't check if the string contains any non-ASCII character. */ -PyAPI_FUNC(PyObject*) _PyUnicode_FromASCII( - const char *buffer, - Py_ssize_t size); - -/* Compute the maximum character of the substring unicode[start:end]. - Return 127 for an empty string. */ -PyAPI_FUNC(Py_UCS4) _PyUnicode_FindMaxChar ( - PyObject *unicode, - Py_ssize_t start, - Py_ssize_t end); /* --- _PyUnicodeWriter API ----------------------------------------------- */ @@ -508,11 +468,11 @@ typedef struct { unsigned char readonly; } _PyUnicodeWriter ; -/* Initialize a Unicode writer. - * - * By default, the minimum buffer size is 0 character and overallocation is - * disabled. Set min_length, min_char and overallocate attributes to control - * the allocation of the buffer. */ +// Initialize a Unicode writer. +// +// By default, the minimum buffer size is 0 character and overallocation is +// disabled. Set min_length, min_char and overallocate attributes to control +// the allocation of the buffer. PyAPI_FUNC(void) _PyUnicodeWriter_Init(_PyUnicodeWriter *writer); @@ -600,15 +560,6 @@ PyAPI_FUNC(void) _PyUnicodeWriter_Dealloc(_PyUnicodeWriter *writer); -/* Format the object based on the format_spec, as defined in PEP 3101 - (Advanced String Formatting). */ -PyAPI_FUNC(int) _PyUnicode_FormatAdvancedWriter( - _PyUnicodeWriter *writer, - PyObject *obj, - PyObject *format_spec, - Py_ssize_t start, - Py_ssize_t end); - /* --- Manage the default encoding ---------------------------------------- */ /* Returns a pointer to the default encoding (UTF-8) of the @@ -626,171 +577,9 @@ PyAPI_FUNC(int) _PyUnicode_FormatAdvancedWriter( PyAPI_FUNC(const char *) PyUnicode_AsUTF8(PyObject *unicode); +// Alias kept for backward compatibility #define _PyUnicode_AsString PyUnicode_AsUTF8 -/* --- UTF-7 Codecs ------------------------------------------------------- */ - -PyAPI_FUNC(PyObject*) _PyUnicode_EncodeUTF7( - PyObject *unicode, /* Unicode object */ - int base64SetO, /* Encode RFC2152 Set O characters in base64 */ - int base64WhiteSpace, /* Encode whitespace (sp, ht, nl, cr) in base64 */ - const char *errors /* error handling */ - ); - -/* --- UTF-8 Codecs ------------------------------------------------------- */ - -PyAPI_FUNC(PyObject*) _PyUnicode_AsUTF8String( - PyObject *unicode, - const char *errors); - -/* --- UTF-32 Codecs ------------------------------------------------------ */ - -PyAPI_FUNC(PyObject*) _PyUnicode_EncodeUTF32( - PyObject *object, /* Unicode object */ - const char *errors, /* error handling */ - int byteorder /* byteorder to use 0=BOM+native;-1=LE,1=BE */ - ); - -/* --- UTF-16 Codecs ------------------------------------------------------ */ - -/* Returns a Python string object holding the UTF-16 encoded value of - the Unicode data. - - If byteorder is not 0, output is written according to the following - byte order: - - byteorder == -1: little endian - byteorder == 0: native byte order (writes a BOM mark) - byteorder == 1: big endian - - If byteorder is 0, the output string will always start with the - Unicode BOM mark (U+FEFF). In the other two modes, no BOM mark is - prepended. -*/ -PyAPI_FUNC(PyObject*) _PyUnicode_EncodeUTF16( - PyObject* unicode, /* Unicode object */ - const char *errors, /* error handling */ - int byteorder /* byteorder to use 0=BOM+native;-1=LE,1=BE */ - ); - -/* --- Unicode-Escape Codecs ---------------------------------------------- */ - -/* Variant of PyUnicode_DecodeUnicodeEscape that supports partial decoding. */ -PyAPI_FUNC(PyObject*) _PyUnicode_DecodeUnicodeEscapeStateful( - const char *string, /* Unicode-Escape encoded string */ - Py_ssize_t length, /* size of string */ - const char *errors, /* error handling */ - Py_ssize_t *consumed /* bytes consumed */ -); -/* Helper for PyUnicode_DecodeUnicodeEscape that detects invalid escape - chars. */ -PyAPI_FUNC(PyObject*) _PyUnicode_DecodeUnicodeEscapeInternal( - const char *string, /* Unicode-Escape encoded string */ - Py_ssize_t length, /* size of string */ - const char *errors, /* error handling */ - Py_ssize_t *consumed, /* bytes consumed */ - const char **first_invalid_escape /* on return, points to first - invalid escaped char in - string. */ -); - -/* --- Raw-Unicode-Escape Codecs ---------------------------------------------- */ - -/* Variant of PyUnicode_DecodeRawUnicodeEscape that supports partial decoding. */ -PyAPI_FUNC(PyObject*) _PyUnicode_DecodeRawUnicodeEscapeStateful( - const char *string, /* Unicode-Escape encoded string */ - Py_ssize_t length, /* size of string */ - const char *errors, /* error handling */ - Py_ssize_t *consumed /* bytes consumed */ -); - -/* --- Latin-1 Codecs ----------------------------------------------------- */ - -PyAPI_FUNC(PyObject*) _PyUnicode_AsLatin1String( - PyObject* unicode, - const char* errors); - -/* --- ASCII Codecs ------------------------------------------------------- */ - -PyAPI_FUNC(PyObject*) _PyUnicode_AsASCIIString( - PyObject* unicode, - const char* errors); - -/* --- Character Map Codecs ----------------------------------------------- */ - -/* Translate an Unicode object by applying a character mapping table to - it and return the resulting Unicode object. - - The mapping table must map Unicode ordinal integers to Unicode strings, - Unicode ordinal integers or None (causing deletion of the character). - - Mapping tables may be dictionaries or sequences. Unmapped character - ordinals (ones which cause a LookupError) are left untouched and - are copied as-is. -*/ -PyAPI_FUNC(PyObject*) _PyUnicode_EncodeCharmap( - PyObject *unicode, /* Unicode object */ - PyObject *mapping, /* encoding mapping */ - const char *errors /* error handling */ - ); - -/* --- Decimal Encoder ---------------------------------------------------- */ - -/* Coverts a Unicode object holding a decimal value to an ASCII string - for using in int, float and complex parsers. - Transforms code points that have decimal digit property to the - corresponding ASCII digit code points. Transforms spaces to ASCII. - Transforms code points starting from the first non-ASCII code point that - is neither a decimal digit nor a space to the end into '?'. */ - -PyAPI_FUNC(PyObject*) _PyUnicode_TransformDecimalAndSpaceToASCII( - PyObject *unicode /* Unicode object */ - ); - -/* --- Methods & Slots ---------------------------------------------------- */ - -PyAPI_FUNC(PyObject *) _PyUnicode_JoinArray( - PyObject *separator, - PyObject *const *items, - Py_ssize_t seqlen - ); - -/* Test whether a unicode is equal to ASCII identifier. Return 1 if true, - 0 otherwise. The right argument must be ASCII identifier. - Any error occurs inside will be cleared before return. */ -PyAPI_FUNC(int) _PyUnicode_EqualToASCIIId( - PyObject *left, /* Left string */ - _Py_Identifier *right /* Right identifier */ - ); - -/* Test whether a unicode is equal to ASCII string. Return 1 if true, - 0 otherwise. The right argument must be ASCII-encoded string. - Any error occurs inside will be cleared before return. */ -PyAPI_FUNC(int) _PyUnicode_EqualToASCIIString( - PyObject *left, - const char *right /* ASCII-encoded string */ - ); - -/* Externally visible for str.strip(unicode) */ -PyAPI_FUNC(PyObject *) _PyUnicode_XStrip( - PyObject *self, - int striptype, - PyObject *sepobj - ); - -/* Using explicit passed-in values, insert the thousands grouping - into the string pointed to by buffer. For the argument descriptions, - see Objects/stringlib/localeutil.h */ -PyAPI_FUNC(Py_ssize_t) _PyUnicode_InsertThousandsGrouping( - _PyUnicodeWriter *writer, - Py_ssize_t n_buffer, - PyObject *digits, - Py_ssize_t d_pos, - Py_ssize_t n_digits, - Py_ssize_t min_width, - const char *grouping, - PyObject *thousands_sep, - Py_UCS4 *maxchar); /* === Characters Type APIs =============================================== */ @@ -813,14 +602,6 @@ PyAPI_FUNC(int) _PyUnicode_IsTitlecase( Py_UCS4 ch /* Unicode character */ ); -PyAPI_FUNC(int) _PyUnicode_IsXidStart( - Py_UCS4 ch /* Unicode character */ - ); - -PyAPI_FUNC(int) _PyUnicode_IsXidContinue( - Py_UCS4 ch /* Unicode character */ - ); - PyAPI_FUNC(int) _PyUnicode_IsWhitespace( const Py_UCS4 ch /* Unicode character */ ); @@ -829,46 +610,18 @@ PyAPI_FUNC(int) _PyUnicode_IsLinebreak( const Py_UCS4 ch /* Unicode character */ ); -/* Py_DEPRECATED(3.3) */ PyAPI_FUNC(Py_UCS4) _PyUnicode_ToLowercase( +PyAPI_FUNC(Py_UCS4) _PyUnicode_ToLowercase( Py_UCS4 ch /* Unicode character */ ); -/* Py_DEPRECATED(3.3) */ PyAPI_FUNC(Py_UCS4) _PyUnicode_ToUppercase( +PyAPI_FUNC(Py_UCS4) _PyUnicode_ToUppercase( Py_UCS4 ch /* Unicode character */ ); -Py_DEPRECATED(3.3) PyAPI_FUNC(Py_UCS4) _PyUnicode_ToTitlecase( +PyAPI_FUNC(Py_UCS4) _PyUnicode_ToTitlecase( Py_UCS4 ch /* Unicode character */ ); -PyAPI_FUNC(int) _PyUnicode_ToLowerFull( - Py_UCS4 ch, /* Unicode character */ - Py_UCS4 *res - ); - -PyAPI_FUNC(int) _PyUnicode_ToTitleFull( - Py_UCS4 ch, /* Unicode character */ - Py_UCS4 *res - ); - -PyAPI_FUNC(int) _PyUnicode_ToUpperFull( - Py_UCS4 ch, /* Unicode character */ - Py_UCS4 *res - ); - -PyAPI_FUNC(int) _PyUnicode_ToFoldedFull( - Py_UCS4 ch, /* Unicode character */ - Py_UCS4 *res - ); - -PyAPI_FUNC(int) _PyUnicode_IsCaseIgnorable( - Py_UCS4 ch /* Unicode character */ - ); - -PyAPI_FUNC(int) _PyUnicode_IsCased( - Py_UCS4 ch /* Unicode character */ - ); - PyAPI_FUNC(int) _PyUnicode_ToDecimalDigit( Py_UCS4 ch /* Unicode character */ ); @@ -945,19 +698,6 @@ static inline int Py_UNICODE_ISALNUM(Py_UCS4 ch) { /* === Misc functions ===================================================== */ -PyAPI_FUNC(PyObject*) _PyUnicode_FormatLong(PyObject *, int, int, int); - -/* Return an interned Unicode object for an Identifier; may fail if there is no memory.*/ +// Return an interned Unicode object for an Identifier; may fail if there is no +// memory. PyAPI_FUNC(PyObject*) _PyUnicode_FromId(_Py_Identifier*); - -/* Fast equality check when the inputs are known to be exact unicode types - and where the hash values are equal (i.e. a very probable match) */ -PyAPI_FUNC(int) _PyUnicode_EQ(PyObject *, PyObject *); - -/* Equality check. */ -PyAPI_FUNC(int) _PyUnicode_Equal(PyObject *, PyObject *); - -PyAPI_FUNC(int) _PyUnicode_WideCharString_Converter(PyObject *, void *); -PyAPI_FUNC(int) _PyUnicode_WideCharString_Opt_Converter(PyObject *, void *); - -PyAPI_FUNC(Py_ssize_t) _PyUnicode_ScanIdentifier(PyObject *); diff --git a/contrib/tools/python3/Include/cpython/weakrefobject.h b/contrib/tools/python3/Include/cpython/weakrefobject.h index fd79fdc2dcc..28acf7265a0 100644 --- a/contrib/tools/python3/Include/cpython/weakrefobject.h +++ b/contrib/tools/python3/Include/cpython/weakrefobject.h @@ -30,13 +30,20 @@ struct _PyWeakReference { PyWeakReference *wr_prev; PyWeakReference *wr_next; vectorcallfunc vectorcall; -}; -PyAPI_FUNC(Py_ssize_t) _PyWeakref_GetWeakrefCount(PyWeakReference *head); +#ifdef Py_GIL_DISABLED + /* Pointer to the lock used when clearing in free-threaded builds. + * Normally this can be derived from wr_object, but in some cases we need + * to lock after wr_object has been set to Py_None. + */ + PyMutex *weakrefs_lock; +#endif +}; PyAPI_FUNC(void) _PyWeakref_ClearRef(PyWeakReference *self); -static inline PyObject* PyWeakref_GET_OBJECT(PyObject *ref_obj) { +Py_DEPRECATED(3.13) static inline PyObject* PyWeakref_GET_OBJECT(PyObject *ref_obj) +{ PyWeakReference *ref; PyObject *obj; assert(PyWeakref_Check(ref_obj)); |