diff options
| author | YDBot <[email protected]> | 2025-09-24 05:04:56 +0000 |
|---|---|---|
| committer | YDBot <[email protected]> | 2025-09-24 05:04:56 +0000 |
| commit | 58810b86e1b6fecf38478e3ef4fca50481a47c80 (patch) | |
| tree | 0684dd844030d28650202a490a170657190303fe /contrib/python | |
| parent | 7a9abfb04d596ae64b42a4c2af5d196b0c9f57fb (diff) | |
| parent | 5bcb5f9adb3ff5f877b32f402597667962f972f7 (diff) | |
Merge pull request #25697 from ydb-platform/merge-rightlib-250924-0050
Diffstat (limited to 'contrib/python')
40 files changed, 3931 insertions, 320 deletions
diff --git a/contrib/python/cffi/py3/.dist-info/METADATA b/contrib/python/cffi/py3/.dist-info/METADATA index 60b0779f688..67508e56a53 100644 --- a/contrib/python/cffi/py3/.dist-info/METADATA +++ b/contrib/python/cffi/py3/.dist-info/METADATA @@ -1,40 +1,68 @@ -Metadata-Version: 2.1 +Metadata-Version: 2.4 Name: cffi -Version: 1.17.1 +Version: 2.0.0 Summary: Foreign Function Interface for Python calling C code. -Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski -Author-email: [email protected] -License: MIT -Project-URL: Documentation, http://cffi.readthedocs.org/ -Project-URL: Source Code, https://github.com/python-cffi/cffi -Project-URL: Issue Tracker, https://github.com/python-cffi/cffi/issues +Maintainer: Matt Davis, Matt Clay, Matti Picus +License-Expression: MIT +Project-URL: Documentation, https://cffi.readthedocs.io/ Project-URL: Changelog, https://cffi.readthedocs.io/en/latest/whatsnew.html Project-URL: Downloads, https://github.com/python-cffi/cffi/releases Project-URL: Contact, https://groups.google.com/forum/#!forum/python-cffi +Project-URL: Source Code, https://github.com/python-cffi/cffi +Project-URL: Issue Tracker, https://github.com/python-cffi/cffi/issues Classifier: Programming Language :: Python Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3.8 Classifier: Programming Language :: Python :: 3.9 Classifier: Programming Language :: Python :: 3.10 Classifier: Programming Language :: Python :: 3.11 Classifier: Programming Language :: Python :: 3.12 Classifier: Programming Language :: Python :: 3.13 +Classifier: Programming Language :: Python :: 3.14 +Classifier: Programming Language :: Python :: Free Threading :: 2 - Beta Classifier: Programming Language :: Python :: Implementation :: CPython -Classifier: Programming Language :: Python :: Implementation :: PyPy -Classifier: License :: OSI Approved :: MIT License -Requires-Python: >=3.8 +Requires-Python: >=3.9 +Description-Content-Type: text/markdown License-File: LICENSE -Requires-Dist: pycparser +License-File: AUTHORS +Requires-Dist: pycparser; implementation_name != "PyPy" +Dynamic: license-file + +[](https://github.com/python-cffi/cffi/actions/workflows/ci.yaml?query=branch%3Amain++) +[](https://pypi.org/project/cffi) +[][Documentation] CFFI ==== Foreign Function Interface for Python calling C code. -Please see the `Documentation <http://cffi.readthedocs.org/>`_. + +Please see the [Documentation] or uncompiled in the `doc/` subdirectory. + +Download +-------- + +[Download page](https://github.com/python-cffi/cffi/releases) + +Source Code +----------- + +Source code is publicly available on +[GitHub](https://github.com/python-cffi/cffi). Contact ------- -`Mailing list <https://groups.google.com/forum/#!forum/python-cffi>`_ +[Mailing list](https://groups.google.com/forum/#!forum/python-cffi) + +Testing/development tips +------------------------ + +After `git clone` or `wget && tar`, we will get a directory called `cffi` or `cffi-x.x.x`. we call it `repo-directory`. To run tests under CPython, run the following in the `repo-directory`: + + pip install pytest + pip install -e . # editable install of CFFI for local development + pytest src/c/ testing/ + +[Documentation]: http://cffi.readthedocs.org/ diff --git a/contrib/python/cffi/py3/LICENSE b/contrib/python/cffi/py3/LICENSE index 29225eee9ed..0a1dbfb017c 100644 --- a/contrib/python/cffi/py3/LICENSE +++ b/contrib/python/cffi/py3/LICENSE @@ -3,7 +3,7 @@ Except when otherwise stated (look for LICENSE files in directories or information at the beginning of each file) all software and documentation is licensed as follows: - The MIT License + MIT No Attribution Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation @@ -11,10 +11,7 @@ documentation is licensed as follows: restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the - Software is furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included - in all copies or substantial portions of the Software. + Software is furnished to do so. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, diff --git a/contrib/python/cffi/py3/README.md b/contrib/python/cffi/py3/README.md index 58e8b82c725..723624d743d 100644 --- a/contrib/python/cffi/py3/README.md +++ b/contrib/python/cffi/py3/README.md @@ -29,11 +29,10 @@ Contact Testing/development tips ------------------------ -To run tests under CPython, run the following in the source root directory: +After `git clone` or `wget && tar`, we will get a directory called `cffi` or `cffi-x.x.x`. we call it `repo-directory`. To run tests under CPython, run the following in the `repo-directory`: pip install pytest pip install -e . # editable install of CFFI for local development - pytest c/ testing/ - + pytest src/c/ testing/ [Documentation]: http://cffi.readthedocs.org/ diff --git a/contrib/python/cffi/py3/c/_cffi_backend.c b/contrib/python/cffi/py3/c/_cffi_backend.c index 4ba773a1753..2aaccede0d1 100644 --- a/contrib/python/cffi/py3/c/_cffi_backend.c +++ b/contrib/python/cffi/py3/c/_cffi_backend.c @@ -1,8 +1,8 @@ #define PY_SSIZE_T_CLEAN #include <Python.h> #include "structmember.h" - -#define CFFI_VERSION "1.17.1" +#include "misc_thread_common.h" +#define CFFI_VERSION "2.0.0" #ifdef MS_WIN32 #include <windows.h> @@ -14,6 +14,7 @@ #include <errno.h> #include <ffi.h> #include <sys/mman.h> +#include "misc_thread_posix.h" #endif /* this block of #ifs should be kept exactly identical between @@ -204,6 +205,49 @@ # define USE_WRITEUNRAISABLEMSG #endif +#if PY_VERSION_HEX <= 0x030d00a1 +static int PyDict_GetItemRef(PyObject *mp, PyObject *key, PyObject **result) +{ + PyObject *obj = PyDict_GetItemWithError(mp, key); + Py_XINCREF(obj); + *result = obj; + if (obj == NULL) { + if (PyErr_Occurred()) { + return -1; + } + return 0; + } + return 1; +} +#endif + +#if PY_VERSION_HEX < 0x030d00a1 +static int PyWeakref_GetRef(PyObject *ref, PyObject **pobj) +{ + PyObject *obj = PyWeakref_GetObject(ref); + if (obj == NULL) { + *pobj = NULL; + return -1; + } + if (obj == Py_None) { + *pobj = NULL; + return 0; + } + Py_INCREF(obj); + *pobj = obj; + return 1; +} +#endif + + +#ifdef Py_GIL_DISABLED +# define LOCK_UNIQUE_CACHE() PyMutex_Lock(&unique_cache_lock) +# define UNLOCK_UNIQUE_CACHE() PyMutex_Unlock(&unique_cache_lock) +#else +# define LOCK_UNIQUE_CACHE() ((void)0) +# define UNLOCK_UNIQUE_CACHE() ((void)0) +#endif + /************************************************************/ /* base type flag: exactly one of the following: */ @@ -217,7 +261,7 @@ #define CT_UNION 0x080 /* union */ #define CT_FUNCTIONPTR 0x100 /* pointer to function */ #define CT_VOID 0x200 /* void */ -#define CT_PRIMITIVE_COMPLEX 0x400 /* float _Complex, double _Complex */ +#define CT_PRIMITIVE_COMPLEX 0x400 /* _cffi_float/double_complex_t */ /* other flags that may also be set in addition to the base flag: */ #define CT_IS_VOIDCHAR_PTR 0x00001000 @@ -225,15 +269,15 @@ #define CT_IS_OPAQUE 0x00004000 #define CT_IS_ENUM 0x00008000 #define CT_IS_PTR_TO_OWNED 0x00010000 /* only owned if CDataOwning_Type */ -#define CT_CUSTOM_FIELD_POS 0x00020000 +/* unused 0x00020000 */ #define CT_IS_LONGDOUBLE 0x00040000 #define CT_IS_BOOL 0x00080000 #define CT_IS_FILE 0x00100000 #define CT_IS_VOID_PTR 0x00200000 -#define CT_WITH_VAR_ARRAY 0x00400000 /* with open-ended array, anywhere */ +/* unused 0x00400000 */ /* unused 0x00800000 */ -#define CT_LAZY_FIELD_LIST 0x01000000 -#define CT_WITH_PACKED_CHANGE 0x02000000 +/* unused 0x01000000 */ +/* unused 0x02000000 */ #define CT_IS_SIGNED_WCHAR 0x04000000 #define CT_PRIMITIVE_ANY (CT_PRIMITIVE_SIGNED | \ CT_PRIMITIVE_UNSIGNED | \ @@ -241,6 +285,16 @@ CT_PRIMITIVE_FLOAT | \ CT_PRIMITIVE_COMPLEX) +/* flags that are mutable at runtime, stored separately in ct_flags_mut to avoid + races with ct_flags + + these are read and set without using atomic operations because CFFI_LOCK is held + while mutating ct_flags_mut + */ +#define CT_CUSTOM_FIELD_POS 0x00000001 +#define CT_WITH_VAR_ARRAY 0x00000002 /* with open-ended array, anywhere */ +#define CT_WITH_PACKED_CHANGE 0x00000004 + typedef struct _ctypedescr { PyObject_VAR_HEAD @@ -263,8 +317,11 @@ typedef struct _ctypedescr { Py_ssize_t ct_length; /* length of arrays, or -1 if unknown; or alignment of primitive and struct types; always -1 for pointers */ - int ct_flags; /* CT_xxx flags */ - + int ct_flags; /* Immutable CT_xxx flags */ + int ct_flags_mut; /* Mutable CT_xxx flags */ + uint8_t ct_under_construction; + uint8_t ct_lazy_field_list; + uint8_t ct_unrealized_struct_or_union; int ct_name_position; /* index in ct_name of where to put a var name */ char ct_name[1]; /* string, e.g. "int *" for pointers to ints */ } CTypeDescrObject; @@ -397,6 +454,10 @@ typedef struct _cffi_allocator_s { } cffi_allocator_t; static const cffi_allocator_t default_allocator = { NULL, NULL, 0 }; static PyObject *FFIError; + +#ifdef Py_GIL_DISABLED +static PyMutex unique_cache_lock; +#endif static PyObject *unique_cache; /************************************************************/ @@ -414,6 +475,10 @@ ctypedescr_new(int name_size) ct->ct_stuff = NULL; ct->ct_weakreflist = NULL; ct->ct_unique_key = NULL; + ct->ct_lazy_field_list = 0; + ct->ct_under_construction = 0; + ct->ct_unrealized_struct_or_union = 0; + ct->ct_flags_mut = 0; PyObject_GC_Track(ct); return ct; } @@ -450,19 +515,17 @@ ctypedescr_repr(CTypeDescrObject *ct) return PyText_FromFormat("<ctype '%s'>", ct->ct_name); } +static void remove_dead_unique_reference(PyObject *unique_key); + static void ctypedescr_dealloc(CTypeDescrObject *ct) { PyObject_GC_UnTrack(ct); - if (ct->ct_weakreflist != NULL) - PyObject_ClearWeakRefs((PyObject *) ct); + PyObject_ClearWeakRefs((PyObject *) ct); if (ct->ct_unique_key != NULL) { - /* revive dead object temporarily for DelItem */ - Py_SET_REFCNT(ct, 43); - PyDict_DelItem(unique_cache, ct->ct_unique_key); - assert(Py_REFCNT(ct) == 42); - Py_SET_REFCNT(ct, 0); + /* delete the weak reference from unique_cache */ + remove_dead_unique_reference(ct->ct_unique_key); Py_DECREF(ct->ct_unique_key); } Py_XDECREF(ct->ct_itemdescr); @@ -559,18 +622,29 @@ static PyObject *ctypeget_length(CTypeDescrObject *ct, void *context) static PyObject * get_field_name(CTypeDescrObject *ct, CFieldObject *cf); /* forward */ +static int do_realize_lazy_struct(CTypeDescrObject *ct); +/* forward, implemented in realize_c_type.c */ + /* returns 0 if the struct ctype is opaque, 1 if it is not, or -1 if an exception occurs */ -#define force_lazy_struct(ct) \ - ((ct)->ct_stuff != NULL ? 1 : do_realize_lazy_struct(ct)) +static inline int +force_lazy_struct(CTypeDescrObject *ct) +{ + assert(ct->ct_flags & (CT_STRUCT | CT_UNION)); + uint8_t lazy_fields = cffi_check_flag(ct->ct_lazy_field_list); + if (lazy_fields) { + // not realized yet + return do_realize_lazy_struct(ct); + } + return ct->ct_stuff != NULL; +} -static int do_realize_lazy_struct(CTypeDescrObject *ct); -/* forward, implemented in realize_c_type.c */ static PyObject *ctypeget_fields(CTypeDescrObject *ct, void *context) { if (ct->ct_flags & (CT_STRUCT | CT_UNION)) { - if (!(ct->ct_flags & CT_IS_OPAQUE)) { + assert((ct->ct_flags & CT_IS_OPAQUE) == 0); + if (!cffi_check_flag(ct->ct_unrealized_struct_or_union)) { CFieldObject *cf; PyObject *res; if (force_lazy_struct(ct) < 0) @@ -821,13 +895,13 @@ _my_PyLong_AsLongLong(PyObject *ob) { /* (possibly) convert and cast a Python object to a long long. Like PyLong_AsLongLong(), this version accepts a Python int too, and - does convertions from other types of objects. The difference is that + does conversions from other types of objects. The difference is that this version refuses floats. */ #if PY_MAJOR_VERSION < 3 if (PyInt_Check(ob)) { return PyInt_AS_LONG(ob); } - else + else #endif if (PyLong_Check(ob)) { return PyLong_AsLongLong(ob); @@ -863,7 +937,7 @@ _my_PyLong_AsUnsignedLongLong(PyObject *ob, int strict) { /* (possibly) convert and cast a Python object to an unsigned long long. Like PyLong_AsLongLong(), this version accepts a Python int too, and - does convertions from other types of objects. If 'strict', complains + does conversions from other types of objects. If 'strict', complains with OverflowError and refuses floats. If '!strict', rounds floats and masks the result. */ #if PY_MAJOR_VERSION < 3 @@ -1099,12 +1173,14 @@ convert_to_object(char *data, CTypeDescrObject *ct) return NULL; } else if (ct->ct_flags & (CT_STRUCT|CT_UNION)) { + if (force_lazy_struct(ct) < 0) + return NULL; return new_simple_cdata(data, ct); } else if (ct->ct_flags & CT_ARRAY) { if (ct->ct_length < 0) { /* we can't return a <cdata 'int[]'> here, because we don't - know the length to give it. As a compromize, returns + know the length to give it. As a compromise, returns <cdata 'int *'> in this case. */ ct = (CTypeDescrObject *)ct->ct_stuff; } @@ -1439,8 +1515,7 @@ convert_vfield_from_object(char *data, CFieldObject *cf, PyObject *value, if (optvarsize == NULL) { return convert_field_from_object(data, cf, value); } - else if ((cf->cf_type->ct_flags & CT_WITH_VAR_ARRAY) != 0 && - !CData_Check(value)) { + else if ((cf->cf_type->ct_flags_mut & CT_WITH_VAR_ARRAY) && !CData_Check(value)) { Py_ssize_t subsize = cf->cf_type->ct_size; if (convert_struct_from_object(NULL, cf->cf_type, value, &subsize) < 0) return -1; @@ -1686,21 +1761,21 @@ convert_from_object(char *data, CTypeDescrObject *ct, PyObject *init) error. The warning is turned off if both types are pointers to single bytes. */ char *msg = (ct->ct_flags & CT_IS_VOIDCHAR_PTR ? - "implicit cast to 'char *' from a different pointer type: " - "will be forbidden in the future (check that the types " - "are as you expect; use an explicit ffi.cast() if they " - "are correct)" : + "implicit cast to 'char *' from a different pointer type." + "(check that the types are as you expect; use an explicit " + "ffi.cast() if they are correct)" : "implicit cast from 'char *' to a different pointer type: " - "will be forbidden in the future (check that the types " - "are as you expect; use an explicit ffi.cast() if they " - "are correct)"); + "(check that the types are as you expect; use an explicit " + "ffi.cast() if they are correct)"); if ((ct->ct_flags & ctinit->ct_flags & CT_POINTER) && ct->ct_itemdescr->ct_size == 1 && ctinit->ct_itemdescr->ct_size == 1) { - /* no warning */ + /* no error */ } - else if (PyErr_WarnEx(PyExc_UserWarning, msg, 1)) + else { + PyErr_SetString(PyExc_TypeError, msg); return -1; + } } else { expected = "pointer to same type"; @@ -1787,8 +1862,8 @@ convert_from_object(char *data, CTypeDescrObject *ct, PyObject *init) if (ct->ct_flags & (CT_STRUCT|CT_UNION)) { if (CData_Check(init)) { - if (((CDataObject *)init)->c_type == ct && ct->ct_size >= 0) { - memcpy(data, ((CDataObject *)init)->c_data, ct->ct_size); + if (((CDataObject *)init)->c_type == ct && cffi_get_size(ct) >= 0) { + memcpy(data, ((CDataObject *)init)->c_data, cffi_get_size(ct)); return 0; } } @@ -1875,12 +1950,13 @@ get_alignment(CTypeDescrObject *ct) int align; retry: if ((ct->ct_flags & (CT_PRIMITIVE_ANY|CT_STRUCT|CT_UNION)) && - !(ct->ct_flags & CT_IS_OPAQUE)) { - align = ct->ct_length; - if (align == -1 && (ct->ct_flags & CT_LAZY_FIELD_LIST)) { - force_lazy_struct(ct); - align = ct->ct_length; + !((ct->ct_flags & CT_IS_OPAQUE) || cffi_check_flag(ct->ct_unrealized_struct_or_union))) { + if (cffi_check_flag(ct->ct_lazy_field_list)) { + if (force_lazy_struct(ct) < 0) { + return -1; + } } + align = ct->ct_length; } else if (ct->ct_flags & (CT_POINTER|CT_FUNCTIONPTR)) { struct aligncheck_ptr { char x; char *y; }; @@ -1907,8 +1983,7 @@ get_alignment(CTypeDescrObject *ct) static void cdata_dealloc(CDataObject *cd) { - if (cd->c_weakreflist != NULL) - PyObject_ClearWeakRefs((PyObject *) cd); + PyObject_ClearWeakRefs((PyObject *) cd); Py_DECREF(cd->c_type); #ifndef CFFI_MEM_LEAK /* never release anything, tests only */ @@ -2194,7 +2269,7 @@ static Py_ssize_t _cdata_var_byte_size(CDataObject *cd) if (cd->c_type->ct_flags & CT_IS_PTR_TO_OWNED) { cd = (CDataObject *)((CDataObject_own_structptr *)cd)->structobj; } - if (cd->c_type->ct_flags & CT_WITH_VAR_ARRAY) { + if (cd->c_type->ct_flags_mut & CT_WITH_VAR_ARRAY) { return ((CDataObject_own_length *)cd)->length; } return -1; @@ -2238,7 +2313,7 @@ static Py_ssize_t cdataowning_size_bytes(CDataObject *cd) else if (cd->c_type->ct_flags & CT_ARRAY) size = get_array_length(cd) * cd->c_type->ct_itemdescr->ct_size; else - size = cd->c_type->ct_size; + size = cffi_get_size(cd->c_type); } return size; } @@ -2531,7 +2606,7 @@ _cdata_get_indexed_ptr(CDataObject *cd, PyObject *key) cd->c_type->ct_name); return NULL; } - return cd->c_data + i * cd->c_type->ct_itemdescr->ct_size; + return cd->c_data + i * cffi_get_size(cd->c_type->ct_itemdescr); } static PyObject * @@ -2598,16 +2673,21 @@ cdata_slice(CDataObject *cd, PySliceObject *slice) CTypeDescrObject *ct = _cdata_getslicearg(cd, slice, bounds); if (ct == NULL) return NULL; - + CTypeDescrObject *array_type = NULL; + Py_BEGIN_CRITICAL_SECTION(ct); + array_type = (CTypeDescrObject *)ct->ct_stuff; if (ct->ct_stuff == NULL) { - ct->ct_stuff = new_array_type(ct, -1); - if (ct->ct_stuff == NULL) - return NULL; + array_type = (CTypeDescrObject *)new_array_type(ct, -1); + ct->ct_stuff = (PyObject *)array_type; } - ct = (CTypeDescrObject *)ct->ct_stuff; + Py_END_CRITICAL_SECTION(); - cdata = cd->c_data + ct->ct_itemdescr->ct_size * bounds[0]; - return new_sized_cdata(cdata, ct, bounds[1]); + if (array_type == NULL) { + return NULL; + } + + cdata = cd->c_data + array_type->ct_itemdescr->ct_size * bounds[0]; + return new_sized_cdata(cdata, array_type, bounds[1]); } static int @@ -2790,7 +2870,7 @@ _cdata_add_or_sub(PyObject *v, PyObject *w, int sign) cd->c_type->ct_name); return NULL; } - itemsize = ctptr->ct_itemdescr->ct_size; + itemsize = cffi_get_size(ctptr->ct_itemdescr); if (itemsize < 0) { if (ctptr->ct_flags & CT_IS_VOID_PTR) { itemsize = 1; @@ -2970,16 +3050,8 @@ static cif_description_t * fb_prepare_cif(PyObject *fargs, CTypeDescrObject *, Py_ssize_t, ffi_abi); /*forward*/ -static PyObject *new_primitive_type(const char *name); /*forward*/ - -static CTypeDescrObject *_get_ct_int(void) -{ - static CTypeDescrObject *ct_int = NULL; - if (ct_int == NULL) { - ct_int = (CTypeDescrObject *)new_primitive_type("int"); - } - return ct_int; -} +static CTypeDescrObject *_get_ct_int(void); +/* forward, implemented in realize_c_type.c */ static Py_ssize_t _prepare_pointer_call_argument(CTypeDescrObject *ctptr, PyObject *init, @@ -3044,6 +3116,11 @@ _prepare_pointer_call_argument(CTypeDescrObject *ctptr, PyObject *init, goto convert_default; } + if (ctitem->ct_flags & (CT_STRUCT|CT_UNION)) { + if (force_lazy_struct(ctitem) < 0) { + return -1; + } + } if (ctitem->ct_size <= 0) goto convert_default; datasize = MUL_WRAPAROUND(length, ctitem->ct_size); @@ -3278,11 +3355,12 @@ static PyObject *cdata_dir(PyObject *cd, PyObject *noarg) ct = ct->ct_itemdescr; } if ((ct->ct_flags & (CT_STRUCT | CT_UNION)) && - !(ct->ct_flags & CT_IS_OPAQUE)) { - + !(cffi_check_flag(ct->ct_unrealized_struct_or_union))) { + assert((ct->ct_flags & CT_IS_OPAQUE) == 0); /* for non-opaque structs or unions */ if (force_lazy_struct(ct) < 0) return NULL; + assert(ct->ct_stuff); return PyDict_Keys(ct->ct_stuff); } else { @@ -3781,7 +3859,7 @@ convert_struct_to_owning_object(char *data, CTypeDescrObject *ct) "return type is an opaque structure or union"); return NULL; } - if (ct->ct_flags & CT_WITH_VAR_ARRAY) { + if (ct->ct_flags_mut & CT_WITH_VAR_ARRAY) { PyErr_SetString(PyExc_TypeError, "return type is a struct/union with a varsize array member"); return NULL; @@ -3875,7 +3953,11 @@ static PyObject *direct_newp(CTypeDescrObject *ct, PyObject *init, if (ct->ct_flags & CT_POINTER) { dataoffset = offsetof(CDataObject_own_nolength, alignment); ctitem = ct->ct_itemdescr; - datasize = ctitem->ct_size; + if (ctitem->ct_flags & (CT_STRUCT | CT_UNION)) { + if (force_lazy_struct(ctitem) < 0) + return NULL; + } + datasize = cffi_get_size(ctitem); if (datasize < 0) { PyErr_Format(PyExc_TypeError, "cannot instantiate ctype '%s' of unknown size", @@ -3886,10 +3968,7 @@ static PyObject *direct_newp(CTypeDescrObject *ct, PyObject *init, datasize *= 2; /* forcefully add another character: a null */ if (ctitem->ct_flags & (CT_STRUCT | CT_UNION)) { - if (force_lazy_struct(ctitem) < 0) /* for CT_WITH_VAR_ARRAY */ - return NULL; - - if (ctitem->ct_flags & CT_WITH_VAR_ARRAY) { + if (ctitem->ct_flags_mut & CT_WITH_VAR_ARRAY) { assert(ct->ct_flags & CT_IS_PTR_TO_OWNED); dataoffset = offsetof(CDataObject_own_length, alignment); @@ -4511,7 +4590,7 @@ static void *b_do_dlopen(PyObject *args, const char **p_printable_filename, int flags = 0; *p_temp = NULL; *auto_close = 1; - + if (PyTuple_GET_SIZE(args) == 0 || PyTuple_GET_ITEM(args, 0) == Py_None) { PyObject *dummy; if (!PyArg_ParseTuple(args, "|Oi:load_library", @@ -4564,7 +4643,7 @@ static void *b_do_dlopen(PyObject *args, const char **p_printable_filename, sz1 = PyText_GetSize(filename_unicode) + 1; sz1 *= 2; /* should not be needed, but you never know */ w1 = alloca(sizeof(wchar_t) * sz1); - sz1 = PyUnicode_AsWideChar((PyUnicodeObject *)filename_unicode, + sz1 = PyUnicode_AsWideChar(filename_unicode, w1, sz1 - 1); if (sz1 < 0) return NULL; @@ -4639,7 +4718,7 @@ static PyObject *b_load_library(PyObject *self, PyObject *args) dlobj->dl_handle = handle; dlobj->dl_name = strdup(printable_filename); dlobj->dl_auto_close = auto_close; - + error: Py_XDECREF(temp); return (PyObject *)dlobj; @@ -4647,6 +4726,46 @@ static PyObject *b_load_library(PyObject *self, PyObject *args) /************************************************************/ +static PyObject *get_or_insert_unique_type(CTypeDescrObject *x, + PyObject *key) +{ + PyObject *wr, *obj; + + if (PyDict_GetItemRef(unique_cache, key, &wr) < 0) { + return NULL; + } + + if (wr != NULL) { + if (PyWeakref_GetRef(wr, &obj) < 0) { + Py_DECREF(wr); + return NULL; + } + Py_DECREF(wr); + if (obj != NULL) { + return obj; + } + } + + /* Use a weakref so that the dictionary does not keep 'x' alive */ + wr = PyWeakref_NewRef((PyObject *)x, NULL); + if (wr == NULL) { + return NULL; + } + + if (PyDict_SetItem(unique_cache, key, wr) < 0) { + Py_DECREF(wr); + return NULL; + } + + assert(x->ct_unique_key == NULL); + Py_INCREF(key); + x->ct_unique_key = key; /* the key will be freed in ctypedescr_dealloc() */ + + Py_DECREF(wr); + Py_INCREF(x); + return (PyObject *)x; +} + static PyObject *get_unique_type(CTypeDescrObject *x, const void *unique_key[], long keylength) { @@ -4664,45 +4783,48 @@ static PyObject *get_unique_type(CTypeDescrObject *x, array [ctype, length] funcptr [ctresult, ellipsis+abi, num_args, ctargs...] */ - PyObject *key, *y, *res; - void *pkey; - - key = PyBytes_FromStringAndSize(NULL, keylength * sizeof(void *)); - if (key == NULL) - goto error; + PyObject *key, *y; - pkey = PyBytes_AS_STRING(key); - memcpy(pkey, unique_key, keylength * sizeof(void *)); - - y = PyDict_GetItem(unique_cache, key); - if (y != NULL) { - Py_DECREF(key); - Py_INCREF(y); + key = PyBytes_FromStringAndSize((const char *)unique_key, keylength * sizeof(void *)); + if (key == NULL) { Py_DECREF(x); - return y; - } - if (PyDict_SetItem(unique_cache, key, (PyObject *)x) < 0) { - Py_DECREF(key); - goto error; + return NULL; } - /* Haaaack for our reference count hack: gcmodule.c must not see this - dictionary. The problem is that any PyDict_SetItem() notices that - 'x' is tracked and re-tracks the unique_cache dictionary. So here - we re-untrack it again... */ - PyObject_GC_UnTrack(unique_cache); - assert(x->ct_unique_key == NULL); - x->ct_unique_key = key; /* the key will be freed in ctypedescr_dealloc() */ - /* the 'value' in unique_cache doesn't count as 1, but don't use - Py_DECREF(x) here because it will confuse debug builds into thinking - there was an extra DECREF in total. */ - res = (PyObject *)x; - Py_SET_REFCNT(res, Py_REFCNT(res) - 1); - return res; + LOCK_UNIQUE_CACHE(); + y = get_or_insert_unique_type(x, key); + UNLOCK_UNIQUE_CACHE(); - error: + Py_DECREF(key); Py_DECREF(x); - return NULL; + return y; +} + +/* Delete a dead weakref from the unique cache */ +static void remove_dead_unique_reference(PyObject *unique_key) +{ + PyObject *wr; + PyObject *tmp = NULL; + int err = 0; + + LOCK_UNIQUE_CACHE(); + /* We need to check that it's not already been replaced by a live weakref + to a different object. */ + wr = PyDict_GetItemWithError(unique_cache, unique_key); + if (wr != NULL) { + err = PyWeakref_GetRef(wr, &tmp); + if (err == 0) { + /* The weakref is dead, delete it. */ + assert(tmp == NULL); + err = PyDict_DelItem(unique_cache, unique_key); + } + } + UNLOCK_UNIQUE_CACHE(); + + Py_XDECREF(tmp); + if (err < 0) { + PyErr_WriteUnraisable(NULL); + } } /* according to the C standard, these types should be equivalent to the @@ -4727,8 +4849,8 @@ static PyObject *new_primitive_type(const char *name) EPTYPE(f, float, CT_PRIMITIVE_FLOAT ) \ EPTYPE(d, double, CT_PRIMITIVE_FLOAT ) \ EPTYPE(ld, long double, CT_PRIMITIVE_FLOAT | CT_IS_LONGDOUBLE ) \ - EPTYPE2(fc, "float _Complex", cffi_float_complex_t, CT_PRIMITIVE_COMPLEX ) \ - EPTYPE2(dc, "double _Complex", cffi_double_complex_t, CT_PRIMITIVE_COMPLEX ) \ + EPTYPE2(fc, "_cffi_float_complex_t", cffi_float_complex_t, CT_PRIMITIVE_COMPLEX)\ + EPTYPE2(dc, "_cffi_double_complex_t", cffi_double_complex_t, CT_PRIMITIVE_COMPLEX)\ ENUM_PRIMITIVE_TYPES_WCHAR \ EPTYPE2(c16, "char16_t", cffi_char16_t, CT_PRIMITIVE_CHAR ) \ EPTYPE2(c32, "char32_t", cffi_char32_t, CT_PRIMITIVE_CHAR ) \ @@ -4973,7 +5095,7 @@ new_array_type(CTypeDescrObject *ctptr, Py_ssize_t length) return NULL; } ctitem = ctptr->ct_itemdescr; - if (ctitem->ct_size < 0) { + if (cffi_get_size(ctitem) < 0) { PyErr_Format(PyExc_ValueError, "array item of unknown size: '%s'", ctitem->ct_name); return NULL; @@ -4986,8 +5108,8 @@ new_array_type(CTypeDescrObject *ctptr, Py_ssize_t length) } else { sprintf(extra_text, "[%llu]", (unsigned PY_LONG_LONG)length); - arraysize = MUL_WRAPAROUND(length, ctitem->ct_size); - if (length > 0 && (arraysize / length) != ctitem->ct_size) { + arraysize = MUL_WRAPAROUND(length, cffi_get_size(ctitem)); + if (length > 0 && (arraysize / length) != cffi_get_size(ctitem)) { PyErr_SetString(PyExc_OverflowError, "array size would overflow a Py_ssize_t"); return NULL; @@ -5037,7 +5159,8 @@ static PyObject *new_struct_or_union_type(const char *name, int flag) td->ct_size = -1; td->ct_length = -1; - td->ct_flags = flag | CT_IS_OPAQUE; + td->ct_flags = flag; + td->ct_unrealized_struct_or_union = 1; td->ct_extra = NULL; memcpy(td->ct_name, name, namelen + 1); td->ct_name_position = namelen; @@ -5160,34 +5283,27 @@ static int detect_custom_layout(CTypeDescrObject *ct, int sflags, ct->ct_name); return -1; } - ct->ct_flags |= CT_CUSTOM_FIELD_POS; + ct->ct_flags_mut |= CT_CUSTOM_FIELD_POS; } return 0; } #define ROUNDUP_BYTES(bytes, bits) ((bytes) + ((bits) > 0)) -static PyObject *b_complete_struct_or_union(PyObject *self, PyObject *args) + + +static PyObject *b_complete_struct_or_union_lock_held(CTypeDescrObject *ct, + PyObject *fields, + Py_ssize_t totalsize, int totalalignment, int sflags, + int pack) { - CTypeDescrObject *ct; - PyObject *fields, *interned_fields, *ignored; int is_union, alignment; Py_ssize_t byteoffset, i, nb_fields, byteoffsetmax, alignedsize; - int bitoffset; + int bitoffset, fflags; Py_ssize_t byteoffsetorg; - Py_ssize_t totalsize = -1; - int totalalignment = -1; CFieldObject **previous; int prev_bitfield_size, prev_bitfield_free; - int sflags = 0, fflags; - int pack = 0; - - if (!PyArg_ParseTuple(args, "O!O!|Oniii:complete_struct_or_union", - &CTypeDescr_Type, &ct, - &PyList_Type, &fields, - &ignored, &totalsize, &totalalignment, &sflags, - &pack)) - return NULL; + PyObject *interned_fields; sflags = complete_sflags(sflags); if (sflags & SF_PACKED) @@ -5197,20 +5313,16 @@ static PyObject *b_complete_struct_or_union(PyObject *self, PyObject *args) else sflags |= SF_PACKED; - if ((ct->ct_flags & (CT_STRUCT|CT_IS_OPAQUE)) == - (CT_STRUCT|CT_IS_OPAQUE)) { - is_union = 0; - } - else if ((ct->ct_flags & (CT_UNION|CT_IS_OPAQUE)) == - (CT_UNION|CT_IS_OPAQUE)) { - is_union = 1; - } - else { + PyObject *res = NULL; + is_union = ct->ct_flags & CT_UNION; + if (!((ct->ct_flags & CT_UNION) || (ct->ct_flags & CT_STRUCT)) || + !(cffi_check_flag(ct->ct_unrealized_struct_or_union) || cffi_check_flag(ct->ct_under_construction))) { PyErr_SetString(PyExc_TypeError, - "first arg must be a non-initialized struct or union ctype"); - return NULL; + "first arg must be a non-initialized struct or union ctype"); + goto finally; } - ct->ct_flags &= ~(CT_CUSTOM_FIELD_POS | CT_WITH_PACKED_CHANGE); + ct->ct_flags_mut &= ~CT_CUSTOM_FIELD_POS; + ct->ct_flags_mut &= ~CT_WITH_PACKED_CHANGE; alignment = 1; byteoffset = 0; /* the real value is 'byteoffset+bitoffset*8', which */ @@ -5221,7 +5333,7 @@ static PyObject *b_complete_struct_or_union(PyObject *self, PyObject *args) nb_fields = PyList_GET_SIZE(fields); interned_fields = PyDict_New(); if (interned_fields == NULL) - return NULL; + goto finally; previous = (CFieldObject **)&ct->ct_extra; @@ -5235,33 +5347,39 @@ static PyObject *b_complete_struct_or_union(PyObject *self, PyObject *args) &PyText_Type, &fname, &CTypeDescr_Type, &ftype, &fbitsize, &foffset)) - goto error; + goto finally; + + if ((ftype->ct_flags & (CT_STRUCT | CT_UNION)) && + !(cffi_check_flag(ct->ct_unrealized_struct_or_union))) { + assert((ftype->ct_flags & CT_IS_OPAQUE) == 0); + /* force now the type of the nested field */ + if (force_lazy_struct(ftype) < 0) + goto finally; + } - if (ftype->ct_size < 0) { + if (cffi_get_size(ftype) < 0) { if ((ftype->ct_flags & CT_ARRAY) && fbitsize < 0 && (i == nb_fields - 1 || foffset != -1)) { - ct->ct_flags |= CT_WITH_VAR_ARRAY; + ct->ct_flags_mut |= CT_WITH_VAR_ARRAY; } else { PyErr_Format(PyExc_TypeError, "field '%s.%s' has ctype '%s' of unknown size", ct->ct_name, PyText_AS_UTF8(fname), ftype->ct_name); - goto error; + goto finally; } } else if (ftype->ct_flags & (CT_STRUCT|CT_UNION)) { - if (force_lazy_struct(ftype) < 0) /* for CT_WITH_VAR_ARRAY */ - return NULL; - /* GCC (or maybe C99) accepts var-sized struct fields that are not the last field of a larger struct. That's why there is no check here for "last field": we propagate the flag CT_WITH_VAR_ARRAY to any struct that contains either an open- ended array or another struct that recursively contains an open-ended array. */ - if (ftype->ct_flags & CT_WITH_VAR_ARRAY) - ct->ct_flags |= CT_WITH_VAR_ARRAY; + if (ftype->ct_flags_mut & CT_WITH_VAR_ARRAY) { + ct->ct_flags_mut |= CT_WITH_VAR_ARRAY; + } } if (is_union) @@ -5271,7 +5389,7 @@ static PyObject *b_complete_struct_or_union(PyObject *self, PyObject *args) field is an anonymous bitfield or if SF_PACKED */ falignorg = get_alignment(ftype); if (falignorg < 0) - goto error; + goto finally; falign = (pack < falignorg) ? pack : falignorg; do_align = 1; @@ -5309,16 +5427,16 @@ static PyObject *b_complete_struct_or_union(PyObject *self, PyObject *args) byteoffset = (byteoffset + falign-1) & ~(falign-1); if (byteoffsetorg != byteoffset) { - ct->ct_flags |= CT_WITH_PACKED_CHANGE; + ct->ct_flags_mut |= CT_WITH_PACKED_CHANGE; } if (foffset >= 0) { /* a forced field position: ignore the offset just computed, - except to know if we must set CT_CUSTOM_FIELD_POS */ + except to know if we must set CT_CUSTOM_FIELD_POS */ if (detect_custom_layout(ct, sflags, byteoffset, foffset, "wrong offset for field '", PyText_AS_UTF8(fname), "'") < 0) - goto error; + goto finally; byteoffset = foffset; } @@ -5338,17 +5456,17 @@ static PyObject *b_complete_struct_or_union(PyObject *self, PyObject *args) cfsrc->cf_bitsize, cfsrc->cf_flags | fflags); if (*previous == NULL) - goto error; + goto finally; previous = &(*previous)->cf_next; } /* always forbid such structures from being passed by value */ - ct->ct_flags |= CT_CUSTOM_FIELD_POS; + ct->ct_flags_mut |= CT_CUSTOM_FIELD_POS; } else { *previous = _add_field(interned_fields, fname, ftype, byteoffset, bs_flag, -1, fflags); if (*previous == NULL) - goto error; + goto finally; previous = &(*previous)->cf_next; } if (ftype->ct_size >= 0) @@ -5365,7 +5483,7 @@ static PyObject *b_complete_struct_or_union(PyObject *self, PyObject *args) "field '%s.%s' is a bitfield, " "but a fixed offset is specified", ct->ct_name, PyText_AS_UTF8(fname)); - goto error; + goto finally; } if (!(ftype->ct_flags & (CT_PRIMITIVE_SIGNED | @@ -5375,7 +5493,7 @@ static PyObject *b_complete_struct_or_union(PyObject *self, PyObject *args) "field '%s.%s' declared as '%s' cannot be a bit field", ct->ct_name, PyText_AS_UTF8(fname), ftype->ct_name); - goto error; + goto finally; } if (fbitsize > 8 * ftype->ct_size) { PyErr_Format(PyExc_TypeError, @@ -5383,7 +5501,7 @@ static PyObject *b_complete_struct_or_union(PyObject *self, PyObject *args) "exceeds the width of the type", ct->ct_name, PyText_AS_UTF8(fname), ftype->ct_name, fbitsize); - goto error; + goto finally; } /* compute the starting position of the theoretical field @@ -5397,7 +5515,7 @@ static PyObject *b_complete_struct_or_union(PyObject *self, PyObject *args) PyErr_Format(PyExc_TypeError, "field '%s.%s' is declared with :0", ct->ct_name, PyText_AS_UTF8(fname)); - goto error; + goto finally; } if (!(sflags & SF_MSVC_BITFIELDS)) { /* GCC's notion of "ftype :0;" */ @@ -5437,7 +5555,7 @@ static PyObject *b_complete_struct_or_union(PyObject *self, PyObject *args) "with 'packed', gcc would compile field " "'%s.%s' to reuse some bits in the previous " "field", ct->ct_name, PyText_AS_UTF8(fname)); - goto error; + goto finally; } field_offset_bytes += falign; assert(byteoffset < field_offset_bytes); @@ -5488,7 +5606,7 @@ static PyObject *b_complete_struct_or_union(PyObject *self, PyObject *args) field_offset_bytes, bitshift, fbitsize, fflags); if (*previous == NULL) - goto error; + goto finally; previous = &(*previous)->cf_next; } } @@ -5513,12 +5631,12 @@ static PyObject *b_complete_struct_or_union(PyObject *self, PyObject *args) else { if (detect_custom_layout(ct, sflags, alignedsize, totalsize, "wrong total size", "", "") < 0) - goto error; + goto finally; if (totalsize < byteoffsetmax) { PyErr_Format(PyExc_TypeError, "%s cannot be of size %zd: there are fields at least " "up to %zd", ct->ct_name, totalsize, byteoffsetmax); - goto error; + goto finally; } } if (totalalignment < 0) { @@ -5527,23 +5645,48 @@ static PyObject *b_complete_struct_or_union(PyObject *self, PyObject *args) else { if (detect_custom_layout(ct, sflags, alignment, totalalignment, "wrong total alignment", "", "") < 0) - goto error; + goto finally; } - ct->ct_size = totalsize; + cffi_set_size(ct, totalsize); ct->ct_length = totalalignment; ct->ct_stuff = interned_fields; - ct->ct_flags &= ~CT_IS_OPAQUE; + cffi_set_flag(ct->ct_unrealized_struct_or_union, 0); + res = Py_None; + Py_INCREF(res); - Py_INCREF(Py_None); - return Py_None; +finally:; + if (res == NULL) { + ct->ct_extra = NULL; + Py_XDECREF(interned_fields); + } + return res; +} - error: - ct->ct_extra = NULL; - Py_DECREF(interned_fields); - return NULL; +static PyObject *b_complete_struct_or_union(PyObject *self, PyObject *args) +{ + CTypeDescrObject *ct; + PyObject *fields, *ignored; + Py_ssize_t totalsize = -1; + int totalalignment = -1; + int sflags = 0; + int pack = 0; + + if (!PyArg_ParseTuple(args, "O!O!|Oniii:complete_struct_or_union", + &CTypeDescr_Type, &ct, + &PyList_Type, &fields, + &ignored, &totalsize, &totalalignment, &sflags, + &pack)) + return NULL; + + PyObject *res; + CFFI_LOCK(); + res = b_complete_struct_or_union_lock_held(ct, fields, totalsize, totalalignment, sflags, pack); + CFFI_UNLOCK(); + return res; } + struct funcbuilder_s { Py_ssize_t nb_bytes; char *bufferp; @@ -5597,7 +5740,7 @@ static ffi_type *fb_fill_type(struct funcbuilder_s *fb, CTypeDescrObject *ct, return &ffi_type_void; } - if (ct->ct_size <= 0) { + if (cffi_get_size(ct) <= 0) { PyErr_Format(PyExc_TypeError, ct->ct_size < 0 ? "ctype '%s' has incomplete type" : "ctype '%s' has size 0", @@ -5626,7 +5769,7 @@ static ffi_type *fb_fill_type(struct funcbuilder_s *fb, CTypeDescrObject *ct, */ if (force_lazy_struct(ct) < 0) return NULL; - if (ct->ct_flags & CT_CUSTOM_FIELD_POS) { + if (ct->ct_flags_mut & CT_CUSTOM_FIELD_POS) { /* these NotImplementedErrors may be caught and ignored until a real call is made to a function of this type */ return fb_unsupported(ct, place, @@ -5636,7 +5779,7 @@ static ffi_type *fb_fill_type(struct funcbuilder_s *fb, CTypeDescrObject *ct, } /* Another reason: __attribute__((packed)) is not supported by libffi. */ - if (ct->ct_flags & CT_WITH_PACKED_CHANGE) { + if (ct->ct_flags_mut & CT_WITH_PACKED_CHANGE) { return fb_unsupported(ct, place, "It is a 'packed' structure, with a different layout than " "expected by libffi"); @@ -5987,11 +6130,15 @@ static PyObject *new_function_type(PyObject *fargs, /* tuple */ Py_ssize_t i; const void **unique_key; - if ((fresult->ct_size < 0 && !(fresult->ct_flags & CT_VOID)) || + if ((cffi_get_size(fresult) < 0 && !(fresult->ct_flags & CT_VOID)) || (fresult->ct_flags & CT_ARRAY)) { char *msg; if (fresult->ct_flags & CT_IS_OPAQUE) msg = "result type '%s' is opaque"; + else if (cffi_check_flag(fresult->ct_unrealized_struct_or_union)) + msg = "result type '%s' is not yet initialized"; + else if (cffi_check_flag(fresult->ct_under_construction)) + msg = "result type '%s' is under construction"; else msg = "invalid result type: '%s'"; PyErr_Format(PyExc_TypeError, msg, fresult->ct_name); @@ -7655,14 +7802,13 @@ static int _testfunc23(char *p) } #if 0 /* libffi doesn't properly support complexes currently */ - /* also, MSVC might not support _Complex... */ /* if this is enabled one day, remember to also add _Complex * arguments in addition to return values. */ -static float _Complex _testfunc24(float a, float b) +static _cffi_float_complex_t _testfunc24(float a, float b) { return a + I*2.0*b; } -static double _Complex _testfunc25(double a, double b) +static _cffi_double_complex_t _testfunc25(double a, double b) { return a + I*2.0*b; } @@ -8085,6 +8231,10 @@ init_cffi_backend(void) if (m == NULL) INITERROR; +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); +#endif + if (unique_cache == NULL) { unique_cache = PyDict_New(); if (unique_cache == NULL) diff --git a/contrib/python/cffi/py3/c/cdlopen.c b/contrib/python/cffi/py3/c/cdlopen.c index 0ed319b8b59..7db7ae280fb 100644 --- a/contrib/python/cffi/py3/c/cdlopen.c +++ b/contrib/python/cffi/py3/c/cdlopen.c @@ -81,7 +81,7 @@ static PyObject *ffi_dlclose(PyObject *self, PyObject *args) static Py_ssize_t cdl_4bytes(char *src) { - /* read 4 bytes in little-endian order; return it as a signed integer */ + /* read 4 bytes in big-endian order; return it as a signed integer */ signed char *ssrc = (signed char *)src; unsigned char *usrc = (unsigned char *)src; return (ssrc[0] << 24) | (usrc[1] << 16) | (usrc[2] << 8) | usrc[3]; diff --git a/contrib/python/cffi/py3/c/cffi1_module.c b/contrib/python/cffi/py3/c/cffi1_module.c index 06a84fea37c..70d9ee8e0fe 100644 --- a/contrib/python/cffi/py3/c/cffi1_module.c +++ b/contrib/python/cffi/py3/c/cffi1_module.c @@ -130,7 +130,13 @@ static PyObject *_my_Py_InitModule(char *module_name) if (module_def == NULL) return PyErr_NoMemory(); *module_def = local_module_def; - return PyModule_Create(module_def); + PyObject *m = PyModule_Create(module_def); +# ifdef Py_GIL_DISABLED + if (m != NULL) { + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); + } +# endif + return m; #else return Py_InitModule(module_name, NULL); #endif diff --git a/contrib/python/cffi/py3/c/ffi_obj.c b/contrib/python/cffi/py3/c/ffi_obj.c index f154146621e..0c4933eeff6 100644 --- a/contrib/python/cffi/py3/c/ffi_obj.c +++ b/contrib/python/cffi/py3/c/ffi_obj.c @@ -16,6 +16,13 @@ need to call ffi.cdef() to add more information to it. */ +#ifdef MS_WIN32 +#include "misc_win32.h" +#else +#include "misc_thread_posix.h" +#endif +#include "misc_thread_common.h" + #define FFI_COMPLEXITY_OUTPUT 1200 /* xxx should grow as needed */ #define FFIObject_Check(op) PyObject_TypeCheck(op, &FFI_Type) @@ -25,7 +32,6 @@ struct FFIObject_s { PyObject_HEAD PyObject *gc_wrefs, *gc_wrefs_freelist; PyObject *init_once_cache; - struct _cffi_parse_info_s info; char ctx_is_static, ctx_is_nonempty; builder_c_t types_builder; }; @@ -33,8 +39,6 @@ struct FFIObject_s { static FFIObject *ffi_internal_new(PyTypeObject *ffitype, const struct _cffi_type_context_s *static_ctx) { - static _cffi_opcode_t internal_output[FFI_COMPLEXITY_OUTPUT]; - FFIObject *ffi; if (static_ctx != NULL) { ffi = (FFIObject *)PyObject_GC_New(FFIObject, ffitype); @@ -54,9 +58,6 @@ static FFIObject *ffi_internal_new(PyTypeObject *ffitype, ffi->gc_wrefs = NULL; ffi->gc_wrefs_freelist = NULL; ffi->init_once_cache = NULL; - ffi->info.ctx = &ffi->types_builder.ctx; - ffi->info.output = internal_output; - ffi->info.output_size = FFI_COMPLEXITY_OUTPUT; ffi->ctx_is_static = (static_ctx != NULL); ffi->ctx_is_nonempty = (static_ctx != NULL); return ffi; @@ -145,7 +146,8 @@ static PyObject *ffi_fetch_int_constant(FFIObject *ffi, const char *name, #define ACCEPT_ALL (ACCEPT_STRING | ACCEPT_CTYPE | ACCEPT_CDATA) #define CONSIDER_FN_AS_FNPTR 8 -static CTypeDescrObject *_ffi_bad_type(FFIObject *ffi, const char *input_text) +static PyObject *_ffi_bad_type(struct _cffi_parse_info_s *info, + const char *input_text) { size_t length = strlen(input_text); char *extra; @@ -155,7 +157,7 @@ static CTypeDescrObject *_ffi_bad_type(FFIObject *ffi, const char *input_text) } else { char *p; - size_t i, num_spaces = ffi->info.error_location; + size_t i, num_spaces = info->error_location; extra = alloca(length + num_spaces + 4); p = extra; *p++ = '\n'; @@ -173,7 +175,7 @@ static CTypeDescrObject *_ffi_bad_type(FFIObject *ffi, const char *input_text) *p++ = '^'; *p++ = 0; } - PyErr_Format(FFIError, "%s%s", ffi->info.error_message, extra); + PyErr_Format(FFIError, "%s%s", info->error_message, extra); return NULL; } @@ -185,16 +187,28 @@ static CTypeDescrObject *_ffi_type(FFIObject *ffi, PyObject *arg, */ if ((accept & ACCEPT_STRING) && PyText_Check(arg)) { PyObject *types_dict = ffi->types_builder.types_dict; + /* The types_dict keeps the reference alive. Items are never removed */ PyObject *x = PyDict_GetItem(types_dict, arg); if (x == NULL) { const char *input_text = PyText_AS_UTF8(arg); - int err, index = parse_c_type(&ffi->info, input_text); - if (index < 0) - return _ffi_bad_type(ffi, input_text); - - x = realize_c_type_or_func(&ffi->types_builder, - ffi->info.output, index); + struct _cffi_parse_info_s info; + info.ctx = &ffi->types_builder.ctx; + info.output_size = FFI_COMPLEXITY_OUTPUT; + info.output = PyMem_Malloc(FFI_COMPLEXITY_OUTPUT * sizeof(_cffi_opcode_t)); + if (info.output == NULL) { + PyErr_NoMemory(); + return NULL; + } + int index = parse_c_type(&info, input_text); + if (index < 0) { + x = _ffi_bad_type(&info, input_text); + } + else { + x = realize_c_type_or_func(&ffi->types_builder, + info.output, index); + } + PyMem_Free(info.output); if (x == NULL) return NULL; @@ -206,11 +220,12 @@ static CTypeDescrObject *_ffi_type(FFIObject *ffi, PyObject *arg, sure that in any case the next _ffi_type() with the same 'arg' will succeed early, in PyDict_GetItem() above. */ - err = PyDict_SetItem(types_dict, arg, x); - Py_DECREF(x); /* we know it was written in types_dict (unless out - of mem), so there is at least that ref left */ - if (err < 0) + PyObject *value = PyDict_SetDefault(types_dict, arg, x); + if (value == NULL) { return NULL; + } + Py_DECREF(x); /* we know types_dict holds a strong ref */ + x = value; } if (CTypeDescr_Check(x)) @@ -265,7 +280,11 @@ static PyObject *ffi_sizeof(FFIObject *self, PyObject *arg) CTypeDescrObject *ct = _ffi_type(self, arg, ACCEPT_ALL); if (ct == NULL) return NULL; - size = ct->ct_size; + if (ct->ct_flags & (CT_STRUCT | CT_UNION)) { + if (force_lazy_struct(ct) < 0) + return NULL; + } + size = cffi_get_size(ct); if (size < 0) { PyErr_Format(FFIError, "don't know the size of ctype '%s'", ct->ct_name); @@ -1001,16 +1020,21 @@ static PyObject *ffi_init_once(FFIObject *self, PyObject *args, PyObject *kwds) in this function */ /* atomically get or create a new dict (no GIL release) */ + Py_BEGIN_CRITICAL_SECTION(self); cache = self->init_once_cache; if (cache == NULL) { - cache = PyDict_New(); - if (cache == NULL) - return NULL; - self->init_once_cache = cache; + self->init_once_cache = cache = PyDict_New(); + } + Py_END_CRITICAL_SECTION(); + + if (cache == NULL) { + return NULL; } /* get the tuple from cache[tag], or make a new one: (False, lock) */ - tup = PyDict_GetItem(cache, tag); + if (PyDict_GetItemRef(cache, tag, &tup) < 0) { + return NULL; + } if (tup == NULL) { lock = PyThread_allocate_lock(); if (lock == NULL) @@ -1033,8 +1057,6 @@ static PyObject *ffi_init_once(FFIObject *self, PyObject *args, PyObject *kwds) Py_DECREF(x); if (tup == NULL) return NULL; - - Py_DECREF(tup); /* there is still a ref inside the dict */ } res = PyTuple_GET_ITEM(tup, 1); @@ -1042,8 +1064,10 @@ static PyObject *ffi_init_once(FFIObject *self, PyObject *args, PyObject *kwds) if (PyTuple_GET_ITEM(tup, 0) == Py_True) { /* tup == (True, result): return the result. */ + Py_DECREF(tup); return res; } + Py_DECREF(tup); /* tup == (False, lock) */ lockobj = res; diff --git a/contrib/python/cffi/py3/c/lib_obj.c b/contrib/python/cffi/py3/c/lib_obj.c index cf373699733..446b225cb5d 100644 --- a/contrib/python/cffi/py3/c/lib_obj.c +++ b/contrib/python/cffi/py3/c/lib_obj.c @@ -158,11 +158,11 @@ static PyObject *lib_build_cpython_func(LibObject *lib, random even value. But OP_FUNCTION_END is odd, so the condition below still works correctly. */ i = type_index + 1; - while (_CFFI_GETOP(opcodes[i]) != _CFFI_OP_FUNCTION_END) + while (_CFFI_GETOP(_CFFI_LOAD_OP(opcodes[i])) != _CFFI_OP_FUNCTION_END) i++; pfargs = alloca(sizeof(CTypeDescrObject *) * (i - type_index - 1)); i = type_index + 1; - while (_CFFI_GETOP(opcodes[i]) != _CFFI_OP_FUNCTION_END) { + while (_CFFI_GETOP(_CFFI_LOAD_OP(opcodes[i])) != _CFFI_OP_FUNCTION_END) { CTypeDescrObject *ct = realize_c_type(lib->l_types_builder, opcodes, i); if (ct == NULL) goto error; @@ -554,7 +554,7 @@ static PyObject *lib_getattr(LibObject *lib, PyObject *name) Py_INCREF(x); return x; } - /* this hack is for Python 3.5, and also to give a more + /* this hack is for Python 3.5, and also to give a more module-like behavior */ if (strcmp(p, "__name__") == 0) { PyErr_Clear(); diff --git a/contrib/python/cffi/py3/c/malloc_closure.h b/contrib/python/cffi/py3/c/malloc_closure.h index bebb93dc155..f51a70c6748 100644 --- a/contrib/python/cffi/py3/c/malloc_closure.h +++ b/contrib/python/cffi/py3/c/malloc_closure.h @@ -154,23 +154,38 @@ static void more_core(void) /******************************************************************/ +#ifdef Py_GIL_DISABLED +static PyMutex malloc_closure_lock; +# define MALLOC_CLOSURE_LOCK() PyMutex_Lock(&malloc_closure_lock) +# define MALLOC_CLOSURE_UNLOCK() PyMutex_Unlock(&malloc_closure_lock) +#else +# define MALLOC_CLOSURE_LOCK() ((void)0) +# define MALLOC_CLOSURE_UNLOCK() ((void)0) +#endif + /* put the item back into the free list */ static void cffi_closure_free(ffi_closure *p) { + MALLOC_CLOSURE_LOCK(); union mmaped_block *item = (union mmaped_block *)p; item->next = free_list; free_list = item; + MALLOC_CLOSURE_UNLOCK(); } /* return one item from the free list, allocating more if needed */ static ffi_closure *cffi_closure_alloc(void) { union mmaped_block *item; + MALLOC_CLOSURE_LOCK(); if (!free_list) more_core(); - if (!free_list) + if (!free_list) { + MALLOC_CLOSURE_UNLOCK(); return NULL; + } item = free_list; free_list = item->next; + MALLOC_CLOSURE_UNLOCK(); return &item->closure; } diff --git a/contrib/python/cffi/py3/c/minibuffer.h b/contrib/python/cffi/py3/c/minibuffer.h index f3f5ca15c43..d92da317ade 100644 --- a/contrib/python/cffi/py3/c/minibuffer.h +++ b/contrib/python/cffi/py3/c/minibuffer.h @@ -1,6 +1,6 @@ /* Implementation of a C object with the 'buffer' or 'memoryview' - * interface at C-level (as approriate for the version of Python we're + * interface at C-level (as appropriate for the version of Python we're * compiling for), but only a minimal but *consistent* part of the * 'buffer' interface at application level. */ @@ -140,8 +140,7 @@ static void mb_dealloc(MiniBufferObj *ob) { PyObject_GC_UnTrack(ob); - if (ob->mb_weakreflist != NULL) - PyObject_ClearWeakRefs((PyObject *)ob); + PyObject_ClearWeakRefs((PyObject *)ob); Py_XDECREF(ob->mb_keepalive); Py_TYPE(ob)->tp_free((PyObject *)ob); } diff --git a/contrib/python/cffi/py3/c/misc_thread_common.h b/contrib/python/cffi/py3/c/misc_thread_common.h index 7d29634beac..b0710acceec 100644 --- a/contrib/python/cffi/py3/c/misc_thread_common.h +++ b/contrib/python/cffi/py3/c/misc_thread_common.h @@ -1,7 +1,10 @@ +#ifndef CFFI_MISC_THREAD_COMMON_H +#define CFFI_MISC_THREAD_COMMON_H #ifndef WITH_THREAD # error "xxx no-thread configuration not tested, please report if you need that" #endif #include "pythread.h" +#include "inttypes.h" struct cffi_tls_s { @@ -22,7 +25,7 @@ struct cffi_tls_s { #endif }; -static struct cffi_tls_s *get_cffi_tls(void); /* in misc_thread_posix.h +static struct cffi_tls_s *get_cffi_tls(void); /* in misc_thread_posix.h or misc_win32.h */ @@ -312,7 +315,7 @@ static void restore_errno_only(void) /* MESS. We can't use PyThreadState_GET(), because that calls PyThreadState_Get() which fails an assert if the result is NULL. - + * in Python 2.7 and <= 3.4, the variable _PyThreadState_Current is directly available, so use that. @@ -390,3 +393,29 @@ static void gil_release(PyGILState_STATE oldstate) //fprintf(stderr, "%p: gil_release(%d), tls=%p\n", get_cffi_tls(), (int)oldstate, get_cffi_tls()); PyGILState_Release(oldstate); } + + +#if PY_VERSION_HEX <= 0x030d00b3 +# define Py_BEGIN_CRITICAL_SECTION(op) { +# define Py_END_CRITICAL_SECTION() } +#endif + +#ifdef Py_GIL_DISABLED +static PyObject _dummy = {0}; +#define cffi_check_flag(arg) cffi_atomic_load_uint8(&(arg)) +#define cffi_set_flag(arg, value) cffi_atomic_store_uint8(&(arg), (value)) +#define cffi_set_size(arg, value) cffi_atomic_store_ssize(&(arg)->ct_size, (value)) +#define cffi_get_size(arg) cffi_atomic_load_ssize(&(arg)->ct_size) +#define _CFFI_LOAD_OP(arg) cffi_atomic_load(&(arg)) +#else +#define cffi_check_flag(arg) (arg) +#define cffi_set_flag(arg, value) (arg) = (value) +#define cffi_set_size(arg, value) (arg)->ct_size = (value) +#define cffi_get_size(arg) (arg)->ct_size +#define _CFFI_LOAD_OP(arg) (arg) +#endif + +#define CFFI_LOCK() Py_BEGIN_CRITICAL_SECTION(&_dummy) +#define CFFI_UNLOCK() Py_END_CRITICAL_SECTION() + +#endif /* CFFI_MISC_THREAD_COMMON_H */ diff --git a/contrib/python/cffi/py3/c/misc_thread_posix.h b/contrib/python/cffi/py3/c/misc_thread_posix.h index bcc017737bc..a8056302c1a 100644 --- a/contrib/python/cffi/py3/c/misc_thread_posix.h +++ b/contrib/python/cffi/py3/c/misc_thread_posix.h @@ -12,6 +12,8 @@ see a given thread, and keep it alive until the thread is really shut down, using a destructor on the tls key. */ +#ifndef CFFI_MISC_THREAD_POSIX_H +#define CFFI_MISC_THREAD_POSIX_H #include <pthread.h> #include "misc_thread_common.h" @@ -47,3 +49,43 @@ static struct cffi_tls_s *get_cffi_tls(void) #define save_errno save_errno_only #define restore_errno restore_errno_only + +#ifdef Py_GIL_DISABLED +# ifndef __ATOMIC_SEQ_CST +# error "The free threading build needs atomic support" +# endif + +/* Minimal atomic support */ +static void *cffi_atomic_load(void **ptr) +{ + return __atomic_load_n(ptr, __ATOMIC_SEQ_CST); +} + +static void cffi_atomic_store(void **ptr, void *value) +{ + __atomic_store_n(ptr, value, __ATOMIC_SEQ_CST); +} + +static uint8_t cffi_atomic_load_uint8(uint8_t *ptr) +{ + return __atomic_load_n(ptr, __ATOMIC_SEQ_CST); +} + +static void cffi_atomic_store_uint8(uint8_t *ptr, uint8_t value) +{ + __atomic_store_n(ptr, value, __ATOMIC_SEQ_CST); +} + +static Py_ssize_t cffi_atomic_load_ssize(Py_ssize_t *ptr) +{ + return __atomic_load_n(ptr, __ATOMIC_SEQ_CST); +} + +static void cffi_atomic_store_ssize(Py_ssize_t *ptr, Py_ssize_t value) +{ + __atomic_store_n(ptr, value, __ATOMIC_SEQ_CST); +} + +#endif + +#endif /* CFFI_MISC_THREAD_POSIX_H */ diff --git a/contrib/python/cffi/py3/c/misc_win32.h b/contrib/python/cffi/py3/c/misc_win32.h index 0f2aaabd39f..90c63adc68c 100644 --- a/contrib/python/cffi/py3/c/misc_win32.h +++ b/contrib/python/cffi/py3/c/misc_win32.h @@ -1,3 +1,7 @@ +#ifndef CFFI_MISC_WIN32_H +#define CFFI_MISC_WIN32_H + + #include <malloc.h> /* for alloca() */ @@ -226,9 +230,71 @@ static int dlclose(void *handle) static const char *dlerror(void) { static char buf[32]; - DWORD dw = GetLastError(); + DWORD dw = GetLastError(); if (dw == 0) return NULL; sprintf(buf, "error 0x%x", (unsigned int)dw); return buf; } + +/* Minimal atomic support */ +static int cffi_atomic_compare_exchange(void **ptr, void **expected, + void *value) +{ + void *initial = _InterlockedCompareExchangePointer(ptr, value, expected); + if (initial == *expected) { + return 1; + } + *expected = initial; + return 0; +} + +static void *cffi_atomic_load(void **ptr) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(volatile void **)ptr; +#elif defined(_M_ARM64) + return (void *)__ldar64((volatile unsigned __int64 *)ptr); +#else +# error "no implementation of cffi_atomic_load" +#endif +} + +static uint8_t cffi_atomic_load_uint8(uint8_t *ptr) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(volatile uint8_t *)ptr; +#elif defined(_M_ARM64) + return (uint8_t)__ldar8((volatile uint8_t *)ptr); +#else +# error "no implementation of cffi_atomic_load_uint8" +#endif +} + +static Py_ssize_t cffi_atomic_load_ssize(Py_ssize_t *ptr) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(volatile Py_ssize_t *)ptr; +#elif defined(_M_ARM64) + return (Py_ssize_t)__ldar64((volatile unsigned __int64 *)ptr); +#else +# error "no implementation of cffi_atomic_load_ssize" +#endif +} + +static void cffi_atomic_store_ssize(Py_ssize_t *ptr, Py_ssize_t value) +{ + _InterlockedExchangePointer(ptr, value); +} + +static void cffi_atomic_store(void **ptr, void *value) +{ + _InterlockedExchangePointer(ptr, value); +} + +static void cffi_atomic_store_uint8(uint8_t *ptr, uint8_t value) +{ + _InterlockedExchange8(ptr, value); +} + +#endif /* CFFI_MISC_WIN32_H */ diff --git a/contrib/python/cffi/py3/c/realize_c_type.c b/contrib/python/cffi/py3/c/realize_c_type.c index 82629b7ef16..fbfa68ca748 100644 --- a/contrib/python/cffi/py3/c/realize_c_type.c +++ b/contrib/python/cffi/py3/c/realize_c_type.c @@ -1,4 +1,11 @@ +#ifdef MS_WIN32 +#include "misc_win32.h" +#else +#include "misc_thread_posix.h" +#endif + + typedef struct { struct _cffi_type_context_s ctx; /* inlined substructure */ PyObject *types_dict; @@ -10,7 +17,7 @@ typedef struct { static PyObject *all_primitives[_CFFI__NUM_PRIM]; -static CTypeDescrObject *g_ct_voidp, *g_ct_chararray; +static CTypeDescrObject *g_ct_voidp, *g_ct_chararray, *g_ct_int, *g_file_struct; static PyObject *build_primitive_type(int num); /* forward */ @@ -22,6 +29,7 @@ static PyObject *build_primitive_type(int num); /* forward */ static int init_global_types_dict(PyObject *ffi_type_dict) { int err; + Py_ssize_t i; PyObject *ct_void, *ct_char, *ct2, *pnull; /* XXX some leaks in case these functions fail, but well, MemoryErrors during importing an extension module are kind @@ -49,12 +57,40 @@ static int init_global_types_dict(PyObject *ffi_type_dict) return -1; g_ct_chararray = (CTypeDescrObject *)ct2; + g_ct_int = (CTypeDescrObject *)get_primitive_type(_CFFI_PRIM_INT); // 'int' + if (g_ct_int == NULL) + return -1; + + ct2 = new_struct_or_union_type("FILE", + CT_STRUCT | CT_IS_FILE); // 'FILE' + if (ct2 == NULL) + return -1; + g_file_struct = (CTypeDescrObject *)ct2; + pnull = new_simple_cdata(NULL, g_ct_voidp); if (pnull == NULL) return -1; err = PyDict_SetItemString(ffi_type_dict, "NULL", pnull); Py_DECREF(pnull); - return err; + if (err < 0) + return err; + +#ifdef Py_GIL_DISABLED + // Ensure that all primitive types are initialised to avoid race conditions + // on the first access. + for (i = 0; i < _CFFI__NUM_PRIM; i++) { + ct2 = get_primitive_type(i); + if (ct2 == NULL) + return -1; + } +#endif + + return 0; +} + +static CTypeDescrObject *_get_ct_int(void) +{ + return g_ct_int; } static void free_builder_c(builder_c_t *builder, int ctx_is_static) @@ -151,8 +187,8 @@ static PyObject *build_primitive_type(int num) "uint_fast64_t", "intmax_t", "uintmax_t", - "float _Complex", - "double _Complex", + "_cffi_float_complex_t", + "_cffi_double_complex_t", "char16_t", "char32_t", }; @@ -246,11 +282,17 @@ unexpected_fn_type(PyObject *x) CTypeDescrObject *ct = unwrap_fn_as_fnptr(x); char *text1 = ct->ct_name; char *text2 = text1 + ct->ct_name_position + 1; + size_t prefix_size = ct->ct_name_position - 2; + char *buf = PyMem_Malloc(prefix_size + 1); + if (buf == NULL) { + return NULL; + } + memcpy(buf, text1, prefix_size); + buf[prefix_size] = '\0'; assert(text2[-3] == '('); - text2[-3] = '\0'; PyErr_Format(FFIError, "the type '%s%s' is a function type, not a " - "pointer-to-function type", text1, text2); - text2[-3] = '('; + "pointer-to-function type", buf, text2); + PyMem_Free(buf); return NULL; } @@ -323,16 +365,12 @@ _realize_c_struct_or_union(builder_c_t *builder, int sindex) if (sindex == _CFFI__IO_FILE_STRUCT) { /* returns a single global cached opaque type */ - static PyObject *file_struct = NULL; - if (file_struct == NULL) - file_struct = new_struct_or_union_type("FILE", - CT_STRUCT | CT_IS_FILE); - Py_XINCREF(file_struct); - return file_struct; + Py_INCREF(g_file_struct); + return (PyObject *)g_file_struct; } s = &builder->ctx.struct_unions[sindex]; - op2 = builder->ctx.types[s->type_index]; + op2 = _CFFI_LOAD_OP(builder->ctx.types[s->type_index]); if ((((uintptr_t)op2) & 1) == 0) { x = (PyObject *)op2; /* found already in the "primary" slot */ Py_INCREF(x); @@ -342,6 +380,8 @@ _realize_c_struct_or_union(builder_c_t *builder, int sindex) if (!(s->flags & _CFFI_F_EXTERNAL)) { int flags = (s->flags & _CFFI_F_UNION) ? CT_UNION : CT_STRUCT; + int is_opaque = (s->flags & _CFFI_F_OPAQUE); + flags |= is_opaque ? CT_IS_OPAQUE : 0; char *name = alloca(8 + strlen(s->name)); _realize_name(name, (s->flags & _CFFI_F_UNION) ? "union " : "struct ", @@ -353,17 +393,17 @@ _realize_c_struct_or_union(builder_c_t *builder, int sindex) if (x == NULL) return NULL; - if (!(s->flags & _CFFI_F_OPAQUE)) { + if (!is_opaque) { assert(s->first_field_index >= 0); ct = (CTypeDescrObject *)x; ct->ct_size = (Py_ssize_t)s->size; - ct->ct_length = s->alignment; /* may be -1 */ - ct->ct_flags &= ~CT_IS_OPAQUE; - ct->ct_flags |= CT_LAZY_FIELD_LIST; + ct->ct_length = s->alignment; /* may be -1 */ + ct->ct_lazy_field_list = 1; ct->ct_extra = builder; } - else + else { assert(s->first_field_index < 0); + } } else { assert(s->first_field_index < 0); @@ -392,19 +432,29 @@ _realize_c_struct_or_union(builder_c_t *builder, int sindex) } } } + if (ct != NULL) { + cffi_set_flag(ct->ct_unrealized_struct_or_union, 0); + } /* Update the "primary" OP_STRUCT_UNION slot */ assert((((uintptr_t)x) & 1) == 0); assert(builder->ctx.types[s->type_index] == op2); Py_INCREF(x); +#ifdef Py_GIL_DISABLED + cffi_atomic_store(&builder->ctx.types[s->type_index], x); +#else builder->ctx.types[s->type_index] = x; - +#endif if (ct != NULL && s->size == (size_t)-2) { /* oops, this struct is unnamed and we couldn't generate a C expression to get its size. We have to rely on complete_struct_or_union() to compute it now. */ if (do_realize_lazy_struct(ct) < 0) { +#ifdef Py_GIL_DISABLED + cffi_atomic_store(&builder->ctx.types[s->type_index], op2); +#else builder->ctx.types[s->type_index] = op2; +#endif return NULL; } } @@ -466,7 +516,7 @@ realize_c_type_or_func_now(builder_c_t *builder, _cffi_opcode_t op, _cffi_opcode_t op2; e = &builder->ctx.enums[_CFFI_GETARG(op)]; - op2 = builder->ctx.types[e->type_index]; + op2 = _CFFI_LOAD_OP(builder->ctx.types[e->type_index]); if ((((uintptr_t)op2) & 1) == 0) { x = (PyObject *)op2; Py_INCREF(x); @@ -541,7 +591,11 @@ realize_c_type_or_func_now(builder_c_t *builder, _cffi_opcode_t op, assert((((uintptr_t)x) & 1) == 0); assert(builder->ctx.types[e->type_index] == op2); Py_INCREF(x); +#ifdef Py_GIL_DISABLED + cffi_atomic_store(&builder->ctx.types[e->type_index], x); +#else builder->ctx.types[e->type_index] = x; +#endif /* Done, leave without updating the "current" slot because it may be done already above. If not, never mind, the @@ -566,12 +620,12 @@ realize_c_type_or_func_now(builder_c_t *builder, _cffi_opcode_t op, pointer in the 'opcodes' array, and GETOP() returns a random even value. But OP_FUNCTION_END is odd, so the condition below still works correctly. */ - while (_CFFI_GETOP(opcodes[base_index + num_args]) != - _CFFI_OP_FUNCTION_END) + while (_CFFI_GETOP(_CFFI_LOAD_OP(opcodes[base_index + num_args])) + != _CFFI_OP_FUNCTION_END) num_args++; ellipsis = _CFFI_GETARG(opcodes[base_index + num_args]) & 0x01; - abi = _CFFI_GETARG(opcodes[base_index + num_args]) & 0xFE; + abi = _CFFI_GETARG(opcodes[base_index + num_args]) & 0xFE; switch (abi) { case 0: abi = FFI_DEFAULT_ABI; @@ -639,14 +693,25 @@ realize_c_type_or_func_now(builder_c_t *builder, _cffi_opcode_t op, return x; } +#ifdef Py_GIL_DISABLED +#ifdef MS_WIN32 +static __declspec(thread) int _realize_recursion_level; +#elif defined(USE__THREAD) +static __thread int _realize_recursion_level; +#else +#error "Cannot detect thread-local keyword" +#endif +#else static int _realize_recursion_level; +#endif static PyObject * -realize_c_type_or_func(builder_c_t *builder, +realize_c_type_or_func_lock_held(builder_c_t *builder, _cffi_opcode_t opcodes[], int index) { + PyObject *x; - _cffi_opcode_t op = opcodes[index]; + _cffi_opcode_t op = _CFFI_LOAD_OP(opcodes[index]); if ((((uintptr_t)op) & 1) == 0) { x = (PyObject *)op; @@ -670,8 +735,25 @@ realize_c_type_or_func(builder_c_t *builder, assert((((uintptr_t)x) & 1) == 0); assert((((uintptr_t)opcodes[index]) & 1) == 1); Py_INCREF(x); +#ifdef Py_GIL_DISABLED + cffi_atomic_store(&opcodes[index], x); +#else opcodes[index] = x; +#endif } + + return x; +} + + +static PyObject * +realize_c_type_or_func(builder_c_t *builder, + _cffi_opcode_t opcodes[], int index) +{ + PyObject *x; + CFFI_LOCK(); + x = realize_c_type_or_func_lock_held(builder, opcodes, index); + CFFI_UNLOCK(); return x; } @@ -701,19 +783,20 @@ realize_c_func_return_type(builder_c_t *builder, } } -static int do_realize_lazy_struct(CTypeDescrObject *ct) +static int do_realize_lazy_struct_lock_held(CTypeDescrObject *ct) { /* This is called by force_lazy_struct() in _cffi_backend.c */ assert(ct->ct_flags & (CT_STRUCT | CT_UNION)); - if (ct->ct_flags & CT_LAZY_FIELD_LIST) { + if (cffi_check_flag(ct->ct_lazy_field_list)) { builder_c_t *builder; char *p; int n, i, sflags; const struct _cffi_struct_union_s *s; const struct _cffi_field_s *fld; - PyObject *fields, *args, *res; + PyObject *fields, *res; + // opaque types should never set ct->ct_lazy_field_list assert(!(ct->ct_flags & CT_IS_OPAQUE)); builder = ct->ct_extra; @@ -789,19 +872,12 @@ static int do_realize_lazy_struct(CTypeDescrObject *ct) if (s->flags & _CFFI_F_PACKED) sflags |= SF_PACKED; - args = Py_BuildValue("(OOOnii)", ct, fields, Py_None, - (Py_ssize_t)s->size, - s->alignment, - sflags); - Py_DECREF(fields); - if (args == NULL) - return -1; - ct->ct_extra = NULL; - ct->ct_flags |= CT_IS_OPAQUE; - res = b_complete_struct_or_union(NULL, args); - ct->ct_flags &= ~CT_IS_OPAQUE; - Py_DECREF(args); + cffi_set_flag(ct->ct_under_construction, 1); + res = b_complete_struct_or_union_lock_held(ct, fields, s->size, s->alignment, + sflags, 0); + cffi_set_flag(ct->ct_under_construction, 0); + Py_DECREF(fields); if (res == NULL) { ct->ct_extra = builder; @@ -809,12 +885,18 @@ static int do_realize_lazy_struct(CTypeDescrObject *ct) } assert(ct->ct_stuff != NULL); - ct->ct_flags &= ~CT_LAZY_FIELD_LIST; + cffi_set_flag(ct->ct_lazy_field_list, 0); Py_DECREF(res); - return 1; - } - else { - assert(ct->ct_flags & CT_IS_OPAQUE); - return 0; } + return ct->ct_stuff != NULL; +} + + +static int do_realize_lazy_struct(CTypeDescrObject *ct) +{ + int res = 0; + CFFI_LOCK(); + res = do_realize_lazy_struct_lock_held(ct); + CFFI_UNLOCK(); + return res; } diff --git a/contrib/python/cffi/py3/cffi/__init__.py b/contrib/python/cffi/py3/cffi/__init__.py index 2e35a38c9ce..c99ec3d4817 100644 --- a/contrib/python/cffi/py3/cffi/__init__.py +++ b/contrib/python/cffi/py3/cffi/__init__.py @@ -5,8 +5,8 @@ from .api import FFI from .error import CDefError, FFIError, VerificationError, VerificationMissing from .error import PkgConfigError -__version__ = "1.17.1" -__version_info__ = (1, 17, 1) +__version__ = "2.0.0" +__version_info__ = (2, 0, 0) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/contrib/python/cffi/py3/cffi/_embedding.h b/contrib/python/cffi/py3/cffi/_embedding.h index 94d8b30a9e3..64c04f67cac 100644 --- a/contrib/python/cffi/py3/cffi/_embedding.h +++ b/contrib/python/cffi/py3/cffi/_embedding.h @@ -225,7 +225,7 @@ static int _cffi_initialize_python(void) if (f != NULL && f != Py_None) { PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME - "\ncompiled with cffi version: 1.17.1" + "\ncompiled with cffi version: 2.0.0" "\n_cffi_backend module: ", f); modules = PyImport_GetModuleDict(); mod = PyDict_GetItemString(modules, "_cffi_backend"); diff --git a/contrib/python/cffi/py3/cffi/cparser.py b/contrib/python/cffi/py3/cffi/cparser.py index eee83caffbd..dd590d8743d 100644 --- a/contrib/python/cffi/py3/cffi/cparser.py +++ b/contrib/python/cffi/py3/cffi/cparser.py @@ -59,7 +59,7 @@ def _workaround_for_old_pycparser(csource): # for "char***(*const)". This means we can't tell the difference # afterwards. But "char(*const(***))" gives us the right syntax # tree. The issue only occurs if there are several stars in - # sequence with no parenthesis inbetween, just possibly qualifiers. + # sequence with no parenthesis in between, just possibly qualifiers. # Attempt to fix it by adding some parentheses in the source: each # time we see "* const" or "* const *", we add an opening # parenthesis before each star---the hard part is figuring out where diff --git a/contrib/python/cffi/py3/cffi/recompiler.py b/contrib/python/cffi/py3/cffi/recompiler.py index c236807fe81..25bf9e91d9e 100644 --- a/contrib/python/cffi/py3/cffi/recompiler.py +++ b/contrib/python/cffi/py3/cffi/recompiler.py @@ -1,4 +1,4 @@ -import os, sys, io +import io, os, sys, sysconfig from . import ffiplatform, model from .error import VerificationError from .cffi_opcode import * @@ -7,9 +7,9 @@ VERSION_BASE = 0x2601 VERSION_EMBEDDED = 0x2701 VERSION_CHAR16CHAR32 = 0x2801 -USE_LIMITED_API = (sys.platform != 'win32' or sys.version_info < (3, 0) or - sys.version_info >= (3, 5)) - +USE_LIMITED_API = ((sys.platform != 'win32' or sys.version_info < (3, 0) or + sys.version_info >= (3, 5)) and + not sysconfig.get_config_var("Py_GIL_DISABLED")) # free-threaded doesn't yet support limited API class GlobalExpr: def __init__(self, name, address, type_op, size=0, check_value=0): @@ -951,7 +951,7 @@ class Recompiler: if cname is None or fbitsize >= 0: offset = '(size_t)-1' elif named_ptr is not None: - offset = '((char *)&((%s)4096)->%s) - (char *)4096' % ( + offset = '(size_t)(((char *)&((%s)4096)->%s) - (char *)4096)' % ( named_ptr.name, fldname) else: offset = 'offsetof(%s, %s)' % (tp.get_c_name(''), fldname) diff --git a/contrib/python/cffi/py3/cffi/setuptools_ext.py b/contrib/python/cffi/py3/cffi/setuptools_ext.py index 681b49d7ad9..5cdd246fb4d 100644 --- a/contrib/python/cffi/py3/cffi/setuptools_ext.py +++ b/contrib/python/cffi/py3/cffi/setuptools_ext.py @@ -1,5 +1,6 @@ import os import sys +import sysconfig try: basestring @@ -87,6 +88,7 @@ def _set_py_limited_api(Extension, kwds): Recently (2020) we started shipping only >= 3.5 wheels, though. So we'll give it another try and set py_limited_api on Windows >= 3.5. """ + from cffi._shimmed_dist_utils import log from cffi import recompiler if ('py_limited_api' not in kwds and not hasattr(sys, 'gettotalrefcount') @@ -101,6 +103,17 @@ def _set_py_limited_api(Extension, kwds): # try to set 'py_limited_api' anyway. At worst, we get a # warning. kwds['py_limited_api'] = True + + if sysconfig.get_config_var("Py_GIL_DISABLED"): + if kwds.get('py_limited_api'): + log.info("Ignoring py_limited_api=True for free-threaded build.") + + kwds['py_limited_api'] = False + + if kwds.get('py_limited_api') is False: + # avoid setting Py_LIMITED_API if py_limited_api=False + # which _cffi_include.h does unless _CFFI_NO_LIMITED_API is defined + kwds.setdefault("define_macros", []).append(("_CFFI_NO_LIMITED_API", None)) return kwds def _add_c_module(dist, ffi, module_name, source, source_extension, kwds): diff --git a/contrib/python/cffi/py3/cffi/vengine_cpy.py b/contrib/python/cffi/py3/cffi/vengine_cpy.py index eb0b6f70e49..02e6a471d04 100644 --- a/contrib/python/cffi/py3/cffi/vengine_cpy.py +++ b/contrib/python/cffi/py3/cffi/vengine_cpy.py @@ -124,6 +124,9 @@ class VCPythonEngine(object): prnt(' Py_DECREF(lib);') prnt(' return NULL;') prnt(' }') + prnt('#if Py_GIL_DISABLED') + prnt(' PyUnstable_Module_SetGIL(lib, Py_MOD_GIL_NOT_USED);') + prnt('#endif') prnt(' return lib;') prnt('}') prnt() diff --git a/contrib/python/cffi/py3/ya.make b/contrib/python/cffi/py3/ya.make index 90ddf448de4..d2eb8e11ad8 100644 --- a/contrib/python/cffi/py3/ya.make +++ b/contrib/python/cffi/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(1.17.1) +VERSION(2.0.0) LICENSE(MIT) diff --git a/contrib/python/pytest-timeout/py2/.dist-info/METADATA b/contrib/python/pytest-timeout/py2/.dist-info/METADATA new file mode 100644 index 00000000000..6c492ef6c3c --- /dev/null +++ b/contrib/python/pytest-timeout/py2/.dist-info/METADATA @@ -0,0 +1,364 @@ +Metadata-Version: 2.1 +Name: pytest-timeout +Version: 1.4.2 +Summary: py.test plugin to abort hanging tests +Home-page: http://github.com/pytest-dev/pytest-timeout/ +Author: Floris Bruynooghe +Author-email: [email protected] +License: MIT +Platform: UNKNOWN +Classifier: Development Status :: 5 - Production/Stable +Classifier: Environment :: Console +Classifier: Environment :: Plugins +Classifier: Intended Audience :: Developers +Classifier: License :: DFSG approved +Classifier: License :: OSI Approved :: MIT License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Topic :: Software Development :: Testing +Requires-Dist: pytest (>=3.6.0) + +============== +pytest-timeout +============== + +|python| |version| |anaconda| |ci| + +.. |version| image:: https://img.shields.io/pypi/v/pytest-timeout.svg + :target: https://pypi.python.org/pypi/pytest-timeout + +.. |anaconda| image:: https://img.shields.io/conda/vn/conda-forge/pytest-timeout.svg + :target: https://anaconda.org/conda-forge/pytest-timeout + +.. |ci| image:: https://github.com/pytest-dev/pytest-timeout/workflows/build/badge.svg + :target: https://github.com/pytest-dev/pytest-timeout/actions + +.. |python| image:: https://img.shields.io/pypi/pyversions/pytest-timeout.svg + :target: https://pypi.python.org/pypi/pytest-timeout/ + +This is a plugin which will terminate tests after a certain timeout, +assuming the test session isn't being debugged. When aborting a test +it will show a stack dump of all threads running at the time. +This is useful when running tests under a continuous +integration server or simply if you don't know why the test suite hangs. + +.. note:: + + The way this plugin detects whether or not a debugging session is + active is by checking if a trace function is set and if one is, it + check to see if the module it belongs to is present in a set of + known debugging frameworks modules OR if pytest itself drops you + into a pbd session. + +.. note:: + + While by default on POSIX systems pytest will continue to execute + the tests after a test has timed out this is not always possible. + Often the only sure way to interrupt a hanging test is by + terminating the entire process. As this is a hard termination + (``os._exit()``) it will result in no teardown, JUnit XML output + etc. But the plugin will ensure you will have the debugging output + on stderr nevertheless, which is the most important part at this + stage. See below for detailed information on the timeout methods + and their side-effects. + +The pytest-timeout plugin has been tested on python 2.7 or higher, +including 3.X, pypy and pypy3. See tox.ini for currently tested +versions. + + +Usage +===== + +Install is as simple as e.g.:: + + pip install pytest-timeout + +Now you can run tests using a timeout, in seconds, after which they +will be terminated:: + + pytest --timeout=300 + +Alternatively you can mark individual tests as having a timeout:: + + @pytest.mark.timeout(60) + def test_foo(): + pass + +By default the plugin will not time out any tests, you must specify a +valid timeout for the plugin to interrupt long-running tests. A +timeout is always specified as a number of seconds, and can be +defined in a number of ways, from low to high priority: + +1. You can set a global timeout in the `pytest configuration file`__ + using the ``timeout`` option. E.g.:: + + [pytest] + timeout = 300 + +2. The ``PYTEST_TIMEOUT`` environment variable sets a global timeout + overriding a possible value in the configuration file. + +3. The ``--timeout`` command line option sets a global timeout + overriding both the environment variable and configuration option. + +4. Using the ``timeout`` marker_ on test items you can specify + timeouts on a per-item basis:: + + @pytest.mark.timeout(300) + def test_foo(): + pass + +__ https://docs.pytest.org/en/latest/reference.html#ini-options-ref + +.. _marker: https://docs.pytest.org/en/latest/mark.html + +Setting a timeout to 0 seconds disables the timeout, so if you have a +global timeout set you can still disable the timeout by using the +mark. + +Timeout Methods +=============== + +Interrupting tests which hang is not always as simple and can be +platform dependent. Furthermore some methods of terminating a test +might conflict with the code under test itself. The pytest-timeout +plugin tries to pick the most suitable method based on your platform, +but occasionally you may need to specify a specific timeout method +explicitly. + + If a timeout method does not work your safest bet is to use the + *thread* method. + +thread +------ + +This is the surest and most portable method. It is also the default +on systems not supporting the *signal* method. For each test item the +pytest-timeout plugin starts a timer thread which will terminate the +whole process after the specified timeout. When a test item finishes +this timer thread is cancelled and the test run continues. + +The downsides of this method are that there is a relatively large +overhead for running each test and that test runs are not completed. +This means that other pytest features, like e.g. JUnit XML output or +fixture teardown, will not function normally. The second issue might +be alleviated by using the ``--boxed`` option of the pytest-xdist_ +plugin. + +.. _pytest-xdist: https://pypi.org/project/pytest-xdist/ + +The benefit of this method is that it will always work. Furthermore +it will still provide you debugging information by printing the stacks +of all the threads in the application to stderr. + +signal +------ + +If the system supports the SIGALRM signal the *signal* method will be +used by default. This method schedules an alarm when the test item +starts and cancels it when it finishes. If the alarm expires during +the test the signal handler will dump the stack of any other threads +running to stderr and use ``pytest.fail()`` to interrupt the test. + +The benefit of this method is that the pytest process is not +terminated and the test run can complete normally. + +The main issue to look out for with this method is that it may +interfere with the code under test. If the code under test uses +SIGALRM itself things will go wrong and you will have to choose the +*thread* method. + +Specifying the Timeout Method +----------------------------- + +The timeout method can be specified by using the ``timeout_method`` +option in the `pytest configuration file`__, the ``--timeout_method`` +command line parameter or the ``timeout`` marker_. Simply set their +value to the string ``thread`` or ``signal`` to override the default +method. On a marker this is done using the ``method`` keyword:: + + @pytest.mark.timeout(method='thread') + def test_foo(): + pass + +__ https://docs.pytest.org/en/latest/reference.html#ini-options-ref + +.. _marker: https://docs.pytest.org/en/latest/mark.html + +The ``timeout`` Marker API +========================== + +The full signature of the timeout marker is:: + + pytest.mark.timeout(timeout=0, method=DEFAULT_METHOD) + +You can use either positional or keyword arguments for both the +timeout and the method. Neither needs to be present. + +See the marker api documentation_ and examples_ for the various ways +markers can be applied to test items. + +.. _documentation: https://docs.pytest.org/en/latest/mark.html + +.. _examples: https://docs.pytest.org/en/latest/example/markers.html#marking-whole-classes-or-modules + + +Timeouts in Fixture Teardown +============================ + +The plugin will happily terminate timeouts in the finalisers of +fixtures. The timeout specified applies to the entire process of +setting up fixtures, running the tests and finalising the fixtures. +However when a timeout occurs in a fixture finaliser and the test +suite continues, i.e. the signal method is used, it must be realised +that subsequent fixtures which need to be finalised might not have +been executed, which could result in a broken test-suite anyway. In +case of doubt the thread method which terminates the entire process +might result in clearer output. + + +Changelog +========= + +1.4.2 +----- + +- Fix compatibility when run with pytest pre-releases, thanks + Bruno Oliveira, +- Fix detection of third-party debuggers, thanks Bruno Oliveira. + +1.4.1 +----- + +- Fix coverage compatibility which was broken by 1.4.0. + +1.4.0 +----- + +- Better detection of when we are debugging, thanks Mattwmaster58. + +1.3.4 +----- + +- Give the threads a name to help debugging, thanks Thomas Grainger. +- Changed location to https://github.com/pytest-dev/pytest-timeout + because bitbucket is dropping mercurial support. Thanks Thomas + Grainger and Bruno Oliveira. + +1.3.3 +----- + +- Fix support for pytest >= 3.10. + +1.3.2 +----- + +- This changelog was ommitted for the 1.3.2 release and was added + afterwards. Apologies for the confusion. +- Fix pytest 3.7.3 compatibility. The capture API had changed + slightly and this needed fixing. Thanks Bruno Oliveira for the + contribution. + +1.3.1 +----- + +- Fix deprecation warning on Python 3.6. Thanks Mickaƫl Schoentgen +- Create a valid tag for the release. Somehow this didn't happen for + 1.3.0, that tag points to a non-existing commit. + +1.3.0 +----- + +- Make it possible to only run the timeout timer on the test function + and not the whole fixture setup + test + teardown duration. Thanks + Pedro Algarvio for the work! +- Use the new pytest marker API, Thanks Pedro Algarvio for the work! + +1.2.1 +----- + +- Fix for pytest 3.3, thanks Bruno Oliveira. +- Update supported python versions: + - Add CPython 3.6. + - Drop CPyhon 2.6 (as did pytest 3.3) + - Drop CPyhon 3.3 + - Drop CPyhon 3.4 + +1.2.0 +----- + +* Allow using floats as timeout instead of only integers, thanks Tom + Myers. + +1.1.0 +----- + +* Report (default) timeout duration in header, thanks Holger Krekel. + +1.0.0 +----- + +* Bump version to 1.0 to commit to semantic versioning. +* Fix issue #12: Now compatible with pytest 2.8, thanks Holger Krekel. +* No longer test with pexpect on py26 as it is no longer supported +* Require pytest 2.8 and use new hookimpl decorator + +0.5 +--- + +* Timeouts will no longer be triggered when inside an interactive pdb + session started by ``pytest.set_trace()`` / ``pdb.set_trace()``. + +* Add pypy3 environment to tox.ini. + +* Transfer repository to pytest-dev team account. + +0.4 +--- + +* Support timeouts happening in (session scoped) finalizers. + +* Change command line option --timeout_method into --timeout-method + for consistency with pytest + +0.3 +--- + +* Added the PYTEST_TIMEOUT environment variable as a way of specifying + the timeout (closes issue #2). + +* More flexible marker argument parsing: you can now specify the + method using a positional argument. + +* The plugin is now enabled by default. There is no longer a need to + specify ``timeout=0`` in the configuration file or on the command + line simply so that a marker would work. + + +0.2 +--- + +* Add a marker to modify the timeout delay using a @pytest.timeout(N) + syntax, thanks to Laurant Brack for the initial code. + +* Allow the timeout marker to select the timeout method using the + ``method`` keyword argument. + +* Rename the --nosigalrm option to --method=thread to future proof + support for eventlet and gevent. Thanks to Ronny Pfannschmidt for + the hint. + +* Add ``timeout`` and ``timeout_method`` items to the configuration + file so you can enable and configure the plugin using the ini file. + Thanks to Holger Krekel and Ronny Pfannschmidt for the hints. + +* Tested (and fixed) for python 2.6, 2.7 and 3.2. + + diff --git a/contrib/python/pytest-timeout/py2/.dist-info/entry_points.txt b/contrib/python/pytest-timeout/py2/.dist-info/entry_points.txt new file mode 100644 index 00000000000..09aa9536310 --- /dev/null +++ b/contrib/python/pytest-timeout/py2/.dist-info/entry_points.txt @@ -0,0 +1,3 @@ +[pytest11] +timeout = pytest_timeout + diff --git a/contrib/python/pytest-timeout/py2/.dist-info/top_level.txt b/contrib/python/pytest-timeout/py2/.dist-info/top_level.txt new file mode 100644 index 00000000000..34eec7a2ee6 --- /dev/null +++ b/contrib/python/pytest-timeout/py2/.dist-info/top_level.txt @@ -0,0 +1 @@ +pytest_timeout diff --git a/contrib/python/pytest-timeout/py2/LICENSE b/contrib/python/pytest-timeout/py2/LICENSE new file mode 100644 index 00000000000..e968817fc44 --- /dev/null +++ b/contrib/python/pytest-timeout/py2/LICENSE @@ -0,0 +1,22 @@ +The MIT License + +Copyright (C) 2012, 2014 Floris Bruynooghe + + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/contrib/python/pytest-timeout/py2/README.rst b/contrib/python/pytest-timeout/py2/README.rst new file mode 100644 index 00000000000..2113b6a05ed --- /dev/null +++ b/contrib/python/pytest-timeout/py2/README.rst @@ -0,0 +1,336 @@ +============== +pytest-timeout +============== + +|python| |version| |anaconda| |ci| + +.. |version| image:: https://img.shields.io/pypi/v/pytest-timeout.svg + :target: https://pypi.python.org/pypi/pytest-timeout + +.. |anaconda| image:: https://img.shields.io/conda/vn/conda-forge/pytest-timeout.svg + :target: https://anaconda.org/conda-forge/pytest-timeout + +.. |ci| image:: https://github.com/pytest-dev/pytest-timeout/workflows/build/badge.svg + :target: https://github.com/pytest-dev/pytest-timeout/actions + +.. |python| image:: https://img.shields.io/pypi/pyversions/pytest-timeout.svg + :target: https://pypi.python.org/pypi/pytest-timeout/ + +This is a plugin which will terminate tests after a certain timeout, +assuming the test session isn't being debugged. When aborting a test +it will show a stack dump of all threads running at the time. +This is useful when running tests under a continuous +integration server or simply if you don't know why the test suite hangs. + +.. note:: + + The way this plugin detects whether or not a debugging session is + active is by checking if a trace function is set and if one is, it + check to see if the module it belongs to is present in a set of + known debugging frameworks modules OR if pytest itself drops you + into a pbd session. + +.. note:: + + While by default on POSIX systems pytest will continue to execute + the tests after a test has timed out this is not always possible. + Often the only sure way to interrupt a hanging test is by + terminating the entire process. As this is a hard termination + (``os._exit()``) it will result in no teardown, JUnit XML output + etc. But the plugin will ensure you will have the debugging output + on stderr nevertheless, which is the most important part at this + stage. See below for detailed information on the timeout methods + and their side-effects. + +The pytest-timeout plugin has been tested on python 2.7 or higher, +including 3.X, pypy and pypy3. See tox.ini for currently tested +versions. + + +Usage +===== + +Install is as simple as e.g.:: + + pip install pytest-timeout + +Now you can run tests using a timeout, in seconds, after which they +will be terminated:: + + pytest --timeout=300 + +Alternatively you can mark individual tests as having a timeout:: + + @pytest.mark.timeout(60) + def test_foo(): + pass + +By default the plugin will not time out any tests, you must specify a +valid timeout for the plugin to interrupt long-running tests. A +timeout is always specified as a number of seconds, and can be +defined in a number of ways, from low to high priority: + +1. You can set a global timeout in the `pytest configuration file`__ + using the ``timeout`` option. E.g.:: + + [pytest] + timeout = 300 + +2. The ``PYTEST_TIMEOUT`` environment variable sets a global timeout + overriding a possible value in the configuration file. + +3. The ``--timeout`` command line option sets a global timeout + overriding both the environment variable and configuration option. + +4. Using the ``timeout`` marker_ on test items you can specify + timeouts on a per-item basis:: + + @pytest.mark.timeout(300) + def test_foo(): + pass + +__ https://docs.pytest.org/en/latest/reference.html#ini-options-ref + +.. _marker: https://docs.pytest.org/en/latest/mark.html + +Setting a timeout to 0 seconds disables the timeout, so if you have a +global timeout set you can still disable the timeout by using the +mark. + +Timeout Methods +=============== + +Interrupting tests which hang is not always as simple and can be +platform dependent. Furthermore some methods of terminating a test +might conflict with the code under test itself. The pytest-timeout +plugin tries to pick the most suitable method based on your platform, +but occasionally you may need to specify a specific timeout method +explicitly. + + If a timeout method does not work your safest bet is to use the + *thread* method. + +thread +------ + +This is the surest and most portable method. It is also the default +on systems not supporting the *signal* method. For each test item the +pytest-timeout plugin starts a timer thread which will terminate the +whole process after the specified timeout. When a test item finishes +this timer thread is cancelled and the test run continues. + +The downsides of this method are that there is a relatively large +overhead for running each test and that test runs are not completed. +This means that other pytest features, like e.g. JUnit XML output or +fixture teardown, will not function normally. The second issue might +be alleviated by using the ``--boxed`` option of the pytest-xdist_ +plugin. + +.. _pytest-xdist: https://pypi.org/project/pytest-xdist/ + +The benefit of this method is that it will always work. Furthermore +it will still provide you debugging information by printing the stacks +of all the threads in the application to stderr. + +signal +------ + +If the system supports the SIGALRM signal the *signal* method will be +used by default. This method schedules an alarm when the test item +starts and cancels it when it finishes. If the alarm expires during +the test the signal handler will dump the stack of any other threads +running to stderr and use ``pytest.fail()`` to interrupt the test. + +The benefit of this method is that the pytest process is not +terminated and the test run can complete normally. + +The main issue to look out for with this method is that it may +interfere with the code under test. If the code under test uses +SIGALRM itself things will go wrong and you will have to choose the +*thread* method. + +Specifying the Timeout Method +----------------------------- + +The timeout method can be specified by using the ``timeout_method`` +option in the `pytest configuration file`__, the ``--timeout_method`` +command line parameter or the ``timeout`` marker_. Simply set their +value to the string ``thread`` or ``signal`` to override the default +method. On a marker this is done using the ``method`` keyword:: + + @pytest.mark.timeout(method='thread') + def test_foo(): + pass + +__ https://docs.pytest.org/en/latest/reference.html#ini-options-ref + +.. _marker: https://docs.pytest.org/en/latest/mark.html + +The ``timeout`` Marker API +========================== + +The full signature of the timeout marker is:: + + pytest.mark.timeout(timeout=0, method=DEFAULT_METHOD) + +You can use either positional or keyword arguments for both the +timeout and the method. Neither needs to be present. + +See the marker api documentation_ and examples_ for the various ways +markers can be applied to test items. + +.. _documentation: https://docs.pytest.org/en/latest/mark.html + +.. _examples: https://docs.pytest.org/en/latest/example/markers.html#marking-whole-classes-or-modules + + +Timeouts in Fixture Teardown +============================ + +The plugin will happily terminate timeouts in the finalisers of +fixtures. The timeout specified applies to the entire process of +setting up fixtures, running the tests and finalising the fixtures. +However when a timeout occurs in a fixture finaliser and the test +suite continues, i.e. the signal method is used, it must be realised +that subsequent fixtures which need to be finalised might not have +been executed, which could result in a broken test-suite anyway. In +case of doubt the thread method which terminates the entire process +might result in clearer output. + + +Changelog +========= + +1.4.2 +----- + +- Fix compatibility when run with pytest pre-releases, thanks + Bruno Oliveira, +- Fix detection of third-party debuggers, thanks Bruno Oliveira. + +1.4.1 +----- + +- Fix coverage compatibility which was broken by 1.4.0. + +1.4.0 +----- + +- Better detection of when we are debugging, thanks Mattwmaster58. + +1.3.4 +----- + +- Give the threads a name to help debugging, thanks Thomas Grainger. +- Changed location to https://github.com/pytest-dev/pytest-timeout + because bitbucket is dropping mercurial support. Thanks Thomas + Grainger and Bruno Oliveira. + +1.3.3 +----- + +- Fix support for pytest >= 3.10. + +1.3.2 +----- + +- This changelog was ommitted for the 1.3.2 release and was added + afterwards. Apologies for the confusion. +- Fix pytest 3.7.3 compatibility. The capture API had changed + slightly and this needed fixing. Thanks Bruno Oliveira for the + contribution. + +1.3.1 +----- + +- Fix deprecation warning on Python 3.6. Thanks Mickaƫl Schoentgen +- Create a valid tag for the release. Somehow this didn't happen for + 1.3.0, that tag points to a non-existing commit. + +1.3.0 +----- + +- Make it possible to only run the timeout timer on the test function + and not the whole fixture setup + test + teardown duration. Thanks + Pedro Algarvio for the work! +- Use the new pytest marker API, Thanks Pedro Algarvio for the work! + +1.2.1 +----- + +- Fix for pytest 3.3, thanks Bruno Oliveira. +- Update supported python versions: + - Add CPython 3.6. + - Drop CPyhon 2.6 (as did pytest 3.3) + - Drop CPyhon 3.3 + - Drop CPyhon 3.4 + +1.2.0 +----- + +* Allow using floats as timeout instead of only integers, thanks Tom + Myers. + +1.1.0 +----- + +* Report (default) timeout duration in header, thanks Holger Krekel. + +1.0.0 +----- + +* Bump version to 1.0 to commit to semantic versioning. +* Fix issue #12: Now compatible with pytest 2.8, thanks Holger Krekel. +* No longer test with pexpect on py26 as it is no longer supported +* Require pytest 2.8 and use new hookimpl decorator + +0.5 +--- + +* Timeouts will no longer be triggered when inside an interactive pdb + session started by ``pytest.set_trace()`` / ``pdb.set_trace()``. + +* Add pypy3 environment to tox.ini. + +* Transfer repository to pytest-dev team account. + +0.4 +--- + +* Support timeouts happening in (session scoped) finalizers. + +* Change command line option --timeout_method into --timeout-method + for consistency with pytest + +0.3 +--- + +* Added the PYTEST_TIMEOUT environment variable as a way of specifying + the timeout (closes issue #2). + +* More flexible marker argument parsing: you can now specify the + method using a positional argument. + +* The plugin is now enabled by default. There is no longer a need to + specify ``timeout=0`` in the configuration file or on the command + line simply so that a marker would work. + + +0.2 +--- + +* Add a marker to modify the timeout delay using a @pytest.timeout(N) + syntax, thanks to Laurant Brack for the initial code. + +* Allow the timeout marker to select the timeout method using the + ``method`` keyword argument. + +* Rename the --nosigalrm option to --method=thread to future proof + support for eventlet and gevent. Thanks to Ronny Pfannschmidt for + the hint. + +* Add ``timeout`` and ``timeout_method`` items to the configuration + file so you can enable and configure the plugin using the ini file. + Thanks to Holger Krekel and Ronny Pfannschmidt for the hints. + +* Tested (and fixed) for python 2.6, 2.7 and 3.2. diff --git a/contrib/python/pytest-timeout/py2/pytest_timeout.py b/contrib/python/pytest-timeout/py2/pytest_timeout.py new file mode 100644 index 00000000000..34252c06352 --- /dev/null +++ b/contrib/python/pytest-timeout/py2/pytest_timeout.py @@ -0,0 +1,459 @@ +"""Timeout for tests to stop hanging testruns. + +This plugin will dump the stack and terminate the test. This can be +useful when running tests on a continuous integration server. + +If the platform supports SIGALRM this is used to raise an exception in +the test, otherwise os._exit(1) is used. +""" +import inspect +import os +import signal +import sys +import threading +import traceback +from collections import namedtuple +from distutils.version import LooseVersion + +import py +import pytest + + +HAVE_SIGALRM = hasattr(signal, "SIGALRM") +if HAVE_SIGALRM: + DEFAULT_METHOD = "signal" +else: + DEFAULT_METHOD = "thread" +TIMEOUT_DESC = """ +Timeout in seconds before dumping the stacks. Default is 0 which +means no timeout. +""".strip() +METHOD_DESC = """ +Timeout mechanism to use. 'signal' uses SIGALRM if available, +'thread' uses a timer thread. The default is to use 'signal' and fall +back to 'thread'. +""".strip() +FUNC_ONLY_DESC = """ +When set to True, defers the timeout evaluation to only the test +function body, ignoring the time it takes when evaluating any fixtures +used in the test. +""".strip() + +# bdb covers pdb, ipdb, and possibly others +# pydevd covers PyCharm, VSCode, and possibly others +KNOWN_DEBUGGING_MODULES = {"pydevd", "bdb"} +Settings = namedtuple("Settings", ["timeout", "method", "func_only"]) + + +def pytest_addoption(parser): + """Add options to control the timeout plugin.""" + group = parser.getgroup( + "timeout", + "Interrupt test run and dump stacks of all threads after a test times out", + ) + group.addoption("--timeout", type=float, help=TIMEOUT_DESC) + group.addoption( + "--timeout_method", + action="store", + choices=["signal", "thread"], + help="Deprecated, use --timeout-method", + ) + group.addoption( + "--timeout-method", + dest="timeout_method", + action="store", + choices=["signal", "thread"], + help=METHOD_DESC, + ) + parser.addini("timeout", TIMEOUT_DESC) + parser.addini("timeout_method", METHOD_DESC) + parser.addini("timeout_func_only", FUNC_ONLY_DESC, type="bool") + + +def pytest_configure(config): + """Register the marker so it shows up in --markers output.""" + config.addinivalue_line( + "markers", + "timeout(timeout, method=None, func_only=False): Set a timeout, timeout " + "method and func_only evaluation on just one test item. The first " + "argument, *timeout*, is the timeout in seconds while the keyword, " + "*method*, takes the same values as the --timeout_method option. The " + "*func_only* keyword, when set to True, defers the timeout evaluation " + "to only the test function body, ignoring the time it takes when " + "evaluating any fixtures used in the test.", + ) + + settings = get_env_settings(config) + config._env_timeout = settings.timeout + config._env_timeout_method = settings.method + config._env_timeout_func_only = settings.func_only + + [email protected](hookwrapper=True) +def pytest_runtest_protocol(item): + """Hook in timeouts to the runtest protocol. + + If the timeout is set on the entire test, including setup and + teardown, then this hook installs the timeout. Otherwise + pytest_runtest_call is used. + """ + func_only = get_func_only_setting(item) + if func_only is False: + timeout_setup(item) + yield + if func_only is False: + timeout_teardown(item) + + [email protected](hookwrapper=True) +def pytest_runtest_call(item): + """Hook in timeouts to the test function call only. + + If the timeout is set on only the test function this hook installs + the timeout, otherwise pytest_runtest_protocol is used. + """ + func_only = get_func_only_setting(item) + if func_only is True: + timeout_setup(item) + yield + if func_only is True: + timeout_teardown(item) + + [email protected](tryfirst=True) +def pytest_report_header(config): + """Add timeout config to pytest header.""" + if config._env_timeout: + return [ + "timeout: %ss\ntimeout method: %s\ntimeout func_only: %s" + % ( + config._env_timeout, + config._env_timeout_method, + config._env_timeout_func_only, + ) + ] + + [email protected](tryfirst=True) +def pytest_exception_interact(node): + """Stop the timeout when pytest enters pdb in post-mortem mode.""" + timeout_teardown(node) + + +def pytest_enter_pdb(): + """Stop the timeouts when we entered pdb. + + This stops timeouts from triggering when pytest's builting pdb + support notices we entered pdb. + """ + # Since pdb.set_trace happens outside of any pytest control, we don't have + # any pytest ``item`` here, so we cannot use timeout_teardown. Thus, we + # need another way to signify that the timeout should not be performed. + global SUPPRESS_TIMEOUT + SUPPRESS_TIMEOUT = True + + +def is_debugging(trace_func=None): + """Detect if a debugging session is in progress. + + This looks at both pytest's builtin pdb support as well as + externally installed debuggers using some heuristics. + + This is done by checking if either of the following conditions is + true: + + 1. Examines the trace function to see if the module it originates + from is in KNOWN_DEBUGGING_MODULES. + 2. Check is SUPPRESS_TIMEOUT is set to True. + + :param trace_func: the current trace function, if not given will use + sys.gettrace(). Used to unit-test this function. + """ + global SUPPRESS_TIMEOUT, KNOWN_DEBUGGING_MODULES + if SUPPRESS_TIMEOUT: + return True + if trace_func is None: + trace_func = sys.gettrace() + if trace_func and inspect.getmodule(trace_func): + parts = inspect.getmodule(trace_func).__name__.split(".") + for name in KNOWN_DEBUGGING_MODULES: + if name in parts: + return True + return False + + +SUPPRESS_TIMEOUT = False + + +def timeout_setup(item): + """Setup up a timeout trigger and handler.""" + params = get_params(item) + if params.timeout is None or params.timeout <= 0: + return + if params.method == "signal": + + def handler(signum, frame): + __tracebackhide__ = True + timeout_sigalrm(item, params.timeout) + + def cancel(): + signal.setitimer(signal.ITIMER_REAL, 0) + signal.signal(signal.SIGALRM, signal.SIG_DFL) + + item.cancel_timeout = cancel + signal.signal(signal.SIGALRM, handler) + signal.setitimer(signal.ITIMER_REAL, params.timeout) + elif params.method == "thread": + timer = threading.Timer(params.timeout, timeout_timer, (item, params.timeout)) + timer.name = "%s %s" % (__name__, item.nodeid) + + def cancel(): + timer.cancel() + timer.join() + + item.cancel_timeout = cancel + timer.start() + + +def timeout_teardown(item): + """Cancel the timeout trigger if it was set.""" + # When skipping is raised from a pytest_runtest_setup function + # (as is the case when using the pytest.mark.skipif marker) we + # may be called without our setup counterpart having been + # called. + cancel = getattr(item, "cancel_timeout", None) + if cancel: + cancel() + + +def get_env_settings(config): + """Return the configured timeout settings. + + This looks up the settings in the environment and config file. + """ + timeout = config.getvalue("timeout") + if timeout is None: + timeout = _validate_timeout( + os.environ.get("PYTEST_TIMEOUT"), "PYTEST_TIMEOUT environment variable" + ) + if timeout is None: + ini = config.getini("timeout") + if ini: + timeout = _validate_timeout(ini, "config file") + + method = config.getvalue("timeout_method") + if method is None: + ini = config.getini("timeout_method") + if ini: + method = _validate_method(ini, "config file") + if method is None: + method = DEFAULT_METHOD + + func_only = config.getini("timeout_func_only") + if func_only == []: + # No value set + func_only = None + if func_only is not None: + func_only = _validate_func_only(func_only, "config file") + return Settings(timeout, method, func_only or False) + + +def get_func_only_setting(item): + """Return the func_only setting for an item.""" + func_only = None + marker = item.get_closest_marker("timeout") + if marker: + settings = get_params(item, marker=marker) + func_only = _validate_func_only(settings.func_only, "marker") + if func_only is None: + func_only = item.config._env_timeout_func_only + if func_only is None: + func_only = False + return func_only + + +def get_params(item, marker=None): + """Return (timeout, method) for an item.""" + timeout = method = func_only = None + if not marker: + marker = item.get_closest_marker("timeout") + if marker is not None: + settings = _parse_marker(item.get_closest_marker(name="timeout")) + timeout = _validate_timeout(settings.timeout, "marker") + method = _validate_method(settings.method, "marker") + func_only = _validate_func_only(settings.func_only, "marker") + if timeout is None: + timeout = item.config._env_timeout + if method is None: + method = item.config._env_timeout_method + if func_only is None: + func_only = item.config._env_timeout_func_only + return Settings(timeout, method, func_only) + + +def _parse_marker(marker): + """Return (timeout, method) tuple from marker. + + Either could be None. The values are not interpreted, so + could still be bogus and even the wrong type. + """ + if not marker.args and not marker.kwargs: + raise TypeError("Timeout marker must have at least one argument") + timeout = method = func_only = NOTSET = object() + for kw, val in marker.kwargs.items(): + if kw == "timeout": + timeout = val + elif kw == "method": + method = val + elif kw == "func_only": + func_only = val + else: + raise TypeError("Invalid keyword argument for timeout marker: %s" % kw) + if len(marker.args) >= 1 and timeout is not NOTSET: + raise TypeError("Multiple values for timeout argument of timeout marker") + elif len(marker.args) >= 1: + timeout = marker.args[0] + if len(marker.args) >= 2 and method is not NOTSET: + raise TypeError("Multiple values for method argument of timeout marker") + elif len(marker.args) >= 2: + method = marker.args[1] + if len(marker.args) > 2: + raise TypeError("Too many arguments for timeout marker") + if timeout is NOTSET: + timeout = None + if method is NOTSET: + method = None + if func_only is NOTSET: + func_only = None + return Settings(timeout, method, func_only) + + +def _validate_timeout(timeout, where): + if timeout is None: + return None + try: + return float(timeout) + except ValueError: + raise ValueError("Invalid timeout %s from %s" % (timeout, where)) + + +def _validate_method(method, where): + if method is None: + return None + if method not in ["signal", "thread"]: + raise ValueError("Invalid method %s from %s" % (method, where)) + return method + + +def _validate_func_only(func_only, where): + if func_only is None: + return False + if not isinstance(func_only, bool): + raise ValueError("Invalid func_only value %s from %s" % (func_only, where)) + return func_only + + +def timeout_sigalrm(item, timeout): + """Dump stack of threads and raise an exception. + + This will output the stacks of any threads other then the + current to stderr and then raise an AssertionError, thus + terminating the test. + """ + if is_debugging(): + return + __tracebackhide__ = True + nthreads = len(threading.enumerate()) + if nthreads > 1: + write_title("Timeout", sep="+") + dump_stacks() + if nthreads > 1: + write_title("Timeout", sep="+") + pytest.fail("Timeout >%ss" % timeout) + + +def timeout_timer(item, timeout): + """Dump stack of threads and call os._exit(). + + This disables the capturemanager and dumps stdout and stderr. + Then the stacks are dumped and os._exit(1) is called. + """ + if is_debugging(): + return + try: + capman = item.config.pluginmanager.getplugin("capturemanager") + if capman: + pytest_version = LooseVersion(pytest.__version__) + if pytest_version >= LooseVersion("3.7.3"): + capman.suspend_global_capture(item) + stdout, stderr = capman.read_global_capture() + else: + stdout, stderr = capman.suspend_global_capture(item) + else: + stdout, stderr = None, None + write_title("Timeout", sep="+") + caplog = item.config.pluginmanager.getplugin("_capturelog") + if caplog and hasattr(item, "capturelog_handler"): + log = item.capturelog_handler.stream.getvalue() + if log: + write_title("Captured log") + write(log) + if stdout: + write_title("Captured stdout") + write(stdout) + if stderr: + write_title("Captured stderr") + write(stderr) + dump_stacks() + write_title("Timeout", sep="+") + except Exception: + traceback.print_exc() + finally: + sys.stdout.flush() + sys.stderr.flush() + os._exit(1) + + +def dump_stacks(): + """Dump the stacks of all threads except the current thread.""" + current_ident = threading.current_thread().ident + for thread_ident, frame in sys._current_frames().items(): + if thread_ident == current_ident: + continue + for t in threading.enumerate(): + if t.ident == thread_ident: + thread_name = t.name + break + else: + thread_name = "<unknown>" + write_title("Stack of %s (%s)" % (thread_name, thread_ident)) + write("".join(traceback.format_stack(frame))) + + +def write_title(title, stream=None, sep="~"): + """Write a section title. + + If *stream* is None sys.stderr will be used, *sep* is used to + draw the line. + """ + if stream is None: + stream = sys.stderr + width = py.io.get_terminal_width() + fill = int((width - len(title) - 2) / 2) + line = " ".join([sep * fill, title, sep * fill]) + if len(line) < width: + line += sep * (width - len(line)) + stream.write("\n" + line + "\n") + + +def write(text, stream=None): + """Write text to stream. + + Pretty stupid really, only here for symetry with .write_title(). + """ + if stream is None: + stream = sys.stderr + stream.write(text) diff --git a/contrib/python/pytest-timeout/py2/ya.make b/contrib/python/pytest-timeout/py2/ya.make new file mode 100644 index 00000000000..59611f15246 --- /dev/null +++ b/contrib/python/pytest-timeout/py2/ya.make @@ -0,0 +1,27 @@ +# Generated by devtools/yamaker (pypi). + +PY2_LIBRARY() + +VERSION(1.4.2) + +LICENSE(MIT) + +PEERDIR( + contrib/python/pytest +) + +NO_LINT() + +PY_SRCS( + TOP_LEVEL + pytest_timeout.py +) + +RESOURCE_FILES( + PREFIX contrib/python/pytest-timeout/py2/ + .dist-info/METADATA + .dist-info/entry_points.txt + .dist-info/top_level.txt +) + +END() diff --git a/contrib/python/pytest-timeout/py3/.dist-info/METADATA b/contrib/python/pytest-timeout/py3/.dist-info/METADATA new file mode 100644 index 00000000000..90ae6ab7865 --- /dev/null +++ b/contrib/python/pytest-timeout/py3/.dist-info/METADATA @@ -0,0 +1,623 @@ +Metadata-Version: 2.4 +Name: pytest-timeout +Version: 2.4.0 +Summary: pytest plugin to abort hanging tests +Home-page: https://github.com/pytest-dev/pytest-timeout +Author: Floris Bruynooghe +Author-email: [email protected] +License: MIT +Classifier: Development Status :: 5 - Production/Stable +Classifier: Environment :: Console +Classifier: Environment :: Plugins +Classifier: Intended Audience :: Developers +Classifier: License :: DFSG approved +Classifier: License :: OSI Approved :: MIT License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3 :: Only +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 +Classifier: Topic :: Software Development :: Testing +Classifier: Framework :: Pytest +Requires-Python: >=3.7 +Description-Content-Type: text/x-rst +License-File: LICENSE +Requires-Dist: pytest>=7.0.0 +Dynamic: license-file + +============== +pytest-timeout +============== + +|python| |version| |anaconda| |ci| |pre-commit| + +.. |version| image:: https://img.shields.io/pypi/v/pytest-timeout.svg + :target: https://pypi.python.org/pypi/pytest-timeout + +.. |anaconda| image:: https://img.shields.io/conda/vn/conda-forge/pytest-timeout.svg + :target: https://anaconda.org/conda-forge/pytest-timeout + +.. |ci| image:: https://github.com/pytest-dev/pytest-timeout/workflows/build/badge.svg + :target: https://github.com/pytest-dev/pytest-timeout/actions + +.. |python| image:: https://img.shields.io/pypi/pyversions/pytest-timeout.svg + :target: https://pypi.python.org/pypi/pytest-timeout/ + +.. |pre-commit| image:: https://results.pre-commit.ci/badge/github/pytest-dev/pytest-timeout/master.svg + :target: https://results.pre-commit.ci/latest/github/pytest-dev/pytest-timeout/master + + +.. warning:: + + Please read this README carefully and only use this plugin if you + understand the consequences. This plugin is designed to catch + excessively long test durations like deadlocked or hanging tests, + it is not designed for precise timings or performance regressions. + Remember your test suite should aim to be **fast**, with timeouts + being a last resort, not an expected failure mode. + +This plugin will time each test and terminate it when it takes too +long. Termination may or may not be graceful, please see below, but +when aborting it will show a stack dump of all thread running at the +time. This is useful when running tests under a continuous +integration server or simply if you don't know why the test suite +hangs. + +.. note:: + + While by default on POSIX systems pytest will continue to execute + the tests after a test has timed out this is not always possible. + Often the only sure way to interrupt a hanging test is by + terminating the entire process. As this is a hard termination + (``os._exit()``) it will result in no teardown, JUnit XML output + etc. But the plugin will ensure you will have the debugging output + on stderr nevertheless, which is the most important part at this + stage. See below for detailed information on the timeout methods + and their side-effects. + +The pytest-timeout plugin has been tested on Python 3.6 and higher, +including PyPy3. See tox.ini for currently tested versions. + + +Usage +===== + +Install is as simple as e.g.:: + + pip install pytest-timeout + +Now you can run the test suite while setting a timeout in seconds, any +individual test which takes longer than the given duration will be +terminated:: + + pytest --timeout=300 + +Furthermore you can also use a decorator to set the timeout for an +individual test. If combined with the ``--timeout`` flag this will +override the timeout for this individual test: + +.. code:: python + + @pytest.mark.timeout(60) + def test_foo(): + pass + +By default the plugin will not time out any tests, you must specify a +valid timeout for the plugin to interrupt long-running tests. A +timeout is always specified as a number of seconds, and can be +defined in a number of ways, from low to high priority: + +1. You can set a global timeout in the `pytest configuration file`__ + using the ``timeout`` option. E.g.: + + .. code:: ini + + [pytest] + timeout = 300 + +2. The ``PYTEST_TIMEOUT`` environment variable sets a global timeout + overriding a possible value in the configuration file. + +3. The ``--timeout`` command line option sets a global timeout + overriding both the environment variable and configuration option. + +4. Using the ``timeout`` marker_ on test items you can specify + timeouts on a per-item basis: + + .. code:: python + + @pytest.mark.timeout(300) + def test_foo(): + pass + +__ https://docs.pytest.org/en/latest/reference.html#ini-options-ref + +.. _marker: https://docs.pytest.org/en/latest/mark.html + +Setting a timeout to 0 seconds disables the timeout, so if you have a +global timeout set you can still disable the timeout by using the +mark. + +Timeout Methods +=============== + +Interrupting tests which hang is not always as simple and can be +platform dependent. Furthermore some methods of terminating a test +might conflict with the code under test itself. The pytest-timeout +plugin tries to pick the most suitable method based on your platform, +but occasionally you may need to specify a specific timeout method +explicitly. + + If a timeout method does not work your safest bet is to use the + *thread* method. + +thread +------ + +This is the surest and most portable method. It is also the default +on systems not supporting the *signal* method. For each test item the +pytest-timeout plugin starts a timer thread which will terminate the +whole process after the specified timeout. When a test item finishes +this timer thread is cancelled and the test run continues. + +The downsides of this method are that there is a relatively large +overhead for running each test and that test runs are not completed. +This means that other pytest features, like e.g. JUnit XML output or +fixture teardown, will not function normally. The second issue might +be alleviated by using the ``--forked`` option of the pytest-forked_ +plugin. + +.. _pytest-forked: https://pypi.org/project/pytest-forked/ + +The benefit of this method is that it will always work. Furthermore +it will still provide you debugging information by printing the stacks +of all the threads in the application to stderr. + +signal +------ + +If the system supports the SIGALRM signal the *signal* method will be +used by default. This method schedules an alarm when the test item +starts and cancels the alarm when the test finishes. If the alarm expires +during the test the signal handler will dump the stack of any other threads +running to stderr and use ``pytest.fail()`` to interrupt the test. + +The benefit of this method is that the pytest process is not +terminated and the test run can complete normally. + +The main issue to look out for with this method is that it may +interfere with the code under test. If the code under test uses +SIGALRM itself things will go wrong and you will have to choose the +*thread* method. + +Specifying the Timeout Method +----------------------------- + +The timeout method can be specified by using the ``timeout_method`` +option in the `pytest configuration file`__, the ``--timeout_method`` +command line parameter or the ``timeout`` marker_. Simply set their +value to the string ``thread`` or ``signal`` to override the default +method. On a marker this is done using the ``method`` keyword: + +.. code:: python + + @pytest.mark.timeout(method="thread") + def test_foo(): + pass + +__ https://docs.pytest.org/en/latest/reference.html#ini-options-ref + +.. _marker: https://docs.pytest.org/en/latest/mark.html + +The ``timeout`` Marker API +========================== + +The full signature of the timeout marker is: + +.. code:: python + + pytest.mark.timeout(timeout=0, method=DEFAULT_METHOD) + +You can use either positional or keyword arguments for both the +timeout and the method. Neither needs to be present. + +See the marker api documentation_ and examples_ for the various ways +markers can be applied to test items. + +.. _documentation: https://docs.pytest.org/en/latest/mark.html + +.. _examples: https://docs.pytest.org/en/latest/example/markers.html#marking-whole-classes-or-modules + + +Timeouts in Fixture Teardown +============================ + +The plugin will happily terminate timeouts in the finalisers of +fixtures. The timeout specified applies to the entire process of +setting up fixtures, running the tests and finalising the fixtures. +However when a timeout occurs in a fixture finaliser and the test +suite continues, i.e. the signal method is used, it must be realised +that subsequent fixtures which need to be finalised might not have +been executed, which could result in a broken test-suite anyway. In +case of doubt the thread method which terminates the entire process +might result in clearer output. + +Avoiding timeouts in Fixtures +============================= + +The timeout applies to the entire test including any fixtures which +may need to be setup or torn down for the test (the exact affected +fixtures depends on which scope they are and whether other tests will +still use the same fixture). If the timeouts really are too short to +include fixture durations, firstly make the timeouts larger ;). If +this really isn't an option a ``timeout_func_only`` boolean setting +exists which can be set in the pytest ini configuration file, as +documented in ``pytest --help``. + +For the decorated function, a decorator will override +``timeout_func_only = true`` in the pytest ini file to the default +value. If you need to keep this option for a decorated test, you +must specify the option explicitly again: + +.. code:: python + + @pytest.mark.timeout(60, func_only=True) + def test_foo(): + pass + + +Debugger Detection +================== + +This plugin tries to avoid triggering the timeout when a debugger is +detected. This is mostly a convenience so you do not need to remember +to disable the timeout when interactively debugging. + +The way this plugin detects whether or not a debugging session is +active is by checking if a trace function is set and if one is, it +check to see if the module it belongs to is present in a set of known +debugging frameworks modules OR if pytest itself drops you into a pdb +session using ``--pdb`` or similar. + +This functionality can be disabled with the ``--disable-debugger-detection`` flag +or the corresponding ``timeout_disable_debugger_detection`` ini setting / environment +variable. + + +Extending pytest-timeout with plugins +===================================== + +``pytest-timeout`` provides two hooks that can be used for extending the tool. These +hooks are used for setting the timeout timer and cancelling it if the timeout is not +reached. + +For example, ``pytest-asyncio`` can provide asyncio-specific code that generates better +traceback and points on timed out ``await`` instead of the running loop iteration. + +See `pytest hooks documentation +<https://docs.pytest.org/en/latest/how-to/writing_hook_functions.html>`_ for more info +regarding to use custom hooks. + +``pytest_timeout_set_timer`` +---------------------------- + +.. code:: python + + @pytest.hookspec(firstresult=True) + def pytest_timeout_set_timer(item, settings): + """Called at timeout setup. + + 'item' is a pytest node to setup timeout for. + + 'settings' is Settings namedtuple (described below). + + Can be overridden by plugins for alternative timeout implementation strategies. + """ + + +``Settings`` +------------ + +When ``pytest_timeout_set_timer`` is called, ``settings`` argument is passed. + +The argument has ``Settings`` namedtuple type with the following fields: + ++-----------+-------+--------------------------------------------------------+ +|Attribute | Index | Value | ++===========+=======+========================================================+ +| timeout | 0 | timeout in seconds or ``None`` for no timeout | ++-----------+-------+--------------------------------------------------------+ +| method | 1 | Method mechanism, | +| | | ``'signal'`` and ``'thread'`` are supported by default | ++-----------+-------+--------------------------------------------------------+ +| func_only | 2 | Apply timeout to test function only if ``True``, | +| | | wrap all test function and its fixtures otherwise | ++-----------+-------+--------------------------------------------------------+ + +``pytest_timeout_cancel_timer`` +------------------------------- + +.. code:: python + + @pytest.hookspec(firstresult=True) + def pytest_timeout_cancel_timer(item): + """Called at timeout teardown. + + 'item' is a pytest node which was used for timeout setup. + + Can be overridden by plugins for alternative timeout implementation strategies. + """ + +``is_debugging`` +---------------- + +When the timeout occurs, user can open the debugger session. In this case, the timeout +should be discarded. A custom hook can check this case by calling ``is_debugging()`` +function: + +.. code:: python + + import pytest + import pytest_timeout + + + def on_timeout(): + if pytest_timeout.is_debugging(): + return + pytest.fail("+++ Timeout +++") + + + +Session Timeout +=============== + +The above mentioned timeouts are all per test function. +The "per test function" timeouts will stop an individual test +from taking too long. We may also want to limit the time of the entire +set of tests running in one session. A session all of the tests +that will be run with one invokation of pytest. + +A session timeout is set with `--session-timeout` and is in seconds. + +The following example shows a session timeout of 10 minutes (600 seconds):: + + pytest --session-timeout=600 + +You can also set the session timeout the pytest configuration file using the ``session_timeout`` option: + + .. code:: ini + + [pytest] + session_timeout = 600 + +Cooperative timeouts +-------------------- + +Session timeouts are cooperative timeouts. pytest-timeout checks the +session time at the end of each test function, and stops further tests +from running if the session timeout is exceeded. The session will +results in a test failure if this occurs. + +In particular this means if a test does not finish of itself, it will +only be interrupted if there is also a function timeout set. A +session timeout is not enough to ensure that a test-suite is +guaranteed to finish. + +Combining session and function timeouts +--------------------------------------- + +It works fine to combine both session and function timeouts. In fact +when using a session timeout it is recommended to also provide a +function timeout. + +For example, to limit test functions to 5 seconds and the full session +to 100 seconds:: + + pytest --timeout=5 --session-timeout=100 + + +Changelog +========= + +2.4.0 +----- + +- Detect debuggers registered with sys.monitoring. Thanks Rich + Chiodo. +- Make it clear the timeout message comes from pytest-timeout. Thanks + Pedro Brochado. + +2.3.1 +----- + +- Fixup some build errors, mostly README syntax which stopped twine + from uploading. + +2.3.0 +----- + +- Fix debugger detection for recent VSCode, this compiles pydevd using + cython which is now correctly detected. Thanks Adrian Gielniewski. +- Switched to using Pytest's ``TerminalReporter`` instead of writing + directly to ``sys.{stdout,stderr}``. + This change also switches all output from ``sys.stderr`` to ``sys.stdout``. + Thanks Pedro Algarvio. +- Pytest 7.0.0 is now the minimum supported version. Thanks Pedro Algarvio. +- Add ``--session-timeout`` option and ``session_timeout`` setting. + Thanks Brian Okken. + +2.2.0 +----- + +- Add ``--timeout-disable-debugger-detection`` flag, thanks + Michael Peters + +2.1.0 +----- + +- Get terminal width from shutil instead of deprecated py, thanks + Andrew Svetlov. +- Add an API for extending ``pytest-timeout`` functionality + with third-party plugins, thanks Andrew Svetlov. + +2.0.2 +----- + +- Fix debugger detection on OSX, thanks Alexander Pacha. + +2.0.1 +----- + +- Fix Python 2 removal, thanks Nicusor Picatureanu. + +2.0.0 +----- + +- Increase pytest requirement to >=5.0.0. Thanks Dominic Davis-Foster. +- Use thread timeout method when plugin is not called from main + thread to avoid crash. +- Fix pycharm debugger detection so timeouts are not triggered during + debugger usage. +- Dropped support for Python 2, minimum pytest version supported is 5.0.0. + +1.4.2 +----- + +- Fix compatibility when run with pytest pre-releases, thanks + Bruno Oliveira, +- Fix detection of third-party debuggers, thanks Bruno Oliveira. + +1.4.1 +----- + +- Fix coverage compatibility which was broken by 1.4.0. + +1.4.0 +----- + +- Better detection of when we are debugging, thanks Mattwmaster58. + +1.3.4 +----- + +- Give the threads a name to help debugging, thanks Thomas Grainger. +- Changed location to https://github.com/pytest-dev/pytest-timeout + because bitbucket is dropping mercurial support. Thanks Thomas + Grainger and Bruno Oliveira. + +1.3.3 +----- + +- Fix support for pytest >= 3.10. + +1.3.2 +----- + +- This changelog was omitted for the 1.3.2 release and was added + afterwards. Apologies for the confusion. +- Fix pytest 3.7.3 compatibility. The capture API had changed + slightly and this needed fixing. Thanks Bruno Oliveira for the + contribution. + +1.3.1 +----- + +- Fix deprecation warning on Python 3.6. Thanks Mickaƫl Schoentgen +- Create a valid tag for the release. Somehow this didn't happen for + 1.3.0, that tag points to a non-existing commit. + +1.3.0 +----- + +- Make it possible to only run the timeout timer on the test function + and not the whole fixture setup + test + teardown duration. Thanks + Pedro Algarvio for the work! +- Use the new pytest marker API, Thanks Pedro Algarvio for the work! + +1.2.1 +----- + +- Fix for pytest 3.3, thanks Bruno Oliveira. +- Update supported python versions: + - Add CPython 3.6. + - Drop CPyhon 2.6 (as did pytest 3.3) + - Drop CPyhon 3.3 + - Drop CPyhon 3.4 + +1.2.0 +----- + +* Allow using floats as timeout instead of only integers, thanks Tom + Myers. + +1.1.0 +----- + +* Report (default) timeout duration in header, thanks Holger Krekel. + +1.0.0 +----- + +* Bump version to 1.0 to commit to semantic versioning. +* Fix issue #12: Now compatible with pytest 2.8, thanks Holger Krekel. +* No longer test with pexpect on py26 as it is no longer supported +* Require pytest 2.8 and use new hookimpl decorator + +0.5 +--- + +* Timeouts will no longer be triggered when inside an interactive pdb + session started by ``pytest.set_trace()`` / ``pdb.set_trace()``. + +* Add pypy3 environment to tox.ini. + +* Transfer repository to pytest-dev team account. + +0.4 +--- + +* Support timeouts happening in (session scoped) finalizers. + +* Change command line option --timeout_method into --timeout-method + for consistency with pytest + +0.3 +--- + +* Added the PYTEST_TIMEOUT environment variable as a way of specifying + the timeout (closes issue #2). + +* More flexible marker argument parsing: you can now specify the + method using a positional argument. + +* The plugin is now enabled by default. There is no longer a need to + specify ``timeout=0`` in the configuration file or on the command + line simply so that a marker would work. + + +0.2 +--- + +* Add a marker to modify the timeout delay using a @pytest.timeout(N) + syntax, thanks to Laurant Brack for the initial code. + +* Allow the timeout marker to select the timeout method using the + ``method`` keyword argument. + +* Rename the --nosigalrm option to --method=thread to future proof + support for eventlet and gevent. Thanks to Ronny Pfannschmidt for + the hint. + +* Add ``timeout`` and ``timeout_method`` items to the configuration + file so you can enable and configure the plugin using the ini file. + Thanks to Holger Krekel and Ronny Pfannschmidt for the hints. + +* Tested (and fixed) for python 2.6, 2.7 and 3.2. diff --git a/contrib/python/pytest-timeout/py3/.dist-info/entry_points.txt b/contrib/python/pytest-timeout/py3/.dist-info/entry_points.txt new file mode 100644 index 00000000000..f7ac24a8d49 --- /dev/null +++ b/contrib/python/pytest-timeout/py3/.dist-info/entry_points.txt @@ -0,0 +1,2 @@ +[pytest11] +timeout = pytest_timeout diff --git a/contrib/python/pytest-timeout/py3/.dist-info/top_level.txt b/contrib/python/pytest-timeout/py3/.dist-info/top_level.txt new file mode 100644 index 00000000000..34eec7a2ee6 --- /dev/null +++ b/contrib/python/pytest-timeout/py3/.dist-info/top_level.txt @@ -0,0 +1 @@ +pytest_timeout diff --git a/contrib/python/pytest-timeout/py3/LICENSE b/contrib/python/pytest-timeout/py3/LICENSE new file mode 100644 index 00000000000..e968817fc44 --- /dev/null +++ b/contrib/python/pytest-timeout/py3/LICENSE @@ -0,0 +1,22 @@ +The MIT License + +Copyright (C) 2012, 2014 Floris Bruynooghe + + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/contrib/python/pytest-timeout/py3/README.rst b/contrib/python/pytest-timeout/py3/README.rst new file mode 100644 index 00000000000..a426d6f0799 --- /dev/null +++ b/contrib/python/pytest-timeout/py3/README.rst @@ -0,0 +1,588 @@ +============== +pytest-timeout +============== + +|python| |version| |anaconda| |ci| |pre-commit| + +.. |version| image:: https://img.shields.io/pypi/v/pytest-timeout.svg + :target: https://pypi.python.org/pypi/pytest-timeout + +.. |anaconda| image:: https://img.shields.io/conda/vn/conda-forge/pytest-timeout.svg + :target: https://anaconda.org/conda-forge/pytest-timeout + +.. |ci| image:: https://github.com/pytest-dev/pytest-timeout/workflows/build/badge.svg + :target: https://github.com/pytest-dev/pytest-timeout/actions + +.. |python| image:: https://img.shields.io/pypi/pyversions/pytest-timeout.svg + :target: https://pypi.python.org/pypi/pytest-timeout/ + +.. |pre-commit| image:: https://results.pre-commit.ci/badge/github/pytest-dev/pytest-timeout/master.svg + :target: https://results.pre-commit.ci/latest/github/pytest-dev/pytest-timeout/master + + +.. warning:: + + Please read this README carefully and only use this plugin if you + understand the consequences. This plugin is designed to catch + excessively long test durations like deadlocked or hanging tests, + it is not designed for precise timings or performance regressions. + Remember your test suite should aim to be **fast**, with timeouts + being a last resort, not an expected failure mode. + +This plugin will time each test and terminate it when it takes too +long. Termination may or may not be graceful, please see below, but +when aborting it will show a stack dump of all thread running at the +time. This is useful when running tests under a continuous +integration server or simply if you don't know why the test suite +hangs. + +.. note:: + + While by default on POSIX systems pytest will continue to execute + the tests after a test has timed out this is not always possible. + Often the only sure way to interrupt a hanging test is by + terminating the entire process. As this is a hard termination + (``os._exit()``) it will result in no teardown, JUnit XML output + etc. But the plugin will ensure you will have the debugging output + on stderr nevertheless, which is the most important part at this + stage. See below for detailed information on the timeout methods + and their side-effects. + +The pytest-timeout plugin has been tested on Python 3.6 and higher, +including PyPy3. See tox.ini for currently tested versions. + + +Usage +===== + +Install is as simple as e.g.:: + + pip install pytest-timeout + +Now you can run the test suite while setting a timeout in seconds, any +individual test which takes longer than the given duration will be +terminated:: + + pytest --timeout=300 + +Furthermore you can also use a decorator to set the timeout for an +individual test. If combined with the ``--timeout`` flag this will +override the timeout for this individual test: + +.. code:: python + + @pytest.mark.timeout(60) + def test_foo(): + pass + +By default the plugin will not time out any tests, you must specify a +valid timeout for the plugin to interrupt long-running tests. A +timeout is always specified as a number of seconds, and can be +defined in a number of ways, from low to high priority: + +1. You can set a global timeout in the `pytest configuration file`__ + using the ``timeout`` option. E.g.: + + .. code:: ini + + [pytest] + timeout = 300 + +2. The ``PYTEST_TIMEOUT`` environment variable sets a global timeout + overriding a possible value in the configuration file. + +3. The ``--timeout`` command line option sets a global timeout + overriding both the environment variable and configuration option. + +4. Using the ``timeout`` marker_ on test items you can specify + timeouts on a per-item basis: + + .. code:: python + + @pytest.mark.timeout(300) + def test_foo(): + pass + +__ https://docs.pytest.org/en/latest/reference.html#ini-options-ref + +.. _marker: https://docs.pytest.org/en/latest/mark.html + +Setting a timeout to 0 seconds disables the timeout, so if you have a +global timeout set you can still disable the timeout by using the +mark. + +Timeout Methods +=============== + +Interrupting tests which hang is not always as simple and can be +platform dependent. Furthermore some methods of terminating a test +might conflict with the code under test itself. The pytest-timeout +plugin tries to pick the most suitable method based on your platform, +but occasionally you may need to specify a specific timeout method +explicitly. + + If a timeout method does not work your safest bet is to use the + *thread* method. + +thread +------ + +This is the surest and most portable method. It is also the default +on systems not supporting the *signal* method. For each test item the +pytest-timeout plugin starts a timer thread which will terminate the +whole process after the specified timeout. When a test item finishes +this timer thread is cancelled and the test run continues. + +The downsides of this method are that there is a relatively large +overhead for running each test and that test runs are not completed. +This means that other pytest features, like e.g. JUnit XML output or +fixture teardown, will not function normally. The second issue might +be alleviated by using the ``--forked`` option of the pytest-forked_ +plugin. + +.. _pytest-forked: https://pypi.org/project/pytest-forked/ + +The benefit of this method is that it will always work. Furthermore +it will still provide you debugging information by printing the stacks +of all the threads in the application to stderr. + +signal +------ + +If the system supports the SIGALRM signal the *signal* method will be +used by default. This method schedules an alarm when the test item +starts and cancels the alarm when the test finishes. If the alarm expires +during the test the signal handler will dump the stack of any other threads +running to stderr and use ``pytest.fail()`` to interrupt the test. + +The benefit of this method is that the pytest process is not +terminated and the test run can complete normally. + +The main issue to look out for with this method is that it may +interfere with the code under test. If the code under test uses +SIGALRM itself things will go wrong and you will have to choose the +*thread* method. + +Specifying the Timeout Method +----------------------------- + +The timeout method can be specified by using the ``timeout_method`` +option in the `pytest configuration file`__, the ``--timeout_method`` +command line parameter or the ``timeout`` marker_. Simply set their +value to the string ``thread`` or ``signal`` to override the default +method. On a marker this is done using the ``method`` keyword: + +.. code:: python + + @pytest.mark.timeout(method="thread") + def test_foo(): + pass + +__ https://docs.pytest.org/en/latest/reference.html#ini-options-ref + +.. _marker: https://docs.pytest.org/en/latest/mark.html + +The ``timeout`` Marker API +========================== + +The full signature of the timeout marker is: + +.. code:: python + + pytest.mark.timeout(timeout=0, method=DEFAULT_METHOD) + +You can use either positional or keyword arguments for both the +timeout and the method. Neither needs to be present. + +See the marker api documentation_ and examples_ for the various ways +markers can be applied to test items. + +.. _documentation: https://docs.pytest.org/en/latest/mark.html + +.. _examples: https://docs.pytest.org/en/latest/example/markers.html#marking-whole-classes-or-modules + + +Timeouts in Fixture Teardown +============================ + +The plugin will happily terminate timeouts in the finalisers of +fixtures. The timeout specified applies to the entire process of +setting up fixtures, running the tests and finalising the fixtures. +However when a timeout occurs in a fixture finaliser and the test +suite continues, i.e. the signal method is used, it must be realised +that subsequent fixtures which need to be finalised might not have +been executed, which could result in a broken test-suite anyway. In +case of doubt the thread method which terminates the entire process +might result in clearer output. + +Avoiding timeouts in Fixtures +============================= + +The timeout applies to the entire test including any fixtures which +may need to be setup or torn down for the test (the exact affected +fixtures depends on which scope they are and whether other tests will +still use the same fixture). If the timeouts really are too short to +include fixture durations, firstly make the timeouts larger ;). If +this really isn't an option a ``timeout_func_only`` boolean setting +exists which can be set in the pytest ini configuration file, as +documented in ``pytest --help``. + +For the decorated function, a decorator will override +``timeout_func_only = true`` in the pytest ini file to the default +value. If you need to keep this option for a decorated test, you +must specify the option explicitly again: + +.. code:: python + + @pytest.mark.timeout(60, func_only=True) + def test_foo(): + pass + + +Debugger Detection +================== + +This plugin tries to avoid triggering the timeout when a debugger is +detected. This is mostly a convenience so you do not need to remember +to disable the timeout when interactively debugging. + +The way this plugin detects whether or not a debugging session is +active is by checking if a trace function is set and if one is, it +check to see if the module it belongs to is present in a set of known +debugging frameworks modules OR if pytest itself drops you into a pdb +session using ``--pdb`` or similar. + +This functionality can be disabled with the ``--disable-debugger-detection`` flag +or the corresponding ``timeout_disable_debugger_detection`` ini setting / environment +variable. + + +Extending pytest-timeout with plugins +===================================== + +``pytest-timeout`` provides two hooks that can be used for extending the tool. These +hooks are used for setting the timeout timer and cancelling it if the timeout is not +reached. + +For example, ``pytest-asyncio`` can provide asyncio-specific code that generates better +traceback and points on timed out ``await`` instead of the running loop iteration. + +See `pytest hooks documentation +<https://docs.pytest.org/en/latest/how-to/writing_hook_functions.html>`_ for more info +regarding to use custom hooks. + +``pytest_timeout_set_timer`` +---------------------------- + +.. code:: python + + @pytest.hookspec(firstresult=True) + def pytest_timeout_set_timer(item, settings): + """Called at timeout setup. + + 'item' is a pytest node to setup timeout for. + + 'settings' is Settings namedtuple (described below). + + Can be overridden by plugins for alternative timeout implementation strategies. + """ + + +``Settings`` +------------ + +When ``pytest_timeout_set_timer`` is called, ``settings`` argument is passed. + +The argument has ``Settings`` namedtuple type with the following fields: + ++-----------+-------+--------------------------------------------------------+ +|Attribute | Index | Value | ++===========+=======+========================================================+ +| timeout | 0 | timeout in seconds or ``None`` for no timeout | ++-----------+-------+--------------------------------------------------------+ +| method | 1 | Method mechanism, | +| | | ``'signal'`` and ``'thread'`` are supported by default | ++-----------+-------+--------------------------------------------------------+ +| func_only | 2 | Apply timeout to test function only if ``True``, | +| | | wrap all test function and its fixtures otherwise | ++-----------+-------+--------------------------------------------------------+ + +``pytest_timeout_cancel_timer`` +------------------------------- + +.. code:: python + + @pytest.hookspec(firstresult=True) + def pytest_timeout_cancel_timer(item): + """Called at timeout teardown. + + 'item' is a pytest node which was used for timeout setup. + + Can be overridden by plugins for alternative timeout implementation strategies. + """ + +``is_debugging`` +---------------- + +When the timeout occurs, user can open the debugger session. In this case, the timeout +should be discarded. A custom hook can check this case by calling ``is_debugging()`` +function: + +.. code:: python + + import pytest + import pytest_timeout + + + def on_timeout(): + if pytest_timeout.is_debugging(): + return + pytest.fail("+++ Timeout +++") + + + +Session Timeout +=============== + +The above mentioned timeouts are all per test function. +The "per test function" timeouts will stop an individual test +from taking too long. We may also want to limit the time of the entire +set of tests running in one session. A session all of the tests +that will be run with one invokation of pytest. + +A session timeout is set with `--session-timeout` and is in seconds. + +The following example shows a session timeout of 10 minutes (600 seconds):: + + pytest --session-timeout=600 + +You can also set the session timeout the pytest configuration file using the ``session_timeout`` option: + + .. code:: ini + + [pytest] + session_timeout = 600 + +Cooperative timeouts +-------------------- + +Session timeouts are cooperative timeouts. pytest-timeout checks the +session time at the end of each test function, and stops further tests +from running if the session timeout is exceeded. The session will +results in a test failure if this occurs. + +In particular this means if a test does not finish of itself, it will +only be interrupted if there is also a function timeout set. A +session timeout is not enough to ensure that a test-suite is +guaranteed to finish. + +Combining session and function timeouts +--------------------------------------- + +It works fine to combine both session and function timeouts. In fact +when using a session timeout it is recommended to also provide a +function timeout. + +For example, to limit test functions to 5 seconds and the full session +to 100 seconds:: + + pytest --timeout=5 --session-timeout=100 + + +Changelog +========= + +2.4.0 +----- + +- Detect debuggers registered with sys.monitoring. Thanks Rich + Chiodo. +- Make it clear the timeout message comes from pytest-timeout. Thanks + Pedro Brochado. + +2.3.1 +----- + +- Fixup some build errors, mostly README syntax which stopped twine + from uploading. + +2.3.0 +----- + +- Fix debugger detection for recent VSCode, this compiles pydevd using + cython which is now correctly detected. Thanks Adrian Gielniewski. +- Switched to using Pytest's ``TerminalReporter`` instead of writing + directly to ``sys.{stdout,stderr}``. + This change also switches all output from ``sys.stderr`` to ``sys.stdout``. + Thanks Pedro Algarvio. +- Pytest 7.0.0 is now the minimum supported version. Thanks Pedro Algarvio. +- Add ``--session-timeout`` option and ``session_timeout`` setting. + Thanks Brian Okken. + +2.2.0 +----- + +- Add ``--timeout-disable-debugger-detection`` flag, thanks + Michael Peters + +2.1.0 +----- + +- Get terminal width from shutil instead of deprecated py, thanks + Andrew Svetlov. +- Add an API for extending ``pytest-timeout`` functionality + with third-party plugins, thanks Andrew Svetlov. + +2.0.2 +----- + +- Fix debugger detection on OSX, thanks Alexander Pacha. + +2.0.1 +----- + +- Fix Python 2 removal, thanks Nicusor Picatureanu. + +2.0.0 +----- + +- Increase pytest requirement to >=5.0.0. Thanks Dominic Davis-Foster. +- Use thread timeout method when plugin is not called from main + thread to avoid crash. +- Fix pycharm debugger detection so timeouts are not triggered during + debugger usage. +- Dropped support for Python 2, minimum pytest version supported is 5.0.0. + +1.4.2 +----- + +- Fix compatibility when run with pytest pre-releases, thanks + Bruno Oliveira, +- Fix detection of third-party debuggers, thanks Bruno Oliveira. + +1.4.1 +----- + +- Fix coverage compatibility which was broken by 1.4.0. + +1.4.0 +----- + +- Better detection of when we are debugging, thanks Mattwmaster58. + +1.3.4 +----- + +- Give the threads a name to help debugging, thanks Thomas Grainger. +- Changed location to https://github.com/pytest-dev/pytest-timeout + because bitbucket is dropping mercurial support. Thanks Thomas + Grainger and Bruno Oliveira. + +1.3.3 +----- + +- Fix support for pytest >= 3.10. + +1.3.2 +----- + +- This changelog was omitted for the 1.3.2 release and was added + afterwards. Apologies for the confusion. +- Fix pytest 3.7.3 compatibility. The capture API had changed + slightly and this needed fixing. Thanks Bruno Oliveira for the + contribution. + +1.3.1 +----- + +- Fix deprecation warning on Python 3.6. Thanks Mickaƫl Schoentgen +- Create a valid tag for the release. Somehow this didn't happen for + 1.3.0, that tag points to a non-existing commit. + +1.3.0 +----- + +- Make it possible to only run the timeout timer on the test function + and not the whole fixture setup + test + teardown duration. Thanks + Pedro Algarvio for the work! +- Use the new pytest marker API, Thanks Pedro Algarvio for the work! + +1.2.1 +----- + +- Fix for pytest 3.3, thanks Bruno Oliveira. +- Update supported python versions: + - Add CPython 3.6. + - Drop CPyhon 2.6 (as did pytest 3.3) + - Drop CPyhon 3.3 + - Drop CPyhon 3.4 + +1.2.0 +----- + +* Allow using floats as timeout instead of only integers, thanks Tom + Myers. + +1.1.0 +----- + +* Report (default) timeout duration in header, thanks Holger Krekel. + +1.0.0 +----- + +* Bump version to 1.0 to commit to semantic versioning. +* Fix issue #12: Now compatible with pytest 2.8, thanks Holger Krekel. +* No longer test with pexpect on py26 as it is no longer supported +* Require pytest 2.8 and use new hookimpl decorator + +0.5 +--- + +* Timeouts will no longer be triggered when inside an interactive pdb + session started by ``pytest.set_trace()`` / ``pdb.set_trace()``. + +* Add pypy3 environment to tox.ini. + +* Transfer repository to pytest-dev team account. + +0.4 +--- + +* Support timeouts happening in (session scoped) finalizers. + +* Change command line option --timeout_method into --timeout-method + for consistency with pytest + +0.3 +--- + +* Added the PYTEST_TIMEOUT environment variable as a way of specifying + the timeout (closes issue #2). + +* More flexible marker argument parsing: you can now specify the + method using a positional argument. + +* The plugin is now enabled by default. There is no longer a need to + specify ``timeout=0`` in the configuration file or on the command + line simply so that a marker would work. + + +0.2 +--- + +* Add a marker to modify the timeout delay using a @pytest.timeout(N) + syntax, thanks to Laurant Brack for the initial code. + +* Allow the timeout marker to select the timeout method using the + ``method`` keyword argument. + +* Rename the --nosigalrm option to --method=thread to future proof + support for eventlet and gevent. Thanks to Ronny Pfannschmidt for + the hint. + +* Add ``timeout`` and ``timeout_method`` items to the configuration + file so you can enable and configure the plugin using the ini file. + Thanks to Holger Krekel and Ronny Pfannschmidt for the hints. + +* Tested (and fixed) for python 2.6, 2.7 and 3.2. diff --git a/contrib/python/pytest-timeout/py3/pytest_timeout.py b/contrib/python/pytest-timeout/py3/pytest_timeout.py new file mode 100644 index 00000000000..7c220699e23 --- /dev/null +++ b/contrib/python/pytest-timeout/py3/pytest_timeout.py @@ -0,0 +1,558 @@ +"""Timeout for tests to stop hanging testruns. + +This plugin will dump the stack and terminate the test. This can be +useful when running tests on a continuous integration server. + +If the platform supports SIGALRM this is used to raise an exception in +the test, otherwise os._exit(1) is used. +""" +import inspect +import os +import signal +import sys +import threading +import time +import traceback +from collections import namedtuple + +import pytest + + +__all__ = ("is_debugging", "Settings") +SESSION_TIMEOUT_KEY = pytest.StashKey[float]() +SESSION_EXPIRE_KEY = pytest.StashKey[float]() +PYTEST_FAILURE_MESSAGE = "Timeout (>%ss) from pytest-timeout." + +HAVE_SIGALRM = hasattr(signal, "SIGALRM") +if HAVE_SIGALRM: + DEFAULT_METHOD = "signal" +else: + DEFAULT_METHOD = "thread" +TIMEOUT_DESC = """ +Timeout in seconds before dumping the stacks. Default is 0 which +means no timeout. +""".strip() +METHOD_DESC = """ +Timeout mechanism to use. 'signal' uses SIGALRM, 'thread' uses a timer +thread. If unspecified 'signal' is used on platforms which support +SIGALRM, otherwise 'thread' is used. +""".strip() +FUNC_ONLY_DESC = """ +When set to True, defers the timeout evaluation to only the test +function body, ignoring the time it takes when evaluating any fixtures +used in the test. +""".strip() +DISABLE_DEBUGGER_DETECTION_DESC = """ +When specified, disables debugger detection. breakpoint(), pdb.set_trace(), etc. +will be interrupted by the timeout. +""".strip() +SESSION_TIMEOUT_DESC = """ +Timeout in seconds for entire session. Default is None which +means no timeout. Timeout is checked between tests, and will not interrupt a test +in progress. +""".strip() + +# bdb covers pdb, ipdb, and possibly others +# pydevd covers PyCharm, VSCode, and possibly others +KNOWN_DEBUGGING_MODULES = {"pydevd", "bdb", "pydevd_frame_evaluator"} +Settings = namedtuple( + "Settings", ["timeout", "method", "func_only", "disable_debugger_detection"] +) + + +def pytest_addoption(parser): + """Add options to control the timeout plugin.""" + group = parser.getgroup( + "timeout", + "Interrupt test run and dump stacks of all threads after a test times out", + ) + group.addoption("--timeout", type=float, help=TIMEOUT_DESC) + group.addoption( + "--timeout_method", + action="store", + choices=["signal", "thread"], + help="Deprecated, use --timeout-method", + ) + group.addoption( + "--timeout-method", + dest="timeout_method", + action="store", + choices=["signal", "thread"], + help=METHOD_DESC, + ) + group.addoption( + "--timeout-disable-debugger-detection", + dest="timeout_disable_debugger_detection", + action="store_true", + help=DISABLE_DEBUGGER_DETECTION_DESC, + ) + group.addoption( + "--session-timeout", + action="store", + dest="session_timeout", + default=None, + type=float, + metavar="SECONDS", + help=SESSION_TIMEOUT_DESC, + ) + parser.addini("timeout", TIMEOUT_DESC) + parser.addini("timeout_method", METHOD_DESC) + parser.addini("timeout_func_only", FUNC_ONLY_DESC, type="bool", default=False) + parser.addini( + "timeout_disable_debugger_detection", + DISABLE_DEBUGGER_DETECTION_DESC, + type="bool", + default=False, + ) + parser.addini("session_timeout", SESSION_TIMEOUT_DESC) + + +class TimeoutHooks: + """Timeout specific hooks.""" + + @pytest.hookspec(firstresult=True) + def pytest_timeout_set_timer(item, settings): + """Called at timeout setup. + + 'item' is a pytest node to setup timeout for. + + Can be overridden by plugins for alternative timeout implementation strategies. + + """ + + @pytest.hookspec(firstresult=True) + def pytest_timeout_cancel_timer(item): + """Called at timeout teardown. + + 'item' is a pytest node which was used for timeout setup. + + Can be overridden by plugins for alternative timeout implementation strategies. + + """ + + +def pytest_addhooks(pluginmanager): + """Register timeout-specific hooks.""" + pluginmanager.add_hookspecs(TimeoutHooks) + + +def pytest_configure(config): + """Register the marker so it shows up in --markers output.""" + config.addinivalue_line( + "markers", + "timeout(timeout, method=None, func_only=False, " + "disable_debugger_detection=False): Set a timeout, timeout " + "method and func_only evaluation on just one test item. The first " + "argument, *timeout*, is the timeout in seconds while the keyword, " + "*method*, takes the same values as the --timeout-method option. The " + "*func_only* keyword, when set to True, defers the timeout evaluation " + "to only the test function body, ignoring the time it takes when " + "evaluating any fixtures used in the test. The " + "*disable_debugger_detection* keyword, when set to True, disables " + "debugger detection, allowing breakpoint(), pdb.set_trace(), etc. " + "to be interrupted", + ) + + settings = get_env_settings(config) + config._env_timeout = settings.timeout + config._env_timeout_method = settings.method + config._env_timeout_func_only = settings.func_only + config._env_timeout_disable_debugger_detection = settings.disable_debugger_detection + + timeout = config.getoption("session_timeout") + if timeout is None: + ini = config.getini("session_timeout") + if ini: + timeout = _validate_timeout(config.getini("session_timeout"), "config file") + if timeout is not None: + expire_time = time.time() + timeout + else: + expire_time = 0 + timeout = 0 + config.stash[SESSION_TIMEOUT_KEY] = timeout + config.stash[SESSION_EXPIRE_KEY] = expire_time + + [email protected](hookwrapper=True) +def pytest_runtest_protocol(item): + """Hook in timeouts to the runtest protocol. + + If the timeout is set on the entire test, including setup and + teardown, then this hook installs the timeout. Otherwise + pytest_runtest_call is used. + """ + hooks = item.config.pluginmanager.hook + settings = _get_item_settings(item) + is_timeout = settings.timeout is not None and settings.timeout > 0 + if is_timeout and settings.func_only is False: + hooks.pytest_timeout_set_timer(item=item, settings=settings) + yield + if is_timeout and settings.func_only is False: + hooks.pytest_timeout_cancel_timer(item=item) + + # check session timeout + expire_time = item.session.config.stash[SESSION_EXPIRE_KEY] + if expire_time and (expire_time < time.time()): + timeout = item.session.config.stash[SESSION_TIMEOUT_KEY] + item.session.shouldfail = f"session-timeout: {timeout} sec exceeded" + + [email protected](hookwrapper=True) +def pytest_runtest_call(item): + """Hook in timeouts to the test function call only. + + If the timeout is set on only the test function this hook installs + the timeout, otherwise pytest_runtest_protocol is used. + """ + hooks = item.config.pluginmanager.hook + settings = _get_item_settings(item) + is_timeout = settings.timeout is not None and settings.timeout > 0 + if is_timeout and settings.func_only is True: + hooks.pytest_timeout_set_timer(item=item, settings=settings) + yield + if is_timeout and settings.func_only is True: + hooks.pytest_timeout_cancel_timer(item=item) + + [email protected](tryfirst=True) +def pytest_report_header(config): + """Add timeout config to pytest header.""" + timeout_header = [] + + if config._env_timeout: + timeout_header.append( + "timeout: %ss\ntimeout method: %s\ntimeout func_only: %s" + % ( + config._env_timeout, + config._env_timeout_method, + config._env_timeout_func_only, + ) + ) + + session_timeout = config.getoption("session_timeout") + if session_timeout: + timeout_header.append("session timeout: %ss" % session_timeout) + if timeout_header: + return timeout_header + + [email protected](tryfirst=True) +def pytest_exception_interact(node): + """Stop the timeout when pytest enters pdb in post-mortem mode.""" + hooks = node.config.pluginmanager.hook + hooks.pytest_timeout_cancel_timer(item=node) + + +def pytest_enter_pdb(): + """Stop the timeouts when we entered pdb. + + This stops timeouts from triggering when pytest's builting pdb + support notices we entered pdb. + """ + # Since pdb.set_trace happens outside of any pytest control, we don't have + # any pytest ``item`` here, so we cannot use timeout_teardown. Thus, we + # need another way to signify that the timeout should not be performed. + global SUPPRESS_TIMEOUT + SUPPRESS_TIMEOUT = True + + +def is_debugging(trace_func=None): + """Detect if a debugging session is in progress. + + This looks at both pytest's builtin pdb support as well as + externally installed debuggers using some heuristics. + + This is done by checking if either of the following conditions is + true: + + 1. Examines the trace function to see if the module it originates + from is in KNOWN_DEBUGGING_MODULES. + 2. Check is SUPPRESS_TIMEOUT is set to True. + + :param trace_func: the current trace function, if not given will use + sys.gettrace(). Used to unit-test this function. + """ + global SUPPRESS_TIMEOUT, KNOWN_DEBUGGING_MODULES + if SUPPRESS_TIMEOUT: + return True + if trace_func is None: + trace_func = sys.gettrace() + trace_module = None + if trace_func: + trace_module = inspect.getmodule(trace_func) or inspect.getmodule( + trace_func.__class__ + ) + if trace_module: + parts = trace_module.__name__.split(".") + for name in KNOWN_DEBUGGING_MODULES: + if any(part.startswith(name) for part in parts): + return True + + # For 3.12, sys.monitoring is used for tracing. Check if any debugger has been registered. + if hasattr(sys, "monitoring"): + return sys.monitoring.get_tool(sys.monitoring.DEBUGGER_ID) != None + return False + + +SUPPRESS_TIMEOUT = False + + [email protected](trylast=True) +def pytest_timeout_set_timer(item, settings): + """Setup up a timeout trigger and handler.""" + timeout_method = settings.method + if ( + timeout_method == "signal" + and threading.current_thread() is not threading.main_thread() + ): + timeout_method = "thread" + + if timeout_method == "signal": + + def handler(signum, frame): + __tracebackhide__ = True + timeout_sigalrm(item, settings) + + def cancel(): + signal.setitimer(signal.ITIMER_REAL, 0) + signal.signal(signal.SIGALRM, signal.SIG_DFL) + + item.cancel_timeout = cancel + signal.signal(signal.SIGALRM, handler) + signal.setitimer(signal.ITIMER_REAL, settings.timeout) + elif timeout_method == "thread": + timer = threading.Timer(settings.timeout, timeout_timer, (item, settings)) + timer.name = "%s %s" % (__name__, item.nodeid) + + def cancel(): + timer.cancel() + timer.join() + + item.cancel_timeout = cancel + timer.start() + return True + + [email protected](trylast=True) +def pytest_timeout_cancel_timer(item): + """Cancel the timeout trigger if it was set.""" + # When skipping is raised from a pytest_runtest_setup function + # (as is the case when using the pytest.mark.skipif marker) we + # may be called without our setup counterpart having been + # called. + cancel = getattr(item, "cancel_timeout", None) + if cancel: + cancel() + return True + + +def get_env_settings(config): + """Return the configured timeout settings. + + This looks up the settings in the environment and config file. + """ + timeout = config.getvalue("timeout") + if timeout is None: + timeout = _validate_timeout( + os.environ.get("PYTEST_TIMEOUT"), "PYTEST_TIMEOUT environment variable" + ) + if timeout is None: + ini = config.getini("timeout") + if ini: + timeout = _validate_timeout(ini, "config file") + + method = config.getvalue("timeout_method") + if method is None: + ini = config.getini("timeout_method") + if ini: + method = _validate_method(ini, "config file") + if method is None: + method = DEFAULT_METHOD + + func_only = config.getini("timeout_func_only") + + disable_debugger_detection = config.getvalue("timeout_disable_debugger_detection") + if disable_debugger_detection is None: + ini = config.getini("timeout_disable_debugger_detection") + if ini: + disable_debugger_detection = _validate_disable_debugger_detection( + ini, "config file" + ) + + return Settings(timeout, method, func_only, disable_debugger_detection) + + +def _get_item_settings(item, marker=None): + """Return (timeout, method) for an item.""" + timeout = method = func_only = disable_debugger_detection = None + if not marker: + marker = item.get_closest_marker("timeout") + if marker is not None: + settings = _parse_marker(item.get_closest_marker(name="timeout")) + timeout = _validate_timeout(settings.timeout, "marker") + method = _validate_method(settings.method, "marker") + func_only = _validate_func_only(settings.func_only, "marker") + disable_debugger_detection = _validate_disable_debugger_detection( + settings.disable_debugger_detection, "marker" + ) + if timeout is None: + timeout = item.config._env_timeout + if method is None: + method = item.config._env_timeout_method + if func_only is None: + func_only = item.config._env_timeout_func_only + if disable_debugger_detection is None: + disable_debugger_detection = item.config._env_timeout_disable_debugger_detection + return Settings(timeout, method, func_only, disable_debugger_detection) + + +def _parse_marker(marker): + """Return (timeout, method) tuple from marker. + + Either could be None. The values are not interpreted, so + could still be bogus and even the wrong type. + """ + if not marker.args and not marker.kwargs: + raise TypeError("Timeout marker must have at least one argument") + timeout = method = func_only = NOTSET = object() + for kw, val in marker.kwargs.items(): + if kw == "timeout": + timeout = val + elif kw == "method": + method = val + elif kw == "func_only": + func_only = val + else: + raise TypeError("Invalid keyword argument for timeout marker: %s" % kw) + if len(marker.args) >= 1 and timeout is not NOTSET: + raise TypeError("Multiple values for timeout argument of timeout marker") + elif len(marker.args) >= 1: + timeout = marker.args[0] + if len(marker.args) >= 2 and method is not NOTSET: + raise TypeError("Multiple values for method argument of timeout marker") + elif len(marker.args) >= 2: + method = marker.args[1] + if len(marker.args) > 2: + raise TypeError("Too many arguments for timeout marker") + if timeout is NOTSET: + timeout = None + if method is NOTSET: + method = None + if func_only is NOTSET: + func_only = None + return Settings(timeout, method, func_only, None) + + +def _validate_timeout(timeout, where): + if timeout is None: + return None + try: + return float(timeout) + except ValueError: + raise ValueError("Invalid timeout %s from %s" % (timeout, where)) + + +def _validate_method(method, where): + if method is None: + return None + if method not in ["signal", "thread"]: + raise ValueError("Invalid method %s from %s" % (method, where)) + return method + + +def _validate_func_only(func_only, where): + if func_only is None: + return None + if not isinstance(func_only, bool): + raise ValueError("Invalid func_only value %s from %s" % (func_only, where)) + return func_only + + +def _validate_disable_debugger_detection(disable_debugger_detection, where): + if disable_debugger_detection is None: + return None + if not isinstance(disable_debugger_detection, bool): + raise ValueError( + "Invalid disable_debugger_detection value %s from %s" + % (disable_debugger_detection, where) + ) + return disable_debugger_detection + + +def timeout_sigalrm(item, settings): + """Dump stack of threads and raise an exception. + + This will output the stacks of any threads other then the + current to stderr and then raise an AssertionError, thus + terminating the test. + """ + if not settings.disable_debugger_detection and is_debugging(): + return + __tracebackhide__ = True + nthreads = len(threading.enumerate()) + terminal = item.config.get_terminal_writer() + if nthreads > 1: + terminal.sep("+", title="Timeout") + dump_stacks(terminal) + if nthreads > 1: + terminal.sep("+", title="Timeout") + pytest.fail(PYTEST_FAILURE_MESSAGE % settings.timeout) + + +def timeout_timer(item, settings): + """Dump stack of threads and call os._exit(). + + This disables the capturemanager and dumps stdout and stderr. + Then the stacks are dumped and os._exit(1) is called. + """ + if not settings.disable_debugger_detection and is_debugging(): + return + terminal = item.config.get_terminal_writer() + try: + capman = item.config.pluginmanager.getplugin("capturemanager") + if capman: + capman.suspend_global_capture(item) + stdout, stderr = capman.read_global_capture() + else: + stdout, stderr = None, None + terminal.sep("+", title="Timeout") + caplog = item.config.pluginmanager.getplugin("_capturelog") + if caplog and hasattr(item, "capturelog_handler"): + log = item.capturelog_handler.stream.getvalue() + if log: + terminal.sep("~", title="Captured log") + terminal.write(log) + if stdout: + terminal.sep("~", title="Captured stdout") + terminal.write(stdout) + if stderr: + terminal.sep("~", title="Captured stderr") + terminal.write(stderr) + dump_stacks(terminal) + terminal.sep("+", title="Timeout") + except Exception: + traceback.print_exc() + finally: + terminal.flush() + sys.stdout.flush() + sys.stderr.flush() + os._exit(1) + + +def dump_stacks(terminal): + """Dump the stacks of all threads except the current thread.""" + current_ident = threading.current_thread().ident + for thread_ident, frame in sys._current_frames().items(): + if thread_ident == current_ident: + continue + for t in threading.enumerate(): + if t.ident == thread_ident: + thread_name = t.name + break + else: + thread_name = "<unknown>" + terminal.sep("~", title="Stack of %s (%s)" % (thread_name, thread_ident)) + terminal.write("".join(traceback.format_stack(frame))) diff --git a/contrib/python/pytest-timeout/py3/ya.make b/contrib/python/pytest-timeout/py3/ya.make new file mode 100644 index 00000000000..5f98848548d --- /dev/null +++ b/contrib/python/pytest-timeout/py3/ya.make @@ -0,0 +1,27 @@ +# Generated by devtools/yamaker (pypi). + +PY3_LIBRARY() + +VERSION(2.4.0) + +LICENSE(MIT) + +PEERDIR( + contrib/python/pytest +) + +NO_LINT() + +PY_SRCS( + TOP_LEVEL + pytest_timeout.py +) + +RESOURCE_FILES( + PREFIX contrib/python/pytest-timeout/py3/ + .dist-info/METADATA + .dist-info/entry_points.txt + .dist-info/top_level.txt +) + +END() diff --git a/contrib/python/pytest-timeout/ya.make b/contrib/python/pytest-timeout/ya.make new file mode 100644 index 00000000000..8373facce89 --- /dev/null +++ b/contrib/python/pytest-timeout/ya.make @@ -0,0 +1,20 @@ +PY23_LIBRARY() + +LICENSE(Service-Py23-Proxy) + +VERSION(Service-proxy-version) + +IF (PYTHON2) + PEERDIR(contrib/python/pytest-timeout/py2) +ELSE() + PEERDIR(contrib/python/pytest-timeout/py3) +ENDIF() + +NO_LINT() + +END() + +RECURSE( + py2 + py3 +) diff --git a/contrib/python/xmltodict/py3/.dist-info/METADATA b/contrib/python/xmltodict/py3/.dist-info/METADATA index 6462e0a6b5c..8f05caf86de 100644 --- a/contrib/python/xmltodict/py3/.dist-info/METADATA +++ b/contrib/python/xmltodict/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.4 Name: xmltodict -Version: 0.15.0 +Version: 0.15.1 Summary: Makes working with XML feel like you are working with JSON Home-page: https://github.com/martinblech/xmltodict Author: Martin Blech @@ -12,9 +12,6 @@ Classifier: License :: OSI Approved :: MIT License Classifier: Operating System :: OS Independent Classifier: Programming Language :: Python Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3.6 -Classifier: Programming Language :: Python :: 3.7 -Classifier: Programming Language :: Python :: 3.8 Classifier: Programming Language :: Python :: 3.9 Classifier: Programming Language :: Python :: 3.10 Classifier: Programming Language :: Python :: 3.11 @@ -22,7 +19,7 @@ Classifier: Programming Language :: Python :: 3.12 Classifier: Programming Language :: Python :: 3.13 Classifier: Programming Language :: Python :: Implementation :: PyPy Classifier: Topic :: Text Processing :: Markup :: XML -Requires-Python: >=3.6 +Requires-Python: >=3.9 Description-Content-Type: text/markdown License-File: LICENSE Dynamic: author diff --git a/contrib/python/xmltodict/py3/tests/test_dicttoxml.py b/contrib/python/xmltodict/py3/tests/test_dicttoxml.py index 67e3a880979..1fa5ba78316 100644 --- a/contrib/python/xmltodict/py3/tests/test_dicttoxml.py +++ b/contrib/python/xmltodict/py3/tests/test_dicttoxml.py @@ -263,3 +263,77 @@ xmlns:b="http://b.com/"><x a:attr="val">1</x><a:y>2</a:y><b:z>3</b:z></root>''' xml = unparse({"a": {"@attr": "1<middle>2", "#text": "x"}}, full_document=False) # The generated XML should contain escaped '<' and '>' within the attribute value self.assertIn('attr="1<middle>2"', xml) + + def test_rejects_tag_name_starting_with_question(self): + with self.assertRaises(ValueError): + unparse({"?pi": "data"}, full_document=False) + + def test_rejects_tag_name_starting_with_bang(self): + with self.assertRaises(ValueError): + unparse({"!decl": "data"}, full_document=False) + + def test_rejects_attribute_name_starting_with_question(self): + with self.assertRaises(ValueError): + unparse({"a": {"@?weird": "x"}}, full_document=False) + + def test_rejects_attribute_name_starting_with_bang(self): + with self.assertRaises(ValueError): + unparse({"a": {"@!weird": "x"}}, full_document=False) + + def test_rejects_xmlns_prefix_starting_with_question_or_bang(self): + with self.assertRaises(ValueError): + unparse({"a": {"@xmlns": {"?p": "http://e/"}}}, full_document=False) + with self.assertRaises(ValueError): + unparse({"a": {"@xmlns": {"!p": "http://e/"}}}, full_document=False) + + def test_rejects_non_string_names(self): + class Weird: + def __str__(self): + return "bad>name" + + # Non-string element key + with self.assertRaises(ValueError): + unparse({Weird(): "x"}, full_document=False) + # Non-string attribute key + with self.assertRaises(ValueError): + unparse({"a": {Weird(): "x"}}, full_document=False) + + def test_rejects_tag_name_with_slash(self): + with self.assertRaises(ValueError): + unparse({"bad/name": "x"}, full_document=False) + + def test_rejects_tag_name_with_whitespace(self): + for name in ["bad name", "bad\tname", "bad\nname"]: + with self.assertRaises(ValueError): + unparse({name: "x"}, full_document=False) + + def test_rejects_attribute_name_with_slash(self): + with self.assertRaises(ValueError): + unparse({"a": {"@bad/name": "x"}}, full_document=False) + + def test_rejects_attribute_name_with_whitespace(self): + for name in ["@bad name", "@bad\tname", "@bad\nname"]: + with self.assertRaises(ValueError): + unparse({"a": {name: "x"}}, full_document=False) + + def test_rejects_xmlns_prefix_with_slash_or_whitespace(self): + # Slash + with self.assertRaises(ValueError): + unparse({"a": {"@xmlns": {"bad/prefix": "http://e/"}}}, full_document=False) + # Whitespace + with self.assertRaises(ValueError): + unparse({"a": {"@xmlns": {"bad prefix": "http://e/"}}}, full_document=False) + + def test_rejects_names_with_quotes_and_equals(self): + # Element names + for name in ['a"b', "a'b", "a=b"]: + with self.assertRaises(ValueError): + unparse({name: "x"}, full_document=False) + # Attribute names + for name in ['@a"b', "@a'b", "@a=b"]: + with self.assertRaises(ValueError): + unparse({"a": {name: "x"}}, full_document=False) + # xmlns prefixes + for prefix in ['a"b', "a'b", "a=b"]: + with self.assertRaises(ValueError): + unparse({"a": {"@xmlns": {prefix: "http://e/"}}}, full_document=False) diff --git a/contrib/python/xmltodict/py3/xmltodict.py b/contrib/python/xmltodict/py3/xmltodict.py index c8491b354bd..4b6852ab23c 100644 --- a/contrib/python/xmltodict/py3/xmltodict.py +++ b/contrib/python/xmltodict/py3/xmltodict.py @@ -14,7 +14,7 @@ if tuple(map(int, platform.python_version_tuple()[:2])) < (3, 7): from inspect import isgenerator __author__ = 'Martin Blech' -__version__ = "0.15.0" +__version__ = "0.15.1" __license__ = 'MIT' @@ -368,7 +368,46 @@ def _has_angle_brackets(value): return isinstance(value, str) and ("<" in value or ">" in value) +def _has_invalid_name_chars(value): + """Return True if value (a str) contains any disallowed name characters. + + Disallowed: '<', '>', '/', or any whitespace character. + Non-string values return False. + """ + if not isinstance(value, str): + return False + if "<" in value or ">" in value or "/" in value: + return True + # Check for any whitespace (spaces, tabs, newlines, etc.) + return any(ch.isspace() for ch in value) + + +def _validate_name(value, kind): + """Validate an element/attribute name for XML safety. + + Raises ValueError with a specific reason when invalid. + + kind: 'element' or 'attribute' (used in error messages) + """ + if not isinstance(value, str): + raise ValueError(f"{kind} name must be a string") + if value.startswith("?") or value.startswith("!"): + raise ValueError(f'Invalid {kind} name: cannot start with "?" or "!"') + if "<" in value or ">" in value: + raise ValueError(f'Invalid {kind} name: "<" or ">" not allowed') + if "/" in value: + raise ValueError(f'Invalid {kind} name: "/" not allowed') + if '"' in value or "'" in value: + raise ValueError(f"Invalid {kind} name: quotes not allowed") + if "=" in value: + raise ValueError(f'Invalid {kind} name: "=" not allowed') + if any(ch.isspace() for ch in value): + raise ValueError(f"Invalid {kind} name: whitespace not allowed") + + def _process_namespace(name, namespaces, ns_sep=':', attr_prefix='@'): + if not isinstance(name, str): + return name if not namespaces: return name try: @@ -402,8 +441,7 @@ def _emit(key, value, content_handler, return key, value = result # Minimal validation to avoid breaking out of tag context - if _has_angle_brackets(key): - raise ValueError('Invalid element name: "<" or ">" not allowed') + _validate_name(key, "element") if not hasattr(value, '__iter__') or isinstance(value, (str, dict)): value = [value] for index, v in enumerate(value): @@ -427,23 +465,19 @@ def _emit(key, value, content_handler, if ik == cdata_key: cdata = iv continue - if ik.startswith(attr_prefix): + if isinstance(ik, str) and ik.startswith(attr_prefix): ik = _process_namespace(ik, namespaces, namespace_separator, attr_prefix) if ik == '@xmlns' and isinstance(iv, dict): for k, v in iv.items(): - if _has_angle_brackets(k): - raise ValueError( - 'Invalid attribute name: "<" or ">" not allowed' - ) + _validate_name(k, "attribute") attr = 'xmlns{}'.format(f':{k}' if k else '') attrs[attr] = str(v) continue if not isinstance(iv, str): iv = str(iv) attr_name = ik[len(attr_prefix) :] - if _has_angle_brackets(attr_name): - raise ValueError('Invalid attribute name: "<" or ">" not allowed') + _validate_name(attr_name, "attribute") attrs[attr_name] = iv continue children.append((ik, iv)) diff --git a/contrib/python/xmltodict/py3/ya.make b/contrib/python/xmltodict/py3/ya.make index 5dbf4dca8ff..19f2aa40d76 100644 --- a/contrib/python/xmltodict/py3/ya.make +++ b/contrib/python/xmltodict/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(0.15.0) +VERSION(0.15.1) LICENSE(MIT) |
