diff options
99 files changed, 2458 insertions, 1485 deletions
diff --git a/.github/TESTOWNERS b/.github/TESTOWNERS index 24df5b39587..c75a4079c54 100644 --- a/.github/TESTOWNERS +++ b/.github/TESTOWNERS @@ -163,6 +163,7 @@ /ydb/core/external_sources/s3 @ydb-platform/fq /ydb/core/resource_pools @ydb-platform/fq /ydb/services/workload_manager @ydb-platform/fq +/ydb/services/metadata @ydb-platform/fq #@ydb-platform/ui-backend /ydb/core/viewer @ydb-platform/ui-backend diff --git a/.github/config/muted_ya_tsan.txt b/.github/config/muted_ya_tsan.txt index 1b1d32b3f1c..5b103aab554 100644 --- a/.github/config/muted_ya_tsan.txt +++ b/.github/config/muted_ya_tsan.txt @@ -53,6 +53,8 @@ ydb/core/blobstorage/ut_blobstorage/ut_vdisk_internals unittest.[*/*] chunk ydb/core/blobstorage/ut_mirror3of4 Mirror3of4.ReplicationHuge ydb/core/blobstorage/ut_mirror3of4 Mirror3of4.ReplicationSmall ydb/core/blobstorage/ut_mirror3of4 unittest.sole chunk +ydb/core/blobstorage/ut_vdisk2 VDiskTest.HugeBlobWrite +ydb/core/blobstorage/ut_vdisk2 unittest.[*/*] chunk ydb/core/blobstorage/vdisk/synclog/ut_real_pdisk TBlobStorageSyncLogRealPDisk.LocalSyncDataCutWaitRetriesAfterInsufficientCut ydb/core/blobstorage/vdisk/synclog/ut_real_pdisk TBlobStorageSyncLogRealPDisk.LocalSyncDataUnsuccessfulFullRecoveryDoesNotExhaustLogChunks ydb/core/http_proxy/ut/sqs_topic_ut unittest.[*/*] chunk @@ -95,8 +97,10 @@ ydb/core/kqp/ut/rbo KqpRboYql.TPCDS_YQL ydb/core/kqp/ut/rbo unittest.[*/*] chunk ydb/core/kqp/ut/runtime KqpScanLogs.GraceJoin-EnabledLogs ydb/core/kqp/ut/runtime unittest.[*/*] chunk +ydb/core/kqp/ut/scheme KqpOlapScheme.TenThousandColumns ydb/core/kqp/ut/scheme KqpScheme.BuildingUniqIndexAllowsTableModificationsPublicApi+EnableIndexStreamWrite ydb/core/kqp/ut/scheme KqpScheme.BuildingUniqIndexAllowsTableModificationsPublicApi-EnableIndexStreamWrite +ydb/core/kqp/ut/scheme KqpScheme.BuildingUniqIndexAllowsTableModificationsSql+EnableIndexStreamWrite ydb/core/kqp/ut/scheme KqpScheme.BuildingUniqIndexAllowsTableModificationsSql-EnableIndexStreamWrite ydb/core/kqp/ut/scheme KqpScheme.BuildingUniqIndexDeniesTableModificationsPublicApi-EnableIndexStreamWrite ydb/core/kqp/ut/scheme KqpScheme.BuildingUniqIndexDeniesTableModificationsSql-EnableIndexStreamWrite @@ -137,6 +141,7 @@ ydb/core/transfer/ut/column_table unittest.sole chunk ydb/core/transfer/ut/functional Transfer.CheckCommittedOffsetEmptyBatch_Remote ydb/core/transfer/ut/functional Transfer.LocalTopic_BigMessage ydb/core/transfer/ut/functional Transfer.MessageField_Attributes_Remote +ydb/core/transfer/ut/functional unittest.sole chunk ydb/core/transfer/ut/row_table Transfer_RowTable.ComplexKey ydb/core/transfer/ut/row_table unittest.sole chunk ydb/core/tx/columnshard/backup/iscan/ut IScan.MultiExport @@ -624,6 +629,7 @@ ydb/tests/functional/tpc/medium test_diff_processing.py.TestTpcdsDiffProcessing. ydb/tests/functional/tpc/medium test_diff_processing.py.TestTpcdsDiffProcessing.test_tpcds[CheckCanonicalPolicy.WARNING] ydb/tests/functional/tpc/medium test_diff_processing.py.TestTpchDiffProcessing.test_tpch[CheckCanonicalPolicy.ERROR] ydb/tests/functional/tpc/medium test_diff_processing.py.TestTpchDiffProcessing.test_tpch[CheckCanonicalPolicy.NO] +ydb/tests/functional/tpc/medium test_diff_processing.py.TestTpchDiffProcessing.test_tpch[CheckCanonicalPolicy.WARNING] ydb/tests/functional/tpc/medium test_import_csv.py.TestExternalImportCsv.test ydb/tests/functional/tpc/medium test_import_csv.py.TestExternalImportCsvArrow.test ydb/tests/functional/tpc/medium test_tpcc.py.TestTpccW10T4SdkCompaction.test diff --git a/contrib/python/cffi/py3/.dist-info/METADATA b/contrib/python/cffi/py3/.dist-info/METADATA index 67508e56a53..6f3e2a45b7a 100644 --- a/contrib/python/cffi/py3/.dist-info/METADATA +++ b/contrib/python/cffi/py3/.dist-info/METADATA @@ -1,30 +1,29 @@ Metadata-Version: 2.4 Name: cffi -Version: 2.0.0 +Version: 2.1.0 Summary: Foreign Function Interface for Python calling C code. Author: Armin Rigo, Maciej Fijalkowski -Maintainer: Matt Davis, Matt Clay, Matti Picus -License-Expression: MIT +Maintainer: Matt Davis, Matt Clay +License-Expression: MIT-0 Project-URL: Documentation, https://cffi.readthedocs.io/ Project-URL: Changelog, https://cffi.readthedocs.io/en/latest/whatsnew.html -Project-URL: Downloads, https://github.com/python-cffi/cffi/releases +Project-URL: Download, https://github.com/python-cffi/cffi/releases Project-URL: Contact, https://groups.google.com/forum/#!forum/python-cffi Project-URL: Source Code, https://github.com/python-cffi/cffi Project-URL: Issue Tracker, https://github.com/python-cffi/cffi/issues Classifier: Programming Language :: Python Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3.9 Classifier: Programming Language :: Python :: 3.10 Classifier: Programming Language :: Python :: 3.11 Classifier: Programming Language :: Python :: 3.12 Classifier: Programming Language :: Python :: 3.13 Classifier: Programming Language :: Python :: 3.14 +Classifier: Programming Language :: Python :: 3.15 Classifier: Programming Language :: Python :: Free Threading :: 2 - Beta Classifier: Programming Language :: Python :: Implementation :: CPython -Requires-Python: >=3.9 +Requires-Python: >=3.10 Description-Content-Type: text/markdown License-File: LICENSE -License-File: AUTHORS Requires-Dist: pycparser; implementation_name != "PyPy" Dynamic: license-file @@ -59,7 +58,7 @@ Contact Testing/development tips ------------------------ -After `git clone` or `wget && tar`, we will get a directory called `cffi` or `cffi-x.x.x`. we call it `repo-directory`. To run tests under CPython, run the following in the `repo-directory`: +After `git clone` or `wget && tar`, we will get a directory called `cffi` or `cffi-x.x.x`. We call it `repo-directory`. To run tests under CPython, run the following in the `repo-directory`: pip install pytest pip install -e . # editable install of CFFI for local development diff --git a/contrib/python/cffi/py3/.dist-info/entry_points.txt b/contrib/python/cffi/py3/.dist-info/entry_points.txt index 4b0274f2333..46bb21b8418 100644 --- a/contrib/python/cffi/py3/.dist-info/entry_points.txt +++ b/contrib/python/cffi/py3/.dist-info/entry_points.txt @@ -1,2 +1,5 @@ +[console_scripts] +cffi-gen-src = cffi._cffi_gen_src:run + [distutils.setup_keywords] cffi_modules = cffi.setuptools_ext:cffi_modules diff --git a/contrib/python/cffi/py3/README.md b/contrib/python/cffi/py3/README.md index 723624d743d..fc7a235dc12 100644 --- a/contrib/python/cffi/py3/README.md +++ b/contrib/python/cffi/py3/README.md @@ -29,7 +29,7 @@ Contact Testing/development tips ------------------------ -After `git clone` or `wget && tar`, we will get a directory called `cffi` or `cffi-x.x.x`. we call it `repo-directory`. To run tests under CPython, run the following in the `repo-directory`: +After `git clone` or `wget && tar`, we will get a directory called `cffi` or `cffi-x.x.x`. We call it `repo-directory`. To run tests under CPython, run the following in the `repo-directory`: pip install pytest pip install -e . # editable install of CFFI for local development diff --git a/contrib/python/cffi/py3/c/_cffi_backend.c b/contrib/python/cffi/py3/c/_cffi_backend.c index 2aaccede0d1..74176b5746c 100644 --- a/contrib/python/cffi/py3/c/_cffi_backend.c +++ b/contrib/python/cffi/py3/c/_cffi_backend.c @@ -2,7 +2,7 @@ #include <Python.h> #include "structmember.h" #include "misc_thread_common.h" -#define CFFI_VERSION "2.0.0" +#define CFFI_VERSION "2.1.0" #ifdef MS_WIN32 #include <windows.h> @@ -142,69 +142,10 @@ #include "malloc_closure.h" -#if PY_MAJOR_VERSION >= 3 -# define STR_OR_BYTES "bytes" -# define PyText_Type PyUnicode_Type -# define PyText_Check PyUnicode_Check -# define PyTextAny_Check PyUnicode_Check -# define PyText_FromFormat PyUnicode_FromFormat -# define PyText_AsUTF8 PyUnicode_AsUTF8 -# define PyText_AS_UTF8 PyUnicode_AsUTF8 -# if PY_VERSION_HEX >= 0x03030000 -# define PyText_GetSize PyUnicode_GetLength -# else -# define PyText_GetSize PyUnicode_GetSize -# endif -# define PyText_FromString PyUnicode_FromString -# define PyText_FromStringAndSize PyUnicode_FromStringAndSize -# define PyText_InternInPlace PyUnicode_InternInPlace -# define PyText_InternFromString PyUnicode_InternFromString -# define PyIntOrLong_Check PyLong_Check -#else -# define STR_OR_BYTES "str" -# define PyText_Type PyString_Type -# define PyText_Check PyString_Check -# define PyTextAny_Check(op) (PyString_Check(op) || PyUnicode_Check(op)) -# define PyText_FromFormat PyString_FromFormat -# define PyText_AsUTF8 PyString_AsString -# define PyText_AS_UTF8 PyString_AS_STRING -# define PyText_GetSize PyString_Size -# define PyText_FromString PyString_FromString -# define PyText_FromStringAndSize PyString_FromStringAndSize -# define PyText_InternInPlace PyString_InternInPlace -# define PyText_InternFromString PyString_InternFromString -# define PyIntOrLong_Check(op) (PyInt_Check(op) || PyLong_Check(op)) -#endif - -#if PY_MAJOR_VERSION >= 3 -# define PyInt_FromLong PyLong_FromLong -# define PyInt_FromSsize_t PyLong_FromSsize_t -# define PyInt_AsSsize_t PyLong_AsSsize_t -# define PyInt_AsLong PyLong_AsLong -#endif - -#if PY_MAJOR_VERSION >= 3 -/* This is the default on Python3 and constant has been removed. */ -# define Py_TPFLAGS_CHECKTYPES 0 -#endif - -#if PY_MAJOR_VERSION < 3 -# undef PyCapsule_GetPointer -# undef PyCapsule_New -# define PyCapsule_GetPointer(capsule, name) \ - (PyCObject_AsVoidPtr(capsule)) -# define PyCapsule_New(pointer, name, destructor) \ - (PyCObject_FromVoidPtr(pointer, destructor)) -#endif - #if PY_VERSION_HEX < 0x030900a4 # define Py_SET_REFCNT(obj, val) (Py_REFCNT(obj) = (val)) #endif -#if PY_VERSION_HEX >= 0x03080000 -# define USE_WRITEUNRAISABLEMSG -#endif - #if PY_VERSION_HEX <= 0x030d00a1 static int PyDict_GetItemRef(PyObject *mp, PyObject *key, PyObject **result) { @@ -435,10 +376,7 @@ typedef struct { #endif #include "minibuffer.h" - -#if PY_MAJOR_VERSION >= 3 -# include "file_emulator.h" -#endif +#include "file_emulator.h" #ifdef PyUnicode_KIND /* Python >= 3.3 */ # include "wchar_helper_3.h" @@ -512,7 +450,7 @@ ctypedescr_new_on_top(CTypeDescrObject *ct_base, const char *extra_text, static PyObject * ctypedescr_repr(CTypeDescrObject *ct) { - return PyText_FromFormat("<ctype '%s'>", ct->ct_name); + return PyUnicode_FromFormat("<ctype '%s'>", ct->ct_name); } static void remove_dead_unique_reference(PyObject *unique_key); @@ -588,12 +526,12 @@ static PyObject *ctypeget_kind(CTypeDescrObject *ct, void *context) else result = "?"; - return PyText_FromString(result); + return PyUnicode_FromString(result); } static PyObject *ctypeget_cname(CTypeDescrObject *ct, void *context) { - return PyText_FromString(ct->ct_name); + return PyUnicode_FromString(ct->ct_name); } static PyObject *ctypeget_item(CTypeDescrObject *ct, void *context) @@ -609,7 +547,7 @@ static PyObject *ctypeget_length(CTypeDescrObject *ct, void *context) { if (ct->ct_flags & CT_ARRAY) { if (ct->ct_length >= 0) { - return PyInt_FromSsize_t(ct->ct_length); + return PyLong_FromSsize_t(ct->ct_length); } else { Py_INCREF(Py_None); @@ -763,7 +701,7 @@ ctypedescr_dir(PyObject *ct, PyObject *noarg) } else { Py_DECREF(x); - x = PyText_FromString(gsdef->name); + x = PyUnicode_FromString(gsdef->name); err = (x != NULL) ? PyList_Append(res, x) : -1; Py_XDECREF(x); if (err < 0) { @@ -897,12 +835,6 @@ _my_PyLong_AsLongLong(PyObject *ob) Like PyLong_AsLongLong(), this version accepts a Python int too, and does conversions from other types of objects. The difference is that this version refuses floats. */ -#if PY_MAJOR_VERSION < 3 - if (PyInt_Check(ob)) { - return PyInt_AS_LONG(ob); - } - else -#endif if (PyLong_Check(ob)) { return PyLong_AsLongLong(ob); } @@ -920,7 +852,7 @@ _my_PyLong_AsLongLong(PyObject *ob) if (io == NULL) return -1; - if (PyIntOrLong_Check(io)) { + if (PyLong_Check(io)) { res = _my_PyLong_AsLongLong(io); } else { @@ -940,15 +872,6 @@ _my_PyLong_AsUnsignedLongLong(PyObject *ob, int strict) does conversions from other types of objects. If 'strict', complains with OverflowError and refuses floats. If '!strict', rounds floats and masks the result. */ -#if PY_MAJOR_VERSION < 3 - if (PyInt_Check(ob)) { - long value1 = PyInt_AS_LONG(ob); - if (strict && value1 < 0) - goto negative; - return (unsigned PY_LONG_LONG)(PY_LONG_LONG)value1; - } - else -#endif if (PyLong_Check(ob)) { if (strict) { if (_PyLong_Sign(ob) < 0) @@ -973,7 +896,7 @@ _my_PyLong_AsUnsignedLongLong(PyObject *ob, int strict) if (io == NULL) return (unsigned PY_LONG_LONG)-1; - if (PyIntOrLong_Check(io)) { + if (PyLong_Check(io)) { res = _my_PyLong_AsUnsignedLongLong(io, strict); } else { @@ -1192,7 +1115,7 @@ convert_to_object(char *data, CTypeDescrObject *ct) /*READ(data, ct->ct_size)*/ value = read_raw_signed_data(data, ct->ct_size); if (ct->ct_flags & CT_PRIMITIVE_FITS_LONG) - return PyInt_FromLong((long)value); + return PyLong_FromLong((long)value); else return PyLong_FromLongLong(value); } @@ -1216,7 +1139,7 @@ convert_to_object(char *data, CTypeDescrObject *ct) Py_INCREF(x); return x; } - return PyInt_FromLong((long)value); + return PyLong_FromLong((long)value); } else return PyLong_FromUnsignedLongLong(value); @@ -1273,7 +1196,7 @@ convert_to_object_bitfield(char *data, CFieldObject *cf) result = ((PY_LONG_LONG)value) - (PY_LONG_LONG)shiftforsign; if (ct->ct_flags & CT_PRIMITIVE_FITS_LONG) - return PyInt_FromLong((long)result); + return PyLong_FromLong((long)result); else return PyLong_FromLongLong(result); } @@ -1285,7 +1208,7 @@ convert_to_object_bitfield(char *data, CFieldObject *cf) value = (value >> cf->cf_bitshift) & valuemask; if (ct->ct_flags & CT_PRIMITIVE_FITS_LONG) - return PyInt_FromLong((long)value); + return PyLong_FromLong((long)value); else return PyLong_FromUnsignedLongLong(value); } @@ -1300,7 +1223,7 @@ static int _convert_overflow(PyObject *init, const char *ct_name) if (s == NULL) return -1; PyErr_Format(PyExc_OverflowError, "integer %s does not fit '%s'", - PyText_AS_UTF8(s), ct_name); + PyUnicode_AsUTF8(s), ct_name); Py_DECREF(s); return -1; } @@ -1318,7 +1241,7 @@ static int _convert_to_char(PyObject *init) return *(unsigned char *)data; } PyErr_Format(PyExc_TypeError, - "initializer for ctype 'char' must be a "STR_OR_BYTES + "initializer for ctype 'char' must be a bytes" " of length 1, not %.200s", Py_TYPE(init)->tp_name); return -1; } @@ -1582,13 +1505,13 @@ convert_array_from_object(char *data, CTypeDescrObject *ct, PyObject *init) char *srcdata; Py_ssize_t n; if (!PyBytes_Check(init)) { - expected = STR_OR_BYTES" or list or tuple"; + expected = "bytes or list or tuple"; goto cannot_convert; } n = PyBytes_GET_SIZE(init); if (ct->ct_length >= 0 && n > ct->ct_length) { PyErr_Format(PyExc_IndexError, - "initializer "STR_OR_BYTES" is too long for '%s' " + "initializer bytes is too long for '%s' " "(got %zd characters)", ct->ct_name, n); return -1; } @@ -1923,9 +1846,9 @@ convert_from_object_bitfield(char *data, CFieldObject *cf, PyObject *init) PyErr_Format(PyExc_OverflowError, "value %s outside the range allowed by the " "bit field width: %s <= x <= %s", - PyText_AS_UTF8(svalue), - PyText_AS_UTF8(sfmin), - PyText_AS_UTF8(sfmax)); + PyUnicode_AsUTF8(svalue), + PyUnicode_AsUTF8(sfmin), + PyUnicode_AsUTF8(sfmax)); skip: Py_XDECREF(svalue); Py_XDECREF(sfmin); @@ -2181,9 +2104,9 @@ static PyObject *convert_cdata_to_enum_string(CDataObject *cd, int both) if (o == NULL) d_value = NULL; else { - d_value = PyText_FromFormat("%s: %s", - PyText_AS_UTF8(o), - PyText_AS_UTF8(d_value)); + d_value = PyUnicode_FromFormat("%s: %s", + PyUnicode_AsUTF8(o), + PyUnicode_AsUTF8(d_value)); Py_DECREF(o); } } @@ -2211,7 +2134,7 @@ static PyObject *cdata_repr(CDataObject *cd) /*READ(cd->c_data, sizeof(long double)*/ lvalue = read_raw_longdouble_data(cd->c_data); sprintf(buffer, "%LE", lvalue); - s = PyText_FromString(buffer); + s = PyUnicode_FromString(buffer); } else { PyObject *o = convert_to_object(cd->c_data, cd->c_type); @@ -2222,14 +2145,14 @@ static PyObject *cdata_repr(CDataObject *cd) } } else if ((cd->c_type->ct_flags & CT_ARRAY) && cd->c_type->ct_length < 0) { - s = PyText_FromFormat("sliced length %zd", get_array_length(cd)); + s = PyUnicode_FromFormat("sliced length %zd", get_array_length(cd)); } else { if (cd->c_data != NULL) { - s = PyText_FromFormat("%p", cd->c_data); + s = PyUnicode_FromFormat("%p", cd->c_data); } else - s = PyText_FromString("NULL"); + s = PyUnicode_FromString("NULL"); } if (s == NULL) return NULL; @@ -2240,9 +2163,9 @@ static PyObject *cdata_repr(CDataObject *cd) extra = " &"; else extra = ""; - result = PyText_FromFormat("<cdata '%s%s' %s>", + result = PyUnicode_FromFormat("<cdata '%s%s' %s>", cd->c_type->ct_name, extra, - PyText_AsUTF8(s)); + PyUnicode_AsUTF8(s)); Py_DECREF(s); return result; } @@ -2252,8 +2175,8 @@ static PyObject *_cdata_repr2(CDataObject *cd, char *text, PyObject *x) PyObject *res, *s = PyObject_Repr(x); if (s == NULL) return NULL; - res = PyText_FromFormat("<cdata '%s' %s %s>", - cd->c_type->ct_name, text, PyText_AsUTF8(s)); + res = PyUnicode_FromFormat("<cdata '%s' %s %s>", + cd->c_type->ct_name, text, PyUnicode_AsUTF8(s)); Py_DECREF(s); return res; } @@ -2280,7 +2203,7 @@ static PyObject *_frombuf_repr(CDataObject *cd, const char *cd_type_name) Py_buffer *view = ((CDataObject_frombuf *)cd)->bufferview; const char *obj_tp_name; if (view->obj == NULL) { - return PyText_FromFormat( + return PyUnicode_FromFormat( "<cdata '%s' buffer RELEASED>", cd_type_name); } @@ -2289,7 +2212,7 @@ static PyObject *_frombuf_repr(CDataObject *cd, const char *cd_type_name) if (cd->c_type->ct_flags & CT_ARRAY) { Py_ssize_t buflen = get_array_length(cd); - return PyText_FromFormat( + return PyUnicode_FromFormat( "<cdata '%s' buffer len %zd from '%.200s' object>", cd_type_name, buflen, @@ -2297,7 +2220,7 @@ static PyObject *_frombuf_repr(CDataObject *cd, const char *cd_type_name) } else { - return PyText_FromFormat( + return PyUnicode_FromFormat( "<cdata '%s' buffer from '%.200s' object>", cd_type_name, obj_tp_name); @@ -2321,7 +2244,7 @@ static Py_ssize_t cdataowning_size_bytes(CDataObject *cd) static PyObject *cdataowning_repr(CDataObject *cd) { Py_ssize_t size = cdataowning_size_bytes(cd); - return PyText_FromFormat("<cdata '%s' owning %zd bytes>", + return PyUnicode_FromFormat("<cdata '%s' owning %zd bytes>", cd->c_type->ct_name, size); } @@ -2378,37 +2301,33 @@ static PyObject *cdata_int(CDataObject *cd) long value; /*READ(cd->c_data, cd->c_type->ct_size)*/ value = (long)read_raw_signed_data(cd->c_data, cd->c_type->ct_size); - return PyInt_FromLong(value); + return PyLong_FromLong(value); } if (cd->c_type->ct_flags & (CT_PRIMITIVE_SIGNED|CT_PRIMITIVE_UNSIGNED)) { PyObject *result = convert_to_object(cd->c_data, cd->c_type); if (result != NULL && PyBool_Check(result)) - result = PyInt_FromLong(PyInt_AsLong(result)); + result = PyLong_FromLong(PyLong_AsLong(result)); return result; } else if (cd->c_type->ct_flags & CT_PRIMITIVE_CHAR) { /*READ(cd->c_data, cd->c_type->ct_size)*/ switch (cd->c_type->ct_size) { case sizeof(char): - return PyInt_FromLong((unsigned char)cd->c_data[0]); + return PyLong_FromLong((unsigned char)cd->c_data[0]); case 2: - return PyInt_FromLong((long)*(cffi_char16_t *)cd->c_data); + return PyLong_FromLong((long)*(cffi_char16_t *)cd->c_data); case 4: if (cd->c_type->ct_flags & CT_IS_SIGNED_WCHAR) - return PyInt_FromLong((long)*(int32_t *)cd->c_data); + return PyLong_FromLong((long)*(int32_t *)cd->c_data); else if (sizeof(long) > 4) - return PyInt_FromLong(*(uint32_t *)cd->c_data); + return PyLong_FromLong(*(uint32_t *)cd->c_data); else return PyLong_FromUnsignedLong(*(uint32_t *)cd->c_data); } } else if (cd->c_type->ct_flags & CT_PRIMITIVE_FLOAT) { PyObject *o = cdata_float(cd); -#if PY_MAJOR_VERSION < 3 - PyObject *r = o ? PyNumber_Int(o) : NULL; -#else PyObject *r = o ? PyNumber_Long(o) : NULL; -#endif Py_XDECREF(o); return r; } @@ -2417,19 +2336,6 @@ static PyObject *cdata_int(CDataObject *cd) return NULL; } -#if PY_MAJOR_VERSION < 3 -static PyObject *cdata_long(CDataObject *cd) -{ - PyObject *res = cdata_int(cd); - if (res != NULL && PyInt_CheckExact(res)) { - PyObject *o = PyLong_FromLong(PyInt_AS_LONG(res)); - Py_DECREF(res); - res = o; - } - return res; -} -#endif - static PyObject *cdata_float(CDataObject *cd) { if (cd->c_type->ct_flags & CT_PRIMITIVE_FLOAT) { @@ -2526,10 +2432,6 @@ static PyObject *cdata_richcompare(PyObject *v, PyObject *w, int op) return pyres; } -#if PY_MAJOR_VERSION < 3 -typedef long Py_hash_t; -#endif - static Py_hash_t cdata_hash(PyObject *v) { if (((CDataObject *)v)->c_type->ct_flags & CT_PRIMITIVE_ANY) { @@ -2618,13 +2520,13 @@ _cdata_getslicearg(CDataObject *cd, PySliceObject *slice, Py_ssize_t bounds[]) Py_ssize_t start, stop; CTypeDescrObject *ct; - start = PyInt_AsSsize_t(slice->start); + start = PyLong_AsSsize_t(slice->start); if (start == -1 && PyErr_Occurred()) { if (slice->start == Py_None) PyErr_SetString(PyExc_IndexError, "slice start must be specified"); return NULL; } - stop = PyInt_AsSsize_t(slice->stop); + stop = PyLong_AsSsize_t(slice->stop); if (stop == -1 && PyErr_Occurred()) { if (slice->stop == Py_None) PyErr_SetString(PyExc_IndexError, "slice stop must be specified"); @@ -2706,6 +2608,11 @@ cdata_ass_slice(CDataObject *cd, PySliceObject *slice, PyObject *v) cdata = cd->c_data + itemsize * bounds[0]; length = bounds[1]; + if (v == NULL) { + PyErr_SetString(PyExc_TypeError, + "'del x[n]' not supported for cdata objects"); + return -1; + } if (CData_Check(v)) { CTypeDescrObject *ctv = ((CDataObject *)v)->c_type; if ((ctv->ct_flags & CT_ARRAY) && (ctv->ct_itemdescr == ct) && @@ -2926,11 +2833,7 @@ cdata_sub(PyObject *v, PyObject *w) } diff = diff / itemsize; } -#if PY_MAJOR_VERSION < 3 - return PyInt_FromSsize_t(diff); -#else return PyLong_FromSsize_t(diff); -#endif } return _cdata_add_or_sub(v, w, -1); @@ -2943,7 +2846,7 @@ _cdata_attr_errmsg(char *errmsg, CDataObject *cd, PyObject *attr) if (!PyErr_ExceptionMatches(PyExc_AttributeError)) return; PyErr_Clear(); - text = PyText_AsUTF8(attr); + text = PyUnicode_AsUTF8(attr); if (text == NULL) return; PyErr_Format(PyExc_AttributeError, errmsg, cd->c_type->ct_name, text); @@ -3233,11 +3136,7 @@ cdata_call(CDataObject *cd, PyObject *args, PyObject *kwds) } PyTuple_SET_ITEM(fvarargs, i, (PyObject *)ct); } -#if PY_MAJOR_VERSION < 3 - fabi = PyInt_AS_LONG(PyTuple_GET_ITEM(signature, 0)); -#else fabi = PyLong_AS_LONG(PyTuple_GET_ITEM(signature, 0)); -#endif cif_descr = fb_prepare_cif(fvarargs, fresult, nargs_declared, fabi); if (cif_descr == NULL) goto error; @@ -3460,9 +3359,6 @@ static PyNumberMethods CData_as_number = { (binaryfunc)cdata_add, /*nb_add*/ (binaryfunc)cdata_sub, /*nb_subtract*/ 0, /*nb_multiply*/ -#if PY_MAJOR_VERSION < 3 - 0, /*nb_divide*/ -#endif 0, /*nb_remainder*/ 0, /*nb_divmod*/ 0, /*nb_power*/ @@ -3476,15 +3372,8 @@ static PyNumberMethods CData_as_number = { 0, /*nb_and*/ 0, /*nb_xor*/ 0, /*nb_or*/ -#if PY_MAJOR_VERSION < 3 - 0, /*nb_coerce*/ -#endif (unaryfunc)cdata_int, /*nb_int*/ -#if PY_MAJOR_VERSION < 3 - (unaryfunc)cdata_long, /*nb_long*/ -#else - 0, -#endif + 0, /*nb_reserved*/ (unaryfunc)cdata_float, /*nb_float*/ 0, /*nb_oct*/ 0, /*nb_hex*/ @@ -3530,7 +3419,7 @@ static PyTypeObject CData_Type = { (getattrofunc)cdata_getattro, /* tp_getattro */ (setattrofunc)cdata_setattro, /* tp_setattro */ 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_CHECKTYPES, /* tp_flags */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ "The internal base type for CData objects. Use FFI.CData to access " "it. Always check with isinstance(): subtypes are sometimes returned " "on CPython, for performance reasons.", /* tp_doc */ @@ -3574,7 +3463,7 @@ static PyTypeObject CDataOwning_Type = { 0, /* inherited */ /* tp_getattro */ 0, /* inherited */ /* tp_setattro */ 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_CHECKTYPES, /* tp_flags */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ "This is an internal subtype of _CDataBase for performance only on " "CPython. Check with isinstance(x, ffi.CData).", /* tp_doc */ 0, /* tp_traverse */ @@ -3617,8 +3506,7 @@ static PyTypeObject CDataOwningGC_Type = { 0, /* inherited */ /* tp_getattro */ 0, /* inherited */ /* tp_setattro */ 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_CHECKTYPES /* tp_flags */ - | Py_TPFLAGS_HAVE_GC, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ "This is an internal subtype of _CDataBase for performance only on " "CPython. Check with isinstance(x, ffi.CData).", /* tp_doc */ (traverseproc)cdataowninggc_traverse, /* tp_traverse */ @@ -3661,8 +3549,7 @@ static PyTypeObject CDataFromBuf_Type = { 0, /* inherited */ /* tp_getattro */ 0, /* inherited */ /* tp_setattro */ 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_CHECKTYPES /* tp_flags */ - | Py_TPFLAGS_HAVE_GC, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ "This is an internal subtype of _CDataBase for performance only on " "CPython. Check with isinstance(x, ffi.CData).", /* tp_doc */ (traverseproc)cdatafrombuf_traverse, /* tp_traverse */ @@ -3705,7 +3592,7 @@ static PyTypeObject CDataGCP_Type = { 0, /* inherited */ /* tp_getattro */ 0, /* inherited */ /* tp_setattro */ 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_CHECKTYPES /* tp_flags */ + Py_TPFLAGS_DEFAULT /* tp_flags */ #ifdef Py_TPFLAGS_HAVE_FINALIZE | Py_TPFLAGS_HAVE_FINALIZE #endif @@ -4072,12 +3959,6 @@ _my_PyObject_AsBool(PyObject *ob) PyNumberMethods *nb; int res; -#if PY_MAJOR_VERSION < 3 - if (PyInt_Check(ob)) { - return PyInt_AS_LONG(ob) != 0; - } - else -#endif if (PyLong_Check(ob)) { return _PyLong_Sign(ob) != 0; } @@ -4111,7 +3992,7 @@ _my_PyObject_AsBool(PyObject *ob) if (io == NULL) return -1; - if (PyIntOrLong_Check(io) || PyFloat_Check(io)) { + if (PyLong_Check(io) || PyFloat_Check(io)) { res = _my_PyObject_AsBool(io); } else { @@ -4145,17 +4026,6 @@ static CDataObject *cast_to_integer_or_char(CTypeDescrObject *ct, PyObject *ob) (CT_POINTER|CT_FUNCTIONPTR|CT_ARRAY)) { value = (Py_intptr_t)((CDataObject *)ob)->c_data; } -#if PY_MAJOR_VERSION < 3 - else if (PyString_Check(ob)) { - if (PyString_GET_SIZE(ob) != 1) { - PyErr_Format(PyExc_TypeError, - "cannot cast string of length %zd to ctype '%s'", - PyString_GET_SIZE(ob), ct->ct_name); - return NULL; - } - value = (unsigned char)PyString_AS_STRING(ob)[0]; - } -#endif else if (PyUnicode_Check(ob)) { char err_buf[80]; cffi_char32_t ordinal; @@ -4421,7 +4291,7 @@ static void dl_dealloc(DynLibObject *dlobj) static PyObject *dl_repr(DynLibObject *dlobj) { - return PyText_FromFormat("<clibrary '%s'>", dlobj->dl_name); + return PyUnicode_FromFormat("<clibrary '%s'>", dlobj->dl_name); } static int dl_check_closed(DynLibObject *dlobj) @@ -4616,8 +4486,8 @@ static void *b_do_dlopen(PyObject *args, const char **p_printable_filename, PyErr_Format(PyExc_RuntimeError, "cannot call dlopen(NULL)"); return NULL; } - *p_temp = PyText_FromFormat("%p", handle); - *p_printable_filename = PyText_AsUTF8(*p_temp); + *p_temp = PyUnicode_FromFormat("%p", handle); + *p_printable_filename = PyUnicode_AsUTF8(*p_temp); *auto_close = 0; return handle; } @@ -4630,17 +4500,11 @@ static void *b_do_dlopen(PyObject *args, const char **p_printable_filename, { Py_ssize_t sz1; wchar_t *w1; -#if PY_MAJOR_VERSION < 3 - s = PyUnicode_AsUTF8String(s); - if (s == NULL) - return NULL; - *p_temp = s; -#endif - *p_printable_filename = PyText_AsUTF8(s); + *p_printable_filename = PyUnicode_AsUTF8(s); if (*p_printable_filename == NULL) return NULL; - sz1 = PyText_GetSize(filename_unicode) + 1; + sz1 = PyUnicode_GetLength(filename_unicode) + 1; sz1 *= 2; /* should not be needed, but you never know */ w1 = alloca(sizeof(wchar_t) * sz1); sz1 = PyUnicode_AsWideChar(filename_unicode, @@ -4656,18 +4520,7 @@ static void *b_do_dlopen(PyObject *args, const char **p_printable_filename, if (!PyArg_ParseTuple(args, "et|i:load_library", Py_FileSystemDefaultEncoding, &filename_or_null, &flags)) return NULL; -#if PY_MAJOR_VERSION < 3 - if (PyUnicode_Check(s)) - { - s = PyUnicode_AsUTF8String(s); - if (s == NULL) { - PyMem_Free(filename_or_null); - return NULL; - } - *p_temp = s; - } -#endif - *p_printable_filename = PyText_AsUTF8(s); + *p_printable_filename = PyUnicode_AsUTF8(s); if (*p_printable_filename == NULL) { PyMem_Free(filename_or_null); return NULL; @@ -5206,7 +5059,7 @@ _add_field(PyObject *interned_fields, PyObject *fname, CTypeDescrObject *ftype, cf->cf_flags = flags; Py_INCREF(fname); - PyText_InternInPlace(&fname); + PyUnicode_InternInPlace(&fname); prev_size = PyDict_Size(interned_fields); err = PyDict_SetItem(interned_fields, fname, (PyObject *)cf); Py_DECREF(fname); @@ -5216,7 +5069,7 @@ _add_field(PyObject *interned_fields, PyObject *fname, CTypeDescrObject *ftype, if (PyDict_Size(interned_fields) != prev_size + 1) { PyErr_Format(PyExc_KeyError, "duplicate field name '%s'", - PyText_AS_UTF8(fname)); + PyUnicode_AsUTF8(fname)); return NULL; } return cf; /* borrowed reference */ @@ -5303,7 +5156,7 @@ static PyObject *b_complete_struct_or_union_lock_held(CTypeDescrObject *ct, Py_ssize_t byteoffsetorg; CFieldObject **previous; int prev_bitfield_size, prev_bitfield_free; - PyObject *interned_fields; + PyObject *interned_fields = NULL; sflags = complete_sflags(sflags); if (sflags & SF_PACKED) @@ -5344,7 +5197,7 @@ static PyObject *b_complete_struct_or_union_lock_held(CTypeDescrObject *ct, Py_ssize_t foffset = -1; if (!PyArg_ParseTuple(PyList_GET_ITEM(fields, i), "O!O!|in:list item", - &PyText_Type, &fname, + &PyUnicode_Type, &fname, &CTypeDescr_Type, &ftype, &fbitsize, &foffset)) goto finally; @@ -5365,7 +5218,7 @@ static PyObject *b_complete_struct_or_union_lock_held(CTypeDescrObject *ct, else { PyErr_Format(PyExc_TypeError, "field '%s.%s' has ctype '%s' of unknown size", - ct->ct_name, PyText_AS_UTF8(fname), + ct->ct_name, PyUnicode_AsUTF8(fname), ftype->ct_name); goto finally; } @@ -5396,7 +5249,7 @@ static PyObject *b_complete_struct_or_union_lock_held(CTypeDescrObject *ct, if (!(sflags & SF_GCC_ARM_BITFIELDS) && fbitsize >= 0) { if (!(sflags & SF_MSVC_BITFIELDS)) { /* GCC: anonymous bitfields (of any size) don't cause alignment */ - do_align = PyText_GetSize(fname) > 0; + do_align = PyUnicode_GetLength(fname) > 0; } else { /* MSVC: zero-sized bitfields don't cause alignment */ @@ -5435,12 +5288,12 @@ static PyObject *b_complete_struct_or_union_lock_held(CTypeDescrObject *ct, except to know if we must set CT_CUSTOM_FIELD_POS */ if (detect_custom_layout(ct, sflags, byteoffset, foffset, "wrong offset for field '", - PyText_AS_UTF8(fname), "'") < 0) + PyUnicode_AsUTF8(fname), "'") < 0) goto finally; byteoffset = foffset; } - if (PyText_GetSize(fname) == 0 && + if (PyUnicode_GetLength(fname) == 0 && ftype->ct_flags & (CT_STRUCT|CT_UNION)) { /* a nested anonymous struct or union */ CFieldObject *cfsrc = (CFieldObject *)ftype->ct_extra; @@ -5482,7 +5335,7 @@ static PyObject *b_complete_struct_or_union_lock_held(CTypeDescrObject *ct, PyErr_Format(PyExc_TypeError, "field '%s.%s' is a bitfield, " "but a fixed offset is specified", - ct->ct_name, PyText_AS_UTF8(fname)); + ct->ct_name, PyUnicode_AsUTF8(fname)); goto finally; } @@ -5491,7 +5344,7 @@ static PyObject *b_complete_struct_or_union_lock_held(CTypeDescrObject *ct, CT_PRIMITIVE_CHAR))) { PyErr_Format(PyExc_TypeError, "field '%s.%s' declared as '%s' cannot be a bit field", - ct->ct_name, PyText_AS_UTF8(fname), + ct->ct_name, PyUnicode_AsUTF8(fname), ftype->ct_name); goto finally; } @@ -5499,7 +5352,7 @@ static PyObject *b_complete_struct_or_union_lock_held(CTypeDescrObject *ct, PyErr_Format(PyExc_TypeError, "bit field '%s.%s' is declared '%s:%d', which " "exceeds the width of the type", - ct->ct_name, PyText_AS_UTF8(fname), + ct->ct_name, PyUnicode_AsUTF8(fname), ftype->ct_name, fbitsize); goto finally; } @@ -5511,10 +5364,10 @@ static PyObject *b_complete_struct_or_union_lock_held(CTypeDescrObject *ct, field_offset_bytes &= ~(falign - 1); if (fbitsize == 0) { - if (PyText_GetSize(fname) > 0) { + if (PyUnicode_GetLength(fname) > 0) { PyErr_Format(PyExc_TypeError, "field '%s.%s' is declared with :0", - ct->ct_name, PyText_AS_UTF8(fname)); + ct->ct_name, PyUnicode_AsUTF8(fname)); goto finally; } if (!(sflags & SF_MSVC_BITFIELDS)) { @@ -5554,7 +5407,7 @@ static PyObject *b_complete_struct_or_union_lock_held(CTypeDescrObject *ct, PyErr_Format(PyExc_NotImplementedError, "with 'packed', gcc would compile field " "'%s.%s' to reuse some bits in the previous " - "field", ct->ct_name, PyText_AS_UTF8(fname)); + "field", ct->ct_name, PyUnicode_AsUTF8(fname)); goto finally; } field_offset_bytes += falign; @@ -5600,7 +5453,7 @@ static PyObject *b_complete_struct_or_union_lock_held(CTypeDescrObject *ct, if (sflags & SF_GCC_BIG_ENDIAN) bitshift = 8 * ftype->ct_size - fbitsize - bitshift; - if (PyText_GetSize(fname) > 0) { + if (PyUnicode_GetLength(fname) > 0) { *previous = _add_field(interned_fields, fname, ftype, field_offset_bytes, bitshift, fbitsize, @@ -6173,7 +6026,7 @@ static PyObject *new_function_type(PyObject *fargs, /* tuple */ fct->ct_stuff = PyTuple_New(2 + funcbuilder.nargs); if (fct->ct_stuff == NULL) goto error; - fabiobj = PyInt_FromLong(fabi); + fabiobj = PyLong_FromLong(fabi); if (fabiobj == NULL) goto error; PyTuple_SET_ITEM(fct->ct_stuff, 0, fabiobj); @@ -6284,7 +6137,6 @@ static void _my_PyErr_WriteUnraisable(PyObject *t, PyObject *v, PyObject *tb, char *extra_error_line) { /* like PyErr_WriteUnraisable(), but write a full traceback */ -#ifdef USE_WRITEUNRAISABLEMSG /* PyErr_WriteUnraisable actually writes the full traceback anyway from Python 3.4, but we can't really get the formatting of the @@ -6314,43 +6166,13 @@ static void _my_PyErr_WriteUnraisable(PyObject *t, PyObject *v, PyObject *tb, #if PY_VERSION_HEX >= 0x030D0000 PyErr_FormatUnraisable("Exception ignored %S", s); #else - _PyErr_WriteUnraisableMsg(PyText_AS_UTF8(s), NULL); + _PyErr_WriteUnraisableMsg(PyUnicode_AsUTF8(s), NULL); #endif Py_DECREF(s); } else PyErr_WriteUnraisable(obj); /* best effort */ PyErr_Clear(); - -#else - - /* version for Python 2.7 and < 3.8 */ - PyObject *f; -#if PY_MAJOR_VERSION >= 3 - /* jump through hoops to ensure the tb is attached to v, on Python 3 */ - PyErr_NormalizeException(&t, &v, &tb); - if (tb == NULL) { - tb = Py_None; - Py_INCREF(tb); - } - PyException_SetTraceback(v, tb); -#endif - f = PySys_GetObject("stderr"); - if (f != NULL) { - if (obj != NULL) { - PyFile_WriteString(objdescr, f); - PyFile_WriteObject(obj, f, 0); - PyFile_WriteString(":\n", f); - } - if (extra_error_line != NULL) - PyFile_WriteString(extra_error_line, f); - PyErr_Display(t, v, tb); - } - Py_XDECREF(t); - Py_XDECREF(v); - Py_XDECREF(tb); - -#endif } static void general_invoke_callback(int decode_args_from_libffi, @@ -6400,11 +6222,7 @@ static void general_invoke_callback(int decode_args_from_libffi, goto error; if (convert_from_object_fficallback(result, SIGNATURE(1), py_res, decode_args_from_libffi) < 0) { -#ifdef USE_WRITEUNRAISABLEMSG extra_error_line = ", trying to convert the result back to C"; -#else - extra_error_line = "Trying to convert the result back to C:\n"; -#endif goto error; } done: @@ -6456,16 +6274,9 @@ static void general_invoke_callback(int decode_args_from_libffi, _my_PyErr_WriteUnraisable(exc1, val1, tb1, "From cffi callback ", py_ob, extra_error_line); -#ifdef USE_WRITEUNRAISABLEMSG _my_PyErr_WriteUnraisable(exc2, val2, tb2, "during handling of the above exception by 'onerror'", NULL, NULL); -#else - extra_error_line = ("\nDuring the call to 'onerror', " - "another exception occurred:\n\n"); - _my_PyErr_WriteUnraisable(exc2, val2, tb2, - NULL, NULL, extra_error_line); -#endif _cffi_stop_error_capture(ecap); } } @@ -6533,14 +6344,6 @@ static PyObject *prepare_callback_info_tuple(CTypeDescrObject *ct, infotuple = Py_BuildValue("OOOO", ct, ob, py_rawerr, onerror_ob); Py_DECREF(py_rawerr); -#if defined(WITH_THREAD) && PY_VERSION_HEX < 0x03070000 - /* We must setup the GIL here, in case the callback is invoked in - some other non-Pythonic thread. This is the same as ctypes. - But PyEval_InitThreads() is always a no-op from CPython 3.7 - (the call from ctypes was removed some time later I think). */ - PyEval_InitThreads(); -#endif - return infotuple; } @@ -6714,19 +6517,7 @@ static PyObject *b_new_enum_type(PyObject *self, PyObject *args) PyObject *value = PyTuple_GET_ITEM(enumvalues, i); tmpkey = PyTuple_GET_ITEM(enumerators, i); Py_INCREF(tmpkey); - if (!PyText_Check(tmpkey)) { -#if PY_MAJOR_VERSION < 3 - if (PyUnicode_Check(tmpkey)) { - const char *text = PyText_AsUTF8(tmpkey); - if (text == NULL) - goto error; - Py_DECREF(tmpkey); - tmpkey = PyString_FromString(text); - if (tmpkey == NULL) - goto error; - } - else -#endif + if (!PyUnicode_Check(tmpkey)) { { PyErr_SetString(PyExc_TypeError, "enumerators must be a list of strings"); @@ -6782,7 +6573,7 @@ static PyObject *b_alignof(PyObject *self, PyObject *arg) align = get_alignment((CTypeDescrObject *)arg); if (align < 0) return NULL; - return PyInt_FromLong(align); + return PyLong_FromLong(align); } static Py_ssize_t direct_sizeof_cdata(CDataObject *cd) @@ -6820,7 +6611,7 @@ static PyObject *b_sizeof(PyObject *self, PyObject *arg) "expected a 'cdata' or 'ctype' object"); return NULL; } - return PyInt_FromSsize_t(size); + return PyLong_FromSsize_t(size); } static PyObject *b_typeof(PyObject *self, PyObject *arg) @@ -6844,7 +6635,7 @@ static CTypeDescrObject *direct_typeoffsetof(CTypeDescrObject *ct, CTypeDescrObject *res; CFieldObject *cf; - if (PyTextAny_Check(fieldname)) { + if (PyUnicode_Check(fieldname)) { if (!following && (ct->ct_flags & CT_POINTER)) ct = ct->ct_itemdescr; if (!(ct->ct_flags & (CT_STRUCT|CT_UNION))) { @@ -6871,7 +6662,7 @@ static CTypeDescrObject *direct_typeoffsetof(CTypeDescrObject *ct, *offset = cf->cf_offset; } else { - Py_ssize_t index = PyInt_AsSsize_t(fieldname); + Py_ssize_t index = PyLong_AsSsize_t(fieldname); if (index < 0 && PyErr_Occurred()) { PyErr_SetString(PyExc_TypeError, "field name or array index expected"); @@ -6961,7 +6752,7 @@ static PyObject *b_getcname(PyObject *self, PyObject *args) memcpy(p, ct->ct_name + ct->ct_name_position, namelen - ct->ct_name_position); - return PyText_FromStringAndSize(s, namelen + replacelen); + return PyUnicode_FromStringAndSize(s, namelen + replacelen); } static PyObject *b_string(PyObject *self, PyObject *args, PyObject *kwds) @@ -6985,7 +6776,7 @@ static PyObject *b_string(PyObject *self, PyObject *args, PyObject *kwds) if (s != NULL) { PyErr_Format(PyExc_RuntimeError, "cannot use string() on %s", - PyText_AS_UTF8(s)); + PyUnicode_AsUTF8(s)); Py_DECREF(s); } return NULL; @@ -7105,7 +6896,7 @@ static PyObject *b_unpack(PyObject *self, PyObject *args, PyObject *kwds) if (s != NULL) { PyErr_Format(PyExc_RuntimeError, "cannot use unpack() on %s", - PyText_AS_UTF8(s)); + PyUnicode_AsUTF8(s)); Py_DECREF(s); } return NULL; @@ -7193,13 +6984,13 @@ static PyObject *b_unpack(PyObject *self, PyObject *args, PyObject *kwds) default: x = convert_to_object(src, ctitem); break; /* special cases for performance only */ - case 0: x = PyInt_FromLong(*(signed char *)src); break; - case 1: x = PyInt_FromLong(*(short *)src); break; - case 2: x = PyInt_FromLong(*(int *)src); break; - case 3: x = PyInt_FromLong(*(long *)src); break; - case 4: x = PyInt_FromLong(*(unsigned char *)src); break; - case 5: x = PyInt_FromLong(*(unsigned short *)src); break; - case 6: x = PyInt_FromLong((long)*(unsigned int *)src); break; + case 0: x = PyLong_FromLong(*(signed char *)src); break; + case 1: x = PyLong_FromLong(*(short *)src); break; + case 2: x = PyLong_FromLong(*(int *)src); break; + case 3: x = PyLong_FromLong(*(long *)src); break; + case 4: x = PyLong_FromLong(*(unsigned char *)src); break; + case 5: x = PyLong_FromLong(*(unsigned short *)src); break; + case 6: x = PyLong_FromLong((long)*(unsigned int *)src); break; case 7: x = PyLong_FromUnsignedLong(*(unsigned long *)src); break; case 8: x = PyFloat_FromDouble(*(float *)src); break; case 9: x = PyFloat_FromDouble(*(double *)src); break; @@ -7283,12 +7074,12 @@ static PyObject *b_get_errno(PyObject *self, PyObject *noarg) restore_errno_only(); err = errno; errno = 0; - return PyInt_FromLong(err); + return PyLong_FromLong(err); } static PyObject *b_set_errno(PyObject *self, PyObject *arg) { - long ival = PyInt_AsLong(arg); + long ival = PyLong_AsLong(arg); if (ival == -1 && PyErr_Occurred()) return NULL; else if (ival < INT_MIN || ival > INT_MAX) { @@ -7370,46 +7161,6 @@ static PyObject *b_from_handle(PyObject *self, PyObject *arg) static int _my_PyObject_GetContiguousBuffer(PyObject *x, Py_buffer *view, int writable_only) { -#if PY_MAJOR_VERSION < 3 - /* Some objects only support the buffer interface and CPython doesn't - translate it into the memoryview interface, mess. Hack a very - minimal content for 'view'. Don't care if the other fields are - uninitialized: we only call PyBuffer_Release(), which only reads - 'view->obj'. */ - PyBufferProcs *pb = x->ob_type->tp_as_buffer; - if (pb && !pb->bf_releasebuffer) { - /* we used to try all three in some vaguely sensible order, - i.e. first the write. But trying to call the write on a - read-only buffer fails with TypeError. So we use a less- - sensible order now. See test_from_buffer_more_cases. - - If 'writable_only', we only try bf_getwritebuffer. - */ - readbufferproc proc = NULL; - if (!writable_only) { - proc = (readbufferproc)pb->bf_getreadbuffer; - if (!proc) - proc = (readbufferproc)pb->bf_getcharbuffer; - } - if (!proc) - proc = (readbufferproc)pb->bf_getwritebuffer; - - if (proc && pb->bf_getsegcount) { - if ((*pb->bf_getsegcount)(x, NULL) != 1) { - PyErr_SetString(PyExc_TypeError, - "expected a single-segment buffer object"); - return -1; - } - view->len = (*proc)(x, 0, &view->buf); - if (view->len < 0) - return -1; - view->obj = x; - Py_INCREF(x); - return 0; - } - } -#endif - if (PyObject_GetBuffer(x, view, writable_only ? PyBUF_WRITABLE : PyBUF_SIMPLE) < 0) return -1; @@ -7857,30 +7608,6 @@ static PyObject *b__testfunc(PyObject *self, PyObject *args) return PyLong_FromVoidPtr(f); } -#if PY_MAJOR_VERSION < 3 -static Py_ssize_t _test_segcountproc(PyObject *o, Py_ssize_t *ignored) -{ - return 1; -} -static Py_ssize_t _test_getreadbuf(PyObject *o, Py_ssize_t i, void **r) -{ - static char buf[] = "RDB"; - *r = buf; - return 3; -} -static Py_ssize_t _test_getwritebuf(PyObject *o, Py_ssize_t i, void **r) -{ - static char buf[] = "WRB"; - *r = buf; - return 3; -} -static Py_ssize_t _test_getcharbuf(PyObject *o, Py_ssize_t i, char **r) -{ - static char buf[] = "CHB"; - *r = buf; - return 3; -} -#endif static int _test_getbuf(PyObject *self, Py_buffer *view, int flags) { static char buf[] = "GTB"; @@ -7903,14 +7630,6 @@ static PyObject *b__testbuff(PyObject *self, PyObject *args) assert(obj->tp_as_buffer != NULL); -#if PY_MAJOR_VERSION < 3 - obj->tp_as_buffer->bf_getsegcount = &_test_segcountproc; - obj->tp_flags |= Py_TPFLAGS_HAVE_GETCHARBUFFER; - obj->tp_flags |= Py_TPFLAGS_HAVE_NEWBUFFER; - if (methods & 1) obj->tp_as_buffer->bf_getreadbuffer = &_test_getreadbuf; - if (methods & 2) obj->tp_as_buffer->bf_getwritebuffer = &_test_getwritebuf; - if (methods & 4) obj->tp_as_buffer->bf_getcharbuffer = &_test_getcharbuf; -#endif if (methods & 8) obj->tp_as_buffer->bf_getbuffer = &_test_getbuf; if (methods & 16) obj->tp_as_buffer->bf_getbuffer = &_test_getbuf_ro; @@ -8052,7 +7771,7 @@ static PyObject *_cffi_get_struct_layout(Py_ssize_t nums[]) return NULL; while (--count >= 0) { - PyObject *o = PyInt_FromSsize_t(nums[count]); + PyObject *o = PyLong_FromSsize_t(nums[count]); if (o == NULL) { Py_DECREF(result); return NULL; @@ -8173,7 +7892,6 @@ static struct { const char *name; int value; } all_dlopen_flags[] = { /************************************************************/ -#if PY_MAJOR_VERSION >= 3 static struct PyModuleDef FFIBackendModuleDef = { PyModuleDef_HEAD_INIT, "_cffi_backend", @@ -8186,12 +7904,6 @@ static struct PyModuleDef FFIBackendModuleDef = { PyMODINIT_FUNC PyInit__cffi_backend(void) -#else -#define INITERROR return - -PyMODINIT_FUNC -init_cffi_backend(void) -#endif { PyObject *m, *v; int i; @@ -8214,19 +7926,15 @@ init_cffi_backend(void) }; v = PySys_GetObject("version"); - if (v == NULL || !PyText_Check(v) || - strncmp(PyText_AS_UTF8(v), PY_VERSION, 3) != 0) { + if (v == NULL || !PyUnicode_Check(v) || + strncmp(PyUnicode_AsUTF8(v), PY_VERSION, 3) != 0) { PyErr_Format(PyExc_ImportError, "this module was compiled for Python %c%c%c", PY_VERSION[0], PY_VERSION[1], PY_VERSION[2]); INITERROR; } -#if PY_MAJOR_VERSION >= 3 m = PyModule_Create(&FFIBackendModuleDef); -#else - m = Py_InitModule("_cffi_backend", FFIBackendMethods); -#endif if (m == NULL) INITERROR; @@ -8259,11 +7967,11 @@ init_cffi_backend(void) } if (!init_done) { - v = PyText_FromString("_cffi_backend"); + v = PyUnicode_FromString("_cffi_backend"); if (v == NULL || PyDict_SetItemString(CData_Type.tp_dict, "__module__", v) < 0) INITERROR; - v = PyText_FromString("<cdata>"); + v = PyUnicode_FromString("<cdata>"); if (v == NULL || PyDict_SetItemString(CData_Type.tp_dict, "__name__", v) < 0) INITERROR; @@ -8275,7 +7983,7 @@ init_cffi_backend(void) if (v == NULL || PyModule_AddObject(m, "_C_API", v) < 0) INITERROR; - v = PyText_FromString(CFFI_VERSION); + v = PyUnicode_FromString(CFFI_VERSION); if (v == NULL || PyModule_AddObject(m, "__version__", v) < 0) INITERROR; @@ -8312,9 +8020,7 @@ init_cffi_backend(void) if (init_ffi_lib(m) < 0) INITERROR; -#if PY_MAJOR_VERSION >= 3 if (init_file_emulator() < 0) INITERROR; return m; -#endif } diff --git a/contrib/python/cffi/py3/c/call_python.c b/contrib/python/cffi/py3/c/call_python.c index 8c6703cc26b..57e5f31402f 100644 --- a/contrib/python/cffi/py3/c/call_python.c +++ b/contrib/python/cffi/py3/c/call_python.c @@ -1,16 +1,7 @@ -#if PY_VERSION_HEX >= 0x03080000 -# define HAVE_PYINTERPSTATE_GETDICT -#endif - - static PyObject *_current_interp_key(void) { PyInterpreterState *interp = PyThreadState_GET()->interp; -#ifdef HAVE_PYINTERPSTATE_GETDICT return PyInterpreterState_GetDict(interp); /* shared reference */ -#else - return interp->modules; -#endif } static PyObject *_get_interpstate_dict(void) @@ -33,11 +24,7 @@ static PyObject *_get_interpstate_dict(void) } interp = tstate->interp; -#ifdef HAVE_PYINTERPSTATE_GETDICT interpdict = PyInterpreterState_GetDict(interp); /* shared reference */ -#else - interpdict = interp->builtins; -#endif if (interpdict == NULL) { /* subinterpreter was cleared already, or is being cleared right now, to a point that is too much for us to continue */ @@ -47,7 +34,7 @@ static PyObject *_get_interpstate_dict(void) /* from there on, we know the (sub-)interpreter is still valid */ if (attr_name == NULL) { - attr_name = PyText_InternFromString("__cffi_backend_extern_py"); + attr_name = PyUnicode_InternFromString("__cffi_backend_extern_py"); if (attr_name == NULL) goto error; } @@ -90,7 +77,7 @@ static PyObject *_ffi_def_extern_decorator(PyObject *outer_args, PyObject *fn) name = PyObject_GetAttrString(fn, "__name__"); if (name == NULL) return NULL; - s = PyText_AsUTF8(name); + s = PyUnicode_AsUTF8(name); if (s == NULL) { Py_DECREF(name); return NULL; diff --git a/contrib/python/cffi/py3/c/cdlopen.c b/contrib/python/cffi/py3/c/cdlopen.c index 7db7ae280fb..d07fcd25ae6 100644 --- a/contrib/python/cffi/py3/c/cdlopen.c +++ b/contrib/python/cffi/py3/c/cdlopen.c @@ -7,7 +7,7 @@ static void *cdlopen_fetch(PyObject *libname, void *libhandle, if (libhandle == NULL) { PyErr_Format(FFIError, "library '%s' has been closed", - PyText_AS_UTF8(libname)); + PyUnicode_AsUTF8(libname)); return NULL; } @@ -16,7 +16,7 @@ static void *cdlopen_fetch(PyObject *libname, void *libhandle, if (address == NULL) { const char *error = dlerror(); PyErr_Format(FFIError, "symbol '%s' not found in library '%s': %s", - symbol, PyText_AS_UTF8(libname), error); + symbol, PyUnicode_AsUTF8(libname), error); } return address; } @@ -32,7 +32,7 @@ static int cdlopen_close(PyObject *libname, void *libhandle) if (libhandle != NULL && dlclose(libhandle) != 0) { const char *error = dlerror(); PyErr_Format(FFIError, "closing library '%s': %s", - PyText_AS_UTF8(libname), error); + PyUnicode_AsUTF8(libname), error); return -1; } return 0; @@ -195,13 +195,6 @@ static int ffiobj_init(PyObject *self, PyObject *args, PyObject *kwds) _CFFI_GETOP(nglobs[i].type_op) == _CFFI_OP_ENUM) { PyObject *o = PyTuple_GET_ITEM(globals, i * 2 + 1); nglobs[i].address = &_cdl_realize_global_int; -#if PY_MAJOR_VERSION < 3 - if (PyInt_Check(o)) { - nintconsts[i].neg = PyInt_AS_LONG(o) <= 0; - nintconsts[i].value = (long long)PyInt_AS_LONG(o); - } - else -#endif { nintconsts[i].neg = PyObject_RichCompareBool(o, Py_False, Py_LE); diff --git a/contrib/python/cffi/py3/c/cffi1_module.c b/contrib/python/cffi/py3/c/cffi1_module.c index 70d9ee8e0fe..7c16b2b7073 100644 --- a/contrib/python/cffi/py3/c/cffi1_module.c +++ b/contrib/python/cffi/py3/c/cffi1_module.c @@ -46,7 +46,7 @@ static int init_ffi_lib(PyObject *m) return -1; for (i = 0; all_dlopen_flags[i].name != NULL; i++) { - x = PyInt_FromLong(all_dlopen_flags[i].value); + x = PyLong_FromLong(all_dlopen_flags[i].value); if (x == NULL) return -1; res = PyDict_SetItemString(FFI_Type.tp_dict, @@ -116,7 +116,6 @@ static int make_included_tuples(char *module_name, static PyObject *_my_Py_InitModule(char *module_name) { -#if PY_MAJOR_VERSION >= 3 struct PyModuleDef *module_def, local_module_def = { PyModuleDef_HEAD_INIT, module_name, @@ -137,9 +136,6 @@ static PyObject *_my_Py_InitModule(char *module_name) } # endif return m; -#else - return Py_InitModule(module_name, NULL); -#endif } static PyObject *b_init_cffi_1_0_external_module(PyObject *self, PyObject *arg) @@ -211,12 +207,10 @@ static PyObject *b_init_cffi_1_0_external_module(PyObject *self, PyObject *arg) (PyObject *)lib) < 0) return NULL; -#if PY_MAJOR_VERSION >= 3 /* add manually 'module_name' in sys.modules: it seems that Py_InitModule() is not enough to do that */ if (PyDict_SetItemString(modules_dict, module_name, m) < 0) return NULL; -#endif return m; } diff --git a/contrib/python/cffi/py3/c/cglob.c b/contrib/python/cffi/py3/c/cglob.c index e97767c9d75..6c6fd4bae2e 100644 --- a/contrib/python/cffi/py3/c/cglob.c +++ b/contrib/python/cffi/py3/c/cglob.c @@ -74,7 +74,7 @@ static void *fetch_global_var_addr(GlobSupportObject *gs) } if (data == NULL) { PyErr_Format(FFIError, "global variable '%s' is at address NULL", - PyText_AS_UTF8(gs->gs_name)); + PyUnicode_AsUTF8(gs->gs_name)); return NULL; } return data; diff --git a/contrib/python/cffi/py3/c/commontypes.c b/contrib/python/cffi/py3/c/commontypes.c index 2337bf99447..a15c3ae87b2 100644 --- a/contrib/python/cffi/py3/c/commontypes.c +++ b/contrib/python/cffi/py3/c/commontypes.c @@ -205,7 +205,7 @@ static PyObject *b__get_common_types(PyObject *self, PyObject *arg) size_t i; for (i = 0; i < num_common_simple_types; i++) { const char *s = common_simple_types[i]; - PyObject *o = PyText_FromString(s + strlen(s) + 1); + PyObject *o = PyUnicode_FromString(s + strlen(s) + 1); if (o == NULL) return NULL; err = PyDict_SetItemString(arg, s, o); diff --git a/contrib/python/cffi/py3/c/ffi_obj.c b/contrib/python/cffi/py3/c/ffi_obj.c index 0c4933eeff6..fbba28941ce 100644 --- a/contrib/python/cffi/py3/c/ffi_obj.c +++ b/contrib/python/cffi/py3/c/ffi_obj.c @@ -185,13 +185,13 @@ static CTypeDescrObject *_ffi_type(FFIObject *ffi, PyObject *arg, /* Returns the CTypeDescrObject from the user-supplied 'arg'. Does not return a new reference! */ - if ((accept & ACCEPT_STRING) && PyText_Check(arg)) { + if ((accept & ACCEPT_STRING) && PyUnicode_Check(arg)) { PyObject *types_dict = ffi->types_builder.types_dict; /* The types_dict keeps the reference alive. Items are never removed */ PyObject *x = PyDict_GetItem(types_dict, arg); if (x == NULL) { - const char *input_text = PyText_AS_UTF8(arg); + const char *input_text = PyUnicode_AsUTF8(arg); struct _cffi_parse_info_s info; info.ctx = &ffi->types_builder.ctx; info.output_size = FFI_COMPLEXITY_OUTPUT; @@ -241,17 +241,6 @@ static CTypeDescrObject *_ffi_type(FFIObject *ffi, PyObject *arg, else if ((accept & ACCEPT_CDATA) && CData_Check(arg)) { return ((CDataObject *)arg)->c_type; } -#if PY_MAJOR_VERSION < 3 - else if (PyUnicode_Check(arg)) { - CTypeDescrObject *result; - arg = PyUnicode_AsASCIIString(arg); - if (arg == NULL) - return NULL; - result = _ffi_type(ffi, arg, accept); - Py_DECREF(arg); - return result; - } -#endif else { const char *m1 = (accept & ACCEPT_STRING) ? "string" : ""; const char *m2 = (accept & ACCEPT_CTYPE) ? "ctype object" : ""; @@ -291,7 +280,7 @@ static PyObject *ffi_sizeof(FFIObject *self, PyObject *arg) return NULL; } } - return PyInt_FromSsize_t(size); + return PyLong_FromSsize_t(size); } PyDoc_STRVAR(ffi_alignof_doc, @@ -308,7 +297,7 @@ static PyObject *ffi_alignof(FFIObject *self, PyObject *arg) align = get_alignment(ct); if (align < 0) return NULL; - return PyInt_FromLong(align); + return PyLong_FromLong(align); } PyDoc_STRVAR(ffi_typeof_doc, @@ -526,7 +515,7 @@ static PyObject *ffi_offsetof(FFIObject *self, PyObject *args) return NULL; offset += ofs1; } - return PyInt_FromSsize_t(offset); + return PyLong_FromSsize_t(offset); } PyDoc_STRVAR(ffi_addressof_doc, @@ -639,9 +628,7 @@ static PyObject *ffi_getctype(FFIObject *self, PyObject *args, PyObject *kwds) CTypeDescrObject *ct; size_t replace_with_len; static char *keywords[] = {"cdecl", "replace_with", NULL}; -#if PY_MAJOR_VERSION >= 3 PyObject *u; -#endif if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|s:getctype", keywords, &c_decl, &replace_with)) @@ -675,14 +662,12 @@ static PyObject *ffi_getctype(FFIObject *self, PyObject *args, PyObject *kwds) if (add_paren) p[replace_with_len] = ')'; -#if PY_MAJOR_VERSION >= 3 /* bytes -> unicode string */ u = PyUnicode_DecodeLatin1(PyBytes_AS_STRING(res), PyBytes_GET_SIZE(res), NULL); Py_DECREF(res); res = u; -#endif return res; } @@ -931,7 +916,7 @@ static PyObject *ffi_list_types(FFIObject *self, PyObject *noargs) goto error; for (i = 0; i < n1; i++) { - o = PyText_FromString(self->types_builder.ctx.typenames[i].name); + o = PyUnicode_FromString(self->types_builder.ctx.typenames[i].name); if (o == NULL) goto error; PyList_SET_ITEM(lst[0], i, o); @@ -945,7 +930,7 @@ static PyObject *ffi_list_types(FFIObject *self, PyObject *noargs) if (s->name[0] == '$') continue; - o = PyText_FromString(s->name); + o = PyUnicode_FromString(s->name); if (o == NULL) goto error; index = (s->flags & _CFFI_F_UNION) ? 2 : 1; @@ -990,14 +975,6 @@ PyDoc_STRVAR(ffi_init_once_doc, "of function() is done. If function() raises an exception, it is\n" "propagated and nothing is cached."); -#if PY_MAJOR_VERSION < 3 -/* PyCapsule_New is redefined to be PyCObject_FromVoidPtr in _cffi_backend, - which gives 2.6 compatibility; but the destructor signature is different */ -static void _free_init_once_lock(void *lock) -{ - PyThread_free_lock((PyThread_type_lock)lock); -} -#else static void _free_init_once_lock(PyObject *capsule) { PyThread_type_lock lock; @@ -1005,7 +982,6 @@ static void _free_init_once_lock(PyObject *capsule) if (lock != NULL) PyThread_free_lock(lock); } -#endif static PyObject *ffi_init_once(FFIObject *self, PyObject *args, PyObject *kwds) { diff --git a/contrib/python/cffi/py3/c/file_emulator.h b/contrib/python/cffi/py3/c/file_emulator.h index 82a34c0c72f..bfdcdeceac2 100644 --- a/contrib/python/cffi/py3/c/file_emulator.h +++ b/contrib/python/cffi/py3/c/file_emulator.h @@ -51,7 +51,7 @@ static FILE *PyFile_AsFile(PyObject *ob_file) ob_mode = PyObject_GetAttrString(ob_file, "mode"); if (ob_mode == NULL) goto fail; - mode = PyText_AsUTF8(ob_mode); + mode = PyUnicode_AsUTF8(ob_mode); if (mode == NULL) goto fail; diff --git a/contrib/python/cffi/py3/c/lib_obj.c b/contrib/python/cffi/py3/c/lib_obj.c index 446b225cb5d..946db3b39dd 100644 --- a/contrib/python/cffi/py3/c/lib_obj.c +++ b/contrib/python/cffi/py3/c/lib_obj.c @@ -123,8 +123,8 @@ static int lib_traverse(LibObject *lib, visitproc visit, void *arg) static PyObject *lib_repr(LibObject *lib) { - return PyText_FromFormat("<Lib object for '%.200s'>", - PyText_AS_UTF8(lib->l_libname)); + return PyUnicode_FromFormat("<Lib object for '%.200s'>", + PyUnicode_AsUTF8(lib->l_libname)); } static PyObject *lib_build_cpython_func(LibObject *lib, @@ -143,7 +143,7 @@ static PyObject *lib_build_cpython_func(LibObject *lib, int i, type_index = _CFFI_GETARG(g->type_op); _cffi_opcode_t *opcodes = lib->l_types_builder->ctx.types; static const char *const format = ";\n\nCFFI C function from %s.lib"; - const char *libname = PyText_AS_UTF8(lib->l_libname); + const char *libname = PyUnicode_AsUTF8(lib->l_libname); struct funcbuilder_s funcbuilder; /* return type: */ @@ -226,7 +226,7 @@ static PyObject *lib_build_and_cache_attr(LibObject *lib, PyObject *name, const struct _cffi_global_s *g; CTypeDescrObject *ct; builder_c_t *types_builder = lib->l_types_builder; - const char *s = PyText_AsUTF8(name); + const char *s = PyUnicode_AsUTF8(name); if (s == NULL) return NULL; @@ -281,7 +281,7 @@ static PyObject *lib_build_and_cache_attr(LibObject *lib, PyObject *name, PyErr_Format(PyExc_AttributeError, "cffi library '%.200s' has no function, constant " "or global variable named '%.200s'", - PyText_AS_UTF8(lib->l_libname), s); + PyUnicode_AsUTF8(lib->l_libname), s); return NULL; } @@ -478,7 +478,7 @@ static PyObject *_lib_dir1(LibObject *lib, int ignore_global_vars) if (op == _CFFI_OP_GLOBAL_VAR || op == _CFFI_OP_GLOBAL_VAR_F) continue; } - s = PyText_FromString(g[i].name); + s = PyUnicode_FromString(g[i].name); if (s == NULL) goto error; PyList_SET_ITEM(lst, count, s); @@ -502,7 +502,7 @@ static PyObject *_lib_dict(LibObject *lib) return NULL; for (i = 0; i < total; i++) { - name = PyText_FromString(g[i].name); + name = PyUnicode_FromString(g[i].name); if (name == NULL) goto error; @@ -534,7 +534,7 @@ static PyObject *lib_getattr(LibObject *lib, PyObject *name) missing: /*** ATTRIBUTEERROR IS SET HERE ***/ - p = PyText_AsUTF8(name); + p = PyUnicode_AsUTF8(name); if (p == NULL) return NULL; if (strcmp(p, "__all__") == 0) { @@ -558,16 +558,14 @@ static PyObject *lib_getattr(LibObject *lib, PyObject *name) module-like behavior */ if (strcmp(p, "__name__") == 0) { PyErr_Clear(); - return PyText_FromFormat("%s.lib", PyText_AS_UTF8(lib->l_libname)); + return PyUnicode_FromFormat("%s.lib", PyUnicode_AsUTF8(lib->l_libname)); } -#if PY_MAJOR_VERSION >= 3 if (strcmp(p, "__loader__") == 0 || strcmp(p, "__spec__") == 0) { /* some more module-like behavior hacks */ PyErr_Clear(); Py_INCREF(Py_None); return Py_None; } -#endif return NULL; } @@ -587,7 +585,7 @@ static int lib_setattr(LibObject *lib, PyObject *name, PyObject *val) PyErr_Format(PyExc_AttributeError, "cannot write to function or constant '%.200s'", - PyText_Check(name) ? PyText_AS_UTF8(name) : "?"); + PyUnicode_Check(name) ? PyUnicode_AsUTF8(name) : "?"); return -1; } @@ -645,7 +643,7 @@ static LibObject *lib_internal_new(FFIObject *ffi, const char *module_name, LibObject *lib; PyObject *libname, *dict; - libname = PyText_FromString(module_name); + libname = PyUnicode_FromString(module_name); if (libname == NULL) goto err1; @@ -712,7 +710,7 @@ static PyObject *address_of_global_var(PyObject *args) /* rebuild a string from 'varname', to do typechecks and to force a unicode back to a plain string (on python 2) */ - o_varname = PyText_FromString(varname); + o_varname = PyUnicode_FromString(varname); if (o_varname == NULL) return NULL; diff --git a/contrib/python/cffi/py3/c/minibuffer.h b/contrib/python/cffi/py3/c/minibuffer.h index d92da317ade..89e400fa069 100644 --- a/contrib/python/cffi/py3/c/minibuffer.h +++ b/contrib/python/cffi/py3/c/minibuffer.h @@ -50,7 +50,7 @@ static int mb_ass_item(MiniBufferObj *self, Py_ssize_t idx, PyObject *other) } else { PyErr_Format(PyExc_TypeError, - "must assign a "STR_OR_BYTES + "must assign a bytes" " of length 1, not %.200s", Py_TYPE(other)->tp_name); return -1; } @@ -85,29 +85,6 @@ static int mb_ass_slice(MiniBufferObj *self, return 0; } -#if PY_MAJOR_VERSION < 3 -static Py_ssize_t mb_getdata(MiniBufferObj *self, Py_ssize_t idx, void **pp) -{ - *pp = self->mb_data; - return self->mb_size; -} - -static Py_ssize_t mb_getsegcount(MiniBufferObj *self, Py_ssize_t *lenp) -{ - if (lenp) - *lenp = self->mb_size; - return 1; -} - -static PyObject *mb_str(MiniBufferObj *self) -{ - /* Python 2: we want str(buffer) to behave like buffer[:], because - that's what bytes(buffer) does on Python 3 and there is no way - we can prevent this. */ - return PyString_FromStringAndSize(self->mb_data, self->mb_size); -} -#endif - static int mb_getbuf(MiniBufferObj *self, Py_buffer *view, int flags) { return PyBuffer_FillInfo(view, (PyObject *)self, @@ -126,12 +103,6 @@ static PySequenceMethods mb_as_sequence = { }; static PyBufferProcs mb_as_buffer = { -#if PY_MAJOR_VERSION < 3 - (readbufferproc)mb_getdata, - (writebufferproc)mb_getdata, - (segcountproc)mb_getsegcount, - (charbufferproc)mb_getdata, -#endif (getbufferproc)mb_getbuf, (releasebufferproc)0, }; @@ -234,16 +205,8 @@ mb_richcompare(PyObject *self, PyObject *other, int op) return res; } -#if PY_MAJOR_VERSION >= 3 /* pfffffffffffff pages of copy-paste from listobject.c */ -/* pfffffffffffff#2: the PySlice_GetIndicesEx() *macro* should not - be called, because C extension modules compiled with it differ - on ABI between 3.6.0, 3.6.1 and 3.6.2. */ -#if PY_VERSION_HEX < 0x03070000 && defined(PySlice_GetIndicesEx) && !defined(PYPY_VERSION) -#undef PySlice_GetIndicesEx -#endif - static PyObject *mb_subscript(MiniBufferObj *self, PyObject *item) { if (PyIndex_Check(item)) { @@ -280,6 +243,11 @@ static PyObject *mb_subscript(MiniBufferObj *self, PyObject *item) static int mb_ass_subscript(MiniBufferObj* self, PyObject* item, PyObject* value) { + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, + "'del x[n]' not supported for buffer objects"); + return -1; + } if (PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); if (i == -1 && PyErr_Occurred()) @@ -317,13 +285,6 @@ static PyMappingMethods mb_as_mapping = { (binaryfunc)mb_subscript, /*mp_subscript*/ (objobjargproc)mb_ass_subscript, /*mp_ass_subscript*/ }; -#endif - -#if PY_MAJOR_VERSION >= 3 -# define MINIBUF_TPFLAGS 0 -#else -# define MINIBUF_TPFLAGS (Py_TPFLAGS_HAVE_GETCHARBUFFER | Py_TPFLAGS_HAVE_NEWBUFFER) -#endif PyDoc_STRVAR(ffi_buffer_doc, "ffi.buffer(cdata[, byte_size]):\n" @@ -354,23 +315,14 @@ static PyTypeObject MiniBuffer_Type = { 0, /* tp_repr */ 0, /* tp_as_number */ &mb_as_sequence, /* tp_as_sequence */ -#if PY_MAJOR_VERSION < 3 - 0, /* tp_as_mapping */ -#else &mb_as_mapping, /* tp_as_mapping */ -#endif 0, /* tp_hash */ 0, /* tp_call */ -#if PY_MAJOR_VERSION < 3 - (reprfunc)mb_str, /* tp_str */ -#else 0, /* tp_str */ -#endif PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ &mb_as_buffer, /* tp_as_buffer */ - (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | - MINIBUF_TPFLAGS), /* tp_flags */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ ffi_buffer_doc, /* tp_doc */ (traverseproc)mb_traverse, /* tp_traverse */ (inquiry)mb_clear, /* tp_clear */ diff --git a/contrib/python/cffi/py3/c/misc_thread_common.h b/contrib/python/cffi/py3/c/misc_thread_common.h index b0710acceec..766dcb2ad7d 100644 --- a/contrib/python/cffi/py3/c/misc_thread_common.h +++ b/contrib/python/cffi/py3/c/misc_thread_common.h @@ -328,20 +328,12 @@ static void restore_errno_only(void) It was added in 3.5.2 but should never be used in 3.5.x because it is not available in 3.5.0 or 3.5.1. */ -#if PY_VERSION_HEX >= 0x03050100 && PY_VERSION_HEX < 0x03060000 -PyAPI_DATA(void *volatile) _PyThreadState_Current; -#endif - static PyThreadState *get_current_ts(void) { #if PY_VERSION_HEX >= 0x030D0000 return PyThreadState_GetUnchecked(); -#elif PY_VERSION_HEX >= 0x03060000 - return _PyThreadState_UncheckedGet(); -#elif defined(_Py_atomic_load_relaxed) - return (PyThreadState*)_Py_atomic_load_relaxed(&_PyThreadState_Current); #else - return (PyThreadState*)_PyThreadState_Current; /* assume atomic read */ + return _PyThreadState_UncheckedGet(); #endif } diff --git a/contrib/python/cffi/py3/c/misc_win32.h b/contrib/python/cffi/py3/c/misc_win32.h index 90c63adc68c..4fe723a727d 100644 --- a/contrib/python/cffi/py3/c/misc_win32.h +++ b/contrib/python/cffi/py3/c/misc_win32.h @@ -64,7 +64,6 @@ static void restore_errno(void) /************************************************************/ -#if PY_MAJOR_VERSION >= 3 static PyObject *b_getwinerror(PyObject *self, PyObject *args, PyObject *kwds) { int err = -1; @@ -113,54 +112,6 @@ static PyObject *b_getwinerror(PyObject *self, PyObject *args, PyObject *kwds) LocalFree(s_buf); return v; } -#else -static PyObject *b_getwinerror(PyObject *self, PyObject *args, PyObject *kwds) -{ - int err = -1; - int len; - char *s; - char *s_buf = NULL; /* Free via LocalFree */ - char s_small_buf[40]; /* Room for "Windows Error 0xFFFFFFFFFFFFFFFF" */ - PyObject *v; - static char *keywords[] = {"code", NULL}; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|i", keywords, &err)) - return NULL; - - if (err == -1) { - struct cffi_tls_s *p = get_cffi_tls(); - if (p == NULL) - return PyErr_NoMemory(); - err = p->saved_lasterror; - } - - len = FormatMessage( - /* Error API error */ - FORMAT_MESSAGE_ALLOCATE_BUFFER | - FORMAT_MESSAGE_FROM_SYSTEM | - FORMAT_MESSAGE_IGNORE_INSERTS, - NULL, /* no message source */ - err, - MAKELANGID(LANG_NEUTRAL, - SUBLANG_DEFAULT), /* Default language */ - (LPTSTR) &s_buf, - 0, /* size not used */ - NULL); /* no args */ - if (len==0) { - /* Only seen this in out of mem situations */ - sprintf(s_small_buf, "Windows Error 0x%X", err); - s = s_small_buf; - } else { - s = s_buf; - /* remove trailing cr/lf and dots */ - while (len > 0 && (s[len-1] <= ' ' || s[len-1] == '.')) - s[--len] = '\0'; - } - v = Py_BuildValue("(is)", err, s); - LocalFree(s_buf); - return v; -} -#endif /************************************************************/ @@ -251,7 +202,9 @@ static int cffi_atomic_compare_exchange(void **ptr, void **expected, static void *cffi_atomic_load(void **ptr) { -#if defined(_M_X64) || defined(_M_IX86) +#if defined(__GNUC__) || defined(__clang__) + return __atomic_load_n(ptr, __ATOMIC_SEQ_CST); +#elif defined(_M_X64) || defined(_M_IX86) return *(volatile void **)ptr; #elif defined(_M_ARM64) return (void *)__ldar64((volatile unsigned __int64 *)ptr); @@ -262,7 +215,9 @@ static void *cffi_atomic_load(void **ptr) static uint8_t cffi_atomic_load_uint8(uint8_t *ptr) { -#if defined(_M_X64) || defined(_M_IX86) +#if defined(__GNUC__) || defined(__clang__) + return __atomic_load_n(ptr, __ATOMIC_SEQ_CST); +#elif defined(_M_X64) || defined(_M_IX86) return *(volatile uint8_t *)ptr; #elif defined(_M_ARM64) return (uint8_t)__ldar8((volatile uint8_t *)ptr); @@ -273,7 +228,9 @@ static uint8_t cffi_atomic_load_uint8(uint8_t *ptr) static Py_ssize_t cffi_atomic_load_ssize(Py_ssize_t *ptr) { -#if defined(_M_X64) || defined(_M_IX86) +#if defined(__GNUC__) || defined(__clang__) + return __atomic_load_n(ptr, __ATOMIC_SEQ_CST); +#elif defined(_M_X64) || defined(_M_IX86) return *(volatile Py_ssize_t *)ptr; #elif defined(_M_ARM64) return (Py_ssize_t)__ldar64((volatile unsigned __int64 *)ptr); @@ -284,17 +241,29 @@ static Py_ssize_t cffi_atomic_load_ssize(Py_ssize_t *ptr) static void cffi_atomic_store_ssize(Py_ssize_t *ptr, Py_ssize_t value) { +#if defined(__GNUC__) || defined(__clang__) + __atomic_store_n(ptr, value, __ATOMIC_SEQ_CST); +#else _InterlockedExchangePointer(ptr, value); +#endif } static void cffi_atomic_store(void **ptr, void *value) { +#if defined(__GNUC__) || defined(__clang__) + __atomic_store_n(ptr, value, __ATOMIC_SEQ_CST); +#else _InterlockedExchangePointer(ptr, value); +#endif } static void cffi_atomic_store_uint8(uint8_t *ptr, uint8_t value) { +#if defined(__GNUC__) || defined(__clang__) + __atomic_store_n(ptr, value, __ATOMIC_SEQ_CST); +#else _InterlockedExchange8(ptr, value); +#endif } #endif /* CFFI_MISC_WIN32_H */ diff --git a/contrib/python/cffi/py3/c/realize_c_type.c b/contrib/python/cffi/py3/c/realize_c_type.c index fbfa68ca748..993a07d122f 100644 --- a/contrib/python/cffi/py3/c/realize_c_type.c +++ b/contrib/python/cffi/py3/c/realize_c_type.c @@ -29,7 +29,6 @@ static PyObject *build_primitive_type(int num); /* forward */ static int init_global_types_dict(PyObject *ffi_type_dict) { int err; - Py_ssize_t i; PyObject *ct_void, *ct_char, *ct2, *pnull; /* XXX some leaks in case these functions fail, but well, MemoryErrors during importing an extension module are kind @@ -78,7 +77,7 @@ static int init_global_types_dict(PyObject *ffi_type_dict) #ifdef Py_GIL_DISABLED // Ensure that all primitive types are initialised to avoid race conditions // on the first access. - for (i = 0; i < _CFFI__NUM_PRIM; i++) { + for (Py_ssize_t i = 0; i < _CFFI__NUM_PRIM; i++) { ct2 = get_primitive_type(i); if (ct2 == NULL) return -1; @@ -247,13 +246,13 @@ static PyObject *realize_global_int(builder_c_t *builder, int gindex) case 0: if (value <= (unsigned long long)LONG_MAX) - return PyInt_FromLong((long)value); + return PyLong_FromLong((long)value); else return PyLong_FromUnsignedLongLong(value); case 1: if ((long long)value >= (long long)LONG_MIN) - return PyInt_FromLong((long)value); + return PyLong_FromLong((long)value); else return PyLong_FromLongLong((long long)value); @@ -551,7 +550,7 @@ realize_c_type_or_func_now(builder_c_t *builder, _cffi_opcode_t op, j = 0; while (p[j] != ',' && p[j] != '\0') j++; - tmp = PyText_FromStringAndSize(p, j); + tmp = PyUnicode_FromStringAndSize(p, j); if (tmp == NULL) break; PyTuple_SET_ITEM(enumerators, i, tmp); diff --git a/contrib/python/cffi/py3/cffi/__init__.py b/contrib/python/cffi/py3/cffi/__init__.py index c99ec3d4817..da1875cc253 100644 --- a/contrib/python/cffi/py3/cffi/__init__.py +++ b/contrib/python/cffi/py3/cffi/__init__.py @@ -5,8 +5,8 @@ from .api import FFI from .error import CDefError, FFIError, VerificationError, VerificationMissing from .error import PkgConfigError -__version__ = "2.0.0" -__version_info__ = (2, 0, 0) +__version__ = "2.1.0" +__version_info__ = (2, 1, 0) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/contrib/python/cffi/py3/cffi/_cffi_errors.h b/contrib/python/cffi/py3/cffi/_cffi_errors.h index 158e0590346..0931379ccd2 100644 --- a/contrib/python/cffi/py3/cffi/_cffi_errors.h +++ b/contrib/python/cffi/py3/cffi/_cffi_errors.h @@ -36,11 +36,7 @@ static PyObject *_cffi_start_error_capture(void) if (result == NULL) goto error; -#if PY_MAJOR_VERSION >= 3 bi = PyImport_ImportModule("builtins"); -#else - bi = PyImport_ImportModule("__builtin__"); -#endif if (bi == NULL) goto error; PyDict_SetItemString(result, "__builtins__", bi); @@ -81,15 +77,9 @@ static PyObject *_cffi_start_error_capture(void) static DWORD WINAPI _cffi_bootstrap_dialog(LPVOID ignored) { Sleep(666); /* may be interrupted if the whole process is closing */ -#if PY_MAJOR_VERSION >= 3 MessageBoxW(NULL, (wchar_t *)_cffi_bootstrap_text, L"Python-CFFI error", MB_OK | MB_ICONERROR); -#else - MessageBoxA(NULL, (char *)_cffi_bootstrap_text, - "Python-CFFI error", - MB_OK | MB_ICONERROR); -#endif _cffi_bootstrap_text = NULL; return 0; } @@ -111,11 +101,7 @@ static void _cffi_stop_error_capture(PyObject *ecap) /* Show a dialog box, but in a background thread, and never show multiple dialog boxes at once. */ -#if PY_MAJOR_VERSION >= 3 text = PyUnicode_AsWideCharString(s, NULL); -#else - text = PyString_AsString(s); -#endif _cffi_bootstrap_text = text; diff --git a/contrib/python/cffi/py3/cffi/_cffi_gen_src.py b/contrib/python/cffi/py3/cffi/_cffi_gen_src.py new file mode 100644 index 00000000000..3d4fb7e3d9c --- /dev/null +++ b/contrib/python/cffi/py3/cffi/_cffi_gen_src.py @@ -0,0 +1,216 @@ +# Integrated from the cffi-buildtool project by Rose Davidson +# (https://github.com/inklesspen/cffi-buildtool), under the following +# license: +# +# MIT License +# +# Copyright (c) 2024, Rose Davidson +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice (including the +# next paragraph) shall be included in all copies or substantial portions +# of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +"""Implementation of the ``cffi-gen-src`` command-line tool. + +This module is private; the command line is the only supported +interface. Two subcommands: + +``exec-python`` + Execute a Python script that constructs a :class:`cffi.FFI` + (the same kind of script that the CFFI docs' "Main mode of usage" + describes) and emit the generated C source. + +``read-sources`` + Create the :class:`cffi.FFI` from a separate ``cdef`` file and C + source prelude, then emit the generated C source. +""" + +import argparse +import io +import os +import sys + +from .api import FFI + + +def _execfile(pysrc, filename, globs): + compiled = compile(source=pysrc, filename=filename, mode='exec') + exec(compiled, globs, globs) + + +def find_ffi_in_python_script(pysrc, filename, ffivar): + """Execute ``pysrc`` and return the :class:`FFI` object it defines. + + The script is executed with ``__name__`` set to ``"cffi.gen_src"``, + so a trailing ``if __name__ == "__main__": ffibuilder.compile()`` + block in the script is skipped. + + ``ffivar`` is the name bound by the script to the :class:`FFI` + object, or to a callable that returns one. + + Raises :class:`NameError` if the name is not bound by the script, + or :class:`TypeError` if the name does not resolve to an + :class:`FFI` instance. + """ + filename = os.path.abspath(filename) + globs = {'__name__': 'cffi.gen_src', '__file__': filename} + old_path = sys.path[:] + sys.path.insert(0, os.path.dirname(filename)) + try: + _execfile(pysrc, filename, globs) + if ffivar not in globs: + raise NameError( + "Expected to find the FFI object with the name %r, " + "but it was not found." % (ffivar,) + ) + ffi = globs[ffivar] + if not isinstance(ffi, FFI) and callable(ffi): + # Maybe it's a callable that returns a FFI + ffi = ffi() + if not isinstance(ffi, FFI): + raise TypeError( + "Found an object with the name %r but it was not an " + "instance of cffi.api.FFI" % (ffivar,) + ) + return ffi + finally: + sys.path[:] = old_path + + +def make_ffi_from_sources(modulename, cdef, csrc): + """Create an :class:`FFI` from ``cdef`` text and a C source prelude.""" + ffibuilder = FFI() + ffibuilder.cdef(cdef) + ffibuilder.set_source(modulename, csrc) + return ffibuilder + + +def generate_c_source(ffi): + """Return the C source that :meth:`FFI.emit_c_code` would write.""" + output = io.StringIO() + ffi.emit_c_code(output) + return output.getvalue() + + +def write_c_source(output, generated): + if output == '-': + sys.stdout.write(generated) + return + with open(output, 'w', encoding='utf-8') as f: + f.write(generated) + + +def same_input_file(file1, file2): + if file1 is file2: + return True + try: + return os.path.samefile(file1.name, file2.name) + except OSError: + return False + + +def exec_python(*, output, pyfile, ffi_var): + with pyfile: + ffi = find_ffi_in_python_script(pyfile.read(), pyfile.name, ffi_var) + generated = generate_c_source(ffi) + write_c_source(output, generated) + + +def read_sources(*, output, module_name, cdef_input, csrc_input): + with csrc_input, cdef_input: + csrc = csrc_input.read() + cdef = cdef_input.read() + ffi = make_ffi_from_sources(module_name, cdef, csrc) + generated = generate_c_source(ffi) + write_c_source(output, generated) + + +def _prog(): + # The same parser serves both documented invocations; make --help + # and usage messages show the one that was actually used. + argv0 = os.path.basename(sys.argv[0]) if sys.argv else '' + if argv0.startswith('cffi-gen-src'): + return 'cffi-gen-src' + return 'python -m cffi.gen_src' + + +parser = argparse.ArgumentParser( + prog=_prog(), + description='Generate CFFI C source for extension modules.', +) +subparsers = parser.add_subparsers(dest='mode') + +exec_python_parser = subparsers.add_parser( + 'exec-python', + help='Execute a Python script that defines an FFI object', +) +exec_python_parser.add_argument( + '--ffi-var', + default='ffibuilder', + help="Name of the FFI object in the Python script; defaults to 'ffibuilder'.", +) +exec_python_parser.add_argument( + 'pyfile', + type=argparse.FileType('r', encoding='utf-8'), + help='Path to the Python script', +) +exec_python_parser.add_argument( + 'output', + help='Output path for the C source', +) + +read_sources_parser = subparsers.add_parser( + 'read-sources', + help='Read cdef and C source prelude files that define an FFI object', +) +read_sources_parser.add_argument( + 'module_name', + help='Full name of the generated module, including packages', +) +read_sources_parser.add_argument( + 'cdef', + type=argparse.FileType('r', encoding='utf-8'), + help='File containing C definitions', +) +read_sources_parser.add_argument( + 'csrc', + type=argparse.FileType('r', encoding='utf-8'), + help='File containing C source prelude', +) +read_sources_parser.add_argument( + 'output', + help='Output path for the C source', +) + + +def run(args=None): + args = parser.parse_args(args=args) + if args.mode == 'exec-python': + exec_python(output=args.output, pyfile=args.pyfile, ffi_var=args.ffi_var) + elif args.mode == 'read-sources': + if same_input_file(args.cdef, args.csrc): + parser.error('cdef and csrc are the same file and should not be') + read_sources( + output=args.output, + module_name=args.module_name, + cdef_input=args.cdef, + csrc_input=args.csrc, + ) + else: + parser.error('a subcommand is required: exec-python or read-sources') + parser.exit(0) diff --git a/contrib/python/cffi/py3/cffi/_cffi_include.h b/contrib/python/cffi/py3/cffi/_cffi_include.h index 908a1d7343f..c8c6d343bcc 100644 --- a/contrib/python/cffi/py3/cffi/_cffi_include.h +++ b/contrib/python/cffi/py3/cffi/_cffi_include.h @@ -28,8 +28,13 @@ #if !defined(_CFFI_USE_EMBEDDING) && !defined(Py_LIMITED_API) # ifdef _MSC_VER # if !defined(_DEBUG) && !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) && !defined(_CFFI_NO_LIMITED_API) -# define Py_LIMITED_API +# if !defined(Py_GIL_DISABLED) +# define Py_LIMITED_API +# else +# define Py_LIMITED_API 0x030f0000 +# endif # endif + # include <pyconfig.h> /* sanity-check: Py_LIMITED_API will cause crashes if any of these are also defined. Normally, the Python file PC/pyconfig.h does not @@ -49,7 +54,11 @@ # else # include <pyconfig.h> # if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) && !defined(_CFFI_NO_LIMITED_API) -# define Py_LIMITED_API +# if !defined(Py_GIL_DISABLED) +# define Py_LIMITED_API +# else +# define Py_LIMITED_API 0x030f0000 +# endif # endif # endif #endif @@ -59,6 +68,9 @@ extern "C" { #endif #include <stddef.h> +#include <stdlib.h> +#include <string.h> + #include "parse_c_type.h" /* this block of #ifs should be kept exactly identical between @@ -128,13 +140,9 @@ extern "C" { #ifndef PYPY_VERSION -#if PY_MAJOR_VERSION >= 3 -# define PyInt_FromLong PyLong_FromLong -#endif - #define _cffi_from_c_double PyFloat_FromDouble #define _cffi_from_c_float PyFloat_FromDouble -#define _cffi_from_c_long PyInt_FromLong +#define _cffi_from_c_long PyLong_FromLong #define _cffi_from_c_ulong PyLong_FromUnsignedLong #define _cffi_from_c_longlong PyLong_FromLongLong #define _cffi_from_c_ulonglong PyLong_FromUnsignedLongLong @@ -146,12 +154,12 @@ extern "C" { #define _cffi_from_c_int(x, type) \ (((type)-1) > 0 ? /* unsigned */ \ (sizeof(type) < sizeof(long) ? \ - PyInt_FromLong((long)x) : \ + PyLong_FromLong((long)x) : \ sizeof(type) == sizeof(long) ? \ PyLong_FromUnsignedLong((unsigned long)x) : \ PyLong_FromUnsignedLongLong((unsigned long long)x)) : \ (sizeof(type) <= sizeof(long) ? \ - PyInt_FromLong((long)x) : \ + PyLong_FromLong((long)x) : \ PyLong_FromLongLong((long long)x))) #define _cffi_to_c_int(o, type) \ diff --git a/contrib/python/cffi/py3/cffi/_embedding.h b/contrib/python/cffi/py3/cffi/_embedding.h index 64c04f67cac..12ff017fb83 100644 --- a/contrib/python/cffi/py3/cffi/_embedding.h +++ b/contrib/python/cffi/py3/cffi/_embedding.h @@ -177,9 +177,6 @@ static int _cffi_initialize_python(void) if (PyDict_SetItemString(global_dict, "__builtins__", builtins) < 0) goto error; x = PyEval_EvalCode( -#if PY_MAJOR_VERSION < 3 - (PyCodeObject *) -#endif pycode, global_dict, global_dict); if (x == NULL) goto error; @@ -225,7 +222,7 @@ static int _cffi_initialize_python(void) if (f != NULL && f != Py_None) { PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME - "\ncompiled with cffi version: 2.0.0" + "\ncompiled with cffi version: 2.1.0" "\n_cffi_backend module: ", f); modules = PyImport_GetModuleDict(); mod = PyDict_GetItemString(modules, "_cffi_backend"); @@ -247,10 +244,6 @@ static int _cffi_initialize_python(void) goto done; } -#if PY_VERSION_HEX < 0x03080000 -PyAPI_DATA(char *) _PyParser_TokenNames[]; /* from CPython */ -#endif - static int _cffi_carefully_make_gil(void) { /* This does the basic initialization of Python. It can be called @@ -295,27 +288,6 @@ static int _cffi_carefully_make_gil(void) */ #ifdef WITH_THREAD -# if PY_VERSION_HEX < 0x03080000 - char *volatile *lock = (char *volatile *)_PyParser_TokenNames; - char *old_value, *locked_value; - - while (1) { /* spin loop */ - old_value = *lock; - locked_value = old_value + 1; - if (old_value[0] == 'E') { - assert(old_value[1] == 'N'); - if (cffi_compare_and_swap(lock, old_value, locked_value)) - break; - } - else { - assert(old_value[0] == 'N'); - /* should ideally do a spin loop instruction here, but - hard to do it portably and doesn't really matter I - think: PyEval_InitThreads() should be very fast, and - this is only run at start-up anyway. */ - } - } -# else # if PY_VERSION_HEX < 0x030C0000 int volatile *lock = (int volatile *)&PyCapsule_Type.tp_version_tag; int old_value, locked_value = -42; @@ -348,26 +320,16 @@ static int _cffi_carefully_make_gil(void) this is only run at start-up anyway. */ } } -# endif #endif /* call Py_InitializeEx() */ if (!Py_IsInitialized()) { _cffi_py_initialize(); -#if PY_VERSION_HEX < 0x03070000 - PyEval_InitThreads(); -#endif PyEval_SaveThread(); /* release the GIL */ /* the returned tstate must be the one that has been stored into the autoTLSkey by _PyGILState_Init() called from Py_Initialize(). */ } else { -#if PY_VERSION_HEX < 0x03070000 - /* PyEval_InitThreads() is always a no-op from CPython 3.7 */ - PyGILState_STATE state = PyGILState_Ensure(); - PyEval_InitThreads(); - PyGILState_Release(state); -#endif } #ifdef WITH_THREAD diff --git a/contrib/python/cffi/py3/cffi/api.py b/contrib/python/cffi/py3/cffi/api.py index 5a474f3da92..ec8ee7dfd2b 100644 --- a/contrib/python/cffi/py3/cffi/api.py +++ b/contrib/python/cffi/py3/cffi/api.py @@ -4,13 +4,6 @@ from .error import CDefError from . import model try: - callable -except NameError: - # Python 3.1 - from collections import Callable - callable = lambda x: isinstance(x, Callable) - -try: basestring except NameError: # Python 3.x @@ -20,7 +13,7 @@ _unspecified = object() -class FFI(object): +class FFI: r''' The main top-level class that you instantiate once, or once per module. @@ -414,7 +407,7 @@ class FFI(object): if (replace_with.startswith('*') and '&[' in self._backend.getcname(cdecl, '&')): replace_with = '(%s)' % replace_with - elif replace_with and not replace_with[0] in '[(': + elif replace_with and replace_with[0] not in '[(': replace_with = ' ' + replace_with return self._backend.getcname(cdecl, replace_with) @@ -909,7 +902,7 @@ def _make_ffi_library(ffi, libname, flags): raise AttributeError(name) accessors[name](name) # - class FFILibrary(object): + class FFILibrary: def __getattr__(self, name): make_accessor(name) return getattr(self, name) diff --git a/contrib/python/cffi/py3/cffi/backend_ctypes.py b/contrib/python/cffi/py3/cffi/backend_ctypes.py index e7956a79cfb..12cd16259db 100644 --- a/contrib/python/cffi/py3/cffi/backend_ctypes.py +++ b/contrib/python/cffi/py3/cffi/backend_ctypes.py @@ -12,7 +12,7 @@ else: class CTypesType(type): pass -class CTypesData(object): +class CTypesData: __metaclass__ = CTypesType __slots__ = ['__weakref__'] __name__ = '<cdata>' @@ -270,7 +270,7 @@ class CTypesBaseStructOrUnion(CTypesData): return CTypesData.__repr__(self, c_name or self._get_c_name(' &')) -class CTypesBackend(object): +class CTypesBackend: PRIMITIVE_TYPES = { 'char': ctypes.c_char, @@ -336,7 +336,6 @@ class CTypesBackend(object): if novalue is not None: raise TypeError("None expected, got %s object" % (type(novalue).__name__,)) - return None CTypesVoid._fix_class() return CTypesVoid @@ -387,7 +386,7 @@ class CTypesBackend(object): return ctype() return ctype(CTypesPrimitive._to_ctypes(init)) - if kind == 'int' or kind == 'byte': + if kind in {'int', 'byte'}: @classmethod def _cast_from(cls, source): source = _cast_source_to_int(source) @@ -435,7 +434,7 @@ class CTypesBackend(object): _cast_to_integer = __int__ - if kind == 'int' or kind == 'byte' or kind == 'bool': + if kind in {'int', 'byte', 'bool'}: @staticmethod def _to_ctypes(x): if not isinstance(x, (int, long)): @@ -558,15 +557,15 @@ class CTypesBackend(object): def __setitem__(self, index, value): self._as_ctype_ptr[index] = BItem._to_ctypes(value) - if kind == 'charp' or kind == 'voidp': + if kind in {'charp', 'voidp'}: @classmethod def _arg_to_ctypes(cls, *value): if value and isinstance(value[0], bytes): return ctypes.c_char_p(value[0]) else: - return super(CTypesPtr, cls)._arg_to_ctypes(*value) + return super()._arg_to_ctypes(*value) - if kind == 'charp' or kind == 'bytep': + if kind in {'charp', 'bytep'}: def _to_string(self, maxlen): if maxlen < 0: maxlen = sys.maxsize @@ -581,7 +580,7 @@ class CTypesBackend(object): if getattr(self, '_own', False): return 'owning %d bytes' % ( ctypes.sizeof(self._as_ctype_ptr.contents),) - return super(CTypesPtr, self)._get_own_repr() + return super()._get_own_repr() # if (BItem is self.ffi._get_cached_btype(model.void_type) or BItem is self.ffi._get_cached_btype(model.PrimitiveType('char'))): @@ -663,7 +662,7 @@ class CTypesBackend(object): raise IndexError self._blob[index] = BItem._to_ctypes(value) - if kind == 'char' or kind == 'byte': + if kind in {'char', 'byte'}: def _to_string(self, maxlen): if maxlen < 0: maxlen = len(self._blob) @@ -677,7 +676,7 @@ class CTypesBackend(object): def _get_own_repr(self): if getattr(self, '_own', False): return 'owning %d bytes' % (ctypes.sizeof(self._blob),) - return super(CTypesArray, self)._get_own_repr() + return super()._get_own_repr() def _convert_to_address(self, BClass): if BClass in (CTypesPtr, None) or BClass._automatic_casts: @@ -917,7 +916,7 @@ class CTypesBackend(object): def _get_own_repr(self): if getattr(self, '_own_callback', None) is not None: return 'calling %r' % (self._own_callback,) - return super(CTypesFunctionPtr, self)._get_own_repr() + return super()._get_own_repr() def __call__(self, *args): if has_varargs: @@ -1094,7 +1093,7 @@ class CTypesBackend(object): return BTypePtr._from_ctypes(ptr) -class CTypesLibrary(object): +class CTypesLibrary: def __init__(self, backend, cdll): self.backend = backend diff --git a/contrib/python/cffi/py3/cffi/cffi_opcode.py b/contrib/python/cffi/py3/cffi/cffi_opcode.py index 6421df62134..c964ba13e69 100644 --- a/contrib/python/cffi/py3/cffi/cffi_opcode.py +++ b/contrib/python/cffi/py3/cffi/cffi_opcode.py @@ -1,6 +1,6 @@ from .error import VerificationError -class CffiOp(object): +class CffiOp: def __init__(self, op, arg): self.op = op self.arg = arg diff --git a/contrib/python/cffi/py3/cffi/cparser.py b/contrib/python/cffi/py3/cffi/cparser.py index dd590d8743d..1896b7ef5dc 100644 --- a/contrib/python/cffi/py3/cffi/cparser.py +++ b/contrib/python/cffi/py3/cffi/cparser.py @@ -292,7 +292,7 @@ def _common_type_names(csource): return words_used -class Parser(object): +class Parser: def __init__(self): self._declarations = {} @@ -804,11 +804,10 @@ class Parser(object): raise AssertionError("kind = %r" % (kind,)) if name is not None: self._declare(key, tp) - else: - if kind == 'enum' and type.values is not None: - raise NotImplementedError( - "enum %s: the '{}' declaration should appear on the first " - "time the enum is mentioned, not later" % explicit_name) + elif kind == 'enum' and type.values is not None: + raise NotImplementedError( + "enum %s: the '{}' declaration should appear on the first " + "time the enum is mentioned, not later" % explicit_name) if not tp.forcename: tp.force_the_name(force_name) if tp.forcename and '$' in tp.name: diff --git a/contrib/python/cffi/py3/cffi/gen_src.py b/contrib/python/cffi/py3/cffi/gen_src.py new file mode 100644 index 00000000000..8eb57b16952 --- /dev/null +++ b/contrib/python/cffi/py3/cffi/gen_src.py @@ -0,0 +1,5 @@ +"""Entry point so ``python -m cffi.gen_src`` works.""" + +if __name__ == '__main__': + from cffi._cffi_gen_src import run + run() diff --git a/contrib/python/cffi/py3/cffi/model.py b/contrib/python/cffi/py3/cffi/model.py index e5f4cae3e84..8bade1d6dca 100644 --- a/contrib/python/cffi/py3/cffi/model.py +++ b/contrib/python/cffi/py3/cffi/model.py @@ -22,7 +22,7 @@ def qualify(quals, replace_with): return replace_with -class BaseTypeByIdentity(object): +class BaseTypeByIdentity: is_array_type = False is_raw_function = False @@ -34,7 +34,7 @@ class BaseTypeByIdentity(object): if replace_with: if replace_with.startswith('*') and '&[' in result: replace_with = '(%s)' % replace_with - elif not replace_with[0] in '[(': + elif replace_with[0] not in '[(': replace_with = ' ' + replace_with replace_with = qualify(quals, replace_with) result = result.replace('&', replace_with) @@ -371,8 +371,7 @@ class StructOrUnion(StructOrUnionOrEnum): if (name == '' and isinstance(type, StructOrUnion) and expand_anonymous_struct_union): # nested anonymous struct/union - for result in type.enumfields(): - yield result + yield from type.enumfields() else: yield (name, type, bitsize, quals) diff --git a/contrib/python/cffi/py3/cffi/pkgconfig.py b/contrib/python/cffi/py3/cffi/pkgconfig.py index 5c93f15a60e..e1034ad68a6 100644 --- a/contrib/python/cffi/py3/cffi/pkgconfig.py +++ b/contrib/python/cffi/py3/cffi/pkgconfig.py @@ -31,7 +31,7 @@ def call(libname, flag, encoding=sys.getfilesystemencoding()): a.append(libname) try: pc = subprocess.Popen(a, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - except EnvironmentError as e: + except OSError as e: raise PkgConfigError("cannot run pkg-config: %s" % (str(e).strip(),)) bout, berr = pc.communicate() diff --git a/contrib/python/cffi/py3/cffi/recompiler.py b/contrib/python/cffi/py3/cffi/recompiler.py index 25bf9e91d9e..fbafc011c97 100644 --- a/contrib/python/cffi/py3/cffi/recompiler.py +++ b/contrib/python/cffi/py3/cffi/recompiler.py @@ -7,9 +7,11 @@ VERSION_BASE = 0x2601 VERSION_EMBEDDED = 0x2701 VERSION_CHAR16CHAR32 = 0x2801 +FREE_THREADED_BUILD = sysconfig.get_config_var("Py_GIL_DISABLED") USE_LIMITED_API = ((sys.platform != 'win32' or sys.version_info < (3, 0) or sys.version_info >= (3, 5)) and - not sysconfig.get_config_var("Py_GIL_DISABLED")) # free-threaded doesn't yet support limited API + # free-threaded build doesn't support the stable ABI until Python 3.15 + (not FREE_THREADED_BUILD or sys.version_info >= (3, 15))) class GlobalExpr: def __init__(self, name, address, type_op, size=0, check_value=0): @@ -93,9 +95,18 @@ class EnumExpr: self.allenums = allenums def as_c_expr(self): - return (' { "%s", %d, _cffi_prim_int(%s, %s),\n' - ' "%s" },' % (self.name, self.type_index, - self.size, self.signed, self.allenums)) + lines = [' { "%s", %d, _cffi_prim_int(%s, %s),' % (self.name, self.type_index, + self.size, self.signed,)] + pending = 0 + while len(self.allenums) > pending + 110: + j = self.allenums.find(',', pending + 100) + if j < 0: + break + j += 1 + lines.append(' "%s"' % (self.allenums[pending:j],)) + pending = j + lines.append(' "%s" },' % (self.allenums[pending:],)) + return '\n'.join(lines) def as_python_expr(self): prim_index = { @@ -315,11 +326,8 @@ class Recompiler: prnt('#ifdef PYPY_VERSION') prnt('# define _CFFI_PYTHON_STARTUP_FUNC _cffi_pypyinit_%s' % ( base_module_name,)) - prnt('#elif PY_MAJOR_VERSION >= 3') - prnt('# define _CFFI_PYTHON_STARTUP_FUNC PyInit_%s' % ( - base_module_name,)) prnt('#else') - prnt('# define _CFFI_PYTHON_STARTUP_FUNC init%s' % ( + prnt('# define _CFFI_PYTHON_STARTUP_FUNC PyInit_%s' % ( base_module_name,)) prnt('#endif') lines = self._rel_readlines('_embedding.h') @@ -427,35 +435,22 @@ class Recompiler: prnt(' }') prnt(' p[0] = (const void *)0x%x;' % self._version) prnt(' p[1] = &_cffi_type_context;') - prnt('#if PY_MAJOR_VERSION >= 3') prnt(' return NULL;') - prnt('#endif') prnt('}') # on Windows, distutils insists on putting init_cffi_xyz in # 'export_symbols', so instead of fighting it, just give up and # give it one prnt('# ifdef _MSC_VER') prnt(' PyMODINIT_FUNC') - prnt('# if PY_MAJOR_VERSION >= 3') prnt(' PyInit_%s(void) { return NULL; }' % (base_module_name,)) - prnt('# else') - prnt(' init%s(void) { }' % (base_module_name,)) prnt('# endif') - prnt('# endif') - prnt('#elif PY_MAJOR_VERSION >= 3') + prnt('#else') prnt('PyMODINIT_FUNC') prnt('PyInit_%s(void)' % (base_module_name,)) prnt('{') prnt(' return _cffi_init("%s", 0x%x, &_cffi_type_context);' % ( self.module_name, self._version)) prnt('}') - prnt('#else') - prnt('PyMODINIT_FUNC') - prnt('init%s(void)' % (base_module_name,)) - prnt('{') - prnt(' _cffi_init("%s", 0x%x, &_cffi_type_context);' % ( - self.module_name, self._version)) - prnt('}') prnt('#endif') prnt() prnt('#ifdef __GNUC__') @@ -552,8 +547,7 @@ class Recompiler: tovar, errcode) return # - elif (isinstance(tp, model.StructOrUnionOrEnum) or - isinstance(tp, model.BasePrimitiveType)): + elif (isinstance(tp, (model.StructOrUnionOrEnum, model.BasePrimitiveType))): # a struct (not a struct pointer) as a function argument; # or, a complex (the same code works) self._prnt(' if (_cffi_to_c((char *)&%s, _cffi_type(%d), %s) < 0)' @@ -1313,10 +1307,6 @@ class Recompiler: s = s.encode('utf-8') # -> bytes else: s.decode('utf-8') # got bytes, check for valid utf-8 - try: - s.decode('ascii') - except UnicodeDecodeError: - s = b'# -*- encoding: utf8 -*-\n' + s for line in s.splitlines(True): comment = line if type('//') is bytes: # python2 @@ -1415,7 +1405,7 @@ else: def write(self, s): if isinstance(s, unicode): s = s.encode('ascii') - super(NativeIO, self).write(s) + super().write(s) def _is_file_like(maybefile): # compare to xml.etree.ElementTree._get_writer @@ -1437,11 +1427,11 @@ def _make_c_or_py_source(ffi, module_name, preamble, target_file, verbose): try: with open(target_file, 'r') as f1: if f1.read(len(output) + 1) != output: - raise IOError + raise OSError if verbose: print("(already up-to-date)") return False # already up-to-date - except IOError: + except OSError: tmp_file = '%s.~%d' % (target_file, os.getpid()) with open(tmp_file, 'w') as f1: f1.write(output) diff --git a/contrib/python/cffi/py3/cffi/setuptools_ext.py b/contrib/python/cffi/py3/cffi/setuptools_ext.py index 5cdd246fb4d..946afb45a6b 100644 --- a/contrib/python/cffi/py3/cffi/setuptools_ext.py +++ b/contrib/python/cffi/py3/cffi/setuptools_ext.py @@ -105,10 +105,12 @@ def _set_py_limited_api(Extension, kwds): kwds['py_limited_api'] = True if sysconfig.get_config_var("Py_GIL_DISABLED"): - if kwds.get('py_limited_api'): - log.info("Ignoring py_limited_api=True for free-threaded build.") - - kwds['py_limited_api'] = False + if sys.version_info < (3, 15): + if kwds.get('py_limited_api'): + log.info("Ignoring py_limited_api=True for free-threaded build.") + kwds['py_limited_api'] = False + else: + kwds.setdefault("define_macros", []).append(("Py_TARGET_ABI3T", "0x030f0000")) if kwds.get('py_limited_api') is False: # avoid setting Py_LIMITED_API if py_limited_api=False diff --git a/contrib/python/cffi/py3/cffi/vengine_cpy.py b/contrib/python/cffi/py3/cffi/vengine_cpy.py index 02e6a471d04..4d2ea43e5d9 100644 --- a/contrib/python/cffi/py3/cffi/vengine_cpy.py +++ b/contrib/python/cffi/py3/cffi/vengine_cpy.py @@ -7,7 +7,7 @@ from .error import VerificationError from . import _imp_emulation as imp -class VCPythonEngine(object): +class VCPythonEngine: _class_key = 'x' _gen_python_module = True @@ -102,8 +102,6 @@ class VCPythonEngine(object): # standard init. modname = self.verifier.get_module_name() constants = self._chained_list_constants[False] - prnt('#if PY_MAJOR_VERSION >= 3') - prnt() prnt('static struct PyModuleDef _cffi_module_def = {') prnt(' PyModuleDef_HEAD_INIT,') prnt(' "%s",' % modname) @@ -130,21 +128,6 @@ class VCPythonEngine(object): prnt(' return lib;') prnt('}') prnt() - prnt('#else') - prnt() - prnt('PyMODINIT_FUNC') - prnt('init%s(void)' % modname) - prnt('{') - prnt(' PyObject *lib;') - prnt(' lib = Py_InitModule("%s", _cffi_methods);' % modname) - prnt(' if (lib == NULL)') - prnt(' return;') - prnt(' if (%s < 0 || _cffi_init() < 0)' % (constants,)) - prnt(' return;') - prnt(' return;') - prnt('}') - prnt() - prnt('#endif') def load_library(self, flags=None): # XXX review all usages of 'self' here! @@ -183,7 +166,7 @@ class VCPythonEngine(object): # it will invoke the chained list of functions that will really # build (notably) the constant objects, as <cdata> if they are # pointers, and store them as attributes on the 'library' object. - class FFILibrary(object): + class FFILibrary: _cffi_python_module = module _cffi_ffi = self.ffi _cffi_dir = [] @@ -873,21 +856,9 @@ cffimod_header = r''' # define _cffi_double_complex_t double _Complex #endif -#if PY_MAJOR_VERSION < 3 -# undef PyCapsule_CheckExact -# undef PyCapsule_GetPointer -# define PyCapsule_CheckExact(capsule) (PyCObject_Check(capsule)) -# define PyCapsule_GetPointer(capsule, name) \ - (PyCObject_AsVoidPtr(capsule)) -#endif - -#if PY_MAJOR_VERSION >= 3 -# define PyInt_FromLong PyLong_FromLong -#endif - #define _cffi_from_c_double PyFloat_FromDouble #define _cffi_from_c_float PyFloat_FromDouble -#define _cffi_from_c_long PyInt_FromLong +#define _cffi_from_c_long PyLong_FromLong #define _cffi_from_c_ulong PyLong_FromUnsignedLong #define _cffi_from_c_longlong PyLong_FromLongLong #define _cffi_from_c_ulonglong PyLong_FromUnsignedLongLong @@ -899,21 +870,21 @@ cffimod_header = r''' #define _cffi_from_c_int_const(x) \ (((x) > 0) ? \ ((unsigned long long)(x) <= (unsigned long long)LONG_MAX) ? \ - PyInt_FromLong((long)(x)) : \ + PyLong_FromLong((long)(x)) : \ PyLong_FromUnsignedLongLong((unsigned long long)(x)) : \ ((long long)(x) >= (long long)LONG_MIN) ? \ - PyInt_FromLong((long)(x)) : \ + PyLong_FromLong((long)(x)) : \ PyLong_FromLongLong((long long)(x))) #define _cffi_from_c_int(x, type) \ (((type)-1) > 0 ? /* unsigned */ \ (sizeof(type) < sizeof(long) ? \ - PyInt_FromLong((long)x) : \ + PyLong_FromLong((long)x) : \ sizeof(type) == sizeof(long) ? \ PyLong_FromUnsignedLong((unsigned long)x) : \ PyLong_FromUnsignedLongLong((unsigned long long)x)) : \ (sizeof(type) <= sizeof(long) ? \ - PyInt_FromLong((long)x) : \ + PyLong_FromLong((long)x) : \ PyLong_FromLongLong((long long)x))) #define _cffi_to_c_int(o, type) \ diff --git a/contrib/python/cffi/py3/cffi/vengine_gen.py b/contrib/python/cffi/py3/cffi/vengine_gen.py index bffc82122c3..0209c794ac7 100644 --- a/contrib/python/cffi/py3/cffi/vengine_gen.py +++ b/contrib/python/cffi/py3/cffi/vengine_gen.py @@ -8,7 +8,7 @@ from . import model from .error import VerificationError -class VGenericEngine(object): +class VGenericEngine: _class_key = 'g' _gen_python_module = False diff --git a/contrib/python/cffi/py3/cffi/verifier.py b/contrib/python/cffi/py3/cffi/verifier.py index e392a2b7fda..96baadb6f76 100644 --- a/contrib/python/cffi/py3/cffi/verifier.py +++ b/contrib/python/cffi/py3/cffi/verifier.py @@ -24,10 +24,10 @@ else: def write(self, s): if isinstance(s, unicode): s = s.encode('ascii') - super(NativeIO, self).write(s) + super().write(s) -class Verifier(object): +class Verifier: def __init__(self, ffi, preamble, tmpdir=None, modulename=None, ext_package=None, tag='', force_generic_engine=False, @@ -182,7 +182,7 @@ class Verifier(object): # Determine if this matches the current file if os.path.exists(self.sourcefilename): with open(self.sourcefilename, "r") as fp: - needs_written = not (fp.read() == source_data) + needs_written = fp.read() != source_data else: needs_written = True diff --git a/contrib/python/cffi/py3/patches/01-arcadia.patch b/contrib/python/cffi/py3/patches/01-arcadia.patch index 23331873652..8ab50be29d7 100644 --- a/contrib/python/cffi/py3/patches/01-arcadia.patch +++ b/contrib/python/cffi/py3/patches/01-arcadia.patch @@ -92,16 +92,8 @@ if (*w > 0xFFFF) { --- contrib/python/cffi/py3/cffi/recompiler.py (index) +++ contrib/python/cffi/py3/cffi/recompiler.py (working tree) -@@ -287,10 +287,8 @@ class Recompiler: - self.write_c_source_to_f(f, preamble) - - def _rel_readlines(self, filename): -- g = open(os.path.join(os.path.dirname(__file__), filename), 'r') -- lines = g.readlines() -- g.close() -- return lines +@@ -287,2 +287,2 @@ class Recompiler: +- with open(os.path.join(os.path.dirname(__file__), filename), 'r') as g: +- return g.readlines() + import pkgutil + return pkgutil.get_data('cffi', filename).decode('utf-8').splitlines(True) - - def write_c_source_to_f(self, f, preamble): - self._f = f diff --git a/contrib/python/cffi/py3/ya.make b/contrib/python/cffi/py3/ya.make index d2eb8e11ad8..a526c12160c 100644 --- a/contrib/python/cffi/py3/ya.make +++ b/contrib/python/cffi/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(2.0.0) +VERSION(2.1.0) LICENSE(MIT) @@ -31,6 +31,7 @@ PY_REGISTER( PY_SRCS( TOP_LEVEL cffi/__init__.py + cffi/_cffi_gen_src.py cffi/_imp_emulation.py cffi/_shimmed_dist_utils.py cffi/api.py @@ -40,6 +41,7 @@ PY_SRCS( cffi/cparser.py cffi/error.py cffi/ffiplatform.py + cffi/gen_src.py cffi/lock.py cffi/model.py cffi/pkgconfig.py diff --git a/contrib/python/idna/py3/.dist-info/METADATA b/contrib/python/idna/py3/.dist-info/METADATA index 3dd388cb837..9a2b49a74ee 100644 --- a/contrib/python/idna/py3/.dist-info/METADATA +++ b/contrib/python/idna/py3/.dist-info/METADATA @@ -1,9 +1,9 @@ Metadata-Version: 2.4 Name: idna -Version: 3.15 +Version: 3.16 Summary: Internationalized Domain Names in Applications (IDNA) Author-email: Kim Davies <[email protected]> -Requires-Python: >=3.8 +Requires-Python: >=3.9 Description-Content-Type: text/markdown License-Expression: BSD-3-Clause Classifier: Development Status :: 5 - Production/Stable @@ -13,7 +13,6 @@ Classifier: Operating System :: OS Independent Classifier: Programming Language :: Python Classifier: Programming Language :: Python :: 3 Classifier: Programming Language :: Python :: 3 :: Only -Classifier: Programming Language :: Python :: 3.8 Classifier: Programming Language :: Python :: 3.9 Classifier: Programming Language :: Python :: 3.10 Classifier: Programming Language :: Python :: 3.11 @@ -36,40 +35,18 @@ Provides-Extra: all # Internationalized Domain Names in Applications (IDNA) -Support for [Internationalized Domain Names in -Applications (IDNA)](https://tools.ietf.org/html/rfc5891) -and [Unicode IDNA Compatibility Processing](https://unicode.org/reports/tr46/). - -The latest versions of these standards supplied here provide -more comprehensive language coverage and reduce the potential of -allowing domains with known security vulnerabilities. This library -is a suitable replacement for the "encodings.idna" -module that comes with the Python standard library, but which -only supports an older superseded IDNA specification from 2003. - -Basic functions are simply executed: - -```pycon ->>> import idna ->>> idna.encode('ドメイン.テスト') -b'xn--eckwd4c7c.xn--zckzah' ->>> print(idna.decode('xn--eckwd4c7c.xn--zckzah')) -ドメイン.テスト -``` - - -## Installation - -This package is available for installation from PyPI via the -typical mechanisms, such as: - -```bash -$ python3 -m pip install idna -``` - +Support for [Internationalized Domain Names in Applications +(IDNA)](https://tools.ietf.org/html/rfc5891) and [Unicode IDNA +Compatibility Processing](https://unicode.org/reports/tr46/). It +supersedes the standard library's `encodings.idna`, which only +implements the 2003 specification, offering broader script coverage and +limiting domains with known security vulnerabilities. ## Usage +Package may be installed from [PyPI](https://pypi.org/project/idna/) via +the typical methods (e.g. `python3 -m pip install idna`) + For typical usage, the `encode` and `decode` functions will take a domain name argument and perform a conversion to ASCII-compatible encoding (known as A-labels), or to Unicode strings (known as U-labels) @@ -84,12 +61,7 @@ b'xn--eckwd4c7c.xn--zckzah' ``` Conversions can be applied at a per-label basis using the `ulabel` or -`alabel` functions if necessary: - -```pycon ->>> idna.alabel('测试') -b'xn--0zwm56d' -``` +`alabel` functions for specialized use cases. ### Compatibility Mapping (UTS #46) @@ -102,10 +74,9 @@ conversion operations. This functionality, known as a specification to be a local user-interface issue distinct from IDNA conversion functionality. -For example, "Königsgäßchen" is not a permissible label as *LATIN -CAPITAL LETTER K* is not allowed (nor are capital letters in general). -UTS 46 will convert this into lower case prior to applying the IDNA -conversion. +For example, "Königsgäßchen" is not a permissible label as capital letters +are not allowed. UTS 46 will convert this into lower case prior to applying +the IDNA conversion. ```pycon >>> import idna @@ -114,81 +85,80 @@ conversion. idna.core.InvalidCodepoint: Codepoint U+004B at position 1 of 'Königsgäßchen' not allowed >>> idna.encode('Königsgäßchen', uts46=True) b'xn--knigsgchen-b4a3dun' ->>> print(idna.decode('xn--knigsgchen-b4a3dun')) -königsgäßchen +>>> idna.decode('xn--knigsgchen-b4a3dun') +'königsgäßchen' ``` ## Exceptions -All errors raised during the conversion following the specification -should raise an exception derived from the `idna.IDNAError` base -class. +All errors raised during conversion derive from the `idna.IDNAError` +base class. The more specific exceptions are: -More specific exceptions that may be generated as `idna.IDNABidiError` -when the error reflects an illegal combination of left-to-right and -right-to-left characters in a label; `idna.InvalidCodepoint` when -a specific codepoint is an illegal character in an IDN label (i.e. -INVALID); and `idna.InvalidCodepointContext` when the codepoint is -illegal based on its position in the string (i.e. it is CONTEXTO or CONTEXTJ -but the contextual requirements are not satisfied.) +* `idna.IDNABidiError` — raised when a label contains an illegal + combination of left-to-right and right-to-left characters. +* `idna.InvalidCodepoint` — raised when a label contains a codepoint + that is INVALID for IDNA. +* `idna.InvalidCodepointContext` — raised when a CONTEXTO or CONTEXTJ + codepoint appears in a position whose contextual requirements are + not satisfied. -## Building and Diagnostics -The IDNA and UTS 46 functionality relies upon pre-calculated lookup -tables for performance. These tables are derived from computing against -eligibility criteria in the respective standards using the command-line -script `tools/idna-data`. +## Command-line tool -This tool will fetch relevant codepoint data from the Unicode repository -and perform the required calculations to identify eligibility. There are -three main modes: +The package supports command-line usage to convert domain names +between their Unicode and ASCII-compatible forms. It can be run either +as a module (`python3 -m idna`) or, once installed (such as with `uv +tool` or `pipx`), via the `idna` script: -* `idna-data make-libdata`. Generates `idnadata.py` and - `uts46data.py`, the pre-calculated lookup tables used for IDNA and - UTS 46 conversions. Implementers who wish to track this library against - a different Unicode version may use this tool to manually generate a - different version of the `idnadata.py` and `uts46data.py` files. +```bash +$ uv tool install idna +$ idna xn--e1afmkfd.xn--p1ai +пример.рф +$ idna пример.рф +xn--e1afmkfd.xn--p1ai +``` -* `idna-data make-table`. Generate a table of the IDNA disposition - (e.g. PVALID, CONTEXTJ, CONTEXTO) in the format found in Appendix - B.1 of RFC 5892 and the pre-computed tables published by [IANA](https://www.iana.org/). +With no mode flag the direction is chosen automatically: inputs +containing an `xn--` label are decoded, anything else is encoded. Pass +`-e`/`--encode` or `-d`/`--decode` to force a specific direction. -* `idna-data U+0061`. Prints debugging output on the various - properties associated with an individual Unicode codepoint (in this - case, U+0061), that are used to assess the IDNA and UTS 46 status of a - codepoint. This is helpful in debugging or analysis. +Multiple domains may be supplied at once, either as positional arguments +or by piping one domain per line on standard input. When more than +one domain is supplied without explicitly asking to encode or decode, +the direction is picked from the first input and that mode is applied +to every remaining input. Use `-e`/`--encode` or `-d`/`--decode` to +override the heuristic if the first input is ambiguous. -The tool accepts a number of arguments, described using `idna-data -h`. -Most notably, the `--version` argument allows the specification -of the version of Unicode to be used in computing the table data. For -example, `idna-data --version 9.0.0 make-libdata` will generate -library data against Unicode 9.0.0. +UTS #46 mapping is applied by default, which lets the tool accept +inputs that aren't strictly valid IDNA 2008 by normalising them first: +```bash +$ idna ΠΑΡΆΔΕΙΓΜΑ.ΕΛ +xn--hxajbheg2az3al.xn--qxam +``` -## Additional Notes +Pass `--strict` to disable UTS #46 and apply IDNA 2008 rules verbatim; +the same input will then be rejected. -* **Packages**. The latest tagged release version is published in the - [Python Package Index](https://pypi.org/project/idna/). +Conversion failures are reported on stderr together with the +offending input; processing continues with the remaining domains and +the tool exits with a non-zero status if any conversion failed. -* **Version support**. This library supports Python 3.8 and higher. - As this library serves as a low-level toolkit for a variety of - applications, many of which strive for broad compatibility with older - Python versions, there is no rush to remove older interpreter support. - Support for older versions are likely to be removed from new releases - as automated tests can no longer easily be run, i.e. once the Python - version is officially end-of-life. -* **Testing**. The library has a test suite based on each rule of the - IDNA specification, as well as tests that are provided as part of the - Unicode Technical Standard 46, [Unicode IDNA Compatibility Processing](https://unicode.org/reports/tr46/). +## Additional Notes + +* **Version support**. This library supports Python 3.9 and higher. + As this library serves as a low-level toolkit for a variety of + applications, we strive to support all versions of Python that are + not beyond end-of-life. * **Emoji**. It is an occasional request to support emoji domains in this library. Encoding of symbols like emoji is expressly prohibited by the IDNA technical standard, and emoji domains are broadly phased out across the domain industry due to associated security risks. -* **Transitional processing**. Unicode 16.0.0 removed transitional - processing so the `transitional` argument for the encode() method - no longer has any effect and will be removed at a later date. +* **Regenerating lookup tables**. The IDNA and UTS 46 functionality + relies upon pre-calculated lookup tables, generated using the + `idna-data` script in [`tools/`](tools/README.md). diff --git a/contrib/python/idna/py3/.dist-info/entry_points.txt b/contrib/python/idna/py3/.dist-info/entry_points.txt new file mode 100644 index 00000000000..59ca7ac02ea --- /dev/null +++ b/contrib/python/idna/py3/.dist-info/entry_points.txt @@ -0,0 +1,3 @@ +[console_scripts] +idna=idna.cli:main + diff --git a/contrib/python/idna/py3/README.md b/contrib/python/idna/py3/README.md index 5772219126b..77e4e4e0a8d 100644 --- a/contrib/python/idna/py3/README.md +++ b/contrib/python/idna/py3/README.md @@ -1,39 +1,17 @@ # Internationalized Domain Names in Applications (IDNA) -Support for [Internationalized Domain Names in -Applications (IDNA)](https://tools.ietf.org/html/rfc5891) -and [Unicode IDNA Compatibility Processing](https://unicode.org/reports/tr46/). - -The latest versions of these standards supplied here provide -more comprehensive language coverage and reduce the potential of -allowing domains with known security vulnerabilities. This library -is a suitable replacement for the "encodings.idna" -module that comes with the Python standard library, but which -only supports an older superseded IDNA specification from 2003. - -Basic functions are simply executed: - -```pycon ->>> import idna ->>> idna.encode('ドメイン.テスト') -b'xn--eckwd4c7c.xn--zckzah' ->>> print(idna.decode('xn--eckwd4c7c.xn--zckzah')) -ドメイン.テスト -``` - - -## Installation - -This package is available for installation from PyPI via the -typical mechanisms, such as: - -```bash -$ python3 -m pip install idna -``` - +Support for [Internationalized Domain Names in Applications +(IDNA)](https://tools.ietf.org/html/rfc5891) and [Unicode IDNA +Compatibility Processing](https://unicode.org/reports/tr46/). It +supersedes the standard library's `encodings.idna`, which only +implements the 2003 specification, offering broader script coverage and +limiting domains with known security vulnerabilities. ## Usage +Package may be installed from [PyPI](https://pypi.org/project/idna/) via +the typical methods (e.g. `python3 -m pip install idna`) + For typical usage, the `encode` and `decode` functions will take a domain name argument and perform a conversion to ASCII-compatible encoding (known as A-labels), or to Unicode strings (known as U-labels) @@ -48,12 +26,7 @@ b'xn--eckwd4c7c.xn--zckzah' ``` Conversions can be applied at a per-label basis using the `ulabel` or -`alabel` functions if necessary: - -```pycon ->>> idna.alabel('测试') -b'xn--0zwm56d' -``` +`alabel` functions for specialized use cases. ### Compatibility Mapping (UTS #46) @@ -66,10 +39,9 @@ conversion operations. This functionality, known as a specification to be a local user-interface issue distinct from IDNA conversion functionality. -For example, "Königsgäßchen" is not a permissible label as *LATIN -CAPITAL LETTER K* is not allowed (nor are capital letters in general). -UTS 46 will convert this into lower case prior to applying the IDNA -conversion. +For example, "Königsgäßchen" is not a permissible label as capital letters +are not allowed. UTS 46 will convert this into lower case prior to applying +the IDNA conversion. ```pycon >>> import idna @@ -78,80 +50,79 @@ conversion. idna.core.InvalidCodepoint: Codepoint U+004B at position 1 of 'Königsgäßchen' not allowed >>> idna.encode('Königsgäßchen', uts46=True) b'xn--knigsgchen-b4a3dun' ->>> print(idna.decode('xn--knigsgchen-b4a3dun')) -königsgäßchen +>>> idna.decode('xn--knigsgchen-b4a3dun') +'königsgäßchen' ``` ## Exceptions -All errors raised during the conversion following the specification -should raise an exception derived from the `idna.IDNAError` base -class. +All errors raised during conversion derive from the `idna.IDNAError` +base class. The more specific exceptions are: -More specific exceptions that may be generated as `idna.IDNABidiError` -when the error reflects an illegal combination of left-to-right and -right-to-left characters in a label; `idna.InvalidCodepoint` when -a specific codepoint is an illegal character in an IDN label (i.e. -INVALID); and `idna.InvalidCodepointContext` when the codepoint is -illegal based on its position in the string (i.e. it is CONTEXTO or CONTEXTJ -but the contextual requirements are not satisfied.) +* `idna.IDNABidiError` — raised when a label contains an illegal + combination of left-to-right and right-to-left characters. +* `idna.InvalidCodepoint` — raised when a label contains a codepoint + that is INVALID for IDNA. +* `idna.InvalidCodepointContext` — raised when a CONTEXTO or CONTEXTJ + codepoint appears in a position whose contextual requirements are + not satisfied. -## Building and Diagnostics -The IDNA and UTS 46 functionality relies upon pre-calculated lookup -tables for performance. These tables are derived from computing against -eligibility criteria in the respective standards using the command-line -script `tools/idna-data`. +## Command-line tool -This tool will fetch relevant codepoint data from the Unicode repository -and perform the required calculations to identify eligibility. There are -three main modes: +The package supports command-line usage to convert domain names +between their Unicode and ASCII-compatible forms. It can be run either +as a module (`python3 -m idna`) or, once installed (such as with `uv +tool` or `pipx`), via the `idna` script: -* `idna-data make-libdata`. Generates `idnadata.py` and - `uts46data.py`, the pre-calculated lookup tables used for IDNA and - UTS 46 conversions. Implementers who wish to track this library against - a different Unicode version may use this tool to manually generate a - different version of the `idnadata.py` and `uts46data.py` files. +```bash +$ uv tool install idna +$ idna xn--e1afmkfd.xn--p1ai +пример.рф +$ idna пример.рф +xn--e1afmkfd.xn--p1ai +``` -* `idna-data make-table`. Generate a table of the IDNA disposition - (e.g. PVALID, CONTEXTJ, CONTEXTO) in the format found in Appendix - B.1 of RFC 5892 and the pre-computed tables published by [IANA](https://www.iana.org/). +With no mode flag the direction is chosen automatically: inputs +containing an `xn--` label are decoded, anything else is encoded. Pass +`-e`/`--encode` or `-d`/`--decode` to force a specific direction. -* `idna-data U+0061`. Prints debugging output on the various - properties associated with an individual Unicode codepoint (in this - case, U+0061), that are used to assess the IDNA and UTS 46 status of a - codepoint. This is helpful in debugging or analysis. +Multiple domains may be supplied at once, either as positional arguments +or by piping one domain per line on standard input. When more than +one domain is supplied without explicitly asking to encode or decode, +the direction is picked from the first input and that mode is applied +to every remaining input. Use `-e`/`--encode` or `-d`/`--decode` to +override the heuristic if the first input is ambiguous. -The tool accepts a number of arguments, described using `idna-data -h`. -Most notably, the `--version` argument allows the specification -of the version of Unicode to be used in computing the table data. For -example, `idna-data --version 9.0.0 make-libdata` will generate -library data against Unicode 9.0.0. +UTS #46 mapping is applied by default, which lets the tool accept +inputs that aren't strictly valid IDNA 2008 by normalising them first: +```bash +$ idna ΠΑΡΆΔΕΙΓΜΑ.ΕΛ +xn--hxajbheg2az3al.xn--qxam +``` -## Additional Notes +Pass `--strict` to disable UTS #46 and apply IDNA 2008 rules verbatim; +the same input will then be rejected. -* **Packages**. The latest tagged release version is published in the - [Python Package Index](https://pypi.org/project/idna/). +Conversion failures are reported on stderr together with the +offending input; processing continues with the remaining domains and +the tool exits with a non-zero status if any conversion failed. -* **Version support**. This library supports Python 3.8 and higher. - As this library serves as a low-level toolkit for a variety of - applications, many of which strive for broad compatibility with older - Python versions, there is no rush to remove older interpreter support. - Support for older versions are likely to be removed from new releases - as automated tests can no longer easily be run, i.e. once the Python - version is officially end-of-life. -* **Testing**. The library has a test suite based on each rule of the - IDNA specification, as well as tests that are provided as part of the - Unicode Technical Standard 46, [Unicode IDNA Compatibility Processing](https://unicode.org/reports/tr46/). +## Additional Notes + +* **Version support**. This library supports Python 3.9 and higher. + As this library serves as a low-level toolkit for a variety of + applications, we strive to support all versions of Python that are + not beyond end-of-life. * **Emoji**. It is an occasional request to support emoji domains in this library. Encoding of symbols like emoji is expressly prohibited by the IDNA technical standard, and emoji domains are broadly phased out across the domain industry due to associated security risks. -* **Transitional processing**. Unicode 16.0.0 removed transitional - processing so the `transitional` argument for the encode() method - no longer has any effect and will be removed at a later date. +* **Regenerating lookup tables**. The IDNA and UTS 46 functionality + relies upon pre-calculated lookup tables, generated using the + `idna-data` script in [`tools/`](tools/README.md). diff --git a/contrib/python/idna/py3/idna/__main__.py b/contrib/python/idna/py3/idna/__main__.py new file mode 100644 index 00000000000..dbdd0661721 --- /dev/null +++ b/contrib/python/idna/py3/idna/__main__.py @@ -0,0 +1,6 @@ +import sys + +from .cli import main + +if __name__ == "__main__": + sys.exit(main()) diff --git a/contrib/python/idna/py3/idna/cli.py b/contrib/python/idna/py3/idna/cli.py new file mode 100644 index 00000000000..4acda2c0f03 --- /dev/null +++ b/contrib/python/idna/py3/idna/cli.py @@ -0,0 +1,128 @@ +"""Command-line interface for the :mod:`idna` package. + +Invoked via ``python -m idna``. See :func:`main` for the entry point. +""" + +import argparse +import sys +from collections.abc import Iterable +from itertools import chain +from typing import IO, Optional + +from . import IDNAError, decode, encode +from .core import _alabel_prefix, _unicode_dots_re +from .package_data import __version__ + + +def _looks_like_alabel(s: str) -> bool: + """Return True if any label in ``s`` carries the ``xn--`` ACE prefix.""" + prefix = _alabel_prefix.decode("ascii") + return any(label.lower().startswith(prefix) for label in _unicode_dots_re.split(s)) + + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="python -m idna", + description=( + "Convert a domain name between its Unicode (U-label) and " + "ASCII-compatible (A-label) forms. With no mode flag, the " + "direction is chosen from the first input — if it contains " + "an xn-- label the stream is decoded, otherwise it is " + "encoded — and the same mode is applied to every remaining " + "input. UTS #46 mapping is applied by default; pass " + "--strict to disable it. When no domains are given on the " + "command line and stdin is piped, one domain per line is " + "read from stdin." + ), + ) + mode = parser.add_mutually_exclusive_group() + mode.add_argument( + "-e", + "--encode", + dest="mode", + action="store_const", + const="encode", + help="Encode the input to its ASCII A-label form.", + ) + mode.add_argument( + "-d", + "--decode", + dest="mode", + action="store_const", + const="decode", + help="Decode the input from its ASCII A-label form.", + ) + parser.add_argument( + "--strict", + action="store_true", + help="Disable the default UTS #46 mapping and apply IDNA 2008 rules verbatim.", + ) + parser.add_argument( + "--version", + action="version", + version=f"idna {__version__}", + ) + parser.add_argument( + "domain", + nargs="*", + help="One or more domain names to convert. Omit to read from stdin.", + ) + return parser + + +def _iter_stdin(stream: IO[str]) -> Iterable[str]: + """Yield non-empty stripped lines from ``stream``, ignoring blanks.""" + for line in stream: + stripped = line.strip() + if stripped: + yield stripped + + +def _convert_one(domain: str, mode: str, uts46: bool) -> bool: + """Convert ``domain`` and write the result; return ``False`` on failure.""" + try: + if mode == "decode": + print(decode(domain, uts46=uts46)) + else: + print(encode(domain, uts46=uts46).decode("ascii")) + except IDNAError as err: + print(f"idna: {mode} failed for {domain!r}: {err}", file=sys.stderr) + return False + return True + + +def main(argv: Optional[list[str]] = None) -> int: + """Entry point for ``python -m idna``. + + When more than one domain is supplied (via positional arguments or + piped stdin) and no mode flag is given, the first input determines + the direction and that mode is applied uniformly to the rest. + + :param argv: Argument list excluding the program name. Defaults to + :data:`sys.argv` when ``None``. + :returns: ``0`` on success, ``1`` if any conversion fails. + """ + parser = _build_parser() + args = parser.parse_args(argv) + uts46 = not args.strict + + if args.domain: + domains: Iterable[str] = args.domain + elif not sys.stdin.isatty(): + domains = _iter_stdin(sys.stdin) + else: + parser.error("a domain argument is required when stdin is a terminal") + + iterator = iter(domains) + first = next(iterator, None) + if first is None: + return 0 + + mode = args.mode or ("decode" if _looks_like_alabel(first) else "encode") + + results = [_convert_one(domain, mode, uts46) for domain in chain([first], iterator)] + return 0 if all(results) else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/contrib/python/idna/py3/idna/codec.py b/contrib/python/idna/py3/idna/codec.py index 280dc3972cb..83b42fe42b5 100644 --- a/contrib/python/idna/py3/idna/codec.py +++ b/contrib/python/idna/py3/idna/codec.py @@ -1,5 +1,5 @@ import codecs -from typing import Any, Optional, Tuple +from typing import Any, Optional from .core import IDNAError, _unicode_dots_re, alabel, decode, encode, ulabel @@ -15,7 +15,7 @@ class Codec(codecs.Codec): raises :exc:`~idna.IDNAError`. """ - def encode(self, data: str, errors: str = "strict") -> Tuple[bytes, int]: # ty: ignore[invalid-method-override] + def encode(self, data: str, errors: str = "strict") -> tuple[bytes, int]: # ty: ignore[invalid-method-override] if errors != "strict": raise IDNAError(f'Unsupported error handling "{errors}"') @@ -24,7 +24,7 @@ class Codec(codecs.Codec): return encode(data), len(data) - def decode(self, data: bytes, errors: str = "strict") -> Tuple[str, int]: # ty: ignore[invalid-method-override] + def decode(self, data: bytes, errors: str = "strict") -> tuple[str, int]: # ty: ignore[invalid-method-override] if errors != "strict": raise IDNAError(f'Unsupported error handling "{errors}"') @@ -46,7 +46,7 @@ class IncrementalEncoder(codecs.BufferedIncrementalEncoder): Only the ``"strict"`` error handler is supported. """ - def _buffer_encode(self, data: str, errors: str, final: bool) -> Tuple[bytes, int]: # ty: ignore[invalid-method-override] + def _buffer_encode(self, data: str, errors: str, final: bool) -> tuple[bytes, int]: # ty: ignore[invalid-method-override] if errors != "strict": raise IDNAError(f'Unsupported error handling "{errors}"') @@ -89,7 +89,7 @@ class IncrementalDecoder(codecs.BufferedIncrementalDecoder): Only the ``"strict"`` error handler is supported. """ - def _buffer_decode(self, data: Any, errors: str, final: bool) -> Tuple[str, int]: # ty: ignore[invalid-method-override] + def _buffer_decode(self, data: Any, errors: str, final: bool) -> tuple[str, int]: # ty: ignore[invalid-method-override] if errors != "strict": raise IDNAError(f'Unsupported error handling "{errors}"') diff --git a/contrib/python/idna/py3/idna/core.py b/contrib/python/idna/py3/idna/core.py index b6f9442deb3..da45b2ae9e4 100644 --- a/contrib/python/idna/py3/idna/core.py +++ b/contrib/python/idna/py3/idna/core.py @@ -27,26 +27,18 @@ _bidi_joiner_r_or_d = frozenset({ord("R"), ord("D")}) class IDNAError(UnicodeError): """Base exception for all IDNA-encoding related problems""" - pass - class IDNABidiError(IDNAError): """Exception when bidirectional requirements are not satisfied""" - pass - class InvalidCodepoint(IDNAError): """Exception when a disallowed or unallocated codepoint is used""" - pass - class InvalidCodepointContext(IDNAError): """Exception when the codepoint is not valid in the context it is used""" - pass - def _combining_class(cp: int) -> int: v = unicodedata.combining(chr(cp)) @@ -118,7 +110,7 @@ def check_bidi(label: str, check_ltr: bool = False) -> bool: direction = unicodedata.bidirectional(cp) if direction == "": # String likely comes from a newer version of Unicode - raise IDNABidiError(f"Unknown directionality in label {repr(label)} at position {idx}") + raise IDNABidiError(f"Unknown directionality in label {label!r} at position {idx}") if direction in _bidi_rtl_categories: bidi_label = True if not bidi_label and not check_ltr: @@ -131,7 +123,7 @@ def check_bidi(label: str, check_ltr: bool = False) -> bool: elif direction == "L": rtl = False else: - raise IDNABidiError(f"First codepoint in label {repr(label)} must be directionality L, R or AL") + raise IDNABidiError(f"First codepoint in label {label!r} must be directionality L, R or AL") valid_ending = False number_type: Optional[str] = None @@ -151,9 +143,8 @@ def check_bidi(label: str, check_ltr: bool = False) -> bool: if direction in _bidi_rtl_numeric: if not number_type: number_type = direction - else: - if number_type != direction: - raise IDNABidiError("Can not mix numeral types in a right-to-left label") + elif number_type != direction: + raise IDNABidiError("Can not mix numeral types in a right-to-left label") else: # Bidi rule 5 if direction not in _bidi_ltr_allowed: @@ -239,11 +230,10 @@ def valid_contextj(label: str, pos: int) -> bool: joining_type = idnadata.joining_types().get(ord(label[i])) if joining_type == ord("T"): continue - elif joining_type in _bidi_joiner_l_or_d: + if joining_type in _bidi_joiner_l_or_d: ok = True break - else: - break + break if not ok: return False @@ -253,18 +243,16 @@ def valid_contextj(label: str, pos: int) -> bool: joining_type = idnadata.joining_types().get(ord(label[i])) if joining_type == ord("T"): continue - elif joining_type in _bidi_joiner_r_or_d: + if joining_type in _bidi_joiner_r_or_d: ok = True break - else: - break + break return ok if cp_value == 0x200D: return pos > 0 and _combining_class(ord(label[pos - 1])) == _virama_combining_class - else: - return False + return False def valid_contexto(label: str, pos: int, exception: bool = False) -> bool: @@ -286,17 +274,17 @@ def valid_contexto(label: str, pos: int, exception: bool = False) -> bool: if cp_value == 0x00B7: return 0 < pos < len(label) - 1 and ord(label[pos - 1]) == 0x006C and ord(label[pos + 1]) == 0x006C - elif cp_value == 0x0375: + if cp_value == 0x0375: if pos < len(label) - 1 and len(label) > 1: return _is_script(label[pos + 1], "Greek") return False - elif cp_value == 0x05F3 or cp_value == 0x05F4: + if cp_value in {0x05F3, 0x05F4}: if pos > 0: return _is_script(label[pos - 1], "Hebrew") return False - elif cp_value == 0x30FB: + if cp_value == 0x30FB: for cp in label: if cp == "\u30fb": continue @@ -304,10 +292,10 @@ def valid_contexto(label: str, pos: int, exception: bool = False) -> bool: return True return False - elif 0x660 <= cp_value <= 0x669: + if 0x660 <= cp_value <= 0x669: return not any(0x6F0 <= ord(cp) <= 0x06F9 for cp in label) - elif 0x6F0 <= cp_value <= 0x6F9: + if 0x6F0 <= cp_value <= 0x6F9: return not any(0x660 <= ord(cp) <= 0x0669 for cp in label) return False @@ -349,23 +337,19 @@ def check_label(label: Union[str, bytes, bytearray]) -> None: cp_value = ord(cp) if intranges_contain(cp_value, idnadata.codepoint_classes["PVALID"]): continue - elif intranges_contain(cp_value, idnadata.codepoint_classes["CONTEXTJ"]): + if intranges_contain(cp_value, idnadata.codepoint_classes["CONTEXTJ"]): try: if not valid_contextj(label, pos): - raise InvalidCodepointContext( - f"Joiner {_unot(cp_value)} not allowed at position {pos + 1} in {repr(label)}" - ) + raise InvalidCodepointContext(f"Joiner {_unot(cp_value)} not allowed at position {pos + 1} in {label!r}") except ValueError as err: raise IDNAError( - f"Unknown codepoint adjacent to joiner {_unot(cp_value)} at position {pos + 1} in {repr(label)}" + f"Unknown codepoint adjacent to joiner {_unot(cp_value)} at position {pos + 1} in {label!r}" ) from err elif intranges_contain(cp_value, idnadata.codepoint_classes["CONTEXTO"]): if not valid_contexto(label, pos): - raise InvalidCodepointContext( - f"Codepoint {_unot(cp_value)} not allowed at position {pos + 1} in {repr(label)}" - ) + raise InvalidCodepointContext(f"Codepoint {_unot(cp_value)} not allowed at position {pos + 1} in {label!r}") else: - raise InvalidCodepoint(f"Codepoint {_unot(cp_value)} at position {pos + 1} of {repr(label)} not allowed") + raise InvalidCodepoint(f"Codepoint {_unot(cp_value)} at position {pos + 1} of {label!r} not allowed") check_bidi(label) @@ -385,12 +369,13 @@ def alabel(label: str) -> bytes: """ try: label_bytes = label.encode("ascii") + except UnicodeEncodeError: + pass + else: ulabel(label_bytes) if not valid_label_length(label_bytes): raise IDNAError("Label too long") return label_bytes - except UnicodeEncodeError: - pass check_label(label) label_bytes = _alabel_prefix + _punycode(label) @@ -492,7 +477,7 @@ def uts46_remap(domain: str, std3_rules: bool = True, transitional: bool = False elif status == "I": continue else: - raise InvalidCodepoint(f"Codepoint {_unot(code_point)} not allowed at position {pos + 1} in {repr(domain)}") + raise InvalidCodepoint(f"Codepoint {_unot(code_point)} not allowed at position {pos + 1} in {domain!r}") return unicodedata.normalize("NFC", output) diff --git a/contrib/python/idna/py3/idna/idnadata.py b/contrib/python/idna/py3/idna/idnadata.py index 8335f26059c..d768948790b 100644 --- a/contrib/python/idna/py3/idna/idnadata.py +++ b/contrib/python/idna/py3/idna/idnadata.py @@ -1,7 +1,6 @@ # This file is automatically generated by tools/idna-data -from functools import lru_cache -from typing import Dict +from functools import cache __version__ = "17.0.0" @@ -105,8 +104,8 @@ scripts = { } -@lru_cache(None) -def joining_types() -> Dict[int, int]: +@cache +def joining_types() -> dict[int, int]: return { 0xAD: 84, 0x300: 84, diff --git a/contrib/python/idna/py3/idna/intranges.py b/contrib/python/idna/py3/idna/intranges.py index ea3455bb8da..19d77810caf 100644 --- a/contrib/python/idna/py3/idna/intranges.py +++ b/contrib/python/idna/py3/idna/intranges.py @@ -6,10 +6,9 @@ in the original list?" in time O(log(# runs)). """ import bisect -from typing import List, Tuple -def intranges_from_list(list_: List[int]) -> Tuple[int, ...]: +def intranges_from_list(list_: list[int]) -> tuple[int, ...]: """Represent a list of integers as a sequence of ranges: ((start_0, end_0), (start_1, end_1), ...), such that the original integers are exactly those x such that start_i <= x < end_i for some i. @@ -34,11 +33,11 @@ def _encode_range(start: int, end: int) -> int: return (start << 32) | end -def _decode_range(r: int) -> Tuple[int, int]: +def _decode_range(r: int) -> tuple[int, int]: return (r >> 32), (r & ((1 << 32) - 1)) -def intranges_contain(int_: int, ranges: Tuple[int, ...]) -> bool: +def intranges_contain(int_: int, ranges: tuple[int, ...]) -> bool: """Determine if `int_` falls into one of the ranges in `ranges`.""" tuple_ = _encode_range(int_, 0) pos = bisect.bisect_left(ranges, tuple_) diff --git a/contrib/python/idna/py3/idna/package_data.py b/contrib/python/idna/py3/idna/package_data.py index 75debb75d6f..560eb22d4d0 100644 --- a/contrib/python/idna/py3/idna/package_data.py +++ b/contrib/python/idna/py3/idna/package_data.py @@ -1 +1 @@ -__version__ = "3.15" +__version__ = "3.16" diff --git a/contrib/python/idna/py3/idna/uts46data.py b/contrib/python/idna/py3/idna/uts46data.py index e13aa90fb86..f728fd96967 100644 --- a/contrib/python/idna/py3/idna/uts46data.py +++ b/contrib/python/idna/py3/idna/uts46data.py @@ -1,13 +1,13 @@ # This file is automatically generated by tools/idna-data -from typing import Tuple, Union +from typing import Union """IDNA Mapping Table from UTS46.""" __version__ = "17.0.0" -uts46data: Tuple[Union[Tuple[int, str], Tuple[int, str, str]], ...] = ( +uts46data: tuple[Union[tuple[int, str], tuple[int, str, str]], ...] = ( (0x0, "V"), (0x1, "V"), (0x2, "V"), diff --git a/contrib/python/idna/py3/patches/01-fix-tests.patch b/contrib/python/idna/py3/patches/01-fix-tests.patch new file mode 100644 index 00000000000..ebd7c115b2e --- /dev/null +++ b/contrib/python/idna/py3/patches/01-fix-tests.patch @@ -0,0 +1,26 @@ +--- contrib/python/idna/py3/tests/test_idna_cli.py (index) ++++ contrib/python/idna/py3/tests/test_idna_cli.py (working tree) +@@ -278,6 +278,7 @@ class CLIModuleEntryTests(unittest.TestCase): + check=False, + capture_output=True, + text=True, ++ env={"Y_PYTHON_ENTRY_POINT": ":main"}, + ) + self.assertEqual(result.returncode, 0) + self.assertEqual(result.stdout.strip(), "テスト") +@@ -291,6 +292,7 @@ class CLIModuleEntryTests(unittest.TestCase): + check=False, + capture_output=True, + text=True, ++ env={"Y_PYTHON_ENTRY_POINT": ":main"}, + ) + self.assertEqual(result.returncode, 0) + self.assertEqual( +@@ -305,6 +307,7 @@ class CLIModuleEntryTests(unittest.TestCase): + check=False, + capture_output=True, + text=True, ++ env={"Y_PYTHON_ENTRY_POINT": ":main"}, + ) + self.assertEqual(result.returncode, 1) + self.assertEqual(result.stdout, "") diff --git a/contrib/python/idna/py3/tests/test_idna.py b/contrib/python/idna/py3/tests/test_idna.py index 65eecfd88ee..e6029aede1f 100644 --- a/contrib/python/idna/py3/tests/test_idna.py +++ b/contrib/python/idna/py3/tests/test_idna.py @@ -2,14 +2,13 @@ import unittest import warnings -from typing import List, Tuple import idna class IDNATests(unittest.TestCase): def setUp(self): - self.tld_strings: List[Tuple[str, bytes]] = [ + self.tld_strings: list[tuple[str, bytes]] = [ ("\u6d4b\u8bd5", b"xn--0zwm56d"), ("\u092a\u0930\u0940\u0915\u094d\u0937\u093e", b"xn--11b5bs3a9aj6g"), ("\ud55c\uad6d", b"xn--3e0b707e"), @@ -104,7 +103,7 @@ class IDNATests(unittest.TestCase): import codecs import time - import idna.codec # noqa: F401 (register the idna2008 codec) + import idna.codec # register the idna2008 codec payload = "・" * 8000 + "漢" start = time.perf_counter() diff --git a/contrib/python/idna/py3/tests/test_idna_cli.py b/contrib/python/idna/py3/tests/test_idna_cli.py new file mode 100644 index 00000000000..fa79aa66b9a --- /dev/null +++ b/contrib/python/idna/py3/tests/test_idna_cli.py @@ -0,0 +1,318 @@ +#!/usr/bin/env python + +import io +import subprocess +import sys +import unittest +from contextlib import contextmanager, redirect_stderr, redirect_stdout +from unittest import mock + +from idna import cli +from idna.package_data import __version__ + + +def run_cli(*argv, stdin=None): + """Invoke ``cli.main`` with ``argv`` and capture (rc, stdout, stderr). + + Pass ``stdin`` as a string to simulate a piped stdin; omit it to + simulate an interactive terminal. + """ + out = io.StringIO() + err = io.StringIO() + with redirect_stdout(out), redirect_stderr(err), _patched_stdin(stdin): + rc = cli.main(list(argv)) + return rc, out.getvalue(), err.getvalue() + + +class _FakeStdin(io.StringIO): + """``io.StringIO`` with a configurable ``isatty`` result.""" + + def __init__(self, data: str, is_tty: bool) -> None: + super().__init__(data) + self._is_tty = is_tty + + def isatty(self) -> bool: + return self._is_tty + + +@contextmanager +def _patched_stdin(data): + """Replace ``sys.stdin`` with a fake stream and isatty() result.""" + stream = _FakeStdin("" if data is None else data, is_tty=data is None) + with mock.patch.object(sys, "stdin", stream): + yield + + +class CLIAutoDetectTests(unittest.TestCase): + def test_autodetect_encodes_unicode_input(self): + rc, out, err = run_cli("пример.рф") + self.assertEqual(rc, 0) + self.assertEqual(out.strip(), "xn--e1afmkfd.xn--p1ai") + self.assertEqual(err, "") + + def test_autodetect_decodes_alabel(self): + rc, out, _ = run_cli("xn--e1afmkfd.xn--p1ai") + self.assertEqual(rc, 0) + self.assertEqual(out.strip(), "пример.рф") + + def test_autodetect_decodes_when_any_label_is_alabel(self): + # Mixed label: at least one xn-- label triggers decode mode. + rc, out, _ = run_cli("xn--11b5bs3a9aj6g.example") + self.assertEqual(rc, 0) + self.assertEqual(out.strip(), "परीक्षा.example") + + def test_autodetect_encodes_plain_ascii_input(self): + rc, out, _ = run_cli("example.com") + self.assertEqual(rc, 0) + self.assertEqual(out.strip(), "example.com") + + def test_looks_like_alabel_is_case_insensitive(self): + self.assertTrue(cli._looks_like_alabel("XN--ZCKZAH")) + self.assertTrue(cli._looks_like_alabel("foo.Xn--ZcKzAh")) + self.assertFalse(cli._looks_like_alabel("example.com")) + self.assertFalse(cli._looks_like_alabel("xnfoo.example")) + + def test_looks_like_alabel_handles_unicode_dots(self): + # IDEOGRAPHIC FULL STOP should split labels for the purpose of + # spotting an A-label, matching the behavior of idna.encode/decode. + self.assertTrue(cli._looks_like_alabel("foo。xn--zckzah")) + + +class CLIExplicitModeTests(unittest.TestCase): + def test_encode_flag_forces_encode(self): + rc, out, _ = run_cli("-e", "παράδειγμα") + self.assertEqual(rc, 0) + self.assertEqual(out.strip(), "xn--hxajbheg2az3al") + + def test_long_encode_flag_forces_encode(self): + rc, out, _ = run_cli("--encode", "한국") + self.assertEqual(rc, 0) + self.assertEqual(out.strip(), "xn--3e0b707e") + + def test_decode_flag_forces_decode(self): + rc, out, _ = run_cli("-d", "xn--hxajbheg2az3al") + self.assertEqual(rc, 0) + self.assertEqual(out.strip(), "παράδειγμα") + + def test_long_decode_flag_forces_decode(self): + rc, out, _ = run_cli("--decode", "xn--3e0b707e") + self.assertEqual(rc, 0) + self.assertEqual(out.strip(), "한국") + + def test_encode_and_decode_are_mutually_exclusive(self): + # argparse exits via SystemExit (code 2) for mutually-exclusive errors. + with self.assertRaises(SystemExit) as ctx, redirect_stderr(io.StringIO()): + cli.main(["-e", "-d", "example.com"]) + self.assertEqual(ctx.exception.code, 2) + + +class CLIUTS46Tests(unittest.TestCase): + def test_uts46_is_applied_by_default(self): + # Uppercase Greek is not PVALID under IDNA 2008 alone; UTS #46 + # case-folds it so encoding succeeds. + rc, out, _ = run_cli("ΠΑΡΆΔΕΙΓΜΑ.ΕΛ") + self.assertEqual(rc, 0) + self.assertEqual(out.strip(), "xn--hxajbheg2az3al.xn--qxam") + + def test_strict_disables_uts46(self): + rc, out, err = run_cli("--strict", "ΠΑΡΆΔΕΙΓΜΑ.ΕΛ") + self.assertEqual(rc, 1) + self.assertEqual(out, "") + self.assertIn("encode failed", err) + self.assertIn("U+03A0", err) + + def test_strict_does_not_affect_valid_inputs(self): + rc, out, _ = run_cli("--strict", "пример.рф") + self.assertEqual(rc, 0) + self.assertEqual(out.strip(), "xn--e1afmkfd.xn--p1ai") + + def test_uts46_applies_to_explicit_decode(self): + # Trailing FULLWIDTH FULL STOP (U+FF0E) is mapped to "." by UTS #46 + # so the input still decodes as a single A-label. + rc, out, _ = run_cli("-d", "xn--zckzah．") + self.assertEqual(rc, 0) + self.assertEqual(out.strip(), "テスト.") + + +class CLIErrorHandlingTests(unittest.TestCase): + def test_invalid_input_returns_nonzero_and_writes_to_stderr(self): + rc, out, err = run_cli("foo_bar") + self.assertEqual(rc, 1) + self.assertEqual(out, "") + self.assertTrue(err.startswith("idna: encode failed for 'foo_bar':")) + + def test_invalid_alabel_decode_returns_nonzero(self): + # Trailing hyphen after the xn-- prefix is rejected by ulabel(). + rc, out, err = run_cli("-d", "xn--") + self.assertEqual(rc, 1) + self.assertEqual(out, "") + self.assertIn("decode failed", err) + + def test_missing_argument_with_tty_stdin_is_an_argparse_error(self): + # Interactive shell (no piped stdin) and no domain arg → argparse exits 2. + with self.assertRaises(SystemExit) as ctx, redirect_stderr(io.StringIO()), _patched_stdin(None): + cli.main([]) + self.assertEqual(ctx.exception.code, 2) + + +class CLIVersionTests(unittest.TestCase): + def test_version_flag(self): + buf = io.StringIO() + with self.assertRaises(SystemExit) as ctx, redirect_stdout(buf): + cli.main(["--version"]) + self.assertEqual(ctx.exception.code, 0) + self.assertIn(__version__, buf.getvalue()) + + +class CLIMultipleDomainsTests(unittest.TestCase): + def test_multiple_unicode_positional_domains_are_all_encoded(self): + rc, out, _ = run_cli("пример.рф", "παράδειγμα", "한국") + self.assertEqual(rc, 0) + self.assertEqual( + out.splitlines(), + ["xn--e1afmkfd.xn--p1ai", "xn--hxajbheg2az3al", "xn--3e0b707e"], + ) + + def test_multiple_alabel_positional_domains_are_all_decoded(self): + rc, out, _ = run_cli("xn--e1afmkfd.xn--p1ai", "xn--hxajbheg2az3al", "xn--3e0b707e") + self.assertEqual(rc, 0) + self.assertEqual(out.splitlines(), ["пример.рф", "παράδειγμα", "한국"]) + + def test_mode_flag_applies_to_every_positional_domain(self): + rc, out, _ = run_cli("-e", "한국", "παράδειγμα") + self.assertEqual(rc, 0) + self.assertEqual( + out.splitlines(), + ["xn--3e0b707e", "xn--hxajbheg2az3al"], + ) + + def test_failures_are_reported_per_domain_and_processing_continues(self): + rc, out, err = run_cli("пример.рф", "foo_bar", "παράδειγμα") + self.assertEqual(rc, 1) + self.assertEqual( + out.splitlines(), + ["xn--e1afmkfd.xn--p1ai", "xn--hxajbheg2az3al"], + ) + self.assertIn("encode failed for 'foo_bar'", err) + + +class CLIModeLockTests(unittest.TestCase): + """The first input picks the mode; the rest of the stream follows.""" + + def test_first_unicode_input_locks_subsequent_inputs_to_encode(self): + # 'foo_bar' is invalid in either direction; the stderr mode word + # reveals which mode the second input was processed in. + rc, out, err = run_cli("пример.рф", "foo_bar") + self.assertEqual(rc, 1) + self.assertEqual(out.splitlines(), ["xn--e1afmkfd.xn--p1ai"]) + self.assertIn("encode failed for 'foo_bar'", err) + + def test_first_alabel_input_locks_subsequent_inputs_to_decode(self): + rc, out, err = run_cli("xn--zckzah", "foo_bar") + self.assertEqual(rc, 1) + self.assertEqual(out.splitlines(), ["テスト"]) + self.assertIn("decode failed for 'foo_bar'", err) + + def test_explicit_mode_flag_overrides_first_input_heuristic(self): + # First input looks like an A-label but -e forces encode for everything. + rc, out, err = run_cli("-e", "xn--zckzah", "foo_bar") + self.assertEqual(rc, 1) + self.assertEqual(out.splitlines(), ["xn--zckzah"]) + self.assertIn("encode failed for 'foo_bar'", err) + + def test_stdin_first_line_locks_mode_for_the_stream(self): + rc, out, err = run_cli(stdin="xn--zckzah\nfoo_bar\n") + self.assertEqual(rc, 1) + self.assertEqual(out.splitlines(), ["テスト"]) + self.assertIn("decode failed for 'foo_bar'", err) + + +class CLIStdinTests(unittest.TestCase): + def test_reads_one_domain_per_line_from_piped_stdin(self): + rc, out, _ = run_cli(stdin="пример.рф\nπαράδειγμα\n한국\n") + self.assertEqual(rc, 0) + self.assertEqual( + out.splitlines(), + ["xn--e1afmkfd.xn--p1ai", "xn--hxajbheg2az3al", "xn--3e0b707e"], + ) + + def test_blank_lines_in_stdin_are_skipped(self): + rc, out, _ = run_cli(stdin="\nпример.рф\n\n \nπαράδειγμα\n") + self.assertEqual(rc, 0) + self.assertEqual( + out.splitlines(), + ["xn--e1afmkfd.xn--p1ai", "xn--hxajbheg2az3al"], + ) + + def test_empty_stdin_exits_cleanly_with_no_output(self): + rc, out, err = run_cli(stdin="") + self.assertEqual(rc, 0) + self.assertEqual(out, "") + self.assertEqual(err, "") + + def test_positional_domains_take_precedence_over_piped_stdin(self): + rc, out, _ = run_cli("example.com", stdin="ignored.example\n") + self.assertEqual(rc, 0) + self.assertEqual(out.splitlines(), ["example.com"]) + + def test_stdin_errors_continue_processing_and_set_exit_code(self): + rc, out, err = run_cli(stdin="пример.рф\nfoo_bar\nπαράδειγμα\n") + self.assertEqual(rc, 1) + self.assertEqual( + out.splitlines(), + ["xn--e1afmkfd.xn--p1ai", "xn--hxajbheg2az3al"], + ) + self.assertIn("encode failed for 'foo_bar'", err) + + def test_decode_flag_applies_to_every_stdin_line(self): + rc, out, _ = run_cli("-d", stdin="xn--hxajbheg2az3al\nxn--3e0b707e\n") + self.assertEqual(rc, 0) + self.assertEqual(out.splitlines(), ["παράδειγμα", "한국"]) + + +class CLIModuleEntryTests(unittest.TestCase): + def test_python_dash_m_idna_runs_cli(self): + # End-to-end: spawn ``python -m idna`` and check the round trip. + result = subprocess.run( + [sys.executable, "-m", "idna", "xn--zckzah"], + check=False, + capture_output=True, + text=True, + env={"Y_PYTHON_ENTRY_POINT": ":main"}, + ) + self.assertEqual(result.returncode, 0) + self.assertEqual(result.stdout.strip(), "テスト") + self.assertEqual(result.stderr, "") + + def test_python_dash_m_idna_reads_piped_stdin(self): + # End-to-end: pipe a list of A-labels and confirm they all decode. + result = subprocess.run( + [sys.executable, "-m", "idna"], + input="xn--e1afmkfd.xn--p1ai\nxn--zckzah\n", + check=False, + capture_output=True, + text=True, + env={"Y_PYTHON_ENTRY_POINT": ":main"}, + ) + self.assertEqual(result.returncode, 0) + self.assertEqual( + result.stdout.splitlines(), + ["пример.рф", "テスト"], + ) + self.assertEqual(result.stderr, "") + + def test_python_dash_m_idna_reports_errors(self): + result = subprocess.run( + [sys.executable, "-m", "idna", "foo_bar"], + check=False, + capture_output=True, + text=True, + env={"Y_PYTHON_ENTRY_POINT": ":main"}, + ) + self.assertEqual(result.returncode, 1) + self.assertEqual(result.stdout, "") + self.assertIn("encode failed", result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/contrib/python/idna/py3/ya.make b/contrib/python/idna/py3/ya.make index 6f0b1006c52..8f96cee300d 100644 --- a/contrib/python/idna/py3/ya.make +++ b/contrib/python/idna/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(3.15) +VERSION(3.16) LICENSE(BSD-3-Clause) @@ -11,6 +11,8 @@ NO_LINT() PY_SRCS( TOP_LEVEL idna/__init__.py + idna/__main__.py + idna/cli.py idna/codec.py idna/compat.py idna/core.py @@ -23,6 +25,7 @@ PY_SRCS( RESOURCE_FILES( PREFIX contrib/python/idna/py3/ .dist-info/METADATA + .dist-info/entry_points.txt .dist-info/top_level.txt idna/py.typed ) diff --git a/contrib/python/yarl/.dist-info/METADATA b/contrib/python/yarl/.dist-info/METADATA index 9870bc4c565..97585939dab 100644 --- a/contrib/python/yarl/.dist-info/METADATA +++ b/contrib/python/yarl/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: yarl -Version: 1.12.1 +Version: 1.13.1 Summary: Yet another URL library Home-page: https://github.com/aio-libs/yarl Author: Andrew Svetlov @@ -262,6 +262,60 @@ Changelog .. towncrier release notes start +1.13.1 +====== + +*(2024-09-27)* + + +Miscellaneous internal changes +------------------------------ + +- Improved performance of calling ``yarl.URL.build()`` with ``authority`` -- by `@bdraco <https://github.com/sponsors/bdraco>`__. + + *Related issues and pull requests on GitHub:* + `#1163 <https://github.com/aio-libs/yarl/issues/1163>`__. + + +---- + + +1.13.0 +====== + +*(2024-09-26)* + + +Bug fixes +--------- + +- Started rejecting ASCII hostnames with invalid characters. For host strings that + look like authority strings, the exception message includes advice on what to do + instead -- by `@mjpieters <https://github.com/sponsors/mjpieters>`__. + + *Related issues and pull requests on GitHub:* + `#880 <https://github.com/aio-libs/yarl/issues/880>`__, `#954 <https://github.com/aio-libs/yarl/issues/954>`__. + +- Fixed IPv6 addresses missing brackets when the ``~yarl.URL`` was converted to a string -- by `@bdraco <https://github.com/sponsors/bdraco>`__. + + *Related issues and pull requests on GitHub:* + `#1157 <https://github.com/aio-libs/yarl/issues/1157>`__, `#1158 <https://github.com/aio-libs/yarl/issues/1158>`__. + + +Features +-------- + +- Added ``~yarl.URL.host_subcomponent`` which returns the ``3986#section-3.2.2`` host subcomponent -- by `@bdraco <https://github.com/sponsors/bdraco>`__. + + The only current practical difference between ``~yarl.URL.raw_host`` and ``~yarl.URL.host_subcomponent`` is that IPv6 addresses are returned bracketed. + + *Related issues and pull requests on GitHub:* + `#1159 <https://github.com/aio-libs/yarl/issues/1159>`__. + + +---- + + 1.12.1 ====== diff --git a/contrib/python/yarl/patches/02-dont-normalize-disable-tests.patch b/contrib/python/yarl/patches/02-dont-normalize-disable-tests.patch index 79c9ccc2d3e..49ecf81c0ff 100644 --- a/contrib/python/yarl/patches/02-dont-normalize-disable-tests.patch +++ b/contrib/python/yarl/patches/02-dont-normalize-disable-tests.patch @@ -8,28 +8,6 @@ def test_is_default_port_for_absolute_url_with_default_port(): url = URL("http://example.com:80") assert url.is_default_port() ---- contrib/python/yarl/tests/test_url_build.py (index) -+++ contrib/python/yarl/tests/test_url_build.py (working tree) -@@ -15,16 +15,19 @@ def test_build_simple(): - assert str(u) == "http://127.0.0.1" - - - def test_url_build_ipv6(): - u = URL.build(scheme="http", host="::1") - assert str(u) == "http://::1" - - - def test_url_build_ipv6_brackets(): - u = URL.build(scheme="http", host="[::1]") - assert str(u) == "http://::1" - - - def test_url_ipv4_in_ipv6(): - u = URL.build(scheme="http", host="2001:db8:122:344::192.0.2.33") - assert str(u) == "http://2001:db8:122:344::c000:221" --- contrib/python/yarl/tests/test_url_update_netloc.py (index) +++ contrib/python/yarl/tests/test_url_update_netloc.py (working tree) @@ -196,6 +196,7 @@ def test_with_port(): diff --git a/contrib/python/yarl/tests/test_cache.py b/contrib/python/yarl/tests/test_cache.py index 46d5b01f159..9b0e75a3551 100644 --- a/contrib/python/yarl/tests/test_cache.py +++ b/contrib/python/yarl/tests/test_cache.py @@ -13,7 +13,7 @@ def test_cache_clear() -> None: def test_cache_info() -> None: info = yarl.cache_info() - assert info.keys() == {"idna_encode", "idna_decode", "ip_address"} + assert info.keys() == {"idna_encode", "idna_decode", "ip_address", "host_validate"} def test_cache_configure_default() -> None: @@ -22,11 +22,17 @@ def test_cache_configure_default() -> None: def test_cache_configure_None() -> None: yarl.cache_configure( - idna_encode_size=None, idna_decode_size=None, ip_address_size=None + idna_encode_size=None, + idna_decode_size=None, + ip_address_size=None, + host_validate_size=None, ) def test_cache_configure_explicit() -> None: yarl.cache_configure( - idna_encode_size=128, idna_decode_size=128, ip_address_size=128 + idna_encode_size=128, + idna_decode_size=128, + ip_address_size=128, + host_validate_size=128, ) diff --git a/contrib/python/yarl/tests/test_url.py b/contrib/python/yarl/tests/test_url.py index 1dd98197fe1..44018c73bce 100644 --- a/contrib/python/yarl/tests/test_url.py +++ b/contrib/python/yarl/tests/test_url.py @@ -176,6 +176,24 @@ def test_raw_host(): assert url.raw_host == url._val.hostname + ("host"), + [ + ("example.com"), + ("[::1]"), + ("xn--gnter-4ya.com"), + ], +) +def test_host_subcomponent(host: str): + url = URL(f"http://{host}") + assert url.host_subcomponent == host + + +def test_host_subcomponent_return_idna_encoded_host(): + url = URL("http://оун-упа.укр") + assert url.host_subcomponent == "xn----8sb1bdhvc.xn--j1amh" + + def test_raw_host_non_ascii(): url = URL("http://оун-упа.укр") assert "xn----8sb1bdhvc.xn--j1amh" == url.raw_host @@ -2041,3 +2059,14 @@ def test_parsing_populates_cache(): assert url._cache["raw_query_string"] == "a=b" assert url._cache["raw_fragment"] == "frag" assert url._cache["scheme"] == "http" + + + ("host", "is_authority"), + [ + *(("other_gen_delim_" + c, False) for c in "[]"), + ], +) +def test_build_with_invalid_ipv6_host(host: str, is_authority: bool): + with pytest.raises(ValueError, match="Invalid IPv6 URL"): + URL(f"http://{host}/") diff --git a/contrib/python/yarl/tests/test_url_build.py b/contrib/python/yarl/tests/test_url_build.py index a0ab77b2d4e..cc9b15f7e4f 100644 --- a/contrib/python/yarl/tests/test_url_build.py +++ b/contrib/python/yarl/tests/test_url_build.py @@ -15,22 +15,24 @@ def test_build_simple(): assert str(u) == "http://127.0.0.1" def test_url_build_ipv6(): u = URL.build(scheme="http", host="::1") - assert str(u) == "http://::1" + assert str(u) == "http://[::1]" -def test_url_build_ipv6_brackets(): - u = URL.build(scheme="http", host="[::1]") - assert str(u) == "http://::1" +def test_url_build_ipv6_brackets_encoded(): + u = URL.build(scheme="http", host="[::1]", encoded=True) + assert str(u) == "http://[::1]" + + +def test_url_build_ipv6_brackets_not_encoded(): + u = URL.build(scheme="http", host="::1", encoded=False) + assert str(u) == "http://[::1]" def test_url_ipv4_in_ipv6(): u = URL.build(scheme="http", host="2001:db8:122:344::192.0.2.33") - assert str(u) == "http://2001:db8:122:344::c000:221" + assert str(u) == "http://[2001:db8:122:344::c000:221]" def test_build_with_scheme(): @@ -118,6 +120,25 @@ def test_build_with_authority_and_host(): URL.build(authority="host.com", host="example.com") + ("host", "is_authority"), + [ + ("user:[email protected]", True), + ("[email protected]", True), + ("host:com", False), + ("not_percent_encoded%Zf", False), + ("still_not_percent_encoded%fZ", False), + *(("other_gen_delim_" + c, False) for c in "/?#[]"), + ], +) +def test_build_with_invalid_host(host: str, is_authority: bool): + match = r"Host '[^']+' cannot contain '[^']+' \(at position \d+\)" + if is_authority: + match += ", if .* use 'authority' instead of 'host'" + with pytest.raises(ValueError, match=f"{match}$"): + URL.build(host=host) + + def test_build_with_authority(): url = URL.build(scheme="http", authority="степан:[email protected]:8000", path="path") assert ( @@ -132,6 +153,31 @@ def test_build_with_authority_without_encoding(): assert str(url) == "http://foo:[email protected]:8000/path" +def test_build_with_authority_empty_host_no_scheme(): + url = URL.build(authority="", path="path") + assert str(url) == "path" + + +def test_build_with_authority_and_only_user(): + url = URL.build(scheme="https", authority="user:@foo.com", path="/path") + assert str(url) == "https://user:@foo.com/path" + + +def test_build_with_authority_with_port(): + url = URL.build(scheme="https", authority="foo.com:8080", path="/path") + assert str(url) == "https://foo.com:8080/path" + + +def test_build_with_authority_with_ipv6(): + url = URL.build(scheme="https", authority="[::1]", path="/path") + assert str(url) == "https://[::1]/path" + + +def test_build_with_authority_with_ipv6_and_port(): + url = URL.build(scheme="https", authority="[::1]:81", path="/path") + assert str(url) == "https://[::1]:81/path" + + def test_query_str(): u = URL.build(scheme="http", host="127.0.0.1", path="/", query_string="arg=value1") assert str(u) == "http://127.0.0.1/?arg=value1" diff --git a/contrib/python/yarl/tests/test_url_parsing.py b/contrib/python/yarl/tests/test_url_parsing.py index 4fe95185e71..0d58c9d09bb 100644 --- a/contrib/python/yarl/tests/test_url_parsing.py +++ b/contrib/python/yarl/tests/test_url_parsing.py @@ -604,3 +604,28 @@ def test_schemes_that_require_host(scheme: str) -> None: ) with pytest.raises(ValueError, match=expect): URL(f"{scheme}://:1") + + + ("url", "hostname", "hostname_without_brackets"), + [ + ("http://[::1]", "[::1]", "::1"), + ("http://[::1]:8080", "[::1]", "::1"), + ("http://127.0.0.1:8080", "127.0.0.1", "127.0.0.1"), + ( + "http://xn--jxagkqfkduily1i.eu", + "xn--jxagkqfkduily1i.eu", + "xn--jxagkqfkduily1i.eu", + ), + ], +) +def test_url_round_trips( + url: str, hostname: str, hostname_without_brackets: str +) -> None: + """Verify that URLs round-trip correctly.""" + parsed = URL(url) + assert parsed._val.hostname == hostname_without_brackets + assert parsed.raw_host == hostname_without_brackets + assert parsed.host_subcomponent == hostname + assert str(parsed) == url + assert str(URL(str(parsed))) == url diff --git a/contrib/python/yarl/tests/test_url_update_netloc.py b/contrib/python/yarl/tests/test_url_update_netloc.py index cfe6ca7d457..a8ff1f9f8b3 100644 --- a/contrib/python/yarl/tests/test_url_update_netloc.py +++ b/contrib/python/yarl/tests/test_url_update_netloc.py @@ -171,6 +171,26 @@ def test_with_host_non_ascii(): assert url2.authority == "оун-упа.укр:123" + ("host", "is_authority"), + [ + ("user:[email protected]", True), + ("[email protected]", True), + ("host:com", False), + ("not_percent_encoded%Zf", False), + ("still_not_percent_encoded%fZ", False), + *(("other_gen_delim_" + c, False) for c in "/?#[]"), + ], +) +def test_with_invalid_host(host: str, is_authority: bool): + url = URL("http://example.com:123") + match = r"Host '[^']+' cannot contain '[^']+' \(at position \d+\)" + if is_authority: + match += ", if .* use 'authority' instead of 'host'" + with pytest.raises(ValueError, match=f"{match}$"): + url.with_host(host=host) + + def test_with_host_percent_encoded(): url = URL("http://%25cf%2580%cf%80:%25cf%2580%cf%[email protected]:123") url2 = url.with_host("%cf%80.org") diff --git a/contrib/python/yarl/ya.make b/contrib/python/yarl/ya.make index 43d5345771f..b272e44b23e 100644 --- a/contrib/python/yarl/ya.make +++ b/contrib/python/yarl/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(1.12.1) +VERSION(1.13.1) LICENSE(Apache-2.0) diff --git a/contrib/python/yarl/yarl/__init__.py b/contrib/python/yarl/yarl/__init__.py index 50d24c0e778..1bc12c9e4f4 100644 --- a/contrib/python/yarl/yarl/__init__.py +++ b/contrib/python/yarl/yarl/__init__.py @@ -8,7 +8,7 @@ from ._url import ( cache_info, ) -__version__ = "1.12.1" +__version__ = "1.13.1" __all__ = ( "URL", diff --git a/contrib/python/yarl/yarl/_url.py b/contrib/python/yarl/yarl/_url.py index 8a48628fcd9..e695800b967 100644 --- a/contrib/python/yarl/yarl/_url.py +++ b/contrib/python/yarl/yarl/_url.py @@ -1,4 +1,5 @@ import math +import re import sys import warnings from collections.abc import Mapping, Sequence @@ -45,6 +46,22 @@ SCHEME_REQUIRES_HOST = frozenset(("http", "https", "ws", "wss", "ftp")) sentinel = object() +# reg-name: unreserved / pct-encoded / sub-delims +# this pattern matches anything that is *not* in those classes. and is only used +# on lower-cased ASCII values. +_not_reg_name = re.compile( + r""" + # any character not in the unreserved or sub-delims sets, plus % + # (validated with the additional check for pct-encoded sequences below) + [^a-z0-9\-._~!$&'()*+,;=%] + | + # % only allowed if it is part of a pct-encoded + # sequence of 2 hex digits. + %(?![0-9a-f]{2}) + """, + re.VERBOSE, +) + SimpleQuery = Union[str, int, float] QueryVariable = Union[SimpleQuery, "Sequence[SimpleQuery]"] Query = Union[ @@ -64,6 +81,7 @@ class CacheInfo(TypedDict): idna_encode: _CacheInfo idna_decode: _CacheInfo ip_address: _CacheInfo + host_validate: _CacheInfo class _SplitResultDict(TypedDict, total=False): @@ -269,7 +287,7 @@ class URL: raise ValueError(msg) else: host = "" - host = cls._encode_host(host) + host = cls._encode_host(host, validate_host=False) raw_user = None if username is None else cls._REQUOTER(username) raw_password = None if password is None else cls._REQUOTER(password) netloc = cls._make_netloc( @@ -350,10 +368,15 @@ class URL: if encoded: netloc = authority else: - tmp = SplitResult("", authority, "", "", "") - port = None if tmp.port == DEFAULT_PORTS.get(scheme) else tmp.port + _user, _password, _host, _port = cls._split_netloc(authority) + port = None if _port == DEFAULT_PORTS.get(scheme) else _port + _host = ( + "" + if _host is None + else cls._encode_host(_host, validate_host=False) + ) netloc = cls._make_netloc( - tmp.username, tmp.password, tmp.hostname, port, encode=True + _user, _password, _host, port, encode=True, encode_host=False ) elif not user and not password and not host and not port: netloc = "" @@ -395,7 +418,7 @@ class URL: netloc=self._make_netloc( self.raw_user, self.raw_password, - self.raw_host, + self.host_subcomponent, port, encode_host=False, ) @@ -647,6 +670,8 @@ class URL: None for relative URLs. + When working with IPv6 addresses, use the `host_subcomponent` property instead + as it will return the host subcomponent with brackets. """ # Use host instead of hostname for sake of shortness # May add .hostname prop later @@ -660,17 +685,36 @@ class URL: None for relative URLs. """ - raw = self.raw_host - if raw is None: + if (raw := self.raw_host) is None: return None - if "%" in raw: - # Hack for scoped IPv6 addresses like - # fe80::2%Перевірка - # presence of '%' sign means only IPv6 address, so idna is useless. + if raw and raw[-1].isdigit() or ":" in raw: + # IP addresses are never IDNA encoded return raw return _idna_decode(raw) @cached_property + def host_subcomponent(self) -> Union[str, None]: + """Return the host subcomponent part of URL. + + None for relative URLs. + + https://datatracker.ietf.org/doc/html/rfc3986#section-3.2.2 + + `IP-literal = "[" ( IPv6address / IPvFuture ) "]"` + + Examples: + - `http://example.com:8080` -> `example.com` + - `http://example.com:80` -> `example.com` + - `https://127.0.0.1:8443` -> `127.0.0.1` + - `https://[::1]:8443` -> `[::1]` + - `http://[::1]` -> `[::1]` + + """ + if (raw := self.raw_host) is None: + return None + return f"[{raw}]" if ":" in raw else raw + + @cached_property def port(self) -> Union[int, None]: """Port part of URL, with scheme-based fallback. @@ -938,7 +982,9 @@ class URL: return prefix + "/".join(_normalize_path_segments(segments)) @classmethod - def _encode_host(cls, host: str, human: bool = False) -> str: + def _encode_host( + cls, host: str, human: bool = False, validate_host: bool = True + ) -> str: if "%" in host: raw_ip, sep, zone = host.partition("%") else: @@ -953,7 +999,8 @@ class URL: # - 127.0.0.1 (last character is a digit) # - 2001:db8::ff00:42:8329 (contains a colon) # - 2001:db8::ff00:42:8329%eth0 (contains a colon) - # - [2001:db8::ff00:42:8329] (contains a colon) + # - [2001:db8::ff00:42:8329] (contains a colon -- brackets should + # have been removed before it gets here) # Rare IP Address formats are not supported per: # https://datatracker.ietf.org/doc/html/rfc3986#section-7.4 # @@ -977,11 +1024,18 @@ class URL: return host host = host.lower() + if human: + return host + # IDNA encoding is slow, # skip it for ASCII-only strings # Don't move the check into _idna_encode() helper # to reduce the cache size - if human or host.isascii(): + if host.isascii(): + # Check for invalid characters explicitly; _idna_encode() does this + # for non-ascii host names. + if validate_host: + _host_validate(host) return host return _idna_encode(host) @@ -1537,12 +1591,31 @@ def _ip_compressed_version(raw_ip: str) -> Tuple[str, int]: return ip.compressed, ip.version +@lru_cache(_MAXCACHE) +def _host_validate(host: str) -> None: + """Validate an ascii host name.""" + invalid = _not_reg_name.search(host) + if invalid is None: + return + value, pos, extra = invalid.group(), invalid.start(), "" + if value == "@" or (value == ":" and "@" in host[pos:]): + # this looks like an authority string + extra = ( + ", if the value includes a username or password, " + "use 'authority' instead of 'host'" + ) + raise ValueError( + f"Host {host!r} cannot contain {value!r} (at position " f"{pos}){extra}" + ) from None + + @rewrite_module def cache_clear() -> None: """Clear all LRU caches.""" _idna_decode.cache_clear() _idna_encode.cache_clear() _ip_compressed_version.cache_clear() + _host_validate.cache_clear() @rewrite_module @@ -1552,6 +1625,7 @@ def cache_info() -> CacheInfo: "idna_encode": _idna_encode.cache_info(), "idna_decode": _idna_decode.cache_info(), "ip_address": _ip_compressed_version.cache_info(), + "host_validate": _host_validate.cache_info(), } @@ -1561,12 +1635,14 @@ def cache_configure( idna_encode_size: Union[int, None] = _MAXCACHE, idna_decode_size: Union[int, None] = _MAXCACHE, ip_address_size: Union[int, None] = _MAXCACHE, + host_validate_size: Union[int, None] = _MAXCACHE, ) -> None: """Configure LRU cache sizes.""" - global _idna_decode, _idna_encode, _ip_compressed_version + global _idna_decode, _idna_encode, _ip_compressed_version, _host_validate _idna_encode = lru_cache(idna_encode_size)(_idna_encode.__wrapped__) _idna_decode = lru_cache(idna_decode_size)(_idna_decode.__wrapped__) _ip_compressed_version = lru_cache(ip_address_size)( _ip_compressed_version.__wrapped__ ) + _host_validate = lru_cache(host_validate_size)(_host_validate.__wrapped__) diff --git a/library/cpp/yt/error/error_code.cpp b/library/cpp/yt/error/error_code.cpp index d82d03214d4..ef9ce0f33c1 100644 --- a/library/cpp/yt/error/error_code.cpp +++ b/library/cpp/yt/error/error_code.cpp @@ -62,11 +62,10 @@ void TErrorCodeRegistry::RegisterErrorCode(int code, const TErrorCodeInfo& error if (code == 119) { return; } - YT_LOG_FATAL( - "Duplicate error code (Code: %v, StoredCodeInfo: %v, NewCodeInfo: %v)", - code, - CodeToInfo_[code], - errorCodeInfo); + YT_TLOG_FATAL("Duplicate error code") + .With("Code", code) + .With("StoredCodeInfo", CodeToInfo_[code]) + .With("NewCodeInfo", errorCodeInfo); } } @@ -91,11 +90,11 @@ void TErrorCodeRegistry::RegisterErrorCodeRange(int from, int to, std::string na TErrorCodeRangeInfo newRange{from, to, std::move(namespaceName), std::move(formatter)}; for (const auto& range : ErrorCodeRanges_) { - YT_LOG_FATAL_IF( + YT_TLOG_FATAL_IF( range.Intersects(newRange), - "Intersecting error code ranges registered (FirstRange: %v, SecondRange: %v)", - range, - newRange); + "Intersecting error code ranges registered") + .With("FirstRange", range) + .With("SecondRange", newRange); } ErrorCodeRanges_.push_back(std::move(newRange)); CheckCodesAgainstRanges(); @@ -105,14 +104,13 @@ void TErrorCodeRegistry::CheckCodesAgainstRanges() const { for (const auto& [code, info] : CodeToInfo_) { for (const auto& range : ErrorCodeRanges_) { - YT_LOG_FATAL_IF( + YT_TLOG_FATAL_IF( range.Contains(code), - "Error code range contains another registered code " - "(Range: %v, Code: %v, RangeCodeInfo: %v, StandaloneCodeInfo: %v)", - range, - code, - range.Get(code), - info); + "Error code range contains another registered code") + .With("Range", range) + .With("Code", code) + .With("RangeCodeInfo", range.Get(code)) + .With("StandaloneCodeInfo", info); } } } diff --git a/library/cpp/yt/logging/backends/stream/unittests/stream_log_manager_ut.cpp b/library/cpp/yt/logging/backends/stream/unittests/stream_log_manager_ut.cpp index cb3e244e3b9..95b2ade9b65 100644 --- a/library/cpp/yt/logging/backends/stream/unittests/stream_log_manager_ut.cpp +++ b/library/cpp/yt/logging/backends/stream/unittests/stream_log_manager_ut.cpp @@ -20,7 +20,7 @@ TEST(TStreamLogManagerTest, Simple) TStringOutput output(str); auto logManager = CreateStreamLogManager(&output); TLogger Logger(logManager.get(), "Test"); - YT_LOG_INFO("Hello world"); + YT_TLOG_INFO("Hello world"); } TVector<TStringBuf> tokens; diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp index e85738f0fbf..09c69eef0d8 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp @@ -600,10 +600,17 @@ static TInterconnectSettings GetInterconnectSettings(const NKikimrConfig::TInter result.EnableUringSQPOLL = config.GetEnableUringSQPOLL(); } - result.EnableInterconnectSessionV2 = config.GetEnableInterconnectSessionV2(); - result.ChecksumInterconnectSessionV2 = config.GetChecksumInterconnectSessionV2(); - result.EnableSQPOLLv2 = config.GetEnableSQPOLLv2(); - result.EnablePreserializeInV2 = config.GetEnablePreserializeInV2(); + if (config.HasV2Config()) { + const auto& v2 = config.GetV2Config(); + result.V2.Enable = v2.GetEnable(); + result.V2.ChecksumEvents = v2.GetChecksumEvents(); + result.V2.EnableSQPOLL = v2.GetEnableSQPOLL(); + result.V2.EnablePreserializeEvents = v2.GetEnablePreserializeEvents(); + result.V2.UringEngineThreads = v2.GetUringEngineThreads(); + result.V2.UringEngineRingsPerShard = v2.GetUringEngineRingsPerShard(); + result.V2.UringEngineSqThreadIdleMs = v2.GetUringEngineSqThreadIdleMs(); + result.V2.ShareRingsAmongThreads = v2.GetShareRingsAmongThreads(); + } return result; } @@ -765,14 +772,17 @@ void TBasicServicesInitializer::InitializeServices(NActors::TActorSystemSetup* s icCommon->ChannelsConfig = channels; icCommon->Settings = settings; - if (settings.EnableInterconnectSessionV2) { + if (settings.V2.Enable) { // Create the shared v2 io_uring data-plane engine once, at startup, and publish it in Common. // The actor system does not exist yet, so the engine is bound to it later (once it is up, // TInterconnectProxyTCP::Registered calls SetActorSystem). CreateUringEngine returns null when // io_uring is unavailable, in which case v2 is simply never negotiated during the handshake. - icCommon->UringEngineV2 = CreateUringEngine(icConfig.GetUringEngineThreadsV2(), + icCommon->UringEngineV2 = CreateUringEngine(settings.V2.UringEngineThreads, interconectCounters->GetSubgroup("subsystem", "uring"), - settings.EnableSQPOLLv2); + settings.V2.EnableSQPOLL, + settings.V2.UringEngineRingsPerShard, + settings.V2.UringEngineSqThreadIdleMs, + settings.V2.ShareRingsAmongThreads); setup->OnActorSystemCreated.push_back([engine = icCommon->UringEngineV2](TActorSystem *actorSystem) { if (engine) { engine->SetActorSystem(actorSystem); diff --git a/ydb/core/kqp/ut/olap/indexes/indexes_ut.cpp b/ydb/core/kqp/ut/olap/indexes/indexes_ut.cpp index 7d367abc7d4..0213cea8bd6 100644 --- a/ydb/core/kqp/ut/olap/indexes/indexes_ut.cpp +++ b/ydb/core/kqp/ut/olap/indexes/indexes_ut.cpp @@ -701,7 +701,7 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { UNIT_ASSERT_C(!status.IsSuccess(), status.GetIssues().ToString()); } - Y_UNIT_TEST(MinMaxIndexUsedInQueries, EUseQueryService, ELocalIndexAsSchemeObject) { + Y_UNIT_TEST(LocalIndexesUsedInQueries, EUseQueryService, ELocalIndexAsSchemeObject) { const bool UseQueryService = (Arg<0>() == EUseQueryService::QueryService); const bool LocalIndexAsSchemeObject = (Arg<1>() == ELocalIndexAsSchemeObject::SchemeObjectEnabled); auto settings = TKikimrSettings() @@ -745,34 +745,47 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { }; assertDDLQueryOk(R"( - CREATE TABLE `/Root/minmax_test_applied_applied` ( + CREATE TABLE `/Root/olap_indexes_applied_test` ( `key` Int32 NOT NULL, `value` String NOT NULL, + `value_bloom` String NOT NULL, PRIMARY KEY (`key`) ) PARTITION BY HASH (`key`) WITH ( STORE = COLUMN ); - ALTER TABLE `/Root/minmax_test_applied_applied` ADD INDEX `value_mm` LOCAL USING min_max ON(`value`); + ALTER TABLE `/Root/olap_indexes_applied_test` + ADD INDEX idx_ngram LOCAL USING bloom_ngram_filter + ON (`value_bloom`) + ; + + ALTER TABLE `/Root/olap_indexes_applied_test` ADD INDEX `value_mm` LOCAL USING min_max ON(`value`); )"); runDMLQuery(R"( $data1 = ListMap(ListFromRange(1, 1500001), ($x) -> { RETURN AsStruct($x AS item); }); - UPSERT INTO `/Root/minmax_test_applied_applied` (`key`, `value`) - SELECT CAST(item AS Int32) AS `key`, "Value_" || CAST(item+1 AS String) AS `value` FROM AS_TABLE($data1); + UPSERT INTO `/Root/olap_indexes_applied_test` (`key`, `value`, `value_bloom`) SELECT + CAST(item AS Int32) AS `key`, + "Value_" || CAST(item AS String) AS `value`, + "Value_" || CAST(item AS String) AS `value_bloom`, + FROM AS_TABLE($data1); )"); runDMLQuery(R"( $data2 = ListMap(ListFromRange(1, 1500001), ($x) -> { RETURN AsStruct($x AS item); }); - UPSERT INTO `/Root/minmax_test_applied_applied` (`key`, `value`) - SELECT CAST(item AS Int32) AS `key`, "Value_" || CAST(item AS String) AS `value` FROM AS_TABLE($data2); + UPSERT INTO `/Root/olap_indexes_applied_test` (`key`, `value`, `value_bloom`) SELECT + CAST(item AS Int32) AS `key`, + "Value_" || CAST(item+1 AS String) AS `value`, + "Value_" || CAST(item+1 AS String) AS `value_bloom`, + FROM AS_TABLE($data2); )"); + csController->WaitCompactions(TDuration::Seconds(5)); struct TQueryResult { ui64 CountResult; - bool MinMaxIndexUsed; + bool LocalIndexUsed; }; auto runQuery = [&](TString text) -> TQueryResult { @@ -782,69 +795,75 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { return TQueryResult { .CountResult = NYT::NodeFromYsonString(ysonArrayWithOneInteger).AsList()[0].AsList()[0].AsUint64(), - .MinMaxIndexUsed = skippedAndApprovedBeforeQuery < skippedAndApprovedAfterQuery + .LocalIndexUsed = skippedAndApprovedBeforeQuery < skippedAndApprovedAfterQuery }; }; TQueryResult resLess = runQuery(R"( - SELECT COUNT(*) FROM `/Root/minmax_test_applied_applied` WHERE `value` < "Value_500000"; + SELECT COUNT(*) FROM `/Root/olap_indexes_applied_test` WHERE `value` < "Value_500000"; )"); UNIT_ASSERT_VALUES_EQUAL_C(resLess.CountResult, 944450, "incorrect result for query with '<' filter over min_max-indexed column"); - UNIT_ASSERT_C(resLess.MinMaxIndexUsed, "query with '<' filter over min_max-indexed column doesn't use min_max index"); + UNIT_ASSERT_C(resLess.LocalIndexUsed, "query with '<' filter over min_max-indexed column doesn't use min_max index"); TQueryResult resGreater = runQuery(R"( - SELECT COUNT(*) FROM `/Root/minmax_test_applied_applied` WHERE `value` > "Value_500000"; + SELECT COUNT(*) FROM `/Root/olap_indexes_applied_test` WHERE `value` > "Value_500000"; )"); UNIT_ASSERT_VALUES_EQUAL_C(resGreater.CountResult, 555549, "incorrect result for query with '>' filter over min_max-indexed column"); - UNIT_ASSERT_C(resGreater.MinMaxIndexUsed, "query with '>' filter over min_max-indexed column doesn't use min_max index"); + UNIT_ASSERT_C(resGreater.LocalIndexUsed, "query with '>' filter over min_max-indexed column doesn't use min_max index"); TQueryResult resLessOrEqual = runQuery(R"( - SELECT COUNT(*) FROM `/Root/minmax_test_applied_applied` WHERE `value` <= "Value_500000"; + SELECT COUNT(*) FROM `/Root/olap_indexes_applied_test` WHERE `value` <= "Value_500000"; )"); UNIT_ASSERT_VALUES_EQUAL_C(resLessOrEqual.CountResult, 944451, "incorrect result for query with '<=' filter over min_max-indexed column"); - UNIT_ASSERT_C(resLessOrEqual.MinMaxIndexUsed, "query with '<=' filter over min_max-indexed column doesn't use min_max index"); + UNIT_ASSERT_C(resLessOrEqual.LocalIndexUsed, "query with '<=' filter over min_max-indexed column doesn't use min_max index"); TQueryResult resGreaterOrEqual = runQuery(R"( - SELECT COUNT(*) FROM `/Root/minmax_test_applied_applied` WHERE `value` >= "Value_500000"; + SELECT COUNT(*) FROM `/Root/olap_indexes_applied_test` WHERE `value` >= "Value_500000"; )"); UNIT_ASSERT_VALUES_EQUAL_C(resGreaterOrEqual.CountResult, 555550, "incorrect result for query with '>=' filter over min_max-indexed column"); - UNIT_ASSERT_C(resGreaterOrEqual.MinMaxIndexUsed, "query with '>=' filter over min_max-indexed column doesn't use min_max index"); + UNIT_ASSERT_C(resGreaterOrEqual.LocalIndexUsed, "query with '>=' filter over min_max-indexed column doesn't use min_max index"); TQueryResult resBetween = runQuery(R"( - SELECT COUNT(*) FROM `/Root/minmax_test_applied_applied` WHERE `value` BETWEEN "Value_500000" AND "Value_500001"; + SELECT COUNT(*) FROM `/Root/olap_indexes_applied_test` WHERE `value` BETWEEN "Value_500000" AND "Value_500001"; )"); UNIT_ASSERT_VALUES_EQUAL_C(resBetween.CountResult, 2, "incorrect result for query with 'BETWEEN' filter over min_max-indexed column"); - UNIT_ASSERT_C(resBetween.MinMaxIndexUsed, "query with 'BETWEEN' filter over min_max-indexed column doesn't use min_max index"); + UNIT_ASSERT_C(resBetween.LocalIndexUsed, "query with 'BETWEEN' filter over min_max-indexed column doesn't use min_max index"); TQueryResult resAnd = runQuery(R"( - SELECT COUNT(*) FROM `/Root/minmax_test_applied_applied` WHERE `value` >= "Value_500000" AND `value` <= "Value_500001"; + SELECT COUNT(*) FROM `/Root/olap_indexes_applied_test` WHERE `value` >= "Value_500000" AND `value` <= "Value_500001"; )"); UNIT_ASSERT_VALUES_EQUAL_C(resAnd.CountResult, 2, "incorrect result for query with '`col` >= ... AND `col` <= ...' filter over min_max-indexed column"); - UNIT_ASSERT_C(resAnd.MinMaxIndexUsed, "query with '`col` >= ... AND `col` <= ...' filter over min_max-indexed column doesn't use min_max index"); + UNIT_ASSERT_C(resAnd.LocalIndexUsed, "query with '`col` >= ... AND `col` <= ...' filter over min_max-indexed column doesn't use min_max index"); TQueryResult resOr = runQuery(R"( - SELECT COUNT(*) FROM `/Root/minmax_test_applied_applied` WHERE `value` >= "Value_500000" OR `value` <= "Value_500001"; + SELECT COUNT(*) FROM `/Root/olap_indexes_applied_test` WHERE `value` >= "Value_500000" OR `value` <= "Value_500001"; )"); UNIT_ASSERT_VALUES_EQUAL_C(resOr.CountResult, 1500000, "incorrect result for query with '`col` >= ... OR `col` <= ...' filter over min_max-indexed column"); - UNIT_ASSERT_C(resOr.MinMaxIndexUsed, "query with '`col` >= ... OR `col` <= ...' filter over min_max-indexed column doesn't use min_max index"); + UNIT_ASSERT_C(resOr.LocalIndexUsed, "query with '`col` >= ... OR `col` <= ...' filter over min_max-indexed column doesn't use min_max index"); TQueryResult resNeq = runQuery(R"( - SELECT COUNT(*) FROM `/Root/minmax_test_applied_applied` WHERE `value` != "Value_500000"; + SELECT COUNT(*) FROM `/Root/olap_indexes_applied_test` WHERE `value` != "Value_500000"; )"); UNIT_ASSERT_VALUES_EQUAL_C(resNeq.CountResult, 1499999, "incorrect result for query with '!=' filter over min_max-indexed column"); - UNIT_ASSERT_C(!resNeq.MinMaxIndexUsed, "query with '!=' filter over min_max-indexed column use min_max index, but it shouldn't"); + UNIT_ASSERT_C(!resNeq.LocalIndexUsed, "query with '!=' filter over min_max-indexed column use min_max index, but it shouldn't"); TQueryResult resIsNotNull = runQuery(R"( - SELECT COUNT(*) FROM `/Root/minmax_test_applied_applied` WHERE `value` IS NOT NULL; + SELECT COUNT(*) FROM `/Root/olap_indexes_applied_test` WHERE `value` IS NOT NULL; )"); UNIT_ASSERT_VALUES_EQUAL_C(resIsNotNull.CountResult, 1500000, "incorrect result for query with 'IS NOT NULL' filter over min_max-indexed column"); - UNIT_ASSERT_C(!resIsNotNull.MinMaxIndexUsed, "query with 'IS NOT NULL' filter over min_max-indexed column use min_max index, but it shouldn't(will use in future, see https://github.com/ydb-platform/ydb/issues/38574)"); + UNIT_ASSERT_C(!resIsNotNull.LocalIndexUsed, "query with 'IS NOT NULL' filter over min_max-indexed column use min_max index, but it shouldn't(will use in future, see https://github.com/ydb-platform/ydb/issues/38574)"); TQueryResult resDistinctFromNull = runQuery(R"( - SELECT COUNT(*) FROM `/Root/minmax_test_applied_applied` WHERE `value` IS DISTINCT FROM NULL; + SELECT COUNT(*) FROM `/Root/olap_indexes_applied_test` WHERE `value` IS DISTINCT FROM NULL; )"); UNIT_ASSERT_VALUES_EQUAL_C(resDistinctFromNull.CountResult, 1500000, "incorrect result for query with 'IS DISCTINCT FROM NULL' filter over min_max-indexed column"); - UNIT_ASSERT_C(!resDistinctFromNull.MinMaxIndexUsed, "query with 'IS DISCTINCT FROM NULL' filter over min_max-indexed column use min_max index, but it shouldn't(will use in future, see https://github.com/ydb-platform/ydb/issues/38574)"); + UNIT_ASSERT_C(!resDistinctFromNull.LocalIndexUsed, "query with 'IS DISCTINCT FROM NULL' filter over min_max-indexed column use min_max index, but it shouldn't(will use in future, see https://github.com/ydb-platform/ydb/issues/38574)"); + + TQueryResult resBloom = runQuery(R"( + SELECT COUNT(*) FROM `/Root/olap_indexes_applied_test` WHERE `value_bloom` LIKE '%alue?_50000%' ESCAPE '?'; + )"); + UNIT_ASSERT_VALUES_EQUAL_C(resBloom.CountResult, 11, "incorrect result for query with 'LIKE' filter over bloom_ngram_filter-indexed column"); + UNIT_ASSERT_C(resBloom.LocalIndexUsed, "query with 'LIKE' filter over bloom_ngram_filter-indexed column doesn't use bloom_ngram_filter index)"); } Y_UNIT_TEST(MinMaxIndexStoredInBSForStringsAndInLocalDBOtherwise, EUseQueryService, ELocalIndexAsSchemeObject) { @@ -1155,6 +1174,113 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { ExecQuery(kikimr, UseQueryService, "ALTER TABLE `/Root/olapTable` DROP INDEX idx_ngram;"); } + // Reproduction of https://github.com/ydb-platform/ydb/issues/45739: + // adding a bloom ngram index without a WITH clause must fall back to defaults + // instead of failing with an internal error. + Y_UNIT_TEST(AddLocalBloomNgramIndexWithoutWithClause, EUseQueryService, ELocalIndexAsSchemeObject) { + const bool UseQueryService = (Arg<0>() == EUseQueryService::QueryService); + const bool LocalIndexAsSchemeObject = (Arg<1>() == ELocalIndexAsSchemeObject::SchemeObjectEnabled); + TKikimrSettings settings{}; + settings.AppConfig.MutableFeatureFlags()->SetEnableLocalIndexAsSchemeObject(LocalIndexAsSchemeObject); + TKikimrRunner kikimr(settings); + + ExecQuery(kikimr, UseQueryService, R"( + CREATE TABLE `/Root/olapBloomWithoutWithClause` ( + `key` Int32 NOT NULL, + `value1` String NOT NULL, + `value2` String NOT NULL, + INDEX idx_ngram1 LOCAL USING bloom_ngram_filter ON (`value1`), + PRIMARY KEY (`key`) + ) + PARTITION BY HASH (`key`) + WITH (STORE = COLUMN); + )"); + + ExecQuery(kikimr, UseQueryService, R"( + --!syntax_v1 + ALTER TABLE `/Root/olapBloomWithoutWithClause` + ADD INDEX idx_ngram2 LOCAL USING bloom_ngram_filter + ON (`value2`); + )"); + } + + // Reproduction of https://github.com/ydb-platform/ydb/issues/45738: + // a bloom ngram index must be allowed on a String column (documented as supported), + // not rejected as an inappropriate column type. + Y_UNIT_TEST(AddLocalBloomNgramIndexOnStringColumn, EUseQueryService, ELocalIndexAsSchemeObject) { + const bool UseQueryService = (Arg<0>() == EUseQueryService::QueryService); + const bool LocalIndexAsSchemeObject = (Arg<1>() == ELocalIndexAsSchemeObject::SchemeObjectEnabled); + auto settings = MakeLocalIndexOnInsertTestSettings(); + settings.AppConfig.MutableFeatureFlags()->SetEnableLocalIndexAsSchemeObject(LocalIndexAsSchemeObject); + settings.AppConfig.MutableColumnShardConfig()->SetReaderClassName("SIMPLE"); + TKikimrRunner kikimr(settings); + + auto csController = NYDBTest::TControllers::RegisterCSControllerGuard<NYDBTest::NColumnShard::TController>(); + csController->SetOverridePeriodicWakeupActivationPeriod(TDuration::Seconds(1)); + csController->SetOverrideLagForCompactionBeforeTierings(TDuration::Seconds(1)); + csController->SetOverrideMemoryLimitForPortionReading(1e+10); + csController->SetOverrideBlobSplitSettings(NOlap::NSplitter::TSplitSettings()); + + ExecQuery(kikimr, UseQueryService, R"( + CREATE TABLE `/Root/olapStringNgram` ( + `key` Int32 NOT NULL, + `value` String NOT NULL, + PRIMARY KEY (`key`) + ) + PARTITION BY HASH (`key`) + WITH (STORE = COLUMN); + )"); + + ExecQuery(kikimr, UseQueryService, R"( + ALTER TABLE `/Root/olapStringNgram` + ADD INDEX idx_ngram LOCAL USING bloom_ngram_filter + ON (`value`) + WITH (ngram_size = 3, false_positive_probability = 0.01, case_sensitive = true); + )"); + + // Data (insert) goes through the query client; index probing uses scan queries, which push + // the LIKE filter down to the shard where the ngram index checker runs. + auto runDML = [&](const TString& query) { + auto result = kikimr.GetQueryClient().ExecuteQuery(query, NYdb::NQuery::TTxControl::NoTx()).GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + }; + auto runScan = [&](const TString& query) -> TString { + auto it = kikimr.GetTableClient().StreamExecuteScanQuery(query).GetValueSync(); + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + return StreamResultToYson(it); + }; + + // Two overlapping writes so each shard accumulates several portions and compaction merges + // them into ngram-indexed portions the reader can skip over. + runDML(R"( + $rows = ListMap(ListFromRange(1, 1500001), ($x) -> { RETURN AsStruct($x AS item); }); + UPSERT INTO `/Root/olapStringNgram` (`key`, `value`) + SELECT CAST(item AS Int32) AS `key`, "value_" || CAST(item + 1 AS String) AS `value` FROM AS_TABLE($rows); + )"); + runDML(R"( + $rows = ListMap(ListFromRange(1, 1500001), ($x) -> { RETURN AsStruct($x AS item); }); + UPSERT INTO `/Root/olapStringNgram` (`key`, `value`) + SELECT CAST(item AS Int32) AS `key`, "value_" || CAST(item AS String) AS `value` FROM AS_TABLE($rows); + )"); + csController->WaitCompactions(TDuration::Seconds(5)); + + // Correctness: the LIKE select returns the expected row. + CompareYson(R"([["value_1234567"]])", + runScan(R"(SELECT `value` FROM `/Root/olapStringNgram` WHERE `value` LIKE "%value_1234567%";)"), + "unexpected LIKE result over ngram-indexed column"); + + // The ngram index must be consulted for a LIKE filter: a substring absent from every row + // must let the index skip the indexed portions instead of scanning them. + const ui64 skipBefore = csController->GetIndexesSkippingOnSelect().Val(); + const ui64 approveBefore = csController->GetIndexesApprovedOnSelect().Val(); + CompareYson(R"([[0u]])", + runScan(R"(SELECT COUNT(*) FROM `/Root/olapStringNgram` WHERE `value` LIKE "%nosuchsubstring%";)"), + "unexpected count for absent-substring LIKE"); + const ui64 consulted = (csController->GetIndexesSkippingOnSelect().Val() - skipBefore) + + (csController->GetIndexesApprovedOnSelect().Val() - approveBefore); + UNIT_ASSERT_C(consulted > 0, "ngram index idx_ngram was not used for the LIKE select"); + } + Y_UNIT_TEST(CreateTableWithLocalBloomFilterIndexAndDropIsCorrect, EUseQueryService, ELocalIndexAsSchemeObject) { const bool UseQueryService = (Arg<0>() == EUseQueryService::QueryService); const bool LocalIndexAsSchemeObject = (Arg<1>() == ELocalIndexAsSchemeObject::SchemeObjectEnabled); diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto index c3c767af2d2..69bb64c6929 100644 --- a/ydb/core/protos/config.proto +++ b/ydb/core/protos/config.proto @@ -549,13 +549,37 @@ message TInterconnectConfig { optional uint32 MaxRdmaRetryBackoffLevel = 61 [default = 8]; // 5s * 2^8 = 1280s, about 21 minutes by default optional bool UseUring = 62 [default = false]; optional bool EnableUringSQPOLL = 63 [default = false]; // only effective when UseUring is set - optional bool EnableInterconnectSessionV2 = 64 [default = false]; - optional bool ChecksumInterconnectSessionV2 = 65 [default = false]; - optional bool EnableSQPOLLv2 = 66 [default = true]; - optional bool EnablePreserializeInV2 = 67 [default = false]; - optional uint32 UringEngineThreadsV2 = 68 [default = 4]; optional bool EnableRdmaSendReceive = 69 [default = false]; + message TV2Config { + optional bool Enable = 1 [default = false]; + optional bool ChecksumEvents = 2 [default = false]; + + // Use io_uring SQPOLL for v2 data-plane rings. If the kernel poller is saturated while shard workers + // still have CPU headroom, set this to false so submissions run on the worker threads instead. + optional bool EnableSQPOLL = 3 [default = true]; + + // If set, this enables event preserialization in IC actor system pool. + optional bool EnablePreserializeEvents = 4 [default = false]; + + // Number of worker threads of ICv2 thread pool (not counting uring internal ones). + optional uint32 UringEngineThreads = 5 [default = 4]; + + // io_uring rings per v2 shard worker (each ring may own a SQPOLL thread). Default 1 preserves legacy + // 1:1 ring/worker topology. Raise this when poller-bound; raise UringEngineThreads when serializer-bound. + optional uint32 UringEngineRingsPerShard = 6 [default = 1]; + + // SQPOLL kernel-thread idle timeout (ms) for v2 rings before the poller sleeps. Only effective when + // EnableSQPOLLv2 is set. Lower values save idle CPU; higher values keep the poller warm for latency. + optional uint32 UringEngineSqThreadIdleMs = 7 [default = 2000]; + + // If enabled, rings of different threads share same kernel thread pools and SQ pollers (effectively creating + // UringEngineRingsPerShard threads instead of UringEngineRingsPerShard * UringEngineThreads). + optional bool ShareRingsAmongThreads = 8 [default = false]; + } + + optional TV2Config V2Config = 70; + // ballast is added to IC handshake frames to ensure correctness of jumbo frames transmission over network optional uint32 HandshakeBallastSize = 14; diff --git a/ydb/core/tx/columnshard/engines/storage/indexes/bloom_ngramm/constructor.cpp b/ydb/core/tx/columnshard/engines/storage/indexes/bloom_ngramm/constructor.cpp index 02e5e901b4c..ad3de52b88e 100644 --- a/ydb/core/tx/columnshard/engines/storage/indexes/bloom_ngramm/constructor.cpp +++ b/ydb/core/tx/columnshard/engines/storage/indexes/bloom_ngramm/constructor.cpp @@ -16,8 +16,9 @@ bool IsSupportedColumnType(const NSchemeShard::TOlapColumnSchema& columnInfo, co const auto extractorProto = dataExtractor.SerializeToProto(); const auto typeId = columnInfo.GetType().GetTypeId(); const bool isUtf8Column = typeId == NScheme::NTypeIds::Utf8; + const bool isStringColumn = typeId == NScheme::NTypeIds::String; const bool isJsonSubColumn = typeId == NScheme::NTypeIds::JsonDocument && extractorProto.HasSubColumn(); - return isUtf8Column || isJsonSubColumn; + return isUtf8Column || isJsonSubColumn || isStringColumn; } } // namespace diff --git a/ydb/core/tx/columnshard/engines/storage/indexes/bloom_ngramm/meta.cpp b/ydb/core/tx/columnshard/engines/storage/indexes/bloom_ngramm/meta.cpp index f789d0d1124..640c06b22d5 100644 --- a/ydb/core/tx/columnshard/engines/storage/indexes/bloom_ngramm/meta.cpp +++ b/ydb/core/tx/columnshard/engines/storage/indexes/bloom_ngramm/meta.cpp @@ -157,7 +157,7 @@ public: template <class TFiller> void FillNGrammHashes(const ui32 nGrammSize, const std::shared_ptr<arrow::Array>& array, TFiller& fillData) { - AFL_VERIFY(array->type_id() == arrow::utf8()->id())("id", array->type()->ToString()); + AFL_VERIFY(array->type_id() == arrow::utf8()->id() || array->type_id() == arrow::binary()->id())("id", array->type()->ToString()); NArrow::SwitchType(array->type_id(), [&](const auto& type) { using TWrap = std::decay_t<decltype(type)>; using T = typename TWrap::T; diff --git a/ydb/core/tx/schemeshard/schemeshard_info_types.h b/ydb/core/tx/schemeshard/schemeshard_info_types.h index a8359d2254b..98563f8dcc4 100644 --- a/ydb/core/tx/schemeshard/schemeshard_info_types.h +++ b/ydb/core/tx/schemeshard/schemeshard_info_types.h @@ -3172,6 +3172,10 @@ struct TTableIndexInfo : public TSimpleRefCount<TTableIndexInfo> { return std::visit([]<typename T>(const T& v) { if constexpr (std::is_same_v<std::monostate, T>) { return TString{}; + } else if constexpr (std::is_same_v<NKikimrSchemeOp::TBloomNGrammFilter, T>) { + TString str; + Y_ENSURE(v.SerializeToString(&str)); + return str; } else { TString str{v.SerializeAsString()}; Y_ENSURE(!str.empty()); diff --git a/ydb/docs/ru/core/concepts/query_execution/index.md b/ydb/docs/ru/core/concepts/query_execution/index.md index d9613c4466c..da166f2bb57 100644 --- a/ydb/docs/ru/core/concepts/query_execution/index.md +++ b/ydb/docs/ru/core/concepts/query_execution/index.md @@ -22,7 +22,7 @@ ## Запросы к топикам {#streaming} -Запросы к [топикам](../datamodel/topic.md) позволяют обрабатывать неограниченные потоки данных. Поскольку поток данных бесконечен, такой запрос не завершается после получения результата, а работает до явной отмены. Есть два варианта таких запросов. +Запросы к [топикам](../datamodel/topic.md) позволяют обрабатывать данные из очередей сообщений. Есть два варианта таких запросов. ### Потоковые запросы @@ -32,6 +32,6 @@ ### Запросы к топикам в табличном режиме -Прочитать данные из топика можно и обычным `SELECT`, указав `STREAMING = "TRUE"` в блоке `WITH`. Без ограничения `LIMIT` такой запрос работает бесконечно, возвращая результат клиенту по мере поступления сообщений. В отличие от потоковых запросов, он не создаёт постоянного объекта схемы и не восстанавливается после сбоев, поэтому предназначен в первую очередь для отладки и проверки данных в топике. +Прочитать данные из топика можно и обычным [`SELECT`](../../yql/reference/syntax/select/topics.md). В отличие от потоковых запросов, он не создаёт постоянного объекта схемы и не восстанавливается после сбоев, поэтому предназначен в первую очередь для отладки и проверки данных в топике. -Подробнее см. в разделе [{#T}](../../yql/reference/syntax/select/streaming.md). +Подробнее см. в разделе [{#T}](topics.md). diff --git a/ydb/docs/ru/core/concepts/query_execution/topics.md b/ydb/docs/ru/core/concepts/query_execution/topics.md index 26a77ea0a8e..3672673f790 100644 --- a/ydb/docs/ru/core/concepts/query_execution/topics.md +++ b/ydb/docs/ru/core/concepts/query_execution/topics.md @@ -37,11 +37,11 @@ SELECT * FROM ext_source.input_topic WITH (FORMAT = json_each_row, SCHEMA = (... ## Чтение из топика {#topic-read} -Чтение из топика можно выполнять в [табличном](#table-read) и [потоковом](#streaming-read) режимах (не путать с [потоковыми запросами](../streaming-query/streaming-query.md)). +Чтение из топика может ограничиваться только текущими данными топика или работать с ожиданием новых записанных сообщений. -### Табличное чтение {#table-read} +### Чтение текущих данных {#table-read} -В табличном режиме чтение выполняется от первого до последнего смещения, хранящегося в топике на момент запуска запроса. Если в топик продолжается запись данных, то запрос остановится после достижения последнего смещения, известного на момент запуска. Указание фильтров по [Служебным полям](#system-metadata) ускоряет чтение, так как чтение происходит только по указанным диапазонам. +В данном режиме чтение выполняется от первого до последнего смещения, хранящегося в топике на момент запуска запроса. Если в топик продолжается запись данных, то запрос остановится после достижения последнего смещения, известного на момент запуска. Указание фильтров по [Служебным полям](#system-metadata) ускоряет чтение, так как чтение происходит только по указанным диапазонам. ```yql SELECT @@ -51,9 +51,9 @@ FROM LIMIT 10; ``` -### Потоковое чтение {#streaming-read} +### Чтение с ожиданием данных {#streaming-read} -Для чтения новых сообщений используйте опцию `WITH (STREAMING = "TRUE")` — подробнее в разделе [Потоковое чтение данных из топика](../../yql/reference/syntax/select/streaming.md). Чтение начинается с текущего момента и продолжается до тех пор, пока не будет прочитано заданное в выражении `LIMIT` количество сообщений. Параметр `LIMIT` обязателен — без него запрос не завершится, так как будет ожидать новые сообщения бесконечно. +Для ожидания новых сообщений используйте опцию `WITH (STREAMING = "TRUE")`. Чтение начинается с текущего момента и продолжается до тех пор, пока не будет прочитано заданное в выражении `LIMIT` количество сообщений. Параметр `LIMIT` обязателен — без него запрос не завершится, так как будет ожидать новые сообщения бесконечно. ```yql diff --git a/ydb/docs/ru/core/recipes/streaming_queries/debug-read.md b/ydb/docs/ru/core/recipes/streaming_queries/debug-read.md index 46c9f856d15..622853532d4 100644 --- a/ydb/docs/ru/core/recipes/streaming_queries/debug-read.md +++ b/ydb/docs/ru/core/recipes/streaming_queries/debug-read.md @@ -13,5 +13,5 @@ ## См. также * [{#T}](../../concepts/query_execution/topics.md) -* [{#T}](../../yql/reference/syntax/select/streaming.md) +* [{#T}](../../yql/reference/syntax/select/topics.md) * [{#T}](../../dev/streaming-query/streaming-query-formats.md) diff --git a/ydb/docs/ru/core/reference/ydb-cli/operation-list.md b/ydb/docs/ru/core/reference/ydb-cli/operation-list.md index 6fc668a8c38..4975d9e9c64 100644 --- a/ydb/docs/ru/core/reference/ydb-cli/operation-list.md +++ b/ydb/docs/ru/core/reference/ydb-cli/operation-list.md @@ -20,7 +20,8 @@ * `import/nfs` — операции импорта с NFS; * `scriptexec` — операции выполнения скриптов; * `incbackup` — операции инкрементального резервного копирования; - * `restore` — операции восстановления из резервной копии. + * `restore` — операции восстановления из резервной копии; + * `setnotnull` — операции выставления ограничения `NOT NULL`. Посмотрите описание команды получения списка фоновых операций: diff --git a/ydb/docs/ru/core/yql/reference/syntax/_includes/column_option_list.md b/ydb/docs/ru/core/yql/reference/syntax/_includes/column_option_list.md index d31065e389c..1764b77b9ed 100644 --- a/ydb/docs/ru/core/yql/reference/syntax/_includes/column_option_list.md +++ b/ydb/docs/ru/core/yql/reference/syntax/_includes/column_option_list.md @@ -19,11 +19,11 @@ ### NULL -Данная колонка может содержать значения `NULL` (по умолчанию). +Опция колонки, которое разрешает запись `NULL` в качестве значений колонки. Это является поведением по умолчанию. ### NOT NULL -Данная колонка не принимает значения `NULL`. +Ограничение колонки, которое запрещает запись `NULL` в качестве значений колонки. По умолчанию это ограничение отсутствует. ### COMPRESSION([algorithm=<algorithm_name>[, level=<value>]]) diff --git a/ydb/docs/ru/core/yql/reference/syntax/_includes/column_option_list_alter.md b/ydb/docs/ru/core/yql/reference/syntax/_includes/column_option_list_alter.md index 71aff46669e..c04c18f81a8 100644 --- a/ydb/docs/ru/core/yql/reference/syntax/_includes/column_option_list_alter.md +++ b/ydb/docs/ru/core/yql/reference/syntax/_includes/column_option_list_alter.md @@ -17,13 +17,12 @@ ### NOT NULL -{% note warning %} - -На текущий момент поддерживается только операция `DROP NOT NULL`. +Существуют две операции: -{% endnote %} +* `SET NOT NULL` — устанавливает ограничение `NOT NULL` для колонки. +* `DROP NOT NULL` — снимает ограничение `NOT NULL` с колонки, снова разрешая значения `NULL`. -Снимает ограничение `NOT NULL` с колонки, снова разрешая значения `NULL`. +Подробнее о работе этих операций можно прочитать [здесь](../alter_table/not_null.md). ### COMPRESSION([algorithm=<algorithm_name>[, level=<value>]]) diff --git a/ydb/docs/ru/core/yql/reference/syntax/alter_table/columns.md b/ydb/docs/ru/core/yql/reference/syntax/alter_table/columns.md index 7024279300b..8390ee072d7 100644 --- a/ydb/docs/ru/core/yql/reference/syntax/alter_table/columns.md +++ b/ydb/docs/ru/core/yql/reference/syntax/alter_table/columns.md @@ -46,7 +46,7 @@ ALTER TABLE episodes ADD COLUMN rate Double (DEFAULT 5.0, NOT NULL); -- альт Изменяет свойства существующей колонки в указанной таблице. Изменение свойства происходит без пересоздания колонки. Некоторые свойства применяются только к свежим записанным данным или в процессе компакшена (детали можно найти в описании конкретного свойства) ```yql -ALTER TABLE table_name ALTER COLUMN column_name SET [FAMILY <family_name>] [DEFAULT <default_value>] [COMPRESSION([algorithm=<algorithm_name>[, level=<value>]])] [ENCODING([OFF|DICT])]; +ALTER TABLE table_name ALTER COLUMN column_name SET [FAMILY <family_name>] [NOT NULL] [DEFAULT <default_value>] [COMPRESSION([algorithm=<algorithm_name>[, level=<value>]])] [ENCODING([OFF|DICT])]; ALTER TABLE table_name ALTER COLUMN column_name DROP [FAMILY] [NOT NULL] [DEFAULT] [COMPRESSION] [ENCODING]; ``` diff --git a/ydb/docs/ru/core/yql/reference/syntax/alter_table/index.md b/ydb/docs/ru/core/yql/reference/syntax/alter_table/index.md index 217b12ebf98..f5fa7d1a512 100644 --- a/ydb/docs/ru/core/yql/reference/syntax/alter_table/index.md +++ b/ydb/docs/ru/core/yql/reference/syntax/alter_table/index.md @@ -17,4 +17,5 @@ ALTER TABLE table_name action1, action2, ..., actionN; {% if backend_name == "YDB" and oss == true %} * Изменение [дополнительных параметров таблиц](set.md). {% endif %} -* [Запуск принудительного компакшена](compact.md).
\ No newline at end of file +* [Запуск принудительного компакшена](compact.md). +* [Выставление и сброс ограничения NOT NULL](not_null.md).
\ No newline at end of file diff --git a/ydb/docs/ru/core/yql/reference/syntax/alter_table/not_null.md b/ydb/docs/ru/core/yql/reference/syntax/alter_table/not_null.md new file mode 100644 index 00000000000..c9227e16933 --- /dev/null +++ b/ydb/docs/ru/core/yql/reference/syntax/alter_table/not_null.md @@ -0,0 +1,43 @@ +# Выставление и сброс ограничения `NOT NULL` + +Ограничение целостности данных на уровне колонки, которое запрещает запись `NULL` в качестве значений. Это ограничение гарантирует, что колонка всегда содержит действительное значение. + +В YDB операция `SET NOT NULL` выполняется как синхронная SQL-операция, ожидающая применения изменения схемы. При этом создаётся фоновая операция для проверки таблицы на наличие `NULL`-значений в существующих данных. + +## Выставление `NOT NULL` + +`SET NOT NULL` устанавливает ограничение `NOT NULL` для указанной колонки. + +Например, следующий запрос установит ограничение `NOT NULL` для колонки `column_name` в таблице `table_name`: + +```yql +ALTER TABLE table_name ALTER COLUMN column_name SET NOT NULL; +``` + +Примечания: + +* `SET NOT NULL` поддерживается только для [строковых таблиц](../../../../concepts/datamodel/table.md#row-oriented-tables). +* `SET NOT NULL` может занять длительное время: перед установкой ограничения YDB проверяет таблицу на наличие `NULL`-значений для существующих данных в указанной колонке. +* SQL-операция выполняется синхронно и ожидает завершения. При этом создаётся фоновая операция для наблюдаемости. +* За ходом выполнения операций можно следить с помощью [CLI-команды](../../../../reference/ydb-cli/operation-list.md) `ydb operation list setnotnull`. Также доступны команды, позволяющие [получить статус конкретной операции](../../../../reference/ydb-cli/operation-get.md), [отменить операцию](../../../../reference/ydb-cli/operation-cancel.md) или [удалить запись о завершённой операции](../../../../reference/ydb-cli/operation-forget.md). +* После запуска операции `SET NOT NULL` и до её завершения в указанную колонку нельзя записывать `NULL`-значения. При попытке записать такие значения, вы получите ошибку вида ``Can't set NULL or optional value to column: <column>. `SET NOT NULL` operation is currently in progress for this column``. +* Если валидация не пройдена, операция `SET NOT NULL` завершится с ошибкой `Validation failed for SET NOT NULL on table ...: one or more columns contain NULL values`. + +## Сброс `NOT NULL` + +`DROP NOT NULL` снимает ограничение `NOT NULL` с указанной колонки. + +Например, следующий запрос снимет ограничение `NOT NULL` с колонки `column_name` в таблице `table_name`: + +```yql +ALTER TABLE table_name ALTER COLUMN column_name DROP NOT NULL; +``` + +Примечание: + +* `DROP NOT NULL` поддерживается только для [строковых таблиц](../../../../concepts/datamodel/table.md#row-oriented-tables). + +## См. также + +* [ALTER COLUMN](columns.md) + diff --git a/ydb/docs/ru/core/yql/reference/syntax/alter_table/toc_i.yaml b/ydb/docs/ru/core/yql/reference/syntax/alter_table/toc_i.yaml index ddf6ca251af..a91b2276111 100644 --- a/ydb/docs/ru/core/yql/reference/syntax/alter_table/toc_i.yaml +++ b/ydb/docs/ru/core/yql/reference/syntax/alter_table/toc_i.yaml @@ -1,9 +1,12 @@ items: - { name: Обзор, href: index.md } - { name: INDEX, href: indexes.md, when: feature_secondary_index } -- { name: COLUMN, href: columns.md } +- name: COLUMN + href: columns.md + items: + - { name: FAMILY, href: family.md, when: backend_name == "YDB" } + - { name: NOT NULL, href: not_null.md, when: backend_name == "YDB" } - { name: SET, href: set.md, when: backend_name == "YDB" } - { name: CHANGEFEED, href: changefeed.md, when: feature_changefeed and backend_name == "YDB" } - { name: RENAME, href: rename.md, when: feature_map_tables } -- { name: FAMILY, href: family.md, when: backend_name == "YDB" } -- { name: COMPACT, href: compact.md, when: backend_name == "YDB" }
\ No newline at end of file +- { name: COMPACT, href: compact.md, when: backend_name == "YDB" } diff --git a/ydb/docs/ru/core/yql/reference/syntax/select/toc_i.yaml b/ydb/docs/ru/core/yql/reference/syntax/select/toc_i.yaml index d198e718af7..f35b392aa9d 100644 --- a/ydb/docs/ru/core/yql/reference/syntax/select/toc_i.yaml +++ b/ydb/docs/ru/core/yql/reference/syntax/select/toc_i.yaml @@ -31,4 +31,4 @@ items: - { name: SAMPLE, href: sample.md, when: feature_tablesample } - { name: TABLESAMPLE, href: sample.md, when: feature_tablesample } - { name: MATCH_RECOGNIZE, href: match_recognize.md, when: feature_match_recogznize } -- { name: STREAMING, href: streaming.md } +- { name: FROM Topic, href: topics.md } diff --git a/ydb/docs/ru/core/yql/reference/syntax/select/streaming.md b/ydb/docs/ru/core/yql/reference/syntax/select/topics.md index 179939d1d12..cc87d2fa5e9 100644 --- a/ydb/docs/ru/core/yql/reference/syntax/select/streaming.md +++ b/ydb/docs/ru/core/yql/reference/syntax/select/topics.md @@ -1,6 +1,6 @@ -# Потоковое чтение данных из топика +# Чтение данных из топика -Можно выполнять чтение данных из [топика](../../../../concepts/datamodel/topic.md) обычным `SELECT` без создания [потокового запроса](../../../../concepts/streaming-query/streaming-query.md). Для этого необходимо указать `STREAMING = "TRUE"` в блоке `WITH` и задать ограничение на количество выходных строк через `LIMIT`, иначе запрос не завершится. +Можно выполнять чтение данных из [топика](../../../../concepts/datamodel/topic.md) обычным `SELECT` без создания [потокового запроса](../../../../concepts/streaming-query/streaming-query.md). {% note warning %} @@ -19,7 +19,9 @@ {% endnote %} -## Пример +## Примеры + +### Чтение текущих данных ```yql SELECT @@ -30,14 +32,32 @@ WITH ( FORMAT = raw, SCHEMA = ( Data String + ) +) +``` + +### Чтение с ожиданием новых данных + +```yql +SELECT + Data +FROM + input_topic -- или внешний топик ext_source.input_topic +WITH ( + FORMAT = raw, + SCHEMA = ( + Data String ), STREAMING = "TRUE" ) LIMIT 1 ``` +Подробнее — [чтение из топика](../../../../concepts/query_execution/topics.md#topic-read). + + ## См. также -* [{#T}](../../../../concepts/query_execution/topics.md#streaming-read) — описание потокового чтения из топика +* [{#T}](../../../../concepts/query_execution/topics.md#topic-read) * [{#T}](../../../../concepts/streaming-query/streaming-query.md) * [{#T}](../create-streaming-query.md) diff --git a/ydb/library/actors/interconnect/events/events.h b/ydb/library/actors/interconnect/events/events.h index 540a5744447..de77e880573 100644 --- a/ydb/library/actors/interconnect/events/events.h +++ b/ydb/library/actors/interconnect/events/events.h @@ -75,6 +75,7 @@ namespace NActors { EvRegisterSession, EvUnregisterSession, EvStop, + EvMigrateDone, //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // nonlocal messages; their indices must be preserved in order to work properly while doing rolling update diff --git a/ydb/library/actors/interconnect/interconnect_common.h b/ydb/library/actors/interconnect/interconnect_common.h index efea50b752b..b70d17f2a87 100644 --- a/ydb/library/actors/interconnect/interconnect_common.h +++ b/ydb/library/actors/interconnect/interconnect_common.h @@ -86,15 +86,30 @@ namespace NActors { bool CollectSubscriptionStackTrace = false; bool UseUring = false; bool EnableUringSQPOLL = false; // only effective when UseUring is set - // Enables negotiation and usage of TInterconnectSessionTCPv2 (no session continuation, no encryption). - // v2 is used only when both peers have this enabled and encryption is not in effect. - bool EnableInterconnectSessionV2 = false; - bool ChecksumInterconnectSessionV2 = false; - // Use io_uring SQPOLL mode for the v2 data-plane rings (kernel-side submission polling). - bool EnableSQPOLLv2 = true; - // Preserialize outgoing events on the session mailbox before handing them to the v2 engine (moves - // serialization cost off the engine's shard worker thread). - bool EnablePreserializeInV2 = false; + + struct { + // Enables negotiation and usage of TInterconnectSessionTCPv2 (no session continuation, no encryption). + // v2 is used only when both peers have this enabled and encryption is not in effect. + bool Enable = false; + bool ChecksumEvents = false; + // Use io_uring SQPOLL mode for the v2 data-plane rings (kernel-side submission polling). + // When the kernel poller is pegged (~100% CPU) while shard workers still have headroom, disable this + // so io_uring_submit/enter runs on the worker thread instead. + bool EnableSQPOLL = true; + // Preserialize outgoing events on the session mailbox before handing them to the v2 engine (moves + // serialization cost off the engine's shard worker thread). + bool EnablePreserializeEvents = false; + // Number of worker threads. + ui32 UringEngineThreads = 4; + // io_uring rings per v2 shard worker (default 1). Each ring may have its own SQPOLL thread, so this + // scales kernel submission-polling independently of the number of serialization workers. + ui32 UringEngineRingsPerShard = 1; + // SQPOLL kernel-thread idle window (ms) for v2 rings before it sleeps. Only used when EnableSQPOLLv2 + // is on. Matches TUringContext::SqThreadIdleMs by default. + ui32 UringEngineSqThreadIdleMs = 2000; + // Enable kernel threads sharing among different worker threads. + bool ShareRingsAmongThreads = false; + } V2; }; struct TWhiteboardSessionStatus { diff --git a/ydb/library/actors/interconnect/interconnect_handshake.cpp b/ydb/library/actors/interconnect/interconnect_handshake.cpp index 4d37e0c2eef..5df9b775e35 100644 --- a/ydb/library/actors/interconnect/interconnect_handshake.cpp +++ b/ydb/library/actors/interconnect/interconnect_handshake.cpp @@ -1033,7 +1033,7 @@ namespace NActors { // v2 session is incompatible with encryption and needs the io_uring data plane; only // request it when encryption is disabled locally and io_uring is available (buffer rings // are used when present, with a fallback to ordinary buffers on older kernels) - request.SetRequestSessionV2(Common->Settings.EnableInterconnectSessionV2 && + request.SetRequestSessionV2(Common->Settings.V2.Enable && Common->Settings.EncryptionMode == EEncryptionMode::DISABLED && TUringContext::IsAvailable()); request.SetHandshakeId(*HandshakeId); @@ -1420,7 +1420,7 @@ namespace NActors { // v2 session is used only when both peers enabled it, encryption is not in effect, and // this side has the io_uring data plane available Params.UseSessionV2 = request.GetRequestSessionV2() && - Common->Settings.EnableInterconnectSessionV2 && !Params.Encryption && + Common->Settings.V2.Enable && !Params.Encryption && TUringContext::IsAvailable(); if (Params.UseExternalDataChannel) { diff --git a/ydb/library/actors/interconnect/interconnect_tcp_session_v2.cpp b/ydb/library/actors/interconnect/interconnect_tcp_session_v2.cpp index b11b1a36866..a0f5ee860ae 100644 --- a/ydb/library/actors/interconnect/interconnect_tcp_session_v2.cpp +++ b/ydb/library/actors/interconnect/interconnect_tcp_session_v2.cpp @@ -79,7 +79,7 @@ namespace NActors { }; SetNonBlock(*Socket, false); EngineHandle = Proxy->Common->UringEngineV2->Register(Socket, SelfId(), - Proxy->Common->Settings.ChecksumInterconnectSessionV2, Params.PeerScopeId, onDisconnectCallback, + Proxy->Common->Settings.V2.ChecksumEvents, Params.PeerScopeId, onDisconnectCallback, SelfId().NodeId() < Proxy->PeerNodeId, ClockSkew, PingRTT); if (!EngineHandle) { LOG_ERROR_IC_SESSION("ICS99", "v2 io_uring engine failed to register the connection"); @@ -153,7 +153,7 @@ namespace NActors { } void TInterconnectSessionTCPv2::EnqueueOutgoing(TAutoPtr<IEventHandle> ev) { - if (Proxy->Common->Settings.EnablePreserializeInV2) { + if (Proxy->Common->Settings.V2.EnablePreserializeEvents) { ev->Preserialize(); } Proxy->Common->UringEngineV2->Send(EngineHandle, std::unique_ptr<IEventHandle>(ev.Release())); diff --git a/ydb/library/actors/interconnect/interconnect_uring_engine.cpp b/ydb/library/actors/interconnect/interconnect_uring_engine.cpp index ed9628cede8..6dc9a8c97fc 100644 --- a/ydb/library/actors/interconnect/interconnect_uring_engine.cpp +++ b/ydb/library/actors/interconnect/interconnect_uring_engine.cpp @@ -1,10 +1,11 @@ #include "interconnect_uring_engine.h" #include "uring_recv_buffer_pool.h" -#include "uring_context.h" // for TUringContext::IsSupported() +#include "uring_context.h" // for TUringContext::IsSupported() / SqThreadIdleMs #include "v2_event_serializer.h" #include "interconnect_direct_session.h" +#include "interconnect_uring_event_queue.h" #include <ydb/library/actors/core/actorsystem.h> #include <ydb/library/actors/core/actor.h> @@ -21,6 +22,7 @@ #include <sys/socket.h> #include <sys/uio.h> #include <sys/timerfd.h> +#include <sys/eventfd.h> #include <cerrno> #include <deque> @@ -42,88 +44,26 @@ namespace NActors { constexpr size_t MaxSpansPerWrite = 64; constexpr size_t SerializeWindowSize = 65536; constexpr size_t MinSerializeWindowSize = 4096; + constexpr ui32 RebalanceTimerMs = 500; // drives MaybeOffload and also issues ping packet + constexpr ui32 OffloadBusyThreshold = 700000; // ppm + constexpr ui32 StealBusyThreshold = 300000; // ppm } - struct TEventPayload { - ui64 Conn; - TIntrusivePtr<IReceiveCallback> Callback; - }; - static_assert(sizeof(TEventPayload) <= sizeof(TActorId)); - - class TIncomingEventQueue { - IEventHandle Stub{0, 0, {}, {}, nullptr, 0}; - std::atomic<IEventHandle*> Head{&Stub}; - std::atomic<IEventHandle*> Tail{&Stub}; - - public: - TIncomingEventQueue() { - Stub.NextLinkPtr.store(0, std::memory_order_relaxed); - } - - bool Push(std::unique_ptr<IEventHandle> ev) { - IEventHandle *last = ev.release(); - last->NextLinkPtr.store(0, std::memory_order_relaxed); - IEventHandle *prev = Head.exchange(last, std::memory_order_acq_rel); - prev->NextLinkPtr.store(reinterpret_cast<uintptr_t>(last), std::memory_order_release); - return prev == &Stub; // if it was the first event - } - - bool IsEmpty() const { - IEventHandle *tail = Tail.load(std::memory_order_relaxed); - IEventHandle *next = reinterpret_cast<IEventHandle*>(tail->NextLinkPtr.load(std::memory_order_acquire)); - IEventHandle *head = Head.load(std::memory_order_acquire); - return tail == &Stub && !next && tail == head; - } - - std::unique_ptr<IEventHandle> Pop() { - for (;;) { - IEventHandle *tail = Tail.load(std::memory_order_relaxed); - IEventHandle *next = reinterpret_cast<IEventHandle*>(tail->NextLinkPtr.load(std::memory_order_acquire)); - IEventHandle *head; - - if (tail == &Stub) { - if (!next) { - if (head = Head.load(std::memory_order_acquire); tail != head) { - continue; - } else { - return nullptr; - } - } - Tail.store(next, std::memory_order_relaxed); - tail = next; - next = reinterpret_cast<IEventHandle*>(tail->NextLinkPtr.load(std::memory_order_acquire)); - } - - if (next) { - Tail.store(next, std::memory_order_relaxed); - return std::unique_ptr<IEventHandle>(tail); - } - - head = Head.load(std::memory_order_acquire); - if (tail != head) { - continue; - } - - Stub.NextLinkPtr.store(0, std::memory_order_relaxed); - IEventHandle *prev = Head.exchange(&Stub, std::memory_order_acq_rel); - prev->NextLinkPtr.store(reinterpret_cast<uintptr_t>(&Stub), std::memory_order_release); - - next = reinterpret_cast<IEventHandle*>(tail->NextLinkPtr.load(std::memory_order_acquire)); - if (next) { - Tail.store(next, std::memory_order_relaxed); - return std::unique_ptr<IEventHandle>(tail); - } - } - } - }; - class TUringEngine final : public IUringEngine { TActorSystem *ActorSystem = nullptr; // bound after construction via SetActorSystem() std::once_flag ActorSystemInitFlag; std::atomic_bool Stopping{false}; + enum class EMigrateState : ui8 { + None = 0, + Draining, + HandedOff, + }; + + // Low 3 bits of the session pointer are used as an io_uring user_data op tag; heap allocation + // alignment of this type is already >= 8 (actually 64 via base/members). struct TRegisteredSession : TEventDeserializer::IEventProcessor { - const ui32 ShardIdx; + std::atomic_uint32_t OwnerShard; const TIntrusivePtr<NInterconnect::TStreamSocket> Socket; const TActorId SessionId; const std::function<void(TDisconnectReason)> OnDisconnectCallback; @@ -141,6 +81,12 @@ namespace NActors { iovec Iov[MaxSpansPerWrite]; size_t IovLen = 0; size_t UnsentBytes = 0; + int ReadPendingRingIdx = -1; + ui32 PreferredRingIdx = 0; + + EMigrateState MigrateState = EMigrateState::None; + ui32 MigrateTargetShard = 0; + ui32 MigrateSourceShard = 0; const std::shared_ptr<std::atomic<int64_t>> ClockSkew; const std::shared_ptr<std::atomic<uint64_t>> PingRTT; @@ -148,11 +94,12 @@ namespace NActors { THashMap<TActorId, TIntrusivePtr<IReceiveCallback>> ReceiveCallbacks; NMonitoring::TDynamicCounters::TCounterPtr EventsReceived; - TRegisteredSession(ui32 shardIdx, TIntrusivePtr<NInterconnect::TStreamSocket> socket, TActorId sessionId, - bool checksumming, TScopeId peerScopeId, std::function<void(TDisconnectReason)> onDisconnectCallback, - TActorSystem *actorSystem, bool sendPings, std::shared_ptr<std::atomic<int64_t>> clockSkew, + TRegisteredSession(ui32 shardIdx, TIntrusivePtr<NInterconnect::TStreamSocket> socket, + TActorId sessionId, bool checksumming, TScopeId peerScopeId, + std::function<void(TDisconnectReason)> onDisconnectCallback, TActorSystem *actorSystem, + bool sendPings, std::shared_ptr<std::atomic<int64_t>> clockSkew, std::shared_ptr<std::atomic<uint64_t>> pingRTT) - : ShardIdx(shardIdx) + : OwnerShard(shardIdx) , Socket(std::move(socket)) , SessionId(sessionId) , OnDisconnectCallback(std::move(onDisconnectCallback)) @@ -244,7 +191,11 @@ namespace NActors { // Advance past exactly the bytes the kernel accepted. A writev can be short (e.g. under // backpressure or on a real network), so drop only fully-sent spans and trim the span that // straddles the boundary; the rest stay queued and are retried by the next writev. - for (size_t remaining = num; remaining && !OutgoingSpans.empty(); OutgoingSpans.pop_front()) { + for (auto& span : OutgoingSpans) { + NSan::CheckMemIsInitialized(span.data(), span.size()); + } + for (size_t remaining = num; remaining; OutgoingSpans.pop_front()) { + Y_DEBUG_ABORT_UNLESS(!OutgoingSpans.empty()); if (TContiguousSpan& front = OutgoingSpans.front(); front.size() <= remaining) { remaining -= front.size(); } else { @@ -338,34 +289,61 @@ namespace NActors { PingValues[2] = pingUs; PingRTT->store(Max(PingValues[0], PingValues[1], PingValues[2])); } + + bool IsMigratable() const { + return MigrateState == EMigrateState::None + && !UnregisterRequested + && !Terminated; + } + }; + + // In-process load signal consumed by rebalancing (not the 15s monitoring scrape path). + struct TShardLoad { + std::atomic_uint64_t Busy{0}; + std::atomic_uint64_t Total{0}; + + ui32 BusyFraction() const { + const ui64 total = Total.load(std::memory_order_relaxed); + if (!total) { + return 0; + } + return Busy.load(std::memory_order_relaxed) * 1'000'000 / total; + } }; class TShard { enum EOperationType { - kOpPipe = 1, + kOpEvent = 1, kOpRead, kOpWrite, kOpTimer, + kOpCancel, }; static const ui64 kOpMask = (1 << 3) - 1; + struct TRingSlot { + io_uring Ring{}; + i64 ItemsToSubmit = 0; + }; + + TUringEngine& Engine; + const ui32 ShardIdx; TIncomingEventQueue IncomingEventQueue; std::thread Worker; - io_uring Ring; - i64 ItemsToSubmit = 0; + std::vector<TRingSlot> Rings; + int EventFd = -1; + ui64 EventFdReadBuffer = 0; + int TimerFd = -1; + ui64 ReadTimerBuffer; std::atomic_bool WaitingForCQ{false}; - int ReadPipe; - int WritePipe; - char ReadPipeBuffer[256]; - - int TimerFd; - char ReadTimerBuffer[256]; + size_t OpShift = 0; struct TSessionHash { size_t operator()(const std::unique_ptr<TRegisteredSession>& p) const { return THash<void*>{}(p.get()); } size_t operator()(const TRegisteredSession *p) const { return THash<void*>{}(p); } + using is_transparent = void; }; struct TSessionEqual { @@ -373,9 +351,12 @@ namespace NActors { bool operator()(const T& x, const T& y) const { return x == y; } bool operator()(const TRegisteredSession *x, const T& y) const { return x == y.get(); } bool operator()(const T& x, const TRegisteredSession *y) const { return x.get() == y; } + using is_transparent = void; }; - THashSet<std::unique_ptr<TRegisteredSession>, TSessionHash, TSessionEqual> Sessions; + std::unordered_set<std::unique_ptr<TRegisteredSession>, TSessionHash, TSessionEqual> Sessions; + // conn -> target shard while OwnerShard still points here during handoff + THashMap<ui64, ui32> MigratingOut; NMonitoring::TDynamicCounters::TCounterPtr SessionsRegistered; NMonitoring::TDynamicCounters::TCounterPtr SessionsUnregistered; @@ -390,11 +371,13 @@ namespace NActors { NMonitoring::TDynamicCounters::TCounterPtr SQEAllocated; NMonitoring::TDynamicCounters::TCounterPtr SubmitCount; NMonitoring::TDynamicCounters::TCounterPtr CQEProcessed; - NMonitoring::TDynamicCounters::TCounterPtr PipeWakeups; + NMonitoring::TDynamicCounters::TCounterPtr EventWakeups; NMonitoring::TDynamicCounters::TCounterPtr PushedAsFirst; NMonitoring::TDynamicCounters::TCounterPtr PushedTotal; NMonitoring::TDynamicCounters::TCounterPtr ReadUnavail; NMonitoring::TDynamicCounters::TCounterPtr WriteUnavail; + NMonitoring::TDynamicCounters::TCounterPtr SessionsMigratedOut; + NMonitoring::TDynamicCounters::TCounterPtr SessionsMigratedIn; NMonitoring::TDynamicCounters::TCounterPtr OtherTotalTime; NMonitoring::TDynamicCounters::TCounterPtr CompleteWaitTotalTime; @@ -420,6 +403,8 @@ namespace NActors { std::vector<ui64> EventToWireTimeVec; + TShardLoad& Load; + private: class TActivityMeasure { TShard& Shard; @@ -449,14 +434,47 @@ namespace NActors { #define ACTIVITY(NAME) if (TActivityMeasure __measure{*this, NAME}; false); else + void InitRing(TRingSlot& slot, bool sqpoll, ui32 sqThreadIdleMs, TRingSlot *shareWith) { + auto tryIt = [&](std::optional<ui32> sqThreadIdleMs) { + io_uring_params params{}; + if (sqpoll) { + params.flags |= IORING_SETUP_SQPOLL; + if (sqThreadIdleMs) { + params.sq_thread_idle = *sqThreadIdleMs; + } + } + if (shareWith) { + params.flags |= IORING_SETUP_ATTACH_WQ; + params.wq_fd = shareWith->Ring.ring_fd; + } + return io_uring_queue_init_params(RingQueueDepth, &slot.Ring, ¶ms) == 0; + }; + if (!tryIt(sqThreadIdleMs) && !tryIt(std::nullopt)) { + Y_ABORT("failed to initialize ring"); + } + } + + void PublishLoadSample(ui64 busyDelta, ui64 totalDelta) { + // Keep a short EWMA-like window by decaying prior samples and adding the latest slice. + constexpr ui64 DecayNum = 3; + constexpr ui64 DecayDen = 4; + const ui64 prevBusy = Load.Busy.load(std::memory_order_relaxed); + const ui64 prevTotal = Load.Total.load(std::memory_order_relaxed); + Load.Busy.store(prevBusy * DecayNum / DecayDen + busyDelta, std::memory_order_relaxed); + Load.Total.store(prevTotal * DecayNum / DecayDen + totalDelta, std::memory_order_relaxed); + } + public: static NMonitoring::IHistogramCollectorPtr TimeCollector() { return NMonitoring::ExponentialHistogram(22, 2, 1000); } - TShard(const NMonitoring::TDynamicCounterPtr& shardCounters, bool sqpoll) + TShard(TUringEngine& engine, ui32 shardIdx, const NMonitoring::TDynamicCounterPtr& shardCounters, bool sqpoll, + ui32 ringsPerShard, ui32 sqThreadIdleMs, TShardLoad& load, TShard *shareRingsWith) #define COUNTER(NAME, DERIV) NAME(shardCounters->GetCounter(#NAME, DERIV)) - : COUNTER(SessionsRegistered, true) + : Engine(engine) + , ShardIdx(shardIdx) + , COUNTER(SessionsRegistered, true) , COUNTER(SessionsUnregistered, true) , COUNTER(EventsSent, true) , COUNTER(EventsReceived, true) @@ -469,11 +487,13 @@ namespace NActors { , COUNTER(SQEAllocated, true) , COUNTER(SubmitCount, true) , COUNTER(CQEProcessed, true) - , COUNTER(PipeWakeups, true) + , COUNTER(EventWakeups, true) , COUNTER(PushedAsFirst, true) , COUNTER(PushedTotal, true) , COUNTER(ReadUnavail, true) , COUNTER(WriteUnavail, true) + , COUNTER(SessionsMigratedOut, true) + , COUNTER(SessionsMigratedIn, true) #define TOTAL_TIME(NAME) NAME(shardCounters->GetCounter("TotalTime/" #NAME, true)) , TOTAL_TIME(OtherTotalTime) , TOTAL_TIME(CompleteWaitTotalTime) @@ -490,57 +510,71 @@ namespace NActors { , SerializeTime(shardCounters->GetNamedHistogram("sensor", "SerializeTime", TimeCollector())) , CompletionsProcessedAtOnce(shardCounters->GetNamedHistogram("sensor", "CompletionsProcessedAtOnce", NMonitoring::ExponentialHistogram(10, 2))) , SubmissionsProcessedAtOnce(shardCounters->GetNamedHistogram("sensor", "SubmissionsProcessedAtOnce", NMonitoring::ExponentialHistogram(12, 2))) + , Load(load) #undef TOTAL_TIME #undef COUNTER { - // initialize ring for this shard - ui32 flags = 0; - if (sqpoll) { - flags |= IORING_SETUP_SQPOLL; - } - if (io_uring_queue_init(RingQueueDepth, &Ring, flags) < 0) { - Y_ABORT("failed to initialize ring"); + EventFd = eventfd(0, 0); + if (EventFd == -1) { + Y_ABORT("eventfd() failed: %s", strerror(errno)); } - // create pipe to kick worker thread when first commands arrive - int fds[2]; - if (pipe(fds) == -1) { - Y_ABORT("pipe() failed: %s", strerror(errno)); + Rings.resize(ringsPerShard); + for (ui32 i = 0; i < Rings.size(); ++i) { + auto& slot = Rings[i]; + InitRing(slot, sqpoll, sqThreadIdleMs, shareRingsWith ? &shareRingsWith->Rings[i] : nullptr); + if (i > 0) { + if (int res = io_uring_register_eventfd(&slot.Ring, EventFd); res < 0) { + Y_ABORT("failed to register eventfd along with ring: %s", strerror(-res)); + } + } } - ReadPipe = fds[0]; - WritePipe = fds[1]; - // set timer TimerFd = timerfd_create(CLOCK_MONOTONIC, 0); Y_ABORT_UNLESS(TimerFd != -1); - // arm timer - itimerspec spec; - memset(&spec, 0, sizeof(spec)); - spec.it_interval.tv_sec = 2; // every two seconds - spec.it_value.tv_sec = 2; // initial expiration - timerfd_settime(TimerFd, 0, &spec, nullptr); - - // start worker thread - Worker = std::thread(std::bind(&TShard::WorkerThread, this)); + // Keep the timer armed for the shard lifetime. Disarming while an io_uring read is + // outstanding would leave a stuck SQE; idle CPU is controlled via sq_thread_idle instead. + itimerspec spec{}; + spec.it_interval.tv_sec = RebalanceTimerMs / 1000; + spec.it_interval.tv_nsec = 1'000'000 * (RebalanceTimerMs % 1000); + spec.it_value = spec.it_interval; + if (timerfd_settime(TimerFd, 0, &spec, nullptr) < 0) { + Y_ABORT("timerfd_settime failed: %s", strerror(errno)); + } } ~TShard() { Stop(); // joins the worker thread, so no completion will be dispatched after this point - io_uring_queue_exit(&Ring); // cancels any still-armed read/write without dispatching them + for (auto& slot : Rings) { + io_uring_queue_exit(&slot.Ring); + } DrainQueue(); // free commands that were enqueued after the worker stopped (teardown races) - close(ReadPipe); - close(WritePipe); + close(EventFd); close(TimerFd); // remaining registered sessions are freed as the Sessions container is destroyed } + void Start() { + Worker = std::thread(std::bind(&TShard::WorkerThread, this)); + } + void Register(std::unique_ptr<TRegisteredSession> session) { ++*SessionsRegistered; SendInternal(reinterpret_cast<ui64>(session.release()), static_cast<ui32>(ENetwork::EvRegisterSession), {}, nullptr); } + void AcceptMigrated(std::unique_ptr<TRegisteredSession> session) { + ++*SessionsMigratedIn; + SendInternal(reinterpret_cast<ui64>(session.release()), static_cast<ui32>(ENetwork::EvRegisterSession), + {}, nullptr); + } + + void Enqueue(ui64 conn, std::unique_ptr<IEventHandle> ev, TIntrusivePtr<IReceiveCallback> replyCallback) { + SendImpl(conn, std::move(ev), std::move(replyCallback)); + } + void Send(ui64 conn, std::unique_ptr<IEventHandle> ev, TIntrusivePtr<IReceiveCallback> replyCallback) { ++*EventsSent; SendImpl(conn, std::move(ev), std::move(replyCallback)); @@ -556,6 +590,10 @@ namespace NActors { SendInternal(conn, static_cast<ui32>(ENetwork::EvRegisterCallback), localActorId, std::move(callback)); } + void NotifyMigrateDone(ui64 conn) { + SendInternal(conn, static_cast<ui32>(ENetwork::EvMigrateDone), {}, nullptr); + } + void Stop() { if (Worker.joinable()) { SendInternal(0, static_cast<ui32>(ENetwork::EvStop), {}, nullptr); @@ -577,44 +615,35 @@ namespace NActors { // the ownership handling of the worker loop: destroys the embedded TEventPayload and reclaims a // TRegisteredSession handed off via an unprocessed EvRegisterSession. void DrainQueue() { - while (std::unique_ptr<IEventHandle> ev{IncomingEventQueue.Pop()}) { - auto& payload = reinterpret_cast<TEventPayload&>(const_cast<TActorId&>(ev->InterconnectSession)); - if (ev->Type == static_cast<ui32>(ENetwork::EvRegisterSession)) { - std::unique_ptr<TRegisteredSession> reclaim(reinterpret_cast<TRegisteredSession*>(payload.Conn)); + for (;;) { + if (auto&& [ev, conn, callback, timestamp] = IncomingEventQueue.Pop(); ev) { + if (ev->Type == static_cast<ui32>(ENetwork::EvRegisterSession)) { + delete reinterpret_cast<TRegisteredSession*>(conn); + } + } else { + break; } - payload.~TEventPayload(); } } void SendImpl(ui64 conn, std::unique_ptr<IEventHandle> ev, TIntrusivePtr<IReceiveCallback> replyCallback) { - new(const_cast<TActorId*>(&ev->InterconnectSession)) TEventPayload{ - .Conn = conn, - .Callback = std::move(replyCallback), - }; - reinterpret_cast<ui64&>(const_cast<TScopeId&>(ev->OriginScopeId)) = GetCycleCountFast(); - const bool first = IncomingEventQueue.Push(std::move(ev)); + const bool first = IncomingEventQueue.Push(std::move(ev), conn, std::move(replyCallback)); if (first) { ++*PushedAsFirst; } ++*PushedTotal; - if (first && WaitingForCQ.load()) { - // this is the first command and we are currently waiting on CQ, so we have to push it through syscall - // writing to special fd - char temp = 0; - int res; - while ((res = write(WritePipe, &temp, 1)) != 1) { - if (res == -1) { - if (errno == EINTR) { - continue; - } else { - Y_ABORT("write() to pipe failed: %s", strerror(errno)); - } + if (first && WaitingForCQ.load(std::memory_order_relaxed)) { + // first command while waiting on CQ: kick the worker via the pipe on ring 0 + const ui64 value = 1; // this commands adds 1 to the counter stored in eventfd + ssize_t res; + while ((res = write(EventFd, &value, sizeof(value))) != sizeof(value)) { + if (res == -1 && errno == EINTR) { + continue; } else { - Y_ABORT_UNLESS(res == 0); - Y_ABORT("write() to pipe failed: zero bytes written"); + Y_ABORT("write() to eventfd failed: %s", strerror(errno)); } } - ++*PipeWakeups; + ++*EventWakeups; } } @@ -622,34 +651,81 @@ namespace NActors { SendImpl(conn, std::make_unique<IEventHandle>(type, 0, TActorId(), sender, nullptr, 0), std::move(callback)); } + bool ForwardIfMigratingOut(ui64 conn, std::unique_ptr<IEventHandle>& ev, TIntrusivePtr<IReceiveCallback>& callback) { + if (const auto it = MigratingOut.find(conn); it != MigratingOut.end()) { + Engine.Shards[it->second]->Enqueue(conn, std::move(ev), std::move(callback)); + return true; + } + return false; + } + // GetSQE returns next available SQ entry, setting up ItemsToSubmit counter in order to commence submission - // on the end of the worker loop - io_uring_sqe *GetSQE(TRegisteredSession *session, EOperationType op) { - io_uring_sqe *sqe = io_uring_get_sqe(&Ring); - if (!sqe) { // submit queue is full: try to submit something to free it up - DoSubmit(); - sqe = io_uring_get_sqe(&Ring); + // on the end of the worker loop. Event/timer control ops always use ring 0. + io_uring_sqe *GetSQE(TRegisteredSession *session, EOperationType op, int ringIdx = -1) { + io_uring_sqe *sqe = nullptr; + + if (ringIdx != -1) { + TRingSlot& ring = Rings[ringIdx]; + sqe = io_uring_get_sqe(&ring.Ring); + if (!sqe) { // submit queue is full: try to submit something to free it up + DoSubmit(ring); + sqe = io_uring_get_sqe(&ring.Ring); + } + } else { + for (size_t i = 0; !sqe && i < Rings.size(); ++i) { + ringIdx = (i + OpShift) % Rings.size(); + TRingSlot& ring = Rings[ringIdx]; + sqe = io_uring_get_sqe(&ring.Ring); + } + for (size_t i = 0; !sqe && i < Rings.size(); ++i) { + ringIdx = (i + OpShift) % Rings.size(); + TRingSlot& ring = Rings[ringIdx]; + DoSubmit(ring); + sqe = io_uring_get_sqe(&ring.Ring); + } + if (sqe) { + ++OpShift; + } } + if (sqe) { - ++ItemsToSubmit; + Y_DEBUG_ABORT_UNLESS(ringIdx != -1); + if (op == kOpRead) { + Y_DEBUG_ABORT_UNLESS(session); + Y_DEBUG_ABORT_UNLESS(session->ReadPendingRingIdx == -1); + session->ReadPendingRingIdx = ringIdx; + } + ++Rings[ringIdx].ItemsToSubmit; uintptr_t sessionId = reinterpret_cast<uintptr_t>(session); Y_ABORT_UNLESS((sessionId & kOpMask) == 0); io_uring_sqe_set_data64(sqe, sessionId | op); - Y_DEBUG_ABORT_UNLESS(op == kOpPipe || op == kOpTimer ? session == nullptr : session != nullptr); + Y_DEBUG_ABORT_UNLESS(op == kOpEvent || op == kOpTimer ? session == nullptr : session != nullptr); ++*SQEAllocated; } return sqe; } + void PutEventReadRequest() { + io_uring_sqe *sqe = GetSQE(nullptr, kOpEvent, 0 /* first ring strict */); + Y_ABORT_UNLESS(sqe, "failed to obtain event SQE: SQ overflow"); + io_uring_prep_read(sqe, EventFd, &EventFdReadBuffer, sizeof(EventFdReadBuffer), -1); + } + + void PutTimer() { + io_uring_sqe *sqe = GetSQE(nullptr, kOpTimer); + Y_ABORT_UNLESS(sqe, "failed to obtain timer SQE: SQ overflow"); + io_uring_prep_read(sqe, TimerFd, &ReadTimerBuffer, sizeof(ReadTimerBuffer), -1); + } + // DoSubmit performs actual io_uring submit operation for all allocated entries during the worker loop - void DoSubmit() { + void DoSubmit(TRingSlot& slot) { ui64 enterTimestamp; ACTIVITY(&SubmitWaitTotalTime) { enterTimestamp = LastActivitySwitchTimestamp; for (;;) { - int res = io_uring_submit(&Ring); + int res = io_uring_submit(&slot.Ring); if (res == -EINTR) { continue; } @@ -662,149 +738,212 @@ namespace NActors { ++*SubmitCount; SubmitExecTime->Collect((LastActivitySwitchTimestamp - enterTimestamp) * Freq); - SubmissionsProcessedAtOnce->Collect(ItemsToSubmit, 1u); - ItemsToSubmit = 0; - } - - void PutPipeReadRequest() { - io_uring_sqe *sqe = GetSQE(nullptr, kOpPipe); - Y_ABORT_UNLESS(sqe, "failed to obtain pipe SQE: SQ overflow"); // TODO(alexvru): handle this somehow - io_uring_prep_read(sqe, ReadPipe, ReadPipeBuffer, sizeof(ReadPipeBuffer), -1); - } - - void PutTimer() { - io_uring_sqe *sqe = GetSQE(nullptr, kOpTimer); - Y_ABORT_UNLESS(sqe, "failed to obtain timer SQE: SQ overflow"); // TODO(alexvru): handle this somehow - io_uring_prep_read(sqe, TimerFd, ReadTimerBuffer, sizeof(ReadTimerBuffer), -1); + SubmissionsProcessedAtOnce->Collect(slot.ItemsToSubmit, 1u); + slot.ItemsToSubmit = 0; } void WorkerThread() { - LastActivitySwitchTimestamp = GetCycleCountFast(); - pthread_setname_np(pthread_self(), "IC_uring"); - // prepare read request in order for sender threads to wake this one up when waiting on CQ - PutPipeReadRequest(); + LastActivitySwitchTimestamp = GetCycleCountFast(); + ui64 loopStartTimestamp = LastActivitySwitchTimestamp; + + // Arm pipe + timer + PutEventReadRequest(); PutTimer(); for (;;) { // submit any pending SQ's (if we have any) - if (ItemsToSubmit) { - DoSubmit(); + for (auto& slot : Rings) { + if (slot.ItemsToSubmit) { + DoSubmit(slot); + } } - // wait for something to happen - WaitingForCQ.store(true); - if (IncomingEventQueue.IsEmpty()) { - io_uring_cqe *cqe; - ui64 enterTimestamp; - ACTIVITY(&CompleteWaitTotalTime) { - enterTimestamp = LastActivitySwitchTimestamp; - if (int res = io_uring_wait_cqe(&Ring, &cqe); res && res != -EINTR) { - Y_ABORT("io_uring_wait_cqe() failed: %s", strerror(-res)); - } - } - CompletionWaitTime->Collect((LastActivitySwitchTimestamp - enterTimestamp) * Freq); + // process pending CQ events from every ring + bool progress = ProcessCompletions(); + + // process pending events and commands + bool stopping = false; + progress |= ProcessPendingCommands(&stopping); + if (stopping) { + break; } - WaitingForCQ.store(false); - // process pending CQ events - io_uring_cqe *cqes[CqeBatchSize]; - i64 completionsProcessedAtOnce = 0; - while (const unsigned n = io_uring_peek_batch_cqe(&Ring, cqes, CqeBatchSize)) { - for (unsigned i = 0; i < n; ++i) { - DispatchCompletion(*cqes[i]); - } - io_uring_cq_advance(&Ring, n); - completionsProcessedAtOnce += n; - if (n < CqeBatchSize) { - break; + ui64 waitStartTimestamp = 0; + if (!progress) { // wait for something to happen -- no progress were made in this loop + WaitingForCQ.store(true, std::memory_order_relaxed); + + // it is critical we first set WaitingForCQ, and then rechecking the queue + if (IncomingEventQueue.IsEmpty()) { + io_uring_cqe *cqe; + ui64 enterTimestamp; + ACTIVITY(&CompleteWaitTotalTime) { + waitStartTimestamp = enterTimestamp = LastActivitySwitchTimestamp; + // wait for the ring waiting for EventFd + if (int res = io_uring_wait_cqe(&Rings.front().Ring, &cqe); res && res != -EINTR) { + Y_ABORT("io_uring_wait_cqe() failed: %s", strerror(-res)); + } + } + CompletionWaitTime->Collect((LastActivitySwitchTimestamp - enterTimestamp) * Freq); } - } - if (completionsProcessedAtOnce) { - CompletionsProcessedAtOnce->Collect(completionsProcessedAtOnce, 1u); + + WaitingForCQ.store(false, std::memory_order_relaxed); } - // process pending events and commands - while (std::unique_ptr<IEventHandle> ev{IncomingEventQueue.Pop()}) { - auto& payload = reinterpret_cast<TEventPayload&>(const_cast<TActorId&>(ev->InterconnectSession)); - const ui64 conn = payload.Conn; - TIntrusivePtr<IReceiveCallback> callback = std::move(payload.Callback); - payload.~TEventPayload(); + const ui64 loopEndTimestamp = GetCycleCountFast(); + const ui64 total = loopEndTimestamp - loopStartTimestamp; + const ui64 wait = waitStartTimestamp ? loopEndTimestamp - waitStartTimestamp : 0; + const ui64 work = wait < total ? total - wait : 0; + PublishLoadSample(work, total); + loopStartTimestamp = loopEndTimestamp; + } + } - const ui64 cycleCountOnSend = reinterpret_cast<const ui64&>(ev->OriginScopeId); - const ui64 cycleCountOnEnter = GetCycleCountFast(); + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + // command processing - switch (ev->Type) { - case static_cast<ui32>(ENetwork::EvRegisterCallback): - if (TRegisteredSession& session = GetSession(conn); callback) { - session.ReceiveCallbacks[ev->Sender] = std::move(callback); - } else { - session.ReceiveCallbacks.erase(ev->Sender); - } - break; + bool ProcessPendingCommands(bool *stopping) { + bool progress = false; + ui64 cycleCountOnEnter = 0; - case static_cast<ui32>(ENetwork::EvRegisterSession): { - std::unique_ptr<TRegisteredSession> session(reinterpret_cast<TRegisteredSession*>(conn)); - const auto [it, inserted] = Sessions.emplace(std::move(session)); - Y_ABORT_UNLESS(inserted); - (*it)->EventsReceived = EventsReceived; - IssueReadForSession(**it); + for (;;) { + auto&& [ev, conn, callback, cycleCountOnSend] = IncomingEventQueue.Pop(); + if (!ev) { + break; + } + progress = true; + + if (!cycleCountOnEnter) { + cycleCountOnEnter = GetCycleCountFast(); + } + + switch (ev->Type) { + case static_cast<ui32>(ENetwork::EvRegisterCallback): + if (ForwardIfMigratingOut(conn, ev, callback)) { break; } + if (TRegisteredSession& session = GetSession(conn); callback) { + session.ReceiveCallbacks[ev->Sender] = std::move(callback); + } else { + session.ReceiveCallbacks.erase(ev->Sender); + } + break; - case static_cast<ui32>(ENetwork::EvUnregisterSession): { - TRegisteredSession& session = GetSession(conn); - // Do NOT free the session while it still has an armed recv or an in-flight - // writev: their io_uring completions carry a raw pointer to this object and - // would dereference freed memory. Mark it terminated (so no new ops are armed) - // and erase only once both are drained. The session actor has already shut the - // socket down before requesting unregistration, so the pending ops complete - // promptly (EOF/EPIPE). - session.Terminated = true; - session.UnregisterRequested = true; - MaybeEraseSession(session); + case static_cast<ui32>(ENetwork::EvRegisterSession): { + std::unique_ptr<TRegisteredSession> session(reinterpret_cast<TRegisteredSession*>(conn)); + const bool migrated = session->MigrateState == EMigrateState::HandedOff; + const ui32 sourceShard = session->MigrateSourceShard; + session->OwnerShard.store(ShardIdx, std::memory_order_release); + session->MigrateState = EMigrateState::None; + session->MigrateSourceShard = ShardIdx; + session->PreferredRingIdx = OpShift++ % Rings.size(); + const auto [it, inserted] = Sessions.emplace(std::move(session)); + Y_ABORT_UNLESS(inserted); + (*it)->EventsReceived = EventsReceived; + IssueReadForSession(**it); + if (migrated && sourceShard != ShardIdx) { + Engine.Shards[sourceShard]->NotifyMigrateDone(conn); + } + break; + } + + case static_cast<ui32>(ENetwork::EvUnregisterSession): { + if (ForwardIfMigratingOut(conn, ev, callback)) { break; } + TRegisteredSession& session = GetSession(conn); + // Do NOT free the session while it still has an armed recv or an in-flight + // writev: their io_uring completions carry a raw pointer to this object and + // would dereference freed memory. Mark it terminated (so no new ops are armed) + // and erase only once both are drained. The session actor has already shut the + // socket down before requesting unregistration, so the pending ops complete + // promptly (EOF/EPIPE). + if (session.MigrateState != EMigrateState::None) { + session.MigrateState = EMigrateState::None; // unregister wins over migrate + } + session.Terminated = true; + session.UnregisterRequested = true; + if (session.ReadPending) { // cancel pending read in order to unregister the session + CancelOp(session, kOpRead, session.ReadPendingRingIdx); + } + MaybeEraseSession(session); + break; + } - case static_cast<ui32>(ENetwork::EvStop): - return; + case static_cast<ui32>(ENetwork::EvMigrateDone): + MigratingOut.erase(conn); + break; - default: { - TRegisteredSession& session = GetSession(conn); - if (callback) { // register callback coming along with the message - session.ReceiveCallbacks[ev->Sender] = std::move(callback); - } - session.Serializer.Push(std::move(ev)); - IssueWritesForSession(session); + case static_cast<ui32>(ENetwork::EvStop): + *stopping = true; + return true; + + default: { + if (ForwardIfMigratingOut(conn, ev, callback)) { break; } + TRegisteredSession& session = GetSession(conn); + if (callback) { // register callback coming along with the message + session.ReceiveCallbacks[ev->Sender] = std::move(callback); + } + session.Serializer.Push(std::move(ev)); + IssueWritesForSession(session); + break; } - - const ui64 cycleCountOnExit = GetCycleCountFast(); - CommandDeliveryTime->Collect(NHPTimer::GetSeconds(cycleCountOnEnter - cycleCountOnSend) * 1e9); - CommandExecTime->Collect(NHPTimer::GetSeconds(cycleCountOnExit - cycleCountOnEnter) * 1e9); } + + const ui64 cycleCountOnExit = GetCycleCountFast(); + CommandDeliveryTime->Collect(NHPTimer::GetSeconds(cycleCountOnEnter - cycleCountOnSend) * 1e9); + CommandExecTime->Collect(NHPTimer::GetSeconds(cycleCountOnExit - cycleCountOnEnter) * 1e9); + cycleCountOnEnter = cycleCountOnExit; } + + return progress; } //////////////////////////////////////////////////////////////////////////////////////////////////////////// // io_uring completion handlers - void DispatchCompletion(io_uring_cqe cqe) { + bool ProcessCompletions() { + bool progress = false; + i64 completionsProcessedAtOnce = 0; + for (auto& slot : Rings) { + io_uring_cqe *cqes[CqeBatchSize]; + while (const unsigned n = io_uring_peek_batch_cqe(&slot.Ring, cqes, CqeBatchSize)) { + progress = true; // we did something with the queues + for (unsigned i = 0; i < n; ++i) { + DispatchCompletion(*cqes[i]); + } + io_uring_cq_advance(&slot.Ring, n); + completionsProcessedAtOnce += n; + if (n < CqeBatchSize) { + break; + } + } + } + if (completionsProcessedAtOnce) { + CompletionsProcessedAtOnce->Collect(completionsProcessedAtOnce, 1u); + } + return progress; + } + + void DispatchCompletion(io_uring_cqe& cqe) { auto *session = reinterpret_cast<TRegisteredSession*>(uintptr_t(cqe.user_data) & ~uintptr_t(kOpMask)); Y_ABORT_UNLESS(!(cqe.flags & IORING_CQE_F_MORE)); // not expecting multiple completions - switch (static_cast<EOperationType>(cqe.user_data & kOpMask)) { - case kOpPipe: - // this operation is used just to break wait-CQE syscall and exit to process some commands; but - // we have to re-arm the pipe + const auto op = static_cast<EOperationType>(cqe.user_data & kOpMask); + switch (op) { + case kOpEvent: Y_DEBUG_ABORT_UNLESS(session == nullptr); - PutPipeReadRequest(); + Y_DEBUG_ABORT_UNLESS(cqe.res == sizeof(EventFdReadBuffer)); + PutEventReadRequest(); break; case kOpRead: Y_DEBUG_ABORT_UNLESS(session != nullptr); - DispatchRead(*session, cqe.res); // TODO(alexvru): maybe handle NONEMPTY (if it won't break equality) + Y_DEBUG_ABORT_UNLESS(session->ReadPendingRingIdx != -1); + session->ReadPendingRingIdx = -1; + DispatchRead(*session, cqe.res); break; case kOpWrite: @@ -817,17 +956,99 @@ namespace NActors { DispatchTimer(); PutTimer(); break; + + case kOpCancel: + // Original op completes with -ECANCELED; the cancel SQE itself needs no action. + break; } ++*CQEProcessed; } void DispatchTimer() { for (auto& session : Sessions) { - if (!session->Terminated && session->SendPings && session->PingRequestSentTimestamp == 0) { + if (!session->Terminated && session->MigrateState == EMigrateState::None && session->SendPings && + session->PingRequestSentTimestamp == 0) { session->SendPingRequest(); IssueWritesForSession(*session); } } + + // Rebalance at most once per ping period to limit churn under short load spikes. + MaybeOffload(); + } + + void MaybeOffload() { + if (Engine.Shards.size() < 2 || Sessions.size() < 2 || Load.BusyFraction() < OffloadBusyThreshold) { + return; + } + + return; // TODO(alexvru): check for out-of-order when processing old shard queue when new shard is registered + + ui32 bestTarget = ShardIdx; + ui32 bestBusy = Max<ui32>(); + for (ui32 i = 0; i < Engine.Shards.size(); ++i) { + if (i == ShardIdx) { + continue; + } + if (const ui32 busy = Engine.ShardLoads[i].BusyFraction(); busy < bestBusy) { + bestBusy = busy; + bestTarget = i; + } + } + if (bestTarget == ShardIdx || bestBusy > StealBusyThreshold) { + return; + } + + TRegisteredSession *candidate = nullptr; + for (auto& session : Sessions) { + if (session->IsMigratable()) { + candidate = session.get(); + break; + } + } + if (!candidate) { + return; + } + + candidate->MigrateState = EMigrateState::Draining; + candidate->MigrateTargetShard = bestTarget; + candidate->MigrateSourceShard = ShardIdx; + + if (candidate->ReadPending) { + CancelOp(*candidate, kOpRead, candidate->ReadPendingRingIdx); + } + + MaybeFinishMigrate(*candidate); + } + + void CancelOp(TRegisteredSession& session, EOperationType op, int ringIdx) { + Y_DEBUG_ABORT_UNLESS(ringIdx != -1); + io_uring_sqe *sqe = GetSQE(&session, kOpCancel, ringIdx); + Y_ABORT_UNLESS(sqe, "failed to obtain cancel SQE"); + const ui64 targetUserData = reinterpret_cast<uintptr_t>(&session) | op; + io_uring_prep_cancel64(sqe, targetUserData, 0); + } + + bool MaybeFinishMigrate(TRegisteredSession& session) { + if (session.MigrateState != EMigrateState::Draining || session.ReadPending || session.WritePending || + session.UnregisterRequested || session.Terminated) { + return false; + } + + const ui64 conn = reinterpret_cast<ui64>(&session); + const ui32 target = session.MigrateTargetShard; + session.MigrateState = EMigrateState::HandedOff; + + auto it = Sessions.find(&session); + Y_DEBUG_ABORT_UNLESS(it != Sessions.end()); + auto node = Sessions.extract(it); + Y_DEBUG_ABORT_UNLESS(node); + + MigratingOut[conn] = target; + ++*SessionsMigratedOut; + Engine.Shards[target]->AcceptMigrated(std::move(node.value())); + + return true; } void DispatchRead(TRegisteredSession& session, i32 res) { @@ -837,6 +1058,9 @@ namespace NActors { if (session.Terminated) { // teardown in progress: don't process further data or re-arm; just let the session drain // toward erasure below + } else if (res == -ECANCELED) { + // cancelled without migrate (should be rare); re-arm unless terminating + IssueReadForSession(session); } else if (res == -EAGAIN) { ++*ReadUnavail; IssueReadForSession(session); @@ -853,16 +1077,19 @@ namespace NActors { IssueWritesForSession(session); } + if (MaybeFinishMigrate(session)) { // NB: may free/move `session` + return; + } MaybeEraseSession(session); // NB: may free `session`; must be the last use } void IssueReadForSession(TRegisteredSession& session) { - if (session.Terminated) { + if (session.Terminated || session.MigrateState != EMigrateState::None) { return; } Y_DEBUG_ABORT_UNLESS(!session.ReadPending); TMutableContiguousSpan span = session.GetReadSpan(); - io_uring_sqe *sqe = GetSQE(&session, kOpRead); + io_uring_sqe *sqe = GetSQE(&session, kOpRead, session.PreferredRingIdx); Y_ABORT_UNLESS(sqe); io_uring_prep_read(sqe, *session.Socket, span.data(), span.size(), -1); session.ReadPending = true; @@ -875,6 +1102,8 @@ namespace NActors { if (session.Terminated) { // teardown in progress: don't retry the write or re-arm; just let the session drain // toward erasure below + } else if (res == -ECANCELED) { + IssueWritesForSession(session); } else if (res == -EAGAIN) { ++*WriteUnavail; SubmitIovec(session); @@ -894,11 +1123,15 @@ namespace NActors { IssueWritesForSession(session); } + if (MaybeFinishMigrate(session)) { + return; + } MaybeEraseSession(session); // NB: may free `session`; must be the last use } void IssueWritesForSession(TRegisteredSession& session) { - if (session.WritePending || session.Terminated || !session.Serializer.IsTrafficPending()) { + if (session.WritePending || session.Terminated || session.MigrateState != EMigrateState::None || + !session.Serializer.IsTrafficPending()) { return; } if (session.Serialize()) { @@ -917,7 +1150,10 @@ namespace NActors { } void SubmitIovec(TRegisteredSession& session) { - io_uring_sqe *sqe = GetSQE(&session, kOpWrite); + if (session.MigrateState != EMigrateState::None) { + return; + } + io_uring_sqe *sqe = GetSQE(&session, kOpWrite, session.PreferredRingIdx); Y_ABORT_UNLESS(sqe); io_uring_prep_writev(sqe, *session.Socket, session.Iov, session.IovLen, -1); session.WritePending = true; @@ -944,17 +1180,30 @@ namespace NActors { }; std::vector<std::unique_ptr<TShard>> Shards; + std::vector<TShardLoad> ShardLoads; std::atomic_uint64_t NextShardIdx; NMonitoring::TDynamicCounterPtr UringCounters; public: - TUringEngine(ui32 numShards, NMonitoring::TDynamicCounterPtr counters, bool sqpoll) + TUringEngine(ui32 numShards, NMonitoring::TDynamicCounterPtr counters, bool sqpoll, ui32 ringsPerShard, + ui32 sqThreadIdleMs, bool shareRingsAmongThreads) : UringCounters(std::move(counters)) { + ShardLoads = std::vector<TShardLoad>(numShards); Shards.reserve(numShards); for (ui32 i = 0; i < numShards; ++i) { - Shards.push_back(std::make_unique<TShard>(UringCounters->GetSubgroup("shard", "0" /*ToString(i)*/), sqpoll)); + Shards.push_back(std::make_unique<TShard>(*this, + i, // shardIdx + UringCounters->GetSubgroup("shard", "0" /*ToString(i)*/), + sqpoll, + ringsPerShard, + sqThreadIdleMs, + ShardLoads[i], + shareRingsAmongThreads && !Shards.empty() ? Shards.front().get() : nullptr)); + } + for (auto& shard : Shards) { + shard->Start(); } } @@ -978,7 +1227,17 @@ namespace NActors { return 0; // engine is shutting down; caller treats 0 as a failed registration and terminates } Y_ABORT_UNLESS(ActorSystem); - const ui32 shardIdx = NextShardIdx++ % Shards.size(); + + // Prefer the currently least-loaded shard (in-process signal); fall back to round-robin. + ui32 shardIdx = NextShardIdx++ % Shards.size(); + ui32 bestBusy = ShardLoads[shardIdx].BusyFraction(); + for (ui32 i = 0; i < ShardLoads.size(); ++i) { + if (const ui32 busy = ShardLoads[i].BusyFraction(); busy < bestBusy) { + bestBusy = busy; + shardIdx = i; + } + } + auto session = std::make_unique<TRegisteredSession>(shardIdx, std::move(socket), sessionActorId, checksumming, peerScopeId, std::move(onDisconnectCallback), ActorSystem, sendPings, std::move(clockSkew), std::move(pingRTT)); @@ -988,7 +1247,8 @@ namespace NActors { } TShard& GetShard(ui64 conn) const { - return *Shards.at(reinterpret_cast<TRegisteredSession*>(conn)->ShardIdx); + // OwnerShard is the indirection for dynamic mapping: stable conn handle, mutable owner. + return *Shards[reinterpret_cast<TRegisteredSession*>(conn)->OwnerShard.load(std::memory_order_acquire)]; } void Send(ui64 conn, std::unique_ptr<IEventHandle> ev, TIntrusivePtr<IReceiveCallback> replyCallback) override { @@ -1028,14 +1288,22 @@ namespace NActors { } }; - TUringEnginePtr CreateUringEngine(ui32 numShards, NMonitoring::TDynamicCounterPtr counters, bool sqpoll) { + TUringEnginePtr CreateUringEngine(ui32 numShards, NMonitoring::TDynamicCounterPtr counters, bool sqpoll, + ui32 ringsPerShard, ui32 sqThreadIdleMs, bool shareRingsAmongThreads) { if (!TUringContext::IsAvailable()) { return nullptr; } if (numShards < 1) { numShards = 1; } - return MakeIntrusive<TUringEngine>(numShards, std::move(counters), sqpoll); + if (ringsPerShard < 1) { + ringsPerShard = 1; + } + if (sqThreadIdleMs < 1) { + sqThreadIdleMs = TUringContext::SqThreadIdleMs; + } + return MakeIntrusive<TUringEngine>(numShards, std::move(counters), sqpoll, ringsPerShard, sqThreadIdleMs, + shareRingsAmongThreads); } } // namespace NActors diff --git a/ydb/library/actors/interconnect/interconnect_uring_engine.h b/ydb/library/actors/interconnect/interconnect_uring_engine.h index 72c58c4cfd9..629670ba9d7 100644 --- a/ydb/library/actors/interconnect/interconnect_uring_engine.h +++ b/ydb/library/actors/interconnect/interconnect_uring_engine.h @@ -48,10 +48,18 @@ namespace NActors { using TUringEnginePtr = TIntrusivePtr<IUringEngine>; - // Creates the engine with `numShards` rings/reaper threads (clamped to >= 1). The engine is created - // without an actor system (it does not exist yet at startup); bind it later via SetActorSystem before the - // first Register. Returns nullptr when io_uring is unavailable (non-Linux or a kernel that cannot create - // a ring). - TUringEnginePtr CreateUringEngine(ui32 numShards, NMonitoring::TDynamicCounterPtr counters, bool sqpoll); + // Creates the engine with `numShards` worker threads (clamped to >= 1), each driving `ringsPerShard` + // io_uring rings (clamped to >= 1). With SQPOLL, each ring has its own kernel poll thread, so + // rings-per-shard scales submission-polling capacity independently of serialization workers. + // The engine is created without an actor system (it does not exist yet at startup); bind it later via + // SetActorSystem before the first Register. Returns nullptr when io_uring is unavailable (non-Linux or a + // kernel that cannot create a ring). + // + // When the kernel SQPOLL thread is saturated while shard workers still have headroom, try + // EnableSQPOLLv2=false (submit from the underutilized worker) or raise ringsPerShard. When workers are + // the bottleneck, raise numShards / lower ringsPerShard. + // sqThreadIdleMs is the SQPOLL kernel-thread idle timeout in milliseconds (ignored when sqpoll is false). + TUringEnginePtr CreateUringEngine(ui32 numShards, NMonitoring::TDynamicCounterPtr counters, bool sqpoll, + ui32 ringsPerShard = 1, ui32 sqThreadIdleMs = 2000, bool shareRingsAmongThreads = false); } diff --git a/ydb/library/actors/interconnect/interconnect_uring_engine_stub.cpp b/ydb/library/actors/interconnect/interconnect_uring_engine_stub.cpp index 59b066426f6..e127ed13720 100644 --- a/ydb/library/actors/interconnect/interconnect_uring_engine_stub.cpp +++ b/ydb/library/actors/interconnect/interconnect_uring_engine_stub.cpp @@ -1,7 +1,7 @@ +#include "interconnect_uring_engine.h" + namespace NActors { - TUringEnginePtr CreateUringEngine(ui32, NMonitoring::TDynamicCounterPtr, bool) { + TUringEnginePtr CreateUringEngine(ui32, NMonitoring::TDynamicCounterPtr, bool, ui32, ui32) { return nullptr; } } - - diff --git a/ydb/library/actors/interconnect/interconnect_uring_event_queue.h b/ydb/library/actors/interconnect/interconnect_uring_event_queue.h new file mode 100644 index 00000000000..e4849f8a614 --- /dev/null +++ b/ydb/library/actors/interconnect/interconnect_uring_event_queue.h @@ -0,0 +1,101 @@ +#pragma once + +#include "interconnect_direct_session.h" + +namespace NActors { + + class TIncomingEventQueue { + IEventHandle Stub{0, 0, {}, {}, nullptr, 0}; + std::atomic<IEventHandle*> Head{&Stub}; + std::atomic<IEventHandle*> Tail{&Stub}; + + struct TEventPayload { + ui64 Conn; + TIntrusivePtr<IReceiveCallback> Callback; + }; + static_assert(sizeof(TEventPayload) <= sizeof(TActorId)); + + public: + TIncomingEventQueue() { + Stub.NextLinkPtr.store(0, std::memory_order_relaxed); + } + + ~TIncomingEventQueue() { + Y_DEBUG_ABORT_UNLESS(IsEmpty()); // ensure this event queue has been properly drained by owner + } + + bool Push(std::unique_ptr<IEventHandle> ev, ui64 conn, TIntrusivePtr<IReceiveCallback> replyCallback) { + // store some metadata in event's unmatching fields + new(const_cast<TActorId*>(&ev->InterconnectSession)) TEventPayload{ + .Conn = conn, + .Callback = std::move(replyCallback), + }; + reinterpret_cast<ui64&>(const_cast<TScopeId&>(ev->OriginScopeId)) = GetCycleCountFast(); + + IEventHandle *last = ev.release(); + last->NextLinkPtr.store(0, std::memory_order_relaxed); + IEventHandle *prev = Head.exchange(last, std::memory_order_acq_rel); + prev->NextLinkPtr.store(reinterpret_cast<uintptr_t>(last), std::memory_order_release); + return prev == &Stub; // if it was the first event + } + + bool IsEmpty() const { + IEventHandle *tail = Tail.load(std::memory_order_relaxed); + IEventHandle *next = reinterpret_cast<IEventHandle*>(tail->NextLinkPtr.load(std::memory_order_acquire)); + IEventHandle *head = Head.load(std::memory_order_acquire); + return tail == &Stub && !next && tail == head; + } + + std::tuple<std::unique_ptr<IEventHandle>, ui64, TIntrusivePtr<IReceiveCallback>, ui64> Pop() { + auto decompose = [&](IEventHandle *ev) { + auto& payload = reinterpret_cast<TEventPayload&>(const_cast<TActorId&>(ev->InterconnectSession)); + ui64 conn = payload.Conn; + TIntrusivePtr<IReceiveCallback> callback = std::move(payload.Callback); + payload.~TEventPayload(); + const ui64 timestamp = reinterpret_cast<const ui64&>(ev->OriginScopeId); + return std::make_tuple(std::unique_ptr<IEventHandle>(ev), conn, std::move(callback), timestamp); + }; + + for (;;) { + IEventHandle *tail = Tail.load(std::memory_order_relaxed); + IEventHandle *next = reinterpret_cast<IEventHandle*>(tail->NextLinkPtr.load(std::memory_order_acquire)); + IEventHandle *head; + + if (tail == &Stub) { + if (!next) { + if (head = Head.load(std::memory_order_acquire); tail != head) { + continue; + } else { + return {}; + } + } + Tail.store(next, std::memory_order_relaxed); + tail = next; + next = reinterpret_cast<IEventHandle*>(tail->NextLinkPtr.load(std::memory_order_acquire)); + } + + if (next) { + Tail.store(next, std::memory_order_relaxed); + return decompose(tail); + } + + head = Head.load(std::memory_order_acquire); + if (tail != head) { + continue; + } + + Stub.NextLinkPtr.store(0, std::memory_order_relaxed); + IEventHandle *prev = Head.exchange(&Stub, std::memory_order_acq_rel); + prev->NextLinkPtr.store(reinterpret_cast<uintptr_t>(&Stub), std::memory_order_release); + + next = reinterpret_cast<IEventHandle*>(tail->NextLinkPtr.load(std::memory_order_acquire)); + if (next) { + Tail.store(next, std::memory_order_relaxed); + return decompose(tail); + } + } + } + }; + + +} // NActors diff --git a/ydb/library/actors/interconnect/ut/lib/node.h b/ydb/library/actors/interconnect/ut/lib/node.h index 5c8cbca4d34..b3cf6a11f21 100644 --- a/ydb/library/actors/interconnect/ut/lib/node.h +++ b/ydb/library/actors/interconnect/ut/lib/node.h @@ -75,17 +75,24 @@ public: } setup.InterconnectCollectSubscriptionStackTrace = common->Settings.CollectSubscriptionStackTrace; - if (common->Settings.EnableInterconnectSessionV2) { + if (common->Settings.V2.Enable) { // Mirror production: create the shared v2 io_uring engine up front and publish it in Common; the // proxy binds it to the actor system on start (SetActorSystem). Shard count is overridable via // YDB_IC_V2_SHARDS so tests can force many connections onto a single ring. - ui32 uringShards = 4; + ui32 uringShards = common->Settings.V2.UringEngineThreads; if (const TString s = GetEnv("YDB_IC_V2_SHARDS"); !s.empty()) { uringShards = FromString<ui32>(s); } + ui32 ringsPerShard = common->Settings.V2.UringEngineRingsPerShard; + if (const TString s = GetEnv("YDB_IC_V2_RINGS_PER_SHARD"); !s.empty()) { + ringsPerShard = FromString<ui32>(s); + } common->UringEngineV2 = CreateUringEngine(uringShards, common->MonCounters->GetSubgroup("subsystem", "uring"), - common->Settings.EnableSQPOLLv2); + common->Settings.V2.EnableSQPOLL, + ringsPerShard, + common->Settings.V2.UringEngineSqThreadIdleMs, + common->Settings.V2.ShareRingsAmongThreads); setup.OnActorSystemCreated.push_back([engine = common->UringEngineV2](TActorSystem *actorSystem) { if (engine) { engine->SetActorSystem(actorSystem); diff --git a/ydb/library/actors/interconnect/ut/v2_session_ut.cpp b/ydb/library/actors/interconnect/ut/v2_session_ut.cpp index 8d93cc19d46..e6949208fc6 100644 --- a/ydb/library/actors/interconnect/ut/v2_session_ut.cpp +++ b/ydb/library/actors/interconnect/ut/v2_session_ut.cpp @@ -308,8 +308,8 @@ namespace { std::unique_ptr<TTestICCluster> MakeV2Cluster(ui32 tcpSocketBufferSize = 0) { auto customizer = [tcpSocketBufferSize](ui32, TInterconnectSettings& settings) { - settings.EnableInterconnectSessionV2 = true; - settings.ChecksumInterconnectSessionV2 = true; + settings.V2.Enable = true; + settings.V2.ChecksumEvents = true; if (tcpSocketBufferSize) { settings.TCPSocketBufferSize = tcpSocketBufferSize; } @@ -505,7 +505,7 @@ Y_UNIT_TEST_SUITE(InterconnectSessionV2) { constexpr ui32 kNodes = 5; auto customizer = [](ui32, TInterconnectSettings& settings) { - settings.EnableInterconnectSessionV2 = true; + settings.V2.Enable = true; }; auto cluster = std::make_unique<TTestICCluster>( kNodes, TChannelsConfig(), nullptr, nullptr, TTestICCluster::EMPTY, @@ -554,7 +554,7 @@ Y_UNIT_TEST_SUITE(InterconnectSessionV2) { constexpr ui32 kNodes = 5; auto customizer = [](ui32, TInterconnectSettings& settings) { - settings.EnableInterconnectSessionV2 = true; + settings.V2.Enable = true; }; auto cluster = std::make_unique<TTestICCluster>( kNodes, TChannelsConfig(), nullptr, nullptr, TTestICCluster::EMPTY, diff --git a/ydb/library/actors/interconnect/v2_event_serializer.cpp b/ydb/library/actors/interconnect/v2_event_serializer.cpp index 2bb23629cbb..0d21c2bf942 100644 --- a/ydb/library/actors/interconnect/v2_event_serializer.cpp +++ b/ydb/library/actors/interconnect/v2_event_serializer.cpp @@ -135,6 +135,7 @@ namespace NActors { memcpy(buffer.UnsafeGetDataMut(), span.data(), span.size()); span = {buffer.data(), span.size()}; buffer.TrimFront(buffer.size() - span.size()); + *bufferProduced = CumulativeProduced + span.size(); } } diff --git a/yt/yt/client/federated/cache.cpp b/yt/yt/client/federated/cache.cpp index 7c126ece2e2..60ce865ddcc 100644 --- a/yt/yt/client/federated/cache.cpp +++ b/yt/yt/client/federated/cache.cpp @@ -59,7 +59,7 @@ private: THROW_ERROR_EXCEPTION_UNLESS( clusters.size() == seenClusters.size(), - "Numbers of desired and configured clusters do not match (Desired: %v, Configured: %v)", + "Numbers of desired and configured clusters do not match: desired %v, configured %v", clusters, seenClusters); diff --git a/yt/yt/client/kafka/requests.cpp b/yt/yt/client/kafka/requests.cpp index f1826cd2795..7c599d0a014 100644 --- a/yt/yt/client/kafka/requests.cpp +++ b/yt/yt/client/kafka/requests.cpp @@ -284,7 +284,7 @@ void TRecordBatch::Deserialize(IKafkaProtocolReader* reader) } } if (reader->GetReadBytesCount() != Length) { - THROW_ERROR_EXCEPTION("Unexpected record batch length (Expected: %v, Actual: %v)", Length, reader->GetReadBytesCount()); + THROW_ERROR_EXCEPTION("Unexpected record batch length: expected %v, actual %v", Length, reader->GetReadBytesCount()); } } else { THROW_ERROR_EXCEPTION("Unsupported MagicByte %v in RecordBatch deserialization", static_cast<int>(MagicByte)); diff --git a/yt/yt/core/concurrency/unittests/bounded_concurrency_invoker_ut.cpp b/yt/yt/core/concurrency/unittests/bounded_concurrency_invoker_ut.cpp index dd8f6596551..afee3288996 100644 --- a/yt/yt/core/concurrency/unittests/bounded_concurrency_invoker_ut.cpp +++ b/yt/yt/core/concurrency/unittests/bounded_concurrency_invoker_ut.cpp @@ -167,7 +167,7 @@ TEST_P(TBoundedConcurrencyInvokerParametrizedReconfigureTest, SetMaxConcurrentIn auto guard = Guard(lock); auto concurrentInvocations = runnedCallbacks - finishedCallbacks; - THROW_ERROR_EXCEPTION_UNLESS(concurrentInvocations <= maxConcurrentInvocations, "Number of concurrent invocations exceeds maximum (ConcurrentInvocations: %v, MaxConcurrentInvocations: %v)", + THROW_ERROR_EXCEPTION_UNLESS(concurrentInvocations <= maxConcurrentInvocations, "Number of concurrent invocations %v exceeds maximum %v", concurrentInvocations, maxConcurrentInvocations); if (callbackIndex > maxConcurrentInvocations) { diff --git a/yt/yt/core/net/address.cpp b/yt/yt/core/net/address.cpp index 16eee1cdb5b..dfe29b605c7 100644 --- a/yt/yt/core/net/address.cpp +++ b/yt/yt/core/net/address.cpp @@ -1222,7 +1222,7 @@ TIP6Address TMtnAddress::ToIP6Address() const ui64 TMtnAddress::GetBytesRangeValue(int leftIndex, int rightIndex) const { if (leftIndex > rightIndex) { - THROW_ERROR_EXCEPTION("Left index is greater than right index (LeftIndex: %v, RightIndex: %v)", + THROW_ERROR_EXCEPTION("Left index %v is greater than right index %v", leftIndex, rightIndex); } @@ -1239,17 +1239,17 @@ ui64 TMtnAddress::GetBytesRangeValue(int leftIndex, int rightIndex) const void TMtnAddress::SetBytesRangeValue(int leftIndex, int rightIndex, ui64 value) { if (leftIndex > rightIndex) { - THROW_ERROR_EXCEPTION("Left index is greater than right index (LeftIndex: %v, RightIndex: %v)", + THROW_ERROR_EXCEPTION("Left index %v is greater than right index %v", leftIndex, rightIndex); } auto bytesInRange = rightIndex - leftIndex; if (value >= (1ull << (8 * bytesInRange))) { - THROW_ERROR_EXCEPTION("Value is too large to be set in [leftIndex; rightIndex) interval (LeftIndex: %v, RightIndex: %v, Value %v)", + THROW_ERROR_EXCEPTION("Value %v is too large to be set in interval [%v, %v)", + value, leftIndex, - rightIndex, - value); + rightIndex); } auto* addressBytes = Address_.GetRawBytes(); diff --git a/yt/yt/core/ytree/polymorphic_yson_struct-inl.h b/yt/yt/core/ytree/polymorphic_yson_struct-inl.h index 595f8fafd70..067956ad5db 100644 --- a/yt/yt/core/ytree/polymorphic_yson_struct-inl.h +++ b/yt/yt/core/ytree/polymorphic_yson_struct-inl.h @@ -236,7 +236,7 @@ void TPolymorphicYsonStruct<TMapping>::MergeWith(const TPolymorphicYsonStruct& o THROW_ERROR_EXCEPTION_UNLESS( GetType() == other.GetType(), - "Can't merge polymorphic yson structs with different types stored (ThisType: %v, OtherType: %v)", + "Cannot merge polymorphic YSON structs with different stored types: %Qlv vs %Qlv", GetType(), other.GetType()); diff --git a/yt/yt/library/formats/arrow_writer.cpp b/yt/yt/library/formats/arrow_writer.cpp index 54756b7cff6..713c23c602f 100644 --- a/yt/yt/library/formats/arrow_writer.cpp +++ b/yt/yt/library/formats/arrow_writer.cpp @@ -898,7 +898,9 @@ void SerializeDateColumn( }, [&] (auto value) { if (value > std::numeric_limits<i32>::max()) { - THROW_ERROR_EXCEPTION("Date value cannot be represented in arrow (Value: %v, MaxAllowedValue: %v)", value, std::numeric_limits<i32>::max()); + THROW_ERROR_EXCEPTION("Date value %v cannot be represented in arrow: maximum allowed value is %v", + value, + std::numeric_limits<i32>::max()); } *currentOutput++ = value; }); @@ -1007,7 +1009,7 @@ void SerializeTimestampColumn( }, [&] (auto value) { if (value > std::numeric_limits<i64>::max()) { - THROW_ERROR_EXCEPTION("Timestamp value cannot be represented in arrow (Value: %v, MaxAllowedValue: %v)", value, std::numeric_limits<i64>::max()); + THROW_ERROR_EXCEPTION("Timestamp value %v cannot be represented in arrow: maximum allowed value is %v", value, std::numeric_limits<i64>::max()); } *currentOutput++ = value; }); |