From 2b9ec015e1cc2f7534725f01a277a0403246a2f3 Mon Sep 17 00:00:00 2001 From: babenko Date: Sat, 25 Jul 2026 11:11:01 +0300 Subject: Get rid of log-style parenthetical tags in error messages in yt/, yp/ Error messages of the form "Something failed (Tag: %v, ...)" follow the logging convention, which is inappropriate for errors. Fold the variable parts into the message text organically or move them to error attributes. commit_hash:edf4f6e61e3a0a9e073a4534ed1ca05d9f22547c --- yt/yt/client/federated/cache.cpp | 2 +- yt/yt/client/kafka/requests.cpp | 2 +- .../concurrency/unittests/bounded_concurrency_invoker_ut.cpp | 2 +- yt/yt/core/net/address.cpp | 10 +++++----- yt/yt/core/ytree/polymorphic_yson_struct-inl.h | 2 +- yt/yt/library/formats/arrow_writer.cpp | 6 ++++-- 6 files changed, 13 insertions(+), 11 deletions(-) diff --git a/yt/yt/client/federated/cache.cpp b/yt/yt/client/federated/cache.cpp index 7c126ece2e2..60ce865ddcc 100644 --- a/yt/yt/client/federated/cache.cpp +++ b/yt/yt/client/federated/cache.cpp @@ -59,7 +59,7 @@ private: THROW_ERROR_EXCEPTION_UNLESS( clusters.size() == seenClusters.size(), - "Numbers of desired and configured clusters do not match (Desired: %v, Configured: %v)", + "Numbers of desired and configured clusters do not match: desired %v, configured %v", clusters, seenClusters); diff --git a/yt/yt/client/kafka/requests.cpp b/yt/yt/client/kafka/requests.cpp index f1826cd2795..7c599d0a014 100644 --- a/yt/yt/client/kafka/requests.cpp +++ b/yt/yt/client/kafka/requests.cpp @@ -284,7 +284,7 @@ void TRecordBatch::Deserialize(IKafkaProtocolReader* reader) } } if (reader->GetReadBytesCount() != Length) { - THROW_ERROR_EXCEPTION("Unexpected record batch length (Expected: %v, Actual: %v)", Length, reader->GetReadBytesCount()); + THROW_ERROR_EXCEPTION("Unexpected record batch length: expected %v, actual %v", Length, reader->GetReadBytesCount()); } } else { THROW_ERROR_EXCEPTION("Unsupported MagicByte %v in RecordBatch deserialization", static_cast(MagicByte)); diff --git a/yt/yt/core/concurrency/unittests/bounded_concurrency_invoker_ut.cpp b/yt/yt/core/concurrency/unittests/bounded_concurrency_invoker_ut.cpp index dd8f6596551..afee3288996 100644 --- a/yt/yt/core/concurrency/unittests/bounded_concurrency_invoker_ut.cpp +++ b/yt/yt/core/concurrency/unittests/bounded_concurrency_invoker_ut.cpp @@ -167,7 +167,7 @@ TEST_P(TBoundedConcurrencyInvokerParametrizedReconfigureTest, SetMaxConcurrentIn auto guard = Guard(lock); auto concurrentInvocations = runnedCallbacks - finishedCallbacks; - THROW_ERROR_EXCEPTION_UNLESS(concurrentInvocations <= maxConcurrentInvocations, "Number of concurrent invocations exceeds maximum (ConcurrentInvocations: %v, MaxConcurrentInvocations: %v)", + THROW_ERROR_EXCEPTION_UNLESS(concurrentInvocations <= maxConcurrentInvocations, "Number of concurrent invocations %v exceeds maximum %v", concurrentInvocations, maxConcurrentInvocations); if (callbackIndex > maxConcurrentInvocations) { diff --git a/yt/yt/core/net/address.cpp b/yt/yt/core/net/address.cpp index 16eee1cdb5b..dfe29b605c7 100644 --- a/yt/yt/core/net/address.cpp +++ b/yt/yt/core/net/address.cpp @@ -1222,7 +1222,7 @@ TIP6Address TMtnAddress::ToIP6Address() const ui64 TMtnAddress::GetBytesRangeValue(int leftIndex, int rightIndex) const { if (leftIndex > rightIndex) { - THROW_ERROR_EXCEPTION("Left index is greater than right index (LeftIndex: %v, RightIndex: %v)", + THROW_ERROR_EXCEPTION("Left index %v is greater than right index %v", leftIndex, rightIndex); } @@ -1239,17 +1239,17 @@ ui64 TMtnAddress::GetBytesRangeValue(int leftIndex, int rightIndex) const void TMtnAddress::SetBytesRangeValue(int leftIndex, int rightIndex, ui64 value) { if (leftIndex > rightIndex) { - THROW_ERROR_EXCEPTION("Left index is greater than right index (LeftIndex: %v, RightIndex: %v)", + THROW_ERROR_EXCEPTION("Left index %v is greater than right index %v", leftIndex, rightIndex); } auto bytesInRange = rightIndex - leftIndex; if (value >= (1ull << (8 * bytesInRange))) { - THROW_ERROR_EXCEPTION("Value is too large to be set in [leftIndex; rightIndex) interval (LeftIndex: %v, RightIndex: %v, Value %v)", + THROW_ERROR_EXCEPTION("Value %v is too large to be set in interval [%v, %v)", + value, leftIndex, - rightIndex, - value); + rightIndex); } auto* addressBytes = Address_.GetRawBytes(); diff --git a/yt/yt/core/ytree/polymorphic_yson_struct-inl.h b/yt/yt/core/ytree/polymorphic_yson_struct-inl.h index 595f8fafd70..067956ad5db 100644 --- a/yt/yt/core/ytree/polymorphic_yson_struct-inl.h +++ b/yt/yt/core/ytree/polymorphic_yson_struct-inl.h @@ -236,7 +236,7 @@ void TPolymorphicYsonStruct::MergeWith(const TPolymorphicYsonStruct& o THROW_ERROR_EXCEPTION_UNLESS( GetType() == other.GetType(), - "Can't merge polymorphic yson structs with different types stored (ThisType: %v, OtherType: %v)", + "Cannot merge polymorphic YSON structs with different stored types: %Qlv vs %Qlv", GetType(), other.GetType()); diff --git a/yt/yt/library/formats/arrow_writer.cpp b/yt/yt/library/formats/arrow_writer.cpp index 54756b7cff6..713c23c602f 100644 --- a/yt/yt/library/formats/arrow_writer.cpp +++ b/yt/yt/library/formats/arrow_writer.cpp @@ -898,7 +898,9 @@ void SerializeDateColumn( }, [&] (auto value) { if (value > std::numeric_limits::max()) { - THROW_ERROR_EXCEPTION("Date value cannot be represented in arrow (Value: %v, MaxAllowedValue: %v)", value, std::numeric_limits::max()); + THROW_ERROR_EXCEPTION("Date value %v cannot be represented in arrow: maximum allowed value is %v", + value, + std::numeric_limits::max()); } *currentOutput++ = value; }); @@ -1007,7 +1009,7 @@ void SerializeTimestampColumn( }, [&] (auto value) { if (value > std::numeric_limits::max()) { - THROW_ERROR_EXCEPTION("Timestamp value cannot be represented in arrow (Value: %v, MaxAllowedValue: %v)", value, std::numeric_limits::max()); + THROW_ERROR_EXCEPTION("Timestamp value %v cannot be represented in arrow: maximum allowed value is %v", value, std::numeric_limits::max()); } *currentOutput++ = value; }); -- cgit v1.3 From fb24da1929620acd460b93c037f189ec4549a28b Mon Sep 17 00:00:00 2001 From: babenko Date: Sat, 25 Jul 2026 11:11:45 +0300 Subject: YT-28451: Migrate library/cpp/yt consumers to YT_TLOG_* API commit_hash:6ab9c84d85a668a8047593f61b5c9e1a7be60377 --- library/cpp/yt/error/error_code.cpp | 30 ++++++++++------------ .../stream/unittests/stream_log_manager_ut.cpp | 2 +- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/library/cpp/yt/error/error_code.cpp b/library/cpp/yt/error/error_code.cpp index d82d03214d4..ef9ce0f33c1 100644 --- a/library/cpp/yt/error/error_code.cpp +++ b/library/cpp/yt/error/error_code.cpp @@ -62,11 +62,10 @@ void TErrorCodeRegistry::RegisterErrorCode(int code, const TErrorCodeInfo& error if (code == 119) { return; } - YT_LOG_FATAL( - "Duplicate error code (Code: %v, StoredCodeInfo: %v, NewCodeInfo: %v)", - code, - CodeToInfo_[code], - errorCodeInfo); + YT_TLOG_FATAL("Duplicate error code") + .With("Code", code) + .With("StoredCodeInfo", CodeToInfo_[code]) + .With("NewCodeInfo", errorCodeInfo); } } @@ -91,11 +90,11 @@ void TErrorCodeRegistry::RegisterErrorCodeRange(int from, int to, std::string na TErrorCodeRangeInfo newRange{from, to, std::move(namespaceName), std::move(formatter)}; for (const auto& range : ErrorCodeRanges_) { - YT_LOG_FATAL_IF( + YT_TLOG_FATAL_IF( range.Intersects(newRange), - "Intersecting error code ranges registered (FirstRange: %v, SecondRange: %v)", - range, - newRange); + "Intersecting error code ranges registered") + .With("FirstRange", range) + .With("SecondRange", newRange); } ErrorCodeRanges_.push_back(std::move(newRange)); CheckCodesAgainstRanges(); @@ -105,14 +104,13 @@ void TErrorCodeRegistry::CheckCodesAgainstRanges() const { for (const auto& [code, info] : CodeToInfo_) { for (const auto& range : ErrorCodeRanges_) { - YT_LOG_FATAL_IF( + YT_TLOG_FATAL_IF( range.Contains(code), - "Error code range contains another registered code " - "(Range: %v, Code: %v, RangeCodeInfo: %v, StandaloneCodeInfo: %v)", - range, - code, - range.Get(code), - info); + "Error code range contains another registered code") + .With("Range", range) + .With("Code", code) + .With("RangeCodeInfo", range.Get(code)) + .With("StandaloneCodeInfo", info); } } } diff --git a/library/cpp/yt/logging/backends/stream/unittests/stream_log_manager_ut.cpp b/library/cpp/yt/logging/backends/stream/unittests/stream_log_manager_ut.cpp index cb3e244e3b9..95b2ade9b65 100644 --- a/library/cpp/yt/logging/backends/stream/unittests/stream_log_manager_ut.cpp +++ b/library/cpp/yt/logging/backends/stream/unittests/stream_log_manager_ut.cpp @@ -20,7 +20,7 @@ TEST(TStreamLogManagerTest, Simple) TStringOutput output(str); auto logManager = CreateStreamLogManager(&output); TLogger Logger(logManager.get(), "Test"); - YT_LOG_INFO("Hello world"); + YT_TLOG_INFO("Hello world"); } TVector tokens; -- cgit v1.3 From 1c03dcf702eb2f6ab1a973dea0261d17c60a043c Mon Sep 17 00:00:00 2001 From: robot-piglet Date: Sat, 25 Jul 2026 20:01:07 +0300 Subject: Intermediate changes commit_hash:c8058ee57bdfc4de3796360968134fe4fff5ee24 --- contrib/python/cffi/py3/.dist-info/METADATA | 15 +- .../python/cffi/py3/.dist-info/entry_points.txt | 3 + contrib/python/cffi/py3/README.md | 2 +- contrib/python/cffi/py3/c/_cffi_backend.c | 498 +++++---------------- contrib/python/cffi/py3/c/call_python.c | 17 +- contrib/python/cffi/py3/c/cdlopen.c | 13 +- contrib/python/cffi/py3/c/cffi1_module.c | 8 +- contrib/python/cffi/py3/c/cglob.c | 2 +- contrib/python/cffi/py3/c/commontypes.c | 2 +- contrib/python/cffi/py3/c/ffi_obj.c | 38 +- contrib/python/cffi/py3/c/file_emulator.h | 2 +- contrib/python/cffi/py3/c/lib_obj.c | 26 +- contrib/python/cffi/py3/c/minibuffer.h | 62 +-- contrib/python/cffi/py3/c/misc_thread_common.h | 10 +- contrib/python/cffi/py3/c/misc_win32.h | 73 +-- contrib/python/cffi/py3/c/realize_c_type.c | 9 +- contrib/python/cffi/py3/cffi/__init__.py | 4 +- contrib/python/cffi/py3/cffi/_cffi_errors.h | 14 - contrib/python/cffi/py3/cffi/_cffi_gen_src.py | 216 +++++++++ contrib/python/cffi/py3/cffi/_cffi_include.h | 26 +- contrib/python/cffi/py3/cffi/_embedding.h | 40 +- contrib/python/cffi/py3/cffi/api.py | 13 +- contrib/python/cffi/py3/cffi/backend_ctypes.py | 25 +- contrib/python/cffi/py3/cffi/cffi_opcode.py | 2 +- contrib/python/cffi/py3/cffi/cparser.py | 11 +- contrib/python/cffi/py3/cffi/gen_src.py | 5 + contrib/python/cffi/py3/cffi/model.py | 7 +- contrib/python/cffi/py3/cffi/pkgconfig.py | 2 +- contrib/python/cffi/py3/cffi/recompiler.py | 52 +-- contrib/python/cffi/py3/cffi/setuptools_ext.py | 10 +- contrib/python/cffi/py3/cffi/vengine_cpy.py | 43 +- contrib/python/cffi/py3/cffi/vengine_gen.py | 2 +- contrib/python/cffi/py3/cffi/verifier.py | 6 +- contrib/python/cffi/py3/patches/01-arcadia.patch | 14 +- contrib/python/cffi/py3/ya.make | 4 +- 35 files changed, 484 insertions(+), 792 deletions(-) create mode 100644 contrib/python/cffi/py3/cffi/_cffi_gen_src.py create mode 100644 contrib/python/cffi/py3/cffi/gen_src.py diff --git a/contrib/python/cffi/py3/.dist-info/METADATA b/contrib/python/cffi/py3/.dist-info/METADATA index 67508e56a53..6f3e2a45b7a 100644 --- a/contrib/python/cffi/py3/.dist-info/METADATA +++ b/contrib/python/cffi/py3/.dist-info/METADATA @@ -1,30 +1,29 @@ Metadata-Version: 2.4 Name: cffi -Version: 2.0.0 +Version: 2.1.0 Summary: Foreign Function Interface for Python calling C code. Author: Armin Rigo, Maciej Fijalkowski -Maintainer: Matt Davis, Matt Clay, Matti Picus -License-Expression: MIT +Maintainer: Matt Davis, Matt Clay +License-Expression: MIT-0 Project-URL: Documentation, https://cffi.readthedocs.io/ Project-URL: Changelog, https://cffi.readthedocs.io/en/latest/whatsnew.html -Project-URL: Downloads, https://github.com/python-cffi/cffi/releases +Project-URL: Download, https://github.com/python-cffi/cffi/releases Project-URL: Contact, https://groups.google.com/forum/#!forum/python-cffi Project-URL: Source Code, https://github.com/python-cffi/cffi Project-URL: Issue Tracker, https://github.com/python-cffi/cffi/issues Classifier: Programming Language :: Python Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3.9 Classifier: Programming Language :: Python :: 3.10 Classifier: Programming Language :: Python :: 3.11 Classifier: Programming Language :: Python :: 3.12 Classifier: Programming Language :: Python :: 3.13 Classifier: Programming Language :: Python :: 3.14 +Classifier: Programming Language :: Python :: 3.15 Classifier: Programming Language :: Python :: Free Threading :: 2 - Beta Classifier: Programming Language :: Python :: Implementation :: CPython -Requires-Python: >=3.9 +Requires-Python: >=3.10 Description-Content-Type: text/markdown License-File: LICENSE -License-File: AUTHORS Requires-Dist: pycparser; implementation_name != "PyPy" Dynamic: license-file @@ -59,7 +58,7 @@ Contact Testing/development tips ------------------------ -After `git clone` or `wget && tar`, we will get a directory called `cffi` or `cffi-x.x.x`. we call it `repo-directory`. To run tests under CPython, run the following in the `repo-directory`: +After `git clone` or `wget && tar`, we will get a directory called `cffi` or `cffi-x.x.x`. We call it `repo-directory`. To run tests under CPython, run the following in the `repo-directory`: pip install pytest pip install -e . # editable install of CFFI for local development diff --git a/contrib/python/cffi/py3/.dist-info/entry_points.txt b/contrib/python/cffi/py3/.dist-info/entry_points.txt index 4b0274f2333..46bb21b8418 100644 --- a/contrib/python/cffi/py3/.dist-info/entry_points.txt +++ b/contrib/python/cffi/py3/.dist-info/entry_points.txt @@ -1,2 +1,5 @@ +[console_scripts] +cffi-gen-src = cffi._cffi_gen_src:run + [distutils.setup_keywords] cffi_modules = cffi.setuptools_ext:cffi_modules diff --git a/contrib/python/cffi/py3/README.md b/contrib/python/cffi/py3/README.md index 723624d743d..fc7a235dc12 100644 --- a/contrib/python/cffi/py3/README.md +++ b/contrib/python/cffi/py3/README.md @@ -29,7 +29,7 @@ Contact Testing/development tips ------------------------ -After `git clone` or `wget && tar`, we will get a directory called `cffi` or `cffi-x.x.x`. we call it `repo-directory`. To run tests under CPython, run the following in the `repo-directory`: +After `git clone` or `wget && tar`, we will get a directory called `cffi` or `cffi-x.x.x`. We call it `repo-directory`. To run tests under CPython, run the following in the `repo-directory`: pip install pytest pip install -e . # editable install of CFFI for local development diff --git a/contrib/python/cffi/py3/c/_cffi_backend.c b/contrib/python/cffi/py3/c/_cffi_backend.c index 2aaccede0d1..74176b5746c 100644 --- a/contrib/python/cffi/py3/c/_cffi_backend.c +++ b/contrib/python/cffi/py3/c/_cffi_backend.c @@ -2,7 +2,7 @@ #include #include "structmember.h" #include "misc_thread_common.h" -#define CFFI_VERSION "2.0.0" +#define CFFI_VERSION "2.1.0" #ifdef MS_WIN32 #include @@ -142,69 +142,10 @@ #include "malloc_closure.h" -#if PY_MAJOR_VERSION >= 3 -# define STR_OR_BYTES "bytes" -# define PyText_Type PyUnicode_Type -# define PyText_Check PyUnicode_Check -# define PyTextAny_Check PyUnicode_Check -# define PyText_FromFormat PyUnicode_FromFormat -# define PyText_AsUTF8 PyUnicode_AsUTF8 -# define PyText_AS_UTF8 PyUnicode_AsUTF8 -# if PY_VERSION_HEX >= 0x03030000 -# define PyText_GetSize PyUnicode_GetLength -# else -# define PyText_GetSize PyUnicode_GetSize -# endif -# define PyText_FromString PyUnicode_FromString -# define PyText_FromStringAndSize PyUnicode_FromStringAndSize -# define PyText_InternInPlace PyUnicode_InternInPlace -# define PyText_InternFromString PyUnicode_InternFromString -# define PyIntOrLong_Check PyLong_Check -#else -# define STR_OR_BYTES "str" -# define PyText_Type PyString_Type -# define PyText_Check PyString_Check -# define PyTextAny_Check(op) (PyString_Check(op) || PyUnicode_Check(op)) -# define PyText_FromFormat PyString_FromFormat -# define PyText_AsUTF8 PyString_AsString -# define PyText_AS_UTF8 PyString_AS_STRING -# define PyText_GetSize PyString_Size -# define PyText_FromString PyString_FromString -# define PyText_FromStringAndSize PyString_FromStringAndSize -# define PyText_InternInPlace PyString_InternInPlace -# define PyText_InternFromString PyString_InternFromString -# define PyIntOrLong_Check(op) (PyInt_Check(op) || PyLong_Check(op)) -#endif - -#if PY_MAJOR_VERSION >= 3 -# define PyInt_FromLong PyLong_FromLong -# define PyInt_FromSsize_t PyLong_FromSsize_t -# define PyInt_AsSsize_t PyLong_AsSsize_t -# define PyInt_AsLong PyLong_AsLong -#endif - -#if PY_MAJOR_VERSION >= 3 -/* This is the default on Python3 and constant has been removed. */ -# define Py_TPFLAGS_CHECKTYPES 0 -#endif - -#if PY_MAJOR_VERSION < 3 -# undef PyCapsule_GetPointer -# undef PyCapsule_New -# define PyCapsule_GetPointer(capsule, name) \ - (PyCObject_AsVoidPtr(capsule)) -# define PyCapsule_New(pointer, name, destructor) \ - (PyCObject_FromVoidPtr(pointer, destructor)) -#endif - #if PY_VERSION_HEX < 0x030900a4 # define Py_SET_REFCNT(obj, val) (Py_REFCNT(obj) = (val)) #endif -#if PY_VERSION_HEX >= 0x03080000 -# define USE_WRITEUNRAISABLEMSG -#endif - #if PY_VERSION_HEX <= 0x030d00a1 static int PyDict_GetItemRef(PyObject *mp, PyObject *key, PyObject **result) { @@ -435,10 +376,7 @@ typedef struct { #endif #include "minibuffer.h" - -#if PY_MAJOR_VERSION >= 3 -# include "file_emulator.h" -#endif +#include "file_emulator.h" #ifdef PyUnicode_KIND /* Python >= 3.3 */ # include "wchar_helper_3.h" @@ -512,7 +450,7 @@ ctypedescr_new_on_top(CTypeDescrObject *ct_base, const char *extra_text, static PyObject * ctypedescr_repr(CTypeDescrObject *ct) { - return PyText_FromFormat("", ct->ct_name); + return PyUnicode_FromFormat("", ct->ct_name); } static void remove_dead_unique_reference(PyObject *unique_key); @@ -588,12 +526,12 @@ static PyObject *ctypeget_kind(CTypeDescrObject *ct, void *context) else result = "?"; - return PyText_FromString(result); + return PyUnicode_FromString(result); } static PyObject *ctypeget_cname(CTypeDescrObject *ct, void *context) { - return PyText_FromString(ct->ct_name); + return PyUnicode_FromString(ct->ct_name); } static PyObject *ctypeget_item(CTypeDescrObject *ct, void *context) @@ -609,7 +547,7 @@ static PyObject *ctypeget_length(CTypeDescrObject *ct, void *context) { if (ct->ct_flags & CT_ARRAY) { if (ct->ct_length >= 0) { - return PyInt_FromSsize_t(ct->ct_length); + return PyLong_FromSsize_t(ct->ct_length); } else { Py_INCREF(Py_None); @@ -763,7 +701,7 @@ ctypedescr_dir(PyObject *ct, PyObject *noarg) } else { Py_DECREF(x); - x = PyText_FromString(gsdef->name); + x = PyUnicode_FromString(gsdef->name); err = (x != NULL) ? PyList_Append(res, x) : -1; Py_XDECREF(x); if (err < 0) { @@ -897,12 +835,6 @@ _my_PyLong_AsLongLong(PyObject *ob) Like PyLong_AsLongLong(), this version accepts a Python int too, and does conversions from other types of objects. The difference is that this version refuses floats. */ -#if PY_MAJOR_VERSION < 3 - if (PyInt_Check(ob)) { - return PyInt_AS_LONG(ob); - } - else -#endif if (PyLong_Check(ob)) { return PyLong_AsLongLong(ob); } @@ -920,7 +852,7 @@ _my_PyLong_AsLongLong(PyObject *ob) if (io == NULL) return -1; - if (PyIntOrLong_Check(io)) { + if (PyLong_Check(io)) { res = _my_PyLong_AsLongLong(io); } else { @@ -940,15 +872,6 @@ _my_PyLong_AsUnsignedLongLong(PyObject *ob, int strict) does conversions from other types of objects. If 'strict', complains with OverflowError and refuses floats. If '!strict', rounds floats and masks the result. */ -#if PY_MAJOR_VERSION < 3 - if (PyInt_Check(ob)) { - long value1 = PyInt_AS_LONG(ob); - if (strict && value1 < 0) - goto negative; - return (unsigned PY_LONG_LONG)(PY_LONG_LONG)value1; - } - else -#endif if (PyLong_Check(ob)) { if (strict) { if (_PyLong_Sign(ob) < 0) @@ -973,7 +896,7 @@ _my_PyLong_AsUnsignedLongLong(PyObject *ob, int strict) if (io == NULL) return (unsigned PY_LONG_LONG)-1; - if (PyIntOrLong_Check(io)) { + if (PyLong_Check(io)) { res = _my_PyLong_AsUnsignedLongLong(io, strict); } else { @@ -1192,7 +1115,7 @@ convert_to_object(char *data, CTypeDescrObject *ct) /*READ(data, ct->ct_size)*/ value = read_raw_signed_data(data, ct->ct_size); if (ct->ct_flags & CT_PRIMITIVE_FITS_LONG) - return PyInt_FromLong((long)value); + return PyLong_FromLong((long)value); else return PyLong_FromLongLong(value); } @@ -1216,7 +1139,7 @@ convert_to_object(char *data, CTypeDescrObject *ct) Py_INCREF(x); return x; } - return PyInt_FromLong((long)value); + return PyLong_FromLong((long)value); } else return PyLong_FromUnsignedLongLong(value); @@ -1273,7 +1196,7 @@ convert_to_object_bitfield(char *data, CFieldObject *cf) result = ((PY_LONG_LONG)value) - (PY_LONG_LONG)shiftforsign; if (ct->ct_flags & CT_PRIMITIVE_FITS_LONG) - return PyInt_FromLong((long)result); + return PyLong_FromLong((long)result); else return PyLong_FromLongLong(result); } @@ -1285,7 +1208,7 @@ convert_to_object_bitfield(char *data, CFieldObject *cf) value = (value >> cf->cf_bitshift) & valuemask; if (ct->ct_flags & CT_PRIMITIVE_FITS_LONG) - return PyInt_FromLong((long)value); + return PyLong_FromLong((long)value); else return PyLong_FromUnsignedLongLong(value); } @@ -1300,7 +1223,7 @@ static int _convert_overflow(PyObject *init, const char *ct_name) if (s == NULL) return -1; PyErr_Format(PyExc_OverflowError, "integer %s does not fit '%s'", - PyText_AS_UTF8(s), ct_name); + PyUnicode_AsUTF8(s), ct_name); Py_DECREF(s); return -1; } @@ -1318,7 +1241,7 @@ static int _convert_to_char(PyObject *init) return *(unsigned char *)data; } PyErr_Format(PyExc_TypeError, - "initializer for ctype 'char' must be a "STR_OR_BYTES + "initializer for ctype 'char' must be a bytes" " of length 1, not %.200s", Py_TYPE(init)->tp_name); return -1; } @@ -1582,13 +1505,13 @@ convert_array_from_object(char *data, CTypeDescrObject *ct, PyObject *init) char *srcdata; Py_ssize_t n; if (!PyBytes_Check(init)) { - expected = STR_OR_BYTES" or list or tuple"; + expected = "bytes or list or tuple"; goto cannot_convert; } n = PyBytes_GET_SIZE(init); if (ct->ct_length >= 0 && n > ct->ct_length) { PyErr_Format(PyExc_IndexError, - "initializer "STR_OR_BYTES" is too long for '%s' " + "initializer bytes is too long for '%s' " "(got %zd characters)", ct->ct_name, n); return -1; } @@ -1923,9 +1846,9 @@ convert_from_object_bitfield(char *data, CFieldObject *cf, PyObject *init) PyErr_Format(PyExc_OverflowError, "value %s outside the range allowed by the " "bit field width: %s <= x <= %s", - PyText_AS_UTF8(svalue), - PyText_AS_UTF8(sfmin), - PyText_AS_UTF8(sfmax)); + PyUnicode_AsUTF8(svalue), + PyUnicode_AsUTF8(sfmin), + PyUnicode_AsUTF8(sfmax)); skip: Py_XDECREF(svalue); Py_XDECREF(sfmin); @@ -2181,9 +2104,9 @@ static PyObject *convert_cdata_to_enum_string(CDataObject *cd, int both) if (o == NULL) d_value = NULL; else { - d_value = PyText_FromFormat("%s: %s", - PyText_AS_UTF8(o), - PyText_AS_UTF8(d_value)); + d_value = PyUnicode_FromFormat("%s: %s", + PyUnicode_AsUTF8(o), + PyUnicode_AsUTF8(d_value)); Py_DECREF(o); } } @@ -2211,7 +2134,7 @@ static PyObject *cdata_repr(CDataObject *cd) /*READ(cd->c_data, sizeof(long double)*/ lvalue = read_raw_longdouble_data(cd->c_data); sprintf(buffer, "%LE", lvalue); - s = PyText_FromString(buffer); + s = PyUnicode_FromString(buffer); } else { PyObject *o = convert_to_object(cd->c_data, cd->c_type); @@ -2222,14 +2145,14 @@ static PyObject *cdata_repr(CDataObject *cd) } } else if ((cd->c_type->ct_flags & CT_ARRAY) && cd->c_type->ct_length < 0) { - s = PyText_FromFormat("sliced length %zd", get_array_length(cd)); + s = PyUnicode_FromFormat("sliced length %zd", get_array_length(cd)); } else { if (cd->c_data != NULL) { - s = PyText_FromFormat("%p", cd->c_data); + s = PyUnicode_FromFormat("%p", cd->c_data); } else - s = PyText_FromString("NULL"); + s = PyUnicode_FromString("NULL"); } if (s == NULL) return NULL; @@ -2240,9 +2163,9 @@ static PyObject *cdata_repr(CDataObject *cd) extra = " &"; else extra = ""; - result = PyText_FromFormat("", + result = PyUnicode_FromFormat("", cd->c_type->ct_name, extra, - PyText_AsUTF8(s)); + PyUnicode_AsUTF8(s)); Py_DECREF(s); return result; } @@ -2252,8 +2175,8 @@ static PyObject *_cdata_repr2(CDataObject *cd, char *text, PyObject *x) PyObject *res, *s = PyObject_Repr(x); if (s == NULL) return NULL; - res = PyText_FromFormat("", - cd->c_type->ct_name, text, PyText_AsUTF8(s)); + res = PyUnicode_FromFormat("", + cd->c_type->ct_name, text, PyUnicode_AsUTF8(s)); Py_DECREF(s); return res; } @@ -2280,7 +2203,7 @@ static PyObject *_frombuf_repr(CDataObject *cd, const char *cd_type_name) Py_buffer *view = ((CDataObject_frombuf *)cd)->bufferview; const char *obj_tp_name; if (view->obj == NULL) { - return PyText_FromFormat( + return PyUnicode_FromFormat( "", cd_type_name); } @@ -2289,7 +2212,7 @@ static PyObject *_frombuf_repr(CDataObject *cd, const char *cd_type_name) if (cd->c_type->ct_flags & CT_ARRAY) { Py_ssize_t buflen = get_array_length(cd); - return PyText_FromFormat( + return PyUnicode_FromFormat( "", cd_type_name, buflen, @@ -2297,7 +2220,7 @@ static PyObject *_frombuf_repr(CDataObject *cd, const char *cd_type_name) } else { - return PyText_FromFormat( + return PyUnicode_FromFormat( "", cd_type_name, obj_tp_name); @@ -2321,7 +2244,7 @@ static Py_ssize_t cdataowning_size_bytes(CDataObject *cd) static PyObject *cdataowning_repr(CDataObject *cd) { Py_ssize_t size = cdataowning_size_bytes(cd); - return PyText_FromFormat("", + return PyUnicode_FromFormat("", cd->c_type->ct_name, size); } @@ -2378,37 +2301,33 @@ static PyObject *cdata_int(CDataObject *cd) long value; /*READ(cd->c_data, cd->c_type->ct_size)*/ value = (long)read_raw_signed_data(cd->c_data, cd->c_type->ct_size); - return PyInt_FromLong(value); + return PyLong_FromLong(value); } if (cd->c_type->ct_flags & (CT_PRIMITIVE_SIGNED|CT_PRIMITIVE_UNSIGNED)) { PyObject *result = convert_to_object(cd->c_data, cd->c_type); if (result != NULL && PyBool_Check(result)) - result = PyInt_FromLong(PyInt_AsLong(result)); + result = PyLong_FromLong(PyLong_AsLong(result)); return result; } else if (cd->c_type->ct_flags & CT_PRIMITIVE_CHAR) { /*READ(cd->c_data, cd->c_type->ct_size)*/ switch (cd->c_type->ct_size) { case sizeof(char): - return PyInt_FromLong((unsigned char)cd->c_data[0]); + return PyLong_FromLong((unsigned char)cd->c_data[0]); case 2: - return PyInt_FromLong((long)*(cffi_char16_t *)cd->c_data); + return PyLong_FromLong((long)*(cffi_char16_t *)cd->c_data); case 4: if (cd->c_type->ct_flags & CT_IS_SIGNED_WCHAR) - return PyInt_FromLong((long)*(int32_t *)cd->c_data); + return PyLong_FromLong((long)*(int32_t *)cd->c_data); else if (sizeof(long) > 4) - return PyInt_FromLong(*(uint32_t *)cd->c_data); + return PyLong_FromLong(*(uint32_t *)cd->c_data); else return PyLong_FromUnsignedLong(*(uint32_t *)cd->c_data); } } else if (cd->c_type->ct_flags & CT_PRIMITIVE_FLOAT) { PyObject *o = cdata_float(cd); -#if PY_MAJOR_VERSION < 3 - PyObject *r = o ? PyNumber_Int(o) : NULL; -#else PyObject *r = o ? PyNumber_Long(o) : NULL; -#endif Py_XDECREF(o); return r; } @@ -2417,19 +2336,6 @@ static PyObject *cdata_int(CDataObject *cd) return NULL; } -#if PY_MAJOR_VERSION < 3 -static PyObject *cdata_long(CDataObject *cd) -{ - PyObject *res = cdata_int(cd); - if (res != NULL && PyInt_CheckExact(res)) { - PyObject *o = PyLong_FromLong(PyInt_AS_LONG(res)); - Py_DECREF(res); - res = o; - } - return res; -} -#endif - static PyObject *cdata_float(CDataObject *cd) { if (cd->c_type->ct_flags & CT_PRIMITIVE_FLOAT) { @@ -2526,10 +2432,6 @@ static PyObject *cdata_richcompare(PyObject *v, PyObject *w, int op) return pyres; } -#if PY_MAJOR_VERSION < 3 -typedef long Py_hash_t; -#endif - static Py_hash_t cdata_hash(PyObject *v) { if (((CDataObject *)v)->c_type->ct_flags & CT_PRIMITIVE_ANY) { @@ -2618,13 +2520,13 @@ _cdata_getslicearg(CDataObject *cd, PySliceObject *slice, Py_ssize_t bounds[]) Py_ssize_t start, stop; CTypeDescrObject *ct; - start = PyInt_AsSsize_t(slice->start); + start = PyLong_AsSsize_t(slice->start); if (start == -1 && PyErr_Occurred()) { if (slice->start == Py_None) PyErr_SetString(PyExc_IndexError, "slice start must be specified"); return NULL; } - stop = PyInt_AsSsize_t(slice->stop); + stop = PyLong_AsSsize_t(slice->stop); if (stop == -1 && PyErr_Occurred()) { if (slice->stop == Py_None) PyErr_SetString(PyExc_IndexError, "slice stop must be specified"); @@ -2706,6 +2608,11 @@ cdata_ass_slice(CDataObject *cd, PySliceObject *slice, PyObject *v) cdata = cd->c_data + itemsize * bounds[0]; length = bounds[1]; + if (v == NULL) { + PyErr_SetString(PyExc_TypeError, + "'del x[n]' not supported for cdata objects"); + return -1; + } if (CData_Check(v)) { CTypeDescrObject *ctv = ((CDataObject *)v)->c_type; if ((ctv->ct_flags & CT_ARRAY) && (ctv->ct_itemdescr == ct) && @@ -2926,11 +2833,7 @@ cdata_sub(PyObject *v, PyObject *w) } diff = diff / itemsize; } -#if PY_MAJOR_VERSION < 3 - return PyInt_FromSsize_t(diff); -#else return PyLong_FromSsize_t(diff); -#endif } return _cdata_add_or_sub(v, w, -1); @@ -2943,7 +2846,7 @@ _cdata_attr_errmsg(char *errmsg, CDataObject *cd, PyObject *attr) if (!PyErr_ExceptionMatches(PyExc_AttributeError)) return; PyErr_Clear(); - text = PyText_AsUTF8(attr); + text = PyUnicode_AsUTF8(attr); if (text == NULL) return; PyErr_Format(PyExc_AttributeError, errmsg, cd->c_type->ct_name, text); @@ -3233,11 +3136,7 @@ cdata_call(CDataObject *cd, PyObject *args, PyObject *kwds) } PyTuple_SET_ITEM(fvarargs, i, (PyObject *)ct); } -#if PY_MAJOR_VERSION < 3 - fabi = PyInt_AS_LONG(PyTuple_GET_ITEM(signature, 0)); -#else fabi = PyLong_AS_LONG(PyTuple_GET_ITEM(signature, 0)); -#endif cif_descr = fb_prepare_cif(fvarargs, fresult, nargs_declared, fabi); if (cif_descr == NULL) goto error; @@ -3460,9 +3359,6 @@ static PyNumberMethods CData_as_number = { (binaryfunc)cdata_add, /*nb_add*/ (binaryfunc)cdata_sub, /*nb_subtract*/ 0, /*nb_multiply*/ -#if PY_MAJOR_VERSION < 3 - 0, /*nb_divide*/ -#endif 0, /*nb_remainder*/ 0, /*nb_divmod*/ 0, /*nb_power*/ @@ -3476,15 +3372,8 @@ static PyNumberMethods CData_as_number = { 0, /*nb_and*/ 0, /*nb_xor*/ 0, /*nb_or*/ -#if PY_MAJOR_VERSION < 3 - 0, /*nb_coerce*/ -#endif (unaryfunc)cdata_int, /*nb_int*/ -#if PY_MAJOR_VERSION < 3 - (unaryfunc)cdata_long, /*nb_long*/ -#else - 0, -#endif + 0, /*nb_reserved*/ (unaryfunc)cdata_float, /*nb_float*/ 0, /*nb_oct*/ 0, /*nb_hex*/ @@ -3530,7 +3419,7 @@ static PyTypeObject CData_Type = { (getattrofunc)cdata_getattro, /* tp_getattro */ (setattrofunc)cdata_setattro, /* tp_setattro */ 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_CHECKTYPES, /* tp_flags */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ "The internal base type for CData objects. Use FFI.CData to access " "it. Always check with isinstance(): subtypes are sometimes returned " "on CPython, for performance reasons.", /* tp_doc */ @@ -3574,7 +3463,7 @@ static PyTypeObject CDataOwning_Type = { 0, /* inherited */ /* tp_getattro */ 0, /* inherited */ /* tp_setattro */ 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_CHECKTYPES, /* tp_flags */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ "This is an internal subtype of _CDataBase for performance only on " "CPython. Check with isinstance(x, ffi.CData).", /* tp_doc */ 0, /* tp_traverse */ @@ -3617,8 +3506,7 @@ static PyTypeObject CDataOwningGC_Type = { 0, /* inherited */ /* tp_getattro */ 0, /* inherited */ /* tp_setattro */ 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_CHECKTYPES /* tp_flags */ - | Py_TPFLAGS_HAVE_GC, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ "This is an internal subtype of _CDataBase for performance only on " "CPython. Check with isinstance(x, ffi.CData).", /* tp_doc */ (traverseproc)cdataowninggc_traverse, /* tp_traverse */ @@ -3661,8 +3549,7 @@ static PyTypeObject CDataFromBuf_Type = { 0, /* inherited */ /* tp_getattro */ 0, /* inherited */ /* tp_setattro */ 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_CHECKTYPES /* tp_flags */ - | Py_TPFLAGS_HAVE_GC, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ "This is an internal subtype of _CDataBase for performance only on " "CPython. Check with isinstance(x, ffi.CData).", /* tp_doc */ (traverseproc)cdatafrombuf_traverse, /* tp_traverse */ @@ -3705,7 +3592,7 @@ static PyTypeObject CDataGCP_Type = { 0, /* inherited */ /* tp_getattro */ 0, /* inherited */ /* tp_setattro */ 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_CHECKTYPES /* tp_flags */ + Py_TPFLAGS_DEFAULT /* tp_flags */ #ifdef Py_TPFLAGS_HAVE_FINALIZE | Py_TPFLAGS_HAVE_FINALIZE #endif @@ -4072,12 +3959,6 @@ _my_PyObject_AsBool(PyObject *ob) PyNumberMethods *nb; int res; -#if PY_MAJOR_VERSION < 3 - if (PyInt_Check(ob)) { - return PyInt_AS_LONG(ob) != 0; - } - else -#endif if (PyLong_Check(ob)) { return _PyLong_Sign(ob) != 0; } @@ -4111,7 +3992,7 @@ _my_PyObject_AsBool(PyObject *ob) if (io == NULL) return -1; - if (PyIntOrLong_Check(io) || PyFloat_Check(io)) { + if (PyLong_Check(io) || PyFloat_Check(io)) { res = _my_PyObject_AsBool(io); } else { @@ -4145,17 +4026,6 @@ static CDataObject *cast_to_integer_or_char(CTypeDescrObject *ct, PyObject *ob) (CT_POINTER|CT_FUNCTIONPTR|CT_ARRAY)) { value = (Py_intptr_t)((CDataObject *)ob)->c_data; } -#if PY_MAJOR_VERSION < 3 - else if (PyString_Check(ob)) { - if (PyString_GET_SIZE(ob) != 1) { - PyErr_Format(PyExc_TypeError, - "cannot cast string of length %zd to ctype '%s'", - PyString_GET_SIZE(ob), ct->ct_name); - return NULL; - } - value = (unsigned char)PyString_AS_STRING(ob)[0]; - } -#endif else if (PyUnicode_Check(ob)) { char err_buf[80]; cffi_char32_t ordinal; @@ -4421,7 +4291,7 @@ static void dl_dealloc(DynLibObject *dlobj) static PyObject *dl_repr(DynLibObject *dlobj) { - return PyText_FromFormat("", dlobj->dl_name); + return PyUnicode_FromFormat("", dlobj->dl_name); } static int dl_check_closed(DynLibObject *dlobj) @@ -4616,8 +4486,8 @@ static void *b_do_dlopen(PyObject *args, const char **p_printable_filename, PyErr_Format(PyExc_RuntimeError, "cannot call dlopen(NULL)"); return NULL; } - *p_temp = PyText_FromFormat("%p", handle); - *p_printable_filename = PyText_AsUTF8(*p_temp); + *p_temp = PyUnicode_FromFormat("%p", handle); + *p_printable_filename = PyUnicode_AsUTF8(*p_temp); *auto_close = 0; return handle; } @@ -4630,17 +4500,11 @@ static void *b_do_dlopen(PyObject *args, const char **p_printable_filename, { Py_ssize_t sz1; wchar_t *w1; -#if PY_MAJOR_VERSION < 3 - s = PyUnicode_AsUTF8String(s); - if (s == NULL) - return NULL; - *p_temp = s; -#endif - *p_printable_filename = PyText_AsUTF8(s); + *p_printable_filename = PyUnicode_AsUTF8(s); if (*p_printable_filename == NULL) return NULL; - sz1 = PyText_GetSize(filename_unicode) + 1; + sz1 = PyUnicode_GetLength(filename_unicode) + 1; sz1 *= 2; /* should not be needed, but you never know */ w1 = alloca(sizeof(wchar_t) * sz1); sz1 = PyUnicode_AsWideChar(filename_unicode, @@ -4656,18 +4520,7 @@ static void *b_do_dlopen(PyObject *args, const char **p_printable_filename, if (!PyArg_ParseTuple(args, "et|i:load_library", Py_FileSystemDefaultEncoding, &filename_or_null, &flags)) return NULL; -#if PY_MAJOR_VERSION < 3 - if (PyUnicode_Check(s)) - { - s = PyUnicode_AsUTF8String(s); - if (s == NULL) { - PyMem_Free(filename_or_null); - return NULL; - } - *p_temp = s; - } -#endif - *p_printable_filename = PyText_AsUTF8(s); + *p_printable_filename = PyUnicode_AsUTF8(s); if (*p_printable_filename == NULL) { PyMem_Free(filename_or_null); return NULL; @@ -5206,7 +5059,7 @@ _add_field(PyObject *interned_fields, PyObject *fname, CTypeDescrObject *ftype, cf->cf_flags = flags; Py_INCREF(fname); - PyText_InternInPlace(&fname); + PyUnicode_InternInPlace(&fname); prev_size = PyDict_Size(interned_fields); err = PyDict_SetItem(interned_fields, fname, (PyObject *)cf); Py_DECREF(fname); @@ -5216,7 +5069,7 @@ _add_field(PyObject *interned_fields, PyObject *fname, CTypeDescrObject *ftype, if (PyDict_Size(interned_fields) != prev_size + 1) { PyErr_Format(PyExc_KeyError, "duplicate field name '%s'", - PyText_AS_UTF8(fname)); + PyUnicode_AsUTF8(fname)); return NULL; } return cf; /* borrowed reference */ @@ -5303,7 +5156,7 @@ static PyObject *b_complete_struct_or_union_lock_held(CTypeDescrObject *ct, Py_ssize_t byteoffsetorg; CFieldObject **previous; int prev_bitfield_size, prev_bitfield_free; - PyObject *interned_fields; + PyObject *interned_fields = NULL; sflags = complete_sflags(sflags); if (sflags & SF_PACKED) @@ -5344,7 +5197,7 @@ static PyObject *b_complete_struct_or_union_lock_held(CTypeDescrObject *ct, Py_ssize_t foffset = -1; if (!PyArg_ParseTuple(PyList_GET_ITEM(fields, i), "O!O!|in:list item", - &PyText_Type, &fname, + &PyUnicode_Type, &fname, &CTypeDescr_Type, &ftype, &fbitsize, &foffset)) goto finally; @@ -5365,7 +5218,7 @@ static PyObject *b_complete_struct_or_union_lock_held(CTypeDescrObject *ct, else { PyErr_Format(PyExc_TypeError, "field '%s.%s' has ctype '%s' of unknown size", - ct->ct_name, PyText_AS_UTF8(fname), + ct->ct_name, PyUnicode_AsUTF8(fname), ftype->ct_name); goto finally; } @@ -5396,7 +5249,7 @@ static PyObject *b_complete_struct_or_union_lock_held(CTypeDescrObject *ct, if (!(sflags & SF_GCC_ARM_BITFIELDS) && fbitsize >= 0) { if (!(sflags & SF_MSVC_BITFIELDS)) { /* GCC: anonymous bitfields (of any size) don't cause alignment */ - do_align = PyText_GetSize(fname) > 0; + do_align = PyUnicode_GetLength(fname) > 0; } else { /* MSVC: zero-sized bitfields don't cause alignment */ @@ -5435,12 +5288,12 @@ static PyObject *b_complete_struct_or_union_lock_held(CTypeDescrObject *ct, except to know if we must set CT_CUSTOM_FIELD_POS */ if (detect_custom_layout(ct, sflags, byteoffset, foffset, "wrong offset for field '", - PyText_AS_UTF8(fname), "'") < 0) + PyUnicode_AsUTF8(fname), "'") < 0) goto finally; byteoffset = foffset; } - if (PyText_GetSize(fname) == 0 && + if (PyUnicode_GetLength(fname) == 0 && ftype->ct_flags & (CT_STRUCT|CT_UNION)) { /* a nested anonymous struct or union */ CFieldObject *cfsrc = (CFieldObject *)ftype->ct_extra; @@ -5482,7 +5335,7 @@ static PyObject *b_complete_struct_or_union_lock_held(CTypeDescrObject *ct, PyErr_Format(PyExc_TypeError, "field '%s.%s' is a bitfield, " "but a fixed offset is specified", - ct->ct_name, PyText_AS_UTF8(fname)); + ct->ct_name, PyUnicode_AsUTF8(fname)); goto finally; } @@ -5491,7 +5344,7 @@ static PyObject *b_complete_struct_or_union_lock_held(CTypeDescrObject *ct, CT_PRIMITIVE_CHAR))) { PyErr_Format(PyExc_TypeError, "field '%s.%s' declared as '%s' cannot be a bit field", - ct->ct_name, PyText_AS_UTF8(fname), + ct->ct_name, PyUnicode_AsUTF8(fname), ftype->ct_name); goto finally; } @@ -5499,7 +5352,7 @@ static PyObject *b_complete_struct_or_union_lock_held(CTypeDescrObject *ct, PyErr_Format(PyExc_TypeError, "bit field '%s.%s' is declared '%s:%d', which " "exceeds the width of the type", - ct->ct_name, PyText_AS_UTF8(fname), + ct->ct_name, PyUnicode_AsUTF8(fname), ftype->ct_name, fbitsize); goto finally; } @@ -5511,10 +5364,10 @@ static PyObject *b_complete_struct_or_union_lock_held(CTypeDescrObject *ct, field_offset_bytes &= ~(falign - 1); if (fbitsize == 0) { - if (PyText_GetSize(fname) > 0) { + if (PyUnicode_GetLength(fname) > 0) { PyErr_Format(PyExc_TypeError, "field '%s.%s' is declared with :0", - ct->ct_name, PyText_AS_UTF8(fname)); + ct->ct_name, PyUnicode_AsUTF8(fname)); goto finally; } if (!(sflags & SF_MSVC_BITFIELDS)) { @@ -5554,7 +5407,7 @@ static PyObject *b_complete_struct_or_union_lock_held(CTypeDescrObject *ct, PyErr_Format(PyExc_NotImplementedError, "with 'packed', gcc would compile field " "'%s.%s' to reuse some bits in the previous " - "field", ct->ct_name, PyText_AS_UTF8(fname)); + "field", ct->ct_name, PyUnicode_AsUTF8(fname)); goto finally; } field_offset_bytes += falign; @@ -5600,7 +5453,7 @@ static PyObject *b_complete_struct_or_union_lock_held(CTypeDescrObject *ct, if (sflags & SF_GCC_BIG_ENDIAN) bitshift = 8 * ftype->ct_size - fbitsize - bitshift; - if (PyText_GetSize(fname) > 0) { + if (PyUnicode_GetLength(fname) > 0) { *previous = _add_field(interned_fields, fname, ftype, field_offset_bytes, bitshift, fbitsize, @@ -6173,7 +6026,7 @@ static PyObject *new_function_type(PyObject *fargs, /* tuple */ fct->ct_stuff = PyTuple_New(2 + funcbuilder.nargs); if (fct->ct_stuff == NULL) goto error; - fabiobj = PyInt_FromLong(fabi); + fabiobj = PyLong_FromLong(fabi); if (fabiobj == NULL) goto error; PyTuple_SET_ITEM(fct->ct_stuff, 0, fabiobj); @@ -6284,7 +6137,6 @@ static void _my_PyErr_WriteUnraisable(PyObject *t, PyObject *v, PyObject *tb, char *extra_error_line) { /* like PyErr_WriteUnraisable(), but write a full traceback */ -#ifdef USE_WRITEUNRAISABLEMSG /* PyErr_WriteUnraisable actually writes the full traceback anyway from Python 3.4, but we can't really get the formatting of the @@ -6314,43 +6166,13 @@ static void _my_PyErr_WriteUnraisable(PyObject *t, PyObject *v, PyObject *tb, #if PY_VERSION_HEX >= 0x030D0000 PyErr_FormatUnraisable("Exception ignored %S", s); #else - _PyErr_WriteUnraisableMsg(PyText_AS_UTF8(s), NULL); + _PyErr_WriteUnraisableMsg(PyUnicode_AsUTF8(s), NULL); #endif Py_DECREF(s); } else PyErr_WriteUnraisable(obj); /* best effort */ PyErr_Clear(); - -#else - - /* version for Python 2.7 and < 3.8 */ - PyObject *f; -#if PY_MAJOR_VERSION >= 3 - /* jump through hoops to ensure the tb is attached to v, on Python 3 */ - PyErr_NormalizeException(&t, &v, &tb); - if (tb == NULL) { - tb = Py_None; - Py_INCREF(tb); - } - PyException_SetTraceback(v, tb); -#endif - f = PySys_GetObject("stderr"); - if (f != NULL) { - if (obj != NULL) { - PyFile_WriteString(objdescr, f); - PyFile_WriteObject(obj, f, 0); - PyFile_WriteString(":\n", f); - } - if (extra_error_line != NULL) - PyFile_WriteString(extra_error_line, f); - PyErr_Display(t, v, tb); - } - Py_XDECREF(t); - Py_XDECREF(v); - Py_XDECREF(tb); - -#endif } static void general_invoke_callback(int decode_args_from_libffi, @@ -6400,11 +6222,7 @@ static void general_invoke_callback(int decode_args_from_libffi, goto error; if (convert_from_object_fficallback(result, SIGNATURE(1), py_res, decode_args_from_libffi) < 0) { -#ifdef USE_WRITEUNRAISABLEMSG extra_error_line = ", trying to convert the result back to C"; -#else - extra_error_line = "Trying to convert the result back to C:\n"; -#endif goto error; } done: @@ -6456,16 +6274,9 @@ static void general_invoke_callback(int decode_args_from_libffi, _my_PyErr_WriteUnraisable(exc1, val1, tb1, "From cffi callback ", py_ob, extra_error_line); -#ifdef USE_WRITEUNRAISABLEMSG _my_PyErr_WriteUnraisable(exc2, val2, tb2, "during handling of the above exception by 'onerror'", NULL, NULL); -#else - extra_error_line = ("\nDuring the call to 'onerror', " - "another exception occurred:\n\n"); - _my_PyErr_WriteUnraisable(exc2, val2, tb2, - NULL, NULL, extra_error_line); -#endif _cffi_stop_error_capture(ecap); } } @@ -6533,14 +6344,6 @@ static PyObject *prepare_callback_info_tuple(CTypeDescrObject *ct, infotuple = Py_BuildValue("OOOO", ct, ob, py_rawerr, onerror_ob); Py_DECREF(py_rawerr); -#if defined(WITH_THREAD) && PY_VERSION_HEX < 0x03070000 - /* We must setup the GIL here, in case the callback is invoked in - some other non-Pythonic thread. This is the same as ctypes. - But PyEval_InitThreads() is always a no-op from CPython 3.7 - (the call from ctypes was removed some time later I think). */ - PyEval_InitThreads(); -#endif - return infotuple; } @@ -6714,19 +6517,7 @@ static PyObject *b_new_enum_type(PyObject *self, PyObject *args) PyObject *value = PyTuple_GET_ITEM(enumvalues, i); tmpkey = PyTuple_GET_ITEM(enumerators, i); Py_INCREF(tmpkey); - if (!PyText_Check(tmpkey)) { -#if PY_MAJOR_VERSION < 3 - if (PyUnicode_Check(tmpkey)) { - const char *text = PyText_AsUTF8(tmpkey); - if (text == NULL) - goto error; - Py_DECREF(tmpkey); - tmpkey = PyString_FromString(text); - if (tmpkey == NULL) - goto error; - } - else -#endif + if (!PyUnicode_Check(tmpkey)) { { PyErr_SetString(PyExc_TypeError, "enumerators must be a list of strings"); @@ -6782,7 +6573,7 @@ static PyObject *b_alignof(PyObject *self, PyObject *arg) align = get_alignment((CTypeDescrObject *)arg); if (align < 0) return NULL; - return PyInt_FromLong(align); + return PyLong_FromLong(align); } static Py_ssize_t direct_sizeof_cdata(CDataObject *cd) @@ -6820,7 +6611,7 @@ static PyObject *b_sizeof(PyObject *self, PyObject *arg) "expected a 'cdata' or 'ctype' object"); return NULL; } - return PyInt_FromSsize_t(size); + return PyLong_FromSsize_t(size); } static PyObject *b_typeof(PyObject *self, PyObject *arg) @@ -6844,7 +6635,7 @@ static CTypeDescrObject *direct_typeoffsetof(CTypeDescrObject *ct, CTypeDescrObject *res; CFieldObject *cf; - if (PyTextAny_Check(fieldname)) { + if (PyUnicode_Check(fieldname)) { if (!following && (ct->ct_flags & CT_POINTER)) ct = ct->ct_itemdescr; if (!(ct->ct_flags & (CT_STRUCT|CT_UNION))) { @@ -6871,7 +6662,7 @@ static CTypeDescrObject *direct_typeoffsetof(CTypeDescrObject *ct, *offset = cf->cf_offset; } else { - Py_ssize_t index = PyInt_AsSsize_t(fieldname); + Py_ssize_t index = PyLong_AsSsize_t(fieldname); if (index < 0 && PyErr_Occurred()) { PyErr_SetString(PyExc_TypeError, "field name or array index expected"); @@ -6961,7 +6752,7 @@ static PyObject *b_getcname(PyObject *self, PyObject *args) memcpy(p, ct->ct_name + ct->ct_name_position, namelen - ct->ct_name_position); - return PyText_FromStringAndSize(s, namelen + replacelen); + return PyUnicode_FromStringAndSize(s, namelen + replacelen); } static PyObject *b_string(PyObject *self, PyObject *args, PyObject *kwds) @@ -6985,7 +6776,7 @@ static PyObject *b_string(PyObject *self, PyObject *args, PyObject *kwds) if (s != NULL) { PyErr_Format(PyExc_RuntimeError, "cannot use string() on %s", - PyText_AS_UTF8(s)); + PyUnicode_AsUTF8(s)); Py_DECREF(s); } return NULL; @@ -7105,7 +6896,7 @@ static PyObject *b_unpack(PyObject *self, PyObject *args, PyObject *kwds) if (s != NULL) { PyErr_Format(PyExc_RuntimeError, "cannot use unpack() on %s", - PyText_AS_UTF8(s)); + PyUnicode_AsUTF8(s)); Py_DECREF(s); } return NULL; @@ -7193,13 +6984,13 @@ static PyObject *b_unpack(PyObject *self, PyObject *args, PyObject *kwds) default: x = convert_to_object(src, ctitem); break; /* special cases for performance only */ - case 0: x = PyInt_FromLong(*(signed char *)src); break; - case 1: x = PyInt_FromLong(*(short *)src); break; - case 2: x = PyInt_FromLong(*(int *)src); break; - case 3: x = PyInt_FromLong(*(long *)src); break; - case 4: x = PyInt_FromLong(*(unsigned char *)src); break; - case 5: x = PyInt_FromLong(*(unsigned short *)src); break; - case 6: x = PyInt_FromLong((long)*(unsigned int *)src); break; + case 0: x = PyLong_FromLong(*(signed char *)src); break; + case 1: x = PyLong_FromLong(*(short *)src); break; + case 2: x = PyLong_FromLong(*(int *)src); break; + case 3: x = PyLong_FromLong(*(long *)src); break; + case 4: x = PyLong_FromLong(*(unsigned char *)src); break; + case 5: x = PyLong_FromLong(*(unsigned short *)src); break; + case 6: x = PyLong_FromLong((long)*(unsigned int *)src); break; case 7: x = PyLong_FromUnsignedLong(*(unsigned long *)src); break; case 8: x = PyFloat_FromDouble(*(float *)src); break; case 9: x = PyFloat_FromDouble(*(double *)src); break; @@ -7283,12 +7074,12 @@ static PyObject *b_get_errno(PyObject *self, PyObject *noarg) restore_errno_only(); err = errno; errno = 0; - return PyInt_FromLong(err); + return PyLong_FromLong(err); } static PyObject *b_set_errno(PyObject *self, PyObject *arg) { - long ival = PyInt_AsLong(arg); + long ival = PyLong_AsLong(arg); if (ival == -1 && PyErr_Occurred()) return NULL; else if (ival < INT_MIN || ival > INT_MAX) { @@ -7370,46 +7161,6 @@ static PyObject *b_from_handle(PyObject *self, PyObject *arg) static int _my_PyObject_GetContiguousBuffer(PyObject *x, Py_buffer *view, int writable_only) { -#if PY_MAJOR_VERSION < 3 - /* Some objects only support the buffer interface and CPython doesn't - translate it into the memoryview interface, mess. Hack a very - minimal content for 'view'. Don't care if the other fields are - uninitialized: we only call PyBuffer_Release(), which only reads - 'view->obj'. */ - PyBufferProcs *pb = x->ob_type->tp_as_buffer; - if (pb && !pb->bf_releasebuffer) { - /* we used to try all three in some vaguely sensible order, - i.e. first the write. But trying to call the write on a - read-only buffer fails with TypeError. So we use a less- - sensible order now. See test_from_buffer_more_cases. - - If 'writable_only', we only try bf_getwritebuffer. - */ - readbufferproc proc = NULL; - if (!writable_only) { - proc = (readbufferproc)pb->bf_getreadbuffer; - if (!proc) - proc = (readbufferproc)pb->bf_getcharbuffer; - } - if (!proc) - proc = (readbufferproc)pb->bf_getwritebuffer; - - if (proc && pb->bf_getsegcount) { - if ((*pb->bf_getsegcount)(x, NULL) != 1) { - PyErr_SetString(PyExc_TypeError, - "expected a single-segment buffer object"); - return -1; - } - view->len = (*proc)(x, 0, &view->buf); - if (view->len < 0) - return -1; - view->obj = x; - Py_INCREF(x); - return 0; - } - } -#endif - if (PyObject_GetBuffer(x, view, writable_only ? PyBUF_WRITABLE : PyBUF_SIMPLE) < 0) return -1; @@ -7857,30 +7608,6 @@ static PyObject *b__testfunc(PyObject *self, PyObject *args) return PyLong_FromVoidPtr(f); } -#if PY_MAJOR_VERSION < 3 -static Py_ssize_t _test_segcountproc(PyObject *o, Py_ssize_t *ignored) -{ - return 1; -} -static Py_ssize_t _test_getreadbuf(PyObject *o, Py_ssize_t i, void **r) -{ - static char buf[] = "RDB"; - *r = buf; - return 3; -} -static Py_ssize_t _test_getwritebuf(PyObject *o, Py_ssize_t i, void **r) -{ - static char buf[] = "WRB"; - *r = buf; - return 3; -} -static Py_ssize_t _test_getcharbuf(PyObject *o, Py_ssize_t i, char **r) -{ - static char buf[] = "CHB"; - *r = buf; - return 3; -} -#endif static int _test_getbuf(PyObject *self, Py_buffer *view, int flags) { static char buf[] = "GTB"; @@ -7903,14 +7630,6 @@ static PyObject *b__testbuff(PyObject *self, PyObject *args) assert(obj->tp_as_buffer != NULL); -#if PY_MAJOR_VERSION < 3 - obj->tp_as_buffer->bf_getsegcount = &_test_segcountproc; - obj->tp_flags |= Py_TPFLAGS_HAVE_GETCHARBUFFER; - obj->tp_flags |= Py_TPFLAGS_HAVE_NEWBUFFER; - if (methods & 1) obj->tp_as_buffer->bf_getreadbuffer = &_test_getreadbuf; - if (methods & 2) obj->tp_as_buffer->bf_getwritebuffer = &_test_getwritebuf; - if (methods & 4) obj->tp_as_buffer->bf_getcharbuffer = &_test_getcharbuf; -#endif if (methods & 8) obj->tp_as_buffer->bf_getbuffer = &_test_getbuf; if (methods & 16) obj->tp_as_buffer->bf_getbuffer = &_test_getbuf_ro; @@ -8052,7 +7771,7 @@ static PyObject *_cffi_get_struct_layout(Py_ssize_t nums[]) return NULL; while (--count >= 0) { - PyObject *o = PyInt_FromSsize_t(nums[count]); + PyObject *o = PyLong_FromSsize_t(nums[count]); if (o == NULL) { Py_DECREF(result); return NULL; @@ -8173,7 +7892,6 @@ static struct { const char *name; int value; } all_dlopen_flags[] = { /************************************************************/ -#if PY_MAJOR_VERSION >= 3 static struct PyModuleDef FFIBackendModuleDef = { PyModuleDef_HEAD_INIT, "_cffi_backend", @@ -8186,12 +7904,6 @@ static struct PyModuleDef FFIBackendModuleDef = { PyMODINIT_FUNC PyInit__cffi_backend(void) -#else -#define INITERROR return - -PyMODINIT_FUNC -init_cffi_backend(void) -#endif { PyObject *m, *v; int i; @@ -8214,19 +7926,15 @@ init_cffi_backend(void) }; v = PySys_GetObject("version"); - if (v == NULL || !PyText_Check(v) || - strncmp(PyText_AS_UTF8(v), PY_VERSION, 3) != 0) { + if (v == NULL || !PyUnicode_Check(v) || + strncmp(PyUnicode_AsUTF8(v), PY_VERSION, 3) != 0) { PyErr_Format(PyExc_ImportError, "this module was compiled for Python %c%c%c", PY_VERSION[0], PY_VERSION[1], PY_VERSION[2]); INITERROR; } -#if PY_MAJOR_VERSION >= 3 m = PyModule_Create(&FFIBackendModuleDef); -#else - m = Py_InitModule("_cffi_backend", FFIBackendMethods); -#endif if (m == NULL) INITERROR; @@ -8259,11 +7967,11 @@ init_cffi_backend(void) } if (!init_done) { - v = PyText_FromString("_cffi_backend"); + v = PyUnicode_FromString("_cffi_backend"); if (v == NULL || PyDict_SetItemString(CData_Type.tp_dict, "__module__", v) < 0) INITERROR; - v = PyText_FromString(""); + v = PyUnicode_FromString(""); if (v == NULL || PyDict_SetItemString(CData_Type.tp_dict, "__name__", v) < 0) INITERROR; @@ -8275,7 +7983,7 @@ init_cffi_backend(void) if (v == NULL || PyModule_AddObject(m, "_C_API", v) < 0) INITERROR; - v = PyText_FromString(CFFI_VERSION); + v = PyUnicode_FromString(CFFI_VERSION); if (v == NULL || PyModule_AddObject(m, "__version__", v) < 0) INITERROR; @@ -8312,9 +8020,7 @@ init_cffi_backend(void) if (init_ffi_lib(m) < 0) INITERROR; -#if PY_MAJOR_VERSION >= 3 if (init_file_emulator() < 0) INITERROR; return m; -#endif } diff --git a/contrib/python/cffi/py3/c/call_python.c b/contrib/python/cffi/py3/c/call_python.c index 8c6703cc26b..57e5f31402f 100644 --- a/contrib/python/cffi/py3/c/call_python.c +++ b/contrib/python/cffi/py3/c/call_python.c @@ -1,16 +1,7 @@ -#if PY_VERSION_HEX >= 0x03080000 -# define HAVE_PYINTERPSTATE_GETDICT -#endif - - static PyObject *_current_interp_key(void) { PyInterpreterState *interp = PyThreadState_GET()->interp; -#ifdef HAVE_PYINTERPSTATE_GETDICT return PyInterpreterState_GetDict(interp); /* shared reference */ -#else - return interp->modules; -#endif } static PyObject *_get_interpstate_dict(void) @@ -33,11 +24,7 @@ static PyObject *_get_interpstate_dict(void) } interp = tstate->interp; -#ifdef HAVE_PYINTERPSTATE_GETDICT interpdict = PyInterpreterState_GetDict(interp); /* shared reference */ -#else - interpdict = interp->builtins; -#endif if (interpdict == NULL) { /* subinterpreter was cleared already, or is being cleared right now, to a point that is too much for us to continue */ @@ -47,7 +34,7 @@ static PyObject *_get_interpstate_dict(void) /* from there on, we know the (sub-)interpreter is still valid */ if (attr_name == NULL) { - attr_name = PyText_InternFromString("__cffi_backend_extern_py"); + attr_name = PyUnicode_InternFromString("__cffi_backend_extern_py"); if (attr_name == NULL) goto error; } @@ -90,7 +77,7 @@ static PyObject *_ffi_def_extern_decorator(PyObject *outer_args, PyObject *fn) name = PyObject_GetAttrString(fn, "__name__"); if (name == NULL) return NULL; - s = PyText_AsUTF8(name); + s = PyUnicode_AsUTF8(name); if (s == NULL) { Py_DECREF(name); return NULL; diff --git a/contrib/python/cffi/py3/c/cdlopen.c b/contrib/python/cffi/py3/c/cdlopen.c index 7db7ae280fb..d07fcd25ae6 100644 --- a/contrib/python/cffi/py3/c/cdlopen.c +++ b/contrib/python/cffi/py3/c/cdlopen.c @@ -7,7 +7,7 @@ static void *cdlopen_fetch(PyObject *libname, void *libhandle, if (libhandle == NULL) { PyErr_Format(FFIError, "library '%s' has been closed", - PyText_AS_UTF8(libname)); + PyUnicode_AsUTF8(libname)); return NULL; } @@ -16,7 +16,7 @@ static void *cdlopen_fetch(PyObject *libname, void *libhandle, if (address == NULL) { const char *error = dlerror(); PyErr_Format(FFIError, "symbol '%s' not found in library '%s': %s", - symbol, PyText_AS_UTF8(libname), error); + symbol, PyUnicode_AsUTF8(libname), error); } return address; } @@ -32,7 +32,7 @@ static int cdlopen_close(PyObject *libname, void *libhandle) if (libhandle != NULL && dlclose(libhandle) != 0) { const char *error = dlerror(); PyErr_Format(FFIError, "closing library '%s': %s", - PyText_AS_UTF8(libname), error); + PyUnicode_AsUTF8(libname), error); return -1; } return 0; @@ -195,13 +195,6 @@ static int ffiobj_init(PyObject *self, PyObject *args, PyObject *kwds) _CFFI_GETOP(nglobs[i].type_op) == _CFFI_OP_ENUM) { PyObject *o = PyTuple_GET_ITEM(globals, i * 2 + 1); nglobs[i].address = &_cdl_realize_global_int; -#if PY_MAJOR_VERSION < 3 - if (PyInt_Check(o)) { - nintconsts[i].neg = PyInt_AS_LONG(o) <= 0; - nintconsts[i].value = (long long)PyInt_AS_LONG(o); - } - else -#endif { nintconsts[i].neg = PyObject_RichCompareBool(o, Py_False, Py_LE); diff --git a/contrib/python/cffi/py3/c/cffi1_module.c b/contrib/python/cffi/py3/c/cffi1_module.c index 70d9ee8e0fe..7c16b2b7073 100644 --- a/contrib/python/cffi/py3/c/cffi1_module.c +++ b/contrib/python/cffi/py3/c/cffi1_module.c @@ -46,7 +46,7 @@ static int init_ffi_lib(PyObject *m) return -1; for (i = 0; all_dlopen_flags[i].name != NULL; i++) { - x = PyInt_FromLong(all_dlopen_flags[i].value); + x = PyLong_FromLong(all_dlopen_flags[i].value); if (x == NULL) return -1; res = PyDict_SetItemString(FFI_Type.tp_dict, @@ -116,7 +116,6 @@ static int make_included_tuples(char *module_name, static PyObject *_my_Py_InitModule(char *module_name) { -#if PY_MAJOR_VERSION >= 3 struct PyModuleDef *module_def, local_module_def = { PyModuleDef_HEAD_INIT, module_name, @@ -137,9 +136,6 @@ static PyObject *_my_Py_InitModule(char *module_name) } # endif return m; -#else - return Py_InitModule(module_name, NULL); -#endif } static PyObject *b_init_cffi_1_0_external_module(PyObject *self, PyObject *arg) @@ -211,12 +207,10 @@ static PyObject *b_init_cffi_1_0_external_module(PyObject *self, PyObject *arg) (PyObject *)lib) < 0) return NULL; -#if PY_MAJOR_VERSION >= 3 /* add manually 'module_name' in sys.modules: it seems that Py_InitModule() is not enough to do that */ if (PyDict_SetItemString(modules_dict, module_name, m) < 0) return NULL; -#endif return m; } diff --git a/contrib/python/cffi/py3/c/cglob.c b/contrib/python/cffi/py3/c/cglob.c index e97767c9d75..6c6fd4bae2e 100644 --- a/contrib/python/cffi/py3/c/cglob.c +++ b/contrib/python/cffi/py3/c/cglob.c @@ -74,7 +74,7 @@ static void *fetch_global_var_addr(GlobSupportObject *gs) } if (data == NULL) { PyErr_Format(FFIError, "global variable '%s' is at address NULL", - PyText_AS_UTF8(gs->gs_name)); + PyUnicode_AsUTF8(gs->gs_name)); return NULL; } return data; diff --git a/contrib/python/cffi/py3/c/commontypes.c b/contrib/python/cffi/py3/c/commontypes.c index 2337bf99447..a15c3ae87b2 100644 --- a/contrib/python/cffi/py3/c/commontypes.c +++ b/contrib/python/cffi/py3/c/commontypes.c @@ -205,7 +205,7 @@ static PyObject *b__get_common_types(PyObject *self, PyObject *arg) size_t i; for (i = 0; i < num_common_simple_types; i++) { const char *s = common_simple_types[i]; - PyObject *o = PyText_FromString(s + strlen(s) + 1); + PyObject *o = PyUnicode_FromString(s + strlen(s) + 1); if (o == NULL) return NULL; err = PyDict_SetItemString(arg, s, o); diff --git a/contrib/python/cffi/py3/c/ffi_obj.c b/contrib/python/cffi/py3/c/ffi_obj.c index 0c4933eeff6..fbba28941ce 100644 --- a/contrib/python/cffi/py3/c/ffi_obj.c +++ b/contrib/python/cffi/py3/c/ffi_obj.c @@ -185,13 +185,13 @@ static CTypeDescrObject *_ffi_type(FFIObject *ffi, PyObject *arg, /* Returns the CTypeDescrObject from the user-supplied 'arg'. Does not return a new reference! */ - if ((accept & ACCEPT_STRING) && PyText_Check(arg)) { + if ((accept & ACCEPT_STRING) && PyUnicode_Check(arg)) { PyObject *types_dict = ffi->types_builder.types_dict; /* The types_dict keeps the reference alive. Items are never removed */ PyObject *x = PyDict_GetItem(types_dict, arg); if (x == NULL) { - const char *input_text = PyText_AS_UTF8(arg); + const char *input_text = PyUnicode_AsUTF8(arg); struct _cffi_parse_info_s info; info.ctx = &ffi->types_builder.ctx; info.output_size = FFI_COMPLEXITY_OUTPUT; @@ -241,17 +241,6 @@ static CTypeDescrObject *_ffi_type(FFIObject *ffi, PyObject *arg, else if ((accept & ACCEPT_CDATA) && CData_Check(arg)) { return ((CDataObject *)arg)->c_type; } -#if PY_MAJOR_VERSION < 3 - else if (PyUnicode_Check(arg)) { - CTypeDescrObject *result; - arg = PyUnicode_AsASCIIString(arg); - if (arg == NULL) - return NULL; - result = _ffi_type(ffi, arg, accept); - Py_DECREF(arg); - return result; - } -#endif else { const char *m1 = (accept & ACCEPT_STRING) ? "string" : ""; const char *m2 = (accept & ACCEPT_CTYPE) ? "ctype object" : ""; @@ -291,7 +280,7 @@ static PyObject *ffi_sizeof(FFIObject *self, PyObject *arg) return NULL; } } - return PyInt_FromSsize_t(size); + return PyLong_FromSsize_t(size); } PyDoc_STRVAR(ffi_alignof_doc, @@ -308,7 +297,7 @@ static PyObject *ffi_alignof(FFIObject *self, PyObject *arg) align = get_alignment(ct); if (align < 0) return NULL; - return PyInt_FromLong(align); + return PyLong_FromLong(align); } PyDoc_STRVAR(ffi_typeof_doc, @@ -526,7 +515,7 @@ static PyObject *ffi_offsetof(FFIObject *self, PyObject *args) return NULL; offset += ofs1; } - return PyInt_FromSsize_t(offset); + return PyLong_FromSsize_t(offset); } PyDoc_STRVAR(ffi_addressof_doc, @@ -639,9 +628,7 @@ static PyObject *ffi_getctype(FFIObject *self, PyObject *args, PyObject *kwds) CTypeDescrObject *ct; size_t replace_with_len; static char *keywords[] = {"cdecl", "replace_with", NULL}; -#if PY_MAJOR_VERSION >= 3 PyObject *u; -#endif if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|s:getctype", keywords, &c_decl, &replace_with)) @@ -675,14 +662,12 @@ static PyObject *ffi_getctype(FFIObject *self, PyObject *args, PyObject *kwds) if (add_paren) p[replace_with_len] = ')'; -#if PY_MAJOR_VERSION >= 3 /* bytes -> unicode string */ u = PyUnicode_DecodeLatin1(PyBytes_AS_STRING(res), PyBytes_GET_SIZE(res), NULL); Py_DECREF(res); res = u; -#endif return res; } @@ -931,7 +916,7 @@ static PyObject *ffi_list_types(FFIObject *self, PyObject *noargs) goto error; for (i = 0; i < n1; i++) { - o = PyText_FromString(self->types_builder.ctx.typenames[i].name); + o = PyUnicode_FromString(self->types_builder.ctx.typenames[i].name); if (o == NULL) goto error; PyList_SET_ITEM(lst[0], i, o); @@ -945,7 +930,7 @@ static PyObject *ffi_list_types(FFIObject *self, PyObject *noargs) if (s->name[0] == '$') continue; - o = PyText_FromString(s->name); + o = PyUnicode_FromString(s->name); if (o == NULL) goto error; index = (s->flags & _CFFI_F_UNION) ? 2 : 1; @@ -990,14 +975,6 @@ PyDoc_STRVAR(ffi_init_once_doc, "of function() is done. If function() raises an exception, it is\n" "propagated and nothing is cached."); -#if PY_MAJOR_VERSION < 3 -/* PyCapsule_New is redefined to be PyCObject_FromVoidPtr in _cffi_backend, - which gives 2.6 compatibility; but the destructor signature is different */ -static void _free_init_once_lock(void *lock) -{ - PyThread_free_lock((PyThread_type_lock)lock); -} -#else static void _free_init_once_lock(PyObject *capsule) { PyThread_type_lock lock; @@ -1005,7 +982,6 @@ static void _free_init_once_lock(PyObject *capsule) if (lock != NULL) PyThread_free_lock(lock); } -#endif static PyObject *ffi_init_once(FFIObject *self, PyObject *args, PyObject *kwds) { diff --git a/contrib/python/cffi/py3/c/file_emulator.h b/contrib/python/cffi/py3/c/file_emulator.h index 82a34c0c72f..bfdcdeceac2 100644 --- a/contrib/python/cffi/py3/c/file_emulator.h +++ b/contrib/python/cffi/py3/c/file_emulator.h @@ -51,7 +51,7 @@ static FILE *PyFile_AsFile(PyObject *ob_file) ob_mode = PyObject_GetAttrString(ob_file, "mode"); if (ob_mode == NULL) goto fail; - mode = PyText_AsUTF8(ob_mode); + mode = PyUnicode_AsUTF8(ob_mode); if (mode == NULL) goto fail; diff --git a/contrib/python/cffi/py3/c/lib_obj.c b/contrib/python/cffi/py3/c/lib_obj.c index 446b225cb5d..946db3b39dd 100644 --- a/contrib/python/cffi/py3/c/lib_obj.c +++ b/contrib/python/cffi/py3/c/lib_obj.c @@ -123,8 +123,8 @@ static int lib_traverse(LibObject *lib, visitproc visit, void *arg) static PyObject *lib_repr(LibObject *lib) { - return PyText_FromFormat("", - PyText_AS_UTF8(lib->l_libname)); + return PyUnicode_FromFormat("", + PyUnicode_AsUTF8(lib->l_libname)); } static PyObject *lib_build_cpython_func(LibObject *lib, @@ -143,7 +143,7 @@ static PyObject *lib_build_cpython_func(LibObject *lib, int i, type_index = _CFFI_GETARG(g->type_op); _cffi_opcode_t *opcodes = lib->l_types_builder->ctx.types; static const char *const format = ";\n\nCFFI C function from %s.lib"; - const char *libname = PyText_AS_UTF8(lib->l_libname); + const char *libname = PyUnicode_AsUTF8(lib->l_libname); struct funcbuilder_s funcbuilder; /* return type: */ @@ -226,7 +226,7 @@ static PyObject *lib_build_and_cache_attr(LibObject *lib, PyObject *name, const struct _cffi_global_s *g; CTypeDescrObject *ct; builder_c_t *types_builder = lib->l_types_builder; - const char *s = PyText_AsUTF8(name); + const char *s = PyUnicode_AsUTF8(name); if (s == NULL) return NULL; @@ -281,7 +281,7 @@ static PyObject *lib_build_and_cache_attr(LibObject *lib, PyObject *name, PyErr_Format(PyExc_AttributeError, "cffi library '%.200s' has no function, constant " "or global variable named '%.200s'", - PyText_AS_UTF8(lib->l_libname), s); + PyUnicode_AsUTF8(lib->l_libname), s); return NULL; } @@ -478,7 +478,7 @@ static PyObject *_lib_dir1(LibObject *lib, int ignore_global_vars) if (op == _CFFI_OP_GLOBAL_VAR || op == _CFFI_OP_GLOBAL_VAR_F) continue; } - s = PyText_FromString(g[i].name); + s = PyUnicode_FromString(g[i].name); if (s == NULL) goto error; PyList_SET_ITEM(lst, count, s); @@ -502,7 +502,7 @@ static PyObject *_lib_dict(LibObject *lib) return NULL; for (i = 0; i < total; i++) { - name = PyText_FromString(g[i].name); + name = PyUnicode_FromString(g[i].name); if (name == NULL) goto error; @@ -534,7 +534,7 @@ static PyObject *lib_getattr(LibObject *lib, PyObject *name) missing: /*** ATTRIBUTEERROR IS SET HERE ***/ - p = PyText_AsUTF8(name); + p = PyUnicode_AsUTF8(name); if (p == NULL) return NULL; if (strcmp(p, "__all__") == 0) { @@ -558,16 +558,14 @@ static PyObject *lib_getattr(LibObject *lib, PyObject *name) module-like behavior */ if (strcmp(p, "__name__") == 0) { PyErr_Clear(); - return PyText_FromFormat("%s.lib", PyText_AS_UTF8(lib->l_libname)); + return PyUnicode_FromFormat("%s.lib", PyUnicode_AsUTF8(lib->l_libname)); } -#if PY_MAJOR_VERSION >= 3 if (strcmp(p, "__loader__") == 0 || strcmp(p, "__spec__") == 0) { /* some more module-like behavior hacks */ PyErr_Clear(); Py_INCREF(Py_None); return Py_None; } -#endif return NULL; } @@ -587,7 +585,7 @@ static int lib_setattr(LibObject *lib, PyObject *name, PyObject *val) PyErr_Format(PyExc_AttributeError, "cannot write to function or constant '%.200s'", - PyText_Check(name) ? PyText_AS_UTF8(name) : "?"); + PyUnicode_Check(name) ? PyUnicode_AsUTF8(name) : "?"); return -1; } @@ -645,7 +643,7 @@ static LibObject *lib_internal_new(FFIObject *ffi, const char *module_name, LibObject *lib; PyObject *libname, *dict; - libname = PyText_FromString(module_name); + libname = PyUnicode_FromString(module_name); if (libname == NULL) goto err1; @@ -712,7 +710,7 @@ static PyObject *address_of_global_var(PyObject *args) /* rebuild a string from 'varname', to do typechecks and to force a unicode back to a plain string (on python 2) */ - o_varname = PyText_FromString(varname); + o_varname = PyUnicode_FromString(varname); if (o_varname == NULL) return NULL; diff --git a/contrib/python/cffi/py3/c/minibuffer.h b/contrib/python/cffi/py3/c/minibuffer.h index d92da317ade..89e400fa069 100644 --- a/contrib/python/cffi/py3/c/minibuffer.h +++ b/contrib/python/cffi/py3/c/minibuffer.h @@ -50,7 +50,7 @@ static int mb_ass_item(MiniBufferObj *self, Py_ssize_t idx, PyObject *other) } else { PyErr_Format(PyExc_TypeError, - "must assign a "STR_OR_BYTES + "must assign a bytes" " of length 1, not %.200s", Py_TYPE(other)->tp_name); return -1; } @@ -85,29 +85,6 @@ static int mb_ass_slice(MiniBufferObj *self, return 0; } -#if PY_MAJOR_VERSION < 3 -static Py_ssize_t mb_getdata(MiniBufferObj *self, Py_ssize_t idx, void **pp) -{ - *pp = self->mb_data; - return self->mb_size; -} - -static Py_ssize_t mb_getsegcount(MiniBufferObj *self, Py_ssize_t *lenp) -{ - if (lenp) - *lenp = self->mb_size; - return 1; -} - -static PyObject *mb_str(MiniBufferObj *self) -{ - /* Python 2: we want str(buffer) to behave like buffer[:], because - that's what bytes(buffer) does on Python 3 and there is no way - we can prevent this. */ - return PyString_FromStringAndSize(self->mb_data, self->mb_size); -} -#endif - static int mb_getbuf(MiniBufferObj *self, Py_buffer *view, int flags) { return PyBuffer_FillInfo(view, (PyObject *)self, @@ -126,12 +103,6 @@ static PySequenceMethods mb_as_sequence = { }; static PyBufferProcs mb_as_buffer = { -#if PY_MAJOR_VERSION < 3 - (readbufferproc)mb_getdata, - (writebufferproc)mb_getdata, - (segcountproc)mb_getsegcount, - (charbufferproc)mb_getdata, -#endif (getbufferproc)mb_getbuf, (releasebufferproc)0, }; @@ -234,16 +205,8 @@ mb_richcompare(PyObject *self, PyObject *other, int op) return res; } -#if PY_MAJOR_VERSION >= 3 /* pfffffffffffff pages of copy-paste from listobject.c */ -/* pfffffffffffff#2: the PySlice_GetIndicesEx() *macro* should not - be called, because C extension modules compiled with it differ - on ABI between 3.6.0, 3.6.1 and 3.6.2. */ -#if PY_VERSION_HEX < 0x03070000 && defined(PySlice_GetIndicesEx) && !defined(PYPY_VERSION) -#undef PySlice_GetIndicesEx -#endif - static PyObject *mb_subscript(MiniBufferObj *self, PyObject *item) { if (PyIndex_Check(item)) { @@ -280,6 +243,11 @@ static PyObject *mb_subscript(MiniBufferObj *self, PyObject *item) static int mb_ass_subscript(MiniBufferObj* self, PyObject* item, PyObject* value) { + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, + "'del x[n]' not supported for buffer objects"); + return -1; + } if (PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); if (i == -1 && PyErr_Occurred()) @@ -317,13 +285,6 @@ static PyMappingMethods mb_as_mapping = { (binaryfunc)mb_subscript, /*mp_subscript*/ (objobjargproc)mb_ass_subscript, /*mp_ass_subscript*/ }; -#endif - -#if PY_MAJOR_VERSION >= 3 -# define MINIBUF_TPFLAGS 0 -#else -# define MINIBUF_TPFLAGS (Py_TPFLAGS_HAVE_GETCHARBUFFER | Py_TPFLAGS_HAVE_NEWBUFFER) -#endif PyDoc_STRVAR(ffi_buffer_doc, "ffi.buffer(cdata[, byte_size]):\n" @@ -354,23 +315,14 @@ static PyTypeObject MiniBuffer_Type = { 0, /* tp_repr */ 0, /* tp_as_number */ &mb_as_sequence, /* tp_as_sequence */ -#if PY_MAJOR_VERSION < 3 - 0, /* tp_as_mapping */ -#else &mb_as_mapping, /* tp_as_mapping */ -#endif 0, /* tp_hash */ 0, /* tp_call */ -#if PY_MAJOR_VERSION < 3 - (reprfunc)mb_str, /* tp_str */ -#else 0, /* tp_str */ -#endif PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ &mb_as_buffer, /* tp_as_buffer */ - (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | - MINIBUF_TPFLAGS), /* tp_flags */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ ffi_buffer_doc, /* tp_doc */ (traverseproc)mb_traverse, /* tp_traverse */ (inquiry)mb_clear, /* tp_clear */ diff --git a/contrib/python/cffi/py3/c/misc_thread_common.h b/contrib/python/cffi/py3/c/misc_thread_common.h index b0710acceec..766dcb2ad7d 100644 --- a/contrib/python/cffi/py3/c/misc_thread_common.h +++ b/contrib/python/cffi/py3/c/misc_thread_common.h @@ -328,20 +328,12 @@ static void restore_errno_only(void) It was added in 3.5.2 but should never be used in 3.5.x because it is not available in 3.5.0 or 3.5.1. */ -#if PY_VERSION_HEX >= 0x03050100 && PY_VERSION_HEX < 0x03060000 -PyAPI_DATA(void *volatile) _PyThreadState_Current; -#endif - static PyThreadState *get_current_ts(void) { #if PY_VERSION_HEX >= 0x030D0000 return PyThreadState_GetUnchecked(); -#elif PY_VERSION_HEX >= 0x03060000 - return _PyThreadState_UncheckedGet(); -#elif defined(_Py_atomic_load_relaxed) - return (PyThreadState*)_Py_atomic_load_relaxed(&_PyThreadState_Current); #else - return (PyThreadState*)_PyThreadState_Current; /* assume atomic read */ + return _PyThreadState_UncheckedGet(); #endif } diff --git a/contrib/python/cffi/py3/c/misc_win32.h b/contrib/python/cffi/py3/c/misc_win32.h index 90c63adc68c..4fe723a727d 100644 --- a/contrib/python/cffi/py3/c/misc_win32.h +++ b/contrib/python/cffi/py3/c/misc_win32.h @@ -64,7 +64,6 @@ static void restore_errno(void) /************************************************************/ -#if PY_MAJOR_VERSION >= 3 static PyObject *b_getwinerror(PyObject *self, PyObject *args, PyObject *kwds) { int err = -1; @@ -113,54 +112,6 @@ static PyObject *b_getwinerror(PyObject *self, PyObject *args, PyObject *kwds) LocalFree(s_buf); return v; } -#else -static PyObject *b_getwinerror(PyObject *self, PyObject *args, PyObject *kwds) -{ - int err = -1; - int len; - char *s; - char *s_buf = NULL; /* Free via LocalFree */ - char s_small_buf[40]; /* Room for "Windows Error 0xFFFFFFFFFFFFFFFF" */ - PyObject *v; - static char *keywords[] = {"code", NULL}; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|i", keywords, &err)) - return NULL; - - if (err == -1) { - struct cffi_tls_s *p = get_cffi_tls(); - if (p == NULL) - return PyErr_NoMemory(); - err = p->saved_lasterror; - } - - len = FormatMessage( - /* Error API error */ - FORMAT_MESSAGE_ALLOCATE_BUFFER | - FORMAT_MESSAGE_FROM_SYSTEM | - FORMAT_MESSAGE_IGNORE_INSERTS, - NULL, /* no message source */ - err, - MAKELANGID(LANG_NEUTRAL, - SUBLANG_DEFAULT), /* Default language */ - (LPTSTR) &s_buf, - 0, /* size not used */ - NULL); /* no args */ - if (len==0) { - /* Only seen this in out of mem situations */ - sprintf(s_small_buf, "Windows Error 0x%X", err); - s = s_small_buf; - } else { - s = s_buf; - /* remove trailing cr/lf and dots */ - while (len > 0 && (s[len-1] <= ' ' || s[len-1] == '.')) - s[--len] = '\0'; - } - v = Py_BuildValue("(is)", err, s); - LocalFree(s_buf); - return v; -} -#endif /************************************************************/ @@ -251,7 +202,9 @@ static int cffi_atomic_compare_exchange(void **ptr, void **expected, static void *cffi_atomic_load(void **ptr) { -#if defined(_M_X64) || defined(_M_IX86) +#if defined(__GNUC__) || defined(__clang__) + return __atomic_load_n(ptr, __ATOMIC_SEQ_CST); +#elif defined(_M_X64) || defined(_M_IX86) return *(volatile void **)ptr; #elif defined(_M_ARM64) return (void *)__ldar64((volatile unsigned __int64 *)ptr); @@ -262,7 +215,9 @@ static void *cffi_atomic_load(void **ptr) static uint8_t cffi_atomic_load_uint8(uint8_t *ptr) { -#if defined(_M_X64) || defined(_M_IX86) +#if defined(__GNUC__) || defined(__clang__) + return __atomic_load_n(ptr, __ATOMIC_SEQ_CST); +#elif defined(_M_X64) || defined(_M_IX86) return *(volatile uint8_t *)ptr; #elif defined(_M_ARM64) return (uint8_t)__ldar8((volatile uint8_t *)ptr); @@ -273,7 +228,9 @@ static uint8_t cffi_atomic_load_uint8(uint8_t *ptr) static Py_ssize_t cffi_atomic_load_ssize(Py_ssize_t *ptr) { -#if defined(_M_X64) || defined(_M_IX86) +#if defined(__GNUC__) || defined(__clang__) + return __atomic_load_n(ptr, __ATOMIC_SEQ_CST); +#elif defined(_M_X64) || defined(_M_IX86) return *(volatile Py_ssize_t *)ptr; #elif defined(_M_ARM64) return (Py_ssize_t)__ldar64((volatile unsigned __int64 *)ptr); @@ -284,17 +241,29 @@ static Py_ssize_t cffi_atomic_load_ssize(Py_ssize_t *ptr) static void cffi_atomic_store_ssize(Py_ssize_t *ptr, Py_ssize_t value) { +#if defined(__GNUC__) || defined(__clang__) + __atomic_store_n(ptr, value, __ATOMIC_SEQ_CST); +#else _InterlockedExchangePointer(ptr, value); +#endif } static void cffi_atomic_store(void **ptr, void *value) { +#if defined(__GNUC__) || defined(__clang__) + __atomic_store_n(ptr, value, __ATOMIC_SEQ_CST); +#else _InterlockedExchangePointer(ptr, value); +#endif } static void cffi_atomic_store_uint8(uint8_t *ptr, uint8_t value) { +#if defined(__GNUC__) || defined(__clang__) + __atomic_store_n(ptr, value, __ATOMIC_SEQ_CST); +#else _InterlockedExchange8(ptr, value); +#endif } #endif /* CFFI_MISC_WIN32_H */ diff --git a/contrib/python/cffi/py3/c/realize_c_type.c b/contrib/python/cffi/py3/c/realize_c_type.c index fbfa68ca748..993a07d122f 100644 --- a/contrib/python/cffi/py3/c/realize_c_type.c +++ b/contrib/python/cffi/py3/c/realize_c_type.c @@ -29,7 +29,6 @@ static PyObject *build_primitive_type(int num); /* forward */ static int init_global_types_dict(PyObject *ffi_type_dict) { int err; - Py_ssize_t i; PyObject *ct_void, *ct_char, *ct2, *pnull; /* XXX some leaks in case these functions fail, but well, MemoryErrors during importing an extension module are kind @@ -78,7 +77,7 @@ static int init_global_types_dict(PyObject *ffi_type_dict) #ifdef Py_GIL_DISABLED // Ensure that all primitive types are initialised to avoid race conditions // on the first access. - for (i = 0; i < _CFFI__NUM_PRIM; i++) { + for (Py_ssize_t i = 0; i < _CFFI__NUM_PRIM; i++) { ct2 = get_primitive_type(i); if (ct2 == NULL) return -1; @@ -247,13 +246,13 @@ static PyObject *realize_global_int(builder_c_t *builder, int gindex) case 0: if (value <= (unsigned long long)LONG_MAX) - return PyInt_FromLong((long)value); + return PyLong_FromLong((long)value); else return PyLong_FromUnsignedLongLong(value); case 1: if ((long long)value >= (long long)LONG_MIN) - return PyInt_FromLong((long)value); + return PyLong_FromLong((long)value); else return PyLong_FromLongLong((long long)value); @@ -551,7 +550,7 @@ realize_c_type_or_func_now(builder_c_t *builder, _cffi_opcode_t op, j = 0; while (p[j] != ',' && p[j] != '\0') j++; - tmp = PyText_FromStringAndSize(p, j); + tmp = PyUnicode_FromStringAndSize(p, j); if (tmp == NULL) break; PyTuple_SET_ITEM(enumerators, i, tmp); diff --git a/contrib/python/cffi/py3/cffi/__init__.py b/contrib/python/cffi/py3/cffi/__init__.py index c99ec3d4817..da1875cc253 100644 --- a/contrib/python/cffi/py3/cffi/__init__.py +++ b/contrib/python/cffi/py3/cffi/__init__.py @@ -5,8 +5,8 @@ from .api import FFI from .error import CDefError, FFIError, VerificationError, VerificationMissing from .error import PkgConfigError -__version__ = "2.0.0" -__version_info__ = (2, 0, 0) +__version__ = "2.1.0" +__version_info__ = (2, 1, 0) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/contrib/python/cffi/py3/cffi/_cffi_errors.h b/contrib/python/cffi/py3/cffi/_cffi_errors.h index 158e0590346..0931379ccd2 100644 --- a/contrib/python/cffi/py3/cffi/_cffi_errors.h +++ b/contrib/python/cffi/py3/cffi/_cffi_errors.h @@ -36,11 +36,7 @@ static PyObject *_cffi_start_error_capture(void) if (result == NULL) goto error; -#if PY_MAJOR_VERSION >= 3 bi = PyImport_ImportModule("builtins"); -#else - bi = PyImport_ImportModule("__builtin__"); -#endif if (bi == NULL) goto error; PyDict_SetItemString(result, "__builtins__", bi); @@ -81,15 +77,9 @@ static PyObject *_cffi_start_error_capture(void) static DWORD WINAPI _cffi_bootstrap_dialog(LPVOID ignored) { Sleep(666); /* may be interrupted if the whole process is closing */ -#if PY_MAJOR_VERSION >= 3 MessageBoxW(NULL, (wchar_t *)_cffi_bootstrap_text, L"Python-CFFI error", MB_OK | MB_ICONERROR); -#else - MessageBoxA(NULL, (char *)_cffi_bootstrap_text, - "Python-CFFI error", - MB_OK | MB_ICONERROR); -#endif _cffi_bootstrap_text = NULL; return 0; } @@ -111,11 +101,7 @@ static void _cffi_stop_error_capture(PyObject *ecap) /* Show a dialog box, but in a background thread, and never show multiple dialog boxes at once. */ -#if PY_MAJOR_VERSION >= 3 text = PyUnicode_AsWideCharString(s, NULL); -#else - text = PyString_AsString(s); -#endif _cffi_bootstrap_text = text; diff --git a/contrib/python/cffi/py3/cffi/_cffi_gen_src.py b/contrib/python/cffi/py3/cffi/_cffi_gen_src.py new file mode 100644 index 00000000000..3d4fb7e3d9c --- /dev/null +++ b/contrib/python/cffi/py3/cffi/_cffi_gen_src.py @@ -0,0 +1,216 @@ +# Integrated from the cffi-buildtool project by Rose Davidson +# (https://github.com/inklesspen/cffi-buildtool), under the following +# license: +# +# MIT License +# +# Copyright (c) 2024, Rose Davidson +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice (including the +# next paragraph) shall be included in all copies or substantial portions +# of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +"""Implementation of the ``cffi-gen-src`` command-line tool. + +This module is private; the command line is the only supported +interface. Two subcommands: + +``exec-python`` + Execute a Python script that constructs a :class:`cffi.FFI` + (the same kind of script that the CFFI docs' "Main mode of usage" + describes) and emit the generated C source. + +``read-sources`` + Create the :class:`cffi.FFI` from a separate ``cdef`` file and C + source prelude, then emit the generated C source. +""" + +import argparse +import io +import os +import sys + +from .api import FFI + + +def _execfile(pysrc, filename, globs): + compiled = compile(source=pysrc, filename=filename, mode='exec') + exec(compiled, globs, globs) + + +def find_ffi_in_python_script(pysrc, filename, ffivar): + """Execute ``pysrc`` and return the :class:`FFI` object it defines. + + The script is executed with ``__name__`` set to ``"cffi.gen_src"``, + so a trailing ``if __name__ == "__main__": ffibuilder.compile()`` + block in the script is skipped. + + ``ffivar`` is the name bound by the script to the :class:`FFI` + object, or to a callable that returns one. + + Raises :class:`NameError` if the name is not bound by the script, + or :class:`TypeError` if the name does not resolve to an + :class:`FFI` instance. + """ + filename = os.path.abspath(filename) + globs = {'__name__': 'cffi.gen_src', '__file__': filename} + old_path = sys.path[:] + sys.path.insert(0, os.path.dirname(filename)) + try: + _execfile(pysrc, filename, globs) + if ffivar not in globs: + raise NameError( + "Expected to find the FFI object with the name %r, " + "but it was not found." % (ffivar,) + ) + ffi = globs[ffivar] + if not isinstance(ffi, FFI) and callable(ffi): + # Maybe it's a callable that returns a FFI + ffi = ffi() + if not isinstance(ffi, FFI): + raise TypeError( + "Found an object with the name %r but it was not an " + "instance of cffi.api.FFI" % (ffivar,) + ) + return ffi + finally: + sys.path[:] = old_path + + +def make_ffi_from_sources(modulename, cdef, csrc): + """Create an :class:`FFI` from ``cdef`` text and a C source prelude.""" + ffibuilder = FFI() + ffibuilder.cdef(cdef) + ffibuilder.set_source(modulename, csrc) + return ffibuilder + + +def generate_c_source(ffi): + """Return the C source that :meth:`FFI.emit_c_code` would write.""" + output = io.StringIO() + ffi.emit_c_code(output) + return output.getvalue() + + +def write_c_source(output, generated): + if output == '-': + sys.stdout.write(generated) + return + with open(output, 'w', encoding='utf-8') as f: + f.write(generated) + + +def same_input_file(file1, file2): + if file1 is file2: + return True + try: + return os.path.samefile(file1.name, file2.name) + except OSError: + return False + + +def exec_python(*, output, pyfile, ffi_var): + with pyfile: + ffi = find_ffi_in_python_script(pyfile.read(), pyfile.name, ffi_var) + generated = generate_c_source(ffi) + write_c_source(output, generated) + + +def read_sources(*, output, module_name, cdef_input, csrc_input): + with csrc_input, cdef_input: + csrc = csrc_input.read() + cdef = cdef_input.read() + ffi = make_ffi_from_sources(module_name, cdef, csrc) + generated = generate_c_source(ffi) + write_c_source(output, generated) + + +def _prog(): + # The same parser serves both documented invocations; make --help + # and usage messages show the one that was actually used. + argv0 = os.path.basename(sys.argv[0]) if sys.argv else '' + if argv0.startswith('cffi-gen-src'): + return 'cffi-gen-src' + return 'python -m cffi.gen_src' + + +parser = argparse.ArgumentParser( + prog=_prog(), + description='Generate CFFI C source for extension modules.', +) +subparsers = parser.add_subparsers(dest='mode') + +exec_python_parser = subparsers.add_parser( + 'exec-python', + help='Execute a Python script that defines an FFI object', +) +exec_python_parser.add_argument( + '--ffi-var', + default='ffibuilder', + help="Name of the FFI object in the Python script; defaults to 'ffibuilder'.", +) +exec_python_parser.add_argument( + 'pyfile', + type=argparse.FileType('r', encoding='utf-8'), + help='Path to the Python script', +) +exec_python_parser.add_argument( + 'output', + help='Output path for the C source', +) + +read_sources_parser = subparsers.add_parser( + 'read-sources', + help='Read cdef and C source prelude files that define an FFI object', +) +read_sources_parser.add_argument( + 'module_name', + help='Full name of the generated module, including packages', +) +read_sources_parser.add_argument( + 'cdef', + type=argparse.FileType('r', encoding='utf-8'), + help='File containing C definitions', +) +read_sources_parser.add_argument( + 'csrc', + type=argparse.FileType('r', encoding='utf-8'), + help='File containing C source prelude', +) +read_sources_parser.add_argument( + 'output', + help='Output path for the C source', +) + + +def run(args=None): + args = parser.parse_args(args=args) + if args.mode == 'exec-python': + exec_python(output=args.output, pyfile=args.pyfile, ffi_var=args.ffi_var) + elif args.mode == 'read-sources': + if same_input_file(args.cdef, args.csrc): + parser.error('cdef and csrc are the same file and should not be') + read_sources( + output=args.output, + module_name=args.module_name, + cdef_input=args.cdef, + csrc_input=args.csrc, + ) + else: + parser.error('a subcommand is required: exec-python or read-sources') + parser.exit(0) diff --git a/contrib/python/cffi/py3/cffi/_cffi_include.h b/contrib/python/cffi/py3/cffi/_cffi_include.h index 908a1d7343f..c8c6d343bcc 100644 --- a/contrib/python/cffi/py3/cffi/_cffi_include.h +++ b/contrib/python/cffi/py3/cffi/_cffi_include.h @@ -28,8 +28,13 @@ #if !defined(_CFFI_USE_EMBEDDING) && !defined(Py_LIMITED_API) # ifdef _MSC_VER # if !defined(_DEBUG) && !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) && !defined(_CFFI_NO_LIMITED_API) -# define Py_LIMITED_API +# if !defined(Py_GIL_DISABLED) +# define Py_LIMITED_API +# else +# define Py_LIMITED_API 0x030f0000 +# endif # endif + # include /* sanity-check: Py_LIMITED_API will cause crashes if any of these are also defined. Normally, the Python file PC/pyconfig.h does not @@ -49,7 +54,11 @@ # else # include # if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) && !defined(_CFFI_NO_LIMITED_API) -# define Py_LIMITED_API +# if !defined(Py_GIL_DISABLED) +# define Py_LIMITED_API +# else +# define Py_LIMITED_API 0x030f0000 +# endif # endif # endif #endif @@ -59,6 +68,9 @@ extern "C" { #endif #include +#include +#include + #include "parse_c_type.h" /* this block of #ifs should be kept exactly identical between @@ -128,13 +140,9 @@ extern "C" { #ifndef PYPY_VERSION -#if PY_MAJOR_VERSION >= 3 -# define PyInt_FromLong PyLong_FromLong -#endif - #define _cffi_from_c_double PyFloat_FromDouble #define _cffi_from_c_float PyFloat_FromDouble -#define _cffi_from_c_long PyInt_FromLong +#define _cffi_from_c_long PyLong_FromLong #define _cffi_from_c_ulong PyLong_FromUnsignedLong #define _cffi_from_c_longlong PyLong_FromLongLong #define _cffi_from_c_ulonglong PyLong_FromUnsignedLongLong @@ -146,12 +154,12 @@ extern "C" { #define _cffi_from_c_int(x, type) \ (((type)-1) > 0 ? /* unsigned */ \ (sizeof(type) < sizeof(long) ? \ - PyInt_FromLong((long)x) : \ + PyLong_FromLong((long)x) : \ sizeof(type) == sizeof(long) ? \ PyLong_FromUnsignedLong((unsigned long)x) : \ PyLong_FromUnsignedLongLong((unsigned long long)x)) : \ (sizeof(type) <= sizeof(long) ? \ - PyInt_FromLong((long)x) : \ + PyLong_FromLong((long)x) : \ PyLong_FromLongLong((long long)x))) #define _cffi_to_c_int(o, type) \ diff --git a/contrib/python/cffi/py3/cffi/_embedding.h b/contrib/python/cffi/py3/cffi/_embedding.h index 64c04f67cac..12ff017fb83 100644 --- a/contrib/python/cffi/py3/cffi/_embedding.h +++ b/contrib/python/cffi/py3/cffi/_embedding.h @@ -177,9 +177,6 @@ static int _cffi_initialize_python(void) if (PyDict_SetItemString(global_dict, "__builtins__", builtins) < 0) goto error; x = PyEval_EvalCode( -#if PY_MAJOR_VERSION < 3 - (PyCodeObject *) -#endif pycode, global_dict, global_dict); if (x == NULL) goto error; @@ -225,7 +222,7 @@ static int _cffi_initialize_python(void) if (f != NULL && f != Py_None) { PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME - "\ncompiled with cffi version: 2.0.0" + "\ncompiled with cffi version: 2.1.0" "\n_cffi_backend module: ", f); modules = PyImport_GetModuleDict(); mod = PyDict_GetItemString(modules, "_cffi_backend"); @@ -247,10 +244,6 @@ static int _cffi_initialize_python(void) goto done; } -#if PY_VERSION_HEX < 0x03080000 -PyAPI_DATA(char *) _PyParser_TokenNames[]; /* from CPython */ -#endif - static int _cffi_carefully_make_gil(void) { /* This does the basic initialization of Python. It can be called @@ -295,27 +288,6 @@ static int _cffi_carefully_make_gil(void) */ #ifdef WITH_THREAD -# if PY_VERSION_HEX < 0x03080000 - char *volatile *lock = (char *volatile *)_PyParser_TokenNames; - char *old_value, *locked_value; - - while (1) { /* spin loop */ - old_value = *lock; - locked_value = old_value + 1; - if (old_value[0] == 'E') { - assert(old_value[1] == 'N'); - if (cffi_compare_and_swap(lock, old_value, locked_value)) - break; - } - else { - assert(old_value[0] == 'N'); - /* should ideally do a spin loop instruction here, but - hard to do it portably and doesn't really matter I - think: PyEval_InitThreads() should be very fast, and - this is only run at start-up anyway. */ - } - } -# else # if PY_VERSION_HEX < 0x030C0000 int volatile *lock = (int volatile *)&PyCapsule_Type.tp_version_tag; int old_value, locked_value = -42; @@ -348,26 +320,16 @@ static int _cffi_carefully_make_gil(void) this is only run at start-up anyway. */ } } -# endif #endif /* call Py_InitializeEx() */ if (!Py_IsInitialized()) { _cffi_py_initialize(); -#if PY_VERSION_HEX < 0x03070000 - PyEval_InitThreads(); -#endif PyEval_SaveThread(); /* release the GIL */ /* the returned tstate must be the one that has been stored into the autoTLSkey by _PyGILState_Init() called from Py_Initialize(). */ } else { -#if PY_VERSION_HEX < 0x03070000 - /* PyEval_InitThreads() is always a no-op from CPython 3.7 */ - PyGILState_STATE state = PyGILState_Ensure(); - PyEval_InitThreads(); - PyGILState_Release(state); -#endif } #ifdef WITH_THREAD diff --git a/contrib/python/cffi/py3/cffi/api.py b/contrib/python/cffi/py3/cffi/api.py index 5a474f3da92..ec8ee7dfd2b 100644 --- a/contrib/python/cffi/py3/cffi/api.py +++ b/contrib/python/cffi/py3/cffi/api.py @@ -3,13 +3,6 @@ from .lock import allocate_lock from .error import CDefError from . import model -try: - callable -except NameError: - # Python 3.1 - from collections import Callable - callable = lambda x: isinstance(x, Callable) - try: basestring except NameError: @@ -20,7 +13,7 @@ _unspecified = object() -class FFI(object): +class FFI: r''' The main top-level class that you instantiate once, or once per module. @@ -414,7 +407,7 @@ class FFI(object): if (replace_with.startswith('*') and '&[' in self._backend.getcname(cdecl, '&')): replace_with = '(%s)' % replace_with - elif replace_with and not replace_with[0] in '[(': + elif replace_with and replace_with[0] not in '[(': replace_with = ' ' + replace_with return self._backend.getcname(cdecl, replace_with) @@ -909,7 +902,7 @@ def _make_ffi_library(ffi, libname, flags): raise AttributeError(name) accessors[name](name) # - class FFILibrary(object): + class FFILibrary: def __getattr__(self, name): make_accessor(name) return getattr(self, name) diff --git a/contrib/python/cffi/py3/cffi/backend_ctypes.py b/contrib/python/cffi/py3/cffi/backend_ctypes.py index e7956a79cfb..12cd16259db 100644 --- a/contrib/python/cffi/py3/cffi/backend_ctypes.py +++ b/contrib/python/cffi/py3/cffi/backend_ctypes.py @@ -12,7 +12,7 @@ else: class CTypesType(type): pass -class CTypesData(object): +class CTypesData: __metaclass__ = CTypesType __slots__ = ['__weakref__'] __name__ = '' @@ -270,7 +270,7 @@ class CTypesBaseStructOrUnion(CTypesData): return CTypesData.__repr__(self, c_name or self._get_c_name(' &')) -class CTypesBackend(object): +class CTypesBackend: PRIMITIVE_TYPES = { 'char': ctypes.c_char, @@ -336,7 +336,6 @@ class CTypesBackend(object): if novalue is not None: raise TypeError("None expected, got %s object" % (type(novalue).__name__,)) - return None CTypesVoid._fix_class() return CTypesVoid @@ -387,7 +386,7 @@ class CTypesBackend(object): return ctype() return ctype(CTypesPrimitive._to_ctypes(init)) - if kind == 'int' or kind == 'byte': + if kind in {'int', 'byte'}: @classmethod def _cast_from(cls, source): source = _cast_source_to_int(source) @@ -435,7 +434,7 @@ class CTypesBackend(object): _cast_to_integer = __int__ - if kind == 'int' or kind == 'byte' or kind == 'bool': + if kind in {'int', 'byte', 'bool'}: @staticmethod def _to_ctypes(x): if not isinstance(x, (int, long)): @@ -558,15 +557,15 @@ class CTypesBackend(object): def __setitem__(self, index, value): self._as_ctype_ptr[index] = BItem._to_ctypes(value) - if kind == 'charp' or kind == 'voidp': + if kind in {'charp', 'voidp'}: @classmethod def _arg_to_ctypes(cls, *value): if value and isinstance(value[0], bytes): return ctypes.c_char_p(value[0]) else: - return super(CTypesPtr, cls)._arg_to_ctypes(*value) + return super()._arg_to_ctypes(*value) - if kind == 'charp' or kind == 'bytep': + if kind in {'charp', 'bytep'}: def _to_string(self, maxlen): if maxlen < 0: maxlen = sys.maxsize @@ -581,7 +580,7 @@ class CTypesBackend(object): if getattr(self, '_own', False): return 'owning %d bytes' % ( ctypes.sizeof(self._as_ctype_ptr.contents),) - return super(CTypesPtr, self)._get_own_repr() + return super()._get_own_repr() # if (BItem is self.ffi._get_cached_btype(model.void_type) or BItem is self.ffi._get_cached_btype(model.PrimitiveType('char'))): @@ -663,7 +662,7 @@ class CTypesBackend(object): raise IndexError self._blob[index] = BItem._to_ctypes(value) - if kind == 'char' or kind == 'byte': + if kind in {'char', 'byte'}: def _to_string(self, maxlen): if maxlen < 0: maxlen = len(self._blob) @@ -677,7 +676,7 @@ class CTypesBackend(object): def _get_own_repr(self): if getattr(self, '_own', False): return 'owning %d bytes' % (ctypes.sizeof(self._blob),) - return super(CTypesArray, self)._get_own_repr() + return super()._get_own_repr() def _convert_to_address(self, BClass): if BClass in (CTypesPtr, None) or BClass._automatic_casts: @@ -917,7 +916,7 @@ class CTypesBackend(object): def _get_own_repr(self): if getattr(self, '_own_callback', None) is not None: return 'calling %r' % (self._own_callback,) - return super(CTypesFunctionPtr, self)._get_own_repr() + return super()._get_own_repr() def __call__(self, *args): if has_varargs: @@ -1094,7 +1093,7 @@ class CTypesBackend(object): return BTypePtr._from_ctypes(ptr) -class CTypesLibrary(object): +class CTypesLibrary: def __init__(self, backend, cdll): self.backend = backend diff --git a/contrib/python/cffi/py3/cffi/cffi_opcode.py b/contrib/python/cffi/py3/cffi/cffi_opcode.py index 6421df62134..c964ba13e69 100644 --- a/contrib/python/cffi/py3/cffi/cffi_opcode.py +++ b/contrib/python/cffi/py3/cffi/cffi_opcode.py @@ -1,6 +1,6 @@ from .error import VerificationError -class CffiOp(object): +class CffiOp: def __init__(self, op, arg): self.op = op self.arg = arg diff --git a/contrib/python/cffi/py3/cffi/cparser.py b/contrib/python/cffi/py3/cffi/cparser.py index dd590d8743d..1896b7ef5dc 100644 --- a/contrib/python/cffi/py3/cffi/cparser.py +++ b/contrib/python/cffi/py3/cffi/cparser.py @@ -292,7 +292,7 @@ def _common_type_names(csource): return words_used -class Parser(object): +class Parser: def __init__(self): self._declarations = {} @@ -804,11 +804,10 @@ class Parser(object): raise AssertionError("kind = %r" % (kind,)) if name is not None: self._declare(key, tp) - else: - if kind == 'enum' and type.values is not None: - raise NotImplementedError( - "enum %s: the '{}' declaration should appear on the first " - "time the enum is mentioned, not later" % explicit_name) + elif kind == 'enum' and type.values is not None: + raise NotImplementedError( + "enum %s: the '{}' declaration should appear on the first " + "time the enum is mentioned, not later" % explicit_name) if not tp.forcename: tp.force_the_name(force_name) if tp.forcename and '$' in tp.name: diff --git a/contrib/python/cffi/py3/cffi/gen_src.py b/contrib/python/cffi/py3/cffi/gen_src.py new file mode 100644 index 00000000000..8eb57b16952 --- /dev/null +++ b/contrib/python/cffi/py3/cffi/gen_src.py @@ -0,0 +1,5 @@ +"""Entry point so ``python -m cffi.gen_src`` works.""" + +if __name__ == '__main__': + from cffi._cffi_gen_src import run + run() diff --git a/contrib/python/cffi/py3/cffi/model.py b/contrib/python/cffi/py3/cffi/model.py index e5f4cae3e84..8bade1d6dca 100644 --- a/contrib/python/cffi/py3/cffi/model.py +++ b/contrib/python/cffi/py3/cffi/model.py @@ -22,7 +22,7 @@ def qualify(quals, replace_with): return replace_with -class BaseTypeByIdentity(object): +class BaseTypeByIdentity: is_array_type = False is_raw_function = False @@ -34,7 +34,7 @@ class BaseTypeByIdentity(object): if replace_with: if replace_with.startswith('*') and '&[' in result: replace_with = '(%s)' % replace_with - elif not replace_with[0] in '[(': + elif replace_with[0] not in '[(': replace_with = ' ' + replace_with replace_with = qualify(quals, replace_with) result = result.replace('&', replace_with) @@ -371,8 +371,7 @@ class StructOrUnion(StructOrUnionOrEnum): if (name == '' and isinstance(type, StructOrUnion) and expand_anonymous_struct_union): # nested anonymous struct/union - for result in type.enumfields(): - yield result + yield from type.enumfields() else: yield (name, type, bitsize, quals) diff --git a/contrib/python/cffi/py3/cffi/pkgconfig.py b/contrib/python/cffi/py3/cffi/pkgconfig.py index 5c93f15a60e..e1034ad68a6 100644 --- a/contrib/python/cffi/py3/cffi/pkgconfig.py +++ b/contrib/python/cffi/py3/cffi/pkgconfig.py @@ -31,7 +31,7 @@ def call(libname, flag, encoding=sys.getfilesystemencoding()): a.append(libname) try: pc = subprocess.Popen(a, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - except EnvironmentError as e: + except OSError as e: raise PkgConfigError("cannot run pkg-config: %s" % (str(e).strip(),)) bout, berr = pc.communicate() diff --git a/contrib/python/cffi/py3/cffi/recompiler.py b/contrib/python/cffi/py3/cffi/recompiler.py index 25bf9e91d9e..fbafc011c97 100644 --- a/contrib/python/cffi/py3/cffi/recompiler.py +++ b/contrib/python/cffi/py3/cffi/recompiler.py @@ -7,9 +7,11 @@ VERSION_BASE = 0x2601 VERSION_EMBEDDED = 0x2701 VERSION_CHAR16CHAR32 = 0x2801 +FREE_THREADED_BUILD = sysconfig.get_config_var("Py_GIL_DISABLED") USE_LIMITED_API = ((sys.platform != 'win32' or sys.version_info < (3, 0) or sys.version_info >= (3, 5)) and - not sysconfig.get_config_var("Py_GIL_DISABLED")) # free-threaded doesn't yet support limited API + # free-threaded build doesn't support the stable ABI until Python 3.15 + (not FREE_THREADED_BUILD or sys.version_info >= (3, 15))) class GlobalExpr: def __init__(self, name, address, type_op, size=0, check_value=0): @@ -93,9 +95,18 @@ class EnumExpr: self.allenums = allenums def as_c_expr(self): - return (' { "%s", %d, _cffi_prim_int(%s, %s),\n' - ' "%s" },' % (self.name, self.type_index, - self.size, self.signed, self.allenums)) + lines = [' { "%s", %d, _cffi_prim_int(%s, %s),' % (self.name, self.type_index, + self.size, self.signed,)] + pending = 0 + while len(self.allenums) > pending + 110: + j = self.allenums.find(',', pending + 100) + if j < 0: + break + j += 1 + lines.append(' "%s"' % (self.allenums[pending:j],)) + pending = j + lines.append(' "%s" },' % (self.allenums[pending:],)) + return '\n'.join(lines) def as_python_expr(self): prim_index = { @@ -315,11 +326,8 @@ class Recompiler: prnt('#ifdef PYPY_VERSION') prnt('# define _CFFI_PYTHON_STARTUP_FUNC _cffi_pypyinit_%s' % ( base_module_name,)) - prnt('#elif PY_MAJOR_VERSION >= 3') - prnt('# define _CFFI_PYTHON_STARTUP_FUNC PyInit_%s' % ( - base_module_name,)) prnt('#else') - prnt('# define _CFFI_PYTHON_STARTUP_FUNC init%s' % ( + prnt('# define _CFFI_PYTHON_STARTUP_FUNC PyInit_%s' % ( base_module_name,)) prnt('#endif') lines = self._rel_readlines('_embedding.h') @@ -427,35 +435,22 @@ class Recompiler: prnt(' }') prnt(' p[0] = (const void *)0x%x;' % self._version) prnt(' p[1] = &_cffi_type_context;') - prnt('#if PY_MAJOR_VERSION >= 3') prnt(' return NULL;') - prnt('#endif') prnt('}') # on Windows, distutils insists on putting init_cffi_xyz in # 'export_symbols', so instead of fighting it, just give up and # give it one prnt('# ifdef _MSC_VER') prnt(' PyMODINIT_FUNC') - prnt('# if PY_MAJOR_VERSION >= 3') prnt(' PyInit_%s(void) { return NULL; }' % (base_module_name,)) - prnt('# else') - prnt(' init%s(void) { }' % (base_module_name,)) prnt('# endif') - prnt('# endif') - prnt('#elif PY_MAJOR_VERSION >= 3') + prnt('#else') prnt('PyMODINIT_FUNC') prnt('PyInit_%s(void)' % (base_module_name,)) prnt('{') prnt(' return _cffi_init("%s", 0x%x, &_cffi_type_context);' % ( self.module_name, self._version)) prnt('}') - prnt('#else') - prnt('PyMODINIT_FUNC') - prnt('init%s(void)' % (base_module_name,)) - prnt('{') - prnt(' _cffi_init("%s", 0x%x, &_cffi_type_context);' % ( - self.module_name, self._version)) - prnt('}') prnt('#endif') prnt() prnt('#ifdef __GNUC__') @@ -552,8 +547,7 @@ class Recompiler: tovar, errcode) return # - elif (isinstance(tp, model.StructOrUnionOrEnum) or - isinstance(tp, model.BasePrimitiveType)): + elif (isinstance(tp, (model.StructOrUnionOrEnum, model.BasePrimitiveType))): # a struct (not a struct pointer) as a function argument; # or, a complex (the same code works) self._prnt(' if (_cffi_to_c((char *)&%s, _cffi_type(%d), %s) < 0)' @@ -1313,10 +1307,6 @@ class Recompiler: s = s.encode('utf-8') # -> bytes else: s.decode('utf-8') # got bytes, check for valid utf-8 - try: - s.decode('ascii') - except UnicodeDecodeError: - s = b'# -*- encoding: utf8 -*-\n' + s for line in s.splitlines(True): comment = line if type('//') is bytes: # python2 @@ -1415,7 +1405,7 @@ else: def write(self, s): if isinstance(s, unicode): s = s.encode('ascii') - super(NativeIO, self).write(s) + super().write(s) def _is_file_like(maybefile): # compare to xml.etree.ElementTree._get_writer @@ -1437,11 +1427,11 @@ def _make_c_or_py_source(ffi, module_name, preamble, target_file, verbose): try: with open(target_file, 'r') as f1: if f1.read(len(output) + 1) != output: - raise IOError + raise OSError if verbose: print("(already up-to-date)") return False # already up-to-date - except IOError: + except OSError: tmp_file = '%s.~%d' % (target_file, os.getpid()) with open(tmp_file, 'w') as f1: f1.write(output) diff --git a/contrib/python/cffi/py3/cffi/setuptools_ext.py b/contrib/python/cffi/py3/cffi/setuptools_ext.py index 5cdd246fb4d..946afb45a6b 100644 --- a/contrib/python/cffi/py3/cffi/setuptools_ext.py +++ b/contrib/python/cffi/py3/cffi/setuptools_ext.py @@ -105,10 +105,12 @@ def _set_py_limited_api(Extension, kwds): kwds['py_limited_api'] = True if sysconfig.get_config_var("Py_GIL_DISABLED"): - if kwds.get('py_limited_api'): - log.info("Ignoring py_limited_api=True for free-threaded build.") - - kwds['py_limited_api'] = False + if sys.version_info < (3, 15): + if kwds.get('py_limited_api'): + log.info("Ignoring py_limited_api=True for free-threaded build.") + kwds['py_limited_api'] = False + else: + kwds.setdefault("define_macros", []).append(("Py_TARGET_ABI3T", "0x030f0000")) if kwds.get('py_limited_api') is False: # avoid setting Py_LIMITED_API if py_limited_api=False diff --git a/contrib/python/cffi/py3/cffi/vengine_cpy.py b/contrib/python/cffi/py3/cffi/vengine_cpy.py index 02e6a471d04..4d2ea43e5d9 100644 --- a/contrib/python/cffi/py3/cffi/vengine_cpy.py +++ b/contrib/python/cffi/py3/cffi/vengine_cpy.py @@ -7,7 +7,7 @@ from .error import VerificationError from . import _imp_emulation as imp -class VCPythonEngine(object): +class VCPythonEngine: _class_key = 'x' _gen_python_module = True @@ -102,8 +102,6 @@ class VCPythonEngine(object): # standard init. modname = self.verifier.get_module_name() constants = self._chained_list_constants[False] - prnt('#if PY_MAJOR_VERSION >= 3') - prnt() prnt('static struct PyModuleDef _cffi_module_def = {') prnt(' PyModuleDef_HEAD_INIT,') prnt(' "%s",' % modname) @@ -130,21 +128,6 @@ class VCPythonEngine(object): prnt(' return lib;') prnt('}') prnt() - prnt('#else') - prnt() - prnt('PyMODINIT_FUNC') - prnt('init%s(void)' % modname) - prnt('{') - prnt(' PyObject *lib;') - prnt(' lib = Py_InitModule("%s", _cffi_methods);' % modname) - prnt(' if (lib == NULL)') - prnt(' return;') - prnt(' if (%s < 0 || _cffi_init() < 0)' % (constants,)) - prnt(' return;') - prnt(' return;') - prnt('}') - prnt() - prnt('#endif') def load_library(self, flags=None): # XXX review all usages of 'self' here! @@ -183,7 +166,7 @@ class VCPythonEngine(object): # it will invoke the chained list of functions that will really # build (notably) the constant objects, as if they are # pointers, and store them as attributes on the 'library' object. - class FFILibrary(object): + class FFILibrary: _cffi_python_module = module _cffi_ffi = self.ffi _cffi_dir = [] @@ -873,21 +856,9 @@ cffimod_header = r''' # define _cffi_double_complex_t double _Complex #endif -#if PY_MAJOR_VERSION < 3 -# undef PyCapsule_CheckExact -# undef PyCapsule_GetPointer -# define PyCapsule_CheckExact(capsule) (PyCObject_Check(capsule)) -# define PyCapsule_GetPointer(capsule, name) \ - (PyCObject_AsVoidPtr(capsule)) -#endif - -#if PY_MAJOR_VERSION >= 3 -# define PyInt_FromLong PyLong_FromLong -#endif - #define _cffi_from_c_double PyFloat_FromDouble #define _cffi_from_c_float PyFloat_FromDouble -#define _cffi_from_c_long PyInt_FromLong +#define _cffi_from_c_long PyLong_FromLong #define _cffi_from_c_ulong PyLong_FromUnsignedLong #define _cffi_from_c_longlong PyLong_FromLongLong #define _cffi_from_c_ulonglong PyLong_FromUnsignedLongLong @@ -899,21 +870,21 @@ cffimod_header = r''' #define _cffi_from_c_int_const(x) \ (((x) > 0) ? \ ((unsigned long long)(x) <= (unsigned long long)LONG_MAX) ? \ - PyInt_FromLong((long)(x)) : \ + PyLong_FromLong((long)(x)) : \ PyLong_FromUnsignedLongLong((unsigned long long)(x)) : \ ((long long)(x) >= (long long)LONG_MIN) ? \ - PyInt_FromLong((long)(x)) : \ + PyLong_FromLong((long)(x)) : \ PyLong_FromLongLong((long long)(x))) #define _cffi_from_c_int(x, type) \ (((type)-1) > 0 ? /* unsigned */ \ (sizeof(type) < sizeof(long) ? \ - PyInt_FromLong((long)x) : \ + PyLong_FromLong((long)x) : \ sizeof(type) == sizeof(long) ? \ PyLong_FromUnsignedLong((unsigned long)x) : \ PyLong_FromUnsignedLongLong((unsigned long long)x)) : \ (sizeof(type) <= sizeof(long) ? \ - PyInt_FromLong((long)x) : \ + PyLong_FromLong((long)x) : \ PyLong_FromLongLong((long long)x))) #define _cffi_to_c_int(o, type) \ diff --git a/contrib/python/cffi/py3/cffi/vengine_gen.py b/contrib/python/cffi/py3/cffi/vengine_gen.py index bffc82122c3..0209c794ac7 100644 --- a/contrib/python/cffi/py3/cffi/vengine_gen.py +++ b/contrib/python/cffi/py3/cffi/vengine_gen.py @@ -8,7 +8,7 @@ from . import model from .error import VerificationError -class VGenericEngine(object): +class VGenericEngine: _class_key = 'g' _gen_python_module = False diff --git a/contrib/python/cffi/py3/cffi/verifier.py b/contrib/python/cffi/py3/cffi/verifier.py index e392a2b7fda..96baadb6f76 100644 --- a/contrib/python/cffi/py3/cffi/verifier.py +++ b/contrib/python/cffi/py3/cffi/verifier.py @@ -24,10 +24,10 @@ else: def write(self, s): if isinstance(s, unicode): s = s.encode('ascii') - super(NativeIO, self).write(s) + super().write(s) -class Verifier(object): +class Verifier: def __init__(self, ffi, preamble, tmpdir=None, modulename=None, ext_package=None, tag='', force_generic_engine=False, @@ -182,7 +182,7 @@ class Verifier(object): # Determine if this matches the current file if os.path.exists(self.sourcefilename): with open(self.sourcefilename, "r") as fp: - needs_written = not (fp.read() == source_data) + needs_written = fp.read() != source_data else: needs_written = True diff --git a/contrib/python/cffi/py3/patches/01-arcadia.patch b/contrib/python/cffi/py3/patches/01-arcadia.patch index 23331873652..8ab50be29d7 100644 --- a/contrib/python/cffi/py3/patches/01-arcadia.patch +++ b/contrib/python/cffi/py3/patches/01-arcadia.patch @@ -92,16 +92,8 @@ if (*w > 0xFFFF) { --- contrib/python/cffi/py3/cffi/recompiler.py (index) +++ contrib/python/cffi/py3/cffi/recompiler.py (working tree) -@@ -287,10 +287,8 @@ class Recompiler: - self.write_c_source_to_f(f, preamble) - - def _rel_readlines(self, filename): -- g = open(os.path.join(os.path.dirname(__file__), filename), 'r') -- lines = g.readlines() -- g.close() -- return lines +@@ -287,2 +287,2 @@ class Recompiler: +- with open(os.path.join(os.path.dirname(__file__), filename), 'r') as g: +- return g.readlines() + import pkgutil + return pkgutil.get_data('cffi', filename).decode('utf-8').splitlines(True) - - def write_c_source_to_f(self, f, preamble): - self._f = f diff --git a/contrib/python/cffi/py3/ya.make b/contrib/python/cffi/py3/ya.make index d2eb8e11ad8..a526c12160c 100644 --- a/contrib/python/cffi/py3/ya.make +++ b/contrib/python/cffi/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(2.0.0) +VERSION(2.1.0) LICENSE(MIT) @@ -31,6 +31,7 @@ PY_REGISTER( PY_SRCS( TOP_LEVEL cffi/__init__.py + cffi/_cffi_gen_src.py cffi/_imp_emulation.py cffi/_shimmed_dist_utils.py cffi/api.py @@ -40,6 +41,7 @@ PY_SRCS( cffi/cparser.py cffi/error.py cffi/ffiplatform.py + cffi/gen_src.py cffi/lock.py cffi/model.py cffi/pkgconfig.py -- cgit v1.3 From 3100c44b5940958527a6678ec67e7085db40c636 Mon Sep 17 00:00:00 2001 From: robot-piglet Date: Sat, 25 Jul 2026 22:39:07 +0300 Subject: Intermediate changes commit_hash:1da6a7b20fba86ceb931f52a3c8bef7817e7f6e3 --- contrib/python/idna/py3/.dist-info/METADATA | 164 +++++------ .../python/idna/py3/.dist-info/entry_points.txt | 3 + contrib/python/idna/py3/README.md | 159 +++++------ contrib/python/idna/py3/idna/__main__.py | 6 + contrib/python/idna/py3/idna/cli.py | 128 +++++++++ contrib/python/idna/py3/idna/codec.py | 10 +- contrib/python/idna/py3/idna/core.py | 61 ++-- contrib/python/idna/py3/idna/idnadata.py | 7 +- contrib/python/idna/py3/idna/intranges.py | 7 +- contrib/python/idna/py3/idna/package_data.py | 2 +- contrib/python/idna/py3/idna/uts46data.py | 4 +- contrib/python/idna/py3/patches/01-fix-tests.patch | 26 ++ contrib/python/idna/py3/tests/test_idna.py | 5 +- contrib/python/idna/py3/tests/test_idna_cli.py | 318 +++++++++++++++++++++ contrib/python/idna/py3/ya.make | 5 +- 15 files changed, 656 insertions(+), 249 deletions(-) create mode 100644 contrib/python/idna/py3/.dist-info/entry_points.txt create mode 100644 contrib/python/idna/py3/idna/__main__.py create mode 100644 contrib/python/idna/py3/idna/cli.py create mode 100644 contrib/python/idna/py3/patches/01-fix-tests.patch create mode 100644 contrib/python/idna/py3/tests/test_idna_cli.py diff --git a/contrib/python/idna/py3/.dist-info/METADATA b/contrib/python/idna/py3/.dist-info/METADATA index 3dd388cb837..9a2b49a74ee 100644 --- a/contrib/python/idna/py3/.dist-info/METADATA +++ b/contrib/python/idna/py3/.dist-info/METADATA @@ -1,9 +1,9 @@ Metadata-Version: 2.4 Name: idna -Version: 3.15 +Version: 3.16 Summary: Internationalized Domain Names in Applications (IDNA) Author-email: Kim Davies -Requires-Python: >=3.8 +Requires-Python: >=3.9 Description-Content-Type: text/markdown License-Expression: BSD-3-Clause Classifier: Development Status :: 5 - Production/Stable @@ -13,7 +13,6 @@ Classifier: Operating System :: OS Independent Classifier: Programming Language :: Python Classifier: Programming Language :: Python :: 3 Classifier: Programming Language :: Python :: 3 :: Only -Classifier: Programming Language :: Python :: 3.8 Classifier: Programming Language :: Python :: 3.9 Classifier: Programming Language :: Python :: 3.10 Classifier: Programming Language :: Python :: 3.11 @@ -36,40 +35,18 @@ Provides-Extra: all # Internationalized Domain Names in Applications (IDNA) -Support for [Internationalized Domain Names in -Applications (IDNA)](https://tools.ietf.org/html/rfc5891) -and [Unicode IDNA Compatibility Processing](https://unicode.org/reports/tr46/). - -The latest versions of these standards supplied here provide -more comprehensive language coverage and reduce the potential of -allowing domains with known security vulnerabilities. This library -is a suitable replacement for the "encodings.idna" -module that comes with the Python standard library, but which -only supports an older superseded IDNA specification from 2003. - -Basic functions are simply executed: - -```pycon ->>> import idna ->>> idna.encode('ドメイン.テスト') -b'xn--eckwd4c7c.xn--zckzah' ->>> print(idna.decode('xn--eckwd4c7c.xn--zckzah')) -ドメイン.テスト -``` - - -## Installation - -This package is available for installation from PyPI via the -typical mechanisms, such as: - -```bash -$ python3 -m pip install idna -``` - +Support for [Internationalized Domain Names in Applications +(IDNA)](https://tools.ietf.org/html/rfc5891) and [Unicode IDNA +Compatibility Processing](https://unicode.org/reports/tr46/). It +supersedes the standard library's `encodings.idna`, which only +implements the 2003 specification, offering broader script coverage and +limiting domains with known security vulnerabilities. ## Usage +Package may be installed from [PyPI](https://pypi.org/project/idna/) via +the typical methods (e.g. `python3 -m pip install idna`) + For typical usage, the `encode` and `decode` functions will take a domain name argument and perform a conversion to ASCII-compatible encoding (known as A-labels), or to Unicode strings (known as U-labels) @@ -84,12 +61,7 @@ b'xn--eckwd4c7c.xn--zckzah' ``` Conversions can be applied at a per-label basis using the `ulabel` or -`alabel` functions if necessary: - -```pycon ->>> idna.alabel('测试') -b'xn--0zwm56d' -``` +`alabel` functions for specialized use cases. ### Compatibility Mapping (UTS #46) @@ -102,10 +74,9 @@ conversion operations. This functionality, known as a specification to be a local user-interface issue distinct from IDNA conversion functionality. -For example, "Königsgäßchen" is not a permissible label as *LATIN -CAPITAL LETTER K* is not allowed (nor are capital letters in general). -UTS 46 will convert this into lower case prior to applying the IDNA -conversion. +For example, "Königsgäßchen" is not a permissible label as capital letters +are not allowed. UTS 46 will convert this into lower case prior to applying +the IDNA conversion. ```pycon >>> import idna @@ -114,81 +85,80 @@ conversion. idna.core.InvalidCodepoint: Codepoint U+004B at position 1 of 'Königsgäßchen' not allowed >>> idna.encode('Königsgäßchen', uts46=True) b'xn--knigsgchen-b4a3dun' ->>> print(idna.decode('xn--knigsgchen-b4a3dun')) -königsgäßchen +>>> idna.decode('xn--knigsgchen-b4a3dun') +'königsgäßchen' ``` ## Exceptions -All errors raised during the conversion following the specification -should raise an exception derived from the `idna.IDNAError` base -class. +All errors raised during conversion derive from the `idna.IDNAError` +base class. The more specific exceptions are: -More specific exceptions that may be generated as `idna.IDNABidiError` -when the error reflects an illegal combination of left-to-right and -right-to-left characters in a label; `idna.InvalidCodepoint` when -a specific codepoint is an illegal character in an IDN label (i.e. -INVALID); and `idna.InvalidCodepointContext` when the codepoint is -illegal based on its position in the string (i.e. it is CONTEXTO or CONTEXTJ -but the contextual requirements are not satisfied.) +* `idna.IDNABidiError` — raised when a label contains an illegal + combination of left-to-right and right-to-left characters. +* `idna.InvalidCodepoint` — raised when a label contains a codepoint + that is INVALID for IDNA. +* `idna.InvalidCodepointContext` — raised when a CONTEXTO or CONTEXTJ + codepoint appears in a position whose contextual requirements are + not satisfied. -## Building and Diagnostics -The IDNA and UTS 46 functionality relies upon pre-calculated lookup -tables for performance. These tables are derived from computing against -eligibility criteria in the respective standards using the command-line -script `tools/idna-data`. +## Command-line tool -This tool will fetch relevant codepoint data from the Unicode repository -and perform the required calculations to identify eligibility. There are -three main modes: +The package supports command-line usage to convert domain names +between their Unicode and ASCII-compatible forms. It can be run either +as a module (`python3 -m idna`) or, once installed (such as with `uv +tool` or `pipx`), via the `idna` script: -* `idna-data make-libdata`. Generates `idnadata.py` and - `uts46data.py`, the pre-calculated lookup tables used for IDNA and - UTS 46 conversions. Implementers who wish to track this library against - a different Unicode version may use this tool to manually generate a - different version of the `idnadata.py` and `uts46data.py` files. +```bash +$ uv tool install idna +$ idna xn--e1afmkfd.xn--p1ai +пример.рф +$ idna пример.рф +xn--e1afmkfd.xn--p1ai +``` -* `idna-data make-table`. Generate a table of the IDNA disposition - (e.g. PVALID, CONTEXTJ, CONTEXTO) in the format found in Appendix - B.1 of RFC 5892 and the pre-computed tables published by [IANA](https://www.iana.org/). +With no mode flag the direction is chosen automatically: inputs +containing an `xn--` label are decoded, anything else is encoded. Pass +`-e`/`--encode` or `-d`/`--decode` to force a specific direction. -* `idna-data U+0061`. Prints debugging output on the various - properties associated with an individual Unicode codepoint (in this - case, U+0061), that are used to assess the IDNA and UTS 46 status of a - codepoint. This is helpful in debugging or analysis. +Multiple domains may be supplied at once, either as positional arguments +or by piping one domain per line on standard input. When more than +one domain is supplied without explicitly asking to encode or decode, +the direction is picked from the first input and that mode is applied +to every remaining input. Use `-e`/`--encode` or `-d`/`--decode` to +override the heuristic if the first input is ambiguous. -The tool accepts a number of arguments, described using `idna-data -h`. -Most notably, the `--version` argument allows the specification -of the version of Unicode to be used in computing the table data. For -example, `idna-data --version 9.0.0 make-libdata` will generate -library data against Unicode 9.0.0. +UTS #46 mapping is applied by default, which lets the tool accept +inputs that aren't strictly valid IDNA 2008 by normalising them first: +```bash +$ idna ΠΑΡΆΔΕΙΓΜΑ.ΕΛ +xn--hxajbheg2az3al.xn--qxam +``` -## Additional Notes +Pass `--strict` to disable UTS #46 and apply IDNA 2008 rules verbatim; +the same input will then be rejected. -* **Packages**. The latest tagged release version is published in the - [Python Package Index](https://pypi.org/project/idna/). +Conversion failures are reported on stderr together with the +offending input; processing continues with the remaining domains and +the tool exits with a non-zero status if any conversion failed. -* **Version support**. This library supports Python 3.8 and higher. - As this library serves as a low-level toolkit for a variety of - applications, many of which strive for broad compatibility with older - Python versions, there is no rush to remove older interpreter support. - Support for older versions are likely to be removed from new releases - as automated tests can no longer easily be run, i.e. once the Python - version is officially end-of-life. -* **Testing**. The library has a test suite based on each rule of the - IDNA specification, as well as tests that are provided as part of the - Unicode Technical Standard 46, [Unicode IDNA Compatibility Processing](https://unicode.org/reports/tr46/). +## Additional Notes + +* **Version support**. This library supports Python 3.9 and higher. + As this library serves as a low-level toolkit for a variety of + applications, we strive to support all versions of Python that are + not beyond end-of-life. * **Emoji**. It is an occasional request to support emoji domains in this library. Encoding of symbols like emoji is expressly prohibited by the IDNA technical standard, and emoji domains are broadly phased out across the domain industry due to associated security risks. -* **Transitional processing**. Unicode 16.0.0 removed transitional - processing so the `transitional` argument for the encode() method - no longer has any effect and will be removed at a later date. +* **Regenerating lookup tables**. The IDNA and UTS 46 functionality + relies upon pre-calculated lookup tables, generated using the + `idna-data` script in [`tools/`](tools/README.md). diff --git a/contrib/python/idna/py3/.dist-info/entry_points.txt b/contrib/python/idna/py3/.dist-info/entry_points.txt new file mode 100644 index 00000000000..59ca7ac02ea --- /dev/null +++ b/contrib/python/idna/py3/.dist-info/entry_points.txt @@ -0,0 +1,3 @@ +[console_scripts] +idna=idna.cli:main + diff --git a/contrib/python/idna/py3/README.md b/contrib/python/idna/py3/README.md index 5772219126b..77e4e4e0a8d 100644 --- a/contrib/python/idna/py3/README.md +++ b/contrib/python/idna/py3/README.md @@ -1,39 +1,17 @@ # Internationalized Domain Names in Applications (IDNA) -Support for [Internationalized Domain Names in -Applications (IDNA)](https://tools.ietf.org/html/rfc5891) -and [Unicode IDNA Compatibility Processing](https://unicode.org/reports/tr46/). - -The latest versions of these standards supplied here provide -more comprehensive language coverage and reduce the potential of -allowing domains with known security vulnerabilities. This library -is a suitable replacement for the "encodings.idna" -module that comes with the Python standard library, but which -only supports an older superseded IDNA specification from 2003. - -Basic functions are simply executed: - -```pycon ->>> import idna ->>> idna.encode('ドメイン.テスト') -b'xn--eckwd4c7c.xn--zckzah' ->>> print(idna.decode('xn--eckwd4c7c.xn--zckzah')) -ドメイン.テスト -``` - - -## Installation - -This package is available for installation from PyPI via the -typical mechanisms, such as: - -```bash -$ python3 -m pip install idna -``` - +Support for [Internationalized Domain Names in Applications +(IDNA)](https://tools.ietf.org/html/rfc5891) and [Unicode IDNA +Compatibility Processing](https://unicode.org/reports/tr46/). It +supersedes the standard library's `encodings.idna`, which only +implements the 2003 specification, offering broader script coverage and +limiting domains with known security vulnerabilities. ## Usage +Package may be installed from [PyPI](https://pypi.org/project/idna/) via +the typical methods (e.g. `python3 -m pip install idna`) + For typical usage, the `encode` and `decode` functions will take a domain name argument and perform a conversion to ASCII-compatible encoding (known as A-labels), or to Unicode strings (known as U-labels) @@ -48,12 +26,7 @@ b'xn--eckwd4c7c.xn--zckzah' ``` Conversions can be applied at a per-label basis using the `ulabel` or -`alabel` functions if necessary: - -```pycon ->>> idna.alabel('测试') -b'xn--0zwm56d' -``` +`alabel` functions for specialized use cases. ### Compatibility Mapping (UTS #46) @@ -66,10 +39,9 @@ conversion operations. This functionality, known as a specification to be a local user-interface issue distinct from IDNA conversion functionality. -For example, "Königsgäßchen" is not a permissible label as *LATIN -CAPITAL LETTER K* is not allowed (nor are capital letters in general). -UTS 46 will convert this into lower case prior to applying the IDNA -conversion. +For example, "Königsgäßchen" is not a permissible label as capital letters +are not allowed. UTS 46 will convert this into lower case prior to applying +the IDNA conversion. ```pycon >>> import idna @@ -78,80 +50,79 @@ conversion. idna.core.InvalidCodepoint: Codepoint U+004B at position 1 of 'Königsgäßchen' not allowed >>> idna.encode('Königsgäßchen', uts46=True) b'xn--knigsgchen-b4a3dun' ->>> print(idna.decode('xn--knigsgchen-b4a3dun')) -königsgäßchen +>>> idna.decode('xn--knigsgchen-b4a3dun') +'königsgäßchen' ``` ## Exceptions -All errors raised during the conversion following the specification -should raise an exception derived from the `idna.IDNAError` base -class. +All errors raised during conversion derive from the `idna.IDNAError` +base class. The more specific exceptions are: -More specific exceptions that may be generated as `idna.IDNABidiError` -when the error reflects an illegal combination of left-to-right and -right-to-left characters in a label; `idna.InvalidCodepoint` when -a specific codepoint is an illegal character in an IDN label (i.e. -INVALID); and `idna.InvalidCodepointContext` when the codepoint is -illegal based on its position in the string (i.e. it is CONTEXTO or CONTEXTJ -but the contextual requirements are not satisfied.) +* `idna.IDNABidiError` — raised when a label contains an illegal + combination of left-to-right and right-to-left characters. +* `idna.InvalidCodepoint` — raised when a label contains a codepoint + that is INVALID for IDNA. +* `idna.InvalidCodepointContext` — raised when a CONTEXTO or CONTEXTJ + codepoint appears in a position whose contextual requirements are + not satisfied. -## Building and Diagnostics -The IDNA and UTS 46 functionality relies upon pre-calculated lookup -tables for performance. These tables are derived from computing against -eligibility criteria in the respective standards using the command-line -script `tools/idna-data`. +## Command-line tool -This tool will fetch relevant codepoint data from the Unicode repository -and perform the required calculations to identify eligibility. There are -three main modes: +The package supports command-line usage to convert domain names +between their Unicode and ASCII-compatible forms. It can be run either +as a module (`python3 -m idna`) or, once installed (such as with `uv +tool` or `pipx`), via the `idna` script: -* `idna-data make-libdata`. Generates `idnadata.py` and - `uts46data.py`, the pre-calculated lookup tables used for IDNA and - UTS 46 conversions. Implementers who wish to track this library against - a different Unicode version may use this tool to manually generate a - different version of the `idnadata.py` and `uts46data.py` files. +```bash +$ uv tool install idna +$ idna xn--e1afmkfd.xn--p1ai +пример.рф +$ idna пример.рф +xn--e1afmkfd.xn--p1ai +``` -* `idna-data make-table`. Generate a table of the IDNA disposition - (e.g. PVALID, CONTEXTJ, CONTEXTO) in the format found in Appendix - B.1 of RFC 5892 and the pre-computed tables published by [IANA](https://www.iana.org/). +With no mode flag the direction is chosen automatically: inputs +containing an `xn--` label are decoded, anything else is encoded. Pass +`-e`/`--encode` or `-d`/`--decode` to force a specific direction. -* `idna-data U+0061`. Prints debugging output on the various - properties associated with an individual Unicode codepoint (in this - case, U+0061), that are used to assess the IDNA and UTS 46 status of a - codepoint. This is helpful in debugging or analysis. +Multiple domains may be supplied at once, either as positional arguments +or by piping one domain per line on standard input. When more than +one domain is supplied without explicitly asking to encode or decode, +the direction is picked from the first input and that mode is applied +to every remaining input. Use `-e`/`--encode` or `-d`/`--decode` to +override the heuristic if the first input is ambiguous. -The tool accepts a number of arguments, described using `idna-data -h`. -Most notably, the `--version` argument allows the specification -of the version of Unicode to be used in computing the table data. For -example, `idna-data --version 9.0.0 make-libdata` will generate -library data against Unicode 9.0.0. +UTS #46 mapping is applied by default, which lets the tool accept +inputs that aren't strictly valid IDNA 2008 by normalising them first: +```bash +$ idna ΠΑΡΆΔΕΙΓΜΑ.ΕΛ +xn--hxajbheg2az3al.xn--qxam +``` -## Additional Notes +Pass `--strict` to disable UTS #46 and apply IDNA 2008 rules verbatim; +the same input will then be rejected. -* **Packages**. The latest tagged release version is published in the - [Python Package Index](https://pypi.org/project/idna/). +Conversion failures are reported on stderr together with the +offending input; processing continues with the remaining domains and +the tool exits with a non-zero status if any conversion failed. -* **Version support**. This library supports Python 3.8 and higher. - As this library serves as a low-level toolkit for a variety of - applications, many of which strive for broad compatibility with older - Python versions, there is no rush to remove older interpreter support. - Support for older versions are likely to be removed from new releases - as automated tests can no longer easily be run, i.e. once the Python - version is officially end-of-life. -* **Testing**. The library has a test suite based on each rule of the - IDNA specification, as well as tests that are provided as part of the - Unicode Technical Standard 46, [Unicode IDNA Compatibility Processing](https://unicode.org/reports/tr46/). +## Additional Notes + +* **Version support**. This library supports Python 3.9 and higher. + As this library serves as a low-level toolkit for a variety of + applications, we strive to support all versions of Python that are + not beyond end-of-life. * **Emoji**. It is an occasional request to support emoji domains in this library. Encoding of symbols like emoji is expressly prohibited by the IDNA technical standard, and emoji domains are broadly phased out across the domain industry due to associated security risks. -* **Transitional processing**. Unicode 16.0.0 removed transitional - processing so the `transitional` argument for the encode() method - no longer has any effect and will be removed at a later date. +* **Regenerating lookup tables**. The IDNA and UTS 46 functionality + relies upon pre-calculated lookup tables, generated using the + `idna-data` script in [`tools/`](tools/README.md). diff --git a/contrib/python/idna/py3/idna/__main__.py b/contrib/python/idna/py3/idna/__main__.py new file mode 100644 index 00000000000..dbdd0661721 --- /dev/null +++ b/contrib/python/idna/py3/idna/__main__.py @@ -0,0 +1,6 @@ +import sys + +from .cli import main + +if __name__ == "__main__": + sys.exit(main()) diff --git a/contrib/python/idna/py3/idna/cli.py b/contrib/python/idna/py3/idna/cli.py new file mode 100644 index 00000000000..4acda2c0f03 --- /dev/null +++ b/contrib/python/idna/py3/idna/cli.py @@ -0,0 +1,128 @@ +"""Command-line interface for the :mod:`idna` package. + +Invoked via ``python -m idna``. See :func:`main` for the entry point. +""" + +import argparse +import sys +from collections.abc import Iterable +from itertools import chain +from typing import IO, Optional + +from . import IDNAError, decode, encode +from .core import _alabel_prefix, _unicode_dots_re +from .package_data import __version__ + + +def _looks_like_alabel(s: str) -> bool: + """Return True if any label in ``s`` carries the ``xn--`` ACE prefix.""" + prefix = _alabel_prefix.decode("ascii") + return any(label.lower().startswith(prefix) for label in _unicode_dots_re.split(s)) + + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="python -m idna", + description=( + "Convert a domain name between its Unicode (U-label) and " + "ASCII-compatible (A-label) forms. With no mode flag, the " + "direction is chosen from the first input — if it contains " + "an xn-- label the stream is decoded, otherwise it is " + "encoded — and the same mode is applied to every remaining " + "input. UTS #46 mapping is applied by default; pass " + "--strict to disable it. When no domains are given on the " + "command line and stdin is piped, one domain per line is " + "read from stdin." + ), + ) + mode = parser.add_mutually_exclusive_group() + mode.add_argument( + "-e", + "--encode", + dest="mode", + action="store_const", + const="encode", + help="Encode the input to its ASCII A-label form.", + ) + mode.add_argument( + "-d", + "--decode", + dest="mode", + action="store_const", + const="decode", + help="Decode the input from its ASCII A-label form.", + ) + parser.add_argument( + "--strict", + action="store_true", + help="Disable the default UTS #46 mapping and apply IDNA 2008 rules verbatim.", + ) + parser.add_argument( + "--version", + action="version", + version=f"idna {__version__}", + ) + parser.add_argument( + "domain", + nargs="*", + help="One or more domain names to convert. Omit to read from stdin.", + ) + return parser + + +def _iter_stdin(stream: IO[str]) -> Iterable[str]: + """Yield non-empty stripped lines from ``stream``, ignoring blanks.""" + for line in stream: + stripped = line.strip() + if stripped: + yield stripped + + +def _convert_one(domain: str, mode: str, uts46: bool) -> bool: + """Convert ``domain`` and write the result; return ``False`` on failure.""" + try: + if mode == "decode": + print(decode(domain, uts46=uts46)) + else: + print(encode(domain, uts46=uts46).decode("ascii")) + except IDNAError as err: + print(f"idna: {mode} failed for {domain!r}: {err}", file=sys.stderr) + return False + return True + + +def main(argv: Optional[list[str]] = None) -> int: + """Entry point for ``python -m idna``. + + When more than one domain is supplied (via positional arguments or + piped stdin) and no mode flag is given, the first input determines + the direction and that mode is applied uniformly to the rest. + + :param argv: Argument list excluding the program name. Defaults to + :data:`sys.argv` when ``None``. + :returns: ``0`` on success, ``1`` if any conversion fails. + """ + parser = _build_parser() + args = parser.parse_args(argv) + uts46 = not args.strict + + if args.domain: + domains: Iterable[str] = args.domain + elif not sys.stdin.isatty(): + domains = _iter_stdin(sys.stdin) + else: + parser.error("a domain argument is required when stdin is a terminal") + + iterator = iter(domains) + first = next(iterator, None) + if first is None: + return 0 + + mode = args.mode or ("decode" if _looks_like_alabel(first) else "encode") + + results = [_convert_one(domain, mode, uts46) for domain in chain([first], iterator)] + return 0 if all(results) else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/contrib/python/idna/py3/idna/codec.py b/contrib/python/idna/py3/idna/codec.py index 280dc3972cb..83b42fe42b5 100644 --- a/contrib/python/idna/py3/idna/codec.py +++ b/contrib/python/idna/py3/idna/codec.py @@ -1,5 +1,5 @@ import codecs -from typing import Any, Optional, Tuple +from typing import Any, Optional from .core import IDNAError, _unicode_dots_re, alabel, decode, encode, ulabel @@ -15,7 +15,7 @@ class Codec(codecs.Codec): raises :exc:`~idna.IDNAError`. """ - def encode(self, data: str, errors: str = "strict") -> Tuple[bytes, int]: # ty: ignore[invalid-method-override] + def encode(self, data: str, errors: str = "strict") -> tuple[bytes, int]: # ty: ignore[invalid-method-override] if errors != "strict": raise IDNAError(f'Unsupported error handling "{errors}"') @@ -24,7 +24,7 @@ class Codec(codecs.Codec): return encode(data), len(data) - def decode(self, data: bytes, errors: str = "strict") -> Tuple[str, int]: # ty: ignore[invalid-method-override] + def decode(self, data: bytes, errors: str = "strict") -> tuple[str, int]: # ty: ignore[invalid-method-override] if errors != "strict": raise IDNAError(f'Unsupported error handling "{errors}"') @@ -46,7 +46,7 @@ class IncrementalEncoder(codecs.BufferedIncrementalEncoder): Only the ``"strict"`` error handler is supported. """ - def _buffer_encode(self, data: str, errors: str, final: bool) -> Tuple[bytes, int]: # ty: ignore[invalid-method-override] + def _buffer_encode(self, data: str, errors: str, final: bool) -> tuple[bytes, int]: # ty: ignore[invalid-method-override] if errors != "strict": raise IDNAError(f'Unsupported error handling "{errors}"') @@ -89,7 +89,7 @@ class IncrementalDecoder(codecs.BufferedIncrementalDecoder): Only the ``"strict"`` error handler is supported. """ - def _buffer_decode(self, data: Any, errors: str, final: bool) -> Tuple[str, int]: # ty: ignore[invalid-method-override] + def _buffer_decode(self, data: Any, errors: str, final: bool) -> tuple[str, int]: # ty: ignore[invalid-method-override] if errors != "strict": raise IDNAError(f'Unsupported error handling "{errors}"') diff --git a/contrib/python/idna/py3/idna/core.py b/contrib/python/idna/py3/idna/core.py index b6f9442deb3..da45b2ae9e4 100644 --- a/contrib/python/idna/py3/idna/core.py +++ b/contrib/python/idna/py3/idna/core.py @@ -27,26 +27,18 @@ _bidi_joiner_r_or_d = frozenset({ord("R"), ord("D")}) class IDNAError(UnicodeError): """Base exception for all IDNA-encoding related problems""" - pass - class IDNABidiError(IDNAError): """Exception when bidirectional requirements are not satisfied""" - pass - class InvalidCodepoint(IDNAError): """Exception when a disallowed or unallocated codepoint is used""" - pass - class InvalidCodepointContext(IDNAError): """Exception when the codepoint is not valid in the context it is used""" - pass - def _combining_class(cp: int) -> int: v = unicodedata.combining(chr(cp)) @@ -118,7 +110,7 @@ def check_bidi(label: str, check_ltr: bool = False) -> bool: direction = unicodedata.bidirectional(cp) if direction == "": # String likely comes from a newer version of Unicode - raise IDNABidiError(f"Unknown directionality in label {repr(label)} at position {idx}") + raise IDNABidiError(f"Unknown directionality in label {label!r} at position {idx}") if direction in _bidi_rtl_categories: bidi_label = True if not bidi_label and not check_ltr: @@ -131,7 +123,7 @@ def check_bidi(label: str, check_ltr: bool = False) -> bool: elif direction == "L": rtl = False else: - raise IDNABidiError(f"First codepoint in label {repr(label)} must be directionality L, R or AL") + raise IDNABidiError(f"First codepoint in label {label!r} must be directionality L, R or AL") valid_ending = False number_type: Optional[str] = None @@ -151,9 +143,8 @@ def check_bidi(label: str, check_ltr: bool = False) -> bool: if direction in _bidi_rtl_numeric: if not number_type: number_type = direction - else: - if number_type != direction: - raise IDNABidiError("Can not mix numeral types in a right-to-left label") + elif number_type != direction: + raise IDNABidiError("Can not mix numeral types in a right-to-left label") else: # Bidi rule 5 if direction not in _bidi_ltr_allowed: @@ -239,11 +230,10 @@ def valid_contextj(label: str, pos: int) -> bool: joining_type = idnadata.joining_types().get(ord(label[i])) if joining_type == ord("T"): continue - elif joining_type in _bidi_joiner_l_or_d: + if joining_type in _bidi_joiner_l_or_d: ok = True break - else: - break + break if not ok: return False @@ -253,18 +243,16 @@ def valid_contextj(label: str, pos: int) -> bool: joining_type = idnadata.joining_types().get(ord(label[i])) if joining_type == ord("T"): continue - elif joining_type in _bidi_joiner_r_or_d: + if joining_type in _bidi_joiner_r_or_d: ok = True break - else: - break + break return ok if cp_value == 0x200D: return pos > 0 and _combining_class(ord(label[pos - 1])) == _virama_combining_class - else: - return False + return False def valid_contexto(label: str, pos: int, exception: bool = False) -> bool: @@ -286,17 +274,17 @@ def valid_contexto(label: str, pos: int, exception: bool = False) -> bool: if cp_value == 0x00B7: return 0 < pos < len(label) - 1 and ord(label[pos - 1]) == 0x006C and ord(label[pos + 1]) == 0x006C - elif cp_value == 0x0375: + if cp_value == 0x0375: if pos < len(label) - 1 and len(label) > 1: return _is_script(label[pos + 1], "Greek") return False - elif cp_value == 0x05F3 or cp_value == 0x05F4: + if cp_value in {0x05F3, 0x05F4}: if pos > 0: return _is_script(label[pos - 1], "Hebrew") return False - elif cp_value == 0x30FB: + if cp_value == 0x30FB: for cp in label: if cp == "\u30fb": continue @@ -304,10 +292,10 @@ def valid_contexto(label: str, pos: int, exception: bool = False) -> bool: return True return False - elif 0x660 <= cp_value <= 0x669: + if 0x660 <= cp_value <= 0x669: return not any(0x6F0 <= ord(cp) <= 0x06F9 for cp in label) - elif 0x6F0 <= cp_value <= 0x6F9: + if 0x6F0 <= cp_value <= 0x6F9: return not any(0x660 <= ord(cp) <= 0x0669 for cp in label) return False @@ -349,23 +337,19 @@ def check_label(label: Union[str, bytes, bytearray]) -> None: cp_value = ord(cp) if intranges_contain(cp_value, idnadata.codepoint_classes["PVALID"]): continue - elif intranges_contain(cp_value, idnadata.codepoint_classes["CONTEXTJ"]): + if intranges_contain(cp_value, idnadata.codepoint_classes["CONTEXTJ"]): try: if not valid_contextj(label, pos): - raise InvalidCodepointContext( - f"Joiner {_unot(cp_value)} not allowed at position {pos + 1} in {repr(label)}" - ) + raise InvalidCodepointContext(f"Joiner {_unot(cp_value)} not allowed at position {pos + 1} in {label!r}") except ValueError as err: raise IDNAError( - f"Unknown codepoint adjacent to joiner {_unot(cp_value)} at position {pos + 1} in {repr(label)}" + f"Unknown codepoint adjacent to joiner {_unot(cp_value)} at position {pos + 1} in {label!r}" ) from err elif intranges_contain(cp_value, idnadata.codepoint_classes["CONTEXTO"]): if not valid_contexto(label, pos): - raise InvalidCodepointContext( - f"Codepoint {_unot(cp_value)} not allowed at position {pos + 1} in {repr(label)}" - ) + raise InvalidCodepointContext(f"Codepoint {_unot(cp_value)} not allowed at position {pos + 1} in {label!r}") else: - raise InvalidCodepoint(f"Codepoint {_unot(cp_value)} at position {pos + 1} of {repr(label)} not allowed") + raise InvalidCodepoint(f"Codepoint {_unot(cp_value)} at position {pos + 1} of {label!r} not allowed") check_bidi(label) @@ -385,12 +369,13 @@ def alabel(label: str) -> bytes: """ try: label_bytes = label.encode("ascii") + except UnicodeEncodeError: + pass + else: ulabel(label_bytes) if not valid_label_length(label_bytes): raise IDNAError("Label too long") return label_bytes - except UnicodeEncodeError: - pass check_label(label) label_bytes = _alabel_prefix + _punycode(label) @@ -492,7 +477,7 @@ def uts46_remap(domain: str, std3_rules: bool = True, transitional: bool = False elif status == "I": continue else: - raise InvalidCodepoint(f"Codepoint {_unot(code_point)} not allowed at position {pos + 1} in {repr(domain)}") + raise InvalidCodepoint(f"Codepoint {_unot(code_point)} not allowed at position {pos + 1} in {domain!r}") return unicodedata.normalize("NFC", output) diff --git a/contrib/python/idna/py3/idna/idnadata.py b/contrib/python/idna/py3/idna/idnadata.py index 8335f26059c..d768948790b 100644 --- a/contrib/python/idna/py3/idna/idnadata.py +++ b/contrib/python/idna/py3/idna/idnadata.py @@ -1,7 +1,6 @@ # This file is automatically generated by tools/idna-data -from functools import lru_cache -from typing import Dict +from functools import cache __version__ = "17.0.0" @@ -105,8 +104,8 @@ scripts = { } -@lru_cache(None) -def joining_types() -> Dict[int, int]: +@cache +def joining_types() -> dict[int, int]: return { 0xAD: 84, 0x300: 84, diff --git a/contrib/python/idna/py3/idna/intranges.py b/contrib/python/idna/py3/idna/intranges.py index ea3455bb8da..19d77810caf 100644 --- a/contrib/python/idna/py3/idna/intranges.py +++ b/contrib/python/idna/py3/idna/intranges.py @@ -6,10 +6,9 @@ in the original list?" in time O(log(# runs)). """ import bisect -from typing import List, Tuple -def intranges_from_list(list_: List[int]) -> Tuple[int, ...]: +def intranges_from_list(list_: list[int]) -> tuple[int, ...]: """Represent a list of integers as a sequence of ranges: ((start_0, end_0), (start_1, end_1), ...), such that the original integers are exactly those x such that start_i <= x < end_i for some i. @@ -34,11 +33,11 @@ def _encode_range(start: int, end: int) -> int: return (start << 32) | end -def _decode_range(r: int) -> Tuple[int, int]: +def _decode_range(r: int) -> tuple[int, int]: return (r >> 32), (r & ((1 << 32) - 1)) -def intranges_contain(int_: int, ranges: Tuple[int, ...]) -> bool: +def intranges_contain(int_: int, ranges: tuple[int, ...]) -> bool: """Determine if `int_` falls into one of the ranges in `ranges`.""" tuple_ = _encode_range(int_, 0) pos = bisect.bisect_left(ranges, tuple_) diff --git a/contrib/python/idna/py3/idna/package_data.py b/contrib/python/idna/py3/idna/package_data.py index 75debb75d6f..560eb22d4d0 100644 --- a/contrib/python/idna/py3/idna/package_data.py +++ b/contrib/python/idna/py3/idna/package_data.py @@ -1 +1 @@ -__version__ = "3.15" +__version__ = "3.16" diff --git a/contrib/python/idna/py3/idna/uts46data.py b/contrib/python/idna/py3/idna/uts46data.py index e13aa90fb86..f728fd96967 100644 --- a/contrib/python/idna/py3/idna/uts46data.py +++ b/contrib/python/idna/py3/idna/uts46data.py @@ -1,13 +1,13 @@ # This file is automatically generated by tools/idna-data -from typing import Tuple, Union +from typing import Union """IDNA Mapping Table from UTS46.""" __version__ = "17.0.0" -uts46data: Tuple[Union[Tuple[int, str], Tuple[int, str, str]], ...] = ( +uts46data: tuple[Union[tuple[int, str], tuple[int, str, str]], ...] = ( (0x0, "V"), (0x1, "V"), (0x2, "V"), diff --git a/contrib/python/idna/py3/patches/01-fix-tests.patch b/contrib/python/idna/py3/patches/01-fix-tests.patch new file mode 100644 index 00000000000..ebd7c115b2e --- /dev/null +++ b/contrib/python/idna/py3/patches/01-fix-tests.patch @@ -0,0 +1,26 @@ +--- contrib/python/idna/py3/tests/test_idna_cli.py (index) ++++ contrib/python/idna/py3/tests/test_idna_cli.py (working tree) +@@ -278,6 +278,7 @@ class CLIModuleEntryTests(unittest.TestCase): + check=False, + capture_output=True, + text=True, ++ env={"Y_PYTHON_ENTRY_POINT": ":main"}, + ) + self.assertEqual(result.returncode, 0) + self.assertEqual(result.stdout.strip(), "テスト") +@@ -291,6 +292,7 @@ class CLIModuleEntryTests(unittest.TestCase): + check=False, + capture_output=True, + text=True, ++ env={"Y_PYTHON_ENTRY_POINT": ":main"}, + ) + self.assertEqual(result.returncode, 0) + self.assertEqual( +@@ -305,6 +307,7 @@ class CLIModuleEntryTests(unittest.TestCase): + check=False, + capture_output=True, + text=True, ++ env={"Y_PYTHON_ENTRY_POINT": ":main"}, + ) + self.assertEqual(result.returncode, 1) + self.assertEqual(result.stdout, "") diff --git a/contrib/python/idna/py3/tests/test_idna.py b/contrib/python/idna/py3/tests/test_idna.py index 65eecfd88ee..e6029aede1f 100644 --- a/contrib/python/idna/py3/tests/test_idna.py +++ b/contrib/python/idna/py3/tests/test_idna.py @@ -2,14 +2,13 @@ import unittest import warnings -from typing import List, Tuple import idna class IDNATests(unittest.TestCase): def setUp(self): - self.tld_strings: List[Tuple[str, bytes]] = [ + self.tld_strings: list[tuple[str, bytes]] = [ ("\u6d4b\u8bd5", b"xn--0zwm56d"), ("\u092a\u0930\u0940\u0915\u094d\u0937\u093e", b"xn--11b5bs3a9aj6g"), ("\ud55c\uad6d", b"xn--3e0b707e"), @@ -104,7 +103,7 @@ class IDNATests(unittest.TestCase): import codecs import time - import idna.codec # noqa: F401 (register the idna2008 codec) + import idna.codec # register the idna2008 codec payload = "・" * 8000 + "漢" start = time.perf_counter() diff --git a/contrib/python/idna/py3/tests/test_idna_cli.py b/contrib/python/idna/py3/tests/test_idna_cli.py new file mode 100644 index 00000000000..fa79aa66b9a --- /dev/null +++ b/contrib/python/idna/py3/tests/test_idna_cli.py @@ -0,0 +1,318 @@ +#!/usr/bin/env python + +import io +import subprocess +import sys +import unittest +from contextlib import contextmanager, redirect_stderr, redirect_stdout +from unittest import mock + +from idna import cli +from idna.package_data import __version__ + + +def run_cli(*argv, stdin=None): + """Invoke ``cli.main`` with ``argv`` and capture (rc, stdout, stderr). + + Pass ``stdin`` as a string to simulate a piped stdin; omit it to + simulate an interactive terminal. + """ + out = io.StringIO() + err = io.StringIO() + with redirect_stdout(out), redirect_stderr(err), _patched_stdin(stdin): + rc = cli.main(list(argv)) + return rc, out.getvalue(), err.getvalue() + + +class _FakeStdin(io.StringIO): + """``io.StringIO`` with a configurable ``isatty`` result.""" + + def __init__(self, data: str, is_tty: bool) -> None: + super().__init__(data) + self._is_tty = is_tty + + def isatty(self) -> bool: + return self._is_tty + + +@contextmanager +def _patched_stdin(data): + """Replace ``sys.stdin`` with a fake stream and isatty() result.""" + stream = _FakeStdin("" if data is None else data, is_tty=data is None) + with mock.patch.object(sys, "stdin", stream): + yield + + +class CLIAutoDetectTests(unittest.TestCase): + def test_autodetect_encodes_unicode_input(self): + rc, out, err = run_cli("пример.рф") + self.assertEqual(rc, 0) + self.assertEqual(out.strip(), "xn--e1afmkfd.xn--p1ai") + self.assertEqual(err, "") + + def test_autodetect_decodes_alabel(self): + rc, out, _ = run_cli("xn--e1afmkfd.xn--p1ai") + self.assertEqual(rc, 0) + self.assertEqual(out.strip(), "пример.рф") + + def test_autodetect_decodes_when_any_label_is_alabel(self): + # Mixed label: at least one xn-- label triggers decode mode. + rc, out, _ = run_cli("xn--11b5bs3a9aj6g.example") + self.assertEqual(rc, 0) + self.assertEqual(out.strip(), "परीक्षा.example") + + def test_autodetect_encodes_plain_ascii_input(self): + rc, out, _ = run_cli("example.com") + self.assertEqual(rc, 0) + self.assertEqual(out.strip(), "example.com") + + def test_looks_like_alabel_is_case_insensitive(self): + self.assertTrue(cli._looks_like_alabel("XN--ZCKZAH")) + self.assertTrue(cli._looks_like_alabel("foo.Xn--ZcKzAh")) + self.assertFalse(cli._looks_like_alabel("example.com")) + self.assertFalse(cli._looks_like_alabel("xnfoo.example")) + + def test_looks_like_alabel_handles_unicode_dots(self): + # IDEOGRAPHIC FULL STOP should split labels for the purpose of + # spotting an A-label, matching the behavior of idna.encode/decode. + self.assertTrue(cli._looks_like_alabel("foo。xn--zckzah")) + + +class CLIExplicitModeTests(unittest.TestCase): + def test_encode_flag_forces_encode(self): + rc, out, _ = run_cli("-e", "παράδειγμα") + self.assertEqual(rc, 0) + self.assertEqual(out.strip(), "xn--hxajbheg2az3al") + + def test_long_encode_flag_forces_encode(self): + rc, out, _ = run_cli("--encode", "한국") + self.assertEqual(rc, 0) + self.assertEqual(out.strip(), "xn--3e0b707e") + + def test_decode_flag_forces_decode(self): + rc, out, _ = run_cli("-d", "xn--hxajbheg2az3al") + self.assertEqual(rc, 0) + self.assertEqual(out.strip(), "παράδειγμα") + + def test_long_decode_flag_forces_decode(self): + rc, out, _ = run_cli("--decode", "xn--3e0b707e") + self.assertEqual(rc, 0) + self.assertEqual(out.strip(), "한국") + + def test_encode_and_decode_are_mutually_exclusive(self): + # argparse exits via SystemExit (code 2) for mutually-exclusive errors. + with self.assertRaises(SystemExit) as ctx, redirect_stderr(io.StringIO()): + cli.main(["-e", "-d", "example.com"]) + self.assertEqual(ctx.exception.code, 2) + + +class CLIUTS46Tests(unittest.TestCase): + def test_uts46_is_applied_by_default(self): + # Uppercase Greek is not PVALID under IDNA 2008 alone; UTS #46 + # case-folds it so encoding succeeds. + rc, out, _ = run_cli("ΠΑΡΆΔΕΙΓΜΑ.ΕΛ") + self.assertEqual(rc, 0) + self.assertEqual(out.strip(), "xn--hxajbheg2az3al.xn--qxam") + + def test_strict_disables_uts46(self): + rc, out, err = run_cli("--strict", "ΠΑΡΆΔΕΙΓΜΑ.ΕΛ") + self.assertEqual(rc, 1) + self.assertEqual(out, "") + self.assertIn("encode failed", err) + self.assertIn("U+03A0", err) + + def test_strict_does_not_affect_valid_inputs(self): + rc, out, _ = run_cli("--strict", "пример.рф") + self.assertEqual(rc, 0) + self.assertEqual(out.strip(), "xn--e1afmkfd.xn--p1ai") + + def test_uts46_applies_to_explicit_decode(self): + # Trailing FULLWIDTH FULL STOP (U+FF0E) is mapped to "." by UTS #46 + # so the input still decodes as a single A-label. + rc, out, _ = run_cli("-d", "xn--zckzah．") + self.assertEqual(rc, 0) + self.assertEqual(out.strip(), "テスト.") + + +class CLIErrorHandlingTests(unittest.TestCase): + def test_invalid_input_returns_nonzero_and_writes_to_stderr(self): + rc, out, err = run_cli("foo_bar") + self.assertEqual(rc, 1) + self.assertEqual(out, "") + self.assertTrue(err.startswith("idna: encode failed for 'foo_bar':")) + + def test_invalid_alabel_decode_returns_nonzero(self): + # Trailing hyphen after the xn-- prefix is rejected by ulabel(). + rc, out, err = run_cli("-d", "xn--") + self.assertEqual(rc, 1) + self.assertEqual(out, "") + self.assertIn("decode failed", err) + + def test_missing_argument_with_tty_stdin_is_an_argparse_error(self): + # Interactive shell (no piped stdin) and no domain arg → argparse exits 2. + with self.assertRaises(SystemExit) as ctx, redirect_stderr(io.StringIO()), _patched_stdin(None): + cli.main([]) + self.assertEqual(ctx.exception.code, 2) + + +class CLIVersionTests(unittest.TestCase): + def test_version_flag(self): + buf = io.StringIO() + with self.assertRaises(SystemExit) as ctx, redirect_stdout(buf): + cli.main(["--version"]) + self.assertEqual(ctx.exception.code, 0) + self.assertIn(__version__, buf.getvalue()) + + +class CLIMultipleDomainsTests(unittest.TestCase): + def test_multiple_unicode_positional_domains_are_all_encoded(self): + rc, out, _ = run_cli("пример.рф", "παράδειγμα", "한국") + self.assertEqual(rc, 0) + self.assertEqual( + out.splitlines(), + ["xn--e1afmkfd.xn--p1ai", "xn--hxajbheg2az3al", "xn--3e0b707e"], + ) + + def test_multiple_alabel_positional_domains_are_all_decoded(self): + rc, out, _ = run_cli("xn--e1afmkfd.xn--p1ai", "xn--hxajbheg2az3al", "xn--3e0b707e") + self.assertEqual(rc, 0) + self.assertEqual(out.splitlines(), ["пример.рф", "παράδειγμα", "한국"]) + + def test_mode_flag_applies_to_every_positional_domain(self): + rc, out, _ = run_cli("-e", "한국", "παράδειγμα") + self.assertEqual(rc, 0) + self.assertEqual( + out.splitlines(), + ["xn--3e0b707e", "xn--hxajbheg2az3al"], + ) + + def test_failures_are_reported_per_domain_and_processing_continues(self): + rc, out, err = run_cli("пример.рф", "foo_bar", "παράδειγμα") + self.assertEqual(rc, 1) + self.assertEqual( + out.splitlines(), + ["xn--e1afmkfd.xn--p1ai", "xn--hxajbheg2az3al"], + ) + self.assertIn("encode failed for 'foo_bar'", err) + + +class CLIModeLockTests(unittest.TestCase): + """The first input picks the mode; the rest of the stream follows.""" + + def test_first_unicode_input_locks_subsequent_inputs_to_encode(self): + # 'foo_bar' is invalid in either direction; the stderr mode word + # reveals which mode the second input was processed in. + rc, out, err = run_cli("пример.рф", "foo_bar") + self.assertEqual(rc, 1) + self.assertEqual(out.splitlines(), ["xn--e1afmkfd.xn--p1ai"]) + self.assertIn("encode failed for 'foo_bar'", err) + + def test_first_alabel_input_locks_subsequent_inputs_to_decode(self): + rc, out, err = run_cli("xn--zckzah", "foo_bar") + self.assertEqual(rc, 1) + self.assertEqual(out.splitlines(), ["テスト"]) + self.assertIn("decode failed for 'foo_bar'", err) + + def test_explicit_mode_flag_overrides_first_input_heuristic(self): + # First input looks like an A-label but -e forces encode for everything. + rc, out, err = run_cli("-e", "xn--zckzah", "foo_bar") + self.assertEqual(rc, 1) + self.assertEqual(out.splitlines(), ["xn--zckzah"]) + self.assertIn("encode failed for 'foo_bar'", err) + + def test_stdin_first_line_locks_mode_for_the_stream(self): + rc, out, err = run_cli(stdin="xn--zckzah\nfoo_bar\n") + self.assertEqual(rc, 1) + self.assertEqual(out.splitlines(), ["テスト"]) + self.assertIn("decode failed for 'foo_bar'", err) + + +class CLIStdinTests(unittest.TestCase): + def test_reads_one_domain_per_line_from_piped_stdin(self): + rc, out, _ = run_cli(stdin="пример.рф\nπαράδειγμα\n한국\n") + self.assertEqual(rc, 0) + self.assertEqual( + out.splitlines(), + ["xn--e1afmkfd.xn--p1ai", "xn--hxajbheg2az3al", "xn--3e0b707e"], + ) + + def test_blank_lines_in_stdin_are_skipped(self): + rc, out, _ = run_cli(stdin="\nпример.рф\n\n \nπαράδειγμα\n") + self.assertEqual(rc, 0) + self.assertEqual( + out.splitlines(), + ["xn--e1afmkfd.xn--p1ai", "xn--hxajbheg2az3al"], + ) + + def test_empty_stdin_exits_cleanly_with_no_output(self): + rc, out, err = run_cli(stdin="") + self.assertEqual(rc, 0) + self.assertEqual(out, "") + self.assertEqual(err, "") + + def test_positional_domains_take_precedence_over_piped_stdin(self): + rc, out, _ = run_cli("example.com", stdin="ignored.example\n") + self.assertEqual(rc, 0) + self.assertEqual(out.splitlines(), ["example.com"]) + + def test_stdin_errors_continue_processing_and_set_exit_code(self): + rc, out, err = run_cli(stdin="пример.рф\nfoo_bar\nπαράδειγμα\n") + self.assertEqual(rc, 1) + self.assertEqual( + out.splitlines(), + ["xn--e1afmkfd.xn--p1ai", "xn--hxajbheg2az3al"], + ) + self.assertIn("encode failed for 'foo_bar'", err) + + def test_decode_flag_applies_to_every_stdin_line(self): + rc, out, _ = run_cli("-d", stdin="xn--hxajbheg2az3al\nxn--3e0b707e\n") + self.assertEqual(rc, 0) + self.assertEqual(out.splitlines(), ["παράδειγμα", "한국"]) + + +class CLIModuleEntryTests(unittest.TestCase): + def test_python_dash_m_idna_runs_cli(self): + # End-to-end: spawn ``python -m idna`` and check the round trip. + result = subprocess.run( + [sys.executable, "-m", "idna", "xn--zckzah"], + check=False, + capture_output=True, + text=True, + env={"Y_PYTHON_ENTRY_POINT": ":main"}, + ) + self.assertEqual(result.returncode, 0) + self.assertEqual(result.stdout.strip(), "テスト") + self.assertEqual(result.stderr, "") + + def test_python_dash_m_idna_reads_piped_stdin(self): + # End-to-end: pipe a list of A-labels and confirm they all decode. + result = subprocess.run( + [sys.executable, "-m", "idna"], + input="xn--e1afmkfd.xn--p1ai\nxn--zckzah\n", + check=False, + capture_output=True, + text=True, + env={"Y_PYTHON_ENTRY_POINT": ":main"}, + ) + self.assertEqual(result.returncode, 0) + self.assertEqual( + result.stdout.splitlines(), + ["пример.рф", "テスト"], + ) + self.assertEqual(result.stderr, "") + + def test_python_dash_m_idna_reports_errors(self): + result = subprocess.run( + [sys.executable, "-m", "idna", "foo_bar"], + check=False, + capture_output=True, + text=True, + env={"Y_PYTHON_ENTRY_POINT": ":main"}, + ) + self.assertEqual(result.returncode, 1) + self.assertEqual(result.stdout, "") + self.assertIn("encode failed", result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/contrib/python/idna/py3/ya.make b/contrib/python/idna/py3/ya.make index 6f0b1006c52..8f96cee300d 100644 --- a/contrib/python/idna/py3/ya.make +++ b/contrib/python/idna/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(3.15) +VERSION(3.16) LICENSE(BSD-3-Clause) @@ -11,6 +11,8 @@ NO_LINT() PY_SRCS( TOP_LEVEL idna/__init__.py + idna/__main__.py + idna/cli.py idna/codec.py idna/compat.py idna/core.py @@ -23,6 +25,7 @@ PY_SRCS( RESOURCE_FILES( PREFIX contrib/python/idna/py3/ .dist-info/METADATA + .dist-info/entry_points.txt .dist-info/top_level.txt idna/py.typed ) -- cgit v1.3 From 4c00d06b256d2c77547ae74d0522c0a7b15d73df Mon Sep 17 00:00:00 2001 From: robot-piglet Date: Sun, 26 Jul 2026 00:00:31 +0300 Subject: Intermediate changes commit_hash:b6fa234df15248845c18fb3f06745b0e7a26ea47 --- contrib/python/yarl/.dist-info/METADATA | 56 ++++++++++- .../patches/02-dont-normalize-disable-tests.patch | 22 ----- contrib/python/yarl/tests/test_cache.py | 12 ++- contrib/python/yarl/tests/test_url.py | 29 ++++++ contrib/python/yarl/tests/test_url_build.py | 62 ++++++++++-- contrib/python/yarl/tests/test_url_parsing.py | 25 +++++ .../python/yarl/tests/test_url_update_netloc.py | 20 ++++ contrib/python/yarl/ya.make | 2 +- contrib/python/yarl/yarl/__init__.py | 2 +- contrib/python/yarl/yarl/_url.py | 106 ++++++++++++++++++--- 10 files changed, 285 insertions(+), 51 deletions(-) diff --git a/contrib/python/yarl/.dist-info/METADATA b/contrib/python/yarl/.dist-info/METADATA index 9870bc4c565..97585939dab 100644 --- a/contrib/python/yarl/.dist-info/METADATA +++ b/contrib/python/yarl/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: yarl -Version: 1.12.1 +Version: 1.13.1 Summary: Yet another URL library Home-page: https://github.com/aio-libs/yarl Author: Andrew Svetlov @@ -262,6 +262,60 @@ Changelog .. towncrier release notes start +1.13.1 +====== + +*(2024-09-27)* + + +Miscellaneous internal changes +------------------------------ + +- Improved performance of calling ``yarl.URL.build()`` with ``authority`` -- by `@bdraco `__. + + *Related issues and pull requests on GitHub:* + `#1163 `__. + + +---- + + +1.13.0 +====== + +*(2024-09-26)* + + +Bug fixes +--------- + +- Started rejecting ASCII hostnames with invalid characters. For host strings that + look like authority strings, the exception message includes advice on what to do + instead -- by `@mjpieters `__. + + *Related issues and pull requests on GitHub:* + `#880 `__, `#954 `__. + +- Fixed IPv6 addresses missing brackets when the ``~yarl.URL`` was converted to a string -- by `@bdraco `__. + + *Related issues and pull requests on GitHub:* + `#1157 `__, `#1158 `__. + + +Features +-------- + +- Added ``~yarl.URL.host_subcomponent`` which returns the ``3986#section-3.2.2`` host subcomponent -- by `@bdraco `__. + + The only current practical difference between ``~yarl.URL.raw_host`` and ``~yarl.URL.host_subcomponent`` is that IPv6 addresses are returned bracketed. + + *Related issues and pull requests on GitHub:* + `#1159 `__. + + +---- + + 1.12.1 ====== diff --git a/contrib/python/yarl/patches/02-dont-normalize-disable-tests.patch b/contrib/python/yarl/patches/02-dont-normalize-disable-tests.patch index 79c9ccc2d3e..49ecf81c0ff 100644 --- a/contrib/python/yarl/patches/02-dont-normalize-disable-tests.patch +++ b/contrib/python/yarl/patches/02-dont-normalize-disable-tests.patch @@ -8,28 +8,6 @@ def test_is_default_port_for_absolute_url_with_default_port(): url = URL("http://example.com:80") assert url.is_default_port() ---- contrib/python/yarl/tests/test_url_build.py (index) -+++ contrib/python/yarl/tests/test_url_build.py (working tree) -@@ -15,16 +15,19 @@ def test_build_simple(): - assert str(u) == "http://127.0.0.1" - - -+@pytest.mark.skip - def test_url_build_ipv6(): - u = URL.build(scheme="http", host="::1") - assert str(u) == "http://::1" - - -+@pytest.mark.skip - def test_url_build_ipv6_brackets(): - u = URL.build(scheme="http", host="[::1]") - assert str(u) == "http://::1" - - -+@pytest.mark.skip - def test_url_ipv4_in_ipv6(): - u = URL.build(scheme="http", host="2001:db8:122:344::192.0.2.33") - assert str(u) == "http://2001:db8:122:344::c000:221" --- contrib/python/yarl/tests/test_url_update_netloc.py (index) +++ contrib/python/yarl/tests/test_url_update_netloc.py (working tree) @@ -196,6 +196,7 @@ def test_with_port(): diff --git a/contrib/python/yarl/tests/test_cache.py b/contrib/python/yarl/tests/test_cache.py index 46d5b01f159..9b0e75a3551 100644 --- a/contrib/python/yarl/tests/test_cache.py +++ b/contrib/python/yarl/tests/test_cache.py @@ -13,7 +13,7 @@ def test_cache_clear() -> None: def test_cache_info() -> None: info = yarl.cache_info() - assert info.keys() == {"idna_encode", "idna_decode", "ip_address"} + assert info.keys() == {"idna_encode", "idna_decode", "ip_address", "host_validate"} def test_cache_configure_default() -> None: @@ -22,11 +22,17 @@ def test_cache_configure_default() -> None: def test_cache_configure_None() -> None: yarl.cache_configure( - idna_encode_size=None, idna_decode_size=None, ip_address_size=None + idna_encode_size=None, + idna_decode_size=None, + ip_address_size=None, + host_validate_size=None, ) def test_cache_configure_explicit() -> None: yarl.cache_configure( - idna_encode_size=128, idna_decode_size=128, ip_address_size=128 + idna_encode_size=128, + idna_decode_size=128, + ip_address_size=128, + host_validate_size=128, ) diff --git a/contrib/python/yarl/tests/test_url.py b/contrib/python/yarl/tests/test_url.py index 1dd98197fe1..44018c73bce 100644 --- a/contrib/python/yarl/tests/test_url.py +++ b/contrib/python/yarl/tests/test_url.py @@ -176,6 +176,24 @@ def test_raw_host(): assert url.raw_host == url._val.hostname +@pytest.mark.parametrize( + ("host"), + [ + ("example.com"), + ("[::1]"), + ("xn--gnter-4ya.com"), + ], +) +def test_host_subcomponent(host: str): + url = URL(f"http://{host}") + assert url.host_subcomponent == host + + +def test_host_subcomponent_return_idna_encoded_host(): + url = URL("http://оун-упа.укр") + assert url.host_subcomponent == "xn----8sb1bdhvc.xn--j1amh" + + def test_raw_host_non_ascii(): url = URL("http://оун-упа.укр") assert "xn----8sb1bdhvc.xn--j1amh" == url.raw_host @@ -2041,3 +2059,14 @@ def test_parsing_populates_cache(): assert url._cache["raw_query_string"] == "a=b" assert url._cache["raw_fragment"] == "frag" assert url._cache["scheme"] == "http" + + +@pytest.mark.parametrize( + ("host", "is_authority"), + [ + *(("other_gen_delim_" + c, False) for c in "[]"), + ], +) +def test_build_with_invalid_ipv6_host(host: str, is_authority: bool): + with pytest.raises(ValueError, match="Invalid IPv6 URL"): + URL(f"http://{host}/") diff --git a/contrib/python/yarl/tests/test_url_build.py b/contrib/python/yarl/tests/test_url_build.py index a0ab77b2d4e..cc9b15f7e4f 100644 --- a/contrib/python/yarl/tests/test_url_build.py +++ b/contrib/python/yarl/tests/test_url_build.py @@ -15,22 +15,24 @@ def test_build_simple(): assert str(u) == "http://127.0.0.1" -@pytest.mark.skip def test_url_build_ipv6(): u = URL.build(scheme="http", host="::1") - assert str(u) == "http://::1" + assert str(u) == "http://[::1]" -@pytest.mark.skip -def test_url_build_ipv6_brackets(): - u = URL.build(scheme="http", host="[::1]") - assert str(u) == "http://::1" +def test_url_build_ipv6_brackets_encoded(): + u = URL.build(scheme="http", host="[::1]", encoded=True) + assert str(u) == "http://[::1]" + + +def test_url_build_ipv6_brackets_not_encoded(): + u = URL.build(scheme="http", host="::1", encoded=False) + assert str(u) == "http://[::1]" -@pytest.mark.skip def test_url_ipv4_in_ipv6(): u = URL.build(scheme="http", host="2001:db8:122:344::192.0.2.33") - assert str(u) == "http://2001:db8:122:344::c000:221" + assert str(u) == "http://[2001:db8:122:344::c000:221]" def test_build_with_scheme(): @@ -118,6 +120,25 @@ def test_build_with_authority_and_host(): URL.build(authority="host.com", host="example.com") +@pytest.mark.parametrize( + ("host", "is_authority"), + [ + ("user:pass@host.com", True), + ("user@host.com", True), + ("host:com", False), + ("not_percent_encoded%Zf", False), + ("still_not_percent_encoded%fZ", False), + *(("other_gen_delim_" + c, False) for c in "/?#[]"), + ], +) +def test_build_with_invalid_host(host: str, is_authority: bool): + match = r"Host '[^']+' cannot contain '[^']+' \(at position \d+\)" + if is_authority: + match += ", if .* use 'authority' instead of 'host'" + with pytest.raises(ValueError, match=f"{match}$"): + URL.build(host=host) + + def test_build_with_authority(): url = URL.build(scheme="http", authority="степан:bar@host.com:8000", path="path") assert ( @@ -132,6 +153,31 @@ def test_build_with_authority_without_encoding(): assert str(url) == "http://foo:bar@host.com:8000/path" +def test_build_with_authority_empty_host_no_scheme(): + url = URL.build(authority="", path="path") + assert str(url) == "path" + + +def test_build_with_authority_and_only_user(): + url = URL.build(scheme="https", authority="user:@foo.com", path="/path") + assert str(url) == "https://user:@foo.com/path" + + +def test_build_with_authority_with_port(): + url = URL.build(scheme="https", authority="foo.com:8080", path="/path") + assert str(url) == "https://foo.com:8080/path" + + +def test_build_with_authority_with_ipv6(): + url = URL.build(scheme="https", authority="[::1]", path="/path") + assert str(url) == "https://[::1]/path" + + +def test_build_with_authority_with_ipv6_and_port(): + url = URL.build(scheme="https", authority="[::1]:81", path="/path") + assert str(url) == "https://[::1]:81/path" + + def test_query_str(): u = URL.build(scheme="http", host="127.0.0.1", path="/", query_string="arg=value1") assert str(u) == "http://127.0.0.1/?arg=value1" diff --git a/contrib/python/yarl/tests/test_url_parsing.py b/contrib/python/yarl/tests/test_url_parsing.py index 4fe95185e71..0d58c9d09bb 100644 --- a/contrib/python/yarl/tests/test_url_parsing.py +++ b/contrib/python/yarl/tests/test_url_parsing.py @@ -604,3 +604,28 @@ def test_schemes_that_require_host(scheme: str) -> None: ) with pytest.raises(ValueError, match=expect): URL(f"{scheme}://:1") + + +@pytest.mark.parametrize( + ("url", "hostname", "hostname_without_brackets"), + [ + ("http://[::1]", "[::1]", "::1"), + ("http://[::1]:8080", "[::1]", "::1"), + ("http://127.0.0.1:8080", "127.0.0.1", "127.0.0.1"), + ( + "http://xn--jxagkqfkduily1i.eu", + "xn--jxagkqfkduily1i.eu", + "xn--jxagkqfkduily1i.eu", + ), + ], +) +def test_url_round_trips( + url: str, hostname: str, hostname_without_brackets: str +) -> None: + """Verify that URLs round-trip correctly.""" + parsed = URL(url) + assert parsed._val.hostname == hostname_without_brackets + assert parsed.raw_host == hostname_without_brackets + assert parsed.host_subcomponent == hostname + assert str(parsed) == url + assert str(URL(str(parsed))) == url diff --git a/contrib/python/yarl/tests/test_url_update_netloc.py b/contrib/python/yarl/tests/test_url_update_netloc.py index cfe6ca7d457..a8ff1f9f8b3 100644 --- a/contrib/python/yarl/tests/test_url_update_netloc.py +++ b/contrib/python/yarl/tests/test_url_update_netloc.py @@ -171,6 +171,26 @@ def test_with_host_non_ascii(): assert url2.authority == "оун-упа.укр:123" +@pytest.mark.parametrize( + ("host", "is_authority"), + [ + ("user:pass@host.com", True), + ("user@host.com", True), + ("host:com", False), + ("not_percent_encoded%Zf", False), + ("still_not_percent_encoded%fZ", False), + *(("other_gen_delim_" + c, False) for c in "/?#[]"), + ], +) +def test_with_invalid_host(host: str, is_authority: bool): + url = URL("http://example.com:123") + match = r"Host '[^']+' cannot contain '[^']+' \(at position \d+\)" + if is_authority: + match += ", if .* use 'authority' instead of 'host'" + with pytest.raises(ValueError, match=f"{match}$"): + url.with_host(host=host) + + def test_with_host_percent_encoded(): url = URL("http://%25cf%2580%cf%80:%25cf%2580%cf%80@example.com:123") url2 = url.with_host("%cf%80.org") diff --git a/contrib/python/yarl/ya.make b/contrib/python/yarl/ya.make index 43d5345771f..b272e44b23e 100644 --- a/contrib/python/yarl/ya.make +++ b/contrib/python/yarl/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(1.12.1) +VERSION(1.13.1) LICENSE(Apache-2.0) diff --git a/contrib/python/yarl/yarl/__init__.py b/contrib/python/yarl/yarl/__init__.py index 50d24c0e778..1bc12c9e4f4 100644 --- a/contrib/python/yarl/yarl/__init__.py +++ b/contrib/python/yarl/yarl/__init__.py @@ -8,7 +8,7 @@ from ._url import ( cache_info, ) -__version__ = "1.12.1" +__version__ = "1.13.1" __all__ = ( "URL", diff --git a/contrib/python/yarl/yarl/_url.py b/contrib/python/yarl/yarl/_url.py index 8a48628fcd9..e695800b967 100644 --- a/contrib/python/yarl/yarl/_url.py +++ b/contrib/python/yarl/yarl/_url.py @@ -1,4 +1,5 @@ import math +import re import sys import warnings from collections.abc import Mapping, Sequence @@ -45,6 +46,22 @@ SCHEME_REQUIRES_HOST = frozenset(("http", "https", "ws", "wss", "ftp")) sentinel = object() +# reg-name: unreserved / pct-encoded / sub-delims +# this pattern matches anything that is *not* in those classes. and is only used +# on lower-cased ASCII values. +_not_reg_name = re.compile( + r""" + # any character not in the unreserved or sub-delims sets, plus % + # (validated with the additional check for pct-encoded sequences below) + [^a-z0-9\-._~!$&'()*+,;=%] + | + # % only allowed if it is part of a pct-encoded + # sequence of 2 hex digits. + %(?![0-9a-f]{2}) + """, + re.VERBOSE, +) + SimpleQuery = Union[str, int, float] QueryVariable = Union[SimpleQuery, "Sequence[SimpleQuery]"] Query = Union[ @@ -64,6 +81,7 @@ class CacheInfo(TypedDict): idna_encode: _CacheInfo idna_decode: _CacheInfo ip_address: _CacheInfo + host_validate: _CacheInfo class _SplitResultDict(TypedDict, total=False): @@ -269,7 +287,7 @@ class URL: raise ValueError(msg) else: host = "" - host = cls._encode_host(host) + host = cls._encode_host(host, validate_host=False) raw_user = None if username is None else cls._REQUOTER(username) raw_password = None if password is None else cls._REQUOTER(password) netloc = cls._make_netloc( @@ -350,10 +368,15 @@ class URL: if encoded: netloc = authority else: - tmp = SplitResult("", authority, "", "", "") - port = None if tmp.port == DEFAULT_PORTS.get(scheme) else tmp.port + _user, _password, _host, _port = cls._split_netloc(authority) + port = None if _port == DEFAULT_PORTS.get(scheme) else _port + _host = ( + "" + if _host is None + else cls._encode_host(_host, validate_host=False) + ) netloc = cls._make_netloc( - tmp.username, tmp.password, tmp.hostname, port, encode=True + _user, _password, _host, port, encode=True, encode_host=False ) elif not user and not password and not host and not port: netloc = "" @@ -395,7 +418,7 @@ class URL: netloc=self._make_netloc( self.raw_user, self.raw_password, - self.raw_host, + self.host_subcomponent, port, encode_host=False, ) @@ -647,6 +670,8 @@ class URL: None for relative URLs. + When working with IPv6 addresses, use the `host_subcomponent` property instead + as it will return the host subcomponent with brackets. """ # Use host instead of hostname for sake of shortness # May add .hostname prop later @@ -660,16 +685,35 @@ class URL: None for relative URLs. """ - raw = self.raw_host - if raw is None: + if (raw := self.raw_host) is None: return None - if "%" in raw: - # Hack for scoped IPv6 addresses like - # fe80::2%Перевірка - # presence of '%' sign means only IPv6 address, so idna is useless. + if raw and raw[-1].isdigit() or ":" in raw: + # IP addresses are never IDNA encoded return raw return _idna_decode(raw) + @cached_property + def host_subcomponent(self) -> Union[str, None]: + """Return the host subcomponent part of URL. + + None for relative URLs. + + https://datatracker.ietf.org/doc/html/rfc3986#section-3.2.2 + + `IP-literal = "[" ( IPv6address / IPvFuture ) "]"` + + Examples: + - `http://example.com:8080` -> `example.com` + - `http://example.com:80` -> `example.com` + - `https://127.0.0.1:8443` -> `127.0.0.1` + - `https://[::1]:8443` -> `[::1]` + - `http://[::1]` -> `[::1]` + + """ + if (raw := self.raw_host) is None: + return None + return f"[{raw}]" if ":" in raw else raw + @cached_property def port(self) -> Union[int, None]: """Port part of URL, with scheme-based fallback. @@ -938,7 +982,9 @@ class URL: return prefix + "/".join(_normalize_path_segments(segments)) @classmethod - def _encode_host(cls, host: str, human: bool = False) -> str: + def _encode_host( + cls, host: str, human: bool = False, validate_host: bool = True + ) -> str: if "%" in host: raw_ip, sep, zone = host.partition("%") else: @@ -953,7 +999,8 @@ class URL: # - 127.0.0.1 (last character is a digit) # - 2001:db8::ff00:42:8329 (contains a colon) # - 2001:db8::ff00:42:8329%eth0 (contains a colon) - # - [2001:db8::ff00:42:8329] (contains a colon) + # - [2001:db8::ff00:42:8329] (contains a colon -- brackets should + # have been removed before it gets here) # Rare IP Address formats are not supported per: # https://datatracker.ietf.org/doc/html/rfc3986#section-7.4 # @@ -977,11 +1024,18 @@ class URL: return host host = host.lower() + if human: + return host + # IDNA encoding is slow, # skip it for ASCII-only strings # Don't move the check into _idna_encode() helper # to reduce the cache size - if human or host.isascii(): + if host.isascii(): + # Check for invalid characters explicitly; _idna_encode() does this + # for non-ascii host names. + if validate_host: + _host_validate(host) return host return _idna_encode(host) @@ -1537,12 +1591,31 @@ def _ip_compressed_version(raw_ip: str) -> Tuple[str, int]: return ip.compressed, ip.version +@lru_cache(_MAXCACHE) +def _host_validate(host: str) -> None: + """Validate an ascii host name.""" + invalid = _not_reg_name.search(host) + if invalid is None: + return + value, pos, extra = invalid.group(), invalid.start(), "" + if value == "@" or (value == ":" and "@" in host[pos:]): + # this looks like an authority string + extra = ( + ", if the value includes a username or password, " + "use 'authority' instead of 'host'" + ) + raise ValueError( + f"Host {host!r} cannot contain {value!r} (at position " f"{pos}){extra}" + ) from None + + @rewrite_module def cache_clear() -> None: """Clear all LRU caches.""" _idna_decode.cache_clear() _idna_encode.cache_clear() _ip_compressed_version.cache_clear() + _host_validate.cache_clear() @rewrite_module @@ -1552,6 +1625,7 @@ def cache_info() -> CacheInfo: "idna_encode": _idna_encode.cache_info(), "idna_decode": _idna_decode.cache_info(), "ip_address": _ip_compressed_version.cache_info(), + "host_validate": _host_validate.cache_info(), } @@ -1561,12 +1635,14 @@ def cache_configure( idna_encode_size: Union[int, None] = _MAXCACHE, idna_decode_size: Union[int, None] = _MAXCACHE, ip_address_size: Union[int, None] = _MAXCACHE, + host_validate_size: Union[int, None] = _MAXCACHE, ) -> None: """Configure LRU cache sizes.""" - global _idna_decode, _idna_encode, _ip_compressed_version + global _idna_decode, _idna_encode, _ip_compressed_version, _host_validate _idna_encode = lru_cache(idna_encode_size)(_idna_encode.__wrapped__) _idna_decode = lru_cache(idna_decode_size)(_idna_decode.__wrapped__) _ip_compressed_version = lru_cache(ip_address_size)( _ip_compressed_version.__wrapped__ ) + _host_validate = lru_cache(host_validate_size)(_host_validate.__wrapped__) -- cgit v1.3