diff options
Diffstat (limited to 'contrib/python/yarl')
| -rw-r--r-- | contrib/python/yarl/.dist-info/METADATA | 124 | ||||
| -rw-r--r-- | contrib/python/yarl/README.rst | 9 | ||||
| -rw-r--r-- | contrib/python/yarl/patches/02-dont-normalize-disable-tests.patch | 22 | ||||
| -rw-r--r-- | contrib/python/yarl/tests/test_cache.py | 12 | ||||
| -rw-r--r-- | contrib/python/yarl/tests/test_helpers.py | 91 | ||||
| -rw-r--r-- | contrib/python/yarl/tests/test_quoting.py | 89 | ||||
| -rw-r--r-- | contrib/python/yarl/tests/test_url.py | 42 | ||||
| -rw-r--r-- | contrib/python/yarl/tests/test_url_build.py | 62 | ||||
| -rw-r--r-- | contrib/python/yarl/tests/test_url_parsing.py | 25 | ||||
| -rw-r--r-- | contrib/python/yarl/tests/test_url_update_netloc.py | 20 | ||||
| -rw-r--r-- | contrib/python/yarl/tests/ya.make | 1 | ||||
| -rw-r--r-- | contrib/python/yarl/ya.make | 7 | ||||
| -rw-r--r-- | contrib/python/yarl/yarl/__init__.py | 2 | ||||
| -rw-r--r-- | contrib/python/yarl/yarl/_helpers.py | 31 | ||||
| -rw-r--r-- | contrib/python/yarl/yarl/_helpers_c.pyi | 6 | ||||
| -rw-r--r-- | contrib/python/yarl/yarl/_helpers_c.pyx | 36 | ||||
| -rw-r--r-- | contrib/python/yarl/yarl/_helpers_py.py | 41 | ||||
| -rw-r--r-- | contrib/python/yarl/yarl/_url.py | 215 |
18 files changed, 517 insertions, 318 deletions
diff --git a/contrib/python/yarl/.dist-info/METADATA b/contrib/python/yarl/.dist-info/METADATA index 9870bc4c565..2995555026d 100644 --- a/contrib/python/yarl/.dist-info/METADATA +++ b/contrib/python/yarl/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: yarl -Version: 1.12.1 +Version: 1.14.0 Summary: Yet another URL library Home-page: https://github.com/aio-libs/yarl Author: Andrew Svetlov @@ -38,6 +38,7 @@ License-File: LICENSE License-File: NOTICE Requires-Dist: idna >=2.0 Requires-Dist: multidict >=4.0 +Requires-Dist: propcache >=0.2.0 yarl ==== @@ -150,9 +151,8 @@ Installation The library is Python 3 only! PyPI contains binary wheels for Linux, Windows and MacOS. If you want to install -``yarl`` on another operating system (like *Alpine Linux*, which is not -manylinux-compliant because of the missing glibc and therefore, cannot be -used with our wheels) the the tarball will be used to compile the library from +``yarl`` on another operating system where wheels are not provided, +the the tarball will be used to compile the library from the source code. It requires a C compiler and and Python headers installed. To skip the compilation you must explicitly opt-in by using a PEP 517 @@ -170,7 +170,7 @@ by this variable. Dependencies ------------ -YARL requires multidict_ library. +YARL requires multidict_ and propcache_ libraries. API documentation @@ -246,6 +246,8 @@ It's *Apache 2* licensed and freely available. .. _multidict: https://github.com/aio-libs/multidict +.. _propcache: https://github.com/aio-libs/propcache + ========= Changelog ========= @@ -262,6 +264,118 @@ Changelog .. towncrier release notes start +1.14.0 +====== + +*(2024-10-08)* + + +Packaging updates and notes for downstreams +------------------------------------------- + +- Switched to using the ``propcache`` package for property caching + -- by `@bdraco <https://github.com/sponsors/bdraco>`__. + + The ``propcache`` package is derived from the property caching + code in ``yarl`` and has been broken out to avoid maintaining it for multiple + projects. + + *Related issues and pull requests on GitHub:* + `#1169 <https://github.com/aio-libs/yarl/issues/1169>`__. + + +Contributor-facing changes +-------------------------- + +- Started testing with Hypothesis -- by `@webknjaz <https://github.com/sponsors/webknjaz>`__ and `@bdraco <https://github.com/sponsors/bdraco>`__. + + Special thanks to `@Zac-HD <https://github.com/sponsors/Zac-HD>`__ for helping us get started with this framework. + + *Related issues and pull requests on GitHub:* + `#860 <https://github.com/aio-libs/yarl/issues/860>`__. + + +Miscellaneous internal changes +------------------------------ + +- Improved performance of ``yarl.URL.is_default_port()`` when no explicit port is set -- by `@bdraco <https://github.com/sponsors/bdraco>`__. + + *Related issues and pull requests on GitHub:* + `#1168 <https://github.com/aio-libs/yarl/issues/1168>`__. + +- Improved performance of converting ``~yarl.URL`` to a string when no explicit port is set -- by `@bdraco <https://github.com/sponsors/bdraco>`__. + + *Related issues and pull requests on GitHub:* + `#1170 <https://github.com/aio-libs/yarl/issues/1170>`__. + +- Improved performance of the ``yarl.URL.origin()`` method -- by `@bdraco <https://github.com/sponsors/bdraco>`__. + + *Related issues and pull requests on GitHub:* + `#1175 <https://github.com/aio-libs/yarl/issues/1175>`__. + +- Improved performance of encoding hosts -- by `@bdraco <https://github.com/sponsors/bdraco>`__. + + *Related issues and pull requests on GitHub:* + `#1176 <https://github.com/aio-libs/yarl/issues/1176>`__. + + +---- + + +1.13.1 +====== + +*(2024-09-27)* + + +Miscellaneous internal changes +------------------------------ + +- Improved performance of calling ``yarl.URL.build()`` with ``authority`` -- by `@bdraco <https://github.com/sponsors/bdraco>`__. + + *Related issues and pull requests on GitHub:* + `#1163 <https://github.com/aio-libs/yarl/issues/1163>`__. + + +---- + + +1.13.0 +====== + +*(2024-09-26)* + + +Bug fixes +--------- + +- Started rejecting ASCII hostnames with invalid characters. For host strings that + look like authority strings, the exception message includes advice on what to do + instead -- by `@mjpieters <https://github.com/sponsors/mjpieters>`__. + + *Related issues and pull requests on GitHub:* + `#880 <https://github.com/aio-libs/yarl/issues/880>`__, `#954 <https://github.com/aio-libs/yarl/issues/954>`__. + +- Fixed IPv6 addresses missing brackets when the ``~yarl.URL`` was converted to a string -- by `@bdraco <https://github.com/sponsors/bdraco>`__. + + *Related issues and pull requests on GitHub:* + `#1157 <https://github.com/aio-libs/yarl/issues/1157>`__, `#1158 <https://github.com/aio-libs/yarl/issues/1158>`__. + + +Features +-------- + +- Added ``~yarl.URL.host_subcomponent`` which returns the ``3986#section-3.2.2`` host subcomponent -- by `@bdraco <https://github.com/sponsors/bdraco>`__. + + The only current practical difference between ``~yarl.URL.raw_host`` and ``~yarl.URL.host_subcomponent`` is that IPv6 addresses are returned bracketed. + + *Related issues and pull requests on GitHub:* + `#1159 <https://github.com/aio-libs/yarl/issues/1159>`__. + + +---- + + 1.12.1 ====== diff --git a/contrib/python/yarl/README.rst b/contrib/python/yarl/README.rst index fe9c856551d..c7a286e6212 100644 --- a/contrib/python/yarl/README.rst +++ b/contrib/python/yarl/README.rst @@ -109,9 +109,8 @@ Installation The library is Python 3 only! PyPI contains binary wheels for Linux, Windows and MacOS. If you want to install -``yarl`` on another operating system (like *Alpine Linux*, which is not -manylinux-compliant because of the missing glibc and therefore, cannot be -used with our wheels) the the tarball will be used to compile the library from +``yarl`` on another operating system where wheels are not provided, +the the tarball will be used to compile the library from the source code. It requires a C compiler and and Python headers installed. To skip the compilation you must explicitly opt-in by using a PEP 517 @@ -129,7 +128,7 @@ by this variable. Dependencies ------------ -YARL requires multidict_ library. +YARL requires multidict_ and propcache_ libraries. API documentation @@ -204,3 +203,5 @@ It's *Apache 2* licensed and freely available. .. _GitHub: https://github.com/aio-libs/yarl .. _multidict: https://github.com/aio-libs/multidict + +.. _propcache: https://github.com/aio-libs/propcache diff --git a/contrib/python/yarl/patches/02-dont-normalize-disable-tests.patch b/contrib/python/yarl/patches/02-dont-normalize-disable-tests.patch index 79c9ccc2d3e..49ecf81c0ff 100644 --- a/contrib/python/yarl/patches/02-dont-normalize-disable-tests.patch +++ b/contrib/python/yarl/patches/02-dont-normalize-disable-tests.patch @@ -8,28 +8,6 @@ def test_is_default_port_for_absolute_url_with_default_port(): url = URL("http://example.com:80") assert url.is_default_port() ---- contrib/python/yarl/tests/test_url_build.py (index) -+++ contrib/python/yarl/tests/test_url_build.py (working tree) -@@ -15,16 +15,19 @@ def test_build_simple(): - assert str(u) == "http://127.0.0.1" - - - def test_url_build_ipv6(): - u = URL.build(scheme="http", host="::1") - assert str(u) == "http://::1" - - - def test_url_build_ipv6_brackets(): - u = URL.build(scheme="http", host="[::1]") - assert str(u) == "http://::1" - - - def test_url_ipv4_in_ipv6(): - u = URL.build(scheme="http", host="2001:db8:122:344::192.0.2.33") - assert str(u) == "http://2001:db8:122:344::c000:221" --- contrib/python/yarl/tests/test_url_update_netloc.py (index) +++ contrib/python/yarl/tests/test_url_update_netloc.py (working tree) @@ -196,6 +196,7 @@ def test_with_port(): diff --git a/contrib/python/yarl/tests/test_cache.py b/contrib/python/yarl/tests/test_cache.py index 46d5b01f159..9b0e75a3551 100644 --- a/contrib/python/yarl/tests/test_cache.py +++ b/contrib/python/yarl/tests/test_cache.py @@ -13,7 +13,7 @@ def test_cache_clear() -> None: def test_cache_info() -> None: info = yarl.cache_info() - assert info.keys() == {"idna_encode", "idna_decode", "ip_address"} + assert info.keys() == {"idna_encode", "idna_decode", "ip_address", "host_validate"} def test_cache_configure_default() -> None: @@ -22,11 +22,17 @@ def test_cache_configure_default() -> None: def test_cache_configure_None() -> None: yarl.cache_configure( - idna_encode_size=None, idna_decode_size=None, ip_address_size=None + idna_encode_size=None, + idna_decode_size=None, + ip_address_size=None, + host_validate_size=None, ) def test_cache_configure_explicit() -> None: yarl.cache_configure( - idna_encode_size=128, idna_decode_size=128, ip_address_size=128 + idna_encode_size=128, + idna_decode_size=128, + ip_address_size=128, + host_validate_size=128, ) diff --git a/contrib/python/yarl/tests/test_helpers.py b/contrib/python/yarl/tests/test_helpers.py deleted file mode 100644 index cdfff129e6d..00000000000 --- a/contrib/python/yarl/tests/test_helpers.py +++ /dev/null @@ -1,91 +0,0 @@ -import platform - -import pytest - -from yarl import _helpers, _helpers_py - -IS_PYPY = platform.python_implementation() == "PyPy" - - -class CachedPropertyMixin: - cached_property = NotImplemented - - def test_cached_property(self) -> None: - class A: - def __init__(self): - self._cache = {} - - @self.cached_property # type: ignore[misc] - def prop(self): - return 1 - - a = A() - assert a.prop == 1 - - def test_cached_property_class(self) -> None: - class A: - def __init__(self): - """Init.""" - # self._cache not set because its never accessed in this test - - @self.cached_property # type: ignore[misc] - def prop(self): - """Docstring.""" - - assert isinstance(A.prop, self.cached_property) - assert A.prop.__doc__ == "Docstring." - - def test_cached_property_assignment(self) -> None: - class A: - def __init__(self): - self._cache = {} - - @self.cached_property # type: ignore[misc] - def prop(self): - """Mock property.""" - - a = A() - - with pytest.raises(AttributeError): - a.prop = 123 - - def test_cached_property_without_cache(self) -> None: - class A: - def __init__(self): - pass - - @self.cached_property # type: ignore[misc] - def prop(self): - """Mock property.""" - - a = A() - - with pytest.raises(AttributeError): - a.prop = 123 - - def test_cached_property_check_without_cache(self) -> None: - class A: - def __init__(self): - pass - - @self.cached_property # type: ignore[misc] - def prop(self): - """Mock property.""" - - a = A() - with pytest.raises(AttributeError): - assert a.prop == 1 - - -class TestPyCachedProperty(CachedPropertyMixin): - cached_property = _helpers_py.cached_property # type: ignore[assignment] - - -if ( - not _helpers.NO_EXTENSIONS - and not IS_PYPY - and hasattr(_helpers, "cached_property_c") -): - - class TestCCachedProperty(CachedPropertyMixin): - cached_property = _helpers.cached_property_c # type: ignore[assignment, attr-defined, unused-ignore] # noqa: E501 diff --git a/contrib/python/yarl/tests/test_quoting.py b/contrib/python/yarl/tests/test_quoting.py index d9b6ae8e4bf..1f9cba6aa40 100644 --- a/contrib/python/yarl/tests/test_quoting.py +++ b/contrib/python/yarl/tests/test_quoting.py @@ -1,4 +1,8 @@ +from typing import Type + import pytest +from hypothesis import assume, example, given, note +from hypothesis import strategies as st from yarl._quoting import NO_EXTENSIONS from yarl._quoting_py import _Quoter as _PyQuoter @@ -16,6 +20,10 @@ if not NO_EXTENSIONS: def unquoter(request): return request.param + quoters = [_PyQuoter, _CQuoter] + quoter_ids = ["PyQuoter", "CQuoter"] + unquoters = [_PyUnquoter, _CUnquoter] + unquoter_ids = ["PyUnquoter", "CUnquoter"] else: @pytest.fixture(params=[_PyQuoter], ids=["py_quoter"]) @@ -26,6 +34,11 @@ else: def unquoter(request): return request.param + quoters = [_PyQuoter] + quoter_ids = ["PyQuoter"] + unquoters = [_PyUnquoter] + unquoter_ids = ["PyUnquoter"] + def hexescape(char): """Escape char as RFC 2396 specifies""" @@ -455,3 +468,79 @@ def test_quoter_path_with_plus(quoter): def test_unquoter_path_with_plus(unquoter): s = "/test/x+y%2Bz/:+%2B/" assert "/test/x+y+z/:++/" == unquoter(unsafe="+")(s) + + +@given(safe=st.text(), protected=st.text(), qs=st.booleans(), requote=st.booleans()) +def test_fuzz__PyQuoter(safe, protected, qs, requote): + """Verify that _PyQuoter can be instantiated with any valid arguments.""" + assert _PyQuoter(safe=safe, protected=protected, qs=qs, requote=requote) + + +@given(ignore=st.text(), unsafe=st.text(), qs=st.booleans()) +def test_fuzz__PyUnquoter(ignore, unsafe, qs): + """Verify that _PyUnquoter can be instantiated with any valid arguments.""" + assert _PyUnquoter(ignore=ignore, unsafe=unsafe, qs=qs) + + +@example(text_input="0") +@given( + text_input=st.text( + alphabet=st.characters(max_codepoint=127, blacklist_characters="%") + ), +) [email protected]("quoter", quoters, ids=quoter_ids) [email protected]("unquoter", unquoters, ids=unquoter_ids) +def test_quote_unquote_parameter( + quoter: Type[_PyQuoter], + unquoter: Type[_PyUnquoter], + text_input: str, +) -> None: + quote = quoter() + unquote = unquoter() + text_quoted = quote(text_input) + note(f"text_quoted={text_quoted!r}") + text_output = unquote(text_quoted) + assert text_input == text_output + + +@example(text_input="0") +@given( + text_input=st.text( + alphabet=st.characters(max_codepoint=127, blacklist_characters="%") + ), +) [email protected]("quoter", quoters, ids=quoter_ids) [email protected]("unquoter", unquoters, ids=unquoter_ids) +def test_quote_unquote_parameter_requote( + quoter: Type[_PyQuoter], + unquoter: Type[_PyUnquoter], + text_input: str, +) -> None: + quote = quoter(requote=True) + unquote = unquoter() + text_quoted = quote(text_input) + note(f"text_quoted={text_quoted!r}") + text_output = unquote(text_quoted) + assert text_input == text_output + + +@example(text_input="0") +@given( + text_input=st.text( + alphabet=st.characters(max_codepoint=127, blacklist_characters="%") + ), +) [email protected]("quoter", quoters, ids=quoter_ids) [email protected]("unquoter", unquoters, ids=unquoter_ids) +def test_quote_unquote_parameter_path_safe( + quoter: Type[_PyQuoter], + unquoter: Type[_PyUnquoter], + text_input: str, +) -> None: + quote = quoter() + unquote = unquoter(ignore="/%", unsafe="+") + assume("+" not in text_input and "/" not in text_input) + text_quoted = quote(text_input) + note(f"text_quoted={text_quoted!r}") + text_output = unquote(text_quoted) + assert text_input == text_output diff --git a/contrib/python/yarl/tests/test_url.py b/contrib/python/yarl/tests/test_url.py index 1dd98197fe1..193f1b584c8 100644 --- a/contrib/python/yarl/tests/test_url.py +++ b/contrib/python/yarl/tests/test_url.py @@ -63,6 +63,11 @@ def test_origin(): assert URL("http://example.com:8888") == url.origin() +def test_origin_with_no_auth(): + url = URL("http://example.com:8888/path/to?a=1&b=2") + assert URL("http://example.com:8888") == url.origin() + + def test_origin_nonascii(): url = URL("http://user:password@оун-упа.укр:8888/path/to?a=1&b=2") assert str(url.origin()) == "http://xn----8sb1bdhvc.xn--j1amh:8888" @@ -176,6 +181,24 @@ def test_raw_host(): assert url.raw_host == url._val.hostname + ("host"), + [ + ("example.com"), + ("[::1]"), + ("xn--gnter-4ya.com"), + ], +) +def test_host_subcomponent(host: str): + url = URL(f"http://{host}") + assert url.host_subcomponent == host + + +def test_host_subcomponent_return_idna_encoded_host(): + url = URL("http://оун-упа.укр") + assert url.host_subcomponent == "xn----8sb1bdhvc.xn--j1amh" + + def test_raw_host_non_ascii(): url = URL("http://оун-упа.укр") assert "xn----8sb1bdhvc.xn--j1amh" == url.raw_host @@ -286,35 +309,43 @@ def test_port_for_implicit_port(): def test_port_for_relative_url(): url = URL("/path/to") assert url.port is None + assert url._port_not_default is None + assert url.explicit_port is None def test_port_for_unknown_scheme(): url = URL("unknown://example.com") assert url.port is None + assert url._port_not_default is None + assert url.explicit_port is None def test_explicit_port_for_explicit_port(): url = URL("http://example.com:8888") assert 8888 == url.explicit_port assert url.explicit_port == url._val.port + assert url._port_not_default == 8888 def test_explicit_port_for_implicit_port(): url = URL("http://example.com") assert url.explicit_port is None assert url.explicit_port == url._val.port + assert url._port_not_default is None def test_explicit_port_for_relative_url(): url = URL("/path/to") assert url.explicit_port is None assert url.explicit_port == url._val.port + assert url._port_not_default is None def test_explicit_port_for_unknown_scheme(): url = URL("unknown://example.com") assert url.explicit_port is None assert url.explicit_port == url._val.port + assert url._port_not_default is None def test_raw_path_string_empty(): @@ -2041,3 +2072,14 @@ def test_parsing_populates_cache(): assert url._cache["raw_query_string"] == "a=b" assert url._cache["raw_fragment"] == "frag" assert url._cache["scheme"] == "http" + + + ("host", "is_authority"), + [ + *(("other_gen_delim_" + c, False) for c in "[]"), + ], +) +def test_build_with_invalid_ipv6_host(host: str, is_authority: bool): + with pytest.raises(ValueError, match="Invalid IPv6 URL"): + URL(f"http://{host}/") diff --git a/contrib/python/yarl/tests/test_url_build.py b/contrib/python/yarl/tests/test_url_build.py index a0ab77b2d4e..cc9b15f7e4f 100644 --- a/contrib/python/yarl/tests/test_url_build.py +++ b/contrib/python/yarl/tests/test_url_build.py @@ -15,22 +15,24 @@ def test_build_simple(): assert str(u) == "http://127.0.0.1" def test_url_build_ipv6(): u = URL.build(scheme="http", host="::1") - assert str(u) == "http://::1" + assert str(u) == "http://[::1]" -def test_url_build_ipv6_brackets(): - u = URL.build(scheme="http", host="[::1]") - assert str(u) == "http://::1" +def test_url_build_ipv6_brackets_encoded(): + u = URL.build(scheme="http", host="[::1]", encoded=True) + assert str(u) == "http://[::1]" + + +def test_url_build_ipv6_brackets_not_encoded(): + u = URL.build(scheme="http", host="::1", encoded=False) + assert str(u) == "http://[::1]" def test_url_ipv4_in_ipv6(): u = URL.build(scheme="http", host="2001:db8:122:344::192.0.2.33") - assert str(u) == "http://2001:db8:122:344::c000:221" + assert str(u) == "http://[2001:db8:122:344::c000:221]" def test_build_with_scheme(): @@ -118,6 +120,25 @@ def test_build_with_authority_and_host(): URL.build(authority="host.com", host="example.com") + ("host", "is_authority"), + [ + ("user:[email protected]", True), + ("[email protected]", True), + ("host:com", False), + ("not_percent_encoded%Zf", False), + ("still_not_percent_encoded%fZ", False), + *(("other_gen_delim_" + c, False) for c in "/?#[]"), + ], +) +def test_build_with_invalid_host(host: str, is_authority: bool): + match = r"Host '[^']+' cannot contain '[^']+' \(at position \d+\)" + if is_authority: + match += ", if .* use 'authority' instead of 'host'" + with pytest.raises(ValueError, match=f"{match}$"): + URL.build(host=host) + + def test_build_with_authority(): url = URL.build(scheme="http", authority="степан:[email protected]:8000", path="path") assert ( @@ -132,6 +153,31 @@ def test_build_with_authority_without_encoding(): assert str(url) == "http://foo:[email protected]:8000/path" +def test_build_with_authority_empty_host_no_scheme(): + url = URL.build(authority="", path="path") + assert str(url) == "path" + + +def test_build_with_authority_and_only_user(): + url = URL.build(scheme="https", authority="user:@foo.com", path="/path") + assert str(url) == "https://user:@foo.com/path" + + +def test_build_with_authority_with_port(): + url = URL.build(scheme="https", authority="foo.com:8080", path="/path") + assert str(url) == "https://foo.com:8080/path" + + +def test_build_with_authority_with_ipv6(): + url = URL.build(scheme="https", authority="[::1]", path="/path") + assert str(url) == "https://[::1]/path" + + +def test_build_with_authority_with_ipv6_and_port(): + url = URL.build(scheme="https", authority="[::1]:81", path="/path") + assert str(url) == "https://[::1]:81/path" + + def test_query_str(): u = URL.build(scheme="http", host="127.0.0.1", path="/", query_string="arg=value1") assert str(u) == "http://127.0.0.1/?arg=value1" diff --git a/contrib/python/yarl/tests/test_url_parsing.py b/contrib/python/yarl/tests/test_url_parsing.py index 4fe95185e71..0d58c9d09bb 100644 --- a/contrib/python/yarl/tests/test_url_parsing.py +++ b/contrib/python/yarl/tests/test_url_parsing.py @@ -604,3 +604,28 @@ def test_schemes_that_require_host(scheme: str) -> None: ) with pytest.raises(ValueError, match=expect): URL(f"{scheme}://:1") + + + ("url", "hostname", "hostname_without_brackets"), + [ + ("http://[::1]", "[::1]", "::1"), + ("http://[::1]:8080", "[::1]", "::1"), + ("http://127.0.0.1:8080", "127.0.0.1", "127.0.0.1"), + ( + "http://xn--jxagkqfkduily1i.eu", + "xn--jxagkqfkduily1i.eu", + "xn--jxagkqfkduily1i.eu", + ), + ], +) +def test_url_round_trips( + url: str, hostname: str, hostname_without_brackets: str +) -> None: + """Verify that URLs round-trip correctly.""" + parsed = URL(url) + assert parsed._val.hostname == hostname_without_brackets + assert parsed.raw_host == hostname_without_brackets + assert parsed.host_subcomponent == hostname + assert str(parsed) == url + assert str(URL(str(parsed))) == url diff --git a/contrib/python/yarl/tests/test_url_update_netloc.py b/contrib/python/yarl/tests/test_url_update_netloc.py index cfe6ca7d457..a8ff1f9f8b3 100644 --- a/contrib/python/yarl/tests/test_url_update_netloc.py +++ b/contrib/python/yarl/tests/test_url_update_netloc.py @@ -171,6 +171,26 @@ def test_with_host_non_ascii(): assert url2.authority == "оун-упа.укр:123" + ("host", "is_authority"), + [ + ("user:[email protected]", True), + ("[email protected]", True), + ("host:com", False), + ("not_percent_encoded%Zf", False), + ("still_not_percent_encoded%fZ", False), + *(("other_gen_delim_" + c, False) for c in "/?#[]"), + ], +) +def test_with_invalid_host(host: str, is_authority: bool): + url = URL("http://example.com:123") + match = r"Host '[^']+' cannot contain '[^']+' \(at position \d+\)" + if is_authority: + match += ", if .* use 'authority' instead of 'host'" + with pytest.raises(ValueError, match=f"{match}$"): + url.with_host(host=host) + + def test_with_host_percent_encoded(): url = URL("http://%25cf%2580%cf%80:%25cf%2580%cf%[email protected]:123") url2 = url.with_host("%cf%80.org") diff --git a/contrib/python/yarl/tests/ya.make b/contrib/python/yarl/tests/ya.make index b93020851f0..73d45a2785a 100644 --- a/contrib/python/yarl/tests/ya.make +++ b/contrib/python/yarl/tests/ya.make @@ -1,6 +1,7 @@ PY3TEST() PEERDIR( + contrib/python/hypothesis contrib/python/yarl ) diff --git a/contrib/python/yarl/ya.make b/contrib/python/yarl/ya.make index 43d5345771f..f8435827974 100644 --- a/contrib/python/yarl/ya.make +++ b/contrib/python/yarl/ya.make @@ -2,13 +2,14 @@ PY3_LIBRARY() -VERSION(1.12.1) +VERSION(1.14.0) LICENSE(Apache-2.0) PEERDIR( contrib/python/idna contrib/python/multidict + contrib/python/propcache ) NO_COMPILER_WARNINGS() @@ -18,15 +19,11 @@ NO_LINT() PY_SRCS( TOP_LEVEL yarl/__init__.py - yarl/_helpers.py - yarl/_helpers_c.pyi - yarl/_helpers_py.py yarl/_quoting.py yarl/_quoting_c.pyi yarl/_quoting_py.py yarl/_url.py CYTHON_C - yarl/_helpers_c.pyx yarl/_quoting_c.pyx ) diff --git a/contrib/python/yarl/yarl/__init__.py b/contrib/python/yarl/yarl/__init__.py index 50d24c0e778..d9703013c6e 100644 --- a/contrib/python/yarl/yarl/__init__.py +++ b/contrib/python/yarl/yarl/__init__.py @@ -8,7 +8,7 @@ from ._url import ( cache_info, ) -__version__ = "1.12.1" +__version__ = "1.14.0" __all__ = ( "URL", diff --git a/contrib/python/yarl/yarl/_helpers.py b/contrib/python/yarl/yarl/_helpers.py deleted file mode 100644 index ac01158c7f5..00000000000 --- a/contrib/python/yarl/yarl/_helpers.py +++ /dev/null @@ -1,31 +0,0 @@ -import os -import sys -from typing import TYPE_CHECKING - -__all__ = ("cached_property",) - - -NO_EXTENSIONS = bool(os.environ.get("YARL_NO_EXTENSIONS")) # type: bool -if sys.implementation.name != "cpython": - NO_EXTENSIONS = True - - -# isort: off -if TYPE_CHECKING: - from ._helpers_py import cached_property as cached_property_py - - cached_property = cached_property_py -elif not NO_EXTENSIONS: # pragma: no branch - try: - from ._helpers_c import cached_property as cached_property_c # type: ignore[attr-defined, unused-ignore] # noqa: E501 - - cached_property = cached_property_c - except ImportError: # pragma: no cover - from ._helpers_py import cached_property as cached_property_py - - cached_property = cached_property_py # type: ignore[assignment, misc] -else: - from ._helpers_py import cached_property as cached_property_py - - cached_property = cached_property_py # type: ignore[assignment, misc] -# isort: on diff --git a/contrib/python/yarl/yarl/_helpers_c.pyi b/contrib/python/yarl/yarl/_helpers_c.pyi deleted file mode 100644 index 69034921219..00000000000 --- a/contrib/python/yarl/yarl/_helpers_c.pyi +++ /dev/null @@ -1,6 +0,0 @@ -from typing import Any - -class cached_property: - def __init__(self, wrapped: Any) -> None: ... - def __get__(self, inst: Any, owner: Any) -> Any: ... - def __set__(self, inst: Any, value: Any) -> None: ... diff --git a/contrib/python/yarl/yarl/_helpers_c.pyx b/contrib/python/yarl/yarl/_helpers_c.pyx deleted file mode 100644 index e6eec375435..00000000000 --- a/contrib/python/yarl/yarl/_helpers_c.pyx +++ /dev/null @@ -1,36 +0,0 @@ -# cython: language_level=3 - -cdef _sentinel = object() - -cdef class cached_property: - """Use as a class method decorator. It operates almost exactly like - the Python `@property` decorator, but it puts the result of the - method it decorates into the instance dict after the first call, - effectively replacing the function it decorates with an instance - variable. It is, in Python parlance, a data descriptor. - - """ - - cdef object wrapped - cdef object name - - def __init__(self, wrapped): - self.wrapped = wrapped - self.name = wrapped.__name__ - - @property - def __doc__(self): - return self.wrapped.__doc__ - - def __get__(self, inst, owner): - if inst is None: - return self - cdef dict cache = inst._cache - val = cache.get(self.name, _sentinel) - if val is _sentinel: - val = self.wrapped(inst) - cache[self.name] = val - return val - - def __set__(self, inst, value): - raise AttributeError("cached property is read-only") diff --git a/contrib/python/yarl/yarl/_helpers_py.py b/contrib/python/yarl/yarl/_helpers_py.py deleted file mode 100644 index 5a18afb5c39..00000000000 --- a/contrib/python/yarl/yarl/_helpers_py.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Various helper functions.""" - -from typing import Any, Callable, Dict, Generic, Optional, Protocol, Type, TypeVar - -_T = TypeVar("_T") - - -class _TSelf(Protocol, Generic[_T]): - _cache: Dict[str, _T] - - -class cached_property(Generic[_T]): - """Use as a class method decorator. - - It operates almost exactly like - the Python `@property` decorator, but it puts the result of the - method it decorates into the instance dict after the first call, - effectively replacing the function it decorates with an instance - variable. It is, in Python parlance, a data descriptor. - """ - - def __init__(self, wrapped: Callable[..., _T]) -> None: - self.wrapped = wrapped - self.__doc__ = wrapped.__doc__ - self.name = wrapped.__name__ - - def __get__(self, inst: _TSelf[_T], owner: Optional[Type[Any]] = None) -> _T: - try: - try: - return inst._cache[self.name] - except KeyError: - val = self.wrapped(inst) - inst._cache[self.name] = val - return val - except AttributeError: - if inst is None: - return self - raise - - def __set__(self, inst: _TSelf[_T], value: _T) -> None: - raise AttributeError("cached property is read-only") diff --git a/contrib/python/yarl/yarl/_url.py b/contrib/python/yarl/yarl/_url.py index 8a48628fcd9..eff4dc3fbd6 100644 --- a/contrib/python/yarl/yarl/_url.py +++ b/contrib/python/yarl/yarl/_url.py @@ -1,4 +1,5 @@ import math +import re import sys import warnings from collections.abc import Mapping, Sequence @@ -31,8 +32,8 @@ from urllib.parse import ( import idna from multidict import MultiDict, MultiDictProxy, istr +from propcache.api import under_cached_property as cached_property -from ._helpers import cached_property from ._quoting import _Quoter, _Unquoter DEFAULT_PORTS = {"http": 80, "https": 443, "ws": 80, "wss": 443, "ftp": 21} @@ -45,6 +46,22 @@ SCHEME_REQUIRES_HOST = frozenset(("http", "https", "ws", "wss", "ftp")) sentinel = object() +# reg-name: unreserved / pct-encoded / sub-delims +# this pattern matches anything that is *not* in those classes. and is only used +# on lower-cased ASCII values. +_not_reg_name = re.compile( + r""" + # any character not in the unreserved or sub-delims sets, plus % + # (validated with the additional check for pct-encoded sequences below) + [^a-z0-9\-._~!$&'()*+,;=%] + | + # % only allowed if it is part of a pct-encoded + # sequence of 2 hex digits. + %(?![0-9a-f]{2}) + """, + re.VERBOSE, +) + SimpleQuery = Union[str, int, float] QueryVariable = Union[SimpleQuery, "Sequence[SimpleQuery]"] Query = Union[ @@ -64,6 +81,7 @@ class CacheInfo(TypedDict): idna_encode: _CacheInfo idna_decode: _CacheInfo ip_address: _CacheInfo + host_validate: _CacheInfo class _SplitResultDict(TypedDict, total=False): @@ -77,6 +95,7 @@ class _SplitResultDict(TypedDict, total=False): class _InternalURLCache(TypedDict, total=False): + _origin: "URL" absolute: bool scheme: str raw_authority: str @@ -269,7 +288,7 @@ class URL: raise ValueError(msg) else: host = "" - host = cls._encode_host(host) + host = cls._encode_host(host, validate_host=False) raw_user = None if username is None else cls._REQUOTER(username) raw_password = None if password is None else cls._REQUOTER(password) netloc = cls._make_netloc( @@ -350,10 +369,15 @@ class URL: if encoded: netloc = authority else: - tmp = SplitResult("", authority, "", "", "") - port = None if tmp.port == DEFAULT_PORTS.get(scheme) else tmp.port + _user, _password, _host, _port = cls._split_netloc(authority) + port = None if _port == DEFAULT_PORTS.get(scheme) else _port + _host = ( + "" + if _host is None + else cls._encode_host(_host, validate_host=False) + ) netloc = cls._make_netloc( - tmp.username, tmp.password, tmp.hostname, port, encode=True + _user, _password, _host, port, encode=True, encode_host=False ) elif not user and not password and not host and not port: netloc = "" @@ -395,7 +419,7 @@ class URL: netloc=self._make_netloc( self.raw_user, self.raw_password, - self.raw_host, + self.host_subcomponent, port, encode_host=False, ) @@ -506,12 +530,12 @@ class URL: Return False for relative URLs. """ - default = self._default_port - explicit = self.explicit_port - if explicit is None: - # A relative URL does not have an implicit port / default port - return default is not None - return explicit == default + if (explicit := self.explicit_port) is None: + # If the explicit port is None, then the URL must be + # using the default port unless its a relative URL + # which does not have an implicit port / default port + return self.absolute + return explicit == self._default_port def origin(self) -> "URL": """Return an URL with scheme, host and port parts only. @@ -520,13 +544,24 @@ class URL: """ # TODO: add a keyword-only option for keeping user/pass maybe? - if not self.absolute: + return self._origin + + @cached_property + def _origin(self) -> "URL": + """Return an URL with scheme, host and port parts only. + + user, password, path, query and fragment are removed. + """ + v = self._val + if not v.netloc: raise ValueError("URL should be absolute") - if not self._val.scheme: + if not v.scheme: raise ValueError("URL should have scheme") - v = self._val - netloc = self._make_netloc(None, None, v.hostname, v.port) - val = v._replace(netloc=netloc, path="", query="", fragment="") + if "@" not in v.netloc: + val = v._replace(path="", query="", fragment="") + else: + netloc = self._make_netloc(None, None, v.hostname, v.port) + val = v._replace(netloc=netloc, path="", query="", fragment="") return URL(val, encoded=True) def relative(self) -> "URL": @@ -575,15 +610,15 @@ class URL: @cached_property def _default_port(self) -> Union[int, None]: """Default port for the scheme or None if not known.""" - return DEFAULT_PORTS.get(self.scheme) + return DEFAULT_PORTS.get(self._val.scheme) @cached_property def _port_not_default(self) -> Union[int, None]: """The port part of URL normalized to None if its the default port.""" - port = self.port - if self._default_port == port: + explicit_port = self.explicit_port + if explicit_port is None or explicit_port == self._default_port: return None - return port + return explicit_port @cached_property def authority(self) -> str: @@ -647,6 +682,8 @@ class URL: None for relative URLs. + When working with IPv6 addresses, use the `host_subcomponent` property instead + as it will return the host subcomponent with brackets. """ # Use host instead of hostname for sake of shortness # May add .hostname prop later @@ -660,17 +697,36 @@ class URL: None for relative URLs. """ - raw = self.raw_host - if raw is None: + if (raw := self.raw_host) is None: return None - if "%" in raw: - # Hack for scoped IPv6 addresses like - # fe80::2%Перевірка - # presence of '%' sign means only IPv6 address, so idna is useless. + if raw and raw[-1].isdigit() or ":" in raw: + # IP addresses are never IDNA encoded return raw return _idna_decode(raw) @cached_property + def host_subcomponent(self) -> Union[str, None]: + """Return the host subcomponent part of URL. + + None for relative URLs. + + https://datatracker.ietf.org/doc/html/rfc3986#section-3.2.2 + + `IP-literal = "[" ( IPv6address / IPvFuture ) "]"` + + Examples: + - `http://example.com:8080` -> `example.com` + - `http://example.com:80` -> `example.com` + - `https://127.0.0.1:8443` -> `127.0.0.1` + - `https://[::1]:8443` -> `[::1]` + - `http://[::1]` -> `[::1]` + + """ + if (raw := self.raw_host) is None: + return None + return f"[{raw}]" if ":" in raw else raw + + @cached_property def port(self) -> Union[int, None]: """Port part of URL, with scheme-based fallback. @@ -725,7 +781,7 @@ class URL: @cached_property def _parsed_query(self) -> List[Tuple[str, str]]: """Parse query part of URL.""" - return parse_qsl(self.raw_query_string, keep_blank_values=True) + return parse_qsl(self._val.query, keep_blank_values=True) @cached_property def query(self) -> "MultiDictProxy[str]": @@ -753,7 +809,7 @@ class URL: Empty string if query is missing. """ - return self._QS_UNQUOTER(self.raw_query_string) + return self._QS_UNQUOTER(self._val.query) @cached_property def path_qs(self) -> str: @@ -765,9 +821,9 @@ class URL: @cached_property def raw_path_qs(self) -> str: """Encoded path of URL with query.""" - if not self.raw_query_string: + if not self._val.query: return self.raw_path - return f"{self.raw_path}?{self.raw_query_string}" + return f"{self.raw_path}?{self._val.query}" @cached_property def raw_fragment(self) -> str: @@ -785,7 +841,7 @@ class URL: Empty string if fragment is missing. """ - return self._UNQUOTER(self.raw_fragment) + return self._UNQUOTER(self._val.fragment) @cached_property def raw_parts(self) -> Tuple[str, ...]: @@ -824,7 +880,7 @@ class URL: """ path = self.raw_path if not path or path == "/": - if self.raw_fragment or self.raw_query_string: + if self._val.fragment or self._val.query: return URL(self._val._replace(query="", fragment=""), encoded=True) return self parts = path.split("/") @@ -938,14 +994,21 @@ class URL: return prefix + "/".join(_normalize_path_segments(segments)) @classmethod - def _encode_host(cls, host: str, human: bool = False) -> str: - if "%" in host: - raw_ip, sep, zone = host.partition("%") - else: - raw_ip = host - sep = zone = "" - - if raw_ip and raw_ip[-1].isdigit() or ":" in raw_ip: + def _encode_host( + cls, host: str, human: bool = False, validate_host: bool = True + ) -> str: + if host and host[-1].isdigit() or ":" in host: + # If the host ends with a digit or contains a colon, its likely + # an IP address. So we check with _ip_compressed_version + # and fall-through if its not an IP address. This is a performance + # optimization to avoid parsing IP addresses as much as possible + # because it is orders of magnitude slower than almost any other + # operation this library does. + if "%" in host: + raw_ip, sep, zone = host.partition("%") + else: + raw_ip = host + sep = zone = "" # Might be an IP address, check it # # IP Addresses can look like: @@ -953,14 +1016,11 @@ class URL: # - 127.0.0.1 (last character is a digit) # - 2001:db8::ff00:42:8329 (contains a colon) # - 2001:db8::ff00:42:8329%eth0 (contains a colon) - # - [2001:db8::ff00:42:8329] (contains a colon) + # - [2001:db8::ff00:42:8329] (contains a colon -- brackets should + # have been removed before it gets here) # Rare IP Address formats are not supported per: # https://datatracker.ietf.org/doc/html/rfc3986#section-7.4 # - # We try to avoid parsing IP addresses as much as possible - # since its orders of magnitude slower than almost any other operation - # this library does. - # # IP parsing is slow, so its wrapped in an LRU try: ip_compressed_version = _ip_compressed_version(raw_ip) @@ -970,18 +1030,23 @@ class URL: # These checks should not happen in the # LRU to keep the cache size small host, version = ip_compressed_version - if sep: - host += "%" + zone if version == 6: - return f"[{host}]" - return host + return f"[{host}%{zone}]" if sep else f"[{host}]" + return f"{host}%{zone}" if sep else host host = host.lower() + if human: + return host + # IDNA encoding is slow, # skip it for ASCII-only strings # Don't move the check into _idna_encode() helper # to reduce the cache size - if human or host.isascii(): + if host.isascii(): + # Check for invalid characters explicitly; _idna_encode() does this + # for non-ascii host names. + if validate_host: + _host_validate(host) return host return _idna_encode(host) @@ -998,26 +1063,24 @@ class URL: ) -> str: if host is None: return "" - quoter = cls._REQUOTER if requote else cls._QUOTER - if encode_host: - ret = cls._encode_host(host) - else: - ret = host + ret = cls._encode_host(host) if encode_host else host if port is not None: ret = f"{ret}:{port}" + if user is None and password is None: + return ret + quoter = cls._REQUOTER if requote else cls._QUOTER if password is not None: if not user: user = "" - else: - if encode: - user = quoter(user) + elif encode: + user = quoter(user) if encode: password = quoter(password) - user = user + ":" + password + user = f"{user}:{password}" elif user and encode: user = quoter(user) if user: - ret = user + "@" + ret + ret = f"{user}@{ret}" return ret @classmethod @@ -1298,7 +1361,7 @@ class URL: new_query_string = self._get_str_query(*args, **kwargs) if not new_query_string: return self - if current_query := self.raw_query_string: + if current_query := self._val.query: # both strings are already encoded so we can use a simple # string join if current_query[-1] == "&": @@ -1362,7 +1425,7 @@ class URL: raise TypeError("Invalid fragment type") else: raw_fragment = self._FRAGMENT_QUOTER(fragment) - if self.raw_fragment == raw_fragment: + if self._val.fragment == raw_fragment: return self return URL(self._val._replace(fragment=raw_fragment), encoded=True) @@ -1496,7 +1559,7 @@ class URL: netloc = self._make_netloc( user, password, host, self.explicit_port, encode_host=False ) - val = SplitResult(self.scheme, netloc, path, query_string, fragment) + val = SplitResult(self._val.scheme, netloc, path, query_string, fragment) return urlunsplit(val) @@ -1537,12 +1600,31 @@ def _ip_compressed_version(raw_ip: str) -> Tuple[str, int]: return ip.compressed, ip.version +@lru_cache(_MAXCACHE) +def _host_validate(host: str) -> None: + """Validate an ascii host name.""" + invalid = _not_reg_name.search(host) + if invalid is None: + return + value, pos, extra = invalid.group(), invalid.start(), "" + if value == "@" or (value == ":" and "@" in host[pos:]): + # this looks like an authority string + extra = ( + ", if the value includes a username or password, " + "use 'authority' instead of 'host'" + ) + raise ValueError( + f"Host {host!r} cannot contain {value!r} (at position " f"{pos}){extra}" + ) from None + + @rewrite_module def cache_clear() -> None: """Clear all LRU caches.""" _idna_decode.cache_clear() _idna_encode.cache_clear() _ip_compressed_version.cache_clear() + _host_validate.cache_clear() @rewrite_module @@ -1552,6 +1634,7 @@ def cache_info() -> CacheInfo: "idna_encode": _idna_encode.cache_info(), "idna_decode": _idna_decode.cache_info(), "ip_address": _ip_compressed_version.cache_info(), + "host_validate": _host_validate.cache_info(), } @@ -1561,12 +1644,14 @@ def cache_configure( idna_encode_size: Union[int, None] = _MAXCACHE, idna_decode_size: Union[int, None] = _MAXCACHE, ip_address_size: Union[int, None] = _MAXCACHE, + host_validate_size: Union[int, None] = _MAXCACHE, ) -> None: """Configure LRU cache sizes.""" - global _idna_decode, _idna_encode, _ip_compressed_version + global _idna_decode, _idna_encode, _ip_compressed_version, _host_validate _idna_encode = lru_cache(idna_encode_size)(_idna_encode.__wrapped__) _idna_decode = lru_cache(idna_decode_size)(_idna_decode.__wrapped__) _ip_compressed_version = lru_cache(ip_address_size)( _ip_compressed_version.__wrapped__ ) + _host_validate = lru_cache(host_validate_size)(_host_validate.__wrapped__) |