diff options
Diffstat (limited to 'contrib/python/yarl')
| -rw-r--r-- | contrib/python/yarl/.dist-info/METADATA | 56 | ||||
| -rw-r--r-- | contrib/python/yarl/patches/02-dont-normalize-disable-tests.patch | 22 | ||||
| -rw-r--r-- | contrib/python/yarl/tests/test_cache.py | 12 | ||||
| -rw-r--r-- | contrib/python/yarl/tests/test_url.py | 29 | ||||
| -rw-r--r-- | contrib/python/yarl/tests/test_url_build.py | 62 | ||||
| -rw-r--r-- | contrib/python/yarl/tests/test_url_parsing.py | 25 | ||||
| -rw-r--r-- | contrib/python/yarl/tests/test_url_update_netloc.py | 20 | ||||
| -rw-r--r-- | contrib/python/yarl/ya.make | 2 | ||||
| -rw-r--r-- | contrib/python/yarl/yarl/__init__.py | 2 | ||||
| -rw-r--r-- | contrib/python/yarl/yarl/_url.py | 106 |
10 files changed, 285 insertions, 51 deletions
diff --git a/contrib/python/yarl/.dist-info/METADATA b/contrib/python/yarl/.dist-info/METADATA index 9870bc4c565..97585939dab 100644 --- a/contrib/python/yarl/.dist-info/METADATA +++ b/contrib/python/yarl/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: yarl -Version: 1.12.1 +Version: 1.13.1 Summary: Yet another URL library Home-page: https://github.com/aio-libs/yarl Author: Andrew Svetlov @@ -262,6 +262,60 @@ Changelog .. towncrier release notes start +1.13.1 +====== + +*(2024-09-27)* + + +Miscellaneous internal changes +------------------------------ + +- Improved performance of calling ``yarl.URL.build()`` with ``authority`` -- by `@bdraco <https://github.com/sponsors/bdraco>`__. + + *Related issues and pull requests on GitHub:* + `#1163 <https://github.com/aio-libs/yarl/issues/1163>`__. + + +---- + + +1.13.0 +====== + +*(2024-09-26)* + + +Bug fixes +--------- + +- Started rejecting ASCII hostnames with invalid characters. For host strings that + look like authority strings, the exception message includes advice on what to do + instead -- by `@mjpieters <https://github.com/sponsors/mjpieters>`__. + + *Related issues and pull requests on GitHub:* + `#880 <https://github.com/aio-libs/yarl/issues/880>`__, `#954 <https://github.com/aio-libs/yarl/issues/954>`__. + +- Fixed IPv6 addresses missing brackets when the ``~yarl.URL`` was converted to a string -- by `@bdraco <https://github.com/sponsors/bdraco>`__. + + *Related issues and pull requests on GitHub:* + `#1157 <https://github.com/aio-libs/yarl/issues/1157>`__, `#1158 <https://github.com/aio-libs/yarl/issues/1158>`__. + + +Features +-------- + +- Added ``~yarl.URL.host_subcomponent`` which returns the ``3986#section-3.2.2`` host subcomponent -- by `@bdraco <https://github.com/sponsors/bdraco>`__. + + The only current practical difference between ``~yarl.URL.raw_host`` and ``~yarl.URL.host_subcomponent`` is that IPv6 addresses are returned bracketed. + + *Related issues and pull requests on GitHub:* + `#1159 <https://github.com/aio-libs/yarl/issues/1159>`__. + + +---- + + 1.12.1 ====== diff --git a/contrib/python/yarl/patches/02-dont-normalize-disable-tests.patch b/contrib/python/yarl/patches/02-dont-normalize-disable-tests.patch index 79c9ccc2d3e..49ecf81c0ff 100644 --- a/contrib/python/yarl/patches/02-dont-normalize-disable-tests.patch +++ b/contrib/python/yarl/patches/02-dont-normalize-disable-tests.patch @@ -8,28 +8,6 @@ def test_is_default_port_for_absolute_url_with_default_port(): url = URL("http://example.com:80") assert url.is_default_port() ---- contrib/python/yarl/tests/test_url_build.py (index) -+++ contrib/python/yarl/tests/test_url_build.py (working tree) -@@ -15,16 +15,19 @@ def test_build_simple(): - assert str(u) == "http://127.0.0.1" - - - def test_url_build_ipv6(): - u = URL.build(scheme="http", host="::1") - assert str(u) == "http://::1" - - - def test_url_build_ipv6_brackets(): - u = URL.build(scheme="http", host="[::1]") - assert str(u) == "http://::1" - - - def test_url_ipv4_in_ipv6(): - u = URL.build(scheme="http", host="2001:db8:122:344::192.0.2.33") - assert str(u) == "http://2001:db8:122:344::c000:221" --- contrib/python/yarl/tests/test_url_update_netloc.py (index) +++ contrib/python/yarl/tests/test_url_update_netloc.py (working tree) @@ -196,6 +196,7 @@ def test_with_port(): diff --git a/contrib/python/yarl/tests/test_cache.py b/contrib/python/yarl/tests/test_cache.py index 46d5b01f159..9b0e75a3551 100644 --- a/contrib/python/yarl/tests/test_cache.py +++ b/contrib/python/yarl/tests/test_cache.py @@ -13,7 +13,7 @@ def test_cache_clear() -> None: def test_cache_info() -> None: info = yarl.cache_info() - assert info.keys() == {"idna_encode", "idna_decode", "ip_address"} + assert info.keys() == {"idna_encode", "idna_decode", "ip_address", "host_validate"} def test_cache_configure_default() -> None: @@ -22,11 +22,17 @@ def test_cache_configure_default() -> None: def test_cache_configure_None() -> None: yarl.cache_configure( - idna_encode_size=None, idna_decode_size=None, ip_address_size=None + idna_encode_size=None, + idna_decode_size=None, + ip_address_size=None, + host_validate_size=None, ) def test_cache_configure_explicit() -> None: yarl.cache_configure( - idna_encode_size=128, idna_decode_size=128, ip_address_size=128 + idna_encode_size=128, + idna_decode_size=128, + ip_address_size=128, + host_validate_size=128, ) diff --git a/contrib/python/yarl/tests/test_url.py b/contrib/python/yarl/tests/test_url.py index 1dd98197fe1..44018c73bce 100644 --- a/contrib/python/yarl/tests/test_url.py +++ b/contrib/python/yarl/tests/test_url.py @@ -176,6 +176,24 @@ def test_raw_host(): assert url.raw_host == url._val.hostname + ("host"), + [ + ("example.com"), + ("[::1]"), + ("xn--gnter-4ya.com"), + ], +) +def test_host_subcomponent(host: str): + url = URL(f"http://{host}") + assert url.host_subcomponent == host + + +def test_host_subcomponent_return_idna_encoded_host(): + url = URL("http://оун-упа.укр") + assert url.host_subcomponent == "xn----8sb1bdhvc.xn--j1amh" + + def test_raw_host_non_ascii(): url = URL("http://оун-упа.укр") assert "xn----8sb1bdhvc.xn--j1amh" == url.raw_host @@ -2041,3 +2059,14 @@ def test_parsing_populates_cache(): assert url._cache["raw_query_string"] == "a=b" assert url._cache["raw_fragment"] == "frag" assert url._cache["scheme"] == "http" + + + ("host", "is_authority"), + [ + *(("other_gen_delim_" + c, False) for c in "[]"), + ], +) +def test_build_with_invalid_ipv6_host(host: str, is_authority: bool): + with pytest.raises(ValueError, match="Invalid IPv6 URL"): + URL(f"http://{host}/") diff --git a/contrib/python/yarl/tests/test_url_build.py b/contrib/python/yarl/tests/test_url_build.py index a0ab77b2d4e..cc9b15f7e4f 100644 --- a/contrib/python/yarl/tests/test_url_build.py +++ b/contrib/python/yarl/tests/test_url_build.py @@ -15,22 +15,24 @@ def test_build_simple(): assert str(u) == "http://127.0.0.1" def test_url_build_ipv6(): u = URL.build(scheme="http", host="::1") - assert str(u) == "http://::1" + assert str(u) == "http://[::1]" -def test_url_build_ipv6_brackets(): - u = URL.build(scheme="http", host="[::1]") - assert str(u) == "http://::1" +def test_url_build_ipv6_brackets_encoded(): + u = URL.build(scheme="http", host="[::1]", encoded=True) + assert str(u) == "http://[::1]" + + +def test_url_build_ipv6_brackets_not_encoded(): + u = URL.build(scheme="http", host="::1", encoded=False) + assert str(u) == "http://[::1]" def test_url_ipv4_in_ipv6(): u = URL.build(scheme="http", host="2001:db8:122:344::192.0.2.33") - assert str(u) == "http://2001:db8:122:344::c000:221" + assert str(u) == "http://[2001:db8:122:344::c000:221]" def test_build_with_scheme(): @@ -118,6 +120,25 @@ def test_build_with_authority_and_host(): URL.build(authority="host.com", host="example.com") + ("host", "is_authority"), + [ + ("user:[email protected]", True), + ("[email protected]", True), + ("host:com", False), + ("not_percent_encoded%Zf", False), + ("still_not_percent_encoded%fZ", False), + *(("other_gen_delim_" + c, False) for c in "/?#[]"), + ], +) +def test_build_with_invalid_host(host: str, is_authority: bool): + match = r"Host '[^']+' cannot contain '[^']+' \(at position \d+\)" + if is_authority: + match += ", if .* use 'authority' instead of 'host'" + with pytest.raises(ValueError, match=f"{match}$"): + URL.build(host=host) + + def test_build_with_authority(): url = URL.build(scheme="http", authority="степан:[email protected]:8000", path="path") assert ( @@ -132,6 +153,31 @@ def test_build_with_authority_without_encoding(): assert str(url) == "http://foo:[email protected]:8000/path" +def test_build_with_authority_empty_host_no_scheme(): + url = URL.build(authority="", path="path") + assert str(url) == "path" + + +def test_build_with_authority_and_only_user(): + url = URL.build(scheme="https", authority="user:@foo.com", path="/path") + assert str(url) == "https://user:@foo.com/path" + + +def test_build_with_authority_with_port(): + url = URL.build(scheme="https", authority="foo.com:8080", path="/path") + assert str(url) == "https://foo.com:8080/path" + + +def test_build_with_authority_with_ipv6(): + url = URL.build(scheme="https", authority="[::1]", path="/path") + assert str(url) == "https://[::1]/path" + + +def test_build_with_authority_with_ipv6_and_port(): + url = URL.build(scheme="https", authority="[::1]:81", path="/path") + assert str(url) == "https://[::1]:81/path" + + def test_query_str(): u = URL.build(scheme="http", host="127.0.0.1", path="/", query_string="arg=value1") assert str(u) == "http://127.0.0.1/?arg=value1" diff --git a/contrib/python/yarl/tests/test_url_parsing.py b/contrib/python/yarl/tests/test_url_parsing.py index 4fe95185e71..0d58c9d09bb 100644 --- a/contrib/python/yarl/tests/test_url_parsing.py +++ b/contrib/python/yarl/tests/test_url_parsing.py @@ -604,3 +604,28 @@ def test_schemes_that_require_host(scheme: str) -> None: ) with pytest.raises(ValueError, match=expect): URL(f"{scheme}://:1") + + + ("url", "hostname", "hostname_without_brackets"), + [ + ("http://[::1]", "[::1]", "::1"), + ("http://[::1]:8080", "[::1]", "::1"), + ("http://127.0.0.1:8080", "127.0.0.1", "127.0.0.1"), + ( + "http://xn--jxagkqfkduily1i.eu", + "xn--jxagkqfkduily1i.eu", + "xn--jxagkqfkduily1i.eu", + ), + ], +) +def test_url_round_trips( + url: str, hostname: str, hostname_without_brackets: str +) -> None: + """Verify that URLs round-trip correctly.""" + parsed = URL(url) + assert parsed._val.hostname == hostname_without_brackets + assert parsed.raw_host == hostname_without_brackets + assert parsed.host_subcomponent == hostname + assert str(parsed) == url + assert str(URL(str(parsed))) == url diff --git a/contrib/python/yarl/tests/test_url_update_netloc.py b/contrib/python/yarl/tests/test_url_update_netloc.py index cfe6ca7d457..a8ff1f9f8b3 100644 --- a/contrib/python/yarl/tests/test_url_update_netloc.py +++ b/contrib/python/yarl/tests/test_url_update_netloc.py @@ -171,6 +171,26 @@ def test_with_host_non_ascii(): assert url2.authority == "оун-упа.укр:123" + ("host", "is_authority"), + [ + ("user:[email protected]", True), + ("[email protected]", True), + ("host:com", False), + ("not_percent_encoded%Zf", False), + ("still_not_percent_encoded%fZ", False), + *(("other_gen_delim_" + c, False) for c in "/?#[]"), + ], +) +def test_with_invalid_host(host: str, is_authority: bool): + url = URL("http://example.com:123") + match = r"Host '[^']+' cannot contain '[^']+' \(at position \d+\)" + if is_authority: + match += ", if .* use 'authority' instead of 'host'" + with pytest.raises(ValueError, match=f"{match}$"): + url.with_host(host=host) + + def test_with_host_percent_encoded(): url = URL("http://%25cf%2580%cf%80:%25cf%2580%cf%[email protected]:123") url2 = url.with_host("%cf%80.org") diff --git a/contrib/python/yarl/ya.make b/contrib/python/yarl/ya.make index 43d5345771f..b272e44b23e 100644 --- a/contrib/python/yarl/ya.make +++ b/contrib/python/yarl/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(1.12.1) +VERSION(1.13.1) LICENSE(Apache-2.0) diff --git a/contrib/python/yarl/yarl/__init__.py b/contrib/python/yarl/yarl/__init__.py index 50d24c0e778..1bc12c9e4f4 100644 --- a/contrib/python/yarl/yarl/__init__.py +++ b/contrib/python/yarl/yarl/__init__.py @@ -8,7 +8,7 @@ from ._url import ( cache_info, ) -__version__ = "1.12.1" +__version__ = "1.13.1" __all__ = ( "URL", diff --git a/contrib/python/yarl/yarl/_url.py b/contrib/python/yarl/yarl/_url.py index 8a48628fcd9..e695800b967 100644 --- a/contrib/python/yarl/yarl/_url.py +++ b/contrib/python/yarl/yarl/_url.py @@ -1,4 +1,5 @@ import math +import re import sys import warnings from collections.abc import Mapping, Sequence @@ -45,6 +46,22 @@ SCHEME_REQUIRES_HOST = frozenset(("http", "https", "ws", "wss", "ftp")) sentinel = object() +# reg-name: unreserved / pct-encoded / sub-delims +# this pattern matches anything that is *not* in those classes. and is only used +# on lower-cased ASCII values. +_not_reg_name = re.compile( + r""" + # any character not in the unreserved or sub-delims sets, plus % + # (validated with the additional check for pct-encoded sequences below) + [^a-z0-9\-._~!$&'()*+,;=%] + | + # % only allowed if it is part of a pct-encoded + # sequence of 2 hex digits. + %(?![0-9a-f]{2}) + """, + re.VERBOSE, +) + SimpleQuery = Union[str, int, float] QueryVariable = Union[SimpleQuery, "Sequence[SimpleQuery]"] Query = Union[ @@ -64,6 +81,7 @@ class CacheInfo(TypedDict): idna_encode: _CacheInfo idna_decode: _CacheInfo ip_address: _CacheInfo + host_validate: _CacheInfo class _SplitResultDict(TypedDict, total=False): @@ -269,7 +287,7 @@ class URL: raise ValueError(msg) else: host = "" - host = cls._encode_host(host) + host = cls._encode_host(host, validate_host=False) raw_user = None if username is None else cls._REQUOTER(username) raw_password = None if password is None else cls._REQUOTER(password) netloc = cls._make_netloc( @@ -350,10 +368,15 @@ class URL: if encoded: netloc = authority else: - tmp = SplitResult("", authority, "", "", "") - port = None if tmp.port == DEFAULT_PORTS.get(scheme) else tmp.port + _user, _password, _host, _port = cls._split_netloc(authority) + port = None if _port == DEFAULT_PORTS.get(scheme) else _port + _host = ( + "" + if _host is None + else cls._encode_host(_host, validate_host=False) + ) netloc = cls._make_netloc( - tmp.username, tmp.password, tmp.hostname, port, encode=True + _user, _password, _host, port, encode=True, encode_host=False ) elif not user and not password and not host and not port: netloc = "" @@ -395,7 +418,7 @@ class URL: netloc=self._make_netloc( self.raw_user, self.raw_password, - self.raw_host, + self.host_subcomponent, port, encode_host=False, ) @@ -647,6 +670,8 @@ class URL: None for relative URLs. + When working with IPv6 addresses, use the `host_subcomponent` property instead + as it will return the host subcomponent with brackets. """ # Use host instead of hostname for sake of shortness # May add .hostname prop later @@ -660,17 +685,36 @@ class URL: None for relative URLs. """ - raw = self.raw_host - if raw is None: + if (raw := self.raw_host) is None: return None - if "%" in raw: - # Hack for scoped IPv6 addresses like - # fe80::2%Перевірка - # presence of '%' sign means only IPv6 address, so idna is useless. + if raw and raw[-1].isdigit() or ":" in raw: + # IP addresses are never IDNA encoded return raw return _idna_decode(raw) @cached_property + def host_subcomponent(self) -> Union[str, None]: + """Return the host subcomponent part of URL. + + None for relative URLs. + + https://datatracker.ietf.org/doc/html/rfc3986#section-3.2.2 + + `IP-literal = "[" ( IPv6address / IPvFuture ) "]"` + + Examples: + - `http://example.com:8080` -> `example.com` + - `http://example.com:80` -> `example.com` + - `https://127.0.0.1:8443` -> `127.0.0.1` + - `https://[::1]:8443` -> `[::1]` + - `http://[::1]` -> `[::1]` + + """ + if (raw := self.raw_host) is None: + return None + return f"[{raw}]" if ":" in raw else raw + + @cached_property def port(self) -> Union[int, None]: """Port part of URL, with scheme-based fallback. @@ -938,7 +982,9 @@ class URL: return prefix + "/".join(_normalize_path_segments(segments)) @classmethod - def _encode_host(cls, host: str, human: bool = False) -> str: + def _encode_host( + cls, host: str, human: bool = False, validate_host: bool = True + ) -> str: if "%" in host: raw_ip, sep, zone = host.partition("%") else: @@ -953,7 +999,8 @@ class URL: # - 127.0.0.1 (last character is a digit) # - 2001:db8::ff00:42:8329 (contains a colon) # - 2001:db8::ff00:42:8329%eth0 (contains a colon) - # - [2001:db8::ff00:42:8329] (contains a colon) + # - [2001:db8::ff00:42:8329] (contains a colon -- brackets should + # have been removed before it gets here) # Rare IP Address formats are not supported per: # https://datatracker.ietf.org/doc/html/rfc3986#section-7.4 # @@ -977,11 +1024,18 @@ class URL: return host host = host.lower() + if human: + return host + # IDNA encoding is slow, # skip it for ASCII-only strings # Don't move the check into _idna_encode() helper # to reduce the cache size - if human or host.isascii(): + if host.isascii(): + # Check for invalid characters explicitly; _idna_encode() does this + # for non-ascii host names. + if validate_host: + _host_validate(host) return host return _idna_encode(host) @@ -1537,12 +1591,31 @@ def _ip_compressed_version(raw_ip: str) -> Tuple[str, int]: return ip.compressed, ip.version +@lru_cache(_MAXCACHE) +def _host_validate(host: str) -> None: + """Validate an ascii host name.""" + invalid = _not_reg_name.search(host) + if invalid is None: + return + value, pos, extra = invalid.group(), invalid.start(), "" + if value == "@" or (value == ":" and "@" in host[pos:]): + # this looks like an authority string + extra = ( + ", if the value includes a username or password, " + "use 'authority' instead of 'host'" + ) + raise ValueError( + f"Host {host!r} cannot contain {value!r} (at position " f"{pos}){extra}" + ) from None + + @rewrite_module def cache_clear() -> None: """Clear all LRU caches.""" _idna_decode.cache_clear() _idna_encode.cache_clear() _ip_compressed_version.cache_clear() + _host_validate.cache_clear() @rewrite_module @@ -1552,6 +1625,7 @@ def cache_info() -> CacheInfo: "idna_encode": _idna_encode.cache_info(), "idna_decode": _idna_decode.cache_info(), "ip_address": _ip_compressed_version.cache_info(), + "host_validate": _host_validate.cache_info(), } @@ -1561,12 +1635,14 @@ def cache_configure( idna_encode_size: Union[int, None] = _MAXCACHE, idna_decode_size: Union[int, None] = _MAXCACHE, ip_address_size: Union[int, None] = _MAXCACHE, + host_validate_size: Union[int, None] = _MAXCACHE, ) -> None: """Configure LRU cache sizes.""" - global _idna_decode, _idna_encode, _ip_compressed_version + global _idna_decode, _idna_encode, _ip_compressed_version, _host_validate _idna_encode = lru_cache(idna_encode_size)(_idna_encode.__wrapped__) _idna_decode = lru_cache(idna_decode_size)(_idna_decode.__wrapped__) _ip_compressed_version = lru_cache(ip_address_size)( _ip_compressed_version.__wrapped__ ) + _host_validate = lru_cache(host_validate_size)(_host_validate.__wrapped__) |