diff options
| author | YDBot <[email protected]> | 2026-06-17 06:37:57 +0000 |
|---|---|---|
| committer | YDBot <[email protected]> | 2026-06-17 06:37:57 +0000 |
| commit | 07fb17aa0d7a896223bc38ed56e67db76c662999 (patch) | |
| tree | 7b8efe9fbc9e5a72ecca2b3f6ea4d3715797de68 /contrib/python | |
| parent | b897077627f74cc3b63e14717c099fb83ebc7350 (diff) | |
| parent | baaf0cf77790185a0ec90fe7569998d65c70227b (diff) | |
Merge pull request #43647 from ydb-platform/merge-rightlib-260617-0136
Diffstat (limited to 'contrib/python')
45 files changed, 844 insertions, 199 deletions
diff --git a/contrib/python/clickhouse-connect/.dist-info/METADATA b/contrib/python/clickhouse-connect/.dist-info/METADATA index fb2f9d27c55..cb9928f01a9 100644 --- a/contrib/python/clickhouse-connect/.dist-info/METADATA +++ b/contrib/python/clickhouse-connect/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.4 Name: clickhouse-connect -Version: 1.1.0 +Version: 1.1.1 Summary: ClickHouse Database Core Driver for Python, Pandas, and Superset Home-page: https://github.com/ClickHouse/clickhouse-connect Author: ClickHouse Inc. diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/_version.py b/contrib/python/clickhouse-connect/clickhouse_connect/_version.py index b2b60a5505a..64c8c21b112 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/_version.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/_version.py @@ -1 +1 @@ -version = "1.1.0" +version = "1.1.1" diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/driver/asyncclient.py b/contrib/python/clickhouse-connect/clickhouse_connect/driver/asyncclient.py index 1e9568f347e..1cadab3e907 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/driver/asyncclient.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/driver/asyncclient.py @@ -45,7 +45,14 @@ from clickhouse_connect.driver.external import ExternalData from clickhouse_connect.driver.insert import InsertContext from clickhouse_connect.driver.models import ColumnDef, SettingDef from clickhouse_connect.driver.options import check_arrow, check_numpy, check_pandas, check_polars -from clickhouse_connect.driver.query import QueryContext, QueryResult, TzMode, TzSource, arrow_buffer +from clickhouse_connect.driver.query import ( + QueryContext, + QueryResult, + TzMode, + TzSource, + arrow_buffer, + returns_empty_string_on_empty_body, +) from clickhouse_connect.driver.streaming import StreamingFileAdapter, StreamingInsertSource, StreamingResponseSource from clickhouse_connect.driver.summary import QuerySummary from clickhouse_connect.driver.transform import NativeTransform @@ -145,6 +152,21 @@ def _release_lease(response: aiohttp.ClientResponse | None) -> None: release() +_REMOTE_CLOSE_ERRORS = (ConnectionResetError, BrokenPipeError) + + +def _is_retryable_async_connection_error(error: aiohttp.ClientConnectionError) -> bool: + if isinstance(error, (aiohttp.ServerTimeoutError, aiohttp.ClientConnectorError, aiohttp.ServerFingerprintMismatch)): + return False + if isinstance(error, aiohttp.ServerDisconnectedError): + return True + if isinstance(error, _REMOTE_CLOSE_ERRORS): + return True + if isinstance(error.__cause__, _REMOTE_CLOSE_ERRORS): + return True + return isinstance(error.__context__, _REMOTE_CLOSE_ERRORS) + + class AsyncClient(Client): valid_transport_settings = { "database", @@ -951,6 +973,8 @@ class AsyncClient(Client): _release_lease(response) if not body: + if returns_empty_string_on_empty_body(cmd): + return "" return QuerySummary(summary) loop = asyncio.get_running_loop() @@ -984,6 +1008,8 @@ class AsyncClient(Client): url = f"{self.url}/ping" timeout = aiohttp.ClientTimeout(total=3.0) get_kwargs: dict[str, Any] = {"timeout": timeout} + if self._proxy_url: + get_kwargs["proxy"] = self._proxy_url if self.server_host_name: get_kwargs["headers"] = {"Host": self.server_host_name} if self._ssl_context is not None: @@ -1994,9 +2020,9 @@ class AsyncClient(Client): continue await self._error_handler(response) - except aiohttp.ServerConnectionError as e: + except aiohttp.ClientConnectionError as e: msg = str(e) - if "Connection reset" in msg or "Remote end closed" in msg or "Cannot connect" in msg or "Server disconnected" in msg: + if _is_retryable_async_connection_error(e): # Always allow at least one retry on a clean connection error so a single stale # keep-alive socket doesn't surface to the caller, and additionally honor the # retries budget when it is larger (e.g. query_retries for reads), so that @@ -2012,6 +2038,7 @@ class AsyncClient(Client): logger.debug("Retrying after connection error from remote host (attempt %s/%s)", attempts, max_attempts) await asyncio.sleep(0.1 * attempts) continue + logger.debug("Non-retryable aiohttp connection error type=%s", type(e).__name__) raise OperationalError(f"Network Error: {msg}") from e except (aiohttp.ClientError, asyncio.TimeoutError) as e: diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/driver/client.py b/contrib/python/clickhouse-connect/clickhouse_connect/driver/client.py index 7c72e570338..e5da4ec5736 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/driver/client.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/driver/client.py @@ -921,7 +921,7 @@ class Client(ABC): :param external_data: ClickHouse "external data" to send with command/query :param transport_settings: Optional dictionary of transport level settings (HTTP headers, etc.) :return: Decoded response from ClickHouse as either a string, int, or sequence of strings, or QuerySummary - if no data returned + if no data returned. Explicitly handled read-style commands can return an empty string for empty results. """ @abstractmethod diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/driver/httpclient.py b/contrib/python/clickhouse-connect/clickhouse_connect/driver/httpclient.py index f9bcf4d690b..8040cd27c5b 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/driver/httpclient.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/driver/httpclient.py @@ -37,7 +37,7 @@ from clickhouse_connect.driver.httputil import ( get_response_data, ) from clickhouse_connect.driver.insert import InsertContext -from clickhouse_connect.driver.query import QueryContext, QueryResult, TzSource +from clickhouse_connect.driver.query import QueryContext, QueryResult, TzSource, returns_empty_string_on_empty_body from clickhouse_connect.driver.summary import QuerySummary from clickhouse_connect.driver.transform import NativeTransform @@ -46,6 +46,8 @@ columns_only_re = re.compile(r"LIMIT 0\s*$", re.IGNORECASE) ex_header = "X-ClickHouse-Exception-Code" ex_tag_header = "X-ClickHouse-Exception-Tag" +_REMOTE_CLOSE_ERRORS = (ConnectionResetError, BrokenPipeError) + class HttpClient(Client): params = {} @@ -478,6 +480,8 @@ class HttpClient(Client): return result except UnicodeDecodeError: return str(response.data) + if returns_empty_string_on_empty_body(cmd): + return "" return QuerySummary(self._summary(response)) def _error_handler(self, response: HTTPResponse, retried: bool = False) -> None: @@ -574,7 +578,8 @@ class HttpClient(Client): # retries budget when it is larger (e.g. query_retries for reads), so that # bursts of stale pooled connections can be drained before giving up. max_attempts = max(2, retries + 1) - if isinstance(ex.__context__, ConnectionResetError) and attempts < max_attempts: + remote_close = isinstance(ex.__context__, _REMOTE_CLOSE_ERRORS) or isinstance(ex.__cause__, _REMOTE_CLOSE_ERRORS) + if remote_close and attempts < max_attempts: # The server closed the connection, probably because the Keep Alive has expired. # We should be safe to retry, as ClickHouse should not have processed anything on # a connection that it killed. @@ -588,6 +593,7 @@ class HttpClient(Client): logger.debug("Retrying remotely closed connection (attempt %s/%s)", attempts, max_attempts) time.sleep(0.1 * attempts) continue + logger.debug("Non-retryable HTTP transport error type=%s", type(ex).__name__) logger.warning("Unexpected Http Driver Exception") err_url = f" ({self.url})" if self.show_clickhouse_errors else "" raise OperationalError(f"Error {ex} executing HTTP request attempt {attempts}{err_url}") from ex diff --git a/contrib/python/clickhouse-connect/clickhouse_connect/driver/query.py b/contrib/python/clickhouse-connect/clickhouse_connect/driver/query.py index 38f6abec512..c17cea1c485 100644 --- a/contrib/python/clickhouse-connect/clickhouse_connect/driver/query.py +++ b/contrib/python/clickhouse-connect/clickhouse_connect/driver/query.py @@ -34,6 +34,14 @@ limit_re = re.compile(r"\s+LIMIT($|\s)", re.IGNORECASE) select_re = re.compile(r"(^|\s)SELECT\s", re.IGNORECASE) insert_re = re.compile(r"(^|\s)INSERT\s*INTO", re.IGNORECASE) command_re = re.compile(r"(^\s*)(" + commands + r")\s", re.IGNORECASE) +row_policy_show_re = re.compile(r"^\s*SHOW\s+(ROW\s+)?POLICIES\b", re.IGNORECASE) +bare_row_policy_show_re = re.compile(r"^\s*SHOW\s+(ROW\s+)?POLICIES\s*$", re.IGNORECASE) + + +def returns_empty_string_on_empty_body(cmd: str | bytes) -> bool: + if not isinstance(cmd, str): + return False + return row_policy_show_re.search(remove_sql_comments(cmd)) is not None class QueryContext(BaseQueryContext): @@ -167,7 +175,7 @@ class QueryContext(BaseQueryContext): @property def is_command(self) -> bool: - return command_re.search(self.uncommented_query) is not None + return command_re.search(self.uncommented_query) is not None or bare_row_policy_show_re.search(self.uncommented_query) is not None def set_parameters(self, parameters: dict[str, Any]): self.parameters = parameters diff --git a/contrib/python/clickhouse-connect/ya.make b/contrib/python/clickhouse-connect/ya.make index f57136f7567..82a5eb6d189 100644 --- a/contrib/python/clickhouse-connect/ya.make +++ b/contrib/python/clickhouse-connect/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(1.1.0) +VERSION(1.1.1) LICENSE(Apache-2.0) diff --git a/contrib/python/hypothesis/py3/.dist-info/METADATA b/contrib/python/hypothesis/py3/.dist-info/METADATA index 5765424bbb5..da12b2f59f0 100644 --- a/contrib/python/hypothesis/py3/.dist-info/METADATA +++ b/contrib/python/hypothesis/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.4 Name: hypothesis -Version: 6.153.2 +Version: 6.155.1 Summary: The property-based testing library for Python Author-email: "David R. MacIver and Zac Hatfield-Dodds" <[email protected]> License-Expression: MPL-2.0 diff --git a/contrib/python/hypothesis/py3/hypothesis/core.py b/contrib/python/hypothesis/py3/hypothesis/core.py index beb28c539a3..44418c255ed 100644 --- a/contrib/python/hypothesis/py3/hypothesis/core.py +++ b/contrib/python/hypothesis/py3/hypothesis/core.py @@ -2328,7 +2328,11 @@ def given( except UnsatisfiedAssumption: status = Status.INVALID return None - except BaseException: + except BaseException as e: + # The engine sets data.interesting_origin in + # _execute_once_for_engine, but fuzz_one_input calls + # execute_once directly, so we replicate it here. + data.interesting_origin = InterestingOrigin.from_exception(e) known = minimal_failures.get(data.interesting_origin) if settings.database is not None and ( known is None or sort_key(data.nodes) <= sort_key(known) diff --git a/contrib/python/hypothesis/py3/hypothesis/extra/numpy.py b/contrib/python/hypothesis/py3/hypothesis/extra/numpy.py index eee98231592..0d93bdb82bb 100644 --- a/contrib/python/hypothesis/py3/hypothesis/extra/numpy.py +++ b/contrib/python/hypothesis/py3/hypothesis/extra/numpy.py @@ -110,6 +110,19 @@ TIME_RESOLUTIONS = ("Y", "M", "D", "h", "m", "s", "ms", "us", "ns", "ps", "fs", NP_FIXED_UNICODE = tuple(int(x) for x in np.__version__.split(".")[:2]) >= (1, 19) +def _reject_dtype_class(dtype: object) -> None: + # A common mistake is to pass a dtype *class*, e.g. np.dtypes.StringDType, + # rather than an instance such as np.dtypes.StringDType(). numpy silently + # coerces such classes to the object dtype, so we reject them with a more + # helpful message than the resulting confusion further down the line. + if isinstance(dtype, type) and issubclass(dtype, np.dtype): + name = getattr(dtype, "__name__", repr(dtype)) + raise InvalidArgument( + f"Cannot infer a strategy from the dtype class {name}; pass an " + f"instance instead, e.g. {name}() rather than {name}." + ) + + @defines_strategy(force_reusable_values=True) def from_dtype( dtype: np.dtype, @@ -137,6 +150,7 @@ def from_dtype( :func:`arrays` which allow a variety of numeric dtypes, as it seamlessly handles the ``width`` or representable bounds for you. """ + _reject_dtype_class(dtype) check_type(np.dtype, dtype, "dtype") kwargs = {k: v for k, v in locals().items() if k != "dtype" and v is not None} @@ -214,6 +228,14 @@ def from_dtype( result = st.text(**compat_kw("alphabet", "min_size", max_size=max_size)).filter( lambda b: b[-1:] != "\0" ) + elif dtype.kind == "T": + # NumPy 2.0+ variable-width strings (StringDType). Unlike the fixed-width + # "U"/"S" dtypes, these store arbitrary Python strings with no length + # limit and no null-termination, so we can use st.text() directly - but + # the UTF-8 backing storage means we must exclude lone surrogates. + if "alphabet" not in kwargs: + kwargs["alphabet"] = st.characters(codec="utf-8") + result = st.text(**compat_kw("alphabet", "min_size", "max_size")) elif dtype.kind in ("m", "M"): if "[" in dtype.str: res = st.just(dtype.str.split("[")[-1][:-1]) @@ -555,6 +577,7 @@ def arrays( lambda s: arrays(dtype, s, elements=elements, fill=fill, unique=unique) ) # From here on, we're only dealing with values and it's relatively simple. + _reject_dtype_class(dtype) dtype = np.dtype(dtype) # type: ignore[arg-type] assert isinstance(dtype, np.dtype) # help mypy out a bit... if elements is None or isinstance(elements, Mapping): diff --git a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/engine.py b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/engine.py index ded037098d6..6605242ba36 100644 --- a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/engine.py +++ b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/engine.py @@ -226,6 +226,7 @@ StatisticsDict = TypedDict( "generate-phase": NotRequired[PhaseStatistics], "reuse-phase": NotRequired[PhaseStatistics], "shrink-phase": NotRequired[PhaseStatistics], + "explain-phase": NotRequired[PhaseStatistics], "stopped-because": NotRequired[str], "targets": NotRequired[dict[str, float]], "nodeid": NotRequired[str], @@ -308,6 +309,8 @@ class ConjectureRunner: self._current_phase: str = "(not a phase)" self.statistics: StatisticsDict = {} self.stats_per_test_case: list[CallStats] = [] + # Time spent in any nested phase, so the enclosing phase can exclude it. + self._nested_phase_seconds: float = 0.0 self.interesting_examples: dict[InterestingOrigin, ConjectureResult] = {} # We use call_count because there may be few possible valid_examples. @@ -379,20 +382,36 @@ class ConjectureRunner: @contextmanager def _log_phase_statistics( - self, phase: Literal["reuse", "generate", "shrink"] + self, phase: Literal["reuse", "generate", "shrink", "explain"] ) -> Generator[None, None, None]: - self.stats_per_test_case.clear() + # Phases may nest - the explain phase runs inside the shrink phase - so + # we save and restore the per-call stats and current phase, exclude the + # duration of any nested phase, and accumulate when a phase is entered + # more than once (the explain phase runs once per shrinking target). + saved_stats = self.stats_per_test_case + saved_phase = self._current_phase + saved_nested_seconds = self._nested_phase_seconds + self.stats_per_test_case = [] + self._current_phase = phase + self._nested_phase_seconds = 0.0 start_time = time.perf_counter() try: - self._current_phase = phase yield finally: - self.statistics[phase + "-phase"] = { # type: ignore - "duration-seconds": time.perf_counter() - start_time, - "test-cases": list(self.stats_per_test_case), - "distinct-failures": len(self.interesting_examples), - "shrinks-successful": self.shrinks, - } + elapsed = time.perf_counter() - start_time + # A phase can be entered more than once (the explain phase runs once + # per shrinking target), so accumulate into any existing bucket. + stats = self.statistics.setdefault( + phase + "-phase", # type: ignore + {"duration-seconds": 0.0, "test-cases": []}, + ) + stats["duration-seconds"] += elapsed - self._nested_phase_seconds + stats["test-cases"] += self.stats_per_test_case + stats["distinct-failures"] = len(self.interesting_examples) + stats["shrinks-successful"] = self.shrinks + self.stats_per_test_case = saved_stats + self._current_phase = saved_phase + self._nested_phase_seconds = saved_nested_seconds + elapsed @property def should_optimise(self) -> bool: diff --git a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/shrinker.py b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/shrinker.py index f36739f0f4e..c8884bcb4ea 100644 --- a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/shrinker.py +++ b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/shrinker.py @@ -491,10 +491,12 @@ class Shrinker: self.explain() def explain(self) -> None: - if not self.should_explain or not self.shrink_target.arg_slices: return + with self.engine._log_phase_statistics("explain"): + self._explain() + def _explain(self) -> None: self.max_stall = 2**100 shrink_target = self.shrink_target nodes = self.nodes @@ -1184,66 +1186,59 @@ class Shrinker: # If this produced something completely invalid we ditch it # here rather than trying to persevere. if attempt.status is Status.OVERRUN: - return False + # Lowering a size-controlling choice can make the realigned (and + # now boring) collection stop triggering the failure, so the test + # draws further and overruns before we see the realignment -- this + # is common in stateful tests, where a non-failing step is followed + # by more steps. Re-run without the length limit to recover the + # realigned tree, which the repair logic below can then act on. + attempt = self.engine.cached_test_function( + [n.value for n in initial_attempt], extend="full" + ) + if attempt.status is Status.OVERRUN: + return False if attempt.status is Status.INVALID: return False - if attempt.misaligned_at is not None: - # we're invalid due to a misalignment in the tree. We'll try to fix - # a very specific type of misalignment here: where we have a node of - # {"size": n} and tried to draw the same node, but with {"size": m < n}. - # This can occur with eg - # - # n = data.draw_integer() - # s = data.draw_string(min_size=n) - # - # where we try lowering n, resulting in the test_function drawing a lower - # min_size than our attempt had for the draw_string node. - # - # We'll now try realigning this tree by: - # * replacing the constraints in our attempt with what test_function tried - # to draw in practice - # * truncating the value of that node to match min_size - # - # This helps in the specific case of drawing a value and then drawing - # a collection of that size...and not much else. In practice this - # helps because this antipattern is fairly common. - - # TODO we'll probably want to apply the same trick as in the valid - # case of this function of preserving from the right instead of - # preserving from the left. see test_can_shrink_variable_string_draws. - - index, attempt_choice_type, attempt_constraints, _attempt_forced = ( - attempt.misaligned_at - ) - node = self.nodes[index] - if node.type != attempt_choice_type: - return False # pragma: no cover - if node.was_forced: - return False # pragma: no cover - - if node.type in {"string", "bytes"}: - # if the size *increased*, we would have to guess what to pad with - # in order to try fixing up this attempt. Just give up. - if node.constraints["min_size"] <= attempt_constraints["min_size"]: - # attempts which increase min_size tend to overrun rather than - # be misaligned, making a covering case difficult. - return False # pragma: no cover - # the size decreased in our attempt. Try again, but truncate the value - # to that size by removing any elements past min_size. - return self.consider_new_nodes( - initial_attempt[: node.index] - + [ - initial_attempt[node.index].copy( - with_constraints=attempt_constraints, - with_value=initial_attempt[node.index].value[ - : attempt_constraints["min_size"] - ], - ) - ] - + initial_attempt[node.index :] - ) + # When we lower a choice that controls the size of a later collection, + # eg + # + # n = data.draw_integer() + # s = data.draw_string(min_size=n, max_size=n) + # + # the recorded value for that collection no longer fits the constraints + # the test function actually used, so the engine realigns the tree by + # substituting a freshly-generated (simplest) value -- discarding + # whatever made the collection interesting. (We can't rely on + # ``attempt.misaligned_at`` to detect this, because the realigned choice + # sequence is often independently cached as an ordinary, non-misaligned + # result.) We detect a string/bytes node whose recorded value is now too + # long, and retry with it truncated to fit. We try preserving content + # from either end, since the interesting part may be at the start or the + # end (see test_can_shrink_variable_string_draws). + for i in range(min(len(initial_attempt), len(attempt.nodes))): + node = initial_attempt[i] + attempt_node = attempt.nodes[i] + if ( + node.type == attempt_node.type + and node.type in {"string", "bytes"} + and not node.was_forced + and len(node.value) > attempt_node.constraints["max_size"] + ): + max_size = attempt_node.constraints["max_size"] + for truncated in (node.value[:max_size], node.value[-max_size:]): + if self.consider_new_nodes( + initial_attempt[:i] + + [ + node.copy( + with_constraints=attempt_node.constraints, + with_value=truncated, + ) + ] + + initial_attempt[i + 1 :] + ): + return True lost_nodes = len(self.nodes) - len(attempt.nodes) if lost_nodes <= 0: @@ -1292,17 +1287,18 @@ class Shrinker: return False def remove_discarded(self): - """Try removing all bytes marked as discarded. + """Try removing all nodes marked as discarded. This is primarily to deal with data that has been ignored while doing rejection sampling - e.g. as a result of an integer range, or a filtered strategy. - Such data will also be handled by the adaptive_example_deletion pass, - but that pass is necessarily more conservative and will try deleting - each interval individually. The common case is that all data drawn and - rejected can just be thrown away immediately in one block, so this pass - will be much faster than trying each one individually when it works. + Such data will also be handled by the ``node_program("X")`` deletion + passes, but those are necessarily more conservative and will try + deleting each contiguous run of nodes individually. The common case is + that all data drawn and rejected can just be thrown away immediately in + one block, so this pass will be much faster than trying each one + individually when it works. returns False if there is discarded data and removing it does not work, otherwise returns True. diff --git a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/shrinking/collection.py b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/shrinking/collection.py index cd51eed2a07..4874ae58162 100644 --- a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/shrinking/collection.py +++ b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/shrinking/collection.py @@ -10,6 +10,7 @@ from collections import Counter +from hypothesis.internal.conjecture.junkdrawer import find_integer from hypothesis.internal.conjecture.shrinking.common import Shrinker from hypothesis.internal.conjecture.shrinking.ordering import Ordering from hypothesis.internal.conjecture.utils import identity @@ -52,11 +53,19 @@ class Collection(Shrinker): zero = self.from_order(0) self.consider([zero] * len(self.current)) - # try deleting each element in turn, starting from the back - # TODO_BETTER_SHRINK: adaptively delete here by deleting larger chunks at once - # if early deletes succeed. use find_integer. turns O(n) into O(log(n)) - for i in reversed(range(len(self.current))): - self.consider(self.current[:i] + self.current[i + 1 :]) + # try deleting elements, starting from the back. We adaptively grow the + # chunk we delete via find_integer, so a run of deletable elements costs + # O(log(n)) calls rather than O(n). + i = len(self.current) - 1 + while i >= 0: + base = self.current + + def delete_k(k, *, i=i, base=base): + # delete the k elements ending at index i, i.e. base[i - k + 1 : i + 1] + return k <= i + 1 and self.consider(base[: i - k + 1] + base[i + 1 :]) + + # advance past the (possibly deleted) chunk; max(k, 1) ensures progress + i -= max(find_integer(delete_k), 1) # then try reordering Ordering.shrink(self.current, self.consider, key=self.to_order) diff --git a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/shrinking/floats.py b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/shrinking/floats.py index f55d3ddc8af..6425418af75 100644 --- a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/shrinking/floats.py +++ b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/shrinking/floats.py @@ -14,7 +14,33 @@ import sys from hypothesis.internal.conjecture.floats import float_to_lex from hypothesis.internal.conjecture.shrinking.common import Shrinker from hypothesis.internal.conjecture.shrinking.integer import Integer -from hypothesis.internal.floats import MAX_PRECISE_INTEGER, float_to_int +from hypothesis.internal.floats import MAX_PRECISE_INTEGER, float_to_int, int_to_float + +# Bit pattern of the boundary float, so we can compute float-grid indices +# relative to it without recomputing the constant on every call. +_BOUNDARY_BITS = float_to_int(float(MAX_PRECISE_INTEGER)) + + +def _float_to_position(f: float) -> int: + """Map a non-negative float to a linear integer position such that adjacent + representable floats correspond to adjacent integers. + + For ``f <= MAX_PRECISE_INTEGER`` the position is just ``int(f)``. Above the + boundary, where the gap between adjacent floats exceeds 1, we extend by the + float's index in the bit-pattern sequence past ``MAX_PRECISE_INTEGER``, so + that decrementing the position by 1 corresponds to ``next_down(f)``. + """ + if f <= MAX_PRECISE_INTEGER: + return int(f) + return MAX_PRECISE_INTEGER + (float_to_int(f) - _BOUNDARY_BITS) + + +def _position_to_float(n: int) -> float: + """Inverse of :func:`_float_to_position` on the integer-valued range. Always + returns an integer-valued, non-negative float.""" + if n <= MAX_PRECISE_INTEGER: + return float(n) + return int_to_float(_BOUNDARY_BITS + (n - MAX_PRECISE_INTEGER)) class Float(Shrinker): @@ -51,13 +77,18 @@ class Float(Shrinker): return True def run_step(self): - # above MAX_PRECISE_INTEGER, all floats are integers. Shrink like one. - # TODO_BETTER_SHRINK: at 2 * MAX_PRECISE_INTEGER, n - 1 == n - 2, and - # Integer.shrink will likely perform badly. We should have a specialized - # big-float shrinker, which mostly follows Integer.shrink but replaces - # n - 1 with next_down(n). + # Above MAX_PRECISE_INTEGER all floats are integers, but the gap between + # adjacent floats is > 1, so consecutive integers are not all + # representable. Integer.shrink would step by n - 1, which rounds straight + # back to n and stalls. We instead shrink on the float grid by delegating + # to Integer with a bijection that maps each representable float to an + # adjacent integer position, so n - 1 always corresponds to next_down(n). if self.current > MAX_PRECISE_INTEGER: - self.delegate(Integer, convert_to=int, convert_from=float) + self.delegate( + Integer, + convert_to=_float_to_position, + convert_from=_position_to_float, + ) return # Finally we get to the important bit: Each of these is a small change diff --git a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/utils.py b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/utils.py index e916fe2e3f1..9f8b41f709e 100644 --- a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/utils.py +++ b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/utils.py @@ -77,6 +77,19 @@ def identity(v: T) -> T: return v +def fisher_yates_shuffle(data: "ConjectureData", ls: list[T]) -> None: + """Shuffle ``ls`` in place, drawing from ``data``. + + Reversed Fisher-Yates shuffle: swap each element with itself or with a + later element. This shrinks i==j for each element, i.e. towards no change, + so a shuffled sequence shrinks back to its original order. We don't + consider the last element as it's always a no-op. + """ + for i in range(len(ls) - 1): + j = data.draw_integer(i, len(ls) - 1) + ls[i], ls[j] = ls[j], ls[i] + + def check_sample( values: type[enum.Enum] | Sequence[T], strategy_name: str ) -> Sequence[T]: diff --git a/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/collections.py b/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/collections.py index 97de24f249b..735145638ac 100644 --- a/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/collections.py +++ b/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/collections.py @@ -10,7 +10,7 @@ import copy import math -from collections.abc import Callable, Iterable +from collections.abc import Callable, Iterable, Mapping from typing import Any, TypeGuard, overload from hypothesis import strategies as st @@ -370,37 +370,37 @@ class UniqueSampledListStrategy(UniqueListStrategy): return result -class FixedDictStrategy(SearchStrategy[dict[Any, Any]]): - """A strategy which produces dicts with a fixed set of keys, given a +class FixedDictStrategy(SearchStrategy[Mapping[Any, Any]]): + """A strategy which produces mappings with a fixed set of keys, given a strategy for each of their equivalent values. - e.g. {'foo' : some_int_strategy} would generate dicts with the single + e.g. {'foo' : some_int_strategy} would generate mappings with the single key 'foo' mapping to some integer. """ def __init__( self, - mapping: dict[Any, SearchStrategy[Any]], + mapping: Mapping[Any, SearchStrategy[Any]], *, - optional: dict[Any, SearchStrategy[Any]] | None, + optional: Mapping[Any, SearchStrategy[Any]] | None, ): super().__init__() dict_type = type(mapping) self.mapping = mapping keys = tuple(mapping.keys()) self.fixed = st.tuples(*[mapping[k] for k in keys]).map( - lambda value: dict_type(zip(keys, value, strict=True)) + lambda value: dict_type(zip(keys, value, strict=True)) # type: ignore ) self.optional = optional - def do_draw(self, data: ConjectureData) -> dict[Any, Any]: + def do_draw(self, data: ConjectureData) -> Mapping[Any, Any]: context = current_build_context() arg_labels: ArgLabelsT = {} - value = type(self.mapping)() + pairs: list[tuple[Any, Any]] = [] for key, strategy in self.mapping.items(): with context.track_arg_label(str(key)) as arg_label: - value[key] = data.draw(strategy) + pairs.append((key, data.draw(strategy))) arg_labels |= arg_label if self.optional is not None: @@ -416,12 +416,17 @@ class FixedDictStrategy(SearchStrategy[dict[Any, Any]]): remaining[-1], remaining[j] = remaining[j], remaining[-1] key = remaining.pop() with context.track_arg_label(str(key)) as arg_label: - value[key] = data.draw(self.optional[key]) + pairs.append((key, data.draw(self.optional[key]))) arg_labels |= arg_label + # Vary the dict's iteration order (#3906). We shuffle after choosing + # the optional keys, so only order varies, not the set of keys. + cu.fisher_yates_shuffle(data, pairs) + value = type(self.mapping)(pairs) # type: ignore + if arg_labels: context.known_object_printers[IDKey(value)].append( - _fixeddict_pprinter(arg_labels, self.mapping) + _fixeddict_pprinter(arg_labels) ) return value diff --git a/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/core.py b/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/core.py index 9ef4a555c41..3df73ed2905 100644 --- a/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/core.py +++ b/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/core.py @@ -87,6 +87,7 @@ from hypothesis.internal.conjecture.utils import ( calc_label_from_name, check_sample, combine_labels, + fisher_yates_shuffle, identity, ) from hypothesis.internal.entropy import get_seeder_and_restorer @@ -498,43 +499,59 @@ def iterables( ).map(PrettyIter) -# this type definition is imprecise, in multiple ways: -# * mapping and optional can be of different types: -# s: dict[str | int, int] = st.fixed_dictionaries( -# {"a": st.integers()}, optional={1: st.integers()} -# ) -# * the values in either mapping or optional need not all be of the same type: -# s: dict[str, int | bool] = st.fixed_dictionaries( -# {"a": st.integers(), "b": st.booleans()} -# ) -# * the arguments may be of any dict-compatible type, in which case the return -# value will be of that type instead of dict +# fixed_dictionaries accepts Mapping rather than the invariant dict so that +# type-checkers can infer the value type even when the per-key strategies are +# heterogeneous: Mapping is covariant in its value type and SearchStrategy is +# covariant in its own, so e.g. `SearchStrategy[int] | SearchStrategy[str]` is +# accepted as `SearchStrategy[int | str]`. The overloads let mapping and +# optional contribute independent key and value types, which are unioned in the +# result. See revealed_types.py for the resulting types. # -# Overloads may help here, but I doubt we'll be able to satisfy all these -# constraints. +# We use fresh typevars rather than the module-level Ex because Ex has a default +# (PEP 696), and a defaulted typevar may not precede a bare one in a signature. # -# Here's some platonic ideal test cases for revealed_types.py, with the understanding -# that some may not be achievable: -# -# ("fixed_dictionaries({'a': booleans()})", "dict[str, bool]"), -# ("fixed_dictionaries({'a': booleans(), 'b': integers()})", "dict[str, bool | int]"), -# ("fixed_dictionaries({}, optional={'a': booleans()})", "dict[str, bool]"), -# ( -# "fixed_dictionaries({'a': booleans()}, optional={1: booleans()})", -# "dict[str | int, bool]", -# ), -# ( -# "fixed_dictionaries({'a': booleans()}, optional={1: integers()})", -# "dict[str | int, bool | int]", -# ), +# The remaining imprecision is that we always report a plain dict, even though +# at runtime the result preserves the concrete (dict-subclass) type of mapping. +K = TypeVar("K") +V = TypeVar("V") +K2 = TypeVar("K2") +V2 = TypeVar("V2") + + +@overload +def fixed_dictionaries( + mapping: Mapping[K, SearchStrategy[V]], +) -> SearchStrategy[dict[K, V]]: # pragma: no cover + ... + + +@overload +def fixed_dictionaries( + # Matching an empty mapping against NoReturn lets the result come solely + # from optional, rather than picking up a spurious `Any` from the empty + # mapping (whose key and value types are otherwise uninferable). + mapping: Mapping[NoReturn, NoReturn], + *, + optional: Mapping[K2, SearchStrategy[V2]], +) -> SearchStrategy[dict[K2, V2]]: # pragma: no cover + ... + + +@overload +def fixed_dictionaries( + mapping: Mapping[K, SearchStrategy[V]], + *, + optional: Mapping[K2, SearchStrategy[V2]], +) -> SearchStrategy[dict[K | K2, V | V2]]: # pragma: no cover + ... @defines_strategy() def fixed_dictionaries( - mapping: dict[T, SearchStrategy[Ex]], + mapping: Mapping[Any, SearchStrategy[Any]], *, - optional: dict[T, SearchStrategy[Ex]] | None = None, -) -> SearchStrategy[dict[T, Ex]]: + optional: Mapping[Any, SearchStrategy[Any]] | None = None, +) -> SearchStrategy[dict[Any, Any]]: """Generates a dictionary of the same type as mapping with a fixed set of keys mapping to strategies. ``mapping`` must be a dict subclass. @@ -548,12 +565,12 @@ def fixed_dictionaries( Examples from this strategy shrink by shrinking each individual value in the generated dictionary, and omitting optional key-value pairs. """ - check_type(dict, mapping, "mapping") + check_type(Mapping, mapping, "mapping") for k, v in mapping.items(): check_strategy(v, f"mapping[{k!r}]") if optional is not None: - check_type(dict, optional, "optional") + check_type(Mapping, optional, "optional") for k, v in optional.items(): check_strategy(v, f"optional[{k!r}]") if type(mapping) != type(optional): @@ -568,7 +585,13 @@ def fixed_dictionaries( f"which is invalid: {set(mapping) & set(optional)!r}" ) - return FixedDictStrategy(mapping, optional=optional) + # FixedDictStrategy honestly types itself as SearchStrategy[Mapping], since + # type(mapping)(pairs) may return any Mapping subclass. We narrow to dict + # here because that's what callers almost always get and find convenient. + return cast( + "SearchStrategy[dict[Any, Any]]", + FixedDictStrategy(mapping, optional=optional), + ) _get_first_item = operator.itemgetter(0) @@ -1303,6 +1326,7 @@ def _from_type_deferred(thing: type[Ex]) -> SearchStrategy[Ex]: _recurse_guard: ContextVar = ContextVar("recurse_guard") +_abstract_recurse_guard: ContextVar = ContextVar("abstract_recurse_guard") def _from_type(thing: type[Ex]) -> SearchStrategy[Ex]: @@ -1552,16 +1576,20 @@ def _from_type(thing: type[Ex]) -> SearchStrategy[Ex]: # a subclass of `thing` and are not themselves a subtype of any other such # type. For example, `Number -> integers() | floats()`, but bools() is # not included because bool is a subclass of int as well as Number. + # Filter to matching subtypes *before* sorting, because computing the repr + # of every registered strategy (just to establish a deterministic order) is + # surprisingly expensive and usually wasted - the matching set is typically + # empty for user-defined types. + matching = [ + (k, v) + for k, v in types._global_type_lookup.items() + if isinstance(k, type) + and issubclass(k, thing) + and sum(types.try_issubclass(k, typ) for typ in types._global_type_lookup) == 1 + ] strategies = [ s - for s in ( - as_strategy(v, thing) - for k, v in sorted(types._global_type_lookup.items(), key=repr) - if isinstance(k, type) - and issubclass(k, thing) - and sum(types.try_issubclass(k, typ) for typ in types._global_type_lookup) - == 1 - ) + for s in (as_strategy(v, thing) for _, v in sorted(matching, key=repr)) if s is not NotImplemented ] if any(not s.is_empty for s in strategies): @@ -1644,12 +1672,35 @@ def _from_type(thing: type[Ex]) -> SearchStrategy[Ex]: "type without any subclasses. Consider using register_type_strategy" ) - subclass_strategies: SearchStrategy = nothing() - for sc in subclasses: - try: - subclass_strategies |= _from_type(sc) - except Exception: - pass + # When subclasses reference `thing` (directly, or via a sibling subclass) + # in their own annotations, naively resolving each subclass would re-resolve + # the entire hierarchy once per reference - which is combinatorially + # expensive for mutually-recursive types. We track the abstract types we're + # currently resolving and defer any recursive reference back to them (by + # returning the cached strategy, so the references share one object - which + # lets recursion in e.g. is_empty checks terminate), so each type is resolved + # only once per pass. We use a guard separate from `_recurse_guard` because + # this catches references regardless of how they reach `_from_type` (e.g. as a + # union arg), and because it must not make `from_type_guarded` treat a + # subclass's required field of type `thing` as unresolvable. + try: + abstract_guard = _abstract_recurse_guard.get() + except LookupError: + _abstract_recurse_guard.set(abstract_guard := set()) + if thing in abstract_guard: + return from_type(thing) + + abstract_guard.add(thing) + try: + substrategies = [] + for sc in subclasses: + try: + substrategies.append(_from_type(sc)) + except Exception: + pass + finally: + abstract_guard.discard(thing) + subclass_strategies = one_of(substrategies) if subclass_strategies.is_empty: # We're unable to resolve subclasses now, but we might be able to later - # so we'll just go back to the mixed distribution. @@ -1840,10 +1891,13 @@ def decimals( factor = Decimal(10) ** -places min_num, max_num = None, None + # Work out the integer bounds exactly: limited-precision division can + # round when the bounds have more than `places` fractional digits, + # which would make ceil/floor over- or undershoot the true bound. if min_value is not None: - min_num = ceil(ctx(min_value).divide(min_value, factor)) + min_num = ceil(Fraction(min_value) / Fraction(factor)) if max_value is not None: - max_num = floor(ctx(max_value).divide(max_value, factor)) + max_num = floor(Fraction(max_value) / Fraction(factor)) if min_num is not None and max_num is not None and min_num > max_num: raise InvalidArgument( f"There are no decimals with {places} places between " @@ -1911,13 +1965,8 @@ class PermutationStrategy(SearchStrategy): self.values = values def do_draw(self, data): - # Reversed Fisher-Yates shuffle: swap each element with itself or with - # a later element. This shrinks i==j for each element, i.e. to no - # change. We don't consider the last element as it's always a no-op. result = list(self.values) - for i in range(len(result) - 1): - j = data.draw_integer(i, len(result) - 1) - result[i], result[j] = result[j], result[i] + fisher_yates_shuffle(data, result) return result diff --git a/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/types.py b/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/types.py index b65fea49398..48e8913b408 100644 --- a/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/types.py +++ b/contrib/python/hypothesis/py3/hypothesis/strategies/_internal/types.py @@ -30,6 +30,7 @@ import uuid import warnings import zoneinfo from collections.abc import Iterator +from contextvars import ContextVar from functools import partial from pathlib import PurePath from types import FunctionType @@ -220,6 +221,13 @@ def type_sorting_key(t): return (is_container, repr(t)) +# Types whose forward references we are currently resolving, used to break the +# recursion in self- or mutually-referential forward references such as +# ``A = list[Union["A", str]]``. Without this we would recurse until hitting a +# RecursionError, which makes resolution depend on the ambient stack depth. +_forward_ref_resolution: ContextVar[list] = ContextVar("forward_ref_resolution") + + def _resolve_forward_ref_in_caller(forward_arg: str) -> typing.Any: """Try to resolve a forward reference name by walking up the call stack. @@ -679,7 +687,20 @@ def from_typing_type(thing): if resolved is None: # pragma: no branch resolved = _resolve_forward_ref_in_caller(thing.__forward_arg__) if resolved is not None and is_a_type(resolved): - return st.from_type(resolved) + try: + in_progress = _forward_ref_resolution.get() + except LookupError: + _forward_ref_resolution.set(in_progress := []) + if resolved in in_progress: + # We're already resolving this type higher up the stack, so this + # is a recursive reference; defer to break the cycle and rely on + # st.from_type's cache to tie the recursive knot. + return st.deferred(lambda r=resolved: st.from_type(r)) + in_progress.append(resolved) + try: + return st.from_type(resolved) + finally: + in_progress.pop() def is_maximal(t): # For each k in the mapping, we use it if it's the most general type diff --git a/contrib/python/hypothesis/py3/hypothesis/vendor/pretty.py b/contrib/python/hypothesis/py3/hypothesis/vendor/pretty.py index 51b11433a36..79b59494cae 100644 --- a/contrib/python/hypothesis/py3/hypothesis/vendor/pretty.py +++ b/contrib/python/hypothesis/py3/hypothesis/vendor/pretty.py @@ -964,10 +964,7 @@ def _tuple_pprinter(arg_labels: ArgLabelsT) -> PrettyPrintFunction: return inner -def _fixeddict_pprinter( - arg_labels: ArgLabelsT, - mapping: dict[Any, Any], -) -> PrettyPrintFunction: +def _fixeddict_pprinter(arg_labels: ArgLabelsT) -> PrettyPrintFunction: """Pretty printer for fixed_dictionaries that shows sub-argument comments.""" def inner(obj: dict, p: RepresentationPrinter, cycle: bool) -> None: @@ -975,8 +972,8 @@ def _fixeddict_pprinter( return p.text("{...}") get = lambda k: _get_slice_comment(p, arg_labels, k) - # Preserve mapping key order, then any optional keys (deduped) - keys = list(dict.fromkeys(k for k in [*mapping, *obj] if k in obj)) + # Print in the dict's actual (possibly permuted) iteration order. + keys = list(obj) has_comments = any(get(k) for k in keys) with p.group(indent=4, open="{", close=""): diff --git a/contrib/python/hypothesis/py3/hypothesis/version.py b/contrib/python/hypothesis/py3/hypothesis/version.py index 7ea0a18e4e5..c513f1ef801 100644 --- a/contrib/python/hypothesis/py3/hypothesis/version.py +++ b/contrib/python/hypothesis/py3/hypothesis/version.py @@ -8,5 +8,5 @@ # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -__version_info__ = (6, 153, 2) +__version_info__ = (6, 155, 1) __version__ = ".".join(map(str, __version_info__)) diff --git a/contrib/python/hypothesis/py3/ya.make b/contrib/python/hypothesis/py3/ya.make index 3390164dd29..30dca82125f 100644 --- a/contrib/python/hypothesis/py3/ya.make +++ b/contrib/python/hypothesis/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(6.153.2) +VERSION(6.155.1) LICENSE(MPL-2.0) diff --git a/contrib/python/pip/.dist-info/METADATA b/contrib/python/pip/.dist-info/METADATA index f591474d6c2..a7ba0c9f0ed 100644 --- a/contrib/python/pip/.dist-info/METADATA +++ b/contrib/python/pip/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.4 Name: pip -Version: 26.1.1 +Version: 26.1.2 Summary: The PyPA recommended tool for installing Python packages. Author-email: The pip developers <[email protected]> Requires-Python: >=3.10 diff --git a/contrib/python/pip/AUTHORS.txt b/contrib/python/pip/AUTHORS.txt index f6291c9bfc8..fc7d8cdaa05 100644 --- a/contrib/python/pip/AUTHORS.txt +++ b/contrib/python/pip/AUTHORS.txt @@ -776,6 +776,7 @@ Sviatoslav Sydorenko Sviatoslav Sydorenko (Святослав Сидоренко) Swat009 Sylvain +Sage Abdullah Takayuki SHIMIZUKAWA Taneli Hukkinen tbeswick diff --git a/contrib/python/pip/pip/__init__.py b/contrib/python/pip/pip/__init__.py index b49ac8288ae..349d57f4741 100644 --- a/contrib/python/pip/pip/__init__.py +++ b/contrib/python/pip/pip/__init__.py @@ -1,6 +1,6 @@ from __future__ import annotations -__version__ = "26.1.1" +__version__ = "26.1.2" def main(args: list[str] | None = None) -> int: diff --git a/contrib/python/pip/pip/_internal/cli/cmdoptions.py b/contrib/python/pip/pip/_internal/cli/cmdoptions.py index a16130131eb..8d0d8cf8e80 100644 --- a/contrib/python/pip/pip/_internal/cli/cmdoptions.py +++ b/contrib/python/pip/pip/_internal/cli/cmdoptions.py @@ -567,7 +567,7 @@ def requirements_from_scripts() -> Option: default=[], dest="requirements_from_scripts", metavar="file", - help="Install dependencies of the given script file" + help="Install dependencies of the given script file " "as defined by PEP 723 inline metadata. ", ) diff --git a/contrib/python/pip/pip/_internal/network/session.py b/contrib/python/pip/pip/_internal/network/session.py index 34083e08942..ecfd79816ec 100644 --- a/contrib/python/pip/pip/_internal/network/session.py +++ b/contrib/python/pip/pip/_internal/network/session.py @@ -351,6 +351,11 @@ class PipSession(requests.Session): # Attach our User Agent to the request self.headers["User-Agent"] = user_agent() + # Pin Accept-Encoding so it doesn't vary with zstd availability (Python + # 3.14+ or backports.zstd); a varying value misses the cache for "Vary: + # Accept-Encoding" responses shared across interpreters (pypa/pip#13979). + self.headers["Accept-Encoding"] = "gzip, deflate" + # Attach our Authentication handler to the session self.auth: MultiDomainBasicAuth = MultiDomainBasicAuth(index_urls=index_urls) diff --git a/contrib/python/pip/pip/_internal/operations/install/wheel.py b/contrib/python/pip/pip/_internal/operations/install/wheel.py index 40097d6a79f..6f9a9833641 100644 --- a/contrib/python/pip/pip/_internal/operations/install/wheel.py +++ b/contrib/python/pip/pip/_internal/operations/install/wheel.py @@ -397,11 +397,25 @@ class MissingCallableSuffix(InstallationError): ) -def _raise_for_invalid_entrypoint(specification: str) -> None: +def _raise_for_invalid_entrypoint(specification: str, scripts_dir: str) -> None: entry = get_export_entry(specification) - if entry is not None and entry.suffix is None: + if entry is None: + return + + if entry.suffix is None: raise MissingCallableSuffix(str(entry)) + # distlib joins the entry point name onto the scripts directory, so a name + # with path separators or ``..`` components can resolve elsewhere. The script + # must resolve to a path strictly inside the scripts directory. + dest = os.path.join(scripts_dir, entry.name) + resolves_to_scripts_dir = os.path.abspath(dest) == os.path.abspath(scripts_dir) + if resolves_to_scripts_dir or not is_within_directory(scripts_dir, dest): + raise InstallationError( + f"Invalid script entry point name {entry.name!r}: the script " + f"would be installed outside the scripts directory ({scripts_dir})." + ) + class PipScriptMaker(ScriptMaker): # Override distlib's default script template with one that @@ -419,7 +433,7 @@ class PipScriptMaker(ScriptMaker): def make( self, specification: str, options: dict[str, Any] | None = None ) -> list[str]: - _raise_for_invalid_entrypoint(specification) + _raise_for_invalid_entrypoint(specification, self.target_dir) return super().make(specification, options) diff --git a/contrib/python/pip/pip/_internal/utils/unpacking.py b/contrib/python/pip/pip/_internal/utils/unpacking.py index 879b40c37ec..8a9b2059ca2 100644 --- a/contrib/python/pip/pip/_internal/utils/unpacking.py +++ b/contrib/python/pip/pip/_internal/utils/unpacking.py @@ -79,12 +79,12 @@ def has_leading_dir(paths: Iterable[str]) -> bool: def is_within_directory(directory: str, target: str) -> bool: """ Return true if the absolute path of target is within the directory + (including when target is equal to the directory). """ abs_directory = os.path.abspath(directory) abs_target = os.path.abspath(target) - prefix = os.path.commonpath([abs_directory, abs_target]) - return prefix == abs_directory + return abs_target == abs_directory or abs_target.startswith(abs_directory + os.sep) def _get_default_mode_plus_executable() -> int: diff --git a/contrib/python/pip/ya.make b/contrib/python/pip/ya.make index e926c4d9019..a8f87969402 100644 --- a/contrib/python/pip/ya.make +++ b/contrib/python/pip/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(26.1.1) +VERSION(26.1.2) LICENSE(MIT) diff --git a/contrib/python/platformdirs/.dist-info/METADATA b/contrib/python/platformdirs/.dist-info/METADATA index f9ac753abd2..d77dee3062d 100644 --- a/contrib/python/platformdirs/.dist-info/METADATA +++ b/contrib/python/platformdirs/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.4 Name: platformdirs -Version: 4.9.6 +Version: 4.10.0 Summary: A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`. Project-URL: Changelog, https://platformdirs.readthedocs.io/en/latest/changelog.html Project-URL: Documentation, https://platformdirs.readthedocs.io @@ -75,14 +75,26 @@ user_config_path("MyApp", "MyCompany") # returns pathlib.Path ## Directory types +**Application directories** — scoped to your app name and version: + - **Data**: Persistent application data (`user_data_dir`, `site_data_dir`) - **Config**: Configuration files and settings (`user_config_dir`, `site_config_dir`) +- **Preference**: User preferences, distinct from config on macOS (`user_preference_dir`) - **Cache**: Cached data that can be regenerated (`user_cache_dir`, `site_cache_dir`) - **State**: Non-essential runtime state like window positions (`user_state_dir`, `site_state_dir`) - **Logs**: Log files (`user_log_dir`, `site_log_dir`) - **Runtime**: Runtime files like sockets and PIDs (`user_runtime_dir`, `site_runtime_dir`) -Each type has both `user_*` (per-user, writable) and `site_*` (system-wide, read-only for users) variants. +App dirs have both `user_*` (per-user, writable) and `site_*` (system-wide, read-only) variants where applicable. + +**User media directories** — standard user-facing folders, not scoped to app name: + +- **Documents** (`user_documents_dir`), **Downloads** (`user_downloads_dir`) +- **Pictures** (`user_pictures_dir`), **Videos** (`user_videos_dir`), **Music** (`user_music_dir`) +- **Desktop** (`user_desktop_dir`), **Projects** (`user_projects_dir`) +- **Public share** (`user_publicshare_dir`), **Templates** (`user_templates_dir`) +- **Fonts** (`user_fonts_dir`) — user-writable font installation directory +- **Executable** (`user_bin_dir`, `site_bin_dir`), **Applications** (`user_applications_dir`, `site_applications_dir`) ## Documentation diff --git a/contrib/python/platformdirs/README.md b/contrib/python/platformdirs/README.md index 7cd5d88e6e6..e2627bd7aa1 100644 --- a/contrib/python/platformdirs/README.md +++ b/contrib/python/platformdirs/README.md @@ -45,14 +45,26 @@ user_config_path("MyApp", "MyCompany") # returns pathlib.Path ## Directory types +**Application directories** — scoped to your app name and version: + - **Data**: Persistent application data (`user_data_dir`, `site_data_dir`) - **Config**: Configuration files and settings (`user_config_dir`, `site_config_dir`) +- **Preference**: User preferences, distinct from config on macOS (`user_preference_dir`) - **Cache**: Cached data that can be regenerated (`user_cache_dir`, `site_cache_dir`) - **State**: Non-essential runtime state like window positions (`user_state_dir`, `site_state_dir`) - **Logs**: Log files (`user_log_dir`, `site_log_dir`) - **Runtime**: Runtime files like sockets and PIDs (`user_runtime_dir`, `site_runtime_dir`) -Each type has both `user_*` (per-user, writable) and `site_*` (system-wide, read-only for users) variants. +App dirs have both `user_*` (per-user, writable) and `site_*` (system-wide, read-only) variants where applicable. + +**User media directories** — standard user-facing folders, not scoped to app name: + +- **Documents** (`user_documents_dir`), **Downloads** (`user_downloads_dir`) +- **Pictures** (`user_pictures_dir`), **Videos** (`user_videos_dir`), **Music** (`user_music_dir`) +- **Desktop** (`user_desktop_dir`), **Projects** (`user_projects_dir`) +- **Public share** (`user_publicshare_dir`), **Templates** (`user_templates_dir`) +- **Fonts** (`user_fonts_dir`) — user-writable font installation directory +- **Executable** (`user_bin_dir`, `site_bin_dir`), **Applications** (`user_applications_dir`, `site_applications_dir`) ## Documentation diff --git a/contrib/python/platformdirs/platformdirs/__init__.py b/contrib/python/platformdirs/platformdirs/__init__.py index 4ab450ea953..e9d3cb672b6 100644 --- a/contrib/python/platformdirs/platformdirs/__init__.py +++ b/contrib/python/platformdirs/platformdirs/__init__.py @@ -344,6 +344,31 @@ def user_desktop_dir() -> str: return PlatformDirs().user_desktop_dir +def user_projects_dir() -> str: + """:returns: projects directory tied to the user""" + return PlatformDirs().user_projects_dir + + +def user_publicshare_dir() -> str: + """:returns: public share directory tied to the user""" + return PlatformDirs().user_publicshare_dir + + +def user_templates_dir() -> str: + """:returns: templates directory tied to the user""" + return PlatformDirs().user_templates_dir + + +def user_fonts_dir() -> str: + """:returns: fonts directory tied to the user""" + return PlatformDirs().user_fonts_dir + + +def user_preference_dir() -> str: + """:returns: preference directory tied to the user""" + return PlatformDirs().user_preference_dir + + def user_bin_dir() -> str: """:returns: bin directory tied to the user""" return PlatformDirs().user_bin_dir @@ -720,6 +745,31 @@ def user_desktop_path() -> Path: return PlatformDirs().user_desktop_path +def user_projects_path() -> Path: + """:returns: projects path tied to the user""" + return PlatformDirs().user_projects_path + + +def user_publicshare_path() -> Path: + """:returns: public share path tied to the user""" + return PlatformDirs().user_publicshare_path + + +def user_templates_path() -> Path: + """:returns: templates path tied to the user""" + return PlatformDirs().user_templates_path + + +def user_fonts_path() -> Path: + """:returns: fonts path tied to the user""" + return PlatformDirs().user_fonts_path + + +def user_preference_path() -> Path: + """:returns: preference path tied to the user""" + return PlatformDirs().user_preference_path + + def user_bin_path() -> Path: """:returns: bin path tied to the user""" return PlatformDirs().user_bin_path @@ -842,16 +892,26 @@ __all__ = [ "user_documents_path", "user_downloads_dir", "user_downloads_path", + "user_fonts_dir", + "user_fonts_path", "user_log_dir", "user_log_path", "user_music_dir", "user_music_path", "user_pictures_dir", "user_pictures_path", + "user_preference_dir", + "user_preference_path", + "user_projects_dir", + "user_projects_path", + "user_publicshare_dir", + "user_publicshare_path", "user_runtime_dir", "user_runtime_path", "user_state_dir", "user_state_path", + "user_templates_dir", + "user_templates_path", "user_videos_dir", "user_videos_path", ] diff --git a/contrib/python/platformdirs/platformdirs/__main__.py b/contrib/python/platformdirs/platformdirs/__main__.py index 2490ffbbe1b..7a52b7d407c 100644 --- a/contrib/python/platformdirs/platformdirs/__main__.py +++ b/contrib/python/platformdirs/platformdirs/__main__.py @@ -15,6 +15,11 @@ PROPS = ( "user_pictures_dir", "user_videos_dir", "user_music_dir", + "user_projects_dir", + "user_publicshare_dir", + "user_templates_dir", + "user_fonts_dir", + "user_preference_dir", "user_bin_dir", "site_bin_dir", "user_applications_dir", diff --git a/contrib/python/platformdirs/platformdirs/_xdg.py b/contrib/python/platformdirs/platformdirs/_xdg.py index 5ffbf93eb57..b2af284ae39 100644 --- a/contrib/python/platformdirs/platformdirs/_xdg.py +++ b/contrib/python/platformdirs/platformdirs/_xdg.py @@ -119,6 +119,34 @@ class XDGMixin(PlatformDirsABC): return super().user_desktop_dir @property + def user_projects_dir(self) -> str: + """:returns: projects directory tied to the user, from ``$XDG_PROJECTS_DIR`` if set, else platform default""" + if path := os.environ.get("XDG_PROJECTS_DIR", "").strip(): + return os.path.expanduser(path) # noqa: PTH111 # API returns str, not Path + return super().user_projects_dir + + @property + def user_publicshare_dir(self) -> str: + """:returns: public share directory tied to the user, from ``$XDG_PUBLICSHARE_DIR`` if set, else platform default""" + if path := os.environ.get("XDG_PUBLICSHARE_DIR", "").strip(): + return os.path.expanduser(path) # noqa: PTH111 # API returns str, not Path + return super().user_publicshare_dir + + @property + def user_templates_dir(self) -> str: + """:returns: templates directory tied to the user, from ``$XDG_TEMPLATES_DIR`` if set, else platform default""" + if path := os.environ.get("XDG_TEMPLATES_DIR", "").strip(): + return os.path.expanduser(path) # noqa: PTH111 # API returns str, not Path + return super().user_templates_dir + + @property + def user_fonts_dir(self) -> str: + """:returns: fonts directory tied to the user, from ``$XDG_DATA_HOME/fonts`` if set, else platform default""" + if path := os.environ.get("XDG_DATA_HOME", "").strip(): + return f"{os.path.expanduser(path)}/fonts" # noqa: PTH111 # API returns str, not Path + return super().user_fonts_dir + + @property def user_applications_dir(self) -> str: """:returns: applications directory tied to the user, from ``$XDG_DATA_HOME`` if set, else platform default""" if path := os.environ.get("XDG_DATA_HOME", "").strip(): diff --git a/contrib/python/platformdirs/platformdirs/android.py b/contrib/python/platformdirs/platformdirs/android.py index 8c798b95e31..885e8c6e154 100644 --- a/contrib/python/platformdirs/platformdirs/android.py +++ b/contrib/python/platformdirs/platformdirs/android.py @@ -108,6 +108,31 @@ class Android(PlatformDirsABC): # noqa: PLR0904 return "/storage/emulated/0/Desktop" @property + def user_projects_dir(self) -> str: + """:returns: projects directory tied to the user e.g. ``/storage/emulated/0/Projects``""" + return "/storage/emulated/0/Projects" + + @property + def user_publicshare_dir(self) -> str: + """:returns: public share directory tied to the user e.g. ``/storage/emulated/0/Public``""" + return "/storage/emulated/0/Public" + + @property + def user_templates_dir(self) -> str: + """:returns: templates directory tied to the user e.g. ``/storage/emulated/0/Templates``""" + return "/storage/emulated/0/Templates" + + @property + def user_fonts_dir(self) -> str: + """:returns: fonts directory tied to the user e.g. ``/storage/emulated/0/fonts``""" + return "/storage/emulated/0/fonts" + + @property + def user_preference_dir(self) -> str: + """:returns: preference directory tied to the user, same as ``user_config_dir``""" + return self.user_config_dir + + @property def user_bin_dir(self) -> str: """:returns: bin directory tied to the user, e.g. ``/data/user/<userid>/<packagename>/files/bin``""" return os.path.join(cast("str", _android_folder()), "files", "bin") # noqa: PTH118 diff --git a/contrib/python/platformdirs/platformdirs/api.py b/contrib/python/platformdirs/platformdirs/api.py index 1e3b9a97c65..91ef902858a 100644 --- a/contrib/python/platformdirs/platformdirs/api.py +++ b/contrib/python/platformdirs/platformdirs/api.py @@ -210,6 +210,31 @@ class PlatformDirsABC(ABC): # noqa: PLR0904 @property @abstractmethod + def user_projects_dir(self) -> str: + """:returns: projects directory tied to the user""" + + @property + @abstractmethod + def user_publicshare_dir(self) -> str: + """:returns: public share directory tied to the user""" + + @property + @abstractmethod + def user_templates_dir(self) -> str: + """:returns: templates directory tied to the user""" + + @property + @abstractmethod + def user_fonts_dir(self) -> str: + """:returns: fonts directory tied to the user""" + + @property + @abstractmethod + def user_preference_dir(self) -> str: + """:returns: preference directory tied to the user""" + + @property + @abstractmethod def user_bin_dir(self) -> str: """:returns: bin directory tied to the user""" @@ -323,6 +348,31 @@ class PlatformDirsABC(ABC): # noqa: PLR0904 return Path(self.user_desktop_dir) @property + def user_projects_path(self) -> Path: + """:returns: projects path tied to the user""" + return Path(self.user_projects_dir) + + @property + def user_publicshare_path(self) -> Path: + """:returns: public share path tied to the user""" + return Path(self.user_publicshare_dir) + + @property + def user_templates_path(self) -> Path: + """:returns: templates path tied to the user""" + return Path(self.user_templates_dir) + + @property + def user_fonts_path(self) -> Path: + """:returns: fonts path tied to the user""" + return Path(self.user_fonts_dir) + + @property + def user_preference_path(self) -> Path: + """:returns: preference path tied to the user""" + return Path(self.user_preference_dir) + + @property def user_bin_path(self) -> Path: """:returns: bin path tied to the user""" return Path(self.user_bin_dir) diff --git a/contrib/python/platformdirs/platformdirs/macos.py b/contrib/python/platformdirs/platformdirs/macos.py index 26447e396e3..9ebc2a0c549 100644 --- a/contrib/python/platformdirs/platformdirs/macos.py +++ b/contrib/python/platformdirs/platformdirs/macos.py @@ -50,6 +50,11 @@ class _MacOSDefaults(PlatformDirsABC): # noqa: PLR0904 return self._first_item_as_path_if_multipath(self.site_data_dir) @property + def site_config_path(self) -> Path: + """:returns: config path shared by users. Only return the first item, even if ``multipath`` is set to ``True``""" + return self._first_item_as_path_if_multipath(self.site_config_dir) + + @property def user_config_dir(self) -> str: """:returns: config directory tied to the user, same as `user_data_dir`""" return self._base_user_app_support_dir() @@ -130,6 +135,31 @@ class _MacOSDefaults(PlatformDirsABC): # noqa: PLR0904 return os.path.expanduser("~/Desktop") # noqa: PTH111 @property + def user_projects_dir(self) -> str: + """:returns: projects directory tied to the user, e.g. ``~/Projects``""" + return os.path.expanduser("~/Projects") # noqa: PTH111 + + @property + def user_publicshare_dir(self) -> str: + """:returns: public share directory tied to the user, e.g. ``~/Public``""" + return os.path.expanduser("~/Public") # noqa: PTH111 # API returns str, not Path + + @property + def user_templates_dir(self) -> str: + """:returns: templates directory tied to the user, e.g. ``~/Templates``""" + return os.path.expanduser("~/Templates") # noqa: PTH111 # API returns str, not Path + + @property + def user_fonts_dir(self) -> str: + """:returns: fonts directory tied to the user, e.g. ``~/Library/Fonts``""" + return os.path.expanduser("~/Library/Fonts") # noqa: PTH111 # API returns str, not Path + + @property + def user_preference_dir(self) -> str: + """:returns: preference directory tied to the user, e.g. ``~/Library/Preferences/AppName``""" + return self._append_app_name_and_version(os.path.expanduser("~/Library/Preferences")) # noqa: PTH111 # API returns str, not Path + + @property def user_bin_dir(self) -> str: """:returns: bin directory tied to the user, e.g. ``~/.local/bin``""" return os.path.expanduser("~/.local/bin") # noqa: PTH111 diff --git a/contrib/python/platformdirs/platformdirs/unix.py b/contrib/python/platformdirs/platformdirs/unix.py index 8ec5a6bc718..500d3a4be92 100644 --- a/contrib/python/platformdirs/platformdirs/unix.py +++ b/contrib/python/platformdirs/platformdirs/unix.py @@ -124,6 +124,31 @@ class _UnixDefaults(PlatformDirsABC): # noqa: PLR0904 return _get_user_media_dir("XDG_DESKTOP_DIR", "~/Desktop") @property + def user_projects_dir(self) -> str: + """:returns: projects directory tied to the user, e.g. ``~/Projects``""" + return _get_user_media_dir("XDG_PROJECTS_DIR", "~/Projects") + + @property + def user_publicshare_dir(self) -> str: + """:returns: public share directory tied to the user, e.g. ``~/Public``""" + return _get_user_media_dir("XDG_PUBLICSHARE_DIR", "~/Public") + + @property + def user_templates_dir(self) -> str: + """:returns: templates directory tied to the user, e.g. ``~/Templates``""" + return _get_user_media_dir("XDG_TEMPLATES_DIR", "~/Templates") + + @property + def user_fonts_dir(self) -> str: + """:returns: fonts directory tied to the user, e.g. ``~/.local/share/fonts``""" + return f"{os.path.expanduser('~/.local/share')}/fonts" # noqa: PTH111 # API returns str, not Path + + @property + def user_preference_dir(self) -> str: + """:returns: preference directory tied to the user, same as ``user_config_dir``""" + return self.user_config_dir + + @property def user_bin_dir(self) -> str: """:returns: bin directory tied to the user, e.g. ``~/.local/bin``""" return os.path.expanduser("~/.local/bin") # noqa: PTH111 diff --git a/contrib/python/platformdirs/platformdirs/version.py b/contrib/python/platformdirs/platformdirs/version.py index 99cae7946ef..a281775fdd8 100644 --- a/contrib/python/platformdirs/platformdirs/version.py +++ b/contrib/python/platformdirs/platformdirs/version.py @@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...] commit_id: str | None __commit_id__: str | None -__version__ = version = '4.9.6' -__version_tuple__ = version_tuple = (4, 9, 6) +__version__ = version = '4.10.0' +__version_tuple__ = version_tuple = (4, 10, 0) __commit_id__ = commit_id = None diff --git a/contrib/python/platformdirs/platformdirs/windows.py b/contrib/python/platformdirs/platformdirs/windows.py index 47403dea5ef..476ffe73605 100644 --- a/contrib/python/platformdirs/platformdirs/windows.py +++ b/contrib/python/platformdirs/platformdirs/windows.py @@ -4,6 +4,7 @@ from __future__ import annotations import os import sys +from pathlib import Path from typing import TYPE_CHECKING, Final from .api import PlatformDirsABC @@ -134,6 +135,31 @@ class Windows(PlatformDirsABC): # noqa: PLR0904 return os.path.normpath(get_win_folder("CSIDL_DESKTOPDIRECTORY")) @property + def user_projects_dir(self) -> str: + r""":returns: projects directory tied to the user, e.g. ``%USERPROFILE%\Projects``""" + return os.path.normpath(os.path.expanduser("~/Projects")) # noqa: PTH111 + + @property + def user_publicshare_dir(self) -> str: + r""":returns: public share directory e.g. ``C:\Users\Public``""" + return os.path.normpath(os.environ.get("PUBLIC", str(Path("~").expanduser().parent / "Public"))) + + @property + def user_templates_dir(self) -> str: + r""":returns: templates directory tied to the user e.g. ``%APPDATA%\Microsoft\Windows\Templates``""" + return os.path.normpath(str(Path(get_win_folder("CSIDL_APPDATA")) / "Microsoft" / "Windows" / "Templates")) + + @property + def user_fonts_dir(self) -> str: + r""":returns: fonts directory tied to the user e.g. ``%LOCALAPPDATA%\Microsoft\Windows\Fonts``""" + return os.path.normpath(str(Path(get_win_folder("CSIDL_LOCAL_APPDATA")) / "Microsoft" / "Windows" / "Fonts")) + + @property + def user_preference_dir(self) -> str: + r""":returns: preference directory tied to the user, same as ``user_config_dir``""" + return self.user_config_dir + + @property def user_bin_dir(self) -> str: r""":returns: bin directory tied to the user, e.g. ``%LOCALAPPDATA%\Programs``""" return os.path.normpath(os.path.join(get_win_folder("CSIDL_LOCAL_APPDATA"), "Programs")) # noqa: PTH118 diff --git a/contrib/python/platformdirs/ya.make b/contrib/python/platformdirs/ya.make index 9155aa43b0a..b30eb7d7837 100644 --- a/contrib/python/platformdirs/ya.make +++ b/contrib/python/platformdirs/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(4.9.6) +VERSION(4.10.0) LICENSE(MIT) diff --git a/contrib/python/pytest/py3/_pytest/fixtures.py b/contrib/python/pytest/py3/_pytest/fixtures.py index f96b081545b..0a46e5bfc06 100644 --- a/contrib/python/pytest/py3/_pytest/fixtures.py +++ b/contrib/python/pytest/py3/_pytest/fixtures.py @@ -1538,6 +1538,7 @@ class FixtureManager: # (discovering matching fixtures for a given name/node is expensive). parentid = parentnode.nodeid + parentnodesids = set(nodes.iterparentnodeids(parentid)) fixturenames_closure = list(initialnames) arg2fixturedefs: Dict[str, Sequence[FixtureDef[Any]]] = {} @@ -1549,7 +1550,7 @@ class FixtureManager: continue if argname in arg2fixturedefs: continue - fixturedefs = self.getfixturedefs(argname, parentid) + fixturedefs = self.getfixturedefs(argname, parentid, _parentnodeids=parentnodesids) if fixturedefs: arg2fixturedefs[argname] = fixturedefs for arg in fixturedefs[-1].argnames: @@ -1574,6 +1575,12 @@ class FixtureManager: args, _ = ParameterSet._parse_parametrize_args(*mark.args, **mark.kwargs) return args + parametrized_argnames = set( + argname + for mark in metafunc.definition.iter_markers("parametrize") + for argname in get_parametrize_mark_argnames(mark) + ) + for argname in metafunc.fixturenames: # Get the FixtureDefs for the argname. fixture_defs = metafunc._arg2fixturedefs.get(argname) @@ -1584,10 +1591,7 @@ class FixtureManager: # If the test itself parametrizes using this argname, give it # precedence. - if any( - argname in get_parametrize_mark_argnames(mark) - for mark in metafunc.definition.iter_markers("parametrize") - ): + if argname in parametrized_argnames: continue # In the common case we only look at the fixture def with the @@ -1721,7 +1725,7 @@ class FixtureManager: self._nodeid_autousenames.setdefault(nodeid or "", []).extend(autousenames) def getfixturedefs( - self, argname: str, nodeid: str + self, argname: str, nodeid: str, /, _parentnodeids: Optional[set[str]] = None ) -> Optional[Sequence[FixtureDef[Any]]]: """Get FixtureDefs for a fixture name which are applicable to a given node. @@ -1738,12 +1742,14 @@ class FixtureManager: fixturedefs = self._arg2fixturedefs[argname] except KeyError: return None - return tuple(self._matchfactories(fixturedefs, nodeid)) + + if not _parentnodeids: + _parentnodeids = set(nodes.iterparentnodeids(nodeid)) + return tuple(self._matchfactories(fixturedefs, _parentnodeids)) def _matchfactories( - self, fixturedefs: Iterable[FixtureDef[Any]], nodeid: str + self, fixturedefs: Iterable[FixtureDef[Any]], _parentnodeids: set[str] ) -> Iterator[FixtureDef[Any]]: - parentnodeids = set(nodes.iterparentnodeids(nodeid)) for fixturedef in fixturedefs: - if fixturedef.baseid in parentnodeids: + if fixturedef.baseid in _parentnodeids: yield fixturedef diff --git a/contrib/python/pytest/py3/_pytest/scope.py b/contrib/python/pytest/py3/_pytest/scope.py index 2c6e23208f2..247a5f34847 100644 --- a/contrib/python/pytest/py3/_pytest/scope.py +++ b/contrib/python/pytest/py3/_pytest/scope.py @@ -53,8 +53,32 @@ class Scope(Enum): return _ALL_SCOPES[index + 1] def __lt__(self, other: "Scope") -> bool: - self_index = _SCOPE_INDICES[self] - other_index = _SCOPE_INDICES[other] + if self == other: + return False + self_index = 0 + if self is Scope.Function: + self_index = 0 + elif self is Scope.Class: + self_index = 1 + elif self is Scope.Module: + self_index = 2 + elif self is Scope.Package: + self_index = 3 + elif self is Scope.Session: + self_index = 4 + + other_index = 0 + if other is Scope.Function: + other_index = 0 + elif other is Scope.Class: + other_index = 1 + elif other is Scope.Module: + other_index = 2 + elif other is Scope.Package: + other_index = 3 + elif other is Scope.Session: + other_index = 4 + return self_index < other_index @classmethod diff --git a/contrib/python/pytest/py3/patches/11-perf-optional-drop-if-upgrade.patch b/contrib/python/pytest/py3/patches/11-perf-optional-drop-if-upgrade.patch new file mode 100644 index 00000000000..2291d7881e0 --- /dev/null +++ b/contrib/python/pytest/py3/patches/11-perf-optional-drop-if-upgrade.patch @@ -0,0 +1,109 @@ +--- contrib/python/pytest/py3/_pytest/fixtures.py (54693a02d00937a28e5176cac9fa36e9ea9a9a4b) ++++ contrib/python/pytest/py3/_pytest/fixtures.py (d19830d8860a577c5e2a88ca771220910c8ad740) +@@ -1538,6 +1538,7 @@ class FixtureManager: + # (discovering matching fixtures for a given name/node is expensive). + + parentid = parentnode.nodeid ++ parentnodesids = set(nodes.iterparentnodeids(parentid)) + fixturenames_closure = list(initialnames) + + arg2fixturedefs: Dict[str, Sequence[FixtureDef[Any]]] = {} +@@ -1549,7 +1550,7 @@ class FixtureManager: + continue + if argname in arg2fixturedefs: + continue +- fixturedefs = self.getfixturedefs(argname, parentid) ++ fixturedefs = self.getfixturedefs(argname, parentid, _parentnodeids=parentnodesids) + if fixturedefs: + arg2fixturedefs[argname] = fixturedefs + for arg in fixturedefs[-1].argnames: +@@ -1574,6 +1575,12 @@ class FixtureManager: + args, _ = ParameterSet._parse_parametrize_args(*mark.args, **mark.kwargs) + return args + ++ parametrized_argnames = set( ++ argname ++ for mark in metafunc.definition.iter_markers("parametrize") ++ for argname in get_parametrize_mark_argnames(mark) ++ ) ++ + for argname in metafunc.fixturenames: + # Get the FixtureDefs for the argname. + fixture_defs = metafunc._arg2fixturedefs.get(argname) +@@ -1584,10 +1591,7 @@ class FixtureManager: + + # If the test itself parametrizes using this argname, give it + # precedence. +- if any( +- argname in get_parametrize_mark_argnames(mark) +- for mark in metafunc.definition.iter_markers("parametrize") +- ): ++ if argname in parametrized_argnames: + continue + + # In the common case we only look at the fixture def with the +@@ -1721,7 +1725,7 @@ class FixtureManager: + self._nodeid_autousenames.setdefault(nodeid or "", []).extend(autousenames) + + def getfixturedefs( +- self, argname: str, nodeid: str ++ self, argname: str, nodeid: str, /, _parentnodeids: Optional[set[str]] = None + ) -> Optional[Sequence[FixtureDef[Any]]]: + """Get FixtureDefs for a fixture name which are applicable + to a given node. +@@ -1738,12 +1742,14 @@ class FixtureManager: + fixturedefs = self._arg2fixturedefs[argname] + except KeyError: + return None +- return tuple(self._matchfactories(fixturedefs, nodeid)) ++ ++ if not _parentnodeids: ++ _parentnodeids = set(nodes.iterparentnodeids(nodeid)) ++ return tuple(self._matchfactories(fixturedefs, _parentnodeids)) + + def _matchfactories( +- self, fixturedefs: Iterable[FixtureDef[Any]], nodeid: str ++ self, fixturedefs: Iterable[FixtureDef[Any]], _parentnodeids: set[str] + ) -> Iterator[FixtureDef[Any]]: +- parentnodeids = set(nodes.iterparentnodeids(nodeid)) + for fixturedef in fixturedefs: +- if fixturedef.baseid in parentnodeids: ++ if fixturedef.baseid in _parentnodeids: + yield fixturedef +--- contrib/python/pytest/py3/_pytest/scope.py (54693a02d00937a28e5176cac9fa36e9ea9a9a4b) ++++ contrib/python/pytest/py3/_pytest/scope.py (d19830d8860a577c5e2a88ca771220910c8ad740) +@@ -53,8 +53,32 @@ class Scope(Enum): + return _ALL_SCOPES[index + 1] + + def __lt__(self, other: "Scope") -> bool: +- self_index = _SCOPE_INDICES[self] +- other_index = _SCOPE_INDICES[other] ++ if self == other: ++ return False ++ self_index = 0 ++ if self is Scope.Function: ++ self_index = 0 ++ elif self is Scope.Class: ++ self_index = 1 ++ elif self is Scope.Module: ++ self_index = 2 ++ elif self is Scope.Package: ++ self_index = 3 ++ elif self is Scope.Session: ++ self_index = 4 ++ ++ other_index = 0 ++ if other is Scope.Function: ++ other_index = 0 ++ elif other is Scope.Class: ++ other_index = 1 ++ elif other is Scope.Module: ++ other_index = 2 ++ elif other is Scope.Package: ++ other_index = 3 ++ elif other is Scope.Session: ++ other_index = 4 ++ + return self_index < other_index + + @classmethod diff --git a/contrib/python/wheel/ya.make b/contrib/python/wheel/ya.make index b0c0af58c78..1bf947a7442 100644 --- a/contrib/python/wheel/ya.make +++ b/contrib/python/wheel/ya.make @@ -12,6 +12,11 @@ PEERDIR( NO_LINT() +NO_CHECK_IMPORTS( + wheel._bdist_wheel + wheel.bdist_wheel +) + PY_SRCS( TOP_LEVEL wheel/__init__.py |
