Intermediate changes

16 files changed, 174 insertions, 47 deletions

diff --git a/contrib/python/hypothesis/py3/.dist-info/METADATA b/contrib/python/hypothesis/py3/.dist-info/METADATA
index 0d4287365b..57cf50eec1 100644
--- a/contrib/python/hypothesis/py3/.dist-info/METADATA
+++ b/contrib/python/hypothesis/py3/.dist-info/METADATA
@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hypothesis
-Version: 6.103.0
+Version: 6.103.1
 Summary: A library for property-based testing
 Home-page: https://hypothesis.works
 Author: David R. MacIver and Zac Hatfield-Dodds
diff --git a/contrib/python/hypothesis/py3/hypothesis/core.py b/contrib/python/hypothesis/py3/hypothesis/core.py
index ccd5c43b6e..9af0381fea 100644
--- a/contrib/python/hypothesis/py3/hypothesis/core.py
+++ b/contrib/python/hypothesis/py3/hypothesis/core.py
@@ -77,7 +77,10 @@ from hypothesis.internal.compat import (
 )
 from hypothesis.internal.conjecture.data import ConjectureData, Status
 from hypothesis.internal.conjecture.engine import BUFFER_SIZE, ConjectureRunner
-from hypothesis.internal.conjecture.junkdrawer import ensure_free_stackframes
+from hypothesis.internal.conjecture.junkdrawer import (
+    ensure_free_stackframes,
+    gc_cumulative_time,
+)
 from hypothesis.internal.conjecture.shrinker import sort_key
 from hypothesis.internal.entropy import deterministic_PRNG
 from hypothesis.internal.escalation import (
@@ -820,21 +823,34 @@ class StateForActualGivenExecution:
         self._string_repr = ""
         text_repr = None
         if self.settings.deadline is None and not TESTCASE_CALLBACKS:
-            test = self.test
+
+            @proxies(self.test)
+            def test(*args, **kwargs):
+                with ensure_free_stackframes():
+                    return self.test(*args, **kwargs)
+
         else:
 
             @proxies(self.test)
             def test(*args, **kwargs):
                 arg_drawtime = math.fsum(data.draw_times.values())
+                arg_stateful = math.fsum(data._stateful_run_times.values())
+                arg_gctime = gc_cumulative_time()
                 start = time.perf_counter()
                 try:
-                    result = self.test(*args, **kwargs)
+                    with ensure_free_stackframes():
+                        result = self.test(*args, **kwargs)
                 finally:
                     finish = time.perf_counter()
                     in_drawtime = math.fsum(data.draw_times.values()) - arg_drawtime
-                    runtime = datetime.timedelta(seconds=finish - start - in_drawtime)
+                    in_stateful = (
+                        math.fsum(data._stateful_run_times.values()) - arg_stateful
+                    )
+                    in_gctime = gc_cumulative_time() - arg_gctime
+                    runtime = finish - start - in_drawtime - in_stateful - in_gctime
                     self._timing_features = {
-                        "execute:test": finish - start - in_drawtime,
+                        "execute:test": runtime,
+                        "overall:gc": in_gctime,
                         **data.draw_times,
                         **data._stateful_run_times,
                     }
@@ -842,8 +858,10 @@ class StateForActualGivenExecution:
                 if (current_deadline := self.settings.deadline) is not None:
                     if not is_final:
                         current_deadline = (current_deadline // 4) * 5
-                    if runtime >= current_deadline:
-                        raise DeadlineExceeded(runtime, self.settings.deadline)
+                    if runtime >= current_deadline.total_seconds():
+                        raise DeadlineExceeded(
+                            datetime.timedelta(seconds=runtime), self.settings.deadline
+                        )
                 return result
 
         def run(data):
diff --git a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/data.py b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/data.py
index 3fc6658e08..10ae727c2d 100644
--- a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/data.py
+++ b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/data.py
@@ -44,7 +44,11 @@ from hypothesis.errors import Frozen, InvalidArgument, StopTest
 from hypothesis.internal.cache import LRUReusedCache
 from hypothesis.internal.compat import add_note, floor, int_from_bytes, int_to_bytes
 from hypothesis.internal.conjecture.floats import float_to_lex, lex_to_float
-from hypothesis.internal.conjecture.junkdrawer import IntList, uniform
+from hypothesis.internal.conjecture.junkdrawer import (
+    IntList,
+    gc_cumulative_time,
+    uniform,
+)
 from hypothesis.internal.conjecture.utils import (
     INT_SIZES,
     INT_SIZES_SAMPLER,
@@ -1980,6 +1984,7 @@ class ConjectureData:
         self.testcounter = global_test_counter
         global_test_counter += 1
         self.start_time = time.perf_counter()
+        self.gc_start_time = gc_cumulative_time()
         self.events: Dict[str, Union[str, int, float]] = {}
         self.forced_indices: "Set[int]" = set()
         self.interesting_origin: Optional[InterestingOrigin] = None
@@ -2420,6 +2425,7 @@ class ConjectureData:
             # where we cache something expensive, this led to Flaky deadline errors!
             # See https://github.com/HypothesisWorks/hypothesis/issues/2108
             start_time = time.perf_counter()
+            gc_start_time = gc_cumulative_time()
 
         strategy.validate()
 
@@ -2443,7 +2449,10 @@ class ConjectureData:
                 try:
                     return strategy.do_draw(self)
                 finally:
-                    self.draw_times[key] = time.perf_counter() - start_time
+                    # Subtract the time spent in GC to avoid overcounting, as it is
+                    # accounted for at the overall example level.
+                    in_gctime = gc_cumulative_time() - gc_start_time
+                    self.draw_times[key] = time.perf_counter() - start_time - in_gctime
             except Exception as err:
                 add_note(err, f"while generating {key[9:]!r} from {strategy!r}")
                 raise
@@ -2520,6 +2529,7 @@ class ConjectureData:
             assert isinstance(self.buffer, bytes)
             return
         self.finish_time = time.perf_counter()
+        self.gc_finish_time = gc_cumulative_time()
         assert len(self.buffer) == self.index
 
         # Always finish by closing all remaining examples so that we have a
diff --git a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/engine.py b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/engine.py
index 89a8c7829c..efeb284dbf 100644
--- a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/engine.py
+++ b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/engine.py
@@ -168,6 +168,7 @@ class CallStats(TypedDict):
     status: str
     runtime: float
     drawtime: float
+    gctime: float
     events: List[str]
 
 
@@ -298,7 +299,9 @@ class ConjectureRunner:
         """
         # We ensure that the test has this much stack space remaining, no
         # matter the size of the stack when called, to de-flake RecursionErrors
-        # (#2494, #3671).
+        # (#2494, #3671). Note, this covers the data generation part of the test;
+        # the actual test execution is additionally protected at the call site
+        # in hypothesis.core.execute_once.
         with ensure_free_stackframes():
             try:
                 self._test_function(data)
@@ -430,6 +433,7 @@ class ConjectureRunner:
                     "status": data.status.name.lower(),
                     "runtime": data.finish_time - data.start_time,
                     "drawtime": math.fsum(data.draw_times.values()),
+                    "gctime": data.gc_finish_time - data.gc_start_time,
                     "events": sorted(
                         k if v == "" else f"{k}: {v}" for k, v in data.events.items()
                     ),
diff --git a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/junkdrawer.py b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/junkdrawer.py
index 716b5d3d82..7dbab5b971 100644
--- a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/junkdrawer.py
+++ b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/junkdrawer.py
@@ -13,7 +13,9 @@ obviously belong anywhere else. If you spot a better home for
 anything that lives here, please move it."""
 
 import array
+import gc
 import sys
+import time
 import warnings
 from random import Random
 from typing import (
@@ -413,3 +415,52 @@ class SelfOrganisingList(Generic[T]):
                 self.__values.append(value)
                 return value
         raise NotFound("No values satisfying condition")
+
+
+_gc_initialized = False
+_gc_start = 0
+_gc_cumulative_time = 0
+
+
+def gc_cumulative_time() -> float:
+    global _gc_initialized
+    if not _gc_initialized:
+        if hasattr(gc, "callbacks"):
+            # CPython
+            def gc_callback(phase, info):
+                global _gc_start, _gc_cumulative_time
+                try:
+                    now = time.perf_counter()
+                    if phase == "start":
+                        _gc_start = now
+                    elif phase == "stop" and _gc_start > 0:
+                        _gc_cumulative_time += now - _gc_start  # pragma: no cover # ??
+                except RecursionError:  # pragma: no cover
+                    # Avoid flakiness via UnraisableException, which is caught and
+                    # warned by pytest. The actual callback (this function) is
+                    # validated to never trigger a RecursionError itself when
+                    # when called by gc.collect.
+                    # Anyway, we should hit the same error on "start"
+                    # and "stop", but to ensure we don't get out of sync we just
+                    # signal that there is no matching start.
+                    _gc_start = 0
+                    return
+
+            gc.callbacks.insert(0, gc_callback)
+        elif hasattr(gc, "hooks"):  # pragma: no cover  # pypy only
+            # PyPy
+            def hook(stats):
+                global _gc_cumulative_time
+                try:
+                    _gc_cumulative_time += stats.duration
+                except RecursionError:
+                    pass
+
+            if gc.hooks.on_gc_minor is None:
+                gc.hooks.on_gc_minor = hook
+            if gc.hooks.on_gc_collect_step is None:
+                gc.hooks.on_gc_collect_step = hook
+
+        _gc_initialized = True
+
+    return _gc_cumulative_time
diff --git a/contrib/python/hypothesis/py3/hypothesis/internal/escalation.py b/contrib/python/hypothesis/py3/hypothesis/internal/escalation.py
index c3c678d239..b85d9fcdc9 100644
--- a/contrib/python/hypothesis/py3/hypothesis/internal/escalation.py
+++ b/contrib/python/hypothesis/py3/hypothesis/internal/escalation.py
@@ -87,9 +87,12 @@ def get_trimmed_traceback(exception=None):
     else:
         tb = exception.__traceback__
     # Avoid trimming the traceback if we're in verbose mode, or the error
-    # was raised inside Hypothesis
+    # was raised inside Hypothesis. Additionally, the environment variable
+    # HYPOTHESIS_NO_TRACEBACK_TRIM is respected if nonempty, because verbose
+    # mode is prohibitively slow when debugging strategy recursion errors.
     if (
         tb is None
+        or os.environ.get("HYPOTHESIS_NO_TRACEBACK_TRIM", None)
         or hypothesis.settings.default.verbosity >= hypothesis.Verbosity.debug
         or is_hypothesis_file(traceback.extract_tb(tb)[-1][0])
         and not isinstance(exception, _Trimmable)
diff --git a/contrib/python/hypothesis/py3/hypothesis/internal/scrutineer.py b/contrib/python/hypothesis/py3/hypothesis/internal/scrutineer.py
index 39352844b4..d99e767c1d 100644
--- a/contrib/python/hypothesis/py3/hypothesis/internal/scrutineer.py
+++ b/contrib/python/hypothesis/py3/hypothesis/internal/scrutineer.py
@@ -10,6 +10,7 @@
 
 import functools
 import os
+import re
 import subprocess
 import sys
 import types
@@ -58,15 +59,18 @@ class Tracer:
         self._previous_location = None
 
     def trace(self, frame, event, arg):
-        if event == "call":
-            return self.trace
-        elif event == "line":
-            # manual inlining of self.trace_line for performance.
-            fname = frame.f_code.co_filename
-            if should_trace_file(fname):
-                current_location = (fname, frame.f_lineno)
-                self.branches.add((self._previous_location, current_location))
-                self._previous_location = current_location
+        try:
+            if event == "call":
+                return self.trace
+            elif event == "line":
+                # manual inlining of self.trace_line for performance.
+                fname = frame.f_code.co_filename
+                if should_trace_file(fname):
+                    current_location = (fname, frame.f_lineno)
+                    self.branches.add((self._previous_location, current_location))
+                    self._previous_location = current_location
+        except RecursionError:
+            pass
 
     def trace_line(self, code: types.CodeType, line_number: int) -> None:
         fname = code.co_filename
@@ -104,19 +108,38 @@ UNHELPFUL_LOCATIONS = (
     # a contextmanager; this is probably after the fault has been triggered.
     # Similar reasoning applies to a few other standard-library modules: even
     # if the fault was later, these still aren't useful locations to report!
-    f"{sep}contextlib.py",
-    f"{sep}inspect.py",
-    f"{sep}re.py",
-    f"{sep}re{sep}__init__.py",  # refactored in Python 3.11
-    f"{sep}warnings.py",
+    # Note: The list is post-processed, so use plain "/" for separator here.
+    "/contextlib.py",
+    "/inspect.py",
+    "/re.py",
+    "/re/__init__.py",  # refactored in Python 3.11
+    "/warnings.py",
     # Quite rarely, the first AFNP line is in Pytest's internals.
-    f"{sep}_pytest{sep}assertion{sep}__init__.py",
-    f"{sep}_pytest{sep}assertion{sep}rewrite.py",
-    f"{sep}_pytest{sep}_io{sep}saferepr.py",
-    f"{sep}pluggy{sep}_result.py",
+    "/_pytest/_io/saferepr.py",
+    "/_pytest/assertion/*.py",
+    "/_pytest/config/__init__.py",
+    "/_pytest/pytester.py",
+    "/pluggy/_*.py",
+    "/reprlib.py",
+    "/typing.py",
+    "/conftest.py",
 )
 
 
+def _glob_to_re(locs):
+    """Translate a list of glob patterns to a combined regular expression.
+    Only the * wildcard is supported, and patterns including special
+    characters will only work by chance."""
+    # fnmatch.translate is not an option since its "*" consumes path sep
+    return "|".join(
+        loc.replace("*", r"[^/]+")
+        .replace(".", re.escape("."))
+        .replace("/", re.escape(sep))
+        + r"\Z"  # right anchored
+        for loc in locs
+    )
+
+
 def get_explaining_locations(traces):
     # Traces is a dict[interesting_origin | None, set[frozenset[tuple[str, int]]]]
     # Each trace in the set might later become a Counter instead of frozenset.
@@ -159,8 +182,9 @@ def get_explaining_locations(traces):
     # The last step is to filter out explanations that we know would be uninformative.
     # When this is the first AFNP location, we conclude that Scrutineer missed the
     # real divergence (earlier in the trace) and drop that unhelpful explanation.
+    filter_regex = re.compile(_glob_to_re(UNHELPFUL_LOCATIONS))
     return {
-        origin: {loc for loc in afnp_locs if not loc[0].endswith(UNHELPFUL_LOCATIONS)}
+        origin: {loc for loc in afnp_locs if not filter_regex.search(loc[0])}
         for origin, afnp_locs in explanations.items()
     }
 
diff --git a/contrib/python/hypothesis/py3/hypothesis/stateful.py b/contrib/python/hypothesis/py3/hypothesis/stateful.py
index 8c8272df7b..067d9c77c6 100644
--- a/contrib/python/hypothesis/py3/hypothesis/stateful.py
+++ b/contrib/python/hypothesis/py3/hypothesis/stateful.py
@@ -50,6 +50,7 @@ from hypothesis.errors import InvalidArgument, InvalidDefinition
 from hypothesis.internal.compat import add_note
 from hypothesis.internal.conjecture import utils as cu
 from hypothesis.internal.conjecture.engine import BUFFER_SIZE
+from hypothesis.internal.conjecture.junkdrawer import gc_cumulative_time
 from hypothesis.internal.healthcheck import fail_health_check
 from hypothesis.internal.observability import TESTCASE_CALLBACKS
 from hypothesis.internal.reflection import (
@@ -158,6 +159,7 @@ def run_state_machine_as_test(state_machine_factory, *, settings=None, _min_step
                     must_stop = True
 
                 start_draw = perf_counter()
+                start_gc = gc_cumulative_time()
                 if cd.draw_boolean(p=2**-16, forced=must_stop):
                     break
                 steps_run += 1
@@ -175,7 +177,8 @@ def run_state_machine_as_test(state_machine_factory, *, settings=None, _min_step
                     rule, data = cd.draw(machine._rules_strategy)
                 draw_label = f"generate:rule:{rule.function.__name__}"
                 cd.draw_times.setdefault(draw_label, 0.0)
-                cd.draw_times[draw_label] += perf_counter() - start_draw
+                in_gctime = gc_cumulative_time() - start_gc
+                cd.draw_times[draw_label] += perf_counter() - start_draw - in_gctime
 
                 # Pretty-print the values this rule was called with *before* calling
                 # _add_result_to_targets, to avoid printing arguments which are also
@@ -196,8 +199,10 @@ def run_state_machine_as_test(state_machine_factory, *, settings=None, _min_step
 
                     label = f"execute:rule:{rule.function.__name__}"
                     start = perf_counter()
+                    start_gc = gc_cumulative_time()
                     result = rule.function(machine, **data)
-                    cd._stateful_run_times[label] += perf_counter() - start
+                    in_gctime = gc_cumulative_time() - start_gc
+                    cd._stateful_run_times[label] += perf_counter() - start - in_gctime
 
                     if rule.targets:
                         if isinstance(result, MultipleResults):
diff --git a/contrib/python/hypothesis/py3/hypothesis/statistics.py b/contrib/python/hypothesis/py3/hypothesis/statistics.py
index 33e4ea6706..7425db7bcb 100644
--- a/contrib/python/hypothesis/py3/hypothesis/statistics.py
+++ b/contrib/python/hypothesis/py3/hypothesis/statistics.py
@@ -48,7 +48,8 @@ def format_ms(times):
     """
     ordered = sorted(times)
     n = len(ordered) - 1
-    assert n >= 0
+    if n < 0 or any(math.isnan(t) for t in ordered):
+        return "NaN ms"
     lower = int(ordered[int(math.floor(n * 0.05))] * 1000)
     upper = int(ordered[int(math.ceil(n * 0.95))] * 1000)
     if upper == 0:
diff --git a/contrib/python/hypothesis/py3/hypothesis/version.py b/contrib/python/hypothesis/py3/hypothesis/version.py
index a3a5493aa1..0260a87746 100644
--- a/contrib/python/hypothesis/py3/hypothesis/version.py
+++ b/contrib/python/hypothesis/py3/hypothesis/version.py
@@ -8,5 +8,5 @@
 # v. 2.0. If a copy of the MPL was not distributed with this file, You can
 # obtain one at https://mozilla.org/MPL/2.0/.
 
-__version_info__ = (6, 103, 0)
+__version_info__ = (6, 103, 1)
 __version__ = ".".join(map(str, __version_info__))
diff --git a/contrib/python/hypothesis/py3/ya.make b/contrib/python/hypothesis/py3/ya.make
index 35e83e2000..c1289cb94f 100644
--- a/contrib/python/hypothesis/py3/ya.make
+++ b/contrib/python/hypothesis/py3/ya.make
@@ -2,7 +2,7 @@
 
 PY3_LIBRARY()
 
-VERSION(6.103.0)
+VERSION(6.103.1)
 
 LICENSE(MPL-2.0)
 
diff --git a/contrib/python/ydb/py3/.dist-info/METADATA b/contrib/python/ydb/py3/.dist-info/METADATA
index d75da64fc1..cccddeb427 100644
--- a/contrib/python/ydb/py3/.dist-info/METADATA
+++ b/contrib/python/ydb/py3/.dist-info/METADATA
@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ydb
-Version: 3.11.4
+Version: 3.12.2
 Summary: YDB Python SDK
 Home-page: http://github.com/ydb-platform/ydb-python-sdk
 Author: Yandex LLC
diff --git a/contrib/python/ydb/py3/ya.make b/contrib/python/ydb/py3/ya.make
index 12ac0b1600..cec1a8a4ef 100644
--- a/contrib/python/ydb/py3/ya.make
+++ b/contrib/python/ydb/py3/ya.make
@@ -2,7 +2,7 @@
 
 PY3_LIBRARY()
 
-VERSION(3.11.4)
+VERSION(3.12.2)
 
 LICENSE(Apache-2.0)
 
diff --git a/contrib/python/ydb/py3/ydb/aio/credentials.py b/contrib/python/ydb/py3/ydb/aio/credentials.py
index 48db925eba..18e1b7e0a9 100644
--- a/contrib/python/ydb/py3/ydb/aio/credentials.py
+++ b/contrib/python/ydb/py3/ydb/aio/credentials.py
@@ -71,7 +71,7 @@ class AbstractExpiringTokenCredentials(credentials.AbstractExpiringTokenCredenti
         try:
             auth_metadata = await self._make_token_request()
             await self._cached_token.update(auth_metadata["access_token"])
-            self.update_expiration_info(auth_metadata)
+            self._update_expiration_info(auth_metadata)
             self.logger.info(
                 "Token refresh successful. current_time %s, refresh_in %s",
                 current_time,
diff --git a/contrib/python/ydb/py3/ydb/iam/auth.py b/contrib/python/ydb/py3/ydb/iam/auth.py
index 7b4fa4e8d9..5fd179d2af 100644
--- a/contrib/python/ydb/py3/ydb/iam/auth.py
+++ b/contrib/python/ydb/py3/ydb/iam/auth.py
@@ -16,8 +16,13 @@ try:
     from yandex.cloud.iam.v1 import iam_token_service_pb2_grpc
     from yandex.cloud.iam.v1 import iam_token_service_pb2
 except ImportError:
-    iam_token_service_pb2_grpc = None
-    iam_token_service_pb2 = None
+    try:
+        # This attempt is to enable the IAM auth inside the YDB repository on GitHub
+        from ydb.public.api.client.yc_public.iam import iam_token_service_pb2_grpc
+        from ydb.public.api.client.yc_public.iam import iam_token_service_pb2
+    except ImportError:
+        iam_token_service_pb2_grpc = None
+        iam_token_service_pb2 = None
 
 try:
     import requests
@@ -99,14 +104,20 @@ class BaseJWTCredentials(abc.ABC):
     @classmethod
     def from_file(cls, key_file, iam_endpoint=None, iam_channel_credentials=None):
         with open(os.path.expanduser(key_file), "r") as r:
-            output = json.loads(r.read())
-        account_id = output.get("service_account_id", None)
+            key = r.read()
+
+        return cls.from_content(key, iam_endpoint=iam_endpoint, iam_channel_credentials=iam_channel_credentials)
+
+    @classmethod
+    def from_content(cls, key, iam_endpoint=None, iam_channel_credentials=None):
+        key_json = json.loads(key)
+        account_id = key_json.get("service_account_id", None)
         if account_id is None:
-            account_id = output.get("user_account_id", None)
+            account_id = key_json.get("user_account_id", None)
         return cls(
             account_id,
-            output["id"],
-            output["private_key"],
+            key_json["id"],
+            key_json["private_key"],
             iam_endpoint=iam_endpoint,
             iam_channel_credentials=iam_channel_credentials,
         )
diff --git a/contrib/python/ydb/py3/ydb/ydb_version.py b/contrib/python/ydb/py3/ydb/ydb_version.py
index b3b3b4cbd5..671c282292 100644
--- a/contrib/python/ydb/py3/ydb/ydb_version.py
+++ b/contrib/python/ydb/py3/ydb/ydb_version.py
@@ -1 +1 @@
-VERSION = "3.11.4"
+VERSION = "3.12.2"